From c2aeba611e6e882b60517fed39f6c998fdbdc8eb Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Fri, 6 Nov 2026 07:28:22 +0100 Subject: [PATCH 01/15] feat(tiles): manually reload linked tile archives --- crates/data/src/mbtiles.rs | 182 +++++++---- crates/engine/src/cmd/plugins.rs | 2 +- crates/engine/src/cmd/repair.rs | 72 ++++- crates/engine/tests/portable_projects.rs | 2 + .../tests/portable_projects/reload_tiles.rs | 289 ++++++++++++++++++ crates/tiles/src/cache.rs | 168 +++++++++- crates/tiles/src/mbtiles.rs | 98 ++++-- crates/tiles/src/pmtiles.rs | 263 ++++++++++++---- crates/tiles/src/source.rs | 33 +- crates/ui-egui/src/chrome.rs | 2 +- crates/ui-egui/src/panes.rs | 17 +- docs/user-guide/maps.md | 20 ++ 12 files changed, 968 insertions(+), 180 deletions(-) create mode 100644 crates/engine/tests/portable_projects/reload_tiles.rs diff --git a/crates/data/src/mbtiles.rs b/crates/data/src/mbtiles.rs index d03f035127..371166df64 100644 --- a/crates/data/src/mbtiles.rs +++ b/crates/data/src/mbtiles.rs @@ -287,8 +287,7 @@ mod file { use std::collections::HashMap; use std::path::{Path, PathBuf}; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::{Mutex, OnceLock}; + use std::sync::Mutex; use rusqlite::{Connection, OpenFlags, OptionalExtension, params}; @@ -296,12 +295,16 @@ mod file { /// Open connections kept per file, for the tile readers to share. const POOL: usize = 4; - /// Goes up when connections are closed, so one in use then isn't put back for reuse. - static GENERATION: AtomicU64 = AtomicU64::new(0); + #[derive(Default)] + struct Pool { + connections: HashMap>, + epochs: HashMap, + generation: u64, + } - fn pool() -> &'static Mutex>> { - static P: OnceLock>>> = OnceLock::new(); - P.get_or_init(Mutex::default) + fn pool() -> &'static Mutex { + static P: std::sync::LazyLock> = std::sync::LazyLock::new(Mutex::default); + &P } fn open(path: &Path) -> Result { @@ -316,8 +319,10 @@ mod file { /// Runs `f` with a connection to `path`, from the pool or newly opened, and puts it back. fn with(path: &Path, f: impl FnOnce(&Connection) -> Result) -> Result { - let generation = GENERATION.load(Ordering::SeqCst); - let pooled = pool().lock().unwrap_or_else(|e| e.into_inner()).get_mut(path).and_then(Vec::pop); + let (generation, epoch, pooled) = { + let mut all = pool().lock().unwrap_or_else(|e| e.into_inner()); + (all.generation, all.epochs.get(path).copied().unwrap_or(0), all.connections.get_mut(path).and_then(Vec::pop)) + }; let conn = match pooled { Some(c) => c, None => open(path)?, @@ -325,14 +330,14 @@ mod file { let r = f(&conn); let mut all = pool().lock().unwrap_or_else(|e| e.into_inner()); // Closed while in use (the file changed, or the cache was cleared): not reused. - if GENERATION.load(Ordering::SeqCst) != generation { + if all.generation != generation || all.epochs.get(path).copied().unwrap_or(0) != epoch { return r; } // A few files are open at a time: past that many, start again rather than grow. - if all.len() >= 16 && !all.contains_key(path) { - all.clear(); + if all.connections.len() >= 16 && !all.connections.contains_key(path) { + all.connections.clear(); } - let slot = all.entry(path.to_path_buf()).or_default(); + let slot = all.connections.entry(path.to_path_buf()).or_default(); if slot.len() < POOL { slot.push(conn); } @@ -342,55 +347,76 @@ mod file { /// Forgets the open connections to `path` (it changed on disk, or was closed). pub fn close(path: &Path) { let mut all = pool().lock().unwrap_or_else(|e| e.into_inner()); - GENERATION.fetch_add(1, Ordering::SeqCst); - all.remove(path); + let epoch = all.epochs.entry(path.to_path_buf()).or_default(); + *epoch = epoch.wrapping_add(1); + all.connections.remove(path); } /// Forgets every open connection. pub fn close_all() { let mut all = pool().lock().unwrap_or_else(|e| e.into_inner()); - GENERATION.fetch_add(1, Ordering::SeqCst); - all.clear(); + all.generation = all.generation.wrapping_add(1); + all.connections.clear(); } /// The metadata of the MBTiles file at `path`. pub fn open_path(path: &Path) -> Result { - with(path, |c| { - let sql = |e: rusqlite::Error| format!("{}: not a readable MBTiles file ({e})", path.display()); - let mut meta = Vec::new(); - if let Ok(mut st) = c.prepare("SELECT name, value FROM metadata LIMIT 1000") { - let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))).map_err(sql)?; - meta.extend(rows.filter_map(Result::ok)); + with(path, |c| read_info(c, path)) + } + + /// Validates a new connection without borrowing or populating the retained pool. + pub fn read_info_path(path: &Path) -> Result { + let c = open(path)?; + let info = read_info(&c, path)?; + // Even complete metadata must not hide a damaged first tile page. This bounded read + // checks the table/view is usable without scanning or decoding the tile set. + let _: Option>> = c + .query_row("SELECT substr(tile_data, 1, 16) FROM tiles LIMIT 1", [], |row| row.get(0)) + .optional() + .map_err(|e| format!("{}: not a readable MBTiles file ({e})", path.display()))?; + Ok(info) + } + + fn read_info(c: &Connection, path: &Path) -> Result { + let sql = |e: rusqlite::Error| format!("{}: not a readable MBTiles file ({e})", path.display()); + let mut meta = Vec::new(); + let has_metadata: bool = c + .query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name = 'metadata' COLLATE NOCASE)", [], |r| r.get(0)) + .map_err(sql)?; + if has_metadata { + let mut st = c.prepare("SELECT name, value FROM metadata LIMIT 1000").map_err(sql)?; + let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))).map_err(sql)?; + for row in rows { + meta.push(row.map_err(sql)?); } - // That it is an MBTiles file: the tiles table (or view) has its columns. Nothing is read. - c.prepare("SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles LIMIT 0").map_err(sql)?; - // What the metadata says, if it is usable: a zoom that reads as a number, a format - // RustGIS knows. - let says = |k: &str, ok: &dyn Fn(&str) -> bool| { - meta.iter().find(|(n, _)| n.eq_ignore_ascii_case(k)).is_some_and(|(_, v)| ok(v.trim())) - }; - let zoom = |v: &str| v.parse::().is_ok_and(f64::is_finite); - let format = |v: &str| super::Format::from_name(v) != super::Format::Unknown; - // The tiles are looked at only for what the metadata leaves out: finding the zooms can - // scan the whole table (or join a deduplicated layout's view). - let zooms = if says("minzoom", &zoom) && says("maxzoom", &zoom) { - None - } else { - let z: (Option, Option) = - c.query_row("SELECT min(zoom_level), max(zoom_level) FROM tiles", [], |r| Ok((r.get(0)?, r.get(1)?))).map_err(sql)?; - match z { - (Some(a), Some(b)) => Some((a.clamp(0, 24) as u8, b.clamp(0, 24) as u8)), - _ => None, - } - }; - // A tile's first bytes say its format when the metadata doesn't. - let first: Option> = if says("format", &format) { - None - } else { - c.query_row("SELECT substr(tile_data, 1, 16) FROM tiles LIMIT 1", [], |r| r.get(0)).optional().map_err(sql)? - }; - Ok(Info::from_metadata(&meta, zooms, first.as_deref())) - }) + } + // That it is an MBTiles file: the tiles table (or view) has its columns. Nothing is read. + c.prepare("SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles LIMIT 0").map_err(sql)?; + // What the metadata says, if it is usable: a zoom that reads as a number, a format + // RustGIS knows. + let says = + |k: &str, ok: &dyn Fn(&str) -> bool| meta.iter().find(|(n, _)| n.eq_ignore_ascii_case(k)).is_some_and(|(_, v)| ok(v.trim())); + let zoom = |v: &str| v.parse::().is_ok_and(f64::is_finite); + let format = |v: &str| super::Format::from_name(v) != super::Format::Unknown; + // The tiles are looked at only for what the metadata leaves out: finding the zooms can + // scan the whole table (or join a deduplicated layout's view). + let zooms = if says("minzoom", &zoom) && says("maxzoom", &zoom) { + None + } else { + let z: (Option, Option) = + c.query_row("SELECT min(zoom_level), max(zoom_level) FROM tiles", [], |r| Ok((r.get(0)?, r.get(1)?))).map_err(sql)?; + match z { + (Some(a), Some(b)) => Some((a.clamp(0, 24) as u8, b.clamp(0, 24) as u8)), + _ => None, + } + }; + // A tile's first bytes say its format when the metadata doesn't. + let first: Option> = if says("format", &format) { + None + } else { + c.query_row("SELECT substr(tile_data, 1, 16) FROM tiles LIMIT 1", [], |r| r.get(0)).optional().map_err(sql)? + }; + Ok(Info::from_metadata(&meta, zooms, first.as_deref())) } /// The tile at z/x/y (`y` from the north) of the file at `path`, `None` when it has none. @@ -414,10 +440,60 @@ mod file { } }) } + + #[cfg(test)] + mod reload_tests { + use super::*; + + #[test] + fn closing_one_file_rejects_its_old_connection_but_keeps_another_file() { + let root = std::env::temp_dir().join(format!("rustgis-mbtiles-pool-reload-{}", std::process::id())); + std::fs::create_dir_all(&root).unwrap(); + let path = root.join("reload.mbtiles"); + let other = root.join("other.mbtiles"); + for p in [&path, &other] { + let c = Connection::open(p).unwrap(); + c.execute_batch( + "CREATE TABLE IF NOT EXISTS tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + DELETE FROM tiles; INSERT INTO tiles VALUES(0,0,0,x'01');", + ) + .unwrap(); + } + assert_eq!(tile_path(&other, 0, 0, 0).unwrap(), Some(vec![1])); + let (started_tx, started_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let read_path = path.clone(); + let worker = std::thread::spawn(move || { + with(&read_path, |c| { + let tile: Vec = c.query_row("SELECT tile_data FROM tiles LIMIT 1", [], |r| r.get(0)).map_err(|e| e.to_string())?; + started_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + Ok(tile) + }) + .unwrap() + }); + started_rx.recv_timeout(std::time::Duration::from_secs(10)).unwrap(); + close(&path); + assert!(pool().lock().unwrap().connections.get(&other).is_some_and(|v| !v.is_empty())); + release_tx.send(()).unwrap(); + assert_eq!(worker.join().unwrap(), vec![1]); + assert!( + pool().lock().unwrap().connections.get(&path).is_none_or(Vec::is_empty), + "a read admitted before close returned its obsolete connection" + ); + let c = Connection::open(&path).unwrap(); + c.execute_batch("UPDATE tiles SET tile_data=x'02'").unwrap(); + drop(c); + assert_eq!(tile_path(&path, 0, 0, 0).unwrap(), Some(vec![2])); + close(&path); + close(&other); + std::fs::remove_dir_all(root).unwrap(); + } + } } #[cfg(not(target_arch = "wasm32"))] -pub use file::{close, close_all, open_path, tile_path}; +pub use file::{close, close_all, open_path, read_info_path, tile_path}; #[cfg(test)] mod tests { diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index 1603f632e5..6ba28e8988 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -306,7 +306,7 @@ const NEVER: &[&str] = &[ const REACHES_FILES: &[&str] = &["project.save", "project.saveAs", "project.log"]; /// Commands that read a file a layer names, not their parameters. -const READS_FILES: &[&str] = &["gpkg.discardEdits"]; +const READS_FILES: &[&str] = &["gpkg.discardEdits", "map.reloadTiles"]; /// Parameter keys that name a file, folder or address. (`data` and `text` carry content, which /// plugins may pass: `layer.addDataBytes`, `map.import` from text.) diff --git a/crates/engine/src/cmd/repair.rs b/crates/engine/src/cmd/repair.rs index c5b123b31e..7eabd13d1b 100644 --- a/crates/engine/src/cmd/repair.rs +++ b/crates/engine/src/cmd/repair.rs @@ -11,6 +11,8 @@ pub fn commands() -> Vec { r#"{"layer"?: name|id (default: the active layer), "path": "the file or folder (a Zarr store, a File Geodatabase) the layer's data moved to", "table"?: "the dataset in a GeoPackage, GDAL or DuckDB file, or a raster's subdataset (default: the one the layer reads now)"}"#), C::new("map.repairTiles", "Repair Tile Archive", "Layers", repair_tiles).params( r#"{"name"?: text | "index"?: number (0 = top) | "id"?: catalog id, "path": "the .pmtiles or .mbtiles file the archive moved to"}"#), + C::new("map.reloadTiles", "Reload Data", "Layers", reload_tiles).params( + r#"{"name": text | "index": number (0 = top) | "id": catalog id} (give one selector) — rereads a local PMTiles or MBTiles archive at its linked path, keeping its presentation; undo cannot restore overwritten file contents"#), ] } @@ -125,30 +127,70 @@ fn repair_tiles(s: &mut Session, p: &Value) -> CmdResult { } #[cfg(not(target_arch = "wasm32"))] { - use rustgis_render::rustgis_tiles::{cache, mbtiles, pmtiles}; let i = super::tiles::find(s, p)?.ok_or("name, index or id is required")?; let t = s.map().tiles.get(i).ok_or("the tile layer no longer exists")?; if t.archive_file().is_none() { return Err(format!("{}: only a tile archive read from a file (PMTiles or MBTiles) can be repaired", t.name)); } let full = replacement(p)?; - let mut new = t.clone(); - new.url = full.clone(); - new.error = None; - new.repair()?; - let vector = match new.kind { - rustgis_map::TileKind::MbTiles => mbtiles::info(&full)?.format.is_vector(), - _ => pmtiles::info_blocking(&full, web_time::Duration::from_secs(10))?.tile_type.is_vector(), - }; - if (!t.layers.is_empty() || !t.style_layers.is_empty()) && !vector { - return Err(format!( - "{full} holds raster tiles; {} draws vector tiles with its own layers or style: choose a vector tile archive", - t.name - )); + let new = archive_candidate(t, &full)?; + rustgis_render::rustgis_tiles::cache::reload_archive(&t.url); + // The replacement path may already have been drawn by another layer. + if t.url != full { + rustgis_render::rustgis_tiles::cache::reload_archive(&full); } - cache::forget(&t.url); let slot = s.map_mut().tiles.get_mut(i).ok_or("the tile layer no longer exists")?; *slot = new; Ok(json!({"index": i, "path": full})) } } + +/// Repair and reload share one validation policy. Reading a candidate must not evict retained +/// tiles/metadata or reuse a connection to an archive replaced at the same path. +#[cfg(not(target_arch = "wasm32"))] +fn archive_candidate(t: &rustgis_map::TileLayer, path: &str) -> Result { + use rustgis_render::rustgis_tiles::{mbtiles, pmtiles}; + let mut new = t.clone(); + new.url = path.to_string(); + new.error = None; + new.repair()?; + let vector = match new.kind { + rustgis_map::TileKind::MbTiles => mbtiles::read_info_local(path)?.format.is_vector(), + _ => pmtiles::read_info_local(path)?.tile_type.is_vector(), + }; + if (!t.layers.is_empty() || !t.style_layers.is_empty()) && !vector { + return Err(format!( + "{path} holds raster tiles; {} draws vector tiles with its own layers or style: choose a vector tile archive", + t.name + )); + } + Ok(new) +} + +fn reload_tiles(s: &mut Session, p: &Value) -> CmdResult { + #[cfg(target_arch = "wasm32")] + { + let _ = (s, p); + Err("the web app can't reload files on this computer: add the file again with Add Data".into()) + } + #[cfg(not(target_arch = "wasm32"))] + { + let i = super::tiles::find(s, p)?.ok_or("name, index or id is required")?; + let t = s.map().tiles.get(i).ok_or("the tile layer no longer exists")?; + let path = t.archive_file().ok_or_else(|| { + format!( + "{}: Reload Data needs a linked local PMTiles or MBTiles file; add snapshots/uploads again, or use Reload Web Layers for web addresses", + t.name + ) + })?.to_string(); + let new = archive_candidate(t, &path).map_err(|e| { + format!("{} ({path}): {e}; correct the file and retry Reload Data, or use Repair Data Source if it moved", t.name) + })?; + rustgis_render::rustgis_tiles::cache::reload_archive(&path); + let slot = s.map_mut().tiles.get_mut(i).ok_or("the tile layer no longer exists")?; + *slot = new; + let message = format!("Reloaded {} from {path}; undo cannot restore overwritten file contents", slot.name); + s.notify(message, false); + Ok(json!({"index": i, "path": path})) + } +} diff --git a/crates/engine/tests/portable_projects.rs b/crates/engine/tests/portable_projects.rs index 74a64522f0..92e68d8fd0 100644 --- a/crates/engine/tests/portable_projects.rs +++ b/crates/engine/tests/portable_projects.rs @@ -2,6 +2,8 @@ #[path = "portable_projects/common.rs"] mod common; +#[path = "portable_projects/reload_tiles.rs"] +mod reload_tiles; #[path = "portable_projects/repair.rs"] mod repair; diff --git a/crates/engine/tests/portable_projects/reload_tiles.rs b/crates/engine/tests/portable_projects/reload_tiles.rs new file mode 100644 index 0000000000..b34eae8665 --- /dev/null +++ b/crates/engine/tests/portable_projects/reload_tiles.rs @@ -0,0 +1,289 @@ +use super::common::{Fixture, session}; +use rustgis_engine::{ + Session, + map::TileLayer, + render::{ + self, + rustgis_tiles::{cache, mbtiles, pmtiles, source}, + }, +}; +use serde_json::{Value, json}; +use std::{path::Path, sync::Arc, time::Duration}; + +fn picture(color: [u8; 4]) -> Vec { + let mut p = tiny_skia::Pixmap::new(256, 256).unwrap(); + p.fill(tiny_skia::Color::from_rgba8(color[0], color[1], color[2], color[3])); + p.encode_png().unwrap() +} + +fn write_archive(path: &Path, color: [u8; 4], credit: &str) { + let png = picture(color); + if path.extension().unwrap() == "pmtiles" { + let a = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Png, + gzipped_tiles: false, + bounds: [-180.0, -85.0, 180.0, 85.0], + metadata: json!({"attribution": credit}), + }; + std::fs::write(path, pmtiles::write(vec![(0, 0, 0, png)], &a).unwrap()).unwrap(); + } else { + let c = rusqlite::Connection::open(path).unwrap(); + c.execute_batch("CREATE TABLE IF NOT EXISTS metadata (name TEXT, value TEXT); CREATE TABLE IF NOT EXISTS tiles (zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_data BLOB); BEGIN IMMEDIATE; DELETE FROM metadata; DELETE FROM tiles;").unwrap(); + c.execute("INSERT INTO metadata VALUES ('format','png'),('minzoom','0'),('maxzoom','0'),('attribution',?1)", [credit]).unwrap(); + c.execute("INSERT INTO tiles VALUES (0,0,0,?1)", [png]).unwrap(); + c.execute_batch("COMMIT").unwrap(); + } +} + +fn tile_url(t: &TileLayer) -> String { + let template = if t.kind == rustgis_engine::map::TileKind::MbTiles { mbtiles::template(&t.url) } else { pmtiles::template(&t.url) }; + template.replace("{z}", "0").replace("{x}", "0").replace("{y}", "0") +} + +fn decoded(t: &TileLayer) -> Arc { + let url = tile_url(t); + for _ in 0..100 { + match cache::get_tile(&url, cache::Want::Image) { + cache::State::Ready(p) => return p, + cache::State::Failed(e) => panic!("{e}"), + cache::State::Pending => cache::wait(Duration::from_millis(100)), + } + } + panic!("tile did not arrive: {url}"); +} + +fn style(t: &TileLayer) -> Arc { + for _ in 0..100 { + if t.kind == rustgis_engine::map::TileKind::MbTiles { + // MBTiles metadata workers have their own bounded wait, not cache::in_flight. + let _ = mbtiles::info_ready(&t.url, Duration::from_millis(100)).unwrap(); + } + match source::resolve(&render::tile_spec(t)) { + source::Resolve::Ready(r) => return r, + source::Resolve::Failed(e) => panic!("{e}"), + source::Resolve::Pending => cache::wait(Duration::from_millis(100)), + } + } + panic!("style did not resolve"); +} + +fn assert_color(p: &cache::Payload, expected: [u8; 4]) { + let cache::Payload::Image(p) = p else { panic!("not an image") }; + assert_eq!(p.data().get(..4), Some(expected.as_slice())); +} + +fn assert_render(s: &mut Session, expected: [u8; 4], credit: &str) { + let v = render::View { width: 300.0, height: 200.0, pixels_per_point: 1.0, camera: s.map().camera, sky: false }; + for _ in 0..100 { + let r = render::render(s.map_mut(), &v).unwrap(); + if !r.tiles_pending { + assert_eq!(r.pixmap.pixel(150, 100).unwrap().red(), expected[0]); + assert_eq!(r.pixmap.pixel(150, 100).unwrap().green(), expected[1]); + assert_eq!(r.pixmap.pixel(150, 100).unwrap().blue(), expected[2]); + assert!(r.attribution.iter().any(|a| a.contains(credit)), "{:?}", r.attribution); + return; + } + cache::wait(Duration::from_millis(100)); + } + panic!("render remained pending"); +} + +#[test] +fn manual_archive_reload_reads_new_pixels_and_credits_without_touching_other_sources() { + for ext in ["pmtiles", "mbtiles"] { + let f = Fixture::new(&format!("reload-{ext}")); + let path = f.file(&format!("linked.{ext}")); + let other = f.file(&format!("unrelated.{ext}")); + write_archive(&path, [255, 0, 0, 255], "Old credits"); + write_archive(&other, [0, 0, 255, 255], "Unrelated credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url": other, "name":"Other", "overlay":true})).unwrap(); + s.run("map.addTiles", &json!({"url": path, "name":"Linked", "overlay":true})).unwrap(); + s.run("map.goToXY", &json!({"lon":10,"lat":20,"scale":50_000_000})).unwrap(); + let i = s.map().tiles.iter().position(|t| t.name == "Linked").unwrap(); + s.map_mut().tiles[i].id = "linked-archive".into(); + s.map_mut().tiles[i].blend = rustgis_engine::map::effects::Blend::Normal; + s.map_mut().tiles[i].expanded = true; + let before = s.map().tiles.clone(); + let camera = s.map().camera; + let other_tile = before.iter().find(|t| t.name == "Other").unwrap(); + let other_pixels = decoded(other_tile); + let other_style = style(other_tile); + let old_pixels = decoded(&before[i]); + assert_color(&old_pixels, [255, 0, 0, 255]); + assert!(style(&before[i]).attribution.iter().any(|a| a.contains("Old credits"))); + assert_render(&mut s, [255, 0, 0, 255], "Old credits"); + write_archive(&path, [0, 255, 0, 255], "New credits"); + // Existing retained data stays until the explicit command. + assert!(Arc::ptr_eq(&decoded(&before[i]), &old_pixels)); + assert_eq!(s.run("map.reloadTiles", &json!({"id":"linked-archive"})).unwrap(), json!({"index":i,"path":path})); + assert_eq!(s.map().tiles, before); + assert_eq!(s.map().camera, camera); + assert_color(&decoded(&before[i]), [0, 255, 0, 255]); + let fresh = style(&before[i]); + assert!(fresh.attribution.iter().any(|a| a.contains("New credits"))); + assert!(!fresh.attribution.iter().any(|a| a.contains("Old credits"))); + assert!(Arc::ptr_eq(&decoded(other_tile), &other_pixels)); + assert!(Arc::ptr_eq(&style(other_tile), &other_style)); + assert_render(&mut s, [0, 255, 0, 255], "New credits"); + assert_eq!(s.undo_label(), Some("Reload Data")); + s.run("edit.undo", &Value::Null).unwrap(); + assert_eq!(s.map().tiles, before); + assert_color(&decoded(&before[i]), [0, 255, 0, 255]); // Undo is not file recovery. + s.run("edit.redo", &Value::Null).unwrap(); + assert_eq!(s.map().camera, camera); + // All three selector forms, including case-insensitive names. + s.run("map.reloadTiles", &json!({"name":"lInKeD"})).unwrap(); + s.run("map.reloadTiles", &json!({"index":i})).unwrap(); + rustgis_engine::data::mbtiles::close(&path); + rustgis_engine::data::mbtiles::close(&other); + } +} + +fn rejected(s: &mut Session, params: Value, expected: &str) { + let before = s.map().tiles.clone(); + let undo = s.undo_label(); + let mark = s.notice_mark(); + let error = s.run("map.reloadTiles", ¶ms).unwrap_err(); + assert!(error.contains(expected), "{error}"); + assert_eq!(s.map().tiles, before); + assert_eq!(s.undo_label(), undo); + assert!(s.notices_since(mark).iter().any(|n| n.error && n.text.contains(expected))); +} + +#[test] +fn failed_archive_reload_retains_tiles_configuration_and_allows_retry() { + for ext in ["pmtiles", "mbtiles"] { + let f = Fixture::new(&format!("reload-failure-{ext}")); + let path = f.file(&format!("linked.{ext}")); + write_archive(&path, [255, 0, 0, 255], "Retained credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path})).unwrap(); + let before = s.map().tiles[0].clone(); + let pixels = decoded(&before); + let old_style = style(&before); + // No live read is held by this test while changing the file on Windows. + rustgis_engine::data::mbtiles::close(&path); + std::fs::remove_file(&path).unwrap(); + rejected(&mut s, json!({"index":0}), "retry Reload Data"); + std::fs::write(&path, b"truncated").unwrap(); + rejected(&mut s, json!({"index":0}), "retry Reload Data"); + assert!(Arc::ptr_eq(&decoded(&before), &pixels)); + assert!(Arc::ptr_eq(&style(&before), &old_style)); + write_archive_after_truncation(&path, [0, 255, 0, 255], "Recovered credits"); + s.run("map.reloadTiles", &json!({"index":0})).unwrap(); + assert_color(&decoded(&before), [0, 255, 0, 255]); + assert_eq!(s.map().tiles[0], before); + rustgis_engine::data::mbtiles::close(&path); + } +} + +fn write_archive_after_truncation(path: &Path, color: [u8; 4], credit: &str) { + std::fs::remove_file(path).unwrap(); + write_archive(path, color, credit); +} + +#[test] +fn locked_mbtiles_reload_fails_without_evicting_retained_data_then_retries() { + let f = Fixture::new("reload-locked"); + let path = f.file("locked.mbtiles"); + write_archive(&path, [255, 0, 0, 255], "Before lock"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path})).unwrap(); + let before = s.map().tiles[0].clone(); + let pixels = decoded(&before); + let c = rusqlite::Connection::open(&path).unwrap(); + c.execute_batch("BEGIN EXCLUSIVE; UPDATE metadata SET value='After lock' WHERE name='attribution';").unwrap(); + rejected(&mut s, json!({"index":0}), "retry Reload Data"); + assert!(Arc::ptr_eq(&decoded(&before), &pixels)); + c.execute_batch("COMMIT").unwrap(); + s.run("map.reloadTiles", &json!({"index":0})).unwrap(); + assert!(style(&before).attribution.iter().any(|a| a.contains("After lock"))); + rustgis_engine::data::mbtiles::close(&path); +} + +// Unix permits replacing an open archive's directory entry. Windows may refuse the rename +// while SQLite holds the original file; this test does not promise otherwise. +#[cfg(unix)] +#[test] +fn replacing_an_open_mbtiles_archive_reloads_the_new_file_not_its_pooled_inode() { + let f = Fixture::new("reload-mbtiles-replaced-inode"); + let path = f.file("linked.mbtiles"); + let replacement = f.file("replacement.mbtiles"); + write_archive(&path, [255, 0, 0, 255], "Old inode"); + write_archive(&replacement, [0, 255, 0, 255], "Replacement inode"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path})).unwrap(); + let before = s.map().tiles[0].clone(); + assert_color(&decoded(&before), [255, 0, 0, 255]); + assert!(style(&before).attribution.iter().any(|a| a.contains("Old inode"))); + std::fs::rename(&replacement, &path).unwrap(); + s.run("map.reloadTiles", &json!({"index":0})).unwrap(); + assert_eq!(s.map().tiles[0], before); + assert_color(&decoded(&before), [0, 255, 0, 255]); + assert!(style(&before).attribution.iter().any(|a| a.contains("Replacement inode"))); + rustgis_engine::data::mbtiles::close(&path); +} + +#[test] +fn incompatible_reload_keeps_archive_presentation_and_undo_restores_prior_error() { + for ext in ["pmtiles", "mbtiles"] { + let f = Fixture::new(&format!("reload-incompatible-{ext}")); + let path = f.file(&format!("vector.{ext}")); + write_vector(&path); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path})).unwrap(); + s.map_mut().tiles[0].layers = vec!["places".into()]; + s.map_mut().tiles[0].style_layers = + r##"[{"id":"places","type":"circle","source-layer":"places","paint":{"circle-color":"#2266aa"}}]"##.into(); + s.map_mut().tiles[0].hidden_layers = vec!["roads".into()]; + s.map_mut().tiles[0].opacity = 0.4; + s.map_mut().tiles[0].visible = false; + s.map_mut().tiles[0].blend = rustgis_engine::map::effects::Blend::Multiply; + s.map_mut().tiles[0].id = "presentation".into(); + s.map_mut().tiles[0].error = Some("old unavailable archive".into()); + let before = s.map().tiles[0].clone(); + write_archive(&path, [0, 255, 0, 255], "Raster now"); + rejected(&mut s, json!({"index":0}), "raster tiles"); + write_vector(&path); + s.run("map.reloadTiles", &json!({"index":0})).unwrap(); + let mut expected = before.clone(); + expected.error = None; + assert_eq!(s.map().tiles[0], expected); + s.run("edit.undo", &Value::Null).unwrap(); + assert_eq!(s.map().tiles[0], before); + rustgis_engine::data::mbtiles::close(&path); + } +} + +fn write_vector(path: &Path) { + if path.extension().unwrap() == "pmtiles" { + super::common::archive_file(path); + } else { + write_archive(path, [255, 0, 0, 255], "Vector"); + let c = rusqlite::Connection::open(path).unwrap(); + c.execute("UPDATE metadata SET value='pbf' WHERE name='format'", []).unwrap(); + let tile = rustgis_engine::render::rustgis_tiles::mvt::encode(&rustgis_engine::render::rustgis_tiles::mvt::Tile { layers: vec![] }); + c.execute("UPDATE tiles SET tile_data=?1", [tile]).unwrap(); + } +} + +#[test] +fn archive_reload_rejects_remote_upload_and_invalid_selectors() { + let mut s = session(); + s.run("map.addTiles", &json!({"url":"https://example.invalid/a.pmtiles"})).unwrap(); + rejected(&mut s, json!({"index":0}), "linked local PMTiles or MBTiles"); + // Uploaded archives are snapshots, not live paths. + s.map_mut().tiles[0] = TileLayer::new("Snapshot", rustgis_engine::map::TileKind::MbTiles, "upload:local.mbtiles"); + rejected(&mut s, json!({"index":0}), "add snapshots/uploads again"); + for (p, expected) in [ + (json!({}), "required"), + (json!({"index":-1}), "no tile layer"), + (json!({"index":0.5}), "no tile layer"), + (json!({"index":8}), "no tile layer"), + (json!({"name":"absent"}), "no tile layer"), + (json!({"id":"absent"}), "no tile layer"), + ] { + rejected(&mut s, p, expected); + } +} diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 2d0b4bf1d0..881de853dd 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -84,6 +84,7 @@ struct Slot { used: u64, /// When the map last asked for it ([`get`]): a waiting tile not asked for lately is dropped. asked: Instant, + epoch: u64, } #[derive(Default)] @@ -95,6 +96,22 @@ struct Cache { latest: Option, } +impl Cache { + fn forget_archive(&mut self, archive: &str) { + let mut dropped = 0; + self.slots.retain(|key, slot| { + if resource_of(key) != archive { + return true; + } + if let Entry::Ready(_, bytes) = &slot.entry { + dropped += bytes; + } + false + }); + self.bytes = self.bytes.saturating_sub(dropped); + } +} + fn cache() -> &'static Mutex { static C: OnceLock> = OnceLock::new(); C.get_or_init(Mutex::default) @@ -107,6 +124,26 @@ fn lock() -> std::sync::MutexGuard<'static, Cache> { static GENERATION: AtomicU64 = AtomicU64::new(0); static IN_FLIGHT: AtomicUsize = AtomicUsize::new(0); +/// Source revisions are locked before any retained cache. Publication and reload share this +/// lock; callbacks and I/O must always run after releasing it. +fn archive_epochs() -> &'static Mutex> { + static E: std::sync::LazyLock>> = std::sync::LazyLock::new(Mutex::default); + &E +} + +pub(crate) fn archive_epoch(url: &str) -> u64 { + archive_epochs().lock().unwrap_or_else(|e| e.into_inner()).get(resource_of(url)).copied().unwrap_or(0) +} + +pub(crate) fn if_archive_current(url: &str, epoch: u64, f: impl FnOnce() -> T) -> Option { + let epochs = archive_epochs().lock().unwrap_or_else(|e| e.into_inner()); + if epochs.get(resource_of(url)).copied().unwrap_or(0) != epoch { + drop(epochs); + return None; + } + Some(f()) +} + fn fetcher() -> &'static Mutex>> { static F: OnceLock>>> = OnceLock::new(); F.get_or_init(Mutex::default) @@ -206,6 +243,8 @@ fn request(url: &str, want: Want, omit: &'static [u16]) -> State { if let Some(base) = crate::warp::direct(url) { return request(base, want, omit); } + let epochs = archive_epochs().lock().unwrap_or_else(|e| e.into_inner()); + let epoch = epochs.get(resource_of(url)).copied().unwrap_or(0); let now = Instant::now(); let mut c = lock(); c.tick += 1; @@ -223,6 +262,7 @@ fn request(url: &str, want: Want, omit: &'static [u16]) -> State { } if crate::warp::is_warp(url) { drop(c); + drop(epochs); // Made once its pictures are here; asked again (as the map draws) until then. return match crate::warp::make(url) { crate::warp::Made::Waiting => State::Pending, @@ -239,9 +279,10 @@ fn request(url: &str, want: Want, omit: &'static [u16]) -> State { } }; } - c.slots.insert(url.to_string(), Slot { entry: Entry::Pending, used: tick, asked: now }); + c.slots.insert(url.to_string(), Slot { entry: Entry::Pending, used: tick, asked: now, epoch }); drop(c); - start(url.to_string(), want, omit); + drop(epochs); + start(url.to_string(), want, omit, epoch); State::Pending } @@ -289,7 +330,7 @@ fn resource_of(key: &str) -> &str { let key = split_download_limit(key).0; let key = split_tail(key).0; let key = split_range(key).0; - if key.starts_with(crate::pmtiles::SCHEME) || key.starts_with(crate::cog::SCHEME) { + if key.starts_with(crate::pmtiles::SCHEME) || key.starts_with(crate::mbtiles::SCHEME) || key.starts_with(crate::cog::SCHEME) { return key.split_once('@').map_or(key, |(_, archive)| crate::cog::split(archive).0); } key @@ -322,6 +363,22 @@ pub fn forget(url: &str) { GENERATION.fetch_add(1, Ordering::Relaxed); } +/// Reloads just this archive, including pending reads, metadata, directories, handles and +/// derived styles. Call only after uncached validation succeeds. Older answers are discarded. +pub fn reload_archive(url: &str) { + let mut epochs = archive_epochs().lock().unwrap_or_else(|e| e.into_inner()); + let epoch = epochs.entry(url.to_string()).or_default(); + *epoch = epoch.wrapping_add(1); + lock().forget_archive(url); + let abandoned = crate::pmtiles::reload_archive(url); + crate::mbtiles::reload_archive(url); + crate::source::forget_archive(url); + drop(epochs); + // Dropping callbacks can complete tile answers and start queued reads: never under locks. + drop(abandoned); + notify(); +} + /// Waits (up to `timeout`) for the fetches under way to finish. Native only: in the browser /// fetches finish between frames, so this returns at once. pub fn wait(timeout: Duration) { @@ -375,7 +432,7 @@ pub fn wait_for_news_unless(since: u64, timeout: Duration, stop: impl Fn() -> bo } /// A waiting fetch: what to decode it as, whose diagnostics it goes to, and its expected statuses. -type Job = (Want, Capture, &'static [u16]); +type Job = (Want, Capture, &'static [u16], u64); fn queue() -> &'static Mutex> { static Q: OnceLock>> = OnceLock::new(); @@ -383,7 +440,7 @@ fn queue() -> &'static Mutex> { } /// Asks for a tile: fetched now when its server has room, else it waits its turn. -fn start(url: String, want: Want, omit: &'static [u16]) { +fn start(url: String, want: Want, omit: &'static [u16], epoch: u64) { let capture = Capture::current(); IN_FLIGHT.fetch_add(1, Ordering::Relaxed); let mut swept = Vec::new(); @@ -391,7 +448,7 @@ fn start(url: String, want: Want, omit: &'static [u16]) { let mut q = queue().lock().unwrap_or_else(|e| e.into_inner()); let mut c = lock(); let latest = c.latest.unwrap_or_else(Instant::now); - let now = q.push(url, (want, capture, omit), |u| c.slots.get(u).map(|s| s.asked), latest, &mut swept); + let now = q.push(url, (want, capture, omit, epoch), |u| c.slots.get(u).map(|s| s.asked), latest, &mut swept); // Under the same lock as the choice: a tile asked for again meanwhile isn't forgotten. forget_dropped(&mut c, &swept); now @@ -417,7 +474,7 @@ fn release(host: &str) { /// Called holding the cache's lock taken to choose them, so a tile can't be asked for between. fn forget_dropped(c: &mut Cache, dropped: &[Waiting]) { for w in dropped { - if matches!(c.slots.get(&w.url), Some(Slot { entry: Entry::Pending, .. })) { + if matches!(c.slots.get(&w.url), Some(Slot { entry: Entry::Pending, epoch, .. }) if *epoch == w.job.3) { c.slots.remove(&w.url); } } @@ -450,9 +507,9 @@ fn launch(jobs: Vec>) { } let _lower = Lower; while let Some(w) = pop() { - let (want, capture, omit) = w.job; + let (want, capture, omit, epoch) = w.job; let _scope = capture.enter(); - fetch_tile(w.url, w.host, want, omit); + fetch_tile(w.url, w.host, want, omit, epoch); } } // A tile queued after the last `pop` and before the flag went down is started here. @@ -469,6 +526,7 @@ struct Answer { key: String, host: String, entry: Option, + epoch: u64, } impl Answer { @@ -480,23 +538,27 @@ impl Answer { impl Drop for Answer { fn drop(&mut self) { - match self.entry.take() { + let entry = self.entry.take(); + if_archive_current(&self.key, self.epoch, || match entry { Some(entry) => store(&self.key, entry), - // Never answered (or dropped by the fetcher): forgotten, fetched again when asked for. + // An old unanswered fetch must not remove a newer pending request. None => { let mut c = lock(); if matches!(c.slots.get(&self.key), Some(Slot { entry: Entry::Pending, .. })) { c.slots.remove(&self.key); } } - } + }); release(&self.host); finished(); } } -fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16]) { - let answer = Answer { key: url.clone(), host, entry: None }; +fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16], epoch: u64) { + let answer = Answer { key: url.clone(), host, entry: None, epoch }; + if archive_epoch(&url) != epoch { + return; + } let done: Done = Box::new(move |bytes| { if matches!(&bytes, Err(e) if e == DROPPED) { return; @@ -1063,17 +1125,22 @@ fn store(url: &str, entry: Entry) { return; } let mut c = lock(); + store_in(&mut c, url, entry); +} + +fn store_in(c: &mut Cache, url: &str, entry: Entry) { let added = if let Entry::Ready(_, n) = &entry { *n } else { 0 }; // Dropped while it was fetched (cleared, or evicted): kept anyway, it was asked for. let tick = c.tick; let asked = c.slots.get(url).map_or_else(Instant::now, |s| s.asked); - let old = c.slots.insert(url.to_string(), Slot { entry, used: tick, asked }); + let epoch = c.slots.get(url).map_or(0, |s| s.epoch); + let old = c.slots.insert(url.to_string(), Slot { entry, used: tick, asked, epoch }); if let Some(Slot { entry: Entry::Ready(_, n), .. }) = old { c.bytes = c.bytes.saturating_sub(n); } c.bytes += added; if c.bytes > BUDGET { - evict(&mut c); + evict(c); } } @@ -1842,6 +1909,75 @@ mod native { mod tests { use super::*; + #[test] + fn archive_reload_excludes_old_decodes_and_keeps_other_sources() { + let archive = "/cache-generation-regression.pmtiles"; + let tile = format!("pmtiles:0/0/0@{archive}"); + let other = "mbtiles:0/0/0@/cache-generation-other.mbtiles"; + let old_epoch = archive_epoch(archive); + let mut c = Cache::default(); + store_in(&mut c, &tile, Entry::Ready(Arc::new(Payload::Json(serde_json::json!("old"))), 64)); + store_in(&mut c, other, Entry::Ready(Arc::new(Payload::Empty), 64)); + let other_payload = match &c.slots[other].entry { + Entry::Ready(payload, _) => payload.clone(), + _ => panic!("ready tile"), + }; + let pending = format!("pmtiles:1/0/0@{archive}"); + c.slots.insert(pending.clone(), Slot { entry: Entry::Pending, used: 0, asked: Instant::now(), epoch: old_epoch }); + reload_archive(archive); + c.forget_archive(archive); + assert!(!c.slots.contains_key(&tile) && !c.slots.contains_key(&pending)); + assert_eq!(c.bytes, 64); + let fresh_epoch = archive_epoch(archive); + c.slots.insert(tile.clone(), Slot { entry: Entry::Pending, used: 0, asked: Instant::now(), epoch: fresh_epoch }); + assert!( + if_archive_current(&tile, old_epoch, || { + store_in(&mut c, &tile, Entry::Ready(Arc::new(Payload::Json(serde_json::json!("stale"))), 64)); + }) + .is_none() + ); + assert!(matches!(c.slots[&tile].entry, Entry::Pending)); + if_archive_current(&tile, fresh_epoch, || { + store_in(&mut c, &tile, Entry::Ready(Arc::new(Payload::Json(serde_json::json!("fresh"))), 64)); + }); + assert!(matches!(&c.slots[&tile].entry, Entry::Ready(p, _) if matches!(p.as_ref(), Payload::Json(v) if v == "fresh"))); + assert!(matches!(&c.slots[other].entry, Entry::Ready(p, _) if Arc::ptr_eq(p, &other_payload))); + assert_eq!(resource_of(other), "/cache-generation-other.mbtiles"); + } + + #[test] + fn delayed_archive_answers_cannot_replace_new_pixels_or_remove_new_pending_reads() { + for scheme in ["pmtiles", "mbtiles"] { + let archive = format!("/delayed-answer.{scheme}"); + let key = format!("{scheme}:0/0/0@{archive}"); + let old_epoch = archive_epoch(&archive); + let mut old_picture = Pixmap::new(1, 1).unwrap(); + old_picture.fill(tiny_skia::Color::from_rgba8(255, 0, 0, 255)); + // Hold the production completion object after decoding, as an asynchronous decoder + // does. No worker timing or process-wide fetcher replacement is involved. + IN_FLIGHT.fetch_add(2, Ordering::Relaxed); + let late = Answer { + key: key.clone(), + host: "delayed-answer.test".into(), + epoch: old_epoch, + entry: Some(Entry::Ready(Arc::new(Payload::Image(old_picture)), 4)), + }; + let unanswered = Answer { key: key.clone(), host: "delayed-answer.test".into(), epoch: old_epoch, entry: None }; + reload_archive(&archive); + let current = archive_epoch(&archive); + lock().slots.insert(key.clone(), Slot { entry: Entry::Pending, used: 0, asked: Instant::now(), epoch: current }); + drop(unanswered); + assert!(matches!(lock().slots.get(&key).map(|s| &s.entry), Some(Entry::Pending))); + let mut fresh_picture = Pixmap::new(1, 1).unwrap(); + fresh_picture.fill(tiny_skia::Color::from_rgba8(0, 255, 0, 255)); + let fresh = Arc::new(Payload::Image(fresh_picture)); + store(&key, Entry::Ready(fresh.clone(), 4)); + drop(late); + assert!(Arc::ptr_eq(&peek(&key).unwrap(), &fresh), "an old {scheme} decode replaced the new pixels"); + reload_archive(&archive); + } + } + fn network_records(history: &rustgis_data::diagnostics::History, url: &str) -> Vec { history.records().into_iter().filter(|record| record.network.as_ref().is_some_and(|network| network.url == url)).collect() } diff --git a/crates/tiles/src/mbtiles.rs b/crates/tiles/src/mbtiles.rs index 9ceb885c98..5beb9e93aa 100644 --- a/crates/tiles/src/mbtiles.rs +++ b/crates/tiles/src/mbtiles.rs @@ -107,6 +107,34 @@ pub(crate) fn forget() { rustgis_data::mbtiles::close_all(); } +/// Called under the cache's archive-revision lock. +pub(crate) fn reload_archive(archive: &str) { + infos().lock().unwrap_or_else(|e| e.into_inner()).remove(archive); + #[cfg(not(target_arch = "wasm32"))] + rustgis_data::mbtiles::close(std::path::Path::new(archive.strip_prefix("file://").unwrap_or(archive))); +} + +fn info_epoch(archive: &str) -> (u64, u64) { + (EPOCH.load(Ordering::SeqCst), crate::cache::archive_epoch(archive)) +} + +/// Reads the current file using a fresh, unpooled connection and retains nothing. +pub fn read_info_local(archive: &str) -> Result, String> { + check(archive)?; + if is_upload(archive) { + return Err(format!("{archive}: an upload is not a local MBTiles file")); + } + #[cfg(not(target_arch = "wasm32"))] + { + let path = std::path::Path::new(archive.strip_prefix("file://").unwrap_or(archive)); + rustgis_data::mbtiles::read_info_path(path).map(Arc::new) + } + #[cfg(target_arch = "wasm32")] + { + Err(format!("{archive}: the web app cannot reload local MBTiles paths")) + } +} + /// An archive's metadata. pub fn info(archive: &str) -> Result, String> { check(archive)?; @@ -118,7 +146,7 @@ pub fn info(archive: &str) -> Result, String> { if let Some(result) = cached_info(archive) { return result; } - let epoch = EPOCH.load(Ordering::SeqCst); + let epoch = info_epoch(archive); let r = read_info(archive); remember_info(archive, &r, epoch); r @@ -126,21 +154,27 @@ pub fn info(archive: &str) -> Result, String> { /// Keeps a completion only if no clear happened since its read started; check and insert share /// the same lock as the clear, so an old completion cannot repopulate the new cache. -fn remember_info(archive: &str, result: &Result, String>, epoch: u64) { - let mut all = infos().lock().unwrap_or_else(|e| e.into_inner()); - if EPOCH.load(Ordering::SeqCst) != epoch { - return; - } - if all.len() >= 64 { - all.clear(); - } - all.insert(archive.to_string(), result.clone().map_err(|e| (e, Instant::now()))); +fn remember_info(archive: &str, result: &Result, String>, epoch: (u64, u64)) { + crate::cache::if_archive_current(archive, epoch.1, || { + let mut all = infos().lock().unwrap_or_else(|e| e.into_inner()); + if EPOCH.load(Ordering::SeqCst) != epoch.0 { + return; + } + if all.len() >= 64 { + all.clear(); + } + all.insert(archive.to_string(), result.clone().map_err(|e| (e, Instant::now()))); + }); } /// An existing read has its own deadline. An unadmitted read reports saturation once a worker /// has stalled; workers keep their slots until the OS returns, never spawning unbounded threads. #[cfg(not(target_arch = "wasm32"))] -fn metadata_stalled(reads: &HashMap, archive: &str) -> bool { +fn metadata_stalled(reads: &HashMap, archive: &Q) -> bool +where + K: Eq + std::hash::Hash + std::borrow::Borrow, + Q: Eq + std::hash::Hash + ?Sized, +{ reads.get(archive).map_or_else( || reads.len() >= 8 && reads.values().any(|start| start.elapsed() >= READ_DEADLINE), |start| start.elapsed() >= READ_DEADLINE, @@ -180,43 +214,49 @@ pub fn info_ready(archive: &str, timeout: Duration) -> Result>, #[cfg(not(target_arch = "wasm32"))] { use std::sync::Condvar; - static LOADING: OnceLock<(Mutex>, Condvar)> = OnceLock::new(); - let (pending, ready) = LOADING.get_or_init(|| (Mutex::new(HashMap::new()), Condvar::new())); + type Pending = HashMap<(String, u64), Instant>; + static LOADING: std::sync::LazyLock<(Mutex, Condvar)> = + std::sync::LazyLock::new(|| (Mutex::new(HashMap::new()), Condvar::new())); + let (pending, ready) = &*LOADING; + let epoch = info_epoch(archive); + let key = (archive.to_string(), epoch.1); let mut all = pending.lock().unwrap_or_else(|e| e.into_inner()); if let Some(result) = cached_info(archive) { return result.map(Some); } - if metadata_stalled(&all, archive) { + if metadata_stalled(&all, &key) { return Err(format!( "{archive}: MBTiles metadata read exceeded 30 seconds or all eight workers are occupied by stalled storage; retry after the storage responds" )); } - if !all.contains_key(archive) { + if !all.contains_key(&key) { if all.len() >= 8 { return Ok(None); } - all.insert(archive.to_string(), Instant::now()); + all.insert(key.clone(), Instant::now()); let address = archive.to_string(); + let worker_key = key.clone(); let spawned = std::thread::Builder::new().name("rustgis-mbtiles-info".into()).spawn(move || { - let _ = info(&address); - pending.lock().unwrap_or_else(|e| e.into_inner()).remove(&address); + let result = read_info(&address); + remember_info(&address, &result, epoch); + pending.lock().unwrap_or_else(|e| e.into_inner()).remove(&worker_key); ready.notify_all(); crate::cache::notify(); }); if let Err(error) = spawned { - all.remove(archive); + all.remove(&key); return Err(format!("could not start an MBTiles metadata read: {error}")); } } if !timeout.is_zero() { - let wait = all.get(archive).map_or(Duration::ZERO, |start| metadata_wait(*start, timeout)); - let (reads, _) = ready.wait_timeout_while(all, wait, |all| all.contains_key(archive)).unwrap_or_else(|e| e.into_inner()); + let wait = all.get(&key).map_or(Duration::ZERO, |start| metadata_wait(*start, timeout)); + let (reads, _) = ready.wait_timeout_while(all, wait, |all| all.contains_key(&key)).unwrap_or_else(|e| e.into_inner()); all = reads; } if let Some(result) = cached_info(archive) { return result.map(Some); } - if metadata_stalled(&all, archive) { + if metadata_stalled(&all, &key) { return Err(format!("{archive}: MBTiles metadata read exceeded 30 seconds; retry after the storage responds")); } Ok(None) @@ -281,12 +321,22 @@ mod tests { #[test] fn cleared_metadata_rejects_an_older_completion() { - let epoch = EPOCH.load(Ordering::SeqCst); + let epoch = info_epoch("/old-completion.mbtiles"); forget(); remember_info("/old-completion.mbtiles", &Err("obsolete read".into()), epoch); assert!(cached_info("/old-completion.mbtiles").is_none()); - remember_info("/fresh-completion.mbtiles", &Err("current read".into()), EPOCH.load(Ordering::SeqCst)); + remember_info("/fresh-completion.mbtiles", &Err("current read".into()), info_epoch("/fresh-completion.mbtiles")); assert!(cached_info("/fresh-completion.mbtiles").is_some()); + let archive = "/mbtiles-source-generation.mbtiles"; + let epoch = info_epoch(archive); + crate::cache::reload_archive(archive); + let current = info_epoch(archive); + remember_info(archive, &Err("current error".into()), current); + remember_info(archive, &Err("obsolete error".into()), epoch); + assert_eq!(cached_info(archive).unwrap().unwrap_err(), "current error"); + crate::cache::reload_archive(archive); + remember_info(archive, &Err("obsolete failure after retry".into()), current); + assert!(cached_info(archive).is_none()); } #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/tiles/src/pmtiles.rs b/crates/tiles/src/pmtiles.rs index 90ea8ee01c..93090990e6 100644 --- a/crates/tiles/src/pmtiles.rs +++ b/crates/tiles/src/pmtiles.rs @@ -68,6 +68,7 @@ pub struct Info { data_length: u64, internal: Compression, tile: Compression, + epoch: u64, } impl Info { @@ -410,21 +411,41 @@ pub(crate) fn forget_archive(archive: &str) { leaves().lock().unwrap_or_else(|e| e.into_inner()).retain(|(a, _), l| a != archive || matches!(l, Leaf::Loading(_))); } +/// Pending callbacks are returned for disposal outside the revision lock: dropping a tile's +/// answer may immediately launch another read. +pub(crate) fn reload_archive(archive: &str) -> impl Sized { + let state = archives().lock().unwrap_or_else(|e| e.into_inner()).remove(archive); + let mut all = leaves().lock().unwrap_or_else(|e| e.into_inner()); + let keys: Vec<_> = all.keys().filter(|(a, _)| a == archive).cloned().collect(); + let removed: Vec<_> = keys.into_iter().filter_map(|key| all.remove(&key)).collect(); + (state, removed) +} + /// An archive's header and metadata: `Ok(None)` while they are read (the map draws again once /// they arrive). pub fn info(archive: &str) -> Result>, String> { crate::logins::check(archive).map_err(str::to_string)?; - let mut all = archives().lock().unwrap_or_else(|e| e.into_inner()); - match all.get(archive) { - Some(State::Ready(i)) => return Ok(Some(i.clone())), - Some(State::Loading(_)) => return Ok(None), - Some(State::Failed(e, at)) if at.elapsed() < RETRY => return Err(e.clone()), - _ => {} + let epoch = crate::cache::archive_epoch(archive); + let result = crate::cache::if_archive_current(archive, epoch, || { + let mut all = archives().lock().unwrap_or_else(|e| e.into_inner()); + match all.get(archive) { + Some(State::Ready(i)) => return (Ok(Some(i.clone())), false), + Some(State::Loading(_)) => return (Ok(None), false), + Some(State::Failed(e, at)) if at.elapsed() < RETRY => return (Err(e.clone()), false), + _ => {} + } + all.insert(archive.to_string(), State::Loading(Vec::new())); + (Ok(None), true) + }); + match result { + Some((result, start)) => { + if start { + load(archive.to_string(), epoch); + } + result + } + None => Ok(None), } - all.insert(archive.to_string(), State::Loading(Vec::new())); - drop(all); - load(archive.to_string()); - Ok(None) } /// The archive's info, read now if it hasn't been, waiting up to `timeout` (natively: the @@ -442,29 +463,34 @@ fn with_info(archive: &str, f: Box, String>) + Send> f(Err(e.to_string())); return; } - let mut all = archives().lock().unwrap_or_else(|e| e.into_inner()); - match all.get_mut(archive) { - Some(State::Ready(i)) => { - let i = i.clone(); - drop(all); - f(Ok(i)); + let epoch = crate::cache::archive_epoch(archive); + let result = crate::cache::if_archive_current(archive, epoch, || { + let mut all = archives().lock().unwrap_or_else(|e| e.into_inner()); + match all.get_mut(archive) { + Some(State::Ready(i)) => (Some((f, Ok(i.clone()))), false), + Some(State::Loading(waiting)) => { + waiting.push(f); + (None, false) + } + Some(State::Failed(e, at)) if at.elapsed() < RETRY => (Some((f, Err(e.clone()))), false), + _ => { + all.insert(archive.to_string(), State::Loading(vec![f])); + (None, true) + } } - Some(State::Loading(waiting)) => waiting.push(f), - Some(State::Failed(e, at)) if at.elapsed() < RETRY => { - let e = e.clone(); - drop(all); - f(Err(e)); + }); + if let Some((answer, start)) = result { + if let Some((f, result)) = answer { + f(result); } - _ => { - all.insert(archive.to_string(), State::Loading(vec![f])); - drop(all); - load(archive.to_string()); + if start { + load(archive.to_string(), epoch); } } } /// Reads an archive's header, root directory and metadata, then answers whoever waits. -fn load(archive: String) { +fn load(archive: String, epoch: u64) { let a = archive.clone(); read( &archive, @@ -473,10 +499,11 @@ fn load(archive: String) { Box::new(move |head| { let parsed = head.and_then(|b| parse_head(&b)); match parsed { - Err(e) => finish(&a, Err(e)), + Err(e) => finish(&a, Err(e), epoch), Ok((mut info, meta_at, meta_len)) => { + info.epoch = epoch; if meta_len == 0 { - finish(&a, Ok(info)); + finish(&a, Ok(info), epoch); return; } let a2 = a.clone(); @@ -485,12 +512,8 @@ fn load(archive: String) { meta_at, meta_len, Box::new(move |m| { - // Unreadable metadata only loses the layer names and credits. - if let Ok(m) = m { - let m = info.internal.decode(m).unwrap_or_default(); - info.set_metadata(serde_json::from_slice(&m).unwrap_or(Value::Null)); - } - finish(&a2, Ok(info)); + let result = m.and_then(|m| metadata(&mut info, m, meta_len)).map(|()| info); + finish(&a2, result, epoch); }), ); } @@ -499,13 +522,13 @@ fn load(archive: String) { ); } -fn finish(archive: &str, r: Result) { +fn finish(archive: &str, r: Result, epoch: u64) { let r = crate::logins::check(archive) .map_err(str::to_string) .and(r) .map(Arc::new) .map_err(|e| if rustgis_data::login_address::is_wrapper(archive) { crate::logins::scrub(&e) } else { e }); - let waiting = { + let waiting = crate::cache::if_archive_current(archive, epoch, || { let mut all = archives().lock().unwrap_or_else(|e| e.into_inner()); let old = match &r { Ok(i) => all.insert(archive.to_string(), State::Ready(i.clone())), @@ -518,7 +541,8 @@ fn finish(archive: &str, r: Result) { Some(State::Loading(w)) => w, _ => Vec::new(), } - }; + }) + .unwrap_or_default(); for w in waiting { w(r.clone()); } @@ -533,11 +557,16 @@ pub(crate) fn fetch(url: &str, done: Done) { return; }; let a = archive.clone(); + let epoch = crate::cache::archive_epoch(&archive); with_info( &archive, Box::new(move |info| match info { Err(e) => done(Err(e)), Ok(info) => { + if crate::cache::archive_epoch(&a) != epoch { + done(Err(crate::cache::DROPPED.into())); + return; + } // Outside the archive's zooms, or not a tile of its zoom: none. let n = 1u64 << z; if z < info.min_zoom || z > info.max_zoom || u64::from(x) >= n || u64::from(y) >= n { @@ -554,6 +583,11 @@ pub(crate) fn fetch(url: &str, done: Done) { /// Looks `id` up in a directory, reading leaf directories as needed, then reads the tile. fn find(archive: String, info: Arc, id: u64, dir: Arc>, depth: usize, done: Done) { + let epoch = info.epoch; + if crate::cache::archive_epoch(&archive) != epoch { + done(Err(crate::cache::DROPPED.into())); + return; + } let Some(e) = lookup(&dir, id) else { done(Ok(Vec::new())); return; @@ -589,18 +623,17 @@ fn find(archive: String, info: Arc, id: u64, dir: Arc>, depth: }) }; // One read per leaf: tiles needing a leaf already on its way wait for it. - { + let admitted = crate::cache::if_archive_current(&archive, epoch, || { let mut l = leaves().lock().unwrap_or_else(|e| e.into_inner()); match l.get_mut(&key) { Some(Leaf::Ready(leaf)) => { let leaf = leaf.clone(); drop(l); - next(Ok(leaf)); - return; + return (Some((next, leaf)), false); } Some(Leaf::Loading(waiting)) => { waiting.push(next); - return; + return (None, false); } None => { // Full: one ready leaf makes room (the cache is a working set, not a history). @@ -612,6 +645,16 @@ fn find(archive: String, info: Arc, id: u64, dir: Arc>, depth: l.insert(key.clone(), Leaf::Loading(vec![next])); } } + (None, true) + }); + match admitted { + None => return, + Some((Some((next, leaf)), _)) => { + next(Ok(leaf)); + return; + } + Some((None, false)) => return, + Some((None, true)) => {} } read( &archive, @@ -619,25 +662,29 @@ fn find(archive: String, info: Arc, id: u64, dir: Arc>, depth: e.length, Box::new(move |b| { let leaf = b.and_then(|b| info.internal.decode(b)).and_then(|b| parse_dir(&b)).map(Arc::new); - let waiting = { - let mut l = leaves().lock().unwrap_or_else(|e| e.into_inner()); - let w = match &leaf { - Ok(leaf) => l.insert(key.clone(), Leaf::Ready(leaf.clone())), - // A failed leaf is read again by the next tile that needs it. - Err(_) => l.remove(&key), - }; - match w { - Some(Leaf::Loading(w)) => w, - _ => Vec::new(), - } - }; - for w in waiting { - w(leaf.clone()); - } + finish_leaf(key, leaf, epoch); }), ); } +fn finish_leaf(key: (String, u64), leaf: Result>, String>, epoch: u64) { + let waiting = crate::cache::if_archive_current(&key.0, epoch, || { + let mut all = leaves().lock().unwrap_or_else(|e| e.into_inner()); + let old = match &leaf { + Ok(leaf) => all.insert(key.clone(), Leaf::Ready(leaf.clone())), + Err(_) => all.remove(&key), + }; + match old { + Some(Leaf::Loading(waiting)) => waiting, + _ => Vec::new(), + } + }) + .unwrap_or_default(); + for callback in waiting { + callback(leaf.clone()); + } +} + /// The entry holding `id`: the last at or before it, when it is a leaf directory or its run /// covers `id`. fn lookup(dir: &[Entry], id: u64) -> Option { @@ -693,6 +740,49 @@ pub(crate) fn read_file(archive: &str, at: u64, len: u64) -> Result, Str Ok(buf) } +fn metadata(info: &mut Info, bytes: Vec, expected: u64) -> Result<(), String> { + if bytes.len() as u64 != expected { + return Err("truncated PMTiles metadata".into()); + } + let bytes = info.internal.decode(bytes)?; + let value: Value = serde_json::from_slice(&bytes).map_err(|e| format!("PMTiles metadata: {e}"))?; + if !value.is_object() { + return Err("PMTiles metadata must be a JSON object".into()); + } + info.set_metadata(value); + Ok(()) +} + +/// Reads the current local file without consulting or changing retained caches. All ranges +/// are bounded and freshly opened; failed validation leaves the displayed archive intact. +#[cfg(not(target_arch = "wasm32"))] +pub fn read_info_local(archive: &str) -> Result, String> { + check_local(archive)?; + let head = read_file(archive, 0, HEAD)?; + let path = archive.strip_prefix("file://").unwrap_or(archive); + let file_len = std::fs::metadata(path).map_err(|e| format!("{path}: {e}"))?.len(); + if head.len() < 127 { + return Err("a truncated PMTiles header".into()); + } + let (mut info, at, len) = parse_head(&head)?; + for (offset, length) in [(8, 16), (24, 32), (40, 48), (56, 64)] { + let start = u64_at(&head, offset).ok_or("a truncated PMTiles header")?; + let size = u64_at(&head, length).ok_or("a truncated PMTiles header")?; + if start.checked_add(size).is_none_or(|end| end > file_len) { + return Err("a PMTiles section outside the local file".into()); + } + } + if len != 0 { + metadata(&mut info, read_file(archive, at, len)?, len)?; + } + Ok(Arc::new(info)) +} + +#[cfg(target_arch = "wasm32")] +pub fn read_info_local(archive: &str) -> Result, String> { + Err(format!("{archive}: the web app cannot reload local PMTiles paths")) +} + fn u64_at(b: &[u8], i: usize) -> Option { Some(u64::from_le_bytes(b.get(i..i + 8)?.try_into().ok()?)) } @@ -754,8 +844,12 @@ fn parse_head(b: &[u8]) -> Result<(Info, u64, u64), String> { data_length, internal, tile, + epoch: 0, }; - Ok((info, meta_at, meta_len.min(MAX_BYTES as u64))) + if meta_len > MAX_BYTES as u64 || meta_at.checked_add(meta_len).is_none() { + return Err("PMTiles metadata is too large or has an invalid range".into()); + } + Ok((info, meta_at, meta_len)) } /// A directory: delta-coded tile ids, then run lengths, lengths and offsets, all varints. @@ -820,6 +914,61 @@ fn gunzip(b: &[u8]) -> Result, String> { pub(crate) mod tests { use super::*; + #[test] + fn reloaded_archive_rejects_old_metadata_and_leaf_answers() { + let source = "/pmtiles-stale-completion-regression.pmtiles"; + let old_epoch = crate::cache::archive_epoch(source); + let bytes = archive(&[(0, 0, 0, vec![1])], 1, true, "{}"); + let (old_info, _, _) = parse_head(&bytes).unwrap(); + crate::cache::reload_archive(source); + let (tx, rx) = std::sync::mpsc::channel(); + archives() + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert(source.into(), State::Loading(vec![Box::new(move |result| tx.send(result.map(|info| info.attribution())).unwrap())])); + let key = (source.to_string(), HEAD); + let (leaf_tx, leaf_rx) = std::sync::mpsc::channel(); + leaves() + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert(key.clone(), Leaf::Loading(vec![Box::new(move |result| leaf_tx.send(result.map(|leaf| leaf.len())).unwrap())])); + finish(source, Ok(old_info), old_epoch); + finish_leaf(key.clone(), Ok(Arc::new(Vec::new())), old_epoch); + assert!(matches!(rx.try_recv(), Err(std::sync::mpsc::TryRecvError::Empty))); + assert!(matches!(leaf_rx.try_recv(), Err(std::sync::mpsc::TryRecvError::Empty))); + // Global clears retain pending reads, so this regression is independent of other tests. + assert!(matches!(archives().lock().unwrap_or_else(|e| e.into_inner()).get(source), Some(State::Loading(_)))); + assert!(matches!(leaves().lock().unwrap_or_else(|e| e.into_inner()).get(&key), Some(Leaf::Loading(_)))); + let (mut current, _, _) = parse_head(&bytes).unwrap(); + current.set_metadata(serde_json::json!({"attribution": "fresh credits"})); + let epoch = crate::cache::archive_epoch(source); + finish(source, Ok(current), epoch); + finish_leaf(key, Ok(Arc::new(vec![Entry { tile_id: 0, offset: 0, length: 1, run_length: 1 }])), epoch); + assert_eq!(rx.recv().unwrap().unwrap(), "fresh credits"); + assert_eq!(leaf_rx.recv().unwrap().unwrap(), 1); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn fresh_local_validation_rejects_bad_metadata_and_truncated_sections() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-validation-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + let valid = archive(&[(0, 0, 0, vec![1, 2, 3])], 1, true, r#"{"attribution":"fresh"}"#); + std::fs::write(&path, &valid).unwrap(); + assert_eq!(read_info_local(source).unwrap().attribution(), "fresh"); + for metadata in ["{", "null", "[]"] { + std::fs::write(&path, archive(&[(0, 0, 0, vec![1])], 1, false, metadata)).unwrap(); + assert!(read_info_local(source).is_err(), "{metadata}"); + } + std::fs::write(&path, &valid[..valid.len() - 1]).unwrap(); + assert!(read_info_local(source).is_err()); + let mut truncated_metadata = valid.clone(); + truncated_metadata[24..32].copy_from_slice(&(valid.len() as u64 - 1).to_le_bytes()); + std::fs::write(&path, truncated_metadata).unwrap(); + assert!(read_info_local(source).is_err()); + std::fs::remove_file(path).unwrap(); + } + #[test] fn tile_ids_follow_the_spec() { assert_eq!(tile_id(0, 0, 0), 0); diff --git a/crates/tiles/src/source.rs b/crates/tiles/src/source.rs index 02e6eba252..b2c06dfa5d 100644 --- a/crates/tiles/src/source.rs +++ b/crates/tiles/src/source.rs @@ -55,8 +55,9 @@ pub struct Spec { impl Spec { fn key(&self) -> String { format!( - "{:?}|{}|{}|{}|{}|{}|{}|{}|{}|{}|{}", + "{:?}|{}:{}|{}|{}|{}|{}|{}|{}|{}|{}|{}", self.kind, + self.url.len(), self.url, self.subdomains.join(","), self.min_zoom, @@ -137,6 +138,13 @@ pub(crate) fn forget() { resolved().lock().unwrap_or_else(|e| e.into_inner()).clear(); } +/// Drops only styles resolved directly from this archive. +pub(crate) fn forget_archive(archive: &str) { + let pm = format!("{:?}|{}:{archive}|", Kind::PmTiles, archive.len()); + let mb = format!("{:?}|{}:{archive}|", Kind::MbTiles, archive.len()); + resolved().lock().unwrap_or_else(|e| e.into_inner()).retain(|key, _| !key.starts_with(&pm) && !key.starts_with(&mb)); +} + /// The built-in style for vector tiles without one: it knows the OpenMapTiles, Protomaps and /// Shortbread layer names, and draws water, land use, parks, buildings, roads, boundaries and /// place names from whichever of them the tiles have. @@ -148,6 +156,7 @@ pub fn resolve(spec: &Spec) -> Resolve { return Resolve::Failed(e.to_string()); } let key = spec.key(); + let epoch = cache::archive_epoch(&spec.url); // A PMTiles archive on this computer is a path, not a web address. let local = spec.kind == Kind::PmTiles && pmtiles::is_local(&spec.url); let checked = if spec.kind == Kind::MbTiles { @@ -164,7 +173,8 @@ pub fn resolve(spec: &Spec) -> Resolve { if let Some(r) = cached.as_ref().filter(|r| r.current()) { return Resolve::Ready(r.clone()); } - settle(resolve_anew(spec, key.clone()), cached, spec, key) + let anew = resolve_anew(spec, key.clone(), epoch); + cache::if_archive_current(&spec.url, epoch, || settle(anew, cached, spec, key)).unwrap_or(Resolve::Pending) } /// What resolving `spec` again (`anew`) gives, with `cached`, the expired one under `key`. @@ -188,7 +198,7 @@ fn settle(anew: Resolve, cached: Option>, spec: &Spec, key: String } /// Reads `spec`'s style and resolves its sources, keeping the result under `key`. -fn resolve_anew(spec: &Spec, key: String) -> Resolve { +fn resolve_anew(spec: &Spec, key: String, epoch: u64) -> Resolve { let doc = match spec.kind { Kind::Raster => raster_style(spec), Kind::Vector => match vector_style(spec) { @@ -288,13 +298,16 @@ fn resolve_anew(spec: &Spec, key: String) -> Resolve { if let Err(e) = crate::logins::check(&spec.url) { return Resolve::Failed(e.to_string()); } - let mut all = resolved().lock().unwrap_or_else(|e| e.into_inner()); - // A few layers are in use at a time: past that many, start again rather than grow. - if all.len() >= MAX_RESOLVED { - all.clear(); - } - all.insert(key, r.clone()); - Resolve::Ready(r) + cache::if_archive_current(&spec.url, epoch, || { + let mut all = resolved().lock().unwrap_or_else(|e| e.into_inner()); + // A few layers are in use at a time: past that many, start again rather than grow. + if all.len() >= MAX_RESOLVED { + all.clear(); + } + all.insert(key, r.clone()); + Resolve::Ready(r) + }) + .unwrap_or(Resolve::Pending) } /// A style's sprites with their addresses resolved against `base`, those RustGIS can fetch, and diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index 0ca1b30650..3bd2343bc4 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -974,7 +974,7 @@ pub fn command_search(app: &mut RustGisApp, ctx: &egui::Context) { } } for c in &app.session.registry.commands { - if command_matches(c, &ql) { + if command_matches(c, &ql) && !(cfg!(target_arch = "wasm32") && matches!(c.id, "map.reloadTiles" | "map.repairTiles")) { hits.push((c.id.to_string(), tl!(c.label).to_string(), crate::i18n::location(c.location))); } } diff --git a/crates/ui-egui/src/panes.rs b/crates/ui-egui/src/panes.rs index 8173a33cab..96a644554e 100644 --- a/crates/ui-egui/src/panes.rs +++ b/crates/ui-egui/src/panes.rs @@ -1746,9 +1746,20 @@ fn tile_row(app: &mut RustGisApp, ui: &mut Ui, i: usize, moves: [bool; 2], reord action = Some(("map.setTiles", json!({"index": i, "move": "down"}))); ui.close(); } - if tile.archive_file().is_some() && ui.button(tl!("Repair Data Source…")).clicked() { - action = Some(("ui.dialog", json!({"name": "repairTiles", "index": i}))); - ui.close(); + #[cfg(not(target_arch = "wasm32"))] + if tile.archive_file().is_some() { + if ui + .button(tl!("Reload Data")) + .on_hover_text(tl!("Manually reread this archive from disk without changing its presentation. Undo cannot restore overwritten file contents.")) + .clicked() + { + action = Some(("map.reloadTiles", json!({"index": i}))); + ui.close(); + } + if ui.button(tl!("Repair Data Source…")).clicked() { + action = Some(("ui.dialog", json!({"name": "repairTiles", "index": i}))); + ui.close(); + } } if ui.button(tl!("Remove")).clicked() { action = Some(("map.removeTiles", json!({"index": i}))); diff --git a/docs/user-guide/maps.md b/docs/user-guide/maps.md index eace026222..457aa0d964 100644 --- a/docs/user-guide/maps.md +++ b/docs/user-guide/maps.md @@ -259,6 +259,26 @@ selectable or editable; a tile not yet loaded may need another click once it arr **Reload** in the Basemap gallery (`map.reloadWebLayers`) clears cached tiles and styles and fetches them again. +**Reloading a local tile archive (desktop).** After changing a PMTiles or MBTiles file on disk, +right-click its row in Layers and choose **Reload Data**. CLI, Python, MCP and the Assistant use +`map.reloadTiles={"index":0}`; `"name"` or catalog `"id"` can select the archive instead. +No replacement path is accepted. This is a manual action: RustGIS does not watch the file or detect changes. +It rereads that archive's metadata and tiles without clearing other archives or web layers. +The layer keeps its name, visibility, opacity, blend, drawing order, source-layer visibility, +style and other presentation settings; the map's view stays where it is. Undo cannot restore +file contents that another program has overwritten. + +Reload first checks that the file can be read. If it is missing, unreadable or invalid, a notice +explains the failure and the existing layer and retained cached data are left unchanged. +Finish writing or restore the file, then choose **Reload Data** again. If the file has moved, +choose **Repair Data Source…** instead and select its new location. Repair also checks the +replacement before changing the source; cancelling or choosing an unreadable or invalid archive +leaves the existing source and layer unchanged, so you can retry with a valid file. +Neither action is available for local archives in the web app, or for remote tile services. +On Windows, an open MBTiles file may not be replaceable while RustGIS holds it open: reload +does not bypass that operating-system restriction. Close RustGIS before replacing the file, +then reopen the project, or write to a different path and use **Repair Data Source…**. + **Source layers.** A vector PMTiles or MBTiles archive lists its source layers (the `vector_layers` its metadata names: `water`, `roads`, `pois`…) under its row in the Layers pane: the arrow beside it opens them, each with its own checkbox. Untick one to hide what it draws From 814df3d2166aa46609bccbede1289db57ba15276 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Fri, 6 Nov 2026 15:06:07 +0100 Subject: [PATCH 02/15] test(tiles): synchronize deferred cache fetch completions --- crates/tiles/src/cache.rs | 111 ++++++++++++++++++++++++++++++++------ 1 file changed, 96 insertions(+), 15 deletions(-) diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 881de853dd..a4cf34b559 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -3039,33 +3039,91 @@ mod tests { #[test] fn fetches_through_the_fetcher_once() { + use std::sync::mpsc; + + // The fetcher and cache are process-global. Other tests legitimately clear caches on + // logout; this fixture must neither lose its answers nor intercept their requests. + if std::env::var("RUSTGIS_CACHE_FETCHER_SCENARIO").as_deref() != Ok("once") { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "cache::tests::fetches_through_the_fetcher_once", "--test-threads=1", "--nocapture"]) + .env("RUSTGIS_CACHE_FETCHER_SCENARIO", "once") + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + let ran_one = stdout.lines().any(|line| line.starts_with("test result: ok. 1 passed; 0 failed; 0 ignored;")); + assert!( + output.status.success() && ran_one, + "isolated fetcher scenario failed or did not run exactly one test:\n{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); + return; + } + + fn receive(rx: &mpsc::Receiver<()>) { + rx.recv_timeout(Duration::from_secs(10)).expect("the test fetcher answered or admitted the request"); + } + + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let release_rx = Mutex::new(release_rx); + let (completed_tx, completed_rx) = mpsc::channel(); static CALLS: AtomicUsize = AtomicUsize::new(0); - set_fetcher(|url, done| { + set_fetcher(move |url, done| { + if url == "test://cache/blocker.json" { + entered_tx.send(()).unwrap(); + receive(&release_rx.lock().unwrap()); + done(Ok(b"0".to_vec())); + return; + } CALLS.fetch_add(1, Ordering::Relaxed); - if url.ends_with("ok.json") { done(Ok(b"[1, 2]".to_vec())) } else { done(Err("404".into())) } + if url.ends_with("ok.json") { + done(Ok(b"[1, 2]".to_vec())) + } else { + done(Err("404".into())) + } + completed_tx.send(()).unwrap(); }); - let g = generation(); - assert!(matches!(get("test://cache/ok.json", Want::Json), State::Pending)); - assert!(generation() > g); + // Hold the shared launcher in another fetcher invocation. Even a synchronous fetcher + // need not answer this request before get() returns when a launcher already owns it. + let blocker = std::thread::spawn(|| get("test://cache/blocker.json", Want::Json)); + receive(&entered_rx); + let requested = get("test://cache/ok.json", Want::Json); + let retained = peek("test://cache/ok.json"); + release_tx.send(()).unwrap(); + blocker.join().unwrap(); + assert!(matches!(requested, State::Pending)); + assert!(retained.is_none(), "the request is still pending while the launcher is held"); + receive(&completed_rx); assert!(matches!(get("test://cache/ok.json", Want::Json), State::Ready(_))); assert!(peek("test://cache/ok.json").is_some()); assert!(matches!(get("test://cache/missing.json", Want::Json), State::Pending)); + receive(&completed_rx); assert!(matches!(get("test://cache/missing.json", Want::Json), State::Failed(_))); assert!(peek("test://cache/never.json").is_none()); // A request the fetcher dropped is forgotten, not failed: asked for again, it is fetched. - set_fetcher(|url, done| { + let (completed_tx, completed_rx) = mpsc::channel(); + set_fetcher(move |url, done| { CALLS.fetch_add(1, Ordering::Relaxed); - if url.ends_with("first.json") { done(Err(DROPPED.into())) } else { done(Ok(b"1".to_vec())) } + if url.ends_with("first.json") { + done(Err(DROPPED.into())) + } else { + done(Ok(b"1".to_vec())) + } + completed_tx.send(()).unwrap(); }); assert!(matches!(get("test://cache/first.json", Want::Json), State::Pending)); + receive(&completed_rx); assert!(matches!(get("test://cache/first.json", Want::Json), State::Pending), "fetched again"); + receive(&completed_rx); assert_eq!(CALLS.load(Ordering::Relaxed), 4); // A server has a few fetches under way at once; the newest waiting tile goes next (#635). static HELD: Mutex> = Mutex::new(Vec::new()); - set_fetcher(|url, done| { + let (started_tx, started_rx) = mpsc::channel(); + set_fetcher(move |url, done| { if url.starts_with("test://held/") { HELD.lock().unwrap().push((url, done)); + started_tx.send(()).unwrap(); } else { done(Ok(b"1".to_vec())); } @@ -3074,16 +3132,25 @@ mod tests { for i in 0..n { assert!(matches!(get(&format!("test://held/{i}.json"), Want::Json), State::Pending)); } + for _ in 0..crate::schedule::PER_HOST { + receive(&started_rx); + } assert_eq!(HELD.lock().unwrap().len(), crate::schedule::PER_HOST); - let (_, done) = HELD.lock().unwrap().remove(0); + let (completed_url, done) = HELD.lock().unwrap().remove(0); done(Ok(b"2".to_vec())); + receive(&started_rx); let held: Vec = HELD.lock().unwrap().iter().map(|(u, _)| u.clone()).collect(); assert_eq!(held.len(), crate::schedule::PER_HOST); assert_eq!(held.last().map(String::as_str), Some(format!("test://held/{}.json", n - 1).as_str())); - assert!(matches!(get("test://held/0.json", Want::Json), State::Ready(_))); + assert!(matches!(get(&completed_url, Want::Json), State::Ready(_))); // The rest finish, and so do the waiting ones they make room for. - loop { - let Some((_, done)) = HELD.lock().unwrap().pop() else { break }; + for _ in 1..n { + let (_, done) = loop { + if let Some(job) = HELD.lock().unwrap().pop() { + break job; + } + receive(&started_rx); + }; done(Ok(b"3".to_vec())); } for i in 0..n { @@ -3093,10 +3160,12 @@ mod tests { // A fetcher that drops a request unanswered (or panics) gives its server's place back: // more requests than the cap still all reach it, and each is forgotten, not stuck. static SEEN: AtomicUsize = AtomicUsize::new(0); - set_fetcher(|url, done| { + let (completed_tx, completed_rx) = mpsc::channel(); + set_fetcher(move |url, done| { if url.starts_with("test://lost/") { SEEN.fetch_add(1, Ordering::Relaxed); drop(done); + completed_tx.send(()).unwrap(); } else { done(Ok(b"1".to_vec())); } @@ -3104,15 +3173,20 @@ mod tests { for i in 0..3 * crate::schedule::PER_HOST { assert!(matches!(get(&format!("test://lost/{i}.json"), Want::Json), State::Pending)); } + for _ in 0..3 * crate::schedule::PER_HOST { + receive(&completed_rx); + } assert_eq!(SEEN.load(Ordering::Relaxed), 3 * crate::schedule::PER_HOST); assert!(peek("test://lost/0.json").is_none()); // Waiting for news: a fetch never answered (another's, a slow server's) doesn't hold up // one waiting for what has come. static STUCK: Mutex> = Mutex::new(Vec::new()); - set_fetcher(|url, done| { + let (started_tx, started_rx) = mpsc::channel(); + set_fetcher(move |url, done| { if url.starts_with("test://stuck/") { STUCK.lock().unwrap().push(done); + started_tx.send(()).unwrap(); } else { std::thread::spawn(move || { std::thread::sleep(std::time::Duration::from_millis(50)); @@ -3121,6 +3195,7 @@ mod tests { } }); assert!(matches!(get("test://stuck/0.json", Want::Json), State::Pending)); + receive(&started_rx); let g = generation(); assert!(matches!(get("test://news/0.json", Want::Json), State::Pending)); let t = Instant::now(); @@ -3136,19 +3211,25 @@ mod tests { // Only a map tile's 404 is an expected answer: a picture that isn't a tile (a single // image, a layout's picture, a sprite sheet) and a style keep theirs. static OMITS: Mutex> = Mutex::new(Vec::new()); - set_fetcher_omitting(|url, done, omit| { + let (completed_tx, completed_rx) = mpsc::channel(); + set_fetcher_omitting(move |url, done, omit| { OMITS.lock().unwrap().push((url, omit)); done(Err("http status: 404".into())); + completed_tx.send(()).unwrap(); }); let _ = get_tile("test://omit/tile.png", Want::Image); let _ = get_tile("test://omit/tile.pbf", Want::Vector); let _ = get("test://omit/picture.png", Want::Image); let _ = get("test://omit/style.json", Want::Json); + for _ in 0..4 { + receive(&completed_rx); + } let omits = std::mem::take(&mut *OMITS.lock().unwrap()); let omit = |name: &str| omits.iter().find(|(u, _)| u.ends_with(name)).map(|(_, o)| *o); assert_eq!(omit("tile.png"), Some(&[404, 410][..])); assert_eq!(omit("tile.pbf"), Some(&[404, 410][..])); assert_eq!(omit("picture.png"), Some(&[][..])); assert_eq!(omit("style.json"), Some(&[][..])); + clear_fetcher(); } } From 10bcb517dc096d2418113f50191ff328460c547c Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Fri, 6 Nov 2026 22:27:42 +0100 Subject: [PATCH 03/15] fix(tiles): validate archive reloads before background installation --- crates/data/src/mbtiles.rs | 118 ++++++- crates/engine/src/cmd/items.rs | 1 + crates/engine/src/cmd/reads.rs | 11 + crates/engine/src/cmd/repair.rs | 58 +++- .../tests/portable_projects/reload_tiles.rs | 180 +++++++++++ crates/tiles/src/mbtiles.rs | 31 +- crates/tiles/src/pmtiles.rs | 297 +++++++++++++++++- crates/ui-egui/src/lib.rs | 16 + crates/ui-egui/src/readjob.rs | 134 +++++++- crates/ui-egui/src/repairsource.rs | 6 +- docs/user-guide/maps.md | 18 ++ 11 files changed, 843 insertions(+), 27 deletions(-) diff --git a/crates/data/src/mbtiles.rs b/crates/data/src/mbtiles.rs index 371166df64..51d90d9a9b 100644 --- a/crates/data/src/mbtiles.rs +++ b/crates/data/src/mbtiles.rs @@ -364,17 +364,37 @@ mod file { with(path, |c| read_info(c, path)) } - /// Validates a new connection without borrowing or populating the retained pool. - pub fn read_info_path(path: &Path) -> Result { + /// Reads metadata and one bounded raster sample on a fresh connection, retaining nothing. + /// Vector archives get only a tile-page readability probe; an empty archive has no sample. + pub fn read_info_path(path: &Path) -> Result<(Info, Option>), String> { let c = open(path)?; let info = read_info(&c, path)?; - // Even complete metadata must not hide a damaged first tile page. This bounded read - // checks the table/view is usable without scanning or decoding the tile set. - let _: Option>> = c - .query_row("SELECT substr(tile_data, 1, 16) FROM tiles LIMIT 1", [], |row| row.get(0)) - .optional() - .map_err(|e| format!("{}: not a readable MBTiles file ({e})", path.display()))?; - Ok(info) + let sql = |e: rusqlite::Error| format!("{}: not a readable MBTiles file ({e})", path.display()); + // Even complete metadata must not hide a damaged first tile page. Vector payloads are + // not decoded here; the tiles layer decodes the raster sample before accepting a reload. + let sample = if info.format.is_vector() { + let _: Option>> = + c.query_row("SELECT substr(tile_data, 1, 16) FROM tiles LIMIT 1", [], |row| row.get(0)).optional().map_err(sql)?; + None + } else { + // Check the size before materializing a blob, as the normal tile reader does. + let tile: Option<(i64, Option>)> = c + .query_row( + "SELECT length(tile_data), CASE WHEN length(tile_data) <= ?1 THEN tile_data END FROM tiles LIMIT 1", + [MAX_TILE_BYTES as i64], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(sql)?; + match tile { + Some((len, _)) if len > MAX_TILE_BYTES as i64 => { + return Err(format!("{}: raster sample of {len} bytes exceeds the {MAX_TILE_BYTES}-byte tile limit", path.display())); + } + Some((_, bytes)) => bytes, + None => None, + } + }; + Ok((info, sample)) } fn read_info(c: &Connection, path: &Path) -> Result { @@ -445,6 +465,86 @@ mod file { mod reload_tests { use super::*; + #[test] + fn ordinary_open_refuses_null_and_non_text_metadata_rows() { + let path = std::env::temp_dir().join(format!("rustgis-mbtiles-metadata-types-{}.mbtiles", std::process::id())); + let _ = std::fs::remove_file(&path); + let c = Connection::open(&path).unwrap(); + c.execute_batch( + "CREATE TABLE metadata(name,value); + CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + INSERT INTO metadata VALUES('format','png'),('minzoom','0'),('maxzoom','0');", + ) + .unwrap(); + assert_eq!(open_path(&path).unwrap().format, super::super::Format::Png); + for row in ["(NULL,'unused')", "('unused',NULL)", "(7,'unused')", "('unused',7)", "(x'01','unused')", "('unused',x'01')"] { + c.execute_batch(&format!("INSERT INTO metadata VALUES {row}")).unwrap(); + let error = open_path(&path).unwrap_err(); + assert!(error.contains(&path.display().to_string()), "{error}"); + assert!(error.contains("not a readable MBTiles file"), "{error}"); + c.execute("DELETE FROM metadata WHERE rowid = (SELECT max(rowid) FROM metadata)", []).unwrap(); + assert!(open_path(&path).is_ok(), "valid text rows must remain usable after {row}"); + } + close(&path); + drop(c); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn candidate_raster_sample_is_bounded_and_unpooled() { + let path = std::env::temp_dir().join(format!("rustgis-mbtiles-raster-sample-{}.mbtiles", std::process::id())); + let _ = std::fs::remove_file(&path); + let c = Connection::open(&path).unwrap(); + c.execute_batch( + "CREATE TABLE metadata(name TEXT,value TEXT); + CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + INSERT INTO metadata VALUES('format','png'),('minzoom','0'),('maxzoom','0');", + ) + .unwrap(); + assert!(read_info_path(&path).unwrap().1.is_none(), "an empty archive remains valid"); + let invalid_png = b"\x89PNG\r\n\x1a\n"; + c.execute("INSERT INTO tiles VALUES(0,0,0,?1)", [invalid_png.as_slice()]).unwrap(); + let (info, sample) = read_info_path(&path).unwrap(); + assert_eq!(info.format, super::super::Format::Png); + assert_eq!(sample.as_deref(), Some(invalid_png.as_slice()), "the tiles layer must see more than a discarded signature probe"); + c.execute("UPDATE tiles SET tile_data=zeroblob(?1)", [MAX_TILE_BYTES as i64 + 1]).unwrap(); + let error = read_info_path(&path).unwrap_err(); + assert!(error.contains(&path.display().to_string()), "{error}"); + assert!(error.contains("exceeds") && error.contains(&MAX_TILE_BYTES.to_string()), "{error}"); + c.execute_batch("UPDATE metadata SET value='pbf' WHERE name='format'; UPDATE tiles SET tile_data=x'01';").unwrap(); + assert!(read_info_path(&path).unwrap().1.is_none(), "vector payloads get a readability probe, not image validation"); + assert!(!pool().lock().unwrap().connections.contains_key(&path), "candidate validation must not populate the retained pool"); + drop(c); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn candidate_probe_reads_tile_pages_even_with_complete_metadata() { + let path = std::env::temp_dir().join(format!("rustgis-mbtiles-damaged-tile-page-{}.mbtiles", std::process::id())); + let _ = std::fs::remove_file(&path); + let c = Connection::open(&path).unwrap(); + c.execute_batch( + "CREATE TABLE metadata(name TEXT,value TEXT); + CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + INSERT INTO metadata VALUES('format','pbf'),('minzoom','0'),('maxzoom','0'); + INSERT INTO tiles VALUES(0,0,0,x'01');", + ) + .unwrap(); + let page = c.query_row("SELECT rootpage FROM sqlite_master WHERE name='tiles'", [], |r| r.get::<_, u32>(0)).unwrap() as usize; + let size = c.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0)).unwrap() as usize; + drop(c); + assert!(read_info_path(&path).is_ok()); + let mut bytes = std::fs::read(&path).unwrap(); + bytes[(page - 1) * size] = 0; + std::fs::write(&path, bytes).unwrap(); + assert!(open_path(&path).is_ok(), "complete metadata alone does not step the damaged tile table"); + let error = read_info_path(&path).unwrap_err(); + assert!(error.contains(&path.display().to_string()), "{error}"); + assert!(error.contains("not a readable MBTiles file"), "{error}"); + close(&path); + std::fs::remove_file(path).unwrap(); + } + #[test] fn closing_one_file_rejects_its_old_connection_but_keeps_another_file() { let root = std::env::temp_dir().join(format!("rustgis-mbtiles-pool-reload-{}", std::process::id())); diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 079a2b97fb..0724bbb17e 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -845,6 +845,7 @@ pub(crate) fn touches_files(command: &str, params: &Value) -> bool { || ["import", "export", "open", "save", "add", "package", "upload", "download"].iter().any(|w| verb.starts_with(w)) // Its inputs may be bare file names, which don't look like paths. || id == "raster.mosaic" + || id == "map.reloadtiles" || (id == "gp.run" && params.get("tool").and_then(Value::as_str).and_then(super::gp::tool_id) .and_then(rustgis_analysis::tool).is_some_and(super::gp::writes_files)) || reaches(params) diff --git a/crates/engine/src/cmd/reads.rs b/crates/engine/src/cmd/reads.rs index 3e2099bb46..1530566c07 100644 --- a/crates/engine/src/cmd/reads.rs +++ b/crates/engine/src/cmd/reads.rs @@ -65,10 +65,21 @@ pub fn read_ahead(s: &Session, command: &str, p: &Value) -> Option { "layer.loadAll" => super::layer::load_all_ahead(s, p), #[cfg(not(target_arch = "wasm32"))] "model.run" => Some(super::workflows::model_ahead(s, p)), + #[cfg(not(target_arch = "wasm32"))] + "map.reloadTiles" | "map.repairTiles" => Some(super::repair::archive_ahead(s, command, p)), _ => None, } } +/// Captures the archive selected now, so a queued UI read cannot retarget an index or name. +#[cfg(not(target_arch = "wasm32"))] +pub fn archive_target(s: &Session, p: &Value) -> Option<(usize, rustgis_map::TileLayer)> { + let i = super::tiles::find(s, p).ok()??; + let tile = s.map().tiles.get(i)?; + tile.archive_file()?; + Some((i, tile.clone())) +} + /// `work` as a read for the command to take under `key`. pub(crate) fn ahead(name: &'static str, key: String, work: impl FnOnce() -> T + Send + 'static) -> ReadAhead { ReadAhead { name, key, work: Box::new(move || Box::new(work()) as Box) } diff --git a/crates/engine/src/cmd/repair.rs b/crates/engine/src/cmd/repair.rs index 7eabd13d1b..5e83e53493 100644 --- a/crates/engine/src/cmd/repair.rs +++ b/crates/engine/src/cmd/repair.rs @@ -127,13 +127,14 @@ fn repair_tiles(s: &mut Session, p: &Value) -> CmdResult { } #[cfg(not(target_arch = "wasm32"))] { + let prepared = super::reads::take::>(s, "map.repairTiles"); let i = super::tiles::find(s, p)?.ok_or("name, index or id is required")?; let t = s.map().tiles.get(i).ok_or("the tile layer no longer exists")?; if t.archive_file().is_none() { return Err(format!("{}: only a tile archive read from a file (PMTiles or MBTiles) can be repaired", t.name)); } let full = replacement(p)?; - let new = archive_candidate(t, &full)?; + let new = archive_candidate_after_read(t, &full, s.map().instance_id(), prepared)?; rustgis_render::rustgis_tiles::cache::reload_archive(&t.url); // The replacement path may already have been drawn by another layer. if t.url != full { @@ -167,6 +168,48 @@ fn archive_candidate(t: &rustgis_map::TileLayer, path: &str) -> Result, +} + +/// Fresh validation for the desktop's single read worker; nothing is installed or invalidated. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn archive_ahead(s: &Session, command: &str, p: &Value) -> super::reads::ReadAhead { + let request = (|| { + let i = super::tiles::find(s, p)?.ok_or("name, index or id is required")?; + let t = s.map().tiles.get(i).ok_or("the tile layer no longer exists")?; + let linked = t.archive_file().ok_or("only a linked local PMTiles or MBTiles archive can be reloaded or repaired")?; + let path = if command == "map.repairTiles" { replacement(p)? } else { linked.to_string() }; + Ok::<_, String>((s.map().instance_id(), t.clone(), path)) + })(); + let name = if command == "map.repairTiles" { "Repair Data Source" } else { "Reload Data" }; + super::reads::ahead(name, command.to_string(), move || { + request.map(|(map, original, path)| { + let result = archive_candidate(&original, &path); + ReadArchive { map, original, path, result } + }) + }) +} + +#[cfg(not(target_arch = "wasm32"))] +fn archive_candidate_after_read( + t: &rustgis_map::TileLayer, + path: &str, + map: u64, + prepared: Option>, +) -> Result { + let Some(prepared) = prepared else { return archive_candidate(t, path) }; + let prepared = prepared?; + if prepared.map != map || prepared.original != *t || prepared.path != path { + return Err("the archive or its settings changed while it was read; retry the command".into()); + } + prepared.result +} + fn reload_tiles(s: &mut Session, p: &Value) -> CmdResult { #[cfg(target_arch = "wasm32")] { @@ -175,6 +218,7 @@ fn reload_tiles(s: &mut Session, p: &Value) -> CmdResult { } #[cfg(not(target_arch = "wasm32"))] { + let prepared = super::reads::take::>(s, "map.reloadTiles"); let i = super::tiles::find(s, p)?.ok_or("name, index or id is required")?; let t = s.map().tiles.get(i).ok_or("the tile layer no longer exists")?; let path = t.archive_file().ok_or_else(|| { @@ -183,13 +227,23 @@ fn reload_tiles(s: &mut Session, p: &Value) -> CmdResult { t.name ) })?.to_string(); - let new = archive_candidate(t, &path).map_err(|e| { + let new = archive_candidate_after_read(t, &path, s.map().instance_id(), prepared).map_err(|e| { format!("{} ({path}): {e}; correct the file and retry Reload Data, or use Repair Data Source if it moved", t.name) })?; + let cancel_view = s.style_view.as_ref().is_some_and(|(pending, _, map)| { + *map == s.map().instance_id() + && pending.kind == t.kind + && pending.url == t.url + && pending.bounds == t.bounds + && pending.id == t.id + }); rustgis_render::rustgis_tiles::cache::reload_archive(&path); let slot = s.map_mut().tiles.get_mut(i).ok_or("the tile layer no longer exists")?; *slot = new; let message = format!("Reloaded {} from {path}; undo cannot restore overwritten file contents", slot.name); + if cancel_view { + s.style_view = None; + } s.notify(message, false); Ok(json!({"index": i, "path": path})) } diff --git a/crates/engine/tests/portable_projects/reload_tiles.rs b/crates/engine/tests/portable_projects/reload_tiles.rs index b34eae8665..bfb5e06ff0 100644 --- a/crates/engine/tests/portable_projects/reload_tiles.rs +++ b/crates/engine/tests/portable_projects/reload_tiles.rs @@ -287,3 +287,183 @@ fn archive_reload_rejects_remote_upload_and_invalid_selectors() { rejected(&mut s, p, expected); } } + +#[test] +fn invalid_raster_and_unreadable_tile_pages_do_not_evict_retained_mbtiles() { + for damaged_page in [false, true] { + let f = Fixture::new(if damaged_page { "reload-tile-page" } else { "reload-png-signature" }); + let path = f.file("linked.mbtiles"); + write_archive(&path, [255, 0, 0, 255], "Retained credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path,"overlay":true})).unwrap(); + let before = s.map().tiles[0].clone(); + let pixels = decoded(&before); + let credits = style(&before); + rustgis_engine::data::mbtiles::close(&path); + let c = rusqlite::Connection::open(&path).unwrap(); + if damaged_page { + let page_size = c.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0)).unwrap() as usize; + let root = c.query_row("SELECT rootpage FROM sqlite_master WHERE name='tiles'", [], |r| r.get::<_, u32>(0)).unwrap() as usize; + drop(c); + let mut bytes = std::fs::read(&path).unwrap(); + bytes[(root - 1) * page_size] = 0; + std::fs::write(&path, bytes).unwrap(); + // Metadata is readable; this is not another damaged-header test. + assert!(rustgis_engine::data::mbtiles::open_path(&path).is_ok()); + rustgis_engine::data::mbtiles::close(&path); + } else { + c.execute("UPDATE tiles SET tile_data=?1", [b"\x89PNG\r\n\x1a\n".as_slice()]).unwrap(); + drop(c); + } + rejected(&mut s, json!({"index":0}), "retry Reload Data"); + assert!(Arc::ptr_eq(&decoded(&before), &pixels)); + assert!(Arc::ptr_eq(&style(&before), &credits)); + write_archive_after_truncation(&path, [0, 255, 0, 255], "Recovered credits"); + s.run("map.reloadTiles", &json!({"index":0})).unwrap(); + assert_color(&decoded(&before), [0, 255, 0, 255]); + rustgis_engine::data::mbtiles::close(&path); + } +} + +fn write_leaf_archive(path: &Path, corrupt: bool) { + let archive = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Png, + gzipped_tiles: false, + bounds: [-180.0, -85.0, 180.0, 85.0], + metadata: json!({"attribution":"Leaf credits"}), + }; + let bytes = pmtiles::write(vec![(0, 0, 0, picture([0, 255, 0, 255]))], &archive).unwrap(); + let number = |at| u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()) as usize; + let mut leaf = bytes[number(8)..number(8) + number(16)].to_vec(); + if corrupt { + // One entry's final offset varint has no terminating byte. + *leaf.last_mut().unwrap() = 0x80; + } + assert!(leaf.len() < 128); + let root = [1, 0, 0, leaf.len() as u8, 1]; + let meta = &bytes[number(24)..number(24) + number(32)]; + let data = &bytes[number(56)..number(56) + number(64)]; + let mut header = bytes[..127].to_vec(); + let meta_at = 127 + root.len(); + let leaf_at = meta_at + meta.len(); + let data_at = leaf_at + leaf.len(); + for (at, value) in + [(8, 127), (16, root.len()), (24, meta_at), (32, meta.len()), (40, leaf_at), (48, leaf.len()), (56, data_at), (64, data.len())] + { + header[at..at + 8].copy_from_slice(&(value as u64).to_le_bytes()); + } + std::fs::write(path, [header.as_slice(), root.as_slice(), meta, leaf.as_slice(), data].concat()).unwrap(); +} + +#[test] +fn malformed_leaf_reload_retains_pixels_and_can_retry_with_a_valid_leaf() { + let f = Fixture::new("reload-malformed-leaf"); + let path = f.file("linked.pmtiles"); + write_archive(&path, [255, 0, 0, 255], "Retained credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path,"overlay":true})).unwrap(); + let before = s.map().tiles[0].clone(); + let pixels = decoded(&before); + let credits = style(&before); + let camera = s.map().camera; + write_leaf_archive(&path, true); + rejected(&mut s, json!({"index":0}), "retry Reload Data"); + assert!(Arc::ptr_eq(&decoded(&before), &pixels)); + assert!(Arc::ptr_eq(&style(&before), &credits)); + assert_eq!(s.map().camera, camera); + write_leaf_archive(&path, false); + s.run("map.reloadTiles", &json!({"index":0})).unwrap(); + assert_color(&decoded(&before), [0, 255, 0, 255]); +} + +#[test] +fn untrusted_workflows_cannot_reload_but_user_authored_workflows_can() { + let f = Fixture::new("reload-workflow-trust"); + let path = f.file("linked.pmtiles"); + write_archive(&path, [255, 0, 0, 255], "Before"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path,"overlay":true})).unwrap(); + let tile = s.map().tiles[0].clone(); + let old_pixels = decoded(&tile); + let task = json!({"name":"Reload task","steps":[{"title":"Reload","command":"map.reloadTiles","params":{"index":0}}]}); + let error = s.run("task.import", &json!({"text":task.to_string()})).unwrap_err(); + assert!(error.contains("imported tasks"), "{error}"); + let notebook = json!({"nbformat":4,"metadata":{"language_info":{"name":"rustgis"}},"cells":[ + {"cell_type":"code","source":"map.reloadTiles {\"index\":0}"} + ]}); + s.run("notebook.open", &json!({"text":notebook.to_string()})).unwrap(); + let answer = s.run("notebook.run", &Value::Null).unwrap(); + assert_eq!(answer["error"], 0); + assert!(s.project.notebooks.last().unwrap().cells[0].error); + assert!(Arc::ptr_eq(&decoded(&tile), &old_pixels)); + write_archive(&path, [0, 255, 0, 255], "After"); + s.run("task.new", &task).unwrap(); + s.run("task.run", &json!({"task":"Reload task"})).unwrap(); + assert_color(&decoded(&tile), [0, 255, 0, 255]); + s.run("notebook.new", &json!({"name":"User reload"})).unwrap(); + s.run("notebook.addCell", &json!({"source":"map.reloadTiles {\"index\":0}"})).unwrap(); + let answer = s.run("notebook.run", &Value::Null).unwrap(); + assert!(answer.get("error").is_none(), "{answer}"); +} + +#[test] +fn successful_reload_cancels_only_its_own_pending_archive_framing() { + let f = Fixture::new("reload-pending-view"); + let path = f.file("linked.pmtiles"); + let other = f.file("other.pmtiles"); + for file in [&path, &other] { + let archive = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Png, + gzipped_tiles: false, + bounds: [20.0, 30.0, 25.0, 35.0], + metadata: json!({}), + }; + std::fs::write(file, pmtiles::write(vec![(0, 0, 0, picture([255, 0, 0, 255]))], &archive).unwrap()).unwrap(); + } + let mut s = session(); + s.run("map.addTiles", &json!({"url":path,"overlay":true})).unwrap(); + s.run("map.addTiles", &json!({"url":other,"overlay":true})).unwrap(); + s.style_view = None; + s.run("map.goToXY", &json!({"lon":-40,"lat":-20,"scale":50_000_000})).unwrap(); + let target = s.map().tiles.iter().position(|t| t.url == path.to_str().unwrap()).unwrap(); + let tile = s.map().tiles[target].clone(); + let camera = s.map().camera; + cache::reload_archive(&tile.url); + s.style_view = Some((tile.clone(), camera, s.map().instance_id())); + s.run("map.reloadTiles", &json!({"index":target})).unwrap(); + assert!(s.style_view.is_none(), "late target metadata must no longer frame the view"); + let _ = style(&tile); + assert!(!s.follow_style_view()); + assert_eq!(s.map().camera, camera); + let other_tile = s.map().tiles.iter().find(|t| t.url != tile.url).unwrap().clone(); + cache::reload_archive(&other_tile.url); + s.style_view = Some((other_tile.clone(), camera, s.map().instance_id())); + s.run("map.reloadTiles", &json!({"index":target})).unwrap(); + assert!(s.style_view.as_ref().is_some_and(|(pending, _, _)| pending.url == other_tile.url)); + let _ = style(&other_tile); + assert!(s.follow_style_view(), "another archive's requested framing still works"); + assert_ne!(s.map().camera, camera); +} + +#[test] +fn prepared_archive_validation_does_not_evict_or_overwrite_changed_settings() { + let f = Fixture::new("reload-read-ahead"); + let path = f.file("linked.pmtiles"); + write_archive(&path, [255, 0, 0, 255], "Before"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path,"overlay":true})).unwrap(); + let tile = s.map().tiles[0].clone(); + let old_pixels = decoded(&tile); + write_archive(&path, [0, 255, 0, 255], "After"); + let control = rustgis_engine::analysis::control::Control::stoppable(); + let read = rustgis_engine::cmd::reads::read_ahead(&s, "map.reloadTiles", &json!({"index":0})).unwrap().run(&control).unwrap(); + assert!(Arc::ptr_eq(&decoded(&tile), &old_pixels), "validation alone must not invalidate"); + s.run("map.setTiles", &json!({"index":0,"opacity":0.4})).unwrap(); + read.placed(&mut s); + rejected(&mut s, json!({"index":0}), "settings changed"); + assert_eq!(s.map().tiles[0].opacity, 0.4); + assert!(Arc::ptr_eq(&decoded(&tile), &old_pixels)); + s.run("map.reloadTiles", &json!({"index":0})).unwrap(); + assert_eq!(s.map().tiles[0].opacity, 0.4); + assert_color(&decoded(&tile), [0, 255, 0, 255]); +} diff --git a/crates/tiles/src/mbtiles.rs b/crates/tiles/src/mbtiles.rs index 5beb9e93aa..4ef2baa36d 100644 --- a/crates/tiles/src/mbtiles.rs +++ b/crates/tiles/src/mbtiles.rs @@ -118,7 +118,7 @@ fn info_epoch(archive: &str) -> (u64, u64) { (EPOCH.load(Ordering::SeqCst), crate::cache::archive_epoch(archive)) } -/// Reads the current file using a fresh, unpooled connection and retains nothing. +/// Reads the current file and decodes one raster sample, without using or changing any cache. pub fn read_info_local(archive: &str) -> Result, String> { check(archive)?; if is_upload(archive) { @@ -127,7 +127,11 @@ pub fn read_info_local(archive: &str) -> Result, String> { #[cfg(not(target_arch = "wasm32"))] { let path = std::path::Path::new(archive.strip_prefix("file://").unwrap_or(archive)); - rustgis_data::mbtiles::read_info_path(path).map(Arc::new) + let (info, sample) = rustgis_data::mbtiles::read_info_path(path)?; + if let Some(sample) = sample { + validate_raster_sample(archive, &sample)?; + } + Ok(Arc::new(info)) } #[cfg(target_arch = "wasm32")] { @@ -135,6 +139,13 @@ pub fn read_info_local(archive: &str) -> Result, String> { } } +#[cfg(not(target_arch = "wasm32"))] +fn validate_raster_sample(archive: &str, sample: &[u8]) -> Result<(), String> { + crate::cache::decode_image(sample) + .map(|_| ()) + .map_err(|e| format!("{archive}: cannot decode the MBTiles raster sample ({e}); repair or replace the archive before reloading")) +} + /// An archive's metadata. pub fn info(archive: &str) -> Result, String> { check(archive)?; @@ -339,6 +350,22 @@ mod tests { assert!(cached_info(archive).is_none()); } + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn raster_samples_require_a_decodable_bounded_image() { + let archive = "/invalid-raster-sample.mbtiles"; + for sample in [b"\x89PNG\r\n\x1a\n".as_slice(), b"not a picture", b""] { + let error = validate_raster_sample(archive, sample).unwrap_err(); + assert!(error.contains(archive), "{error}"); + assert!(error.contains("cannot decode the MBTiles raster sample"), "{error}"); + assert!(error.contains("repair or replace"), "{error}"); + } + let valid = tiny_skia::Pixmap::new(2, 2).unwrap().encode_png().unwrap(); + assert!(validate_raster_sample(archive, &valid).is_ok()); + let oversized = tiny_skia::Pixmap::new(4097, 1).unwrap().encode_png().unwrap(); + assert!(validate_raster_sample(archive, &oversized).is_err()); + } + #[cfg(not(target_arch = "wasm32"))] #[test] fn stalled_metadata_reports_errors_without_recycling_worker_slots() { diff --git a/crates/tiles/src/pmtiles.rs b/crates/tiles/src/pmtiles.rs index 93090990e6..82a0c179f0 100644 --- a/crates/tiles/src/pmtiles.rs +++ b/crates/tiles/src/pmtiles.rs @@ -29,6 +29,10 @@ const MAX_ENTRIES: usize = 1 << 20; const MAX_DEPTH: usize = 4; /// Leaf directories kept. const MAX_LEAVES: usize = 256; +/// Fresh validation bounds the entire reachable index, including the root. +#[cfg(not(target_arch = "wasm32"))] +const INDEX_BUDGET: IndexBudget = + IndexBudget { stored: (MAX_BYTES * 4) as u64, decoded: (MAX_BYTES * 4) as u64, entries: MAX_ENTRIES, directories: MAX_LEAVES * 16 }; /// A failed archive is tried again after this long. const RETRY: Duration = Duration::from_secs(60); @@ -114,7 +118,12 @@ impl Compression { }; match self { Compression::None => Ok(b), - Compression::Gzip => gunzip(&b), + Compression::Gzip => { + if !b.starts_with(&[0x1f, 0x8b]) { + return Err("gzip: invalid header".into()); + } + gunzip(&b) + } Compression::Brotli => bounded(&mut brotli_decompressor::Decompressor::new(b.as_slice(), 4096), "Brotli"), Compression::Zstd => unzstd(&b, limit), } @@ -753,10 +762,16 @@ fn metadata(info: &mut Info, bytes: Vec, expected: u64) -> Result<(), String Ok(()) } -/// Reads the current local file without consulting or changing retained caches. All ranges -/// are bounded and freshly opened; failed validation leaves the displayed archive intact. +/// Reads the current local file without consulting or changing retained caches. All reachable +/// directories are validated within fixed aggregate budgets; tile payloads remain lazy. +/// Ranges are bounded and freshly opened; failure leaves the displayed archive intact. #[cfg(not(target_arch = "wasm32"))] pub fn read_info_local(archive: &str) -> Result, String> { + read_info_local_bounded(archive, INDEX_BUDGET) +} + +#[cfg(not(target_arch = "wasm32"))] +fn read_info_local_bounded(archive: &str, mut budget: IndexBudget) -> Result, String> { check_local(archive)?; let head = read_file(archive, 0, HEAD)?; let path = archive.strip_prefix("file://").unwrap_or(archive); @@ -764,7 +779,7 @@ pub fn read_info_local(archive: &str) -> Result, String> { if head.len() < 127 { return Err("a truncated PMTiles header".into()); } - let (mut info, at, len) = parse_head(&head)?; + let (mut info, at, len, root_decoded) = parse_head_detail(&head)?; for (offset, length) in [(8, 16), (24, 32), (40, 48), (56, 64)] { let start = u64_at(&head, offset).ok_or("a truncated PMTiles header")?; let size = u64_at(&head, length).ok_or("a truncated PMTiles header")?; @@ -775,6 +790,8 @@ pub fn read_info_local(archive: &str) -> Result, String> { if len != 0 { metadata(&mut info, read_file(archive, at, len)?, len)?; } + budget.take(u64_at(&head, 16).ok_or("a truncated PMTiles header")?, root_decoded, info.root.len())?; + validate_directories(archive, &info, &info.root, 0, &mut budget, &mut HashMap::new())?; Ok(Arc::new(info)) } @@ -783,6 +800,97 @@ pub fn read_info_local(archive: &str) -> Result, String> { Err(format!("{archive}: the web app cannot reload local PMTiles paths")) } +/// Remaining aggregate work. The decoder also independently caps any one directory at 64 MiB; +/// an over-budget decode is discarded before parsing or following its references. +#[cfg(not(target_arch = "wasm32"))] +#[derive(Clone, Copy)] +struct IndexBudget { + stored: u64, + decoded: u64, + entries: usize, + directories: usize, +} + +#[cfg(not(target_arch = "wasm32"))] +impl IndexBudget { + fn take(&mut self, stored: u64, decoded: u64, entries: usize) -> Result<(), String> { + let error = "PMTiles index exceeds validation budget"; + self.stored = self.stored.checked_sub(stored).ok_or(error)?; + self.decoded = self.decoded.checked_sub(decoded).ok_or(error)?; + self.entries = self.entries.checked_sub(entries).ok_or(error)?; + self.directories = self.directories.checked_sub(1).ok_or(error)?; + Ok(()) + } +} + +/// Completed ranges remember their subtree height, so sharing a directory cannot hide a path +/// exceeding MAX_DEPTH. An unfinished range encountered again is a cycle. +#[cfg(not(target_arch = "wasm32"))] +fn validate_directories( + archive: &str, + info: &Info, + dir: &[Entry], + depth: usize, + budget: &mut IndexBudget, + seen: &mut HashMap<(u64, u64), Option>, +) -> Result { + let mut height = 0; + for e in dir { + let end = e.offset.checked_add(e.length).ok_or("a PMTiles entry range overflows")?; + if e.run_length > 0 { + e.tile_id.checked_add(e.run_length).ok_or("tile ids overflow")?; + if end > info.data_length || info.data_offset.checked_add(end).is_none() { + return Err("a tile outside the archive".into()); + } + if e.length > MAX_BYTES as u64 { + return Err(format!("{} bytes is too large to read from an archive", e.length)); + } + continue; + } + if depth >= MAX_DEPTH { + return Err("PMTiles directories nested too deep".into()); + } + if end > info.leaf_length { + return Err("a leaf directory outside the archive".into()); + } + let at = info.leaf_offset.checked_add(e.offset).ok_or("a leaf directory range overflows")?; + info.leaf_offset.checked_add(end).ok_or("a leaf directory range overflows")?; + if e.length > MAX_BYTES as u64 { + return Err(format!("{} bytes is too large to read from an archive", e.length)); + } + let key = (at, e.length); + let child_height = if let Some(done) = seen.get(&key) { + done.ok_or("cyclic PMTiles directories")? + } else { + // Refuse oversized ranges and too many directories before issuing any I/O. + if budget.directories == 0 || e.length > budget.stored { + return Err("PMTiles index exceeds validation budget".into()); + } + seen.insert(key, None); + let bytes = read_file(archive, at, e.length)?; + if bytes.len() as u64 != e.length { + return Err("a truncated PMTiles leaf directory".into()); + } + let bytes = info.internal.decode(bytes)?; + if bytes.len() as u64 > budget.decoded { + return Err("PMTiles index exceeds validation budget".into()); + } + let child = parse_dir_bounded(&bytes, budget.entries)?; + budget.take(e.length, bytes.len() as u64, child.len())?; + drop(bytes); + let child_height = validate_directories(archive, info, &child, depth + 1, budget, seen)?; + seen.insert(key, Some(child_height)); + child_height + }; + let subtree = child_height + 1; + if depth + subtree > MAX_DEPTH { + return Err("PMTiles directories nested too deep".into()); + } + height = height.max(subtree); + } + Ok(height) +} + fn u64_at(b: &[u8], i: usize) -> Option { Some(u64::from_le_bytes(b.get(i..i + 8)?.try_into().ok()?)) } @@ -793,6 +901,10 @@ fn i32_at(b: &[u8], i: usize) -> Option { /// The header and root directory, and where the metadata is. fn parse_head(b: &[u8]) -> Result<(Info, u64, u64), String> { + parse_head_detail(b).map(|(info, at, len, _)| (info, at, len)) +} + +fn parse_head_detail(b: &[u8]) -> Result<(Info, u64, u64, u64), String> { if b.get(..7) != Some(b"PMTiles".as_slice()) { return Err("not a PMTiles archive".into()); } @@ -849,17 +961,24 @@ fn parse_head(b: &[u8]) -> Result<(Info, u64, u64), String> { if meta_len > MAX_BYTES as u64 || meta_at.checked_add(meta_len).is_none() { return Err("PMTiles metadata is too large or has an invalid range".into()); } - Ok((info, meta_at, meta_len)) + Ok((info, meta_at, meta_len, raw.len() as u64)) } /// A directory: delta-coded tile ids, then run lengths, lengths and offsets, all varints. fn parse_dir(b: &[u8]) -> Result, String> { + parse_dir_bounded(b, MAX_ENTRIES) +} + +fn parse_dir_bounded(b: &[u8], entry_budget: usize) -> Result, String> { let mut r = Varints { b, at: 0 }; let n = usize::try_from(r.next()?).map_err(|_| "a directory too large")?; // Each entry takes at least four bytes. if n > MAX_ENTRIES || n.saturating_mul(4) > b.len() { return Err("a directory with more entries than it can hold".into()); } + if n > entry_budget { + return Err("PMTiles index exceeds validation budget".into()); + } let mut entries = vec![Entry { tile_id: 0, offset: 0, length: 0, run_length: 0 }; n]; let mut last = 0u64; for e in entries.iter_mut() { @@ -878,7 +997,7 @@ fn parse_dir(b: &[u8]) -> Result, String> { let e = entries.get_mut(i).ok_or("a short directory")?; e.offset = match (v, prev) { // Right after the previous entry. - (0, Some(p)) => p.offset.saturating_add(p.length), + (0, Some(p)) => p.offset.checked_add(p.length).ok_or("a directory offset overflows")?, (v, _) => v.saturating_sub(1), }; } @@ -896,6 +1015,9 @@ impl Varints<'_> { for shift in (0..64).step_by(7) { let byte = *self.b.get(self.at).ok_or("a truncated directory")?; self.at += 1; + if shift == 63 && byte > 1 { + return Err("a varint too long".into()); + } v |= u64::from(byte & 0x7f) << shift; if byte & 0x80 == 0 { return Ok(v); @@ -969,6 +1091,167 @@ pub(crate) mod tests { std::fs::remove_file(path).unwrap(); } + #[cfg(not(target_arch = "wasm32"))] + fn indexed_archive(root: &[Entry], leaves: Vec) -> Vec { + let mut head = archive(&[(0, 0, 0, vec![1, 2, 3])], 1, false, "{}")[..127].to_vec(); + let root = write_dir(root); + let meta_at = 127 + root.len() as u64; + let leaf_at = meta_at + 2; + let data_at = leaf_at + leaves.len() as u64; + for (at, value) in [(16, root.len() as u64), (24, meta_at), (40, leaf_at), (48, leaves.len() as u64), (56, data_at)] { + head[at..at + 8].copy_from_slice(&value.to_le_bytes()); + } + [head, root, b"{}".to_vec(), leaves, vec![1, 2, 3]].concat() + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn fresh_local_validation_checks_every_leaf_and_deduplicates_shared_ranges() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-leaves-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + let mut bad = archive(&[(0, 0, 0, vec![1, 2, 3])], 1, true, "{}"); + bad[138] = 0x80; + std::fs::write(&path, bad).unwrap(); + assert_eq!(read_info_local(source).unwrap_err(), "a truncated directory"); + + let first = write_dir(&[Entry { tile_id: 0, offset: 0, length: 3, run_length: 1 }]); + let second = write_dir(&[Entry { tile_id: 1, offset: 0, length: 3, run_length: 1 }]); + let root = [ + Entry { tile_id: 0, offset: 0, length: first.len() as u64, run_length: 0 }, + Entry { tile_id: 1, offset: first.len() as u64, length: second.len() as u64, run_length: 0 }, + Entry { tile_id: 2, offset: 0, length: first.len() as u64, run_length: 0 }, + ]; + let leaves = [first.clone(), second].concat(); + let valid = indexed_archive(&root, leaves.clone()); + std::fs::write(&path, &valid).unwrap(); + assert!(read_info_local(source).is_ok()); + // Exactly enough for the root and two unique leaves; the repeated range costs no work. + let bytes = (write_dir(&root).len() + leaves.len()) as u64; + let exact = IndexBudget { stored: bytes, decoded: bytes, entries: 5, directories: 3 }; + assert!(read_info_local_bounded(source, exact).is_ok()); + for budget in [ + IndexBudget { stored: bytes - 1, ..exact }, + IndexBudget { decoded: bytes - 1, ..exact }, + IndexBudget { entries: 4, ..exact }, + IndexBudget { directories: 2, ..exact }, + ] { + assert_eq!(read_info_local_bounded(source, budget).unwrap_err(), "PMTiles index exceeds validation budget"); + } + + let mut later_bad = valid; + let leaf_at = u64_at(&later_bad, 40).unwrap() as usize; + later_bad[leaf_at + leaves.len() - 1] = 0x80; + std::fs::write(&path, later_bad).unwrap(); + assert_eq!(read_info_local(source).unwrap_err(), "a truncated directory"); + for entry in [ + Entry { tile_id: 0, offset: u64::MAX - 1, length: 2, run_length: 0 }, + Entry { tile_id: 0, offset: 0, length: leaves.len() as u64 + 1, run_length: 0 }, + ] { + std::fs::write(&path, indexed_archive(&[entry], leaves.clone())).unwrap(); + let error = read_info_local(source).unwrap_err(); + assert!(error.contains("overflows") || error == "a leaf directory outside the archive", "{error}"); + } + // Oversized leaf reads fail before opening a file, without a huge fixture. + let (mut info, _, _) = parse_head(&archive(&[(0, 0, 0, vec![1])], 1, false, "{}")).unwrap(); + info.leaf_length = MAX_BYTES as u64 + 1; + let entry = Entry { tile_id: 0, offset: 0, length: info.leaf_length, run_length: 0 }; + let mut budget = INDEX_BUDGET; + assert_eq!( + validate_directories("/nonexistent.pmtiles", &info, &[entry], 0, &mut budget, &mut HashMap::new()).unwrap_err(), + format!("{} bytes is too large to read from an archive", entry.length) + ); + // A declared compression on lazy tile payloads is not decoded during validation. + let mut lazy = archive(&[(0, 0, 0, vec![1, 2, 3])], 1, true, "{}"); + lazy[98] = 2; + std::fs::write(&path, lazy).unwrap(); + assert!(read_info_local(source).is_ok()); + std::fs::remove_file(path).unwrap(); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn fresh_local_validation_rejects_cycles_and_excess_depth_even_when_shared() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-depth-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + let pointer = |offset| Entry { tile_id: 0, offset, length: 5, run_length: 0 }; + let tile = Entry { tile_id: 0, offset: 0, length: 3, run_length: 1 }; + std::fs::write(&path, indexed_archive(&[pointer(0)], write_dir(&[pointer(0)]))).unwrap(); + assert_eq!(read_info_local(source).unwrap_err(), "cyclic PMTiles directories"); + let mutual: Vec = [pointer(5), pointer(0)].iter().flat_map(|entry| write_dir(&[*entry])).collect(); + std::fs::write(&path, indexed_archive(&[pointer(0)], mutual)).unwrap(); + assert_eq!(read_info_local(source).unwrap_err(), "cyclic PMTiles directories"); + for count in [MAX_DEPTH, MAX_DEPTH + 1] { + let leaves: Vec = + (0..count).flat_map(|i| write_dir(&[if i + 1 == count { tile } else { pointer((i as u64 + 1) * 5) }])).collect(); + std::fs::write(&path, indexed_archive(&[pointer(0)], leaves)).unwrap(); + if count == MAX_DEPTH { + assert!(read_info_local(source).is_ok()); + } else { + assert_eq!(read_info_local(source).unwrap_err(), "PMTiles directories nested too deep"); + } + } + // Validate a shared subtree shallowly first, then reach it by an over-deep path. + let leaves: Vec = + [pointer(5), tile, pointer(15), pointer(20), pointer(0)].iter().flat_map(|entry| write_dir(&[*entry])).collect(); + let root = [pointer(0), Entry { tile_id: 1, ..pointer(10) }]; + std::fs::write(&path, indexed_archive(&root, leaves)).unwrap(); + assert_eq!(read_info_local(source).unwrap_err(), "PMTiles directories nested too deep"); + std::fs::remove_file(path).unwrap(); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn ordinary_loaders_reject_present_invalid_metadata_with_specific_errors() { + let dir = std::env::temp_dir().join(format!("rustgis-pmtiles-metadata-errors-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let mut cases = Vec::new(); + for (text, error) in [ + ("{", "PMTiles metadata:"), + ("null", "PMTiles metadata must be a JSON object"), + ("[]", "PMTiles metadata must be a JSON object"), + ("42", "PMTiles metadata must be a JSON object"), + ] { + cases.push((archive(&[(0, 0, 0, vec![1])], 1, false, text), error)); + } + let mut truncated = archive(&[(0, 0, 0, vec![1])], 1, false, "{}"); + truncated[32..40].copy_from_slice(&100u64.to_le_bytes()); + cases.push((truncated, "truncated PMTiles metadata")); + for (code, error) in [(2, "gzip: invalid header"), (3, "Brotli:"), (4, "Zstandard:")] { + let mut invalid = archive_with(&[(0, 0, 0, vec![1])], 1, false, "{}", code); + let at = u64_at(&invalid, 24).unwrap() as usize; + invalid[at] = 0xff; + invalid[32..40].copy_from_slice(&1u64.to_le_bytes()); + cases.push((invalid, error)); + } + let mut oversized = archive(&[(0, 0, 0, vec![1])], 1, false, "{}"); + oversized[32..40].copy_from_slice(&(MAX_BYTES as u64 + 1).to_le_bytes()); + // Store the whole oversized (otherwise valid JSON) section, not merely a header + // claiming an invalid range. The ordinary loader must reject it before reading it. + let metadata_at = u64_at(&oversized, 24).unwrap() as usize; + let metadata_end = metadata_at + MAX_BYTES + 1; + oversized.truncate(metadata_at + 2); + oversized.resize(metadata_end, b' '); + oversized.push(1); + oversized[56..64].copy_from_slice(&(metadata_end as u64).to_le_bytes()); + cases.push((oversized, "PMTiles metadata is too large or has an invalid range")); + for (i, (bytes, expected)) in cases.into_iter().enumerate() { + // Separate paths force both entry points through their ordinary loader, not a + // failure memoized by the preceding assertion. + for loader in ["info", "get"] { + let path = dir.join(format!("{i}-{loader}.pmtiles")); + std::fs::write(&path, &bytes).unwrap(); + let source = path.to_str().unwrap(); + let error = if loader == "info" { + info_blocking(source, std::time::Duration::from_secs(10)).unwrap_err() + } else { + get(&format!("pmtiles:0/0/0@{source}")).unwrap_err() + }; + assert!(error.starts_with(expected), "case {i}, {loader}: {error}"); + } + } + std::fs::remove_dir_all(dir).unwrap(); + } + #[test] fn tile_ids_follow_the_spec() { assert_eq!(tile_id(0, 0, 0), 0); @@ -1158,6 +1441,8 @@ pub(crate) mod tests { let p = dir.join(format!("t{leaf}{code}.pmtiles")); std::fs::write(&p, archive_with(&tiles, 1, leaf, r#"{"vector_layers":[{"id":"roads"}],"attribution":"Test"}"#, code)).unwrap(); let a = p.display().to_string(); + #[cfg(not(target_arch = "wasm32"))] + assert_eq!(read_info_local(&a).unwrap().attribution(), "Test"); let mut i = info(&a).unwrap(); for _ in 0..200 { if i.is_some() { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 59d44c4340..a16705ac60 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -2083,6 +2083,22 @@ impl RustGisApp { /// Runs a command by id: `ui.*` here, everything else in the engine. pub fn run(&mut self, id: &str, params: Value) -> Result { + #[cfg(not(target_arch = "wasm32"))] + if self.reads_off_frame { + let archive = match id { + "map.reloadTiles" => Some("map.reloadTiles"), + "map.repairTiles" => Some("map.repairTiles"), + _ => None, + }; + if let Some(command) = archive { + return readjob::run(self, command, params); + } + } + self.run_now(id, params) + } + + /// Executes a command after any native read-ahead has completed. + fn run_now(&mut self, id: &str, params: Value) -> Result { let _diagnostics = self.session.diagnostics.enter(); if !matches!(id, "ui.menu" | "ui.ribbon" | "ui.language" | "ui.worldview") { self.ui.ribbon_peek = false; diff --git a/crates/ui-egui/src/readjob.rs b/crates/ui-egui/src/readjob.rs index 95e47bed4c..0954932bab 100644 --- a/crates/ui-egui/src/readjob.rs +++ b/crates/ui-egui/src/readjob.rs @@ -35,6 +35,8 @@ struct Asked { opened: u64, map: u64, layer: Option, + /// The exact archive and its presentation when asked, not a reusable list index. + archive: Option<(usize, rustgis_engine::map::TileLayer)>, /// Given what the command answered when it runs, or why it didn't (cancelled, or left). then: Option, } @@ -67,7 +69,7 @@ pub fn run(app: &mut RustGisApp, command: &'static str, p: Value) -> Result) -> Result { if !app.reads_off_frame { - let r = app.run(command, p); + let r = app.run_now(command, p); return answered(app, then, r); } let layer = (command == "layer.loadAll") @@ -80,8 +82,14 @@ pub fn run_then(app: &mut RustGisApp, command: &'static str, p: Value, then: Opt { return Ok(json!({"reading": instance})); } + let archive = matches!(command, "map.reloadTiles" | "map.repairTiles").then(|| reads::archive_target(&app.session, &p)).flatten(); + let mut p = p; + if let Some((i, _)) = &archive { + p["index"] = json!(i); + } let what = match &layer { Some((_, name)) => name.clone(), + None if archive.is_some() => archive.as_ref().map(|(_, tile)| tile.name.clone()).unwrap_or_default(), // A model by its name (one drawn but not yet named has none: see `called`). None if command == "model.run" => p.get("name").and_then(Value::as_str).map(str::trim).unwrap_or_default().to_string(), None => p.get("path").and_then(Value::as_str).map(|path| file_name(path).to_string()).unwrap_or_default(), @@ -93,6 +101,7 @@ pub fn run_then(app: &mut RustGisApp, command: &'static str, p: Value, then: Opt opened: app.session.opened, map: app.session.map().instance_id(), layer: layer.map(|(instance, _)| instance), + archive, then, }; if busy(app) { @@ -145,6 +154,7 @@ enum Gone { Project, Map, Layer, + Archive, } impl std::fmt::Display for Gone { @@ -154,6 +164,7 @@ impl std::fmt::Display for Gone { Gone::Project => tl_raw!("another project was opened"), Gone::Map => tl_raw!("the map it was asked for isn't open"), Gone::Layer => tl_raw!("its layer was removed"), + Gone::Archive => tl_raw!("its tile archive changed or was removed"), }) } } @@ -167,6 +178,11 @@ fn placed(app: &RustGisApp, asked: &Asked) -> Result { if app.session.map().instance_id() != asked.map { return Err(Gone::Map); } + if let Some((i, original)) = &asked.archive + && app.session.map().tiles.get(*i) != Some(original) + { + return Err(Gone::Archive); + } let Some(instance) = asked.layer else { return Ok(asked.params.clone()) }; let l = app.session.map().layers.iter().find(|l| l.instance_id() == instance).ok_or(Gone::Layer)?; let mut p = asked.params.clone(); @@ -180,7 +196,13 @@ fn start(app: &mut RustGisApp, asked: Asked, p: Value) -> Result // Picked on this thread, so a bug in the picking must not take the app down: it reads here. let picked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| reads::read_ahead(&app.session, asked.command, &p))); let Ok(Some(ahead)) = picked else { - let r = app.run(asked.command, p); + if matches!(asked.command, "map.reloadTiles" | "map.repairTiles") { + let why = tl!("The archive could not be prepared for background validation. Retry the command.").to_string(); + app.session.notify(why.clone(), true); + not_run(app, asked.then, why.clone()); + return Err(why); + } + let r = app.run_now(asked.command, p); return answered(app, asked.then, r); }; let name = ahead.name; @@ -196,8 +218,17 @@ fn start(app: &mut RustGisApp, asked: Asked, p: Value) -> Result *out.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(read); }); if let Err(e) = spawned { + if matches!(asked.command, "map.reloadTiles" | "map.repairTiles") { + let why = crate::i18n::fmt( + "The archive validation worker could not start ({error}). Retry the command.", + &[("error", &e.to_string())], + ); + app.session.notify(why.clone(), true); + not_run(app, asked.then, why.clone()); + return Err(why); + } log::warn!("{name} reads here: its thread didn't start: {e}"); - let r = app.run(asked.command, p); + let r = app.run_now(asked.command, p); return answered(app, asked.then, r); } app.reads.job = Some(ReadJob { asked, gone: None, name, control, slot }); @@ -250,7 +281,15 @@ pub fn land(app: &mut RustGisApp) -> bool { Ok(read) => { read.placed(&mut app.session); // A failure is said by the command. - let r = app.run(job.asked.command, p); + let r = app.run_now(job.asked.command, p); + if job.asked.command == "map.repairTiles" + && let Ok(answer) = &r + && let Some(index) = answer.get("index").and_then(Value::as_u64).and_then(|i| usize::try_from(i).ok()) + && let Some(tile) = app.session.map().tiles.get(index) + { + let text = crate::i18n::fmt("Repaired {name}", &[("name", &tile.name)]); + app.session.notify(text, false); + } let _ = answered(app, job.asked.then, r); app.session.drop_read(); app.map.invalidate(); @@ -265,7 +304,7 @@ pub fn land(app: &mut RustGisApp) -> bool { ), false, ); - let r = app.run(job.asked.command, p); + let r = app.run_now(job.asked.command, p); let _ = answered(app, job.asked.then, r); app.map.invalidate(); } @@ -296,7 +335,7 @@ pub fn land(app: &mut RustGisApp) -> bool { for (what, why, then) in &left { not_run(app, *then, crate::i18n::fmt("{what} was left: {why}", &[("what", called(what)), ("why", &why.to_string())])); } - for why in [Gone::Project, Gone::Map, Gone::Layer] { + for why in [Gone::Project, Gone::Map, Gone::Layer, Gone::Archive] { let of: Vec<&String> = left.iter().filter(|(_, w, _)| *w == why).map(|(what, ..)| what).collect(); let reason = why.to_string(); match of.as_slice() { @@ -364,6 +403,7 @@ pub fn status(app: &RustGisApp, ui: &mut Ui, t: &Tokens) { #[allow(clippy::unwrap_used)] mod tests { use super::*; + use std::time::Duration; /// Lands every read under way and waiting, failing past `secs`. fn landed(app: &mut RustGisApp, secs: u64) { @@ -463,4 +503,86 @@ mod tests { assert!(app.session.project.maps[0].layer(id).unwrap().is_streamed()); let _ = std::fs::remove_dir_all(&dir); } + + fn archive(path: &std::path::Path, credit: &str) { + use rustgis_engine::render::rustgis_tiles::{mvt, pmtiles}; + let metadata = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Mvt, + gzipped_tiles: false, + bounds: [-180.0, -85.0, 180.0, 85.0], + metadata: json!({"attribution":credit}), + }; + let bytes = pmtiles::write(vec![(0, 0, 0, mvt::encode(&mvt::Tile { layers: Vec::new() }))], &metadata).unwrap(); + std::fs::write(path, bytes).unwrap(); + } + + #[test] + fn archive_reload_lands_once_and_stale_or_cancelled_reads_keep_retained_metadata() { + use rustgis_engine::render::rustgis_tiles::pmtiles; + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + let dir = std::env::temp_dir().join(format!("rustgis-read-archive-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("source.pmtiles"); + let url = path.to_str().unwrap(); + archive(&path, "Before"); + app.run("map.addTiles", json!({"url":url,"overlay":true})).unwrap(); + let retained = pmtiles::info_blocking(url, Duration::from_secs(10)).unwrap(); + let tile = app.session.map().tiles[0].clone(); + let camera = app.session.map().camera; + archive(&path, "After"); + app.reads_off_frame = true; + let depth = app.session.undo_depth(); + assert_eq!(app.run("map.reloadTiles", json!({"index":0})).unwrap()["started"], "Reload Data"); + assert!(Arc::ptr_eq(&pmtiles::info_blocking(url, Duration::from_secs(10)).unwrap(), &retained)); + assert_eq!(app.session.undo_depth(), depth, "validation has not installed anything"); + landed(&mut app, 60); + let current = pmtiles::info_blocking(url, Duration::from_secs(10)).unwrap(); + assert_eq!(current.attribution(), "After"); + assert_eq!(app.session.map().tiles[0], tile); + assert_eq!(app.session.map().camera, camera); + assert_eq!(app.session.undo_depth(), depth + 1); + + archive(&path, "Must not land"); + app.run("map.reloadTiles", json!({"index":0})).unwrap(); + app.run("map.setTiles", json!({"index":0,"opacity":0.4})).unwrap(); + landed(&mut app, 60); + assert_eq!(app.session.map().tiles[0].opacity, 0.4); + assert!(Arc::ptr_eq(&pmtiles::info_blocking(url, Duration::from_secs(10)).unwrap(), ¤t)); + assert!(app.session.notices.iter().any(|n| n.text.contains("tile archive changed"))); + + app.run("map.reloadTiles", json!({"index":0})).unwrap(); + app.reads.cancel_for_tests(); + landed(&mut app, 60); + assert!(Arc::ptr_eq(&pmtiles::info_blocking(url, Duration::from_secs(10)).unwrap(), ¤t)); + app.run("map.reloadTiles", json!({"index":0})).unwrap(); + app.run("map.new", json!({"name":"Different map"})).unwrap(); + landed(&mut app, 60); + assert!(app.session.map().tiles.is_empty()); + assert!(Arc::ptr_eq(&pmtiles::info_blocking(url, Duration::from_secs(10)).unwrap(), ¤t)); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn archive_repair_does_not_report_success_before_validation_lands() { + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + let dir = std::env::temp_dir().join(format!("rustgis-read-archive-repair-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let original = dir.join("original.pmtiles"); + let replacement = dir.join("replacement.pmtiles"); + archive(&original, "Original"); + archive(&replacement, "Replacement"); + app.run("map.addTiles", json!({"url":original.to_str().unwrap(),"name":"Linked","overlay":true})).unwrap(); + app.reads_off_frame = true; + let mark = app.session.notice_mark(); + assert_eq!( + app.run("map.repairTiles", json!({"index":0,"path":replacement.to_str().unwrap()})).unwrap()["started"], + "Repair Data Source" + ); + assert_eq!(app.session.map().tiles[0].url, original.to_str().unwrap()); + assert!(!app.session.notices_since(mark).iter().any(|n| n.text.contains("Repaired"))); + landed(&mut app, 60); + assert_eq!(app.session.map().tiles[0].url, replacement.to_str().unwrap()); + assert!(app.session.notices_since(mark).iter().any(|n| n.text == "Repaired Linked")); + let _ = std::fs::remove_dir_all(dir); + } } diff --git a/crates/ui-egui/src/repairsource.rs b/crates/ui-egui/src/repairsource.rs index 17233bf0d0..9e65863fd7 100644 --- a/crates/ui-egui/src/repairsource.rs +++ b/crates/ui-egui/src/repairsource.rs @@ -258,8 +258,10 @@ pub fn show(app: &mut RustGisApp, ctx: &egui::Context) { Target::Tiles(index) => app.run("map.repairTiles", json!({"index": index, "path": f.path.trim()})), }; match result { - Ok(_) => { - app.session.notify(crate::i18n::fmt("Repaired {name}", &[("name", &f.name)]), false); + Ok(result) => { + if result.get("started").is_none() && result.get("waiting").is_none() { + app.session.notify(crate::i18n::fmt("Repaired {name}", &[("name", &f.name)]), false); + } close = true; } Err(error) => f.error = Some(error), diff --git a/docs/user-guide/maps.md b/docs/user-guide/maps.md index 457aa0d964..2621340f4c 100644 --- a/docs/user-guide/maps.md +++ b/docs/user-guide/maps.md @@ -222,6 +222,12 @@ Ocean, Atlas, Blank), which need no network, and web tile layers over them: A style's root `center`, `zoom`, `bearing` and `pitch` say where it opens: `map.addTiles` with `"view": true` takes the map there once the style has loaded. +Present PMTiles metadata must be complete, decodable and a JSON object; malformed metadata or a +stored metadata section over **16 MiB** refuses ordinary loading, repair and reload rather than +silently losing credits or source-layer information. This is a metadata limit, not a limit on the +archive's total size; absent metadata remains supported. MBTiles metadata names and values must +be text: unreadable or NULL metadata cells also produce an error rather than being silently skipped. + HTTPS tiles use [platform certificate trust](data.md#https-certificates) on desktop and the command line, including private CAs trusted by the operating system. The web app uses the browser. @@ -268,12 +274,24 @@ The layer keeps its name, visibility, opacity, blend, drawing order, source-laye style and other presentation settings; the map's view stays where it is. Undo cannot restore file contents that another program has overwritten. +On the desktop, reload and repair validate on a background worker; the status bar shows the work +and **Cancel**. Only one read runs at a time. If its project, map, archive or presentation changes +before it lands, the result is left unapplied: retry on the intended target. + Reload first checks that the file can be read. If it is missing, unreadable or invalid, a notice explains the failure and the existing layer and retained cached data are left unchanged. Finish writing or restore the file, then choose **Reload Data** again. If the file has moved, choose **Repair Data Source…** instead and select its new location. Repair also checks the replacement before changing the source; cancelling or choosing an unreadable or invalid archive leaves the existing source and layer unchanged, so you can retry with a valid file. + +Validation reads one size-bounded raster sample from a fresh MBTiles connection and decodes it +before dropping retained caches. PMTiles validation checks all reachable index directories, not +every tile payload. It allows at most **64 MiB stored and 64 MiB decoded index bytes**, **1,048,576 +index entries**, **4,096 unique directories** (including the root), and **four leaf levels**; +each stored directory read is limited to **16 MiB**. An invalid index or an exceeded validation +budget is an explicit error without eviction, even if an unusually large index is otherwise valid. + Neither action is available for local archives in the web app, or for remote tile services. On Windows, an open MBTiles file may not be replaceable while RustGIS holds it open: reload does not bypass that operating-system restriction. Close RustGIS before replacing the file, From 1dc97c105ceb8807357e9d2800602a79a2306089 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Sun, 8 Nov 2026 12:56:43 +0100 Subject: [PATCH 04/15] fix(tiles): reject overlapping PMTiles runs and honor cancellation during index validation --- crates/tiles/src/pmtiles.rs | 85 +++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 3 deletions(-) diff --git a/crates/tiles/src/pmtiles.rs b/crates/tiles/src/pmtiles.rs index 79b429dd81..35bd7fa3f6 100644 --- a/crates/tiles/src/pmtiles.rs +++ b/crates/tiles/src/pmtiles.rs @@ -813,7 +813,8 @@ fn metadata(info: &mut Info, bytes: Vec, expected: u64) -> Result<(), String } /// Reads the current local file without consulting or changing retained caches. All reachable -/// directories are validated within fixed aggregate budgets; tile payloads remain lazy. +/// directories and nonoverlapping tile runs are validated within fixed aggregate budgets; +/// cancellation is checked between entries, while tile payloads remain lazy. /// Ranges are bounded and freshly opened; failure leaves the displayed archive intact. #[cfg(not(target_arch = "wasm32"))] pub fn read_info_local(archive: &str) -> Result, String> { @@ -885,10 +886,14 @@ fn validate_directories( seen: &mut HashMap<(u64, u64), Option>, ) -> Result { let mut height = 0; - for e in dir { + for (i, e) in dir.iter().enumerate() { + rustgis_data::progress::check().map_err(|e| e.to_string())?; let end = e.offset.checked_add(e.length).ok_or("a PMTiles entry range overflows")?; if e.run_length > 0 { - e.tile_id.checked_add(e.run_length).ok_or("tile ids overflow")?; + let tile_end = e.tile_id.checked_add(e.run_length).ok_or("tile ids overflow")?; + if dir.get(i + 1).is_some_and(|next| tile_end > next.tile_id) { + return Err("overlapping PMTiles tile runs".into()); + } if end > info.data_length || info.data_offset.checked_add(end).is_none() { return Err("a tile outside the archive".into()); } @@ -1393,6 +1398,80 @@ pub(crate) mod tests { std::fs::remove_file(path).unwrap(); } + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn fresh_local_validation_checks_run_boundaries_in_roots_and_reachable_leaves() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-run-boundaries-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + for nested in [false, true] { + for next_is_leaf in [false, true] { + // [0, 2) overlaps id 1, is adjacent to id 2, and leaves a gap before id 4. + for next_id in [1, 2, 4] { + let tile = Entry { tile_id: next_id, offset: 0, length: 3, run_length: 1 }; + let mut leaves = if next_is_leaf { write_dir(&[tile]) } else { Vec::new() }; + let next = if next_is_leaf { Entry { length: leaves.len() as u64, run_length: 0, ..tile } } else { tile }; + let entries = [Entry { tile_id: 0, offset: 0, length: 3, run_length: 2 }, next]; + let bytes = if nested { + let directory = write_dir(&entries); + let root = Entry { tile_id: 0, offset: leaves.len() as u64, length: directory.len() as u64, run_length: 0 }; + leaves.extend(directory); + indexed_archive(&[root], leaves) + } else { + indexed_archive(&entries, leaves) + }; + std::fs::write(&path, bytes).unwrap(); + let result = read_info_local(source); + if next_id == 1 { + assert_eq!(result.unwrap_err(), "overlapping PMTiles tile runs", "nested={nested}, next_is_leaf={next_is_leaf}"); + } else { + assert!(result.is_ok(), "nested={nested}, next_is_leaf={next_is_leaf}, next_id={next_id}: {result:?}"); + } + } + } + } + // The final run has no following entry, but its exclusive end must still not overflow. + let entries = [Entry { tile_id: u64::MAX, offset: 0, length: 3, run_length: 1 }]; + std::fs::write(&path, indexed_archive(&entries, Vec::new())).unwrap(); + assert_eq!(read_info_local(source).unwrap_err(), "tile ids overflow"); + std::fs::remove_file(path).unwrap(); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn fresh_local_validation_cancels_before_reading_a_later_malformed_leaf() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct CancelBeforeThirdEntry(AtomicUsize); + impl rustgis_data::progress::Probe for CancelBeforeThirdEntry { + fn cancelled(&self) -> bool { + self.0.fetch_add(1, Ordering::Relaxed) >= 2 + } + fn begin(&self, _: u64) {} + fn advance(&self, _: u64) {} + } + + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-cancel-traversal-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + let first = write_dir(&[Entry { tile_id: 0, offset: 0, length: 3, run_length: 1 }]); + let root = [ + Entry { tile_id: 0, offset: 0, length: first.len() as u64, run_length: 0 }, + Entry { tile_id: 1, offset: first.len() as u64, length: 1, run_length: 0 }, + ]; + let mut leaves = first; + leaves.push(0x80); + std::fs::write(&path, indexed_archive(&root, leaves)).unwrap(); + assert_eq!(read_info_local(source).unwrap_err(), "a truncated directory"); + let probe = Arc::new(CancelBeforeThirdEntry(AtomicUsize::new(0))); + { + let _installed = rustgis_data::progress::install(Some(probe.clone())); + assert_eq!(read_info_local(source).unwrap_err(), rustgis_data::DataError::Cancelled.to_string()); + } + // Root pointer, its reachable tile, then the next root pointer: stop before its I/O. + assert_eq!(probe.0.load(Ordering::Relaxed), 3); + assert_eq!(read_info_local(source).unwrap_err(), "a truncated directory"); + std::fs::remove_file(path).unwrap(); + } + #[cfg(not(target_arch = "wasm32"))] #[test] fn fresh_local_validation_rejects_cycles_and_excess_depth_even_when_shared() { From 696df2956eb0f71b3a5bb10de1cc8d834635a6d5 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Sun, 8 Nov 2026 12:56:43 +0100 Subject: [PATCH 05/15] fix(engine): bind archive approvals to the selected source and reject archives changed after validation --- crates/engine/src/cmd/repair.rs | 154 +++++++- crates/engine/src/tests/review_boundaries.rs | 102 ++++++ .../tests/portable_projects/reload_tiles.rs | 329 +++++++++++++++++- docs/user-guide/maps.md | 13 +- 4 files changed, 576 insertions(+), 22 deletions(-) diff --git a/crates/engine/src/cmd/repair.rs b/crates/engine/src/cmd/repair.rs index 1b1e943c50..d0ca9e6007 100644 --- a/crates/engine/src/cmd/repair.rs +++ b/crates/engine/src/cmd/repair.rs @@ -9,13 +9,19 @@ pub fn commands() -> Vec { vec![ C::new("layer.repairSource", "Repair Data Source", "Data › Data Source", repair_source).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).params( r#"{"layer"?: name|id (default: the active layer), "path": "the file or folder (a Zarr store, a File Geodatabase) the layer's data moved to", "table"?: "the dataset in a GeoPackage, GDAL or DuckDB file, or a raster's subdataset (default: the one the layer reads now)"}"#), - C::new("map.repairTiles", "Repair Tile Archive", "Layers", repair_tiles).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).params( + C::new("map.repairTiles", "Repair Tile Archive", "Layers", repair_tiles).inputs(archive_inputs).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).params( r#"{"name"?: text | "index"?: number (0 = top) | "id"?: catalog id, "path": "the .pmtiles or .mbtiles file the archive moved to"}"#), - C::new("map.reloadTiles", "Reload Data", "Layers", reload_tiles).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).params( + C::new("map.reloadTiles", "Reload Data", "Layers", reload_tiles).inputs(archive_inputs).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).params( r#"{"name": text | "index": number (0 = top) | "id": catalog id} (give one selector) — rereads a local PMTiles or MBTiles archive at its linked path, keeping its presentation; undo cannot restore overwritten file contents"#), ] } +/// Freeze the executor's selected archive without reading it or resolving a replacement path. +fn archive_inputs(input: &super::InputContext<'_>, p: &Value) -> Result { + let i = super::tiles::find(input, p)?.ok_or("name, index or id is required")?; + Ok(super::tiles::tile_inputs(input.map(), [i])) +} + #[cfg(not(target_arch = "wasm32"))] fn replacement(p: &Value) -> Result { let path = super::layer::s_req(p, "path")?.trim(); @@ -133,7 +139,7 @@ fn repair_tiles(s: &mut Session, p: &Value) -> CmdResult { if t.archive_file().is_none() { return Err(format!("{}: only a tile archive read from a file (PMTiles or MBTiles) can be repaired", t.name)); } - let full = replacement(p)?; + let full = replacement(p).map_err(|e| format!("{e}; correct the file and retry Repair Data Source"))?; let new = archive_candidate_after_read(t, &full, s.map().instance_id(), prepared)?; rustgis_render::rustgis_tiles::cache::reload_archive(&t.url); // The replacement path may already have been drawn by another layer. @@ -168,12 +174,108 @@ fn archive_candidate(t: &rustgis_map::TileLayer, path: &str) -> Result Result, String> { + let metadata = match std::fs::metadata(path) { + Ok(metadata) => metadata, + Err(e) if optional && e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => { + return Err(format!("{path}: could not observe archive metadata ({e}); finish writing or restore the file and retry")); + } + }; + // SQLite creates an empty -wal even for a read-only open of a clean WAL-mode archive, + // and cannot remove it again. An empty sidecar holds no commits: same as no sidecar. + if optional && metadata.len() == 0 { + return Ok(None); + } + if !metadata.is_file() { + return Err(format!("{path}: expected an archive file; restore the file and retry")); + } + let modified = metadata + .modified() + .map_err(|e| format!("{path}: could not observe the archive modification time ({e}); check file access and retry"))?; + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; + #[cfg(windows)] + use std::os::windows::fs::MetadataExt; + Ok(Some(Self { + len: metadata.len(), + modified, + #[cfg(unix)] + identity_change: (metadata.dev(), metadata.ino(), metadata.ctime(), metadata.ctime_nsec()), + #[cfg(windows)] + created: metadata.creation_time(), + })) + } +} + +#[cfg(not(target_arch = "wasm32"))] +#[derive(PartialEq, Eq)] +struct ArchiveObservation { + primary: Option, + // SQLite commits can change these without changing the main database. The -shm file + // is reader coordination, not content, and is deliberately not observed. + sqlite: Option<(Option, Option)>, +} + +#[cfg(not(target_arch = "wasm32"))] +impl ArchiveObservation { + fn read(path: &str) -> Result { + let path = path.strip_prefix("file://").unwrap_or(path); + let primary = FileRevision::read(path, false)?; + let sqlite = std::path::Path::new(path).extension().is_some_and(|e| e.eq_ignore_ascii_case("mbtiles")); + let sqlite = if sqlite { + Some((FileRevision::read(&format!("{path}-wal"), true)?, FileRevision::read(&format!("{path}-journal"), true)?)) + } else { + None + }; + Ok(Self { primary, sqlite }) + } + + /// Metadata only: safe for final landing, with no archive scan or decompression. + fn unchanged(&self, path: &str) -> Result<(), String> { + if *self != Self::read(path)? { + return Err(format!("{path}: the archive changed while it was read; finish writing and retry the command")); + } + Ok(()) + } +} + +#[cfg(not(target_arch = "wasm32"))] +struct ValidatedArchive { + layer: rustgis_map::TileLayer, + observation: ArchiveObservation, +} + +#[cfg(not(target_arch = "wasm32"))] +fn observed_candidate(t: &rustgis_map::TileLayer, path: &str) -> Result { + // Keep the archive reader's path boundary ahead of the new metadata I/O too. + rustgis_map::tiles::check_path(path, if t.kind == rustgis_map::TileKind::MbTiles { "mbtiles" } else { "pmtiles" })?; + let observation = ArchiveObservation::read(path)?; + let layer = archive_candidate(t, path)?; + observation.unchanged(path)?; + Ok(ValidatedArchive { layer, observation }) +} + #[cfg(not(target_arch = "wasm32"))] struct ReadArchive { map: u64, original: rustgis_map::TileLayer, path: String, - result: Result, + result: Result, } /// Fresh validation for the desktop's single read worker; nothing is installed or invalidated. @@ -189,7 +291,7 @@ pub(crate) fn archive_ahead(s: &Session, command: &str, p: &Value) -> super::rea let name = if command == "map.repairTiles" { "Repair Data Source" } else { "Reload Data" }; super::reads::ahead(name, command.to_string(), move || { request.map(|(map, original, path)| { - let result = archive_candidate(&original, &path); + let result = observed_candidate(&original, &path); ReadArchive { map, original, path, result } }) }) @@ -202,12 +304,17 @@ fn archive_candidate_after_read( map: u64, prepared: Option>, ) -> Result { - let Some(prepared) = prepared else { return archive_candidate(t, path) }; - let prepared = prepared?; - if prepared.map != map || prepared.original != *t || prepared.path != path { - return Err("the archive or its settings changed while it was read; retry the command".into()); - } - prepared.result + let candidate = if let Some(prepared) = prepared { + let prepared = prepared?; + if prepared.map != map || prepared.original != *t || prepared.path != path { + return Err("the archive or its settings changed while it was read; retry the command".into()); + } + prepared.result? + } else { + observed_candidate(t, path)? + }; + candidate.observation.unchanged(path)?; + Ok(candidate.layer) } fn reload_tiles(s: &mut Session, p: &Value) -> CmdResult { @@ -248,3 +355,28 @@ fn reload_tiles(s: &mut Session, p: &Value) -> CmdResult { Ok(json!({"index": i, "path": path})) } } + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use super::*; + + #[test] + fn empty_sqlite_sidecars_match_absent_ones_but_commits_do_not() { + let dir = std::env::temp_dir().join(format!("rustgis-repair-observation-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("clean.mbtiles"); + std::fs::write(&path, b"main database").unwrap(); + let path = path.display().to_string(); + let observed = ArchiveObservation::read(&path).unwrap(); + for sidecar in ["-wal", "-journal"] { + let sidecar = format!("{path}{sidecar}"); + std::fs::write(&sidecar, b"").unwrap(); + observed.unchanged(&path).expect("an empty sidecar created by a reader is not a commit"); + std::fs::write(&sidecar, b"committed frame").unwrap(); + assert!(observed.unchanged(&path).is_err(), "{sidecar}"); + std::fs::remove_file(&sidecar).unwrap(); + observed.unchanged(&path).unwrap(); + } + std::fs::remove_dir_all(dir).unwrap(); + } +} diff --git a/crates/engine/src/tests/review_boundaries.rs b/crates/engine/src/tests/review_boundaries.rs index 60c8e9bb37..0889535336 100644 --- a/crates/engine/src/tests/review_boundaries.rs +++ b/crates/engine/src/tests/review_boundaries.rs @@ -370,3 +370,105 @@ fn permission_review_script_line_limit_refuses_before_any_edit() { s.run("script.run", &json!({"script":prefix})).unwrap(); assert!(s.map().layer_by_name("Renamed").is_some(), "the exact boundary remains supported"); } + +#[test] +fn permission_review_archive_actions_bind_the_selected_route_and_stack_position() { + for command in ["map.reloadTiles", "map.repairTiles"] { + for selector in [json!({"index":0}), json!({"name":"Review archive"}), json!({"id":"review-archive"})] { + for reorder in [false, true] { + let mut s = world(); + let mut tile = + rustgis_map::TileLayer::new("Review archive", rustgis_map::TileKind::PmTiles, "/not-opened/review-one.pmtiles"); + tile.id = "review-archive".into(); + s.map_mut().tiles = + vec![tile, rustgis_map::TileLayer::new("Other archive", rustgis_map::TileKind::PmTiles, "/not-opened/other.pmtiles")]; + let mut params = selector.clone(); + if command == "map.repairTiles" { + params["path"] = json!("/not-opened/replacement.pmtiles"); + } + let call = Call::command(command, params); + let scope = Scope::assistant(1464, Arc::new(Live)); + let Outcome::NeedsApproval(approved) = s.review(&scope, &call) else { panic!("archive action needs review") }; + let remembered = approved.remember().unwrap(); + if reorder { + s.map_mut().tiles.swap(0, 1); + } else { + s.map_mut().tiles[0].url = "/not-opened/review-two.pmtiles".into(); + } + let Outcome::NeedsApproval(changed) = s.review(&scope, &call) else { panic!("changed action needs review") }; + assert!(!changed.covered_by(&remembered), "{command}: {selector}, reorder={reorder}"); + let before = s.map().tiles.clone(); + let undo = s.undo_depth(); + assert!(s.run_remembered(scope, &remembered, &call).is_err(), "stale consent must fail before opening any archive"); + assert_eq!(s.map().tiles, before); + assert_eq!(s.undo_depth(), undo); + } + } + } +} + +#[test] +fn permission_review_archive_places_are_redacted_but_raw_routes_remain_bound() { + for command in ["map.reloadTiles", "map.repairTiles"] { + let mut s = world(); + s.map_mut().tiles = vec![ + rustgis_map::TileLayer::new( + "Review archive", + rustgis_map::TileKind::PmTiles, + "https://archive-review.invalid/private.pmtiles?token=RGSYNTH_archive_one", + ), + rustgis_map::TileLayer::new("Other archive", rustgis_map::TileKind::PmTiles, "/not-opened/other.pmtiles"), + ]; + // Preserve index > name > id selector precedence, even for a conflicting request. + let mut params = json!({"index":0,"name":"Other archive","id":"not-the-target"}); + if command == "map.repairTiles" { + params["path"] = json!("/not-opened/replacement.pmtiles"); + } + let call = Call::command(command, params); + let scope = Scope::assistant(1465, Arc::new(Live)); + let Outcome::NeedsApproval(approved) = s.review(&scope, &call) else { panic!("archive action needs review") }; + assert!(approved.review().details.iter().flat_map(|d| &d.places).any(|p| p.contains("archive-review.invalid"))); + assert!(!format!("{:?}", approved.review()).contains("RGSYNTH_archive_")); + let remembered = approved.remember().unwrap(); + s.map_mut().tiles[1].url = "/not-opened/unselected-change.pmtiles".into(); + let Outcome::NeedsApproval(unselected) = s.review(&scope, &call) else { panic!("still needs review") }; + assert!(unselected.covered_by(&remembered), "only the executor's selected archive is bound"); + s.map_mut().tiles[0].url = "https://archive-review.invalid/private.pmtiles?token=RGSYNTH_archive_two".into(); + let Outcome::NeedsApproval(changed) = s.review(&scope, &call) else { panic!("still needs review") }; + assert!(!changed.covered_by(&remembered), "redaction must not erase private route identity"); + } +} + +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn permission_review_unchanged_real_archive_reload_and_repair_execute_with_remembered_consent() { + let dir = std::env::temp_dir().join(format!("rustgis-review-archive-actions-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let linked = dir.join("linked.pmtiles"); + let replacement = dir.join("replacement.pmtiles"); + for path in [&linked, &replacement] { + std::fs::write(path, empty_pmtiles([-24.5, 63.3, -13.5, 66.6])).unwrap(); + } + for command in ["map.reloadTiles", "map.repairTiles"] { + let mut s = world(); + s.run("map.addTiles", &json!({"url":linked,"name":"Approved archive","overlay":true})).unwrap(); + let mut params = json!({"name":"Approved archive"}); + if command == "map.repairTiles" { + params["path"] = json!(replacement); + } + let call = Call::command(command, params); + let scope = Scope::assistant(1466, Arc::new(Live)); + let Outcome::NeedsApproval(approved) = s.review(&scope, &call) else { panic!("archive action needs review") }; + assert!(approved.review().details.iter().flat_map(|d| &d.places).any(|p| p.contains("linked.pmtiles"))); + if command == "map.repairTiles" { + assert!(approved.review().details.iter().flat_map(|d| &d.places).any(|p| p.contains("replacement.pmtiles"))); + } + let remembered = approved.remember().unwrap(); + let Outcome::NeedsApproval(unchanged) = s.review(&scope, &call) else { panic!("still needs review") }; + assert!(unchanged.covered_by(&remembered)); + let result = s.run_remembered(scope, &remembered, &call).unwrap(); + assert_eq!(result["path"], json!(if command == "map.repairTiles" { &replacement } else { &linked })); + assert_eq!(s.map().tiles[0].name, "Approved archive"); + } + std::fs::remove_dir_all(dir).unwrap(); +} diff --git a/crates/engine/tests/portable_projects/reload_tiles.rs b/crates/engine/tests/portable_projects/reload_tiles.rs index b845181c73..2f9a0d32f6 100644 --- a/crates/engine/tests/portable_projects/reload_tiles.rs +++ b/crates/engine/tests/portable_projects/reload_tiles.rs @@ -10,6 +10,38 @@ use rustgis_engine::{ use serde_json::{Value, json}; use std::{path::Path, sync::Arc, time::Duration}; +/// Archive reloads share process-wide caches, and an MBTiles forget clears every resolved style. +/// Serializing these tests and dropping each test's archives keeps Arc-identity assertions about +/// retention rather than about another test's cache work or capacity. +struct Scoped { + fixture: Fixture, + _turn: std::sync::MutexGuard<'static, ()>, +} + +impl Scoped { + fn new(label: &str) -> Self { + static TURN: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let turn = TURN.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + Self { fixture: Fixture::new(label), _turn: turn } + } +} + +impl std::ops::Deref for Scoped { + type Target = Fixture; + fn deref(&self) -> &Fixture { + &self.fixture + } +} + +impl Drop for Scoped { + fn drop(&mut self) { + for file in std::fs::read_dir(&self.fixture.root).into_iter().flatten().flatten() { + rustgis_engine::data::mbtiles::close(&file.path()); + cache::reload_archive(&file.path().display().to_string()); + } + } +} + fn picture(color: [u8; 4]) -> Vec { let mut p = tiny_skia::Pixmap::new(256, 256).unwrap(); p.fill(tiny_skia::Color::from_rgba8(color[0], color[1], color[2], color[3])); @@ -91,7 +123,7 @@ fn assert_render(s: &mut Session, expected: [u8; 4], credit: &str) { #[test] fn manual_archive_reload_reads_new_pixels_and_credits_without_touching_other_sources() { for ext in ["pmtiles", "mbtiles"] { - let f = Fixture::new(&format!("reload-{ext}")); + let f = Scoped::new(&format!("reload-{ext}")); let path = f.file(&format!("linked.{ext}")); let other = f.file(&format!("unrelated.{ext}")); write_archive(&path, [255, 0, 0, 255], "Old credits"); @@ -154,7 +186,7 @@ fn rejected(s: &mut Session, params: Value, expected: &str) { #[test] fn failed_archive_reload_retains_tiles_configuration_and_allows_retry() { for ext in ["pmtiles", "mbtiles"] { - let f = Fixture::new(&format!("reload-failure-{ext}")); + let f = Scoped::new(&format!("reload-failure-{ext}")); let path = f.file(&format!("linked.{ext}")); write_archive(&path, [255, 0, 0, 255], "Retained credits"); let mut s = session(); @@ -185,7 +217,7 @@ fn write_archive_after_truncation(path: &Path, color: [u8; 4], credit: &str) { #[test] fn locked_mbtiles_reload_fails_without_evicting_retained_data_then_retries() { - let f = Fixture::new("reload-locked"); + let f = Scoped::new("reload-locked"); let path = f.file("locked.mbtiles"); write_archive(&path, [255, 0, 0, 255], "Before lock"); let mut s = session(); @@ -207,7 +239,7 @@ fn locked_mbtiles_reload_fails_without_evicting_retained_data_then_retries() { #[cfg(unix)] #[test] fn replacing_an_open_mbtiles_archive_reloads_the_new_file_not_its_pooled_inode() { - let f = Fixture::new("reload-mbtiles-replaced-inode"); + let f = Scoped::new("reload-mbtiles-replaced-inode"); let path = f.file("linked.mbtiles"); let replacement = f.file("replacement.mbtiles"); write_archive(&path, [255, 0, 0, 255], "Old inode"); @@ -228,7 +260,7 @@ fn replacing_an_open_mbtiles_archive_reloads_the_new_file_not_its_pooled_inode() #[test] fn incompatible_reload_keeps_archive_presentation_and_undo_restores_prior_error() { for ext in ["pmtiles", "mbtiles"] { - let f = Fixture::new(&format!("reload-incompatible-{ext}")); + let f = Scoped::new(&format!("reload-incompatible-{ext}")); let path = f.file(&format!("vector.{ext}")); write_vector(&path); let mut s = session(); @@ -291,7 +323,7 @@ fn archive_reload_rejects_remote_upload_and_invalid_selectors() { #[test] fn invalid_raster_and_unreadable_tile_pages_do_not_evict_retained_mbtiles() { for damaged_page in [false, true] { - let f = Fixture::new(if damaged_page { "reload-tile-page" } else { "reload-png-signature" }); + let f = Scoped::new(if damaged_page { "reload-tile-page" } else { "reload-png-signature" }); let path = f.file("linked.mbtiles"); write_archive(&path, [255, 0, 0, 255], "Retained credits"); let mut s = session(); @@ -357,7 +389,7 @@ fn write_leaf_archive(path: &Path, corrupt: bool) { #[test] fn malformed_leaf_reload_retains_pixels_and_can_retry_with_a_valid_leaf() { - let f = Fixture::new("reload-malformed-leaf"); + let f = Scoped::new("reload-malformed-leaf"); let path = f.file("linked.pmtiles"); write_archive(&path, [255, 0, 0, 255], "Retained credits"); let mut s = session(); @@ -378,7 +410,7 @@ fn malformed_leaf_reload_retains_pixels_and_can_retry_with_a_valid_leaf() { #[test] fn untrusted_workflows_cannot_reload_but_user_authored_workflows_can() { - let f = Fixture::new("reload-workflow-trust"); + let f = Scoped::new("reload-workflow-trust"); let path = f.file("linked.pmtiles"); write_archive(&path, [255, 0, 0, 255], "Before"); let mut s = session(); @@ -408,7 +440,7 @@ fn untrusted_workflows_cannot_reload_but_user_authored_workflows_can() { #[test] fn successful_reload_cancels_only_its_own_pending_archive_framing() { - let f = Fixture::new("reload-pending-view"); + let f = Scoped::new("reload-pending-view"); let path = f.file("linked.pmtiles"); let other = f.file("other.pmtiles"); for file in [&path, &other] { @@ -447,7 +479,7 @@ fn successful_reload_cancels_only_its_own_pending_archive_framing() { #[test] fn prepared_archive_validation_does_not_evict_or_overwrite_changed_settings() { - let f = Fixture::new("reload-read-ahead"); + let f = Scoped::new("reload-read-ahead"); let path = f.file("linked.pmtiles"); write_archive(&path, [255, 0, 0, 255], "Before"); let mut s = session(); @@ -470,3 +502,280 @@ fn prepared_archive_validation_does_not_evict_or_overwrite_changed_settings() { assert_eq!(s.map().tiles[0].opacity, 0.4); assert_color(&decoded(&tile), [0, 255, 0, 255]); } + +fn archive_read(s: &mut Session, command: &str, params: &Value) -> rustgis_engine::cmd::reads::Read { + let control = rustgis_engine::analysis::control::Control::stoppable(); + rustgis_engine::cmd::reads::read_ahead(s, command, params).unwrap().run(&control).unwrap() +} + +type RetainedArchive = (TileLayer, Arc, Arc); + +fn retain_archive(t: &TileLayer) -> RetainedArchive { + (t.clone(), decoded(t), style(t)) +} + +fn reject_stale_archive(s: &mut Session, read: rustgis_engine::cmd::reads::Read, retained: &[RetainedArchive]) -> String { + let before = serde_json::to_value(&s.project).unwrap(); + let view = s.style_view.clone(); + let undo = s.undo_depth(); + let mark = s.notice_mark(); + let error = rustgis_engine::cmd::reads::read_land(s, read).unwrap_err(); + assert_eq!(serde_json::to_value(&s.project).unwrap(), before, "failed landing must retain source, presentation and view"); + assert_eq!(s.style_view, view, "failed landing must not cancel pending framing"); + assert_eq!(s.undo_depth(), undo); + assert!(s.notices_since(mark).iter().all(|n| n.error), "failed landing must not emit a success notice"); + for (tile, pixels, credits) in retained { + assert!(Arc::ptr_eq(&decoded(tile), pixels), "failed landing must not evict {}", tile.url); + assert!(Arc::ptr_eq(&style(tile), credits), "failed landing must not evict metadata for {}", tile.url); + } + error +} + +#[test] +fn changed_or_removed_archive_after_worker_validation_retains_data_and_retries() { + for ext in ["pmtiles", "mbtiles"] { + for command in ["map.reloadTiles", "map.repairTiles"] { + for remove in [false, true] { + let f = Scoped::new(&format!("archive-freshness-{ext}-{command}-{remove}")); + let linked = f.file(&format!("linked.{ext}")); + let replacement = f.file(&format!("replacement.{ext}")); + write_archive(&linked, [255, 0, 0, 255], "Retained credits"); + write_archive(&replacement, [0, 0, 255, 255], "Replacement credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":linked,"overlay":true})).unwrap(); + s.map_mut().tiles[0].opacity = 0.4; + s.map_mut().tiles[0].expanded = true; + let tile = s.map().tiles[0].clone(); + let mut candidate = tile.clone(); + let mut params = json!({"index":0}); + let path = if command == "map.repairTiles" { + params["path"] = json!(replacement); + candidate.url = replacement.display().to_string(); + &replacement + } else { + &linked + }; + let retained = vec![retain_archive(&tile), retain_archive(&candidate)]; + s.style_view = Some((tile.clone(), s.map().camera, s.map().instance_id())); + let read = archive_read(&mut s, command, ¶ms); + rustgis_engine::data::mbtiles::close(path); + if remove { + std::fs::remove_file(path).unwrap(); + } else { + let len = std::fs::metadata(path).unwrap().len(); + std::fs::OpenOptions::new().write(true).open(path).unwrap().set_len(len + 1).unwrap(); + } + let error = reject_stale_archive(&mut s, read, &retained); + assert!(error.contains("retry"), "{error}"); + let expected = if !remove { + "changed while it was read" + } else if command == "map.repairTiles" { + "there's no such file" + } else { + "observe archive metadata" + }; + assert!(error.contains(expected), "{error}"); + if !remove { + std::fs::remove_file(path).unwrap(); + } + write_archive(path, [0, 255, 0, 255], "Retry credits"); + s.run(command, ¶ms).unwrap(); + assert_eq!(s.map().tiles[0].opacity, 0.4); + assert!(s.map().tiles[0].expanded); + assert_color(&decoded(&s.map().tiles[0]), [0, 255, 0, 255]); + assert!(style(&s.map().tiles[0]).attribution.iter().any(|a| a.contains("Retry credits"))); + rustgis_engine::data::mbtiles::close(&linked); + rustgis_engine::data::mbtiles::close(&replacement); + } + } + } +} + +#[cfg(unix)] +#[test] +fn replacing_validated_archive_directory_entry_rejects_landing_even_with_identical_contents() { + use std::os::unix::fs::MetadataExt; + for ext in ["pmtiles", "mbtiles"] { + for command in ["map.reloadTiles", "map.repairTiles"] { + let f = Scoped::new(&format!("archive-freshness-inode-{ext}-{command}")); + let linked = f.file(&format!("linked.{ext}")); + let candidate_path = f.file(&format!("candidate.{ext}")); + let swapped = f.file(&format!("swapped.{ext}")); + write_archive(&linked, [255, 0, 0, 255], "Retained credits"); + write_archive(&candidate_path, [0, 0, 255, 255], "Candidate credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":linked,"overlay":true})).unwrap(); + let tile = s.map().tiles[0].clone(); + let retained = vec![retain_archive(&tile)]; + s.style_view = Some((tile.clone(), s.map().camera, s.map().instance_id())); + let mut params = json!({"index":0}); + let path = if command == "map.repairTiles" { + params["path"] = json!(candidate_path); + &candidate_path + } else { + &linked + }; + let read = archive_read(&mut s, command, ¶ms); + let old = std::fs::metadata(path).unwrap(); + std::fs::copy(path, &swapped).unwrap(); + std::fs::File::options() + .write(true) + .open(&swapped) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(old.modified().unwrap())) + .unwrap(); + std::fs::rename(&swapped, path).unwrap(); + let new = std::fs::metadata(path).unwrap(); + assert_eq!(old.len(), new.len()); + assert_eq!(old.modified().unwrap(), new.modified().unwrap()); + assert_ne!(old.ino(), new.ino()); + assert!(reject_stale_archive(&mut s, read, &retained).contains("changed while it was read")); + s.run(command, ¶ms).unwrap(); + assert_eq!(s.map().tiles[0].url, path.display().to_string()); + rustgis_engine::data::mbtiles::close(&linked); + rustgis_engine::data::mbtiles::close(&candidate_path); + } + } +} + +#[test] +fn mbtiles_wal_commit_with_writer_open_invalidates_worker_validation_without_eviction() { + for command in ["map.reloadTiles", "map.repairTiles"] { + let f = Scoped::new(&format!("archive-freshness-wal-{command}")); + let linked = f.file("linked.mbtiles"); + let replacement = f.file("replacement.mbtiles"); + write_archive(&linked, [255, 0, 0, 255], "Retained credits"); + write_archive(&replacement, [0, 0, 255, 255], "Replacement credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":linked,"overlay":true})).unwrap(); + let tile = s.map().tiles[0].clone(); + let mut params = json!({"index":0}); + let path = if command == "map.repairTiles" { + params["path"] = json!(replacement); + &replacement + } else { + &linked + }; + let writer = rusqlite::Connection::open(path).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0; UPDATE metadata SET value='Before worker' WHERE name='attribution';", + ) + .unwrap(); + let mut candidate = tile.clone(); + candidate.url = path.display().to_string(); + let retained = vec![retain_archive(&tile), retain_archive(&candidate)]; + s.style_view = Some((tile.clone(), s.map().camera, s.map().instance_id())); + let read = archive_read(&mut s, command, ¶ms); + let before = std::fs::metadata(path).unwrap(); + writer.execute_batch("BEGIN IMMEDIATE; UPDATE metadata SET value='Committed after worker' WHERE name='attribution';").unwrap(); + writer.execute("UPDATE tiles SET tile_data=?1", [picture([0, 255, 0, 255])]).unwrap(); + writer.execute_batch("COMMIT").unwrap(); + let after = std::fs::metadata(path).unwrap(); + assert_eq!(before.len(), after.len(), "this regression must exercise a sidecar-only commit"); + assert_eq!(before.modified().unwrap(), after.modified().unwrap()); + assert!(reject_stale_archive(&mut s, read, &retained).contains("changed while it was read")); + // Keep the writer open: retry must see committed WAL contents, without requiring a checkpoint. + s.run(command, ¶ms).unwrap(); + assert_color(&decoded(&s.map().tiles[0]), [0, 255, 0, 255]); + assert!(style(&s.map().tiles[0]).attribution.iter().any(|a| a.contains("Committed after worker"))); + drop(writer); + rustgis_engine::data::mbtiles::close(&linked); + rustgis_engine::data::mbtiles::close(&replacement); + } +} + +#[test] +fn clean_wal_mode_mbtiles_first_reload_and_repair_ignore_reader_created_empty_wal() { + for command in ["map.reloadTiles", "map.repairTiles"] { + let f = Scoped::new(&format!("archive-clean-wal-{command}")); + let linked = f.file("linked.mbtiles"); + let replacement = f.file("replacement.mbtiles"); + write_archive(&linked, [255, 0, 0, 255], "Retained credits"); + write_archive(&replacement, [0, 255, 0, 255], "Clean WAL credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":linked,"overlay":true})).unwrap(); + let mut params = json!({"index":0}); + let path = if command == "map.repairTiles" { + params["path"] = json!(replacement); + &replacement + } else { + &linked + }; + // A clean close of the last WAL connection removes -wal/-shm; the archive stays WAL mode. + let writer = rusqlite::Connection::open(path).unwrap(); + writer.query_row("PRAGMA journal_mode=WAL", [], |r| r.get::<_, String>(0)).unwrap(); + drop(writer); + rustgis_engine::data::mbtiles::close(&linked); + rustgis_engine::data::mbtiles::close(&replacement); + for sidecar in ["-wal", "-journal"] { + let _ = std::fs::remove_file(format!("{}{sidecar}", path.display())); + } + assert!(!Path::new(&format!("{}-wal", path.display())).exists()); + // Whether or not the reader creates an empty -wal while validating, the first attempt lands. + let read = archive_read(&mut s, command, ¶ms); + rustgis_engine::cmd::reads::read_land(&mut s, read).unwrap(); + let expected = if command == "map.repairTiles" { [0, 255, 0, 255] } else { [255, 0, 0, 255] }; + assert_color(&decoded(&s.map().tiles[0]), expected); + if command == "map.repairTiles" { + assert_eq!(s.map().tiles[0].url, path.display().to_string()); + } + rustgis_engine::data::mbtiles::close(&linked); + rustgis_engine::data::mbtiles::close(&replacement); + } +} + +#[test] +fn archive_observation_begins_in_worker_not_when_work_is_queued() { + for ext in ["pmtiles", "mbtiles"] { + let f = Scoped::new(&format!("archive-worker-observation-{ext}")); + let path = f.file(&format!("linked.{ext}")); + write_archive(&path, [255, 0, 0, 255], "Before queue"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path,"overlay":true})).unwrap(); + let ahead = rustgis_engine::cmd::reads::read_ahead(&mut s, "map.reloadTiles", &json!({"index":0})).unwrap(); + write_archive(&path, [0, 255, 0, 255], "Written before worker starts"); + let read = ahead.run(&rustgis_engine::analysis::control::Control::stoppable()).unwrap(); + rustgis_engine::cmd::reads::read_land(&mut s, read).unwrap(); + assert_color(&decoded(&s.map().tiles[0]), [0, 255, 0, 255]); + rustgis_engine::data::mbtiles::close(&path); + } +} + +#[test] +fn overlapping_pmtiles_runs_reject_reload_and_repair_without_losing_retained_data() { + for command in ["map.reloadTiles", "map.repairTiles"] { + let f = Scoped::new(&format!("archive-overlapping-runs-{command}")); + let linked = f.file("linked.pmtiles"); + let replacement = f.file("replacement.pmtiles"); + write_archive(&linked, [255, 0, 0, 255], "Retained credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":linked,"overlay":true})).unwrap(); + let tile = s.map().tiles[0].clone(); + let retained = vec![retain_archive(&tile)]; + s.style_view = Some((tile.clone(), s.map().camera, s.map().instance_id())); + let archive = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Png, + gzipped_tiles: false, + bounds: [-180.0, -85.0, 180.0, 85.0], + metadata: json!({"attribution":"Valid adjacent runs"}), + }; + let valid = pmtiles::write(vec![(0, 0, 0, picture([0, 255, 0, 255])), (1, 0, 0, picture([0, 0, 255, 255]))], &archive).unwrap(); + assert_eq!(&valid[127..132], &[2, 0, 1, 1, 1], "fixture must contain two adjacent single-tile runs"); + let mut overlapping = valid.clone(); + overlapping[130] = 2; // First [0,1) run becomes [0,2), overlapping the next entry. + let mut params = json!({"index":0}); + let path = if command == "map.repairTiles" { + params["path"] = json!(replacement); + &replacement + } else { + &linked + }; + std::fs::write(path, overlapping).unwrap(); + let read = archive_read(&mut s, command, ¶ms); + assert!(reject_stale_archive(&mut s, read, &retained).contains("overlapping PMTiles tile runs")); + std::fs::write(path, valid).unwrap(); + s.run(command, ¶ms).unwrap(); + assert_color(&decoded(&s.map().tiles[0]), [0, 255, 0, 255]); + } +} diff --git a/docs/user-guide/maps.md b/docs/user-guide/maps.md index 78d5cf8378..a1bc62c61f 100644 --- a/docs/user-guide/maps.md +++ b/docs/user-guide/maps.md @@ -319,7 +319,7 @@ fetches them again. **Reloading a local tile archive (desktop).** After changing a PMTiles or MBTiles file on disk, right-click its row in Layers and choose **Reload Data**. CLI, Python, MCP and the Assistant use `map.reloadTiles={"index":0}`; `"name"` or catalog `"id"` can select the archive instead. -No replacement path is accepted. This is a manual action: RustGIS does not watch the file or detect changes. +No replacement path is accepted. This is a manual action: RustGIS does not watch the file for changes. It rereads that archive's metadata and tiles without clearing other archives or web layers. The layer keeps its name, visibility, opacity, blend, drawing order, source-layer visibility, style and other presentation settings; the map's view stays where it is. Undo cannot restore @@ -329,6 +329,17 @@ On the desktop, reload and repair validate on a background worker; the status ba and **Cancel**. Only one read runs at a time. If its project, map, archive or presentation changes before it lands, the result is left unapplied: retry on the intended target. +Approval binds the selected archive's owning map, stack position and source route, not just its +name or index. Retargeting or reordering it requires reviewing the operation again. + +Each operation observes file size and modification time before validation, checks them again +after validation and immediately before applying the result, and rejects observable changes. +It also checks file identity and change time where the platform exposes them, plus MBTiles +`-wal` and `-journal` existence and revisions (not the reader-coordination `-shm` file). +Metadata that cannot be read is an error, not permission to apply an unchecked result. +These checks are not an immutable snapshot: a rewrite that leaves the same observations, or a +write after the final check, may escape detection. Finish writing before reloading or repairing. + Reload first checks that the file can be read. If it is missing, unreadable or invalid, a notice explains the failure and the existing layer and retained cached data are left unchanged. Finish writing or restore the file, then choose **Reload Data** again. If the file has moved, From f766c1b282c75f33a77f3810233bc865471f0686 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Sun, 8 Nov 2026 12:56:43 +0100 Subject: [PATCH 06/15] fix(relay): validate direct relay archive reloads on the background read worker --- crates/assistant/src/mcp.rs | 123 +++++++++ crates/assistant/src/relay.rs | 81 +++++- crates/assistant/src/relay/archive_tests.rs | 271 ++++++++++++++++++++ crates/assistant/src/tools.rs | 19 ++ crates/engine/src/cmd/reads.rs | 7 + crates/engine/src/policy/dispatch.rs | 33 +++ crates/ui-egui/src/lib.rs | 8 +- crates/ui-egui/src/readjob.rs | 119 ++++++++- docs/mcp.md | 10 +- 9 files changed, 656 insertions(+), 15 deletions(-) create mode 100644 crates/assistant/src/relay/archive_tests.rs diff --git a/crates/assistant/src/mcp.rs b/crates/assistant/src/mcp.rs index 14473fc241..86dcb65818 100644 --- a/crates/assistant/src/mcp.rs +++ b/crates/assistant/src/mcp.rs @@ -88,6 +88,98 @@ pub(crate) fn request_key(root: &str, position: usize) -> String { format!("{root}/{position}") } +/// A relay message paused at an admitted standalone archive command. Batch positions and +/// one-use permits remain owned here; no suffix entry runs until the actual command lands. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) struct ArchiveBatch { + server: Server, + scope: Scope, + permits: Vec<(String, Permit)>, + request: String, + messages: std::vec::IntoIter, + position: usize, + batch: bool, + replies: Vec, + pending: Option<(Value, tools::CommandCompletion)>, + ahead: Option, +} +#[cfg(not(target_arch = "wasm32"))] +pub(crate) enum ArchiveStep { + Done(Option), + Read(Box), +} +#[cfg(not(target_arch = "wasm32"))] +impl ArchiveBatch { + pub(crate) fn take_read(&mut self) -> Option { + self.ahead.take() + } + /// The command's reply reports changes made from the moment it actually runs, not from its + /// admission: the queue may run user work or other reads first. + pub(crate) fn begin_landing(&mut self, s: &Session) { + if let Some((_, completion)) = &mut self.pending { + *completion = tools::CommandCompletion::capture(s); + } + } + pub(crate) fn finish(mut self, s: &mut Session, result: Result) -> ArchiveStep { + if let Some((id, completion)) = self.pending.take() { + self.replies.push(archive_reply(id, completion.finish(s, result))); + } + self.advance(s) + } + fn advance(mut self, s: &mut Session) -> ArchiveStep { + while let Some(msg) = self.messages.next() { + let request = request_key(&self.request, self.position); + self.position += 1; + let archive = (|| { + let id = msg.get("id")?; + if !matches!(id, Value::String(_) | Value::Number(_)) || msg.get("method").and_then(Value::as_str) != Some("tools/call") { + return None; + } + let (name, args) = tool_request(msg.get("params")?).ok()?; + if name != "run_command" + || !matches!(args.get("command").and_then(Value::as_str), Some("map.reloadTiles" | "map.repairTiles")) + { + return None; + } + Some((id.clone(), ToolCall { id: request.clone(), name: name.into(), args })) + })(); + if let Some((id, call)) = archive { + let permit = self.permits.iter().position(|(key, _)| key == &request).map(|i| self.permits.remove(i).1); + let completion = tools::CommandCompletion::capture(s); + let admitted = tools::policy_call(&call) + .and_then(|call| call.ok_or_else(|| "missing archive adapter".to_string())) + .and_then(|call| s.start_scoped_archive_read(self.scope.clone(), &call, permit)); + match admitted { + Ok(ahead) => { + self.pending = Some((id, completion)); + self.ahead = Some(ahead); + return ArchiveStep::Read(Box::new(self)); + } + Err(error) => self.replies.push(archive_reply(id, completion.finish(s, Err(error)))), + } + } else if let Some(reply) = self.server.handle_value(s, &msg, &self.scope, &mut self.permits, &request) { + self.replies.push(reply); + } + } + let reply = if self.replies.is_empty() { + None + } else if self.batch { + Some(Value::Array(self.replies).to_string()) + } else { + self.replies.pop().map(|reply| reply.to_string()) + }; + ArchiveStep::Done(reply) + } +} +#[cfg(not(target_arch = "wasm32"))] +fn archive_reply(id: Value, result: Result) -> Value { + let (text, failed) = match result { + Ok(text) => (text, false), + Err(error) => (error, true), + }; + json!({"jsonrpc":"2.0","id":id,"result":{"content":[{"type":"text","text":text}],"isError":failed}}) +} + impl Server { /// A server with the built-in skills. `rustgis-cli mcp` and the desktop's relay replace them /// with the skills switched on (`skills::installed`): the user's, the plugins' and shared ones. @@ -100,6 +192,37 @@ impl Server { pub fn handle(&self, s: &mut Session, message: &str) -> Option { self.handle_scoped(s, message, self.scope(s), Vec::new(), "") } + #[cfg(not(target_arch = "wasm32"))] + /// Desktop relay ingress; only direct archive commands pause. Scripts and other transports + /// retain their synchronous contract, and malformed messages keep normal JSON-RPC handling. + pub(crate) fn start_archive_batch( + self, + s: &mut Session, + message: &str, + scope: Scope, + permits: Vec<(String, Permit)>, + request: &str, + ) -> ArchiveStep { + let parsed = serde_json::from_str::(message); + let (messages, batch) = match parsed { + Ok(Value::Array(batch)) if !batch.is_empty() && batch.len() <= MAX_BATCH => (batch, true), + Ok(msg) if !msg.is_array() => (vec![msg], false), + _ => return ArchiveStep::Done(self.handle_scoped(s, message, scope, permits, request)), + }; + ArchiveBatch { + server: self, + scope, + permits, + request: request.into(), + messages: messages.into_iter(), + position: 0, + batch, + replies: Vec::new(), + pending: None, + ahead: None, + } + .advance(s) + } /// Named relay/file-bridge ingress; permits are engine plans stored when approval was shown. pub(crate) fn handle_scoped( &self, diff --git a/crates/assistant/src/relay.rs b/crates/assistant/src/relay.rs index 7ca9e34757..8cfe820b22 100644 --- a/crates/assistant/src/relay.rs +++ b/crates/assistant/src/relay.rs @@ -196,6 +196,8 @@ struct Shared { /// The screen snapshot the app is capturing ([`Relay::screen_request`]); nothing else runs /// until it is done. shot: Mutex>, + /// A desktop archive command holds its complete message, including any ordered batch suffix. + archive: Mutex>, } /// What the relay keeps for one agent between its calls. @@ -228,6 +230,12 @@ struct Shot { wait: Duration, } +struct Archive { + req: Request, + batch: Box, + started: Instant, +} + impl Shared { /// Wakes the UI thread so it draws a frame to run queued messages or repaint the client /// count. Does nothing when the app never set a callback with [`Relay::set_wake`]. @@ -251,6 +259,9 @@ impl Shared { if let Some(shot) = lock(&self.shot).take() { shot.req.state.store(CANCELLED, SeqCst); } + if let Some(archive) = lock(&self.archive).take() { + archive.req.state.store(CANCELLED, SeqCst); + } // Pending requests are cancelled before they are dropped, so none can run after stop. for req in lock(&self.requests).drain(..) { let _ = req.state.compare_exchange(PENDING, CANCELLED, SeqCst, SeqCst); @@ -592,6 +603,7 @@ impl Relay { agents: Mutex::new(HashMap::new()), screen: AtomicBool::new(false), shot: Mutex::new(None), + archive: Mutex::new(None), }); let owner = Arc::new(Owner { shared, port, audit: Mutex::new(None), last: Mutex::new(None) }); let accepting = Arc::clone(&owner.shared); @@ -716,6 +728,46 @@ impl Relay { let _ = req.reply.send(Some(reply)); self.owner.shared.notify(); } + /// Takes the already-admitted archive read once. The desktop uses its single read worker; + /// dropping or recreating the invocation is never a native redispatch. + pub fn take_archive_read(&self) -> Option { + lock(&self.owner.shared.archive).as_mut()?.batch.take_read() + } + + /// Starts the reply's change report at the moment the admitted command actually runs (or is + /// known never to run), so work the queue ran meanwhile is not attributed to the agent. + pub fn begin_archive_landing(&self, s: &Session) { + if let Some(archive) = lock(&self.owner.shared.archive).as_mut() { + archive.batch.begin_landing(s); + } + } + + /// Completes the real archive command before running its batch suffix or issuing its reply + /// and audit. The engine has already checked the original owner at worker start and landing. + pub fn finish_archive_read(&self, s: &mut Session, result: Result) { + let Some(archive) = lock(&self.owner.shared.archive).take() else { return }; + let Archive { req, batch, started, .. } = archive; + let layers = self.enter_agent(s, &req.token); + let step = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| (*batch).finish(s, result))) + .unwrap_or_else(|_| crate::mcp::ArchiveStep::Done(Some(internal_error(&req.line)))); + let out = match step { + crate::mcp::ArchiveStep::Read(batch) => { + self.leave_agent(s, &req.token, layers, false); + *lock(&self.owner.shared.archive) = Some(Archive { req, batch, started }); + self.owner.shared.notify(); + return; + } + crate::mcp::ArchiveStep::Done(out) => out, + }; + let picked = selects_layer(&req.line) && reply_ok(out.as_deref()); + self.leave_agent(s, &req.token, layers, picked); + self.audit(&req.line, out.as_deref(), started.elapsed()); + if let Some(action) = action(&req.line, out.as_deref()) { + *lock(&self.owner.last) = Some(action); + } + let _ = req.reply.send(out); + self.owner.shared.notify(); + } /// The last tool call an attached agent ran, if any has run since the relay started. pub fn last_action(&self) -> Option { @@ -788,13 +840,14 @@ impl Relay { Ok(grant) } - /// Runs the messages that arrived since the last call, on `s`. Returns how many ran. The - /// calls go through [`Session::run`], so each is one undo step as everywhere else. A message - /// whose connection already gave up is skipped, never run. + /// Runs queued messages on `s`, returning how many completed. Direct archive commands pause + /// at admission for the app's existing read queue: take_archive_read and finish_archive_read + /// carry them through their real completion, holding their batch suffix and later messages. + /// Commands retain ordinary undo bookkeeping; abandoned queued requests never run. pub fn poll(&self, s: &mut Session) -> usize { *lock(&self.owner.shared.last_poll) = Instant::now(); // One request at a time: a screen snapshot being captured holds the rest back. - if lock(&self.owner.shared.shot).is_some() { + if lock(&self.owner.shared.shot).is_some() || lock(&self.owner.shared.archive).is_some() { return 0; } let pending: Vec = lock(&self.owner.shared.requests).drain(..).collect(); @@ -952,9 +1005,21 @@ impl Relay { if reviewed { std::mem::take(&mut req.plans).into_iter().map(|(id, p)| (id, p.allow())).collect() } else { Vec::new() }; let started = Instant::now(); let out = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - server.handle_scoped(s, &req.line, scope, permits, &req.id.to_string()) + server.start_archive_batch(s, &req.line, scope, permits, &req.id.to_string()) })) - .unwrap_or_else(|_| Some(internal_error(&req.line))); + .unwrap_or_else(|_| crate::mcp::ArchiveStep::Done(Some(internal_error(&req.line)))); + let out = match out { + crate::mcp::ArchiveStep::Done(out) => out, + crate::mcp::ArchiveStep::Read(batch) => { + self.leave_agent(s, &req.token, layers, false); + *lock(&self.owner.shared.archive) = Some(Archive { req, batch, started }); + let mut queue = lock(&self.owner.shared.requests); + for rest in pending.rev() { + queue.push_front(rest); + } + return ran; + } + }; // A layer.select that failed (an unknown layer, a refusal) chose nothing. let picked = selects_layer(&req.line) && reply_ok(out.as_deref()); self.leave_agent(s, &req.token, layers, picked); @@ -1557,6 +1622,10 @@ impl Revocation for GrantLive { } } +#[cfg(test)] +#[path = "relay/archive_tests.rs"] +mod archive_tests; + #[cfg(test)] #[allow(clippy::unwrap_used, clippy::panic)] mod tests { diff --git a/crates/assistant/src/relay/archive_tests.rs b/crates/assistant/src/relay/archive_tests.rs new file mode 100644 index 0000000000..d5cdbea9d6 --- /dev/null +++ b/crates/assistant/src/relay/archive_tests.rs @@ -0,0 +1,271 @@ +//! Real archive reads across the relay's approval, worker and completion boundaries. +#![allow(clippy::unwrap_used, clippy::panic)] + +use super::*; +use rustgis_engine::{ + Project, + cmd::reads, + render::rustgis_tiles::{mvt, pmtiles}, +}; +use serde_json::json; +use std::sync::mpsc::Receiver; + +fn archive(path: &std::path::Path, credit: &str) { + let metadata = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Mvt, + gzipped_tiles: false, + bounds: [-180.0, -85.0, 180.0, 85.0], + metadata: json!({"attribution":credit}), + }; + let bytes = pmtiles::write(vec![(0, 0, 0, mvt::encode(&mvt::Tile { layers: Vec::new() }))], &metadata).unwrap(); + std::fs::write(path, bytes).unwrap(); +} + +fn fixture() -> (tempfile::TempDir, Session, PathBuf, PathBuf) { + let root = tempfile::tempdir().unwrap(); + let original = root.path().join("original.pmtiles"); + let replacement = root.path().join("replacement.pmtiles"); + archive(&original, "Original"); + archive(&replacement, "Replacement"); + let mut session = Session::new(Project::default()); + session.run("map.addTiles", &json!({"url":original,"name":"Linked","overlay":true})).unwrap(); + (root, session, original, replacement) +} + +fn command(id: u64, name: &str, params: Value) -> Value { + json!({"jsonrpc":"2.0","id":id,"method":"tools/call","params":{"name":"run_command","arguments":{"command":name,"params":params}}}) +} + +fn queue(relay: &Relay, grant: &Grant, message: Value) -> (u64, Receiver>) { + let (reply, rx) = sync_channel(1); + let id = relay.owner.shared.next_id.fetch_add(1, SeqCst); + lock(&relay.owner.shared.requests).push_back(Request { + plans: Vec::new(), + approval_required: false, + held: Arc::default(), + id, + decision: Arc::new(AtomicU8::new(PENDING)), + line: message.to_string(), + token: grant.token.clone(), + asked: true, + reply, + state: Arc::new(AtomicU8::new(PENDING)), + }); + (id, rx) +} + +fn worker(ahead: reads::ReadAhead) -> Result { + std::thread::spawn(move || ahead.run(&rustgis_engine::analysis::control::Control::stoppable())).join().unwrap() +} + +fn finish(relay: &Relay, session: &mut Session, done: Result) { + relay.begin_archive_landing(session); + let result = done.and_then(|read| reads::read_land(session, read)); + relay.finish_archive_read(session, result); +} + +fn reply(rx: &Receiver>) -> Value { + serde_json::from_str(&rx.recv_timeout(Duration::from_secs(5)).unwrap().unwrap()).unwrap() +} + +#[test] +fn scoped_archive_reply_reports_only_changes_made_when_it_lands() { + let (_root, mut s, _, replacement) = fixture(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let (_, rx) = queue(&relay, &grant, command(1, "map.repairTiles", json!({"index":0,"path":replacement}))); + assert_eq!(relay.poll(&mut s), 0); + let done = worker(relay.take_archive_read().unwrap()); + // Work the queue ran between admission and landing belongs to its own caller. + s.run("layer.addSample", &json!({"name":"countries"})).unwrap(); + s.notify("Unrelated queued notice", false); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer["result"]["isError"], false, "{answer}"); + let text = answer["result"]["content"][0]["text"].as_str().unwrap(); + assert!(!text.contains("layersAdded") && !text.contains("Countries"), "{text}"); + assert!(!text.contains("Unrelated queued notice"), "{text}"); + assert!(text.contains("replacement") || text.contains("Repaired"), "{text}"); + relay.stop(); +} + +#[test] +fn scoped_archive_capture_rejects_scripts_other_commands_and_nested_ingress() { + use rustgis_engine::policy::Call; + let (_root, mut s, _, _) = fixture(); + for call in [Call::script("map.reloadTiles {\"index\":0}"), Call::command("map.tiles", json!({}))] { + assert!(s.start_scoped_archive_read(Scope::peer(), &call, None).is_err()); + } + let call = Call::command("map.reloadTiles", json!({"index":0})); + let nested = s.with_scope(Scope::peer(), |s| s.start_scoped_archive_read(Scope::peer(), &call, None)).unwrap(); + assert!(nested.is_err(), "a nested owner must not create a new pre-handler escape"); +} + +#[test] +fn direct_allowed_archive_reload_keeps_its_scoped_owner_without_a_permit() { + let (_root, mut s, original, _) = fixture(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let retained = pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap(); + archive(&original, "Allowed scoped reload"); + let (_, rx) = queue(&relay, &grant, command(1, "map.reloadTiles", json!({"index":0}))); + assert_eq!(relay.poll(&mut s), 0); + assert!(grant.pending().is_empty()); + let done = worker(relay.take_archive_read().unwrap()); + assert!(rx.try_recv().is_err()); + assert!(Arc::ptr_eq(&retained, &pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap())); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer["result"]["isError"], false, "{answer}"); + assert!(!Arc::ptr_eq(&retained, &pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap())); + relay.stop(); +} + +#[test] +fn reviewed_direct_archive_repair_waits_for_real_completion_and_keeps_original_actor_and_undo() { + let (root, mut s, original, replacement) = fixture(); + let relay = Relay::start(0).unwrap(); + let log = root.path().join("audit"); + relay.set_audit_dir(Some(log.clone())); + let grant = relay.grant_reviewed(false).unwrap(); + let depth = s.undo_depth(); + let (id, rx) = queue(&relay, &grant, command(1, "map.repairTiles", json!({"index":0,"path":replacement}))); + assert_eq!(relay.poll(&mut s), 0); + assert_eq!(grant.pending().len(), 1); + assert!(relay.take_archive_read().is_none(), "approval must precede even worker creation"); + assert_eq!(s.map().tiles[0].url, original.to_str().unwrap()); + assert!(rx.try_recv().is_err()); + grant.decide(id, true); + assert_eq!(relay.poll(&mut s), 0); + let ahead = relay.take_archive_read().unwrap(); + assert!(ahead.reads()); + assert!(relay.take_archive_read().is_none(), "one admitted invocation, not redispatch"); + let done = worker(ahead); + assert_eq!(s.undo_depth(), depth); + assert_eq!(s.map().tiles[0].url, original.to_str().unwrap()); + assert!(rx.try_recv().is_err(), "worker completion is not command completion"); + assert!(!log.exists() || std::fs::read_dir(&log).unwrap().next().is_none(), "no premature audit"); + s.actor = Some("intervening user".into()); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer["result"]["isError"], false, "{answer}"); + assert_eq!(s.map().tiles[0].url, replacement.to_str().unwrap()); + assert_eq!(s.undo_depth(), depth + 1); + assert!(s.undo_actor().flatten().is_some_and(|actor| actor.starts_with("agent 1 (relay run ")), "{:?}", s.undo_actor()); + let files: Vec<_> = std::fs::read_dir(&log).unwrap().flatten().collect(); + assert_eq!(files.len(), 1); + let audit = std::fs::read_to_string(files[0].path()).unwrap(); + let entry: Value = serde_json::from_str(audit.trim()).unwrap(); + assert_eq!(entry["success"], true, "{entry}"); + s.run("edit.undo", &Value::Null).unwrap(); + assert_eq!(s.map().tiles[0].url, original.to_str().unwrap()); + relay.stop(); +} + +#[test] +fn scoped_archive_revocation_before_worker_and_before_landing_has_no_effect() { + for before_start in [true, false] { + let (_root, mut s, original, replacement) = fixture(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant_reviewed(false).unwrap(); + let depth = s.undo_depth(); + let (id, rx) = queue(&relay, &grant, command(1, "map.repairTiles", json!({"index":0,"path":replacement}))); + relay.poll(&mut s); + grant.decide(id, true); + relay.poll(&mut s); + let ahead = relay.take_archive_read().unwrap(); + let done = if before_start { + drop(grant); + let done = worker(ahead); + assert!(done.is_err(), "revoked authority must not start its read"); + done + } else { + let done = worker(ahead); + assert!(done.is_ok()); + drop(grant); + done + }; + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer["result"]["isError"], true, "{answer}"); + assert_eq!(s.map().tiles[0].url, original.to_str().unwrap()); + assert_eq!(s.undo_depth(), depth); + relay.stop(); + } +} + +#[test] +fn scoped_archive_read_cannot_land_on_a_stale_target() { + let (_root, mut s, original, replacement) = fixture(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant_reviewed(false).unwrap(); + let (id, rx) = queue(&relay, &grant, command(1, "map.repairTiles", json!({"index":0,"path":replacement}))); + relay.poll(&mut s); + grant.decide(id, true); + relay.poll(&mut s); + let done = worker(relay.take_archive_read().unwrap()); + s.run("map.setTiles", &json!({"index":0,"opacity":0.25})).unwrap(); + let depth = s.undo_depth(); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer["result"]["isError"], true, "{answer}"); + assert_eq!(s.map().tiles[0].url, original.to_str().unwrap()); + assert_eq!(s.map().tiles[0].opacity, 0.25); + assert_eq!(s.undo_depth(), depth); + relay.stop(); +} + +#[test] +fn batch_reload_holds_observer_and_later_requests_until_real_landing() { + let (_root, mut s, original, _) = fixture(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + // A real retained metadata object distinguishes validation from installation. + let retained = pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap(); + archive(&original, "Fresh metadata after manual reload"); + s.map_mut().tiles[0].error = Some("Observer must not see the retained error".into()); + let (_, rx) = queue(&relay, &grant, json!([command(7, "map.reloadTiles", json!({"index":0})), command(7, "map.tiles", json!({}))])); + assert_eq!(relay.poll(&mut s), 0); + let done = worker(relay.take_archive_read().unwrap()); + assert_eq!(s.map().tiles[0].error.as_deref(), Some("Observer must not see the retained error")); + assert!(Arc::ptr_eq(&retained, &pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap())); + let (_, later) = queue(&relay, &grant, json!({"jsonrpc":"2.0","id":9,"method":"ping"})); + assert_eq!(relay.poll(&mut s), 0); + assert!(later.try_recv().is_err()); + assert!(rx.try_recv().is_err()); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer.as_array().unwrap().len(), 2, "{answer}"); + assert!(answer.as_array().unwrap().iter().all(|r| r["id"] == 7 && r["result"]["isError"] == false), "{answer}"); + let observed = answer[1]["result"]["content"][0]["text"].as_str().unwrap(); + assert!(observed.contains("Linked"), "{observed}"); + assert!(!observed.contains("Observer must not see the retained error"), "{observed}"); + let fresh = pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap(); + assert!(!Arc::ptr_eq(&retained, &fresh)); + relay.poll(&mut s); + assert_eq!(reply(&later)["id"], 9); + relay.stop(); +} + +/// An approval freezes the state its card showed. A batch suffix whose reviewed input the earlier +/// entry then legitimately changes (a retained load error the reload clears) is refused at its own +/// turn, after the reload landed, instead of running on stale consent. +#[test] +fn reviewed_batch_suffix_keeps_its_preflight_state_across_a_landed_reload() { + let (_root, mut s, _, _) = fixture(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant_reviewed(false).unwrap(); + s.map_mut().tiles[0].error = Some("retained".into()); + let (id, rx) = queue(&relay, &grant, json!([command(7, "map.reloadTiles", json!({"index":0})), command(8, "map.tiles", json!({}))])); + relay.poll(&mut s); + grant.decide(id, true); + relay.poll(&mut s); + let done = worker(relay.take_archive_read().unwrap()); + assert!(rx.try_recv().is_err()); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer[0]["result"]["isError"], false, "{answer}"); + assert_eq!(answer[1]["result"]["isError"], true, "{answer}"); + assert!(s.map().tiles[0].error.is_none(), "the observer's turn came after the reload landed"); + relay.stop(); +} diff --git a/crates/assistant/src/tools.rs b/crates/assistant/src/tools.rs index 880a294d87..0a37c0a67b 100644 --- a/crates/assistant/src/tools.rs +++ b/crates/assistant/src/tools.rs @@ -230,6 +230,25 @@ pub fn run_with_budget(s: &mut Session, call: &ToolCall, tokens: usize, scope: S run_with_budget_inner(s, call, tokens, scope, permit).map_err(|e| public_error(&e)) } +/// Adapter-owned presentation state for one already-admitted archive command. Execution +/// authority stays in the engine's ReadAhead/Read, never in this result formatter. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) struct CommandCompletion { + before: Vec<(String, Vec)>, + mark: u64, +} +#[cfg(not(target_arch = "wasm32"))] +impl CommandCompletion { + pub(crate) fn capture(s: &Session) -> Self { + Self { before: layer_snap(s), mark: s.notice_mark() } + } + pub(crate) fn finish(self, s: &mut Session, result: Result) -> Result { + result + .map(|value| command_result(s, value, &self.before, self.mark, DEFAULT_OUTPUT_TOKENS)) + .map_err(|e| public_error(&error_with_notices(e, s, self.mark))) + } +} + /// Retains the initiating scope and one-use permit while assembling ordinary tool results. fn run_with_budget_inner(s: &mut Session, call: &ToolCall, tokens: usize, scope: Scope, permit: Option) -> Result { if matches!(call.name.as_str(), "run_command" | "run_tool") { diff --git a/crates/engine/src/cmd/reads.rs b/crates/engine/src/cmd/reads.rs index c879ac1445..e8c35bc7f4 100644 --- a/crates/engine/src/cmd/reads.rs +++ b/crates/engine/src/cmd/reads.rs @@ -45,6 +45,13 @@ pub struct ReadAhead { } impl ReadAhead { + /// The immutable registered invocation retained by this read, for queue presentation only. + pub fn command(&self) -> Option<(&'static str, &Value)> { + match self.op.op() { + CommandOp::Registered { spec, params } => Some((spec.id, params)), + _ => None, + } + } /// Whether this command has a read to make before it lands; otherwise landing runs it inline. pub fn reads(&self) -> bool { self.work.is_some() diff --git a/crates/engine/src/policy/dispatch.rs b/crates/engine/src/policy/dispatch.rs index d2f707cc03..3b04a9f2c0 100644 --- a/crates/engine/src/policy/dispatch.rs +++ b/crates/engine/src/policy/dispatch.rs @@ -107,6 +107,39 @@ impl Session { if self.depth > 0 || !self.policy.nodes.is_empty() || self.policy.cursor.is_some() { return Err("background reads can only start before handler dispatch".into()); } + self.capture_read(command, params) + } + /// Admits exactly one standalone archive command before dispatch, retaining its original + /// scope and one-use approval through the worker and landing. Composite/handler capture + /// remains forbidden, including when an approval cursor is already active. + pub fn start_scoped_archive_read( + &mut self, + scope: Scope, + call: &Call, + permit: Option, + ) -> Result { + if self.depth > 0 || !self.policy.nodes.is_empty() || self.policy.cursor.is_some() || !self.policy.scopes.is_empty() { + return Err("background reads can only start before handler dispatch".into()); + } + let CallKind::Command { id, params } = &call.kind else { + return Err("only a standalone archive command can prepare a scoped read".into()); + }; + if !matches!(id.as_str(), "map.reloadTiles" | "map.repairTiles") { + return Err("only a standalone archive command can prepare a scoped read".into()); + } + let capture = |s: &mut Session| s.capture_read(id, params); + match permit { + Some(p) => { + if p.request != call.request { + return Err("this approval belongs to another request".into()); + } + self.run_plan_with(scope, p.plan, call, capture) + } + None => self.with_scope(scope, capture)?, + } + } + /// Private pre-handler capture shared by native reads and the exact scoped archive entry. + fn capture_read(&mut self, command: &str, params: &Value) -> Result { if !matches!(command, "layer.addData" | "layer.loadAll" | "model.run" | "script.run" | "map.reloadTiles" | "map.repairTiles") { return Err("only Add Data, Load All, models, scripts and local tile archives can prepare background work".into()); } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index b2204fd075..0315bd76a5 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -4923,7 +4923,7 @@ impl RustGisApp { self.frame_ctx = Some(ctx.clone()); // Agents the user turned on last time: the relay starts with the app. connectui::resume(self); - if let Some(relay) = &self.relay { + if let Some(relay) = self.relay.clone() { // A relay started before the first frame has no wake yet; now there is a context // to ask for repaints with. if self.relay_wake_pending { @@ -4935,9 +4935,13 @@ impl RustGisApp { relay.set_screen(relayshot::map_shown(self, ctx)); // Only a request that changed the project ends a label move: a read-only one leaves it. let state = self.session.state(); - if relay.poll(&mut self.session) > 0 && self.session.state() != state { + relay.poll(&mut self.session); + if self.session.state() != state { self.map.cancel_label_drag(); } + if let Some(ahead) = relay.take_archive_read() { + readjob::admitted(self, ahead, relay); + } } relayshot::drive(self, ctx); // A script an agent wrote to the file bridge. diff --git a/crates/ui-egui/src/readjob.rs b/crates/ui-egui/src/readjob.rs index 3afc8349e5..0d32881b43 100644 --- a/crates/ui-egui/src/readjob.rs +++ b/crates/ui-egui/src/readjob.rs @@ -41,6 +41,8 @@ struct Asked { archive: Option<(usize, rustgis_engine::map::TileLayer)>, /// Given what the command answered when it runs, or why it didn't (cancelled, or left). then: Option, + /// A direct relay archive call owns its real completion, rather than a Native callback. + relay: Option, } /// What a caller does once the command lands (ModelBuilder shows its run). @@ -128,6 +130,7 @@ pub fn run_then(app: &mut RustGisApp, command: &'static str, mut p: Value, then: layer: layer.map(|(instance, _)| instance), archive, then, + relay: None, }; if busy(app) { app.reads.waiting.push_back(asked); @@ -143,6 +146,47 @@ pub fn run_then(app: &mut RustGisApp, command: &'static str, mut p: Value, then: } } +/// Queues an archive read already admitted by the relay, without dispatching another command. +pub fn admitted(app: &mut RustGisApp, ahead: reads::ReadAhead, relay: rustgis_assistant::relay::Relay) { + let Some((command, params)) = ahead.command() else { + relay.finish_archive_read(&mut app.session, Err(tl!("This read was already started").to_string())); + return; + }; + let params = params.clone(); + let archive = reads::archive_target(&app.session, ¶ms); + let what = archive.as_ref().map(|(_, tile)| tile.name.clone()).unwrap_or_default(); + let asked = Asked { + command, + params, + ahead: Some(ahead), + what, + opened: app.session.opened, + map: app.session.map().instance_id(), + layer: None, + archive, + then: None, + relay: Some(relay), + }; + if busy(app) { + app.reads.waiting.push_back(asked); + } else { + match placed(app, &asked) { + Ok(p) => { + let _ = start(app, asked, p); + } + Err(why) => not_run_asked(app, &asked, why.to_string()), + } + } +} + +fn not_run_asked(app: &mut RustGisApp, asked: &Asked, why: String) { + not_run(app, asked.then, why.clone()); + if let Some(relay) = &asked.relay { + relay.begin_archive_landing(&app.session); + relay.finish_archive_read(&mut app.session, Err(why)); + } +} + /// What the command answered, `r`, given to `then` first. fn answered(app: &mut RustGisApp, then: Option, r: Result) -> Result { if let Some(then) = then { @@ -161,7 +205,7 @@ fn not_run(app: &mut RustGisApp, then: Option, why: String) { /// `asked` left for `why`: its `then` told now, and kept in `left` for the notice. fn leave(app: &mut RustGisApp, left: &mut Vec<(String, Gone)>, asked: &Asked, why: Gone) { let what = called(asked.command, &asked.what).to_string(); - not_run(app, asked.then, crate::i18n::fmt("{what} was left: {why}", &[("what", &what), ("why", &why.to_string())])); + not_run_asked(app, asked, crate::i18n::fmt("{what} was left: {why}", &[("what", &what), ("why", &why.to_string())])); left.push((what, why)); } @@ -260,7 +304,7 @@ fn start(app: &mut RustGisApp, mut asked: Asked, p: Value) -> Result { let notice_mark = app.session.notice_mark(); app.command_failed(asked.command, &error, notice_mark); - not_run(app, asked.then, error.clone()); + not_run_asked(app, &asked, error.clone()); return Err(error); } }; + if let Some(relay) = &asked.relay { + relay.begin_archive_landing(&app.session); + } let r = app.run_read(asked.command, p, read); if asked.command == "map.repairTiles" && let Ok(answer) = &r @@ -297,6 +344,9 @@ fn complete_here(app: &mut RustGisApp, asked: Asked, p: Value, done: Result bool { crate::i18n::fmt("{tool} of {what} was cancelled", &[("tool", tl_raw!(job.name)), ("what", &job.asked.what)]) }; app.session.notify(said.clone(), false); - not_run(app, job.asked.then, said); + not_run_asked(app, &job.asked, said); let waiting = std::mem::take(&mut app.reads.waiting); let n = waiting.len(); let left_too = match n { @@ -341,7 +391,7 @@ pub fn land(app: &mut RustGisApp) -> bool { app.session.notify(left_too.clone(), false); } for asked in waiting { - not_run(app, asked.then, left_too.clone()); + not_run_asked(app, &asked, left_too.clone()); } } else { // The engine consumes the original owner before the callback sees its result. @@ -545,6 +595,65 @@ mod tests { std::fs::write(path, bytes).unwrap(); } + #[test] + fn direct_relay_archive_joins_existing_read_queue_and_answers_only_after_landing() { + use rustgis_assistant::relay::{Relay, read_token_file}; + use std::io::{BufRead, BufReader, Write}; + use std::net::TcpStream; + let _turn = crate::fetching::turn(); + let dir = std::env::temp_dir().join(format!("rustgis-relay-read-queue-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let original = dir.join("original.pmtiles"); + let replacement = dir.join("replacement.pmtiles"); + let csv = dir.join("first.csv"); + archive(&original, "Original"); + archive(&replacement, "Replacement"); + std::fs::write(&csv, "name,x,y\nA,8,47\n").unwrap(); + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + app.run("map.addTiles", json!({"url":original,"name":"Linked","overlay":true})).unwrap(); + app.reads_off_frame = true; + run(&mut app, "layer.addData", json!({"path":csv})).unwrap(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant_reviewed(false).unwrap(); + let mut client = TcpStream::connect(("127.0.0.1", relay.port())).unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let token = read_token_file(grant.token_file()).unwrap(); + writeln!(client, "{}", json!({"jsonrpc":"2.0","method":"rustgis/attach","params":{"token":token,"access":true}})).unwrap(); + writeln!(client, "{}", json!({"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"run_command","arguments":{"command":"map.repairTiles","params":{"index":0,"path":replacement}}}})).unwrap(); + client.flush().unwrap(); + let end = std::time::Instant::now() + Duration::from_secs(10); + loop { + relay.poll(&mut app.session); + if let Some(approval) = grant.pending().first() { + grant.decide(approval.id, true); + } + if let Some(ahead) = relay.take_archive_read() { + admitted(&mut app, ahead, relay.clone()); + break; + } + assert!(std::time::Instant::now() < end, "relay never admitted the archive"); + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(app.reads.waiting.len(), 1, "the relay must reuse the one existing worker queue"); + assert_eq!(app.session.map().tiles[0].url, original.to_str().unwrap()); + client.set_nonblocking(true).unwrap(); + assert_eq!(client.peek(&mut [0u8; 1]).unwrap_err().kind(), std::io::ErrorKind::WouldBlock); + client.set_nonblocking(false).unwrap(); + landed(&mut app, 60); + let mut line = String::new(); + BufReader::new(client).read_line(&mut line).unwrap(); + let reply: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(reply["result"]["isError"], false, "{reply}"); + let text = reply["result"]["content"][0]["text"].as_str().unwrap(); + assert!(text.contains("replacement") || text.contains("Repaired"), "the agent's own change is reported: {text}"); + assert!(!text.contains("layersAdded") && !text.contains("first"), "the queued user read is not the agent's change: {text}"); + assert_eq!(app.session.map().tiles[0].url, replacement.to_str().unwrap()); + assert_eq!(app.session.map().layers.len(), 1); + assert!(app.session.undo_actor().flatten().is_some_and(|actor| actor.starts_with("agent 1 (relay run "))); + relay.stop(); + let _ = std::fs::remove_dir_all(dir); + } + #[test] fn archive_reload_lands_once_and_stale_or_cancelled_reads_keep_retained_metadata() { let _turn = crate::fetching::turn(); diff --git a/docs/mcp.md b/docs/mcp.md index c0fac2fc66..c425efcd68 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -130,8 +130,14 @@ Attached calls run on the open app's session through the same tools and refusal private server. `snapshot` additionally supports a real window capture with `mode: "screen"`. If the map is not on screen, it falls back to an offscreen render and explains why in `note`. The relay holds a screen request until the app has taken the picture, and runs nothing else -meanwhile. File and network commands need `--yes`, unless **Let your Agents Control RustGIS** -is ticked. +meanwhile. Direct `run_command` calls to `map.reloadTiles` and `map.repairTiles` validate local +PMTiles/MBTiles archives on the desktop's existing background-read worker after admission and, +when required, approval. Their replies and audit entries wait for the command to actually land; +later entries in the same JSON-RPC batch and later relay requests wait too. Cancelling, revoking +the caller or changing the original archive target prevents installation. The original agent +still owns the resulting undo step. Arbitrary `run` scripts and file-bridge scripts retain their +existing synchronous execution contract; this does not move their child commands to the worker. +File and network commands need `--yes`, unless **Let your Agents Control RustGIS** is ticked. Without **Full access**, an attached agent cannot replace the open project: `project.new`, `project.open`, `project.openText`, `project.sample` and `project.fromTemplate` From 8be9319f9a241a0b9a900e3bf88a0691cdb76590 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Sun, 8 Nov 2026 20:48:52 +0100 Subject: [PATCH 07/15] fix: preserve archive batch intent and validate nested tile ranges --- crates/assistant/src/mcp.rs | 21 +- crates/assistant/src/relay.rs | 27 ++- crates/assistant/src/relay/archive_tests.rs | 139 ++++++++++++++ crates/assistant/src/tools.rs | 12 ++ crates/tiles/src/pmtiles.rs | 106 ++++++++-- crates/ui-egui/src/mapview.rs | 11 ++ crates/ui-egui/src/readjob.rs | 202 +++++++++++++++++++- 7 files changed, 481 insertions(+), 37 deletions(-) diff --git a/crates/assistant/src/mcp.rs b/crates/assistant/src/mcp.rs index 86dcb65818..7a5a966129 100644 --- a/crates/assistant/src/mcp.rs +++ b/crates/assistant/src/mcp.rs @@ -102,10 +102,12 @@ pub(crate) struct ArchiveBatch { replies: Vec, pending: Option<(Value, tools::CommandCompletion)>, ahead: Option, + /// Successful explicit selections in the current uninterrupted execution segment. + picked: bool, } #[cfg(not(target_arch = "wasm32"))] pub(crate) enum ArchiveStep { - Done(Option), + Done(Option, bool), Read(Box), } #[cfg(not(target_arch = "wasm32"))] @@ -113,6 +115,9 @@ impl ArchiveBatch { pub(crate) fn take_read(&mut self) -> Option { self.ahead.take() } + pub(crate) fn picked(&self) -> bool { + self.picked + } /// The command's reply reports changes made from the moment it actually runs, not from its /// admission: the queue may run user work or other reads first. pub(crate) fn begin_landing(&mut self, s: &Session) { @@ -121,6 +126,7 @@ impl ArchiveBatch { } } pub(crate) fn finish(mut self, s: &mut Session, result: Result) -> ArchiveStep { + self.picked = false; if let Some((id, completion)) = self.pending.take() { self.replies.push(archive_reply(id, completion.finish(s, result))); } @@ -137,7 +143,7 @@ impl ArchiveBatch { } let (name, args) = tool_request(msg.get("params")?).ok()?; if name != "run_command" - || !matches!(args.get("command").and_then(Value::as_str), Some("map.reloadTiles" | "map.repairTiles")) + || !matches!(args.get("command").and_then(Value::as_str).map(str::trim), Some("map.reloadTiles" | "map.repairTiles")) { return None; } @@ -158,6 +164,12 @@ impl ArchiveBatch { Err(error) => self.replies.push(archive_reply(id, completion.finish(s, Err(error)))), } } else if let Some(reply) = self.server.handle_value(s, &msg, &self.scope, &mut self.permits, &request) { + if reply.get("error").is_none() && reply.pointer("/result/isError") == Some(&Value::Bool(false)) { + self.picked |= msg.get("params").is_some_and(|params| { + let Some(name) = params.get("name").and_then(Value::as_str) else { return false }; + params.get("arguments").is_some_and(|args| tools::selects_layer(name, args)) + }); + } self.replies.push(reply); } } @@ -168,7 +180,7 @@ impl ArchiveBatch { } else { self.replies.pop().map(|reply| reply.to_string()) }; - ArchiveStep::Done(reply) + ArchiveStep::Done(reply, self.picked) } } #[cfg(not(target_arch = "wasm32"))] @@ -207,7 +219,7 @@ impl Server { let (messages, batch) = match parsed { Ok(Value::Array(batch)) if !batch.is_empty() && batch.len() <= MAX_BATCH => (batch, true), Ok(msg) if !msg.is_array() => (vec![msg], false), - _ => return ArchiveStep::Done(self.handle_scoped(s, message, scope, permits, request)), + _ => return ArchiveStep::Done(self.handle_scoped(s, message, scope, permits, request), false), }; ArchiveBatch { server: self, @@ -220,6 +232,7 @@ impl Server { replies: Vec::new(), pending: None, ahead: None, + picked: false, } .advance(s) } diff --git a/crates/assistant/src/relay.rs b/crates/assistant/src/relay.rs index 8cfe820b22..4438a40127 100644 --- a/crates/assistant/src/relay.rs +++ b/crates/assistant/src/relay.rs @@ -749,17 +749,16 @@ impl Relay { let Archive { req, batch, started, .. } = archive; let layers = self.enter_agent(s, &req.token); let step = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| (*batch).finish(s, result))) - .unwrap_or_else(|_| crate::mcp::ArchiveStep::Done(Some(internal_error(&req.line)))); - let out = match step { + .unwrap_or_else(|_| crate::mcp::ArchiveStep::Done(Some(internal_error(&req.line)), false)); + let (out, picked) = match step { crate::mcp::ArchiveStep::Read(batch) => { - self.leave_agent(s, &req.token, layers, false); + self.leave_agent(s, &req.token, layers, batch.picked()); *lock(&self.owner.shared.archive) = Some(Archive { req, batch, started }); self.owner.shared.notify(); return; } - crate::mcp::ArchiveStep::Done(out) => out, + crate::mcp::ArchiveStep::Done(out, picked) => (out, picked), }; - let picked = selects_layer(&req.line) && reply_ok(out.as_deref()); self.leave_agent(s, &req.token, layers, picked); self.audit(&req.line, out.as_deref(), started.elapsed()); if let Some(action) = action(&req.line, out.as_deref()) { @@ -1007,11 +1006,11 @@ impl Relay { let out = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { server.start_archive_batch(s, &req.line, scope, permits, &req.id.to_string()) })) - .unwrap_or_else(|_| crate::mcp::ArchiveStep::Done(Some(internal_error(&req.line)))); - let out = match out { - crate::mcp::ArchiveStep::Done(out) => out, + .unwrap_or_else(|_| crate::mcp::ArchiveStep::Done(Some(internal_error(&req.line)), false)); + let (out, picked) = match out { + crate::mcp::ArchiveStep::Done(out, picked) => (out, picked), crate::mcp::ArchiveStep::Read(batch) => { - self.leave_agent(s, &req.token, layers, false); + self.leave_agent(s, &req.token, layers, batch.picked()); *lock(&self.owner.shared.archive) = Some(Archive { req, batch, started }); let mut queue = lock(&self.owner.shared.requests); for rest in pending.rev() { @@ -1020,8 +1019,6 @@ impl Relay { return ran; } }; - // A layer.select that failed (an unknown layer, a refusal) chose nothing. - let picked = selects_layer(&req.line) && reply_ok(out.as_deref()); self.leave_agent(s, &req.token, layers, picked); self.audit(&req.line, out.as_deref(), started.elapsed()); if let Some(a) = action(&req.line, out.as_deref()) { @@ -1092,11 +1089,9 @@ pub(crate) fn reply_ok(reply: Option<&str>) -> bool { pub(crate) fn selects_layer(line: &str) -> bool { let Ok(msg) = serde_json::from_str::(line) else { return false }; let messages = msg.as_array().map(Vec::as_slice).unwrap_or_else(|| std::slice::from_ref(&msg)); - messages.iter().filter_map(|m| m.get("params").and_then(|p| p.get("arguments"))).any(|a| { - a.get("command").and_then(Value::as_str) == Some("layer.select") - || a.get("script").and_then(Value::as_str).is_some_and(|s| { - rustgis_engine::cmd::workflows::script_commands(s).is_ok_and(|lines| lines.iter().any(|(_, id, _)| id == "layer.select")) - }) + messages.iter().filter_map(|m| m.get("params")).any(|params| { + let Some(name) = params.get("name").and_then(Value::as_str) else { return false }; + params.get("arguments").is_some_and(|args| crate::tools::selects_layer(name, args)) }) } diff --git a/crates/assistant/src/relay/archive_tests.rs b/crates/assistant/src/relay/archive_tests.rs index d5cdbea9d6..b244e77e66 100644 --- a/crates/assistant/src/relay/archive_tests.rs +++ b/crates/assistant/src/relay/archive_tests.rs @@ -269,3 +269,142 @@ fn reviewed_batch_suffix_keeps_its_preflight_state_across_a_landed_reload() { assert!(s.map().tiles[0].error.is_none(), "the observer's turn came after the reload landed"); relay.stop(); } + +#[test] +fn padded_archive_commands_use_the_worker_and_keep_admission_and_landing_boundaries() { + for reviewed in [false, true] { + for repair in [false, true] { + let (_root, mut s, original, replacement) = fixture(); + let retained = pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap(); + archive(&original, "Reloaded"); + let relay = Relay::start(0).unwrap(); + let grant = if reviewed { relay.grant_reviewed(false).unwrap() } else { relay.grant(true).unwrap() }; + let name = if repair { " map.repairTiles " } else { " map.reloadTiles " }; + let params = if repair { json!({"index":0,"path":replacement}) } else { json!({"index":0}) }; + let depth = s.undo_depth(); + let (id, rx) = queue(&relay, &grant, command(1, name, params)); + assert_eq!(relay.poll(&mut s), 0); + if reviewed { + assert_eq!(grant.pending().len(), 1); + assert!(relay.take_archive_read().is_none()); + grant.decide(id, true); + assert_eq!(relay.poll(&mut s), 0); + } + let ahead = relay.take_archive_read().unwrap(); + assert!(ahead.reads(), "{name} must use the archive worker"); + let done = worker(ahead); + assert!(rx.try_recv().is_err()); + assert_eq!(s.undo_depth(), depth); + assert_eq!(s.map().tiles[0].url, original.to_str().unwrap()); + assert!(Arc::ptr_eq(&retained, &pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap())); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer["result"]["isError"], false, "{answer}"); + assert_eq!(s.undo_depth(), depth + 1); + assert_eq!(s.map().tiles[0].url, if repair { replacement.to_str().unwrap() } else { original.to_str().unwrap() }); + relay.stop(); + } + } +} + +#[test] +fn padded_archive_commands_without_authorization_do_not_start_a_read() { + for repair in [false, true] { + let (_root, mut s, original, replacement) = fixture(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(false).unwrap(); + let name = if repair { " map.repairTiles " } else { " map.reloadTiles " }; + let params = if repair { json!({"index":0,"path":replacement}) } else { json!({"index":0}) }; + let (_, rx) = queue(&relay, &grant, command(1, name, params)); + assert_eq!(relay.poll(&mut s), 1); + assert!(relay.take_archive_read().is_none()); + let answer = reply(&rx); + assert_eq!(answer["result"]["isError"], true, "{answer}"); + assert_eq!(s.map().tiles[0].url, original.to_str().unwrap()); + relay.stop(); + } +} + +fn layers(s: &mut Session) -> (u32, u32) { + for name in ["A", "B"] { + s.run("layer.template", &json!({"template":"points","name":name})).unwrap(); + } + let a = s.map().layer_by_name("A").unwrap().id; + let b = s.map().layer_by_name("B").unwrap().id; + s.run("layer.select", &json!({"layer":a})).unwrap(); + (a, b) +} + +#[test] +fn selecting_the_active_layer_is_owned_before_each_archive_pause() { + for second_pause in [false, true] { + let (_root, mut s, _, _) = fixture(); + let (a, b) = layers(&mut s); + let before_a = s.map().layer(a).unwrap().renderer.clone(); + let before_b = s.map().layer(b).unwrap().renderer.clone(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let mut batch = Vec::new(); + if second_pause { + batch.push(command(1, "map.reloadTiles", json!({"index":0}))); + } + batch.extend([ + command(2, "layer.select", json!({"layer":a})), + // An unrelated failure must not erase a selection that actually succeeded. + command(3, "layer.select", json!({"layer":"Missing"})), + command(4, "map.reloadTiles", json!({"index":0})), + command(5, "layer.color", json!({"color":"#123456"})), + ]); + let (_, rx) = queue(&relay, &grant, json!(batch)); + assert_eq!(relay.poll(&mut s), 0); + if second_pause { + let done = worker(relay.take_archive_read().unwrap()); + finish(&relay, &mut s, done); + assert!(rx.try_recv().is_err()); + } + assert_eq!(s.active_layer, Some(a)); + assert_eq!(lock(&relay.owner.shared.agents).get(&grant.token).unwrap().active, Some(a)); + let done = worker(relay.take_archive_read().unwrap()); + s.run("layer.select", &json!({"layer":b})).unwrap(); + finish(&relay, &mut s, done); + let answer = reply(&rx); + let replies = answer.as_array().unwrap(); + assert_eq!(replies.last().unwrap()["result"]["isError"], false, "{answer}"); + assert_ne!(s.map().layer(a).unwrap().renderer, before_a, "suffix must change the selected A"); + assert_eq!(s.map().layer(b).unwrap().renderer, before_b, "user-selected B must not be changed"); + assert_eq!(s.active_layer, Some(b), "the user's active layer is restored"); + relay.stop(); + } +} + +#[test] +fn failed_or_unexecuted_selection_does_not_own_a_layer_at_archive_pause() { + let (_root, mut s, _, _) = fixture(); + let (a, b) = layers(&mut s); + let before_a = s.map().layer(a).unwrap().renderer.clone(); + let before_b = s.map().layer(b).unwrap().renderer.clone(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let (_, rx) = queue( + &relay, + &grant, + json!([ + command(1, "layer.select", json!({"layer":"Missing"})), + command(2, "map.reloadTiles", json!({"index":0})), + command(3, "layer.color", json!({"color":"#123456"})), + command(4, "layer.select", json!({"layer":a})), + ]), + ); + assert_eq!(relay.poll(&mut s), 0); + assert_eq!(lock(&relay.owner.shared.agents).get(&grant.token).unwrap().active, None); + let done = worker(relay.take_archive_read().unwrap()); + s.run("layer.select", &json!({"layer":b})).unwrap(); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer[0]["result"]["isError"], true, "{answer}"); + assert_eq!(answer[2]["result"]["isError"], false, "{answer}"); + assert_eq!(s.map().layer(a).unwrap().renderer, before_a); + assert_ne!(s.map().layer(b).unwrap().renderer, before_b, "without an executed selection the suffix follows the user"); + assert_eq!(s.active_layer, Some(b)); + relay.stop(); +} diff --git a/crates/assistant/src/tools.rs b/crates/assistant/src/tools.rs index 0a37c0a67b..54e6ce163f 100644 --- a/crates/assistant/src/tools.rs +++ b/crates/assistant/src/tools.rs @@ -154,6 +154,18 @@ pub fn target(call: &ToolCall) -> Option<(String, Value)> { } } +/// Whether a successfully executed tool explicitly chose a layer, even if it was already active. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn selects_layer(name: &str, args: &Value) -> bool { + match name { + "run_command" => args.get("command").and_then(Value::as_str).map(str::trim) == Some("layer.select"), + "run" => args.get("script").and_then(Value::as_str).is_some_and(|script| { + rustgis_engine::cmd::workflows::script_commands(script).is_ok_and(|lines| lines.iter().any(|(_, id, _)| id == "layer.select")) + }), + _ => false, + } +} + /// The executor's strict envelope is the only input to engine planning. pub fn policy_call(call: &ToolCall) -> Result, String> { let c = match call.name.as_str() { diff --git a/crates/tiles/src/pmtiles.rs b/crates/tiles/src/pmtiles.rs index 35bd7fa3f6..c1d167652a 100644 --- a/crates/tiles/src/pmtiles.rs +++ b/crates/tiles/src/pmtiles.rs @@ -874,8 +874,16 @@ impl IndexBudget { } } -/// Completed ranges remember their subtree height, so sharing a directory cannot hide a path -/// exceeding MAX_DEPTH. An unfinished range encountered again is a cycle. +/// A completed directory's height and greatest exclusive tile-run end, including descendants. +/// Recheck both on every shared reference: its parent may impose a tighter depth or tile boundary. +#[cfg(not(target_arch = "wasm32"))] +#[derive(Clone, Copy, Debug, Default)] +struct DirectorySummary { + height: usize, + tile_end: u64, +} + +/// An unfinished range encountered again is a cycle; completed ranges cost no further I/O or budget. #[cfg(not(target_arch = "wasm32"))] fn validate_directories( archive: &str, @@ -883,9 +891,9 @@ fn validate_directories( dir: &[Entry], depth: usize, budget: &mut IndexBudget, - seen: &mut HashMap<(u64, u64), Option>, -) -> Result { - let mut height = 0; + seen: &mut HashMap<(u64, u64), Option>, +) -> Result { + let mut summary = DirectorySummary::default(); for (i, e) in dir.iter().enumerate() { rustgis_data::progress::check().map_err(|e| e.to_string())?; let end = e.offset.checked_add(e.length).ok_or("a PMTiles entry range overflows")?; @@ -900,6 +908,7 @@ fn validate_directories( if e.length > MAX_BYTES as u64 { return Err(format!("{} bytes is too large to read from an archive", e.length)); } + summary.tile_end = summary.tile_end.max(tile_end); continue; } if depth >= MAX_DEPTH { @@ -914,7 +923,7 @@ fn validate_directories( return Err(format!("{} bytes is too large to read from an archive", e.length)); } let key = (at, e.length); - let child_height = if let Some(done) = seen.get(&key) { + let child_summary = if let Some(done) = seen.get(&key) { done.ok_or("cyclic PMTiles directories")? } else { // Refuse oversized ranges and too many directories before issuing any I/O. @@ -933,17 +942,21 @@ fn validate_directories( let child = parse_dir_bounded(&bytes, budget.entries)?; budget.take(e.length, bytes.len() as u64, child.len())?; drop(bytes); - let child_height = validate_directories(archive, info, &child, depth + 1, budget, seen)?; - seen.insert(key, Some(child_height)); - child_height + let child_summary = validate_directories(archive, info, &child, depth + 1, budget, seen)?; + seen.insert(key, Some(child_summary)); + child_summary }; - let subtree = child_height + 1; + if dir.get(i + 1).is_some_and(|next| child_summary.tile_end > next.tile_id) { + return Err("overlapping PMTiles tile runs".into()); + } + let subtree = child_summary.height + 1; if depth + subtree > MAX_DEPTH { return Err("PMTiles directories nested too deep".into()); } - height = height.max(subtree); + summary.height = summary.height.max(subtree); + summary.tile_end = summary.tile_end.max(child_summary.tile_end); } - Ok(height) + Ok(summary) } fn u64_at(b: &[u8], i: usize) -> Option { @@ -1436,6 +1449,75 @@ pub(crate) mod tests { std::fs::remove_file(path).unwrap(); } + /// Two root pointers; the first reaches [0, 3), optionally through an intermediate directory, + /// while the second reaches an empty directory. `next_id = 2` hides tile 2 from lookup. + #[cfg(not(target_arch = "wasm32"))] + fn ancestor_boundary_archive(nested: bool, next_id: u64) -> Vec { + let mut leaves = write_dir(&[Entry { tile_id: 0, offset: 0, length: 3, run_length: 3 }]); + let mut first = Entry { tile_id: 0, offset: 0, length: leaves.len() as u64, run_length: 0 }; + if nested { + let directory = write_dir(&[first]); + first.offset = leaves.len() as u64; + first.length = directory.len() as u64; + leaves.extend(directory); + } + let empty = write_dir(&[]); + let next = Entry { tile_id: next_id, offset: leaves.len() as u64, length: empty.len() as u64, run_length: 0 }; + leaves.extend(empty); + indexed_archive(&[first, next], leaves) + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn fresh_local_validation_checks_descendant_runs_against_ancestor_boundaries() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-ancestor-boundaries-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + for nested in [false, true] { + // [0, 3) crosses a boundary at 2, ends exactly at 3, and leaves a gap before 5. + for next_id in [2, 3, 5] { + std::fs::write(&path, ancestor_boundary_archive(nested, next_id)).unwrap(); + let result = read_info_local(source); + if next_id == 2 { + assert_eq!(result.unwrap_err(), "overlapping PMTiles tile runs", "nested={nested}"); + } else { + assert!(result.is_ok(), "nested={nested}, next_id={next_id}: {result:?}"); + } + } + } + std::fs::remove_file(path).unwrap(); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn memoized_directory_rechecks_each_parent_boundary_without_io_or_budget() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-memo-boundaries-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + let leaf = write_dir(&[Entry { tile_id: 0, offset: 0, length: 3, run_length: 3 }]); + let pointer = Entry { tile_id: 0, offset: 0, length: leaf.len() as u64, run_length: 0 }; + let bytes = indexed_archive(&[pointer], leaf.clone()); + let (info, _, _) = parse_head(&bytes).unwrap(); + std::fs::write(&path, bytes).unwrap(); + // Exactly one leaf's work is available; the initial visit consumes all of it. + let mut budget = IndexBudget { stored: leaf.len() as u64, decoded: leaf.len() as u64, entries: 1, directories: 1 }; + let mut seen = HashMap::new(); + let summary = validate_directories(source, &info, &[pointer], 0, &mut budget, &mut seen).unwrap(); + assert_eq!((summary.height, summary.tile_end), (1, 3)); + assert_eq!((budget.stored, budget.decoded, budget.entries, budget.directories), (0, 0, 0, 0)); + // Reused references must neither reopen the removed file nor charge the exhausted budget. + std::fs::remove_file(&path).unwrap(); + for next_id in [3, 5, 2] { + let parent = [pointer, Entry { tile_id: next_id, offset: 0, length: 3, run_length: 1 }]; + let result = validate_directories(source, &info, &parent, 0, &mut budget, &mut seen); + if next_id == 2 { + assert_eq!(result.unwrap_err(), "overlapping PMTiles tile runs"); + } else { + let summary = result.unwrap(); + assert_eq!((summary.height, summary.tile_end), (1, next_id + 1)); + } + assert_eq!((budget.stored, budget.decoded, budget.entries, budget.directories), (0, 0, 0, 0)); + } + } + #[cfg(not(target_arch = "wasm32"))] #[test] fn fresh_local_validation_cancels_before_reading_a_later_malformed_leaf() { diff --git a/crates/ui-egui/src/mapview.rs b/crates/ui-egui/src/mapview.rs index 0f12f265cb..48a5a9a059 100644 --- a/crates/ui-egui/src/mapview.rs +++ b/crates/ui-egui/src/mapview.rs @@ -536,6 +536,17 @@ impl MapView { } } + #[cfg(test)] + pub(crate) fn label_dragging_for_tests(&self) -> bool { + self.moving_label.is_some() + } + + #[cfg(test)] + pub(crate) fn label_position_for_tests(&self, text: &str) -> Option { + let label = self.labels.iter().find(|label| label.text == text)?; + Some(self.view_rect?.0.min + vec2(label.x, label.y)) + } + /// Cancels a vertex preview and its pending release together, leaving other map gestures /// alone. The chosen node is cleared because its row/address may now belong to another shape. pub(crate) fn cancel_vertex_drag(&mut self) { diff --git a/crates/ui-egui/src/readjob.rs b/crates/ui-egui/src/readjob.rs index c041fcf1fc..754b7817e0 100644 --- a/crates/ui-egui/src/readjob.rs +++ b/crates/ui-egui/src/readjob.rs @@ -110,8 +110,10 @@ pub fn run_then(app: &mut RustGisApp, command: &'static str, mut p: Value, then: } let archive = matches!(command, "map.reloadTiles" | "map.repairTiles").then(|| reads::archive_target(&app.session, &p)).flatten(); let mut p = p; - if let Some((i, _)) = &archive { - p["index"] = json!(i); + if let Some((i, _)) = &archive + && let Some(params) = p.as_object_mut() + { + params.insert("index".into(), json!(i)); } let what = match &layer { Some((_, name)) => name.clone(), @@ -149,7 +151,7 @@ pub fn run_then(app: &mut RustGisApp, command: &'static str, mut p: Value, then: /// Queues an archive read already admitted by the relay, without dispatching another command. pub fn admitted(app: &mut RustGisApp, ahead: reads::ReadAhead, relay: rustgis_assistant::relay::Relay) { let Some((command, params)) = ahead.command() else { - relay.finish_archive_read(&mut app.session, Err(tl!("This read was already started").to_string())); + finish_relay(app, &relay, Err(tl!("This read was already started").to_string())); return; }; let params = params.clone(); @@ -183,7 +185,16 @@ fn not_run_asked(app: &mut RustGisApp, asked: &Asked, why: String) { not_run(app, asked.then, why.clone()); if let Some(relay) = &asked.relay { relay.begin_archive_landing(&app.session); - relay.finish_archive_read(&mut app.session, Err(why)); + finish_relay(app, relay, Err(why)); + } +} + +/// Resumed relay suffixes have the same interaction guard as calls run by `Relay::poll`. +fn finish_relay(app: &mut RustGisApp, relay: &rustgis_assistant::relay::Relay, result: Result) { + let state = app.session.state(); + relay.finish_archive_read(&mut app.session, result); + if app.session.state() != state { + app.map.cancel_label_drag(); } } @@ -345,7 +356,7 @@ fn complete_here(app: &mut RustGisApp, asked: Asked, p: Value, done: Result Value { + json!({"jsonrpc":"2.0","id":id,"method":"tools/call","params":{"name":"run_command","arguments":{"command":command,"params":params}}}) + } + + fn paused_relay( + app: &mut RustGisApp, + batch: Value, + ) -> (rustgis_assistant::relay::Relay, rustgis_assistant::relay::Grant, std::net::TcpStream, reads::ReadAhead) { + use rustgis_assistant::relay::{Relay, read_token_file}; + use std::io::Write; + use std::net::TcpStream; + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let mut client = TcpStream::connect(("127.0.0.1", relay.port())).unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let token = read_token_file(grant.token_file()).unwrap(); + writeln!(client, "{}", json!({"jsonrpc":"2.0","method":"rustgis/attach","params":{"token":token,"access":true}})).unwrap(); + writeln!(client, "{batch}").unwrap(); + client.flush().unwrap(); + let end = std::time::Instant::now() + Duration::from_secs(10); + loop { + relay.poll(&mut app.session); + if let Some(ahead) = relay.take_archive_read() { + return (relay, grant, client, ahead); + } + assert!(std::time::Instant::now() < end, "relay never paused at its archive read"); + std::thread::sleep(Duration::from_millis(5)); + } + } + + fn label_drag_harness(path: &std::path::Path) -> (egui_kittest::Harness<'static, RustGisApp>, egui::Pos2) { + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + app.run("map.addTiles", json!({"url":path,"name":"Linked","overlay":true})).unwrap(); + app.run("layer.template", json!({"template":"points","name":"Labelled"})).unwrap(); + app.run( + "edit.addFeature", + json!({"layer":"Labelled","geometry":{"type":"Point","coordinates":[1,1]},"attributes":{"name":"Relay drag"}}), + ) + .unwrap(); + app.run("labels.field", json!({"layer":"Labelled","field":"name"})).unwrap(); + app.run("tool.set", json!({"tool":"moveLabel"})).unwrap(); + app.reads_off_frame = true; + let mut h = egui_kittest::Harness::builder() + .with_size(egui::vec2(1400.0, 860.0)) + .with_step_dt(0.016) + .with_max_steps(10_000) + .build_ui_state( + |ui, app: &mut RustGisApp| { + let ctx = ui.ctx().clone(); + app.logic(&ctx); + app.ui(ui); + }, + app, + ); + h.step(); + h.state_mut().run("map.zoomToExtent", json!({"extent":[-1,-1,5,5]})).unwrap(); + let end = std::time::Instant::now() + Duration::from_secs(10); + let at = loop { + h.step(); + if let Some(at) = h.state().map.label_position_for_tests("Relay drag") + && !h.state().map.drawing() + { + break at; + } + assert!(std::time::Instant::now() < end, "the label was never drawn"); + std::thread::sleep(Duration::from_millis(2)); + }; + label_pointer(&mut h, at, None); + label_pointer(&mut h, at, Some(true)); + for i in 1..=6 { + label_pointer(&mut h, at + egui::vec2(0.0, -8.0 * i as f32), None); + } + assert!(h.state().map.label_dragging_for_tests(), "a real label drag must be active"); + (h, at + egui::vec2(0.0, -48.0)) + } + + fn label_pointer(h: &mut egui_kittest::Harness<'_, RustGisApp>, at: egui::Pos2, pressed: Option) { + h.event(egui::Event::PointerMoved(at)); + if let Some(pressed) = pressed { + h.event(egui::Event::PointerButton { + pos: at, + button: egui::PointerButton::Primary, + pressed, + modifiers: egui::Modifiers::NONE, + }); + } + h.step(); + } + + #[test] + fn resumed_relay_edits_cancel_label_drags_on_success_failure_and_cancel_but_reads_preserve_them() { + use std::io::{BufRead, BufReader}; + let _turn = crate::fetching::turn(); + let dir = std::env::temp_dir().join(format!("rustgis-relay-label-drag-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + for outcome in ["success", "failure", "cancel", "not_run"] { + for removes in [true, false] { + let path = dir.join(format!("{outcome}-{removes}.pmtiles")); + archive(&path, "Label drag"); + let (mut h, release) = label_drag_harness(&path); + let suffix = if removes { + relay_command(2, "layer.remove", json!({"layer":"Labelled"})) + } else { + relay_command(2, "map.tiles", json!({})) + }; + let batch = json!([relay_command(1, "map.reloadTiles", json!({"index":0})), suffix]); + let (relay, _grant, client, ahead) = paused_relay(h.state_mut(), batch); + assert!(h.state().map.label_dragging_for_tests()); + if outcome == "failure" { + std::fs::write(&path, b"not a PMTiles archive").unwrap(); + } + admitted(h.state_mut(), ahead, relay.clone()); + if outcome == "cancel" { + h.state().reads.cancel_for_tests(); + } else if outcome == "not_run" { + let job = h.state_mut().reads.job.take().unwrap(); + job.control.cancel(); + not_run_asked(h.state_mut(), &job.asked, "The archive read could not start".into()); + let end = std::time::Instant::now() + Duration::from_secs(10); + while job.slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner).is_none() { + assert!(std::time::Instant::now() < end, "cancelled worker did not stop"); + std::thread::sleep(Duration::from_millis(2)); + } + } + landed(h.state_mut(), 60); + let mut line = String::new(); + BufReader::new(client).read_line(&mut line).unwrap(); + let reply: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(reply[0]["result"]["isError"], outcome != "success", "{outcome}: {reply}"); + assert_eq!(reply[1]["result"]["isError"], false, "{reply}"); + assert_eq!(h.state().map.label_dragging_for_tests(), !removes, "{outcome}: resumed edit guard"); + assert_eq!(h.state().session.map().layer_by_name("Labelled").is_none(), removes); + let mark = h.state().session.notice_mark(); + let depth = h.state().session.undo_depth(); + label_pointer(&mut h, release, Some(false)); + assert!(!h.state().map.label_dragging_for_tests()); + if removes { + assert_eq!(h.state().session.undo_depth(), depth, "release must not dispatch labels.move"); + assert!(h.state().session.notices_since(mark).is_empty(), "no stale labels.move error after removal"); + } else { + assert_eq!(h.state().session.map().layer_by_name("Labelled").unwrap().labels.default.moved.len(), 1); + assert_eq!(h.state().session.undo_depth(), depth + 1, "a read-only suffix preserves the pending move"); + } + relay.stop(); + } + } + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn archive_parameter_normalization_uses_objects_and_rejects_non_objects_without_panicking() { + let _turn = crate::fetching::turn(); + let dir = std::env::temp_dir().join(format!("rustgis-archive-params-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let original = dir.join("original.pmtiles"); + let replacement = dir.join("replacement.pmtiles"); + archive(&original, "Original"); + archive(&replacement, "Replacement"); + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + app.run("map.addTiles", json!({"url":original,"name":"Linked","overlay":true})).unwrap(); + app.reads_off_frame = true; + for command in ["map.reloadTiles", "map.repairTiles"] { + for params in [Value::Null, json!(true), json!(7), json!([]), json!("not an object")] { + assert!(run(&mut app, command, params).is_err()); + assert!(!busy(&app), "hostile parameters must not start a read"); + } + let id = app.session.map().tiles[0].id.clone(); + for mut params in [json!({"index":0}), json!({"name":"Linked"}), json!({"id":id})] { + if command == "map.repairTiles" { + params.as_object_mut().unwrap().insert("path".into(), json!(replacement)); + } + run(&mut app, command, params).unwrap(); + let asked = &app.reads.job.as_ref().unwrap().asked; + assert_eq!(asked.params.get("index"), Some(&json!(0))); + assert_eq!(asked.archive.as_ref().unwrap().0, 0); + landed(&mut app, 60); + } + } + let _ = std::fs::remove_dir_all(dir); + } + #[test] fn files_added_read_on_a_thread_and_land_in_order() { let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); From ff6bb9c4afdd5909a8ebc16978c7f5ba1a2a3207 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Sun, 8 Nov 2026 22:22:11 +0100 Subject: [PATCH 08/15] fix: bind archive continuations to maps and reject hidden tile ranges --- crates/assistant/src/mcp.rs | 64 +++++-- crates/assistant/src/relay.rs | 30 +++- crates/assistant/src/relay/archive_tests.rs | 163 ++++++++++++++++++ .../tests/portable_projects/reload_tiles.rs | 109 ++++++++---- crates/tiles/src/pmtiles.rs | 93 +++++++++- docs/user-guide/maps.md | 9 +- 6 files changed, 400 insertions(+), 68 deletions(-) diff --git a/crates/assistant/src/mcp.rs b/crates/assistant/src/mcp.rs index 7a5a966129..3839ad221c 100644 --- a/crates/assistant/src/mcp.rs +++ b/crates/assistant/src/mcp.rs @@ -126,11 +126,29 @@ impl ArchiveBatch { } } pub(crate) fn finish(mut self, s: &mut Session, result: Result) -> ArchiveStep { + self.finish_pending(s, result); + self.advance(s) + } + /// A changed project or map invalidates the suffix, not the command's actual result. + /// Complete every remaining request without dispatching even a read-only tool. + pub(crate) fn cancel_remaining(mut self, s: &mut Session, result: Result) -> ArchiveStep { + self.finish_pending(s, result); + for msg in self.messages.by_ref() { + match request_identity(&msg) { + Ok(Some((_, id))) => { + self.replies.push(error_value(&id, -32000, "archive batch context changed; remaining request cancelled")); + } + Ok(None) => {} + Err(reply) => self.replies.push(reply), + } + } + self.done() + } + fn finish_pending(&mut self, s: &mut Session, result: Result) { self.picked = false; if let Some((id, completion)) = self.pending.take() { self.replies.push(archive_reply(id, completion.finish(s, result))); } - self.advance(s) } fn advance(mut self, s: &mut Session) -> ArchiveStep { while let Some(msg) = self.messages.next() { @@ -173,6 +191,9 @@ impl ArchiveBatch { self.replies.push(reply); } } + self.done() + } + fn done(mut self) -> ArchiveStep { let reply = if self.replies.is_empty() { None } else if self.batch { @@ -281,22 +302,12 @@ impl Server { permits: &mut Vec<(String, Permit)>, request: &str, ) -> Option { - let Some(obj) = msg.as_object() else { - return Some(error_value(&Value::Null, INVALID_REQUEST, "expected a JSON-RPC object")); + let (method, id) = match request_identity(msg) { + Ok(Some(request)) => request, + Ok(None) => return None, + Err(reply) => return Some(reply), }; - let id = obj.get("id").cloned(); - let Some(method) = obj.get("method").and_then(Value::as_str) else { - // A response to a request of ours (we send none), or junk. - return id - .filter(|_| !obj.contains_key("result") && !obj.contains_key("error")) - .map(|id| error_value(&id, INVALID_REQUEST, "expected a method")); - }; - // Notifications (no id) get no reply. - let id = id?; - if !matches!(id, Value::String(_) | Value::Number(_)) { - return Some(error_value(&Value::Null, INVALID_REQUEST, "id must be a string or a number")); - } - let params = obj.get("params").cloned().unwrap_or(Value::Null); + let params = msg.get("params").cloned().unwrap_or(Value::Null); Some(match self.call(s, method, ¶ms, scope, permits, request) { Ok(result) => json!({"jsonrpc": "2.0", "id": id, "result": result}), Err((code, message)) => error_value(&id, code, &message), @@ -673,6 +684,27 @@ pub fn expects_reply(message: &str) -> bool { } } +/// Applies JSON-RPC identity rules without executing a request. +fn request_identity(msg: &Value) -> Result, Value> { + let Some(obj) = msg.as_object() else { + return Err(error_value(&Value::Null, INVALID_REQUEST, "expected a JSON-RPC object")); + }; + let id = obj.get("id").cloned(); + let Some(method) = obj.get("method").and_then(Value::as_str) else { + // A response to a request of ours (we send none), or junk. + return match id.filter(|_| !obj.contains_key("result") && !obj.contains_key("error")) { + Some(id) => Err(error_value(&id, INVALID_REQUEST, "expected a method")), + None => Ok(None), + }; + }; + // Notifications (no id) get no reply. + let Some(id) = id else { return Ok(None) }; + if !matches!(id, Value::String(_) | Value::Number(_)) { + return Err(error_value(&Value::Null, INVALID_REQUEST, "id must be a string or a number")); + } + Ok(Some((method, id))) +} + /// Projects generated RPC error text while retaining the exact correlation id and code. fn error_value(id: &Value, code: i64, message: &str) -> Value { let message = rustgis_engine::render::rustgis_tiles::logins::scrub_error(message); diff --git a/crates/assistant/src/relay.rs b/crates/assistant/src/relay.rs index 4438a40127..7113ef7707 100644 --- a/crates/assistant/src/relay.rs +++ b/crates/assistant/src/relay.rs @@ -234,6 +234,9 @@ struct Archive { req: Request, batch: Box, started: Instant, + /// The project and selected map at this pause, after the preceding batch segment ran. + opened: u64, + map: u64, } impl Shared { @@ -743,23 +746,32 @@ impl Relay { } /// Completes the real archive command before running its batch suffix or issuing its reply - /// and audit. The engine has already checked the original owner at worker start and landing. + /// and audit. A suffix may resume only in the project and map selected at its latest pause. pub fn finish_archive_read(&self, s: &mut Session, result: Result) { let Some(archive) = lock(&self.owner.shared.archive).take() else { return }; - let Archive { req, batch, started, .. } = archive; - let layers = self.enter_agent(s, &req.token); - let step = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| (*batch).finish(s, result))) - .unwrap_or_else(|_| crate::mcp::ArchiveStep::Done(Some(internal_error(&req.line)), false)); + let Archive { req, batch, started, opened, map } = archive; + // Check before entering the agent: its remembered layer ID is only map-local, and a + // stale continuation must not change the current actor, selection or stored agent state. + let same_context = s.opened == opened && s.map().instance_id() == map; + let layers = same_context.then(|| self.enter_agent(s, &req.token)); + let step = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if same_context { (*batch).finish(s, result) } else { (*batch).cancel_remaining(s, result) } + })) + .unwrap_or_else(|_| crate::mcp::ArchiveStep::Done(Some(internal_error(&req.line)), false)); let (out, picked) = match step { crate::mcp::ArchiveStep::Read(batch) => { - self.leave_agent(s, &req.token, layers, batch.picked()); - *lock(&self.owner.shared.archive) = Some(Archive { req, batch, started }); + if let Some(layers) = layers { + self.leave_agent(s, &req.token, layers, batch.picked()); + } + *lock(&self.owner.shared.archive) = Some(Archive { req, batch, started, opened: s.opened, map: s.map().instance_id() }); self.owner.shared.notify(); return; } crate::mcp::ArchiveStep::Done(out, picked) => (out, picked), }; - self.leave_agent(s, &req.token, layers, picked); + if let Some(layers) = layers { + self.leave_agent(s, &req.token, layers, picked); + } self.audit(&req.line, out.as_deref(), started.elapsed()); if let Some(action) = action(&req.line, out.as_deref()) { *lock(&self.owner.last) = Some(action); @@ -1011,7 +1023,7 @@ impl Relay { crate::mcp::ArchiveStep::Done(out, picked) => (out, picked), crate::mcp::ArchiveStep::Read(batch) => { self.leave_agent(s, &req.token, layers, batch.picked()); - *lock(&self.owner.shared.archive) = Some(Archive { req, batch, started }); + *lock(&self.owner.shared.archive) = Some(Archive { req, batch, started, opened: s.opened, map: s.map().instance_id() }); let mut queue = lock(&self.owner.shared.requests); for rest in pending.rev() { queue.push_front(rest); diff --git a/crates/assistant/src/relay/archive_tests.rs b/crates/assistant/src/relay/archive_tests.rs index b244e77e66..8cd74f3ab6 100644 --- a/crates/assistant/src/relay/archive_tests.rs +++ b/crates/assistant/src/relay/archive_tests.rs @@ -408,3 +408,166 @@ fn failed_or_unexecuted_selection_does_not_own_a_layer_at_archive_pause() { assert_eq!(s.active_layer, Some(b)); relay.stop(); } + +#[test] +fn archive_batch_rejects_changed_context_before_agent_reentry_at_every_pause() { + for second_pause in [false, true] { + for replace_project in [false, true] { + for completion in ["stale", "cancelled", "success"] { + let (_root, mut s, _, _) = fixture(); + let (a, _) = layers(&mut s); + let map_a = s.map().name.clone(); + let renderer_a = s.map().layer(a).unwrap().renderer.clone(); + s.run("map.new", &json!({"name":"Other"})).unwrap(); + let (other, user) = layers(&mut s); + assert_eq!(a, other, "layer IDs are map-local"); + s.run("map.activate", &json!({"name":map_a})).unwrap(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let mut batch = vec![command(1, "layer.select", json!({"layer":a}))]; + if second_pause { + batch.push(command(2, "map.reloadTiles", json!({"index":0}))); + } + batch.extend([ + command(7, "map.reloadTiles", json!({"index":0})), + command(7, "layer.color", json!({"color":"#123456"})), + json!({"jsonrpc":"2.0","id":"observer","method":"ping"}), + json!({"jsonrpc":"2.0","method":"tools/call","params":{"name":"run_command","arguments":{"command":"layer.color","params":{"color":"#654321"}}}}), + json!({"jsonrpc":"2.0","id":"response","result":{}}), + command(9, "map.reloadTiles", json!({"index":0})), + ]); + let (_, rx) = queue(&relay, &grant, json!(batch)); + assert_eq!(relay.poll(&mut s), 0); + if second_pause { + let done = worker(relay.take_archive_read().unwrap()); + finish(&relay, &mut s, done); + assert!(rx.try_recv().is_err()); + } + let done = worker(relay.take_archive_read().unwrap()); + if replace_project { + let opened = s.opened; + s.run("project.new", &json!({"name":"Replacement"})).unwrap(); + assert_ne!(s.opened, opened); + let (replacement, _) = layers(&mut s); + assert_eq!(replacement, a); + } else { + s.run("map.activate", &json!({"name":"Other"})).unwrap(); + } + s.run("layer.select", &json!({"layer":user})).unwrap(); + s.actor = Some("current user".into()); + let before = serde_json::to_value(&s.project).unwrap(); + let agent_active = lock(&relay.owner.shared.agents).get(&grant.token).unwrap().active; + relay.begin_archive_landing(&s); + let result = match completion { + "stale" => done.and_then(|read| reads::read_land(&mut s, read)), + "cancelled" => Err("Cancelled archive fixture".into()), + _ => Ok(json!({"archive":"actual successful result"})), + }; + let failed = result.is_err(); + let expected = result.as_ref().err().cloned().unwrap_or_else(|| "actual successful result".into()); + relay.finish_archive_read(&mut s, result); + let answer = reply(&rx); + let replies = answer.as_array().unwrap(); + let pending = if second_pause { 2 } else { 1 }; + assert_eq!(replies.len(), pending + 4, "{answer}"); + assert_eq!(answer[0]["id"], 1); + assert_eq!(answer[0]["result"]["isError"], false, "{answer}"); + if second_pause { + assert_eq!(answer[1]["id"], 2); + assert_eq!(answer[1]["result"]["isError"], false, "{answer}"); + } + assert_eq!(answer[pending]["id"], 7); + assert_eq!(answer[pending]["result"]["isError"], failed, "{answer}"); + assert!(answer[pending]["result"]["content"][0]["text"].as_str().unwrap().contains(&expected), "{answer}"); + for (reply, id) in replies[pending + 1..].iter().zip([json!(7), json!("observer"), json!(9)]) { + assert_eq!(reply["id"], id, "{answer}"); + assert!(reply.get("error").is_some() || reply["result"]["isError"] == true, "{answer}"); + assert!(reply.to_string().contains("context changed"), "{answer}"); + } + assert_eq!(serde_json::to_value(&s.project).unwrap(), before, "no suffix command may mutate either map"); + if !replace_project { + assert_eq!(s.project.maps[0].layer(a).unwrap().renderer, renderer_a); + assert_eq!(s.map().name, "Other"); + } + assert_eq!(s.active_layer, Some(user)); + assert_eq!(s.actor.as_deref(), Some("current user")); + assert_eq!(lock(&relay.owner.shared.agents).get(&grant.token).unwrap().active, agent_active); + assert!(relay.take_archive_read().is_none(), "the suffix's second archive command must not dispatch"); + relay.finish_archive_read(&mut s, Err("duplicate completion".into())); + assert!(rx.try_recv().is_err(), "one terminal reply only"); + relay.stop(); + } + } + } +} + +#[test] +fn archive_batch_allows_explicit_map_changes_between_pauses() { + let (_root, mut s, original, _) = fixture(); + let (a, _) = layers(&mut s); + let map_a = s.map().name.clone(); + let before_a = s.map().layer(a).unwrap().renderer.clone(); + s.run("map.new", &json!({"name":"Other"})).unwrap(); + s.run("map.addTiles", &json!({"url":original,"name":"Other archive","overlay":true})).unwrap(); + let (b, _) = layers(&mut s); + let before_b = s.map().layer(b).unwrap().renderer.clone(); + s.run("map.activate", &json!({"name":map_a})).unwrap(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let (_, rx) = queue( + &relay, + &grant, + json!([ + command(1, "map.reloadTiles", json!({"index":0})), + command(2, "map.activate", json!({"name":"Other"})), + command(3, "layer.select", json!({"layer":b})), + command(4, "map.reloadTiles", json!({"index":0})), + command(5, "layer.color", json!({"color":"#123456"})), + ]), + ); + assert_eq!(relay.poll(&mut s), 0); + let done = worker(relay.take_archive_read().unwrap()); + finish(&relay, &mut s, done); + assert!(rx.try_recv().is_err()); + assert_eq!(s.map().name, "Other"); + let done = worker(relay.take_archive_read().unwrap()); + finish(&relay, &mut s, done); + let answer = reply(&rx); + let replies = answer.as_array().unwrap(); + assert_eq!(replies.len(), 5, "{answer}"); + assert!(replies.iter().all(|reply| reply["result"]["isError"] == false), "{answer}"); + assert_eq!(s.project.maps[0].layer(a).unwrap().renderer, before_a); + assert_ne!(s.map().layer(b).unwrap().renderer, before_b); + assert_eq!(s.map().name, "Other"); + relay.stop(); +} + +#[test] +fn archive_batch_resumes_same_context_after_validation_failure_or_cancellation() { + for error in ["Archive validation failed", "Cancelled"] { + let (_root, mut s, _, _) = fixture(); + let (a, _) = layers(&mut s); + let before = s.map().layer(a).unwrap().renderer.clone(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let (_, rx) = queue( + &relay, + &grant, + json!([ + command(1, "layer.select", json!({"layer":a})), + command(2, "map.reloadTiles", json!({"index":0})), + command(3, "layer.color", json!({"color":"#123456"})), + ]), + ); + assert_eq!(relay.poll(&mut s), 0); + drop(relay.take_archive_read().unwrap()); + relay.begin_archive_landing(&s); + relay.finish_archive_read(&mut s, Err(error.into())); + let answer = reply(&rx); + assert_eq!(answer[1]["result"]["isError"], true, "{answer}"); + assert!(answer[1]["result"]["content"][0]["text"].as_str().unwrap().contains(error), "{answer}"); + assert_eq!(answer[2]["result"]["isError"], false, "{answer}"); + assert_ne!(s.map().layer(a).unwrap().renderer, before); + relay.stop(); + } +} diff --git a/crates/engine/tests/portable_projects/reload_tiles.rs b/crates/engine/tests/portable_projects/reload_tiles.rs index 2f9a0d32f6..08a63afe45 100644 --- a/crates/engine/tests/portable_projects/reload_tiles.rs +++ b/crates/engine/tests/portable_projects/reload_tiles.rs @@ -742,40 +742,83 @@ fn archive_observation_begins_in_worker_not_when_work_is_queued() { } } +/// Move the writer's uncompressed root index into a leaf and optionally add an intermediate +/// directory. All pointer fields are small fixture constants; tile payloads remain unchanged. +fn pmtiles_with_leaf_pointer(bytes: &[u8], pointer_id: u8, nested: bool) -> Vec { + let at = |offset: usize| u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()) as usize; + assert_eq!(bytes[97], 1, "fixture directories must be uncompressed"); + assert_eq!(at(48), 0, "fixture must start with no leaf directories"); + let mut leaves = bytes[at(8)..at(8) + at(16)].to_vec(); + let leaf_length = u8::try_from(leaves.len()).unwrap(); + let mut root = vec![1, pointer_id, 0, leaf_length, 1]; + if nested { + root[3] = 5; + root[4] = leaf_length + 1; + leaves.extend([1, 0, 0, leaf_length, 1]); + } + let metadata = &bytes[at(24)..at(24) + at(32)]; + let data = &bytes[at(56)..at(56) + at(64)]; + let mut head = bytes[..127].to_vec(); + let metadata_at = 127 + root.len() as u64; + let leaf_at = metadata_at + metadata.len() as u64; + let data_at = leaf_at + leaves.len() as u64; + for (offset, value) in [(8, 127), (16, root.len() as u64), (24, metadata_at), (40, leaf_at), (48, leaves.len() as u64), (56, data_at)] { + head[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + [head, root, metadata.to_vec(), leaves, data.to_vec()].concat() +} + #[test] -fn overlapping_pmtiles_runs_reject_reload_and_repair_without_losing_retained_data() { +fn unreachable_pmtiles_runs_reject_reload_and_repair_without_losing_retained_data() { for command in ["map.reloadTiles", "map.repairTiles"] { - let f = Scoped::new(&format!("archive-overlapping-runs-{command}")); - let linked = f.file("linked.pmtiles"); - let replacement = f.file("replacement.pmtiles"); - write_archive(&linked, [255, 0, 0, 255], "Retained credits"); - let mut s = session(); - s.run("map.addTiles", &json!({"url":linked,"overlay":true})).unwrap(); - let tile = s.map().tiles[0].clone(); - let retained = vec![retain_archive(&tile)]; - s.style_view = Some((tile.clone(), s.map().camera, s.map().instance_id())); - let archive = pmtiles::NewArchive { - tile_type: pmtiles::TileType::Png, - gzipped_tiles: false, - bounds: [-180.0, -85.0, 180.0, 85.0], - metadata: json!({"attribution":"Valid adjacent runs"}), - }; - let valid = pmtiles::write(vec![(0, 0, 0, picture([0, 255, 0, 255])), (1, 0, 0, picture([0, 0, 255, 255]))], &archive).unwrap(); - assert_eq!(&valid[127..132], &[2, 0, 1, 1, 1], "fixture must contain two adjacent single-tile runs"); - let mut overlapping = valid.clone(); - overlapping[130] = 2; // First [0,1) run becomes [0,2), overlapping the next entry. - let mut params = json!({"index":0}); - let path = if command == "map.repairTiles" { - params["path"] = json!(replacement); - &replacement - } else { - &linked - }; - std::fs::write(path, overlapping).unwrap(); - let read = archive_read(&mut s, command, ¶ms); - assert!(reject_stale_archive(&mut s, read, &retained).contains("overlapping PMTiles tile runs")); - std::fs::write(path, valid).unwrap(); - s.run(command, ¶ms).unwrap(); - assert_color(&decoded(&s.map().tiles[0]), [0, 255, 0, 255]); + for defect in ["overlap", "direct lower bound", "nested lower bound"] { + let f = Scoped::new(&format!("archive-unreachable-runs-{command}-{defect}")); + let linked = f.file("linked.pmtiles"); + let replacement = f.file("replacement.pmtiles"); + write_archive(&linked, [255, 0, 0, 255], "Retained credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":linked,"overlay":true})).unwrap(); + s.run("map.setTiles", &json!({"index":0,"opacity":0.4})).unwrap(); + let tile = s.map().tiles[0].clone(); + let retained = vec![retain_archive(&tile)]; + s.style_view = Some((tile.clone(), s.map().camera, s.map().instance_id())); + let archive = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Png, + gzipped_tiles: false, + bounds: [-180.0, -85.0, 180.0, 85.0], + metadata: json!({"attribution":"Valid reachable runs"}), + }; + let valid = pmtiles::write(vec![(0, 0, 0, picture([0, 255, 0, 255])), (1, 0, 0, picture([0, 0, 255, 255]))], &archive).unwrap(); + assert_eq!(&valid[127..132], &[2, 0, 1, 1, 1], "fixture must contain two adjacent single-tile runs"); + let (invalid, corrected, expected) = if defect == "overlap" { + let mut overlapping = valid.clone(); + overlapping[130] = 2; // First [0,1) run becomes [0,2), overlapping the next entry. + (overlapping, valid, "overlapping PMTiles tile runs") + } else { + let nested = defect == "nested lower bound"; + ( + pmtiles_with_leaf_pointer(&valid, 5, nested), + pmtiles_with_leaf_pointer(&valid, 0, nested), + "a PMTiles leaf directory contains tiles below its pointer", + ) + }; + let mut params = json!({"index":0}); + let path = if command == "map.repairTiles" { + params["path"] = json!(replacement); + &replacement + } else { + &linked + }; + std::fs::write(path, invalid).unwrap(); + let read = archive_read(&mut s, command, ¶ms); + assert!(reject_stale_archive(&mut s, read, &retained).contains(expected)); + std::fs::write(path, corrected).unwrap(); + s.run(command, ¶ms).unwrap(); + let mut expected_tile = tile; + expected_tile.url = path.display().to_string(); + assert_eq!(s.map().tiles[0], expected_tile, "retry must preserve presentation and update only the repaired path"); + assert_color(&decoded(&s.map().tiles[0]), [0, 255, 0, 255]); + assert!(style(&s.map().tiles[0]).attribution.iter().any(|a| a.contains("Valid reachable runs"))); + } } } diff --git a/crates/tiles/src/pmtiles.rs b/crates/tiles/src/pmtiles.rs index c1d167652a..c810046389 100644 --- a/crates/tiles/src/pmtiles.rs +++ b/crates/tiles/src/pmtiles.rs @@ -245,7 +245,7 @@ pub struct NewArchive { pub gzipped_tiles: bool, /// Longitude/latitude bounds: west, south, east, north. pub bounds: [f64; 4], - /// The JSON metadata (`vector_layers`, `attribution`, `name`…). + /// The JSON metadata object (`vector_layers`, `attribution`, `name`…); use `{}` when absent. pub metadata: Value, } @@ -253,6 +253,9 @@ pub struct NewArchive { /// stored once, its directories uncompressed, in leaf directories when the root would pass the /// first 16 KB. Tiles given twice keep the last. pub fn write(tiles: Vec<(u8, u32, u32, Vec)>, a: &NewArchive) -> Result, String> { + if !a.metadata.is_object() { + return Err("PMTiles metadata must be a JSON object".into()); + } // A tile outside its zoom would take the id of another zoom's; readers stop at zoom 30. if let Some((z, x, y, _)) = tiles.iter().find(|(z, x, y, _)| *z > 30 || u64::from(*x) >= 1u64 << z || u64::from(*y) >= 1u64 << z) { return Err(format!("{z}/{x}/{y} isn't a tile (zoom 0–30, x and y below 2^zoom)")); @@ -874,12 +877,13 @@ impl IndexBudget { } } -/// A completed directory's height and greatest exclusive tile-run end, including descendants. -/// Recheck both on every shared reference: its parent may impose a tighter depth or tile boundary. +/// A completed directory's height and descendant tile bounds; empty subtrees have no minimum. +/// Recheck these on every shared reference: its parent may impose a tighter depth or tile boundary. #[cfg(not(target_arch = "wasm32"))] #[derive(Clone, Copy, Debug, Default)] struct DirectorySummary { height: usize, + tile_start: Option, tile_end: u64, } @@ -908,6 +912,7 @@ fn validate_directories( if e.length > MAX_BYTES as u64 { return Err(format!("{} bytes is too large to read from an archive", e.length)); } + summary.tile_start = Some(summary.tile_start.map_or(e.tile_id, |start| start.min(e.tile_id))); summary.tile_end = summary.tile_end.max(tile_end); continue; } @@ -946,6 +951,9 @@ fn validate_directories( seen.insert(key, Some(child_summary)); child_summary }; + if child_summary.tile_start.is_some_and(|start| start < e.tile_id) { + return Err("a PMTiles leaf directory contains tiles below its pointer".into()); + } if dir.get(i + 1).is_some_and(|next| child_summary.tile_end > next.tile_id) { return Err("overlapping PMTiles tile runs".into()); } @@ -954,6 +962,9 @@ fn validate_directories( return Err("PMTiles directories nested too deep".into()); } summary.height = summary.height.max(subtree); + if let Some(start) = child_summary.tile_start { + summary.tile_start = Some(summary.tile_start.map_or(start, |current| current.min(start))); + } summary.tile_end = summary.tile_end.max(child_summary.tile_end); } Ok(summary) @@ -1357,7 +1368,7 @@ pub(crate) mod tests { std::fs::write(&path, bad).unwrap(); assert_eq!(read_info_local(source).unwrap_err(), "a truncated directory"); - let first = write_dir(&[Entry { tile_id: 0, offset: 0, length: 3, run_length: 1 }]); + let first = write_dir(&[]); let second = write_dir(&[Entry { tile_id: 1, offset: 0, length: 3, run_length: 1 }]); let root = [ Entry { tile_id: 0, offset: 0, length: first.len() as u64, run_length: 0 }, @@ -1370,12 +1381,12 @@ pub(crate) mod tests { assert!(read_info_local(source).is_ok()); // Exactly enough for the root and two unique leaves; the repeated range costs no work. let bytes = (write_dir(&root).len() + leaves.len()) as u64; - let exact = IndexBudget { stored: bytes, decoded: bytes, entries: 5, directories: 3 }; + let exact = IndexBudget { stored: bytes, decoded: bytes, entries: 4, directories: 3 }; assert!(read_info_local_bounded(source, exact).is_ok()); for budget in [ IndexBudget { stored: bytes - 1, ..exact }, IndexBudget { decoded: bytes - 1, ..exact }, - IndexBudget { entries: 4, ..exact }, + IndexBudget { entries: 3, ..exact }, IndexBudget { directories: 2, ..exact }, ] { assert_eq!(read_info_local_bounded(source, budget).unwrap_err(), "PMTiles index exceeds validation budget"); @@ -1487,6 +1498,67 @@ pub(crate) mod tests { std::fs::remove_file(path).unwrap(); } + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn fresh_local_validation_checks_descendant_tiles_against_pointer_lower_bounds() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-pointer-lower-bounds-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + for nested in [false, true] { + for tile_id in [0, 5] { + for pointer_id in [0, 5, 6] { + let mut leaves = write_dir(&[Entry { tile_id, offset: 0, length: 3, run_length: 1 }]); + let mut pointer = Entry { tile_id: 0, offset: 0, length: leaves.len() as u64, run_length: 0 }; + if nested { + let directory = write_dir(&[pointer]); + pointer.offset = leaves.len() as u64; + pointer.length = directory.len() as u64; + leaves.extend(directory); + } + pointer.tile_id = pointer_id; + std::fs::write(&path, indexed_archive(&[pointer], leaves)).unwrap(); + let result = read_info_local(source); + if pointer_id > tile_id { + assert_eq!( + result.unwrap_err(), + "a PMTiles leaf directory contains tiles below its pointer", + "nested={nested}, pointer_id={pointer_id}, tile_id={tile_id}" + ); + } else { + assert!(result.is_ok(), "nested={nested}, pointer_id={pointer_id}, tile_id={tile_id}: {result:?}"); + } + } + } + } + std::fs::remove_file(path).unwrap(); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn memoized_directory_rechecks_pointer_lower_bounds_without_io_or_budget() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-memo-lower-bounds-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + let leaf = write_dir(&[Entry { tile_id: 5, offset: 0, length: 3, run_length: 1 }]); + let pointer = Entry { tile_id: 0, offset: 0, length: leaf.len() as u64, run_length: 0 }; + let bytes = indexed_archive(&[pointer], leaf.clone()); + let (info, _, _) = parse_head(&bytes).unwrap(); + std::fs::write(&path, bytes).unwrap(); + let mut budget = IndexBudget { stored: leaf.len() as u64, decoded: leaf.len() as u64, entries: 1, directories: 1 }; + let mut seen = HashMap::new(); + validate_directories(source, &info, &[pointer], 0, &mut budget, &mut seen).unwrap(); + assert_eq!((budget.stored, budget.decoded, budget.entries, budget.directories), (0, 0, 0, 0)); + std::fs::remove_file(&path).unwrap(); + // The same completed subtree is legal below its first tile and at equality, but not above. + for tile_id in [0, 5, 6] { + let result = validate_directories(source, &info, &[Entry { tile_id, ..pointer }], 0, &mut budget, &mut seen); + if tile_id > 5 { + assert_eq!(result.unwrap_err(), "a PMTiles leaf directory contains tiles below its pointer"); + } else { + assert!(result.is_ok(), "pointer_id={tile_id}: {result:?}"); + } + assert_eq!((budget.stored, budget.decoded, budget.entries, budget.directories), (0, 0, 0, 0)); + } + } + #[cfg(not(target_arch = "wasm32"))] #[test] fn memoized_directory_rechecks_each_parent_boundary_without_io_or_budget() { @@ -1779,8 +1851,12 @@ pub(crate) mod tests { tile_type, gzipped_tiles: false, bounds: [-10.0, -5.0, 10.0, 5.0], - metadata: serde_json::json!({"vector_layers": [{"id": "roads"}]}), + metadata: serde_json::json!({"vector_layers": [{"id": "roads"}], "attribution": "Written credits"}), }; + for metadata in [Value::Null, serde_json::json!([]), serde_json::json!("credits"), serde_json::json!(1), serde_json::json!(true)] { + let invalid = NewArchive { metadata, ..new(TileType::Mvt) }; + assert_eq!(write(vec![(0, 0, 0, b"tile".to_vec())], &invalid).unwrap_err(), "PMTiles metadata must be a JSON object"); + } // A few tiles, one given twice and two alike (stored once). let small = vec![(0u8, 0u32, 0u32, b"old".to_vec()), (1, 0, 0, b"same".to_vec()), (1, 0, 1, b"same".to_vec()), (0, 0, 0, b"zero".to_vec())]; @@ -1799,6 +1875,9 @@ pub(crate) mod tests { assert_eq!(get(&format!("pmtiles:1/0/0@{a}")).unwrap(), b"same"); assert_eq!(get(&format!("pmtiles:1/0/1@{a}")).unwrap(), b"same"); assert!(get(&format!("pmtiles:1/1/1@{a}")).unwrap().is_empty()); + assert_eq!(info_blocking(&a, std::time::Duration::from_secs(10)).unwrap().attribution(), "Written credits"); + #[cfg(not(target_arch = "wasm32"))] + assert_eq!(read_info_local(&a).unwrap().attribution(), "Written credits"); // Every tile to zoom 7, each different: too many for the root, so in leaves. let many: Vec<(u8, u32, u32, Vec)> = (0..=7u8) .flat_map(|z| (0..1u32 << z).flat_map(move |x| (0..1u32 << z).map(move |y| (z, x, y, format!("{z}/{x}/{y}").into_bytes())))) diff --git a/docs/user-guide/maps.md b/docs/user-guide/maps.md index 7174db298f..c678aac570 100644 --- a/docs/user-guide/maps.md +++ b/docs/user-guide/maps.md @@ -328,9 +328,12 @@ The layer keeps its name, visibility, opacity, blend, drawing order, source-laye style and other presentation settings; the map's view stays where it is. Undo cannot restore file contents that another program has overwritten. -On the desktop, reload and repair validate on a background worker; the status bar shows the work -and **Cancel**. Only one read runs at a time. If its project, map, archive or presentation changes -before it lands, the result is left unapplied: retry on the intended target. +On the desktop, menu actions and direct attached MCP `run_command` calls validate reload and +repair on a background worker; the status bar shows the work and **Cancel**. Only one read runs +at a time. If its project, map, archive or presentation changes before it lands, the result is +left unapplied: retry on the intended target. A paused attached batch also rejects its remaining +commands if the project or selected map changes. Attached MCP `run` scripts and file-bridge +scripts validate inline, without reload status or cancellation during validation. Approval binds the selected archive's owning map, stack position and source route, not just its name or index. Retargeting or reordering it requires reviewing the operation again. From 82a2d8953a111fd5bbbc0c88c5f45a86cffe363d Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Mon, 9 Nov 2026 09:41:50 +0100 Subject: [PATCH 09/15] fix: refuse lossy archive metadata and bound PMTiles writer output --- crates/data/src/mbtiles.rs | 56 ++++++++++++++++++++++++++++++++++--- crates/tiles/src/pmtiles.rs | 27 +++++++++++++++++- docs/user-guide/maps.md | 8 ++++-- 3 files changed, 83 insertions(+), 8 deletions(-) diff --git a/crates/data/src/mbtiles.rs b/crates/data/src/mbtiles.rs index 69e0aa56bf..235515c637 100644 --- a/crates/data/src/mbtiles.rs +++ b/crates/data/src/mbtiles.rs @@ -427,10 +427,13 @@ mod file { .query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name = 'metadata' COLLATE NOCASE)", [], |r| r.get(0)) .map_err(sql)?; if has_metadata { - let mut st = c.prepare("SELECT name, value FROM metadata LIMIT 1000").map_err(sql)?; - let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))).map_err(sql)?; - for row in rows { - meta.push(row.map_err(sql)?); + let mut st = c.prepare("SELECT name, value FROM metadata LIMIT 1001").map_err(sql)?; + let mut rows = st.query([]).map_err(sql)?; + while let Some(row) = rows.next().map_err(sql)? { + if meta.len() == 1000 { + return Err(format!("{}: MBTiles metadata exceeds the 1000-row limit; refusing to omit metadata", path.display())); + } + meta.push((row.get::<_, String>(0).map_err(sql)?, row.get::<_, String>(1).map_err(sql)?)); } } // That it is an MBTiles file: the tiles table (or view) has its columns. Nothing is read. @@ -488,6 +491,51 @@ mod file { mod reload_tests { use super::*; + #[test] + fn metadata_row_limit_is_explicit_and_retryable_for_both_readers() { + let path = std::env::temp_dir().join(format!("rustgis-mbtiles-metadata-limit-{}.mbtiles", std::process::id())); + let _ = std::fs::remove_file(&path); + let c = Connection::open(&path).unwrap(); + c.execute_batch( + "CREATE TABLE metadata(name TEXT,value TEXT); + CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + INSERT INTO metadata VALUES('format','png'),('minzoom','0'),('maxzoom','0');", + ) + .unwrap(); + let tx = c.unchecked_transaction().unwrap(); + { + let mut insert = tx.prepare("INSERT INTO metadata VALUES(?1,'unused')").unwrap(); + for n in 0..996 { + insert.execute([format!("unused-{n}")]).unwrap(); + } + } + tx.execute("INSERT INTO metadata VALUES('attribution','last allowed attribution')", []).unwrap(); + tx.commit().unwrap(); + + let assert_exact_limit = || { + let ordinary = open_path(&path).unwrap(); + let (candidate, sample) = read_info_path(&path).unwrap(); + for info in [&ordinary, &candidate] { + assert_eq!(info.format, super::super::Format::Png); + assert_eq!(info.attribution, "last allowed attribution", "the thousandth metadata row must survive"); + } + assert!(sample.is_none(), "the fixture has no tiles"); + }; + assert_exact_limit(); + + c.execute("INSERT INTO metadata VALUES('json',?1)", [r#"{"vector_layers":[{"id":"overflow"}]}"#]).unwrap(); + for error in [open_path(&path).unwrap_err(), read_info_path(&path).unwrap_err()] { + assert!(error.contains(&path.display().to_string()), "{error}"); + assert!(error.contains("1000-row limit"), "{error}"); + } + c.execute("DELETE FROM metadata WHERE name='json'", []).unwrap(); + assert_exact_limit(); + + close(&path); + drop(c); + std::fs::remove_file(path).unwrap(); + } + #[test] fn ordinary_open_refuses_null_and_non_text_metadata_rows() { let path = std::env::temp_dir().join(format!("rustgis-mbtiles-metadata-types-{}.mbtiles", std::process::id())); diff --git a/crates/tiles/src/pmtiles.rs b/crates/tiles/src/pmtiles.rs index c810046389..12f8f8da72 100644 --- a/crates/tiles/src/pmtiles.rs +++ b/crates/tiles/src/pmtiles.rs @@ -246,6 +246,7 @@ pub struct NewArchive { /// Longitude/latitude bounds: west, south, east, north. pub bounds: [f64; 4], /// The JSON metadata object (`vector_layers`, `attribution`, `name`…); use `{}` when absent. + /// Its serialized JSON must not exceed 16 MiB. pub metadata: Value, } @@ -256,6 +257,10 @@ pub fn write(tiles: Vec<(u8, u32, u32, Vec)>, a: &NewArchive) -> Result MAX_BYTES { + return Err("PMTiles metadata exceeds the 16 MiB limit".into()); + } // A tile outside its zoom would take the id of another zoom's; readers stop at zoom 30. if let Some((z, x, y, _)) = tiles.iter().find(|(z, x, y, _)| *z > 30 || u64::from(*x) >= 1u64 << z || u64::from(*y) >= 1u64 << z) { return Err(format!("{z}/{x}/{y} isn't a tile (zoom 0–30, x and y below 2^zoom)")); @@ -310,7 +315,6 @@ pub fn write(tiles: Vec<(u8, u32, u32, Vec)>, a: &NewArchive) -> Result Date: Mon, 9 Nov 2026 14:37:13 +0100 Subject: [PATCH 10/15] Fix archive reload cancellation and render consistency --- Cargo.toml | 2 +- crates/data/src/mbtiles.rs | 224 +++++++++-- crates/engine/src/cmd/layout.rs | 23 +- crates/engine/src/cmd/presentation.rs | 19 +- crates/engine/src/cmd/report.rs | 32 +- crates/engine/src/cmd/share.rs | 8 +- crates/engine/src/tests.rs | 3 + .../src/tests/archive_render_revision.rs | 372 ++++++++++++++++++ crates/render/src/gpu_scene.rs | 3 +- crates/render/src/layout.rs | 41 +- crates/render/src/lib.rs | 71 +++- crates/render/src/report.rs | 31 +- crates/render/src/stream_tests.rs | 8 +- crates/render/src/svg.rs | 18 +- crates/render/src/tiles.rs | 18 +- crates/tiles/src/cache.rs | 14 +- crates/ui-egui/src/layoutview.rs | 3 +- crates/ui-egui/src/mapgpu.rs | 58 ++- crates/ui-egui/src/mapview.rs | 22 +- crates/ui-egui/src/reportpane.rs | 3 +- crates/ui-egui/src/tilt_drape.rs | 40 +- crates/ui-egui/src/tilt_settle.rs | 5 +- docs/user-guide/maps.md | 4 + 23 files changed, 927 insertions(+), 95 deletions(-) create mode 100644 crates/engine/src/tests/archive_render_revision.rs diff --git a/Cargo.toml b/Cargo.toml index 192d9057ad..885ed9c900 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,7 +80,7 @@ arrow-schema = { version = "60", default-features = false } bytes = "1" flatgeobuf = { version = "6", default-features = false } geozero = { version = "0.15", default-features = false, features = ["with-wkb"] } -rusqlite = { version = "0.41", features = ["bundled", "functions"] } +rusqlite = { version = "0.41", features = ["bundled", "functions", "hooks"] } # PostgreSQL/PostGIS, in pure Rust: the client, and TLS through rustls (ring, as ureq has it). postgres = { version = "0.19", default-features = false } tokio-postgres-rustls = { version = "0.14", default-features = false, features = ["ring"] } diff --git a/crates/data/src/mbtiles.rs b/crates/data/src/mbtiles.rs index 235515c637..c3d12c9b0e 100644 --- a/crates/data/src/mbtiles.rs +++ b/crates/data/src/mbtiles.rs @@ -389,35 +389,72 @@ mod file { /// Reads metadata and one bounded raster sample on a fresh connection, retaining nothing. /// Vector archives get only a tile-page readability probe; an empty archive has no sample. + /// The current progress probe and request predicate cancel SQLite scans on this connection. pub fn read_info_path(path: &Path) -> Result<(Info, Option>), String> { - let c = open(path)?; - let info = read_info(&c, path)?; - let sql = |e: rusqlite::Error| format!("{}: not a readable MBTiles file ({e})", path.display()); - // Even complete metadata must not hide a damaged first tile page. Vector payloads are - // not decoded here; the tiles layer decodes the raster sample before accepting a reload. - let sample = if info.format.is_vector() { - let _: Option>> = - c.query_row("SELECT substr(tile_data, 1, 16) FROM tiles LIMIT 1", [], |row| row.get(0)).optional().map_err(sql)?; - None - } else { - // Check the size before materializing a blob, as the normal tile reader does. - let tile: Option<(i64, Option>)> = c - .query_row( - "SELECT length(tile_data), CASE WHEN length(tile_data) <= ?1 THEN tile_data END FROM tiles LIMIT 1", - [MAX_TILE_BYTES as i64], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .optional() - .map_err(sql)?; - match tile { - Some((len, _)) if len > MAX_TILE_BYTES as i64 => { - return Err(format!("{}: raster sample of {len} bytes exceeds the {MAX_TILE_BYTES}-byte tile limit", path.display())); - } - Some((_, bytes)) => bytes, - None => None, + let still = crate::source::remote::current_still(); + let cancelled = move || crate::progress::check().is_err() || still.as_ref().is_some_and(|s| !s()); + if cancelled() { + return Err(crate::DataError::Cancelled.to_string()); + } + // Latch callback cancellation: SQLite's "interrupted" error is not archive damage, + // even if a request predicate has changed again before the query unwinds. + let interrupted = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let result = (|| { + let mut c = open(path)?; + let poll = cancelled.clone(); + let stopped = interrupted.clone(); + c.progress_handler( + 1000, + Some(move || { + let stop = poll(); + if stop { + stopped.store(true, std::sync::atomic::Ordering::Relaxed); + } + stop + }), + ) + .map_err(|e| format!("{}: cannot install MBTiles cancellation handler ({e})", path.display()))?; + let info = read_info(&c, path)?; + if cancelled() { + return Err(crate::DataError::Cancelled.to_string()); } - }; - Ok((info, sample)) + let sql = |e: rusqlite::Error| format!("{}: not a readable MBTiles file ({e})", path.display()); + // Even complete metadata must not hide a damaged first tile page. Vector payloads are + // not decoded here; the tiles layer decodes the raster sample before accepting a reload. + let sample = if info.format.is_vector() { + let _: Option>> = + c.query_row("SELECT substr(tile_data, 1, 16) FROM tiles LIMIT 1", [], |row| row.get(0)).optional().map_err(sql)?; + None + } else { + // Check the size before materializing a blob, as the normal tile reader does. + let tile: Option<(i64, Option>)> = c + .query_row( + "SELECT length(tile_data), CASE WHEN length(tile_data) <= ?1 THEN tile_data END FROM tiles LIMIT 1", + [MAX_TILE_BYTES as i64], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(sql)?; + match tile { + Some((len, _)) if len > MAX_TILE_BYTES as i64 => { + return Err(format!( + "{}: raster sample of {len} bytes exceeds the {MAX_TILE_BYTES}-byte tile limit", + path.display() + )); + } + Some((_, bytes)) => bytes, + None => None, + } + }; + Ok((info, sample)) + })(); + // Tiny queries may finish before the first SQLite callback; check success and error + // boundaries too, including a failed open. No fresh handle or hook enters the pool. + if interrupted.load(std::sync::atomic::Ordering::Relaxed) || cancelled() { + Err(crate::DataError::Cancelled.to_string()) + } else { + result + } } fn read_info(c: &Connection, path: &Path) -> Result { @@ -491,6 +528,139 @@ mod file { mod reload_tests { use super::*; + struct CancelAfterPolls { + polls: std::sync::atomic::AtomicUsize, + after: usize, + } + + impl CancelAfterPolls { + fn new(after: usize) -> std::sync::Arc { + std::sync::Arc::new(Self { polls: std::sync::atomic::AtomicUsize::new(0), after }) + } + + fn polls(&self) -> usize { + self.polls.load(std::sync::atomic::Ordering::Relaxed) + } + } + + impl crate::progress::Probe for CancelAfterPolls { + fn cancelled(&self) -> bool { + self.polls.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1 >= self.after + } + + fn begin(&self, _: u64) {} + + fn advance(&self, _: u64) {} + } + + #[test] + fn candidate_missing_zoom_scan_cancels_inside_sql_and_retries_cleanly() { + let path = std::env::temp_dir().join(format!("rustgis-mbtiles-cancel-scan-{}.mbtiles", std::process::id())); + let _ = std::fs::remove_file(&path); + let c = Connection::open(&path).unwrap(); + c.execute_batch( + "CREATE TABLE metadata(name TEXT,value TEXT); + CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + INSERT INTO metadata VALUES('format','png'),('minzoom','2'),('maxzoom','7'); + WITH RECURSIVE n(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM n WHERE i<1023) + INSERT INTO tiles SELECT 2+i%6,i,0,x'89504e470d0a1a0a' FROM n;", + ) + .unwrap(); + drop(c); + + // The same small archive with complete zoom metadata needs only the entry and + // completion checks. Cancelling at poll four therefore requires SQLite's hook. + let complete = CancelAfterPolls::new(4); + { + let _installed = crate::progress::install(Some(complete.clone())); + let (info, _) = read_info_path(&path).unwrap(); + assert_eq!((info.min_zoom, info.max_zoom), (2, 7)); + } + assert!(complete.polls() < 4, "complete metadata unexpectedly ran a scan"); + let c = Connection::open(&path).unwrap(); + c.execute("DELETE FROM metadata WHERE name IN ('minzoom','maxzoom')", []).unwrap(); + drop(c); + + let scan = CancelAfterPolls::new(4); + { + let _installed = crate::progress::install(Some(scan.clone())); + assert_eq!(read_info_path(&path).unwrap_err(), crate::DataError::Cancelled.to_string()); + } + assert_eq!(scan.polls(), 4, "the SQL callback must stop at the first cancelled poll"); + assert!(!is_open(&path), "a cancelled fresh validator must not populate the pool"); + + let clean = CancelAfterPolls::new(usize::MAX); + { + let _installed = crate::progress::install(Some(clean.clone())); + let (info, sample) = read_info_path(&path).unwrap(); + assert_eq!((info.min_zoom, info.max_zoom, info.format), (2, 7, super::super::Format::Png)); + assert_eq!(sample.as_deref(), Some(b"\x89PNG\r\n\x1a\n".as_slice())); + } + assert!(clean.polls() > complete.polls() + 3, "the fixed scan must cross several SQLite callback intervals"); + assert!(!is_open(&path), "a successful fresh validator must not populate the pool"); + // A request revoked only during SQL must also latch cancellation, not relabel the + // interruption as a damaged archive when its predicate is live again afterwards. + let request_polls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let polled = request_polls.clone(); + crate::source::remote::with_still( + std::sync::Arc::new(move || polled.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1 != 4), + || assert_eq!(read_info_path(&path).unwrap_err(), crate::DataError::Cancelled.to_string()), + ); + assert_eq!(request_polls.load(std::sync::atomic::Ordering::Relaxed), 4); + assert!(!is_open(&path)); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn candidate_already_cancelled_stops_before_opening_the_file() { + let path = std::env::temp_dir().join(format!("rustgis-mbtiles-cancel-missing-{}.mbtiles", std::process::id())); + let _ = std::fs::remove_file(&path); + let cancelled = CancelAfterPolls::new(1); + { + let _installed = crate::progress::install(Some(cancelled.clone())); + assert_eq!(read_info_path(&path).unwrap_err(), crate::DataError::Cancelled.to_string()); + } + assert_eq!(cancelled.polls(), 1); + assert!(!is_open(&path)); + assert!( + read_info_path(&path).unwrap_err().contains(&path.display().to_string()), + "a clean retry must report the real missing-file error" + ); + + let during_open = CancelAfterPolls::new(2); + { + let _installed = crate::progress::install(Some(during_open.clone())); + assert_eq!(read_info_path(&path).unwrap_err(), crate::DataError::Cancelled.to_string()); + } + assert_eq!(during_open.polls(), 2, "a failed open still checks cancellation at its error boundary"); + } + + #[test] + fn candidate_small_queries_check_cancellation_at_completion() { + let path = std::env::temp_dir().join(format!("rustgis-mbtiles-cancel-small-{}.mbtiles", std::process::id())); + let _ = std::fs::remove_file(&path); + let c = Connection::open(&path).unwrap(); + c.execute_batch( + "CREATE TABLE metadata(name TEXT,value TEXT); + CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + INSERT INTO metadata VALUES('format','pbf'),('minzoom','2'),('maxzoom','7');", + ) + .unwrap(); + drop(c); + // Neither short query reaches 1,000 VM instructions. Cancel after metadata and + // after the sample query must still refuse to publish a successful candidate. + for after in [2, 3] { + let cancelled = CancelAfterPolls::new(after); + let _installed = crate::progress::install(Some(cancelled)); + assert_eq!(read_info_path(&path).unwrap_err(), crate::DataError::Cancelled.to_string()); + assert!(!is_open(&path)); + } + let (info, sample) = read_info_path(&path).unwrap(); + assert_eq!((info.min_zoom, info.max_zoom), (2, 7)); + assert!(sample.is_none()); + std::fs::remove_file(path).unwrap(); + } + #[test] fn metadata_row_limit_is_explicit_and_retryable_for_both_readers() { let path = std::env::temp_dir().join(format!("rustgis-mbtiles-metadata-limit-{}.mbtiles", std::process::id())); diff --git a/crates/engine/src/cmd/layout.rs b/crates/engine/src/cmd/layout.rs index d579fbe647..6f9d8fa1ed 100644 --- a/crates/engine/src/cmd/layout.rs +++ b/crates/engine/src/cmd/layout.rs @@ -3501,6 +3501,18 @@ fn named_fonts_unread(layout: &Layout) -> Option<&'static str> { /// Layout `li` as a PDF (kept as vectors; every map series page when `all`) or a PNG of the /// current page, at `dpi`. pub fn layout_bytes(s: &mut Session, li: usize, pdf: bool, dpi: f64, all: bool) -> Result, String> { + let layout = s.project.layouts.get(li).ok_or("no layout")?; + let revisions = rustgis_render::layout::archive_revisions(layout, &s.project.maps); + let result = layout_bytes_inner(s, li, pdf, dpi, all); + let layout = s.project.layouts.get(li).ok_or("no layout")?; + revisions.finish_revision( + rustgis_render::layout::archive_revisions(layout, &s.project.maps), + Some(result), + "could not draw the page", + )? +} + +fn layout_bytes_inner(s: &mut Session, li: usize, pdf: bool, dpi: f64, all: bool) -> Result, String> { let saved = s.project.layouts.get(li).ok_or("no layout")?.clone(); if let Some(note) = named_fonts_unread(&saved) { s.warn(note); @@ -3550,13 +3562,11 @@ pub fn layout_bytes(s: &mut Session, li: usize, pdf: bool, dpi: f64, all: bool) // a memory budget. if pdf { let (w, h) = (layout.width, layout.height); - let (drawing, pictured) = rustgis_render::layout::render_drawing(layout, &mut s.project.maps, dpi, &info) - .ok_or("the page is too large to draw")?; + let (drawing, pictured) = rustgis_render::layout::render_drawing(layout, &mut s.project.maps, dpi, &info)?; if drawing.overflowed() { pages_pictured.push(page + 1); let layout = s.project.layouts.get(li).ok_or("no layout")?; - let img = - rustgis_render::layout::render(layout, &mut s.project.maps, dpi, &info).ok_or("the page is too large to draw")?; + let img = rustgis_render::layout::render(layout, &mut s.project.maps, dpi, &info)?; out.push(rustgis_render::pdf::Page::new(&img, w, h)); } else { frames_pictured.extend(pictured); @@ -3571,7 +3581,7 @@ pub fn layout_bytes(s: &mut Session, li: usize, pdf: bool, dpi: f64, all: bool) )); } } else { - let img = rustgis_render::layout::render(layout, &mut s.project.maps, dpi, &info).ok_or("the page is too large to draw")?; + let img = rustgis_render::layout::render(layout, &mut s.project.maps, dpi, &info)?; png = Some(img.encode_png().map_err(|e| e.to_string())?); } // A cancelled export stops here, its layout put back below. @@ -3680,8 +3690,7 @@ fn layout_svg(s: &mut Session, li: usize, dpi: f64) -> Result { }; // Rendering borrows the current layout; it does not step or modify the series page. let layout = s.project.layouts.get(li).ok_or("no layout")?; - let (text, pictured) = rustgis_render::svg::layout_svg(layout, &mut s.project.maps, dpi, &info) - .ok_or("could not draw the page; check its map sources and export size")?; + let (text, pictured) = rustgis_render::svg::layout_svg(layout, &mut s.project.maps, dpi, &info)?; pictured_notices(s, &pictured, &[]); Ok(text) } diff --git a/crates/engine/src/cmd/presentation.rs b/crates/engine/src/cmd/presentation.rs index cfcbf73994..4f1311e3e6 100644 --- a/crates/engine/src/cmd/presentation.rs +++ b/crates/engine/src/cmd/presentation.rs @@ -397,7 +397,7 @@ pub fn draw_page(s: &mut Session, i: usize, k: usize, dpi: f64) -> Result &str { /// The presentation as a PDF, a self-contained HTML slideshow or a PNG of its first page, and /// the number of pages written. pub fn export_bytes(s: &mut Session, i: usize, format: &str, dpi: f64) -> Result<(Vec, usize), String> { + let revision = |s: &Session| { + rustgis_render::archive_revisions(s.project.presentations.get(i).into_iter().flat_map(|p| { + p.pages + .iter() + .take(if format == "png" { 1 } else { usize::MAX }) + .filter_map(|page| if let PageKind::Map { map, .. } = &page.kind { Some(page_map(s, map)) } else { None }) + })) + }; + let before = revision(s); + let result = export_bytes_inner(s, i, format, dpi); + before.finish_revision(revision(s), Some(result), "could not draw the presentation")? +} + +fn export_bytes_inner(s: &mut Session, i: usize, format: &str, dpi: f64) -> Result<(Vec, usize), String> { // Only the size and name are copied: the pages can hold many megabytes of pictures, so each // is read in place after it's drawn. let (name, width, height, n) = diff --git a/crates/engine/src/cmd/report.rs b/crates/engine/src/cmd/report.rs index 2d5c36d9ca..3d453ee022 100644 --- a/crates/engine/src/cmd/report.rs +++ b/crates/engine/src/cmd/report.rs @@ -881,23 +881,43 @@ pub fn pages(s: &Session, i: usize) -> Result<(Doc, Vec Result<(Vec, usize), String> { let (doc, pages) = pages(s, i)?; + let revision = |s: &Session| { + pages + .iter() + .take(if pdf { usize::MAX } else { 1 }) + .filter_map(|page| { + let rustgis_render::report::Page::Layout(li) = page else { return None }; + s.project.layouts.get(*li).map(|layout| rustgis_render::layout::archive_revisions(layout, &s.project.maps)) + }) + .max() + .unwrap_or_default() + }; + let before = revision(s); + let result = report_pages_bytes(s, &doc, &pages, dpi, pdf); + before.finish_revision(revision(s), Some(result), "could not draw the report")? +} + +fn report_pages_bytes( + s: &mut Session, + doc: &Doc, + pages: &[rustgis_render::report::Page], + dpi: f64, + pdf: bool, +) -> Result<(Vec, usize), String> { let n = if pdf { pages.len() } else { 1 }; let mut out = Vec::new(); let mut held = 0usize; let (mut frames_pictured, mut pages_pictured) = (Vec::new(), Vec::new()); for k in 0..n { if !pdf { - let (img, ..) = rustgis_render::report::draw_page(&doc, &pages, k, dpi, &s.project.layouts, &mut s.project.maps) - .ok_or("the page is too large to draw")?; + let (img, ..) = rustgis_render::report::draw_page(doc, pages, k, dpi, &s.project.layouts, &mut s.project.maps)?; return Ok((img.encode_png().map_err(|e| e.to_string())?, pages.len())); } - let (page, pictured) = rustgis_render::report::pdf_page(&doc, &pages, k, dpi, &s.project.layouts, &mut s.project.maps) - .ok_or("the page is too large to draw")?; + let (page, pictured) = rustgis_render::report::pdf_page(doc, pages, k, dpi, &s.project.layouts, &mut s.project.maps)?; if page.overflowed() { // Too large for vectors all the same: the page as one picture. pages_pictured.push(k + 1); - let (img, w, h) = rustgis_render::report::draw_page(&doc, &pages, k, dpi, &s.project.layouts, &mut s.project.maps) - .ok_or("the page is too large to draw")?; + let (img, w, h) = rustgis_render::report::draw_page(doc, pages, k, dpi, &s.project.layouts, &mut s.project.maps)?; out.push(rustgis_render::pdf::Page::new(&img, w, h)); } else { frames_pictured.extend(pictured); diff --git a/crates/engine/src/cmd/share.rs b/crates/engine/src/cmd/share.rs index 63b8cb88d4..4d155d1e18 100644 --- a/crates/engine/src/cmd/share.rs +++ b/crates/engine/src/cmd/share.rs @@ -340,11 +340,17 @@ fn export_view( /// The map view on one PDF page (for printing): the page is the view's shape at 96 points per /// inch of screen, kept as vectors with what is pixels by nature drawn at `dpi`. pub fn map_pdf(s: &mut Session, dpi: f64) -> Result, String> { + let revisions = rustgis_render::archive_revisions([s.map()]); + let result = map_pdf_inner(s, dpi); + revisions.finish_revision(rustgis_render::archive_revisions([s.map()]), Some(result), "could not draw the map")? +} + +fn map_pdf_inner(s: &mut Session, dpi: f64) -> Result, String> { let (vw, vh) = s.view_size; let ppp = (dpi / 96.0).clamp(0.5, 6.0); let view = View { width: vw.clamp(16.0, 8000.0), height: vh.clamp(16.0, 8000.0), pixels_per_point: ppp, camera: s.map().camera, sky: false }; - let drawing = rustgis_render::render_drawing(s.map_mut(), &view).ok_or("could not draw the map")?; + let drawing = rustgis_render::render_drawing(s.map_mut(), &view)?; let mm = |pt: f64| pt * 25.4 / 96.0; let (w, h) = (mm(view.width), mm(view.height)); let page = if drawing.overflowed() { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index d58352a3a8..0d1e11abe9 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -34877,3 +34877,6 @@ fn bundled_ca_roots_are_said_once_a_session() { other.say_bundled_roots(true); assert_eq!(said(&other), 1); } + +#[cfg(not(target_arch = "wasm32"))] +mod archive_render_revision; diff --git a/crates/engine/src/tests/archive_render_revision.rs b/crates/engine/src/tests/archive_render_revision.rs new file mode 100644 index 0000000000..5ccac5f546 --- /dev/null +++ b/crates/engine/src/tests/archive_render_revision.rs @@ -0,0 +1,372 @@ +use crate::{data, map, render}; +use data::source::{Batch, FeatureSource, Request, SourceInfo}; +use data::{Dataset, Feature, Geometry, GeometryType}; +use map::{Camera, Map, TileKind, TileLayer, source::SourceLayer}; +use render::rustgis_tiles::{cache, pmtiles}; +use std::sync::{ + Arc, Barrier, + atomic::{AtomicBool, Ordering}, + mpsc, +}; + +struct Fixture(std::path::PathBuf); +impl Fixture { + fn new() -> Self { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + loop { + let path = std::env::temp_dir().join(format!( + "rustgis-render-revision-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + match std::fs::create_dir(&path) { + Ok(()) => return Self(path), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => panic!("creating {}: {e}", path.display()), + } + } + } + + fn path(&self) -> &std::path::Path { + &self.0 + } +} +impl Drop for Fixture { + fn drop(&mut self) { + data::mbtiles::close_unless(|path| !path.starts_with(&self.0)); + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// A real streamed layer between two drawings of the archive. Complete traversal pauses only +/// after the lower drawing has copied its old pixels and credits; the upper drawing has not run. +struct PauseSource { + info: SourceInfo, + ds: Dataset, + once: AtomicBool, + paused: mpsc::SyncSender<()>, + resume: Arc, +} +impl FeatureSource for PauseSource { + fn info(&self) -> &SourceInfo { + &self.info + } + fn read(&self, _: &Request) -> data::Result { + Ok(Batch { features: self.ds.features.to_vec(), matched: 1, complete: true, ..Default::default() }) + } + fn streams_chunks(&self) -> bool { + true + } + fn visit_chunks( + &self, + _: Option, + _: usize, + visit: &mut dyn FnMut(Vec) -> data::Result<()>, + ) -> data::Result { + if !self.once.swap(true, Ordering::SeqCst) { + self.paused.send(()).unwrap(); + self.resume.wait(); + } + visit(self.ds.features.to_vec())?; + Ok(Batch { matched: 1, complete: true, ..Default::default() }) + } +} + +fn archive(path: &std::path::Path, kind: TileKind, green: bool) { + let mut tile = tiny_skia::Pixmap::new(256, 256).unwrap(); + tile.fill(if green { tiny_skia::Color::from_rgba8(0, 220, 0, 255) } else { tiny_skia::Color::from_rgba8(220, 0, 0, 255) }); + let png = tile.encode_png().unwrap(); + let credits = if green { "New archive credits" } else { "Old archive credits" }; + if kind == TileKind::PmTiles { + let options = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Png, + gzipped_tiles: false, + bounds: [-180.0, -85.0, 180.0, 85.0], + metadata: serde_json::json!({"attribution": credits}), + }; + std::fs::write(path, pmtiles::write(vec![(0, 0, 0, png)], &options).unwrap()).unwrap(); + } else { + let db = rusqlite::Connection::open(path).unwrap(); + db.execute_batch("CREATE TABLE metadata(name TEXT,value TEXT); CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); INSERT INTO metadata VALUES('format','png'),('minzoom','0'),('maxzoom','0');").unwrap(); + db.execute("INSERT INTO metadata VALUES('attribution',?1)", [credits]).unwrap(); + db.execute("INSERT INTO tiles VALUES(0,0,0,?1)", [png]).unwrap(); + } +} + +fn paused_map(path: &std::path::Path, kind: TileKind) -> (Map, render::View, mpsc::Receiver<()>, Arc) { + let mut map = Map::new("Archive"); + map.graticule = false; + map.camera = Camera::new(data::Coord::new(0.0, 0.0), 100_000.0); + map.tiles.push(TileLayer::new("Below", kind, path.to_string_lossy())); + let view = render::View { width: 128.0, height: 128.0, pixels_per_point: 1.0, camera: map.camera, sky: false }; + let old = render::render_placed(&mut map, &view).unwrap(); + assert_eq!(old.credits, ["Old archive credits"]); + assert!(old.pixmap.pixel(8, 8).unwrap().red() > 200); + let mut ds = Dataset::new("Between", GeometryType::Polyline, Vec::new()); + ds.push(Some(Geometry::Line(vec![vec![data::Coord::new(0.0, 0.0), data::Coord::new(0.1, 0.1)]])), Vec::new()); + let info = SourceInfo { + name: ds.name.clone(), + format: "test", + geometry_type: ds.geometry_type, + fields: Vec::new(), + extent: ds.extent(), + count: Some(1), + crs: "EPSG:4326".into(), + indexed: true, + notes: Vec::new(), + zm_dropped: false, + }; + let id = map.add_dataset(ds.clone()); + let (paused, receive) = mpsc::sync_channel(1); + let resume = Arc::new(Barrier::new(2)); + map.layer_mut(id).unwrap().source = Some(SourceLayer::from_path( + "pause.shp", + None, + Box::new(PauseSource { info, ds, once: AtomicBool::new(false), paused, resume: resume.clone() }), + )); + let mut above = map.tiles[0].clone(); + above.name = "Above".into(); + above.over = Some(id); + map.tiles.insert(0, above); + (map, view, receive, resume) +} + +fn page(map: &Map) -> map::layout::Layout { + use map::layout::{ElementKind, FrameShape, Layout}; + let mut page = Layout::new("Archive", 40.0, 40.0); + page.add( + "Map", + (0.0, 0.0, 40.0, 40.0), + ElementKind::MapFrame { + map: map.name.clone(), + center: data::Coord::new(0.0, 0.0), + scale: 300_000_000.0, + shape: FrameShape::Rectangle, + grid: None, + extent_of: None, + extent_color: map::Rgba::rgb(0, 0, 0), + extent_width: 1.0, + bearing: 0.0, + pitch: 0.0, + }, + ); + page +} + +#[test] +fn archive_reload_refuses_spanning_exports_without_replacing_destination() { + for kind in [TileKind::PmTiles, TileKind::MbTiles] { + for route in 0..6 { + let dir = Fixture::new(); + let path = dir.path().join(format!("target.{}", kind.code())); + let replacement = dir.path().join(format!("replacement.{}", kind.code())); + archive(&path, kind, false); + archive(&replacement, kind, true); + let (mut map, view, paused, resume) = paused_map(&path, kind); + let mut warm = map.clone(); + warm.layers.clear(); + warm.tiles.truncate(1); + warm.tiles[0].over = None; + let destination = dir.path().join("map.png"); + std::fs::write(&destination, b"existing destination").unwrap(); + let output = destination.clone(); + let worker = std::thread::spawn(move || { + let result = match route { + 0 => render::render_png(&mut map, &view), + 1 => render::render_placed(&mut map, &view).and_then(|p| p.pixmap.encode_png().map_err(|e| e.to_string())), + 2 => render::render_drawing(&mut map, &view) + .map(|drawing| render::pdf::write(&[render::pdf::Page::drawing(drawing, 40.0, 40.0)], "Archive")), + 3 => render::layout::render(&page(&map), &mut [map], 96.0, &Default::default()) + .and_then(|p| p.encode_png().map_err(|e| e.to_string())), + 4 => render::layout::render_drawing(&page(&map), &mut [map], 96.0, &Default::default()) + .map(|(drawing, _)| render::pdf::write(&[render::pdf::Page::drawing(drawing, 40.0, 40.0)], "Archive")), + _ => render::svg::layout_svg(&page(&map), &mut [map], 96.0, &Default::default()).map(|(svg, _)| svg.into_bytes()), + }; + if let Ok(bytes) = &result { + data::io::write_atomic(&output, bytes).unwrap(); + } + result + }); + paused.recv_timeout(std::time::Duration::from_secs(30)).unwrap(); + data::mbtiles::close(&path); + std::fs::rename(&replacement, &path).unwrap(); + cache::reload_archive(path.to_str().unwrap()); + let fresh = render::render_placed(&mut warm, &view); + resume.wait(); + let result = worker.join().unwrap(); + let fresh = fresh.unwrap(); + assert_eq!(fresh.credits, ["New archive credits"]); + assert!(fresh.pixmap.pixel(8, 8).unwrap().green() > 200); + let error = result.unwrap_err(); + assert!(error.contains("archive changed during rendering") && error.contains("export again"), "{error}"); + assert_eq!(std::fs::read(&destination).unwrap(), b"existing destination"); + data::mbtiles::close(&path); + } + } +} + +#[test] +fn archive_revision_ignores_other_sources_and_tile_arrivals() { + let dir = Fixture::new(); + let path = dir.path().join("target.pmtiles"); + archive(&path, TileKind::PmTiles, false); + let mut map = Map::new("Revisions"); + map.tiles.push(TileLayer::new("Target", TileKind::PmTiles, path.to_string_lossy())); + let revision = render::archive_revisions([&map]); + cache::reload_archive("/render-revision-unrelated.mbtiles"); + assert!(revision.current([&map])); + let arrived_before = cache::generation(); + let view = render::View { width: 32.0, height: 32.0, pixels_per_point: 1.0, camera: map.camera, sky: false }; + render::render_placed(&mut map, &view).unwrap(); + assert_ne!(cache::generation(), arrived_before); + assert!(revision.current([&map])); + cache::reload_archive(&map.tiles[0].url); + assert!(!revision.current([&map])); +} + +#[test] +fn a_spanning_render_keeps_progressive_arrivals_and_unrelated_reloads() { + for kind in [TileKind::PmTiles, TileKind::MbTiles] { + let dir = Fixture::new(); + let target = dir.path().join(format!("target.{}", kind.code())); + let unrelated = dir.path().join(format!("other.{}", kind.code())); + archive(&target, kind, false); + archive(&unrelated, kind, true); + let (mut map, view, paused, resume) = paused_map(&target, kind); + let worker = std::thread::spawn(move || render::render_placed(&mut map, &view)); + paused.recv_timeout(std::time::Duration::from_secs(30)).unwrap(); + cache::reload_archive(unrelated.to_str().unwrap()); + let mut other = Map::new("Unrelated"); + other.tiles.push(TileLayer::new("Other", kind, unrelated.to_string_lossy())); + let arrived_before = cache::generation(); + let fresh = render::render_placed(&mut other, &view); + resume.wait(); + let result = worker.join().unwrap().unwrap(); + fresh.unwrap(); + assert_ne!(cache::generation(), arrived_before); + assert_eq!(result.credits, ["Old archive credits"]); + assert!(result.pixmap.pixel(8, 8).unwrap().red() > 200); + data::mbtiles::close(&target); + data::mbtiles::close(&unrelated); + } +} + +#[test] +fn archive_reload_discards_spanning_cpu_gpu_and_drape_passes() { + for kind in [TileKind::PmTiles, TileKind::MbTiles] { + for route in 0..4 { + let dir = Fixture::new(); + let path = dir.path().join(format!("target.{}", kind.code())); + let replacement = dir.path().join(format!("replacement.{}", kind.code())); + archive(&path, kind, false); + archive(&replacement, kind, true); + let (mut map, view, paused, resume) = paused_map(&path, kind); + let mut warm = map.clone(); + warm.layers.clear(); + warm.tiles.truncate(1); + warm.tiles[0].over = None; + let worker = std::thread::spawn(move || match route { + 0 => render::render_backend(&mut map, &view, render::Backend::Cpu).is_none(), + 1 => render::render_backend(&mut map, &view, render::Backend::Gpu).is_none(), + 2 => { + let view = render::View { camera: Camera { pitch: 30.0, ..view.camera }, ..view }; + render::render_tilt_drape(&mut map, &view, 0.0, 128).is_empty() + } + _ => render::render_globe_drape(&mut map, &view, &[data::Extent::new(-10.0, -10.0, 10.0, 10.0)], 128).is_empty(), + }); + paused.recv_timeout(std::time::Duration::from_secs(30)).unwrap(); + data::mbtiles::close(&path); + std::fs::rename(&replacement, &path).unwrap(); + cache::reload_archive(path.to_str().unwrap()); + let fresh = render::render_placed(&mut warm, &view); + resume.wait(); + let refused = worker.join().unwrap(); + fresh.unwrap(); + assert!(refused, "mixed archive revision escaped route {route} ({kind:?})"); + data::mbtiles::close(&path); + } + } +} + +#[test] +fn a_reload_between_series_pages_refuses_the_engine_pdf_without_replacing_destination() { + struct BetweenPages { + pages: std::sync::atomic::AtomicU64, + paused: mpsc::SyncSender<()>, + resume: Arc, + } + impl data::progress::Probe for BetweenPages { + fn cancelled(&self) -> bool { + false + } + fn begin(&self, _: u64) {} + fn advance(&self, n: u64) { + if self.pages.fetch_add(n, Ordering::SeqCst) == 0 { + self.paused.send(()).unwrap(); + self.resume.wait(); + } + } + } + + for kind in [TileKind::PmTiles, TileKind::MbTiles] { + let dir = Fixture::new(); + let path = dir.path().join(format!("target.{}", kind.code())); + let replacement = dir.path().join(format!("replacement.{}", kind.code())); + archive(&path, kind, false); + archive(&replacement, kind, true); + let mut map = Map::new("Archive"); + map.graticule = false; + map.camera = Camera::new(data::Coord::new(0.0, 0.0), 100_000.0); + map.tiles.push(TileLayer::new("Archive", kind, path.to_string_lossy())); + let mut index = Dataset::new("Pages", GeometryType::Point, Vec::new()); + for x in [0.0, 1.0] { + index.push(Some(Geometry::Point(data::Coord::new(x, 0.0))), Vec::new()); + } + map.add_dataset(index); + let layout = page(&map); + let mut warm = map.clone(); + warm.layers.clear(); + let view = render::View { width: 32.0, height: 32.0, pixels_per_point: 1.0, camera: map.camera, sky: false }; + let old = render::render_placed(&mut warm, &view).unwrap(); + assert_eq!(old.credits, ["Old archive credits"], "{:?}", old.gaps); + let mut session = crate::Session::new(crate::Project::default()); + session.project.maps = vec![map]; + session.project.layouts = vec![layout]; + session.active_layout = Some(0); + let series = session.run("layout.series", &serde_json::json!({"layer":"Pages"})).unwrap(); + assert_eq!(series["pages"], 2); + let destination = dir.path().join("series.pdf"); + std::fs::write(&destination, b"existing destination").unwrap(); + // The command's analysis wrapper installs its own progress carrier. Run its actual + // export work under this probe to pause exactly between complete pages, without a hook. + let work = crate::cmd::layout::export_work(&session, &serde_json::json!({"path":destination,"dpi":36,"all":true})).unwrap(); + let (paused, receive) = mpsc::sync_channel(1); + let resume = Arc::new(Barrier::new(2)); + let probe = Arc::new(BetweenPages { pages: std::sync::atomic::AtomicU64::new(0), paused, resume: resume.clone() }); + let source = path.clone(); + let reloaded = std::thread::spawn(move || { + receive.recv_timeout(std::time::Duration::from_secs(30)).unwrap(); + // The layout's page-progress callback runs after page one is complete and before + // page two captures its own render revision: only the whole-document guard spans this. + data::mbtiles::close(&source); + std::fs::rename(replacement, &source).unwrap(); + cache::reload_archive(source.to_str().unwrap()); + let fresh = render::render_placed(&mut warm, &view); + resume.wait(); + fresh + }); + let installed = data::progress::install(Some(probe.clone())); + let wrote = work().unwrap(); + drop(installed); + let fresh = reloaded.join().unwrap().unwrap(); + assert_eq!(fresh.credits, ["New archive credits"]); + assert!(fresh.pixmap.pixel(8, 8).unwrap().green() > 200); + assert_eq!(probe.pages.load(Ordering::SeqCst), 2, "both individually coherent pages were drawn"); + let error = wrote.then.unwrap()(&mut session, &mut serde_json::Value::Null).unwrap_err(); + assert!(error.contains("archive changed during rendering"), "{error}"); + assert_eq!(std::fs::read(&destination).unwrap(), b"existing destination"); + session.run("layout.export", &serde_json::json!({"path":destination,"dpi":36,"all":true})).unwrap(); + let pdf = std::fs::read(&destination).unwrap(); + assert!(pdf.starts_with(b"%PDF") && pdf.windows(b"/Count 2".len()).any(|part| part == b"/Count 2")); + } +} diff --git a/crates/render/src/gpu_scene.rs b/crates/render/src/gpu_scene.rs index bae92d1b33..d2cfcdd3d4 100644 --- a/crates/render/src/gpu_scene.rs +++ b/crates/render/src/gpu_scene.rs @@ -119,7 +119,8 @@ pub enum DrawCmd { /// One raster tile layer the GPU map places again for every frame it shows, from the tile /// cache, for the camera of that frame (see [`crate::tiles::live`]): the scene holds only /// which layer, so panning and zooming never draw the scene again for its tiles (#635). - LiveTiles { spec: rustgis_tiles::Spec, opacity: f32 }, + /// The archive revision binds later live tile acquisition to this scene's credits. + LiveTiles { spec: rustgis_tiles::Spec, opacity: f32, archive_revision: u64 }, /// One vector tile layer's tiles as meshes (see `tile_mesh`): the GPU keeps each mesh's /// buffers across frames, shows the layers of its runs that show at the camera's zoom and /// clips each tile to its square, as MapLibre draws its tiles. diff --git a/crates/render/src/layout.rs b/crates/render/src/layout.rs index 7e0dbdac26..f6b5455c2c 100644 --- a/crates/render/src/layout.rs +++ b/crates/render/src/layout.rs @@ -51,15 +51,33 @@ pub(crate) fn frame_requires_complete(map: &Map, view: &View) -> bool { }) } +/// Only maps placed in visible frames participate in this page's archive revision. +pub fn archive_revisions(layout: &Layout, maps: &[Map]) -> crate::ArchiveRevision { + crate::archive_revisions(layout.elements.iter().filter_map(|e| { + if !e.visible { + return None; + } + let ElementKind::MapFrame { map, .. } = &e.kind else { return None }; + map_index(maps, map).and_then(|i| maps.get(i)) + })) +} + /// Renders a layout at `dpi` (dots per inch). `maps` are the project's maps, found by name. -pub fn render(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Option { +/// Refuses a linked archive changed while composing the page, including its different map frames. +pub fn render(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Result { render_pending(layout, maps, dpi, info).map(|(pm, _)| pm) } /// [`render`], and whether a map frame on the page drew while some of its tiles, or a read of its /// streamed layers or rasters, were still on their way: in the browser, which doesn't wait for /// them, the page is drawn again once they land. -pub fn render_pending(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Option<(Pixmap, bool)> { +pub fn render_pending(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Result<(Pixmap, bool), String> { + let revisions = archive_revisions(layout, maps); + let result = render_pending_inner(layout, maps, dpi, info); + revisions.finish_revision(archive_revisions(layout, maps), result, "the page is too large to draw") +} + +fn render_pending_inner(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Option<(Pixmap, bool)> { let (w, h, k) = page_geometry(layout, dpi); let bounds = tiny_skia::IntRect::from_xywh(0, 0, w, h)?; let mut pending = false; @@ -78,6 +96,17 @@ pub fn render_drawing( maps: &mut [Map], dpi: f64, info: &PageInfo, +) -> Result<(crate::vector::Drawing, Vec<(u32, String)>), String> { + let revisions = archive_revisions(layout, maps); + let result = render_drawing_inner(layout, maps, dpi, info); + revisions.finish_revision(archive_revisions(layout, maps), result, "the page is too large to draw") +} + +fn render_drawing_inner( + layout: &Layout, + maps: &mut [Map], + dpi: f64, + info: &PageInfo, ) -> Option<(crate::vector::Drawing, Vec<(u32, String)>)> { let (w, h, k) = page_geometry(layout, dpi); let bounds = tiny_skia::IntRect::from_xywh(0, 0, w, h)?; @@ -435,9 +464,11 @@ fn draw_page( /// A map frame's map, `w` × `h` points at `ppp` pixels a point, labels and attribution included: /// what a frame shows before its grid, indicator and outline. It waits for the map's tiles, as a -/// page needs them all. -pub fn frame_map(map: &mut Map, camera: rustgis_map::Camera, w: f64, h: f64, ppp: f64) -> Option { - frame_canvas(map, camera, w, h, ppp, false, &mut false).map(Canvas::into_pixmap) +/// page needs them all. Refuses a linked archive changed during the complete frame. +pub fn frame_map(map: &mut Map, camera: rustgis_map::Camera, w: f64, h: f64, ppp: f64) -> Result { + let revisions = crate::archive_revisions([&*map]); + let result = frame_canvas(map, camera, w, h, ppp, false, &mut false).map(Canvas::into_pixmap); + revisions.finish([&*map], result, "could not draw the map frame") } /// [`frame_map`] on a canvas: one that records (see [`crate::canvas`]) when `record` and the diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 200c77f5db..94a8dc4f97 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -318,6 +318,7 @@ pub enum RenderOutput { /// [`render`]; `Backend::Gpu` draws what it can as tessellated meshes and everything else /// (basemap, tiles, rasters, masked/blended/transparent layers) as CPU-rasterized images in /// the same z-order. +/// A pass spanning a linked archive reload returns `None`; redraw against the new source. pub fn render_backend(map: &mut Map, view: &View, backend: Backend) -> Option { render_with_backend(map, view, false, backend) } @@ -1615,8 +1616,44 @@ pub fn view_lonlat(map: &Map, view: &View) -> Extent { if out.is_empty() { Extent::WORLD_LONLAT } else { out } } +/// A source-specific render revision, independent of ordinary tile arrivals. +#[derive(Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord)] +pub struct ArchiveRevision(u64); + +impl ArchiveRevision { + /// Checks against the same maps whose linked archives were captured. + pub fn current<'a>(self, maps: impl IntoIterator) -> bool { + self.0 == archive_revisions(maps).0 + } + + /// Refuses a spanning render before reporting any ordinary drawing failure. + pub(crate) fn finish<'a, T>(self, maps: impl IntoIterator, result: Option, failure: &str) -> Result { + self.finish_revision(archive_revisions(maps), result, failure) + } + + /// Checks a composed output against a newly captured revision of the same source set. + pub fn finish_revision(self, current: Self, result: Option, failure: &str) -> Result { + if self.0 != current.0 { + return Err("a tile archive changed during rendering; export again after the reload finishes".to_string()); + } + result.ok_or_else(|| failure.to_string()) + } +} + +/// Captures only linked tile archives used by these maps, before styles or tiles are read. +/// The token is allocation-free; retain the maps for a check at worker publication. +pub fn archive_revisions<'a>(maps: impl IntoIterator) -> ArchiveRevision { + ArchiveRevision(rustgis_tiles::cache::archive_revision(maps.into_iter().flat_map(|map| { + map.tiles + .iter() + .filter(|t| t.draws() && t.opacity > 0.0 && matches!(t.kind, rustgis_map::TileKind::PmTiles | rustgis_map::TileKind::MbTiles)) + .map(|t| t.url.as_str()) + }))) +} + /// Draws the map for the screen. Layers are drawn bottom-up (the last layer in the list first). /// On high-density screens rasters are computed at logical-pixel resolution, for speed. +/// A pass spanning a linked archive reload returns `None`, without publishing its mixed pixels. pub fn render(map: &mut Map, view: &View) -> Option { render_with(map, view, false) } @@ -1660,6 +1697,13 @@ fn render_with_backend(map: &mut Map, view: &View, full_res_rasters: bool, backe /// [`render_with_backend`], `record` keeping a flat, north-up frame as vectors where it can /// ([`render_frame`]). fn render_any(map: &mut Map, view: &View, full_res_rasters: bool, backend: Backend, record: bool) -> Option { + let revisions = archive_revisions([&*map]); + let output = render_pass(map, view, full_res_rasters, backend, record); + revisions.current([&*map]).then_some(output).flatten() +} + +/// One entire pass, including all bands and backend fallbacks, under `render_any`'s guard. +fn render_pass(map: &mut Map, view: &View, full_res_rasters: bool, backend: Backend, record: bool) -> Option { // A bearing or pitch set directly on the camera may be unnormalised (or no number, which // faces north and looks straight down). let mut camera = view.camera; @@ -4963,6 +5007,7 @@ pub struct DrapePatch { /// ([`rustgis_map::Globe::drape_mesh`]) while it turns, with no drawing per frame. Labels are /// left out (the view hides them while it moves). pub fn render_globe_drape(map: &mut Map, view: &View, boxes: &[Extent], max_side: u32) -> Vec { + let revisions = archive_revisions([&*map]); // A coarse drape may have far fewer pixels than points: it shows the map blurrier, but at // the view's scale. let ppp = if view.pixels_per_point.is_finite() { view.pixels_per_point.clamp(1.0 / 64.0, 4.0) } else { 1.0 }; @@ -5002,7 +5047,7 @@ pub fn render_globe_drape(map: &mut Map, view: &View, boxes: &[Extent], max_side map.label_display = display; map.label_lock = lock; map.label_lock_line_anchors = anchors; - out + if revisions.current([&*map]) { out } else { Vec::new() } } /// A band of a tilted view drawn flat, to lay over the tilted ground while the view moves. @@ -5020,6 +5065,7 @@ pub struct TiltPatch { /// band, square, over the circle round the view and without the margin: it is laid over the view /// the same way while it turns. pub fn render_tilt_drape(map: &mut Map, view: &View, margin: f64, max_side: u32) -> Vec { + let revisions = archive_revisions([&*map]); let ppp = if view.pixels_per_point.is_finite() { view.pixels_per_point.clamp(0.25, 4.0) } else { 1.0 }; let max_side = f64::from(max_side.clamp(64, 8192)); let cam = view.camera; @@ -5054,7 +5100,7 @@ pub fn render_tilt_drape(map: &mut Map, view: &View, margin: f64, max_side: u32) map.label_display = display; map.label_lock = lock; map.label_lock_line_anchors = anchors; - out + if revisions.current([&*map]) { out } else { Vec::new() } } /// A line's longest stretch in view, in points (`None` when none has two points in sight): the @@ -5739,7 +5785,14 @@ fn complete(map: &mut Map, view: &View, record: bool) -> Option { /// [`render_png`]'s map kept as vectors (for a vector PDF) where it can be: flat, north up, /// without view effects; anything else, and what is pixels by nature, as pictures. -pub fn render_drawing(map: &mut Map, view: &View) -> Option { +/// Returns an actionable error if a linked archive changes during the complete drawing. +pub fn render_drawing(map: &mut Map, view: &View) -> Result { + let revisions = archive_revisions([&*map]); + let result = render_drawing_inner(map, view); + revisions.finish([&*map], result, "could not draw the map") +} + +fn render_drawing_inner(map: &mut Map, view: &View) -> Option { let r = render_complete_recorded(map, view)?; let mut pm = match r.recorded { Some(c) => c, @@ -5759,7 +5812,9 @@ pub fn render_drawing(map: &mut Map, view: &View) -> Option { } /// Renders a map with its labels, a scale bar and a north arrow to PNG bytes. +/// Refuses an archive changed during rendering or encoding; a caller must not write a destination on error. pub fn render_png(map: &mut Map, view: &View) -> Result, String> { + let revisions = archive_revisions([&*map]); let r = render_exported(map, view)?; let mut pm = Canvas::from_pixmap(r.pixmap); for l in r.unplaced.iter().chain(&r.labels) { @@ -5767,7 +5822,8 @@ pub fn render_png(map: &mut Map, view: &View) -> Result, String> { } text::scale_bar(&mut pm, view, map.ground_res_at(&view.camera)); draw_attribution(&mut pm, &r.attribution, view.pixels_per_point as f32); - pm.into_pixmap().encode_png().map_err(|e| e.to_string()) + let png = pm.into_pixmap().encode_png().map_err(|e| e.to_string())?; + revisions.finish([&*map], Some(png), "could not encode the image") } /// A map drawn as data ([`render_placed`]), with what it holds besides its pixels. @@ -5787,6 +5843,7 @@ pub struct Placed { /// without the scale bar and the credits, which sit on the screen, not the ground. Refuses what /// [`render_png`] refuses. pub fn render_placed(map: &mut Map, view: &View) -> Result { + let revisions = archive_revisions([&*map]); let r = render_exported(map, view)?; let mut gaps = Vec::new(); if r.tiles_pending { @@ -5810,13 +5867,15 @@ pub fn render_placed(map: &mut Map, view: &View) -> Result { for l in r.unplaced.iter().chain(&r.labels) { text::draw(&mut pm, l, view.pixels_per_point as f32); } - Ok(Placed { pixmap: pm.into_pixmap(), credits: r.attribution, gaps }) + revisions.finish([&*map], Some(Placed { pixmap: pm.into_pixmap(), credits: r.attribution, gaps }), "could not draw the map") } /// The map drawn whole for an export, or why it can't be written: a raster read in place whose /// tiles haven't come, or that is drawn from less than the view needs. fn render_exported(map: &mut Map, view: &View) -> Result { - let r = render_complete(map, view).ok_or("could not allocate the image")?; + let revisions = archive_revisions([&*map]); + let result = render_complete(map, view); + let r = revisions.finish([&*map], result, "could not draw the map")?; // A raster read in place drawn from less than the view needs isn't written: its tiles still // on their way (the browser can't wait; natively the wait ran out), or a coarser level or // tiles it couldn't read (`partial`). diff --git a/crates/render/src/report.rs b/crates/render/src/report.rs index d47be67af3..883c2a58d2 100644 --- a/crates/render/src/report.rs +++ b/crates/render/src/report.rs @@ -267,15 +267,24 @@ const MAX_PIXELS: f64 = if cfg!(target_arch = "wasm32") { 25_000_000.0 } else { /// Draws page `n` (0-based) at `dpi`, returning it and its size in millimetres. `layouts` and /// `maps` are the project's, for layout pages. -pub fn draw_page(doc: &Doc, pages: &[Page], n: usize, dpi: f64, layouts: &[Layout], maps: &mut [Map]) -> Option<(Pixmap, f64, f64)> { - let page = pages.get(n)?; +pub fn draw_page( + doc: &Doc, + pages: &[Page], + n: usize, + dpi: f64, + layouts: &[Layout], + maps: &mut [Map], +) -> Result<(Pixmap, f64, f64), String> { + let page = pages.get(n).ok_or("no such report page")?; let dpi = if dpi.is_finite() { dpi.clamp(24.0, 600.0) } else { 96.0 }; if let Page::Layout(li) = page { - let l = layouts.get(*li)?; + let l = layouts.get(*li).ok_or("no such report layout")?; let info = crate::layout::PageInfo { page: n + 1, pages: pages.len(), page_name: String::new() }; return crate::layout::render(l, maps, dpi, &info).map(|pm| (pm, l.width, l.height)); } - flow_canvas(doc, pages, n, dpi, false).map(|(pm, w, h)| (pm.into_pixmap(), w, h)) + flow_canvas(doc, pages, n, dpi, false) + .map(|(pm, w, h)| (pm.into_pixmap(), w, h)) + .ok_or_else(|| "the page is too large to draw".to_string()) } /// Page `n` as a vector PDF page: a layout's as [`crate::layout::render_drawing`] keeps it, a @@ -288,17 +297,17 @@ pub fn pdf_page( dpi: f64, layouts: &[Layout], maps: &mut [Map], -) -> Option<(crate::pdf::Page, Vec<(u32, String)>)> { - let page = pages.get(n)?; +) -> Result<(crate::pdf::Page, Vec<(u32, String)>), String> { + let page = pages.get(n).ok_or("no such report page")?; let dpi = if dpi.is_finite() { dpi.clamp(24.0, 600.0) } else { 96.0 }; if let Page::Layout(li) = page { - let l = layouts.get(*li)?; + let l = layouts.get(*li).ok_or("no such report layout")?; let info = crate::layout::PageInfo { page: n + 1, pages: pages.len(), page_name: String::new() }; return crate::layout::render_drawing(l, maps, dpi, &info) .map(|(d, pictured)| (crate::pdf::Page::drawing(d, l.width, l.height), pictured)); } - let (pm, w, h) = flow_canvas(doc, pages, n, dpi, true)?; - pm.into_drawing().map(|d| (crate::pdf::Page::drawing(d, w, h), Vec::new())) + let (pm, w, h) = flow_canvas(doc, pages, n, dpi, true).ok_or("the page is too large to draw")?; + pm.into_drawing().map(|d| (crate::pdf::Page::drawing(d, w, h), Vec::new())).ok_or_else(|| "the page is too large to draw".to_string()) } /// Flowing page `n` on a canvas `dpi` dots an inch (recording when `record`), and its size in @@ -540,7 +549,7 @@ mod tests { let (pm, w, _) = draw_page(&d, &pages, 0, 72.0, &[], &mut []).unwrap(); assert_eq!(w, 215.9); assert!(pm.pixels().iter().any(|p| p.red() < 100), "the page has text"); - assert!(draw_page(&d, &pages, 10_000, 72.0, &[], &mut []).is_none()); + assert!(draw_page(&d, &pages, 10_000, 72.0, &[], &mut []).is_err()); } #[test] @@ -550,7 +559,7 @@ mod tests { assert!(!pages.is_empty() && pages.len() <= MAX_PAGES); let d = Doc { title: "Empty".into(), width: 100.0, height: 100.0, blocks: Vec::new() }; assert_eq!(paginate(&d).len(), 1); - assert!(draw_page(&d, &paginate(&d), 0, 1e9, &[], &mut []).is_some()); + assert!(draw_page(&d, &paginate(&d), 0, 1e9, &[], &mut []).is_ok()); } #[test] diff --git a/crates/render/src/stream_tests.rs b/crates/render/src/stream_tests.rs index 040523298b..3276be7640 100644 --- a/crates/render/src/stream_tests.rs +++ b/crates/render/src/stream_tests.rs @@ -554,7 +554,7 @@ fn layout_export_keeps_all_drawn_legend_classes() { } let mut failed = base.clone(); failed.layers[0] = streamed(&base.layers[0], 0); - assert!(layout::render(&page, &mut [failed], 96.0, &layout::PageInfo::default()).is_none()); + assert!(layout::render(&page, &mut [failed], 96.0, &layout::PageInfo::default()).is_err()); } #[test] @@ -1163,7 +1163,7 @@ fn layout_export_rejects_unopened_local_shapefiles_and_required_masks() { map.layer_mut(point).unwrap().mask = Some("MASK".into()); } assert!( - layout::render(&page, std::slice::from_mut(&mut map), 96.0, &layout::PageInfo::default()).is_none(), + layout::render(&page, std::slice::from_mut(&mut map), 96.0, &layout::PageInfo::default()).is_err(), "missing complete source must fail the page: pitch {pitch}, masked {masked}" ); assert!(map.layer(owner).unwrap().source.as_ref().unwrap().error.is_some()); @@ -1173,7 +1173,7 @@ fn layout_export_rejects_unopened_local_shapefiles_and_required_masks() { source.path = Some(dir.path().join("unused.shp").display().to_string()); unused.layer_mut(mask).unwrap().source = Some(source); assert!( - layout::render(&page, &mut [unused], 96.0, &layout::PageInfo::default()).is_some(), + layout::render(&page, &mut [unused], 96.0, &layout::PageInfo::default()).is_ok(), "an unused hidden Shapefile must not fail export" ); @@ -1182,7 +1182,7 @@ fn layout_export_rejects_unopened_local_shapefiles_and_required_masks() { source.path = Some(dir.path().join("missing.gpkg").display().to_string()); ordinary.layer_mut(point).unwrap().source = Some(source); assert!( - layout::render(&page, &mut [ordinary], 96.0, &layout::PageInfo::default()).is_some(), + layout::render(&page, &mut [ordinary], 96.0, &layout::PageInfo::default()).is_ok(), "ordinary sources retain upstream best-effort behavior" ); } diff --git a/crates/render/src/svg.rs b/crates/render/src/svg.rs index 42dde168c0..8a73a58055 100644 --- a/crates/render/src/svg.rs +++ b/crates/render/src/svg.rs @@ -17,8 +17,18 @@ use rustgis_map::layout::{ElementKind, FontFamily, Layout, LineHead, PT_PER_MM, use crate::layout::{MAX_PIXELS, PageInfo, expand, page_geometry, record_element, render_element}; /// Writes `layout` as an SVG document, pictures drawn at `dpi`, and the elements (id and name) -/// drawn as pictures because their vectors would pass the budget; `None` when the page can't be drawn. -pub fn layout_svg(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Option<(String, Vec<(u32, String)>)> { +/// drawn as pictures because their vectors would pass the budget. Refuses a source changed during drawing. +pub fn layout_svg(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Result<(String, Vec<(u32, String)>), String> { + let revisions = crate::layout::archive_revisions(layout, maps); + let result = layout_svg_inner(layout, maps, dpi, info); + revisions.finish_revision( + crate::layout::archive_revisions(layout, maps), + result, + "could not draw the page; check its map sources and export size", + ) +} + +fn layout_svg_inner(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Option<(String, Vec<(u32, String)>)> { let mut defs = vectors::Defs::default(); let mut pictured = Vec::new(); let mut body = elements(layout, maps, dpi, info, &mut defs, &mut pictured)?; @@ -438,7 +448,7 @@ mod tests { assert!(svg.contains(&format!(r#" Option { + let archive = matches!(spec.kind, Kind::PmTiles | Kind::MbTiles).then_some(spec.url.as_str()); + let revision = rustgis_tiles::cache::archive_revision(archive); + let output = live_inner(spec, opacity, view); + (revision == rustgis_tiles::cache::archive_revision(archive)).then_some(output).flatten() +} + +fn live_inner(spec: &Spec, opacity: f32, view: &View) -> Option { let res = view.camera.res; let ppp = view.pixels_per_point as f32; let frame = Frame { @@ -387,7 +394,12 @@ pub(crate) fn draw( && let Some(p) = rustgis_tiles::place_raster(&s, t.opacity, &frame, &fetch, &reach) && crate::flush_acc(pm, g.cmds, g.dirty).is_some() { - g.cmds.push(DrawCmd::LiveTiles { spec: s, opacity: t.opacity }); + let archive_revision = if matches!(s.kind, Kind::PmTiles | Kind::MbTiles) { + rustgis_tiles::cache::archive_revision([s.url.as_str()]) + } else { + 0 + }; + g.cmds.push(DrawCmd::LiveTiles { spec: s, opacity: t.opacity, archive_revision }); let d = rustgis_tiles::Drawn { labels: Vec::new(), attribution: p.attribution, pending: p.pending, error: p.error }; collect(d, t, &mut out); continue; @@ -1459,8 +1471,8 @@ pub(crate) mod tests { for done in std::mem::take(&mut *HELD.lock().unwrap()) { done(Ok(tiny_skia::Pixmap::new(256, 256).unwrap().encode_png().unwrap())); } - assert_eq!(short, Some(true), "drawn before its tiles landed"); + assert_eq!(short, Ok(true), "drawn before its tiles landed"); let whole = crate::layout::render_pending(&page, &mut maps, 72.0, &info).map(|(_, pending)| pending); - assert_eq!(whole, Some(false), "drawn once they have"); + assert_eq!(whole, Ok(false), "drawn once they have"); } } diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index a7379663e6..a37faff281 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -188,6 +188,16 @@ fn archive_epochs() -> &'static Mutex> { &E } +/// The most recent reload of just these archives. Reload stamps increase globally, so any +/// selected source changing increases this maximum; unrelated reloads and tile arrivals do not. +/// The caller retains the source paths and can compare again without allocating a snapshot. +pub fn archive_revision<'a>(archives: impl IntoIterator) -> u64 { + let mut archives = archives.into_iter(); + let Some(first) = archives.next() else { return 0 }; + let epochs = archive_epochs().lock().unwrap_or_else(|e| e.into_inner()); + std::iter::once(first).chain(archives).filter_map(|url| epochs.get(url).copied()).max().unwrap_or(0) +} + pub(crate) fn archive_epoch(url: &str) -> u64 { archive_epochs().lock().unwrap_or_else(|e| e.into_inner()).get(resource_of(url)).copied().unwrap_or(0) } @@ -776,8 +786,8 @@ pub fn forget(url: &str) { /// derived styles. Call only after uncached validation succeeds. Older answers are discarded. pub fn reload_archive(url: &str) { let mut epochs = archive_epochs().lock().unwrap_or_else(|e| e.into_inner()); - let epoch = epochs.entry(url.to_string()).or_default(); - *epoch = epoch.wrapping_add(1); + static NEXT: AtomicU64 = AtomicU64::new(1); + epochs.insert(url.to_string(), NEXT.fetch_add(1, Ordering::Relaxed)); lock().forget_archive(url); let abandoned = crate::pmtiles::reload_archive(url); crate::mbtiles::reload_archive(url); diff --git a/crates/ui-egui/src/layoutview.rs b/crates/ui-egui/src/layoutview.rs index 7b8a3ead88..6de19d22ce 100644 --- a/crates/ui-egui/src/layoutview.rs +++ b/crates/ui-egui/src/layoutview.rs @@ -655,8 +655,7 @@ pub fn show(app: &mut RustGisApp, ui: &mut Ui) { }); if (app.layout.key != Some(key) || due) && size[0] > 2 && size[1] > 2 { let info = page_info(app, li); - if let Some((pm, frames_pending)) = - rustgis_engine::render::layout::render_pending(&layout, &mut app.session.project.maps, dpi, &info) + if let Ok((pm, frames_pending)) = rustgis_engine::render::layout::render_pending(&layout, &mut app.session.project.maps, dpi, &info) { let img = egui::ColorImage::from_rgba_premultiplied([pm.width() as usize, pm.height() as usize], pm.data()); match &mut app.layout.texture { diff --git a/crates/ui-egui/src/mapgpu.rs b/crates/ui-egui/src/mapgpu.rs index 4156e94142..46bd308008 100644 --- a/crates/ui-egui/src/mapgpu.rs +++ b/crates/ui-egui/src/mapgpu.rs @@ -1012,6 +1012,8 @@ pub(crate) struct SceneBuffers { pub tilt: Option, /// Its live raster tile layers, in order, placed for each frame shown ([`LiveFrame`]). pub live: Vec<(rustgis_engine::render::rustgis_tiles::Spec, f32)>, + /// The archive revision of the scene that owns the live layers' attribution. + live_archive_revision: u64, /// Every tile layer of the scene is live: a tile arriving needs no new scene. pub tiles_live: bool, /// The farthest a vertex lies from its origin (map units, [`gpu_scene::reach`]): zoomed in @@ -1034,6 +1036,15 @@ impl SceneBuffers { if self.live.is_empty() { return None; } + let revision = || { + use rustgis_engine::render::rustgis_tiles::{Kind, cache}; + cache::archive_revision( + self.live.iter().filter_map(|(spec, _)| matches!(spec.kind, Kind::PmTiles | Kind::MbTiles).then_some(spec.url.as_str())), + ) + }; + if revision() != self.live_archive_revision { + return Some((LiveFrame { camera, layers: Vec::new() }, false, true)); + } let view = rustgis_engine::render::View { width: self.size[0], height: self.size[1], @@ -1056,6 +1067,9 @@ impl SceneBuffers { } }) .collect(); + if revision() != self.live_archive_revision { + return Some((LiveFrame { camera, layers: Vec::new() }, false, true)); + } Some((LiveFrame { camera, layers }, pending, declined)) } } @@ -1145,6 +1159,7 @@ impl SceneBuffers { let mut has_images = false; let mut has_tiles = false; let mut live = Vec::new(); + let mut live_archive_revision = 0; let tilt = scene.tilt.as_ref(); let mut tile_textures = r.tile_textures.lock().unwrap_or_else(|e| e.into_inner()); let mut tile_meshes = r.tile_meshes.lock().unwrap_or_else(|e| e.into_inner()); @@ -1338,9 +1353,10 @@ impl SceneBuffers { draws.push(SceneDraw::Image { bind, _texture: texture }); has_images = true; } - DrawCmd::LiveTiles { spec, opacity } => { + DrawCmd::LiveTiles { spec, opacity, archive_revision } => { draws.push(SceneDraw::LiveTiles { n: live.len() }); live.push((spec.clone(), *opacity)); + live_archive_revision = live_archive_revision.max(*archive_revision); } DrawCmd::Tiles { origin, tiles } => { let mut vertices = Vec::with_capacity(tiles.len() * 6); @@ -1422,7 +1438,19 @@ impl SceneBuffers { let tiles_live = !live.is_empty() && !has_images && !has_tiles && !scene.cmds.iter().any(|c| matches!(c, DrawCmd::TileMeshes { .. })); let reach = gpu_scene::reach(&scene.cmds); - Some(SceneBuffers { draws, clear: scene.clear, size, ppp, has_images, has_tiles, tilt: scene.tilt, live, tiles_live, reach }) + Some(SceneBuffers { + draws, + clear: scene.clear, + size, + ppp, + has_images, + has_tiles, + tilt: scene.tilt, + live, + tiles_live, + reach, + live_archive_revision, + }) } /// Rewrites every command's camera uniforms for a moved camera — pan/zoom without @@ -1685,6 +1713,32 @@ mod tests { .ok() } + #[test] + fn live_archive_pixels_cannot_outlive_their_scenes_credits() { + use rustgis_engine::{ + map::{TileKind, TileLayer}, + render, + }; + let layer = TileLayer::new("Archive", TileKind::PmTiles, "/live-scene-revision.pmtiles"); + let revision = render::rustgis_tiles::cache::archive_revision([layer.url.as_str()]); + let scene = SceneBuffers { + draws: Vec::new(), + clear: [0.0; 4], + size: [128.0, 128.0], + ppp: 1.0, + has_images: false, + has_tiles: false, + tilt: None, + live: vec![(render::tile_spec(&layer), 1.0)], + tiles_live: true, + reach: 0.0, + live_archive_revision: revision, + }; + render::rustgis_tiles::cache::reload_archive(&layer.url); + let (frame, pending, declined) = scene.live_frame(Camera::world()).unwrap(); + assert!(declined && !pending && frame.layers.is_empty(), "redraw the scene instead of placing new pixels under its old credits"); + } + #[test] fn commands_over_device_limits_are_declined() { let limits = wgpu::Limits { max_buffer_size: 1024, max_texture_dimension_2d: 2048, ..Default::default() }; diff --git a/crates/ui-egui/src/mapview.rs b/crates/ui-egui/src/mapview.rs index 69318fdda1..fd03d0ce15 100644 --- a/crates/ui-egui/src/mapview.rs +++ b/crates/ui-egui/src/mapview.rs @@ -931,12 +931,13 @@ fn main_view(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens) { if let Some((k, done)) = app.map.tilt_settle.take() { app.session.map_mut().adopt_caches(&done.map); // A frame of this view that couldn't be drawn isn't tried again: nothing is to fill. - if same_view(k, key) && done.rendered.is_none() { + if same_view(k, key) && done.rendered.is_none() && done.archives.current([&done.map]) { app.map.filling = false; } // One drawn before tiles or data arrived still shows; the next follows. So does one // of a change made since, for a view whose frame on screen is older still. if (same_view(k, key) || (tiled_flat && shows_finished(k, key, app.map.key))) + && done.archives.current([&done.map]) && let Some(r) = done.rendered { // With tile pictures still to draw, a draft: the next frame draws more. @@ -4240,6 +4241,25 @@ mod tile_redraw_tests { assert!(!keeps_frame(Some(key(3)), key(3), 40)); } + #[test] + fn an_eligible_finished_frame_cannot_cross_its_archive_reload() { + use rustgis_engine::{ + map::{Map, TileKind, TileLayer}, + render, + }; + let mut map = Map::new("Archive publication"); + map.tiles.push(TileLayer::new("Target", TileKind::PmTiles, "/finished-frame-revision.pmtiles")); + let archives = render::archive_revisions([&map]); + let (shown, done, now) = (Key { rev: 4, ..key(0) }, Key { rev: 6, ..key(0) }, Key { rev: 9, ..key(0) }); + let eligible = || shows_finished(done, now, Some(shown)) && archives.current([&map]); + assert!(eligible(), "ordinary progressive finished frames still show"); + render::rustgis_tiles::cache::reload_archive("/finished-frame-unrelated.mbtiles"); + assert!(eligible(), "another source does not revoke this worker's frame"); + render::rustgis_tiles::cache::reload_archive(&map.tiles[0].url); + assert!(shows_finished(done, now, Some(shown)), "the permissive revision fallback alone still accepts it"); + assert!(!eligible(), "archive coherence is also required at adoption"); + } + #[test] fn a_frame_of_an_earlier_change_shows_while_a_slider_is_dragged() { let (on_screen, done, now) = (Key { rev: 4, ..key(0) }, Key { rev: 6, ..key(0) }, Key { rev: 9, ..key(0) }); diff --git a/crates/ui-egui/src/reportpane.rs b/crates/ui-egui/src/reportpane.rs index 398f768a18..c45a092c8f 100644 --- a/crates/ui-egui/src/reportpane.rs +++ b/crates/ui-egui/src/reportpane.rs @@ -330,8 +330,7 @@ fn preview(app: &mut RustGisApp, ui: &mut Ui, i: usize, t: &Tokens) { let (s, page) = (&mut app.session, app.reports.page); let drawn = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> Result<_, String> { let page = page.min(pages.len().saturating_sub(1)); - let (pm, _, _) = rustgis_engine::render::report::draw_page(doc, pages, page, 60.0, &s.project.layouts, &mut s.project.maps) - .ok_or("the page is too large to draw")?; + let (pm, _, _) = rustgis_engine::render::report::draw_page(doc, pages, page, 60.0, &s.project.layouts, &mut s.project.maps)?; Ok((pm, page, pages.len())) })) .unwrap_or_else(|_| Err("the preview failed unexpectedly".into())); diff --git a/crates/ui-egui/src/tilt_drape.rs b/crates/ui-egui/src/tilt_drape.rs index 796a7ed906..d4b344574b 100644 --- a/crates/ui-egui/src/tilt_drape.rs +++ b/crates/ui-egui/src/tilt_drape.rs @@ -63,6 +63,7 @@ struct Drawn { relief: Option<(usize, Vec>)>, shapes: Vec, markers: Vec, + archives: render::ArchiveRevision, } /// The drape of the moving tilted map, kept between frames while the map doesn't change. @@ -188,7 +189,10 @@ impl TiltDrape { } /// Makes `drawn` the drape, its pictures textures (reusing the last drape's). - fn show(&mut self, ctx: &egui::Context, drawn: Drawn) { + fn show(&mut self, ctx: &egui::Context, drawn: Drawn, map: &Map) { + if !drawn.archives.current([map]) { + return; + } // Something to drape: `warm` keeps it up to date again. #[cfg(not(target_arch = "wasm32"))] { @@ -225,6 +229,7 @@ impl TiltDrape { if let Some(drawn) = self.take(map) && drawn.key == key && drawn.ppp == drape_ppp + && drawn.archives.current([&*map]) && self.drape.as_ref().is_none_or(|d| d.built <= drawn.built) { if drawn.patches.is_empty() { @@ -233,7 +238,7 @@ impl TiltDrape { self.drape = None; self.empty = Some((key, drawn.camera)); } else { - self.show(ctx, drawn); + self.show(ctx, drawn, map); } } if self.job.as_ref().is_some_and(|(k, _)| *k != key) { @@ -339,7 +344,7 @@ impl TiltDrape { self.drape = None; return false; } - self.show(ctx, drawn); + self.show(ctx, drawn, map); true } @@ -556,9 +561,21 @@ impl TiltDrape { fn draw(map: &mut Map, view: &View, max_side: u32, key: DrapeKey, tiles: u64, time: f64) -> Drawn { let cam = view.camera; let ppp = view.pixels_per_point as f32; + let archives = render::archive_revisions([&*map]); let patches = render::render_tilt_drape(map, view, MARGIN, max_side); if patches.is_empty() { - return Drawn { key, ppp, camera: cam, tiles, built: time, patches, relief: None, shapes: Vec::new(), markers: Vec::new() }; + return Drawn { + key, + ppp, + camera: cam, + tiles, + built: time, + patches, + relief: None, + shapes: Vec::new(), + markers: Vec::new(), + archives, + }; } // A turned flat map's band is a square over the circle round the view: what stands on it is // taken over all of it, as the view turns there. @@ -586,7 +603,7 @@ fn draw(map: &mut Map, view: &View, max_side: u32, key: DrapeKey, tiles: u64, ti .collect(); (n, heights) }); - Drawn { key, ppp, camera: cam, tiles, built: time, patches, relief, shapes, markers } + Drawn { key, ppp, camera: cam, tiles, built: time, patches, relief, shapes, markers, archives } } /// Adds a convex polygon (screen points, `None` where one couldn't be placed) to `mesh` as a fan @@ -1067,7 +1084,18 @@ mod tests { /// A drape with nothing in it, as a worker hands back for a view over no ground. fn nothing(key: DrapeKey, camera: Camera) -> Drawn { - Drawn { key, ppp: 1.0, camera, tiles: 0, built: 0.0, patches: Vec::new(), relief: None, shapes: Vec::new(), markers: Vec::new() } + Drawn { + key, + ppp: 1.0, + camera, + tiles: 0, + built: 0.0, + patches: Vec::new(), + relief: None, + shapes: Vec::new(), + markers: Vec::new(), + archives: Default::default(), + } } /// Prepares `d` until the drape being drawn is shown. diff --git a/crates/ui-egui/src/tilt_settle.rs b/crates/ui-egui/src/tilt_settle.rs index a1aedb1138..d65f5fbb11 100644 --- a/crates/ui-egui/src/tilt_settle.rs +++ b/crates/ui-egui/src/tilt_settle.rs @@ -21,6 +21,7 @@ pub(crate) struct Done { pub ms: u64, /// Tile pictures stand in for others, or are still to be drawn: it is drawn again. pub incomplete: bool, + pub archives: render::ArchiveRevision, } /// The frame being drawn (at most one at a time), and what it is for. @@ -47,6 +48,7 @@ impl Default for TiltSettle { fn draw(mut map: Map, view: View, backend: Backend) -> Done { let t0 = web_time::Instant::now(); + let archives = render::archive_revisions([&map]); // On the UI thread in the browser, a frame's worth of tile pictures at a time; on a worker // natively, all of them. #[cfg(target_arch = "wasm32")] @@ -57,7 +59,7 @@ fn draw(mut map: Map, view: View, backend: Backend) -> Done { }; #[cfg(not(target_arch = "wasm32"))] let (rendered, incomplete) = (render::render_backend(&mut map, &view, backend), false); - Done { rendered, view, map, ms: t0.elapsed().as_millis() as u64, incomplete } + Done { rendered, view, map, ms: t0.elapsed().as_millis() as u64, incomplete, archives } } impl TiltSettle { @@ -145,6 +147,7 @@ impl TiltSettle { let done = self.ready.take(); if let Some((k, d)) = &done && d.rendered.is_none() + && d.archives.current([&d.map]) { self.failed = Some(*k); } diff --git a/docs/user-guide/maps.md b/docs/user-guide/maps.md index f89e7490f1..7b8932defe 100644 --- a/docs/user-guide/maps.md +++ b/docs/user-guide/maps.md @@ -363,6 +363,8 @@ at a time. If its project, map, archive or presentation changes before it lands, left unapplied: retry on the intended target. A paused attached batch also rejects its remaining commands if the project or selected map changes. Attached MCP `run` scripts and file-bridge scripts validate inline, without reload status or cancellation during validation. +Cancelling MBTiles validation also interrupts SQLite queries used to infer missing zoom metadata. +It cannot preempt a filesystem operation blocked inside the operating system. Approval binds the selected archive's owning map, stack position and source route, not just its name or index. Retargeting or reordering it requires reviewing the operation again. @@ -374,6 +376,8 @@ It also checks file identity and change time where the platform exposes them, pl Metadata that cannot be read is an error, not permission to apply an unchecked result. These checks are not an immutable snapshot: a rewrite that leaves the same observations, or a write after the final check, may escape detection. Finish writing before reloading or repairing. +If an archive is reloaded during a map or layout export, the export refuses a mixed-version +result. Wait for the reload to finish and retry the export. Reload first checks that the file can be read. If it is missing, unreadable or invalid, a notice explains the failure and the existing layer and retained cached data are left unchanged. From 8fe3cf6660759da01b3b904dab4c27f3d6df5174 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Mon, 9 Nov 2026 18:00:04 +0100 Subject: [PATCH 11/15] Name Reload Tile Archive in the tile archive reload docs --- docs/user-guide/maps.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/user-guide/maps.md b/docs/user-guide/maps.md index 7b8932defe..ad244561c0 100644 --- a/docs/user-guide/maps.md +++ b/docs/user-guide/maps.md @@ -349,7 +349,8 @@ selectable or editable; a tile not yet loaded may need another click once it arr fetches them again. **Reloading a local tile archive (desktop).** After changing a PMTiles or MBTiles file on disk, -right-click its row in Layers and choose **Reload Data**. CLI, Python, MCP and the Assistant use +right-click its row in Layers and choose **Reload Data** (**Reload Tile Archive** in Command +Search). CLI, Python, MCP and the Assistant use `map.reloadTiles={"index":0}`; `"name"` or catalog `"id"` can select the archive instead. No replacement path is accepted. This is a manual action: RustGIS does not watch the file for changes. It rereads that archive's metadata and tiles without clearing other archives or web layers. From 09e959e3f20cff9c8f18a4c0f5aa4274bc408444 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Mon, 9 Nov 2026 21:57:43 +0100 Subject: [PATCH 12/15] Keep the MBTiles pool test valid across a concurrent pool clear --- crates/data/src/mbtiles.rs | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/crates/data/src/mbtiles.rs b/crates/data/src/mbtiles.rs index c3d12c9b0e..4b5e5477d5 100644 --- a/crates/data/src/mbtiles.rs +++ b/crates/data/src/mbtiles.rs @@ -800,6 +800,9 @@ mod file { ) .unwrap(); } + // Another test's new path can clear the process-global pool when 16 are tracked: the + // other file is only expected to survive `close` if no clear happened around it. + let generation = pool().lock().unwrap().generation; assert_eq!(tile_path(&other, 0, 0, 0).unwrap(), Some(vec![1])); let (started_tx, started_rx) = std::sync::mpsc::channel(); let (release_tx, release_rx) = std::sync::mpsc::channel(); @@ -815,7 +818,26 @@ mod file { }); started_rx.recv_timeout(std::time::Duration::from_secs(10)).unwrap(); close(&path); - assert!(pool().lock().unwrap().connections.get(&other).is_some_and(|v| !v.is_empty())); + let mut kept = { + let all = pool().lock().unwrap(); + (all.generation == generation).then(|| all.connections.get(&other).is_some_and(|v| !v.is_empty())) + }; + for _ in 0..8 { + if kept.is_some() { + break; + } + // The pool was cleared by a concurrent test: seed both files again and close + // `path` once more, so the retry still shows `close` drops only its own file. + let generation = pool().lock().unwrap().generation; + assert_eq!(tile_path(&other, 0, 0, 0).unwrap(), Some(vec![1])); + assert_eq!(tile_path(&path, 0, 0, 0).unwrap(), Some(vec![1])); + close(&path); + let all = pool().lock().unwrap(); + kept = (all.generation == generation).then(|| { + all.connections.get(&other).is_some_and(|v| !v.is_empty()) && all.connections.get(&path).is_none_or(Vec::is_empty) + }); + } + assert_eq!(kept, Some(true), "closing one file must keep another file's connection"); release_tx.send(()).unwrap(); assert_eq!(worker.join().unwrap(), vec![1]); assert!( From f57ea79814f416d5cb0d3e91ba10f823c16ff22d Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Mon, 9 Nov 2026 21:57:43 +0100 Subject: [PATCH 13/15] Observe MBTiles sidecars beside both the link and its target --- crates/engine/src/cmd/repair.rs | 54 ++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/crates/engine/src/cmd/repair.rs b/crates/engine/src/cmd/repair.rs index 0a339a19d9..09572c9c7a 100644 --- a/crates/engine/src/cmd/repair.rs +++ b/crates/engine/src/cmd/repair.rs @@ -544,12 +544,13 @@ struct FileRevision { #[cfg(not(target_arch = "wasm32"))] impl FileRevision { - fn read(path: &str, optional: bool) -> Result, String> { + fn read(path: &std::path::Path, optional: bool) -> Result, String> { + let shown = path.display(); let metadata = match std::fs::metadata(path) { Ok(metadata) => metadata, Err(e) if optional && e.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(e) => { - return Err(format!("{path}: could not observe archive metadata ({e}); finish writing or restore the file and retry")); + return Err(format!("{shown}: could not observe archive metadata ({e}); finish writing or restore the file and retry")); } }; // SQLite creates an empty -wal even for a read-only open of a clean WAL-mode archive, @@ -558,11 +559,11 @@ impl FileRevision { return Ok(None); } if !metadata.is_file() { - return Err(format!("{path}: expected an archive file; restore the file and retry")); + return Err(format!("{shown}: expected an archive file; restore the file and retry")); } let modified = metadata .modified() - .map_err(|e| format!("{path}: could not observe the archive modification time ({e}); check file access and retry"))?; + .map_err(|e| format!("{shown}: could not observe the archive modification time ({e}); check file access and retry"))?; #[cfg(unix)] use std::os::unix::fs::MetadataExt; #[cfg(windows)] @@ -584,17 +585,30 @@ struct ArchiveObservation { primary: Option, // SQLite commits can change these without changing the main database. The -shm file // is reader coordination, not content, and is deliberately not observed. - sqlite: Option<(Option, Option)>, + sqlite: Option>>, } #[cfg(not(target_arch = "wasm32"))] impl ArchiveObservation { fn read(path: &str) -> Result { let path = path.strip_prefix("file://").unwrap_or(path); - let primary = FileRevision::read(path, false)?; + let primary = FileRevision::read(std::path::Path::new(path), false)?; let sqlite = std::path::Path::new(path).extension().is_some_and(|e| e.eq_ignore_ascii_case("mbtiles")); let sqlite = if sqlite { - Some((FileRevision::read(&format!("{path}-wal"), true)?, FileRevision::read(&format!("{path}-journal"), true)?)) + // Unix SQLite names its sidecars from the resolved database path, so commits through + // a link land beside the target; Windows names them from the path as given. Observe + // beside both when they differ, so neither VFS's commits escape. + let given = std::path::Path::new(path); + let resolved = std::fs::canonicalize(given).ok().filter(|resolved| resolved != given); + let mut sidecars = Vec::new(); + for base in std::iter::once(given).chain(resolved.as_deref()) { + for suffix in ["-wal", "-journal"] { + let mut sidecar = base.as_os_str().to_owned(); + sidecar.push(suffix); + sidecars.push(FileRevision::read(std::path::Path::new(&sidecar), true)?); + } + } + Some(sidecars) } else { None }; @@ -735,4 +749,30 @@ mod tests { } std::fs::remove_dir_all(dir).unwrap(); } + + #[cfg(unix)] + #[test] + fn linked_mbtiles_observes_the_sidecars_beside_its_target_and_beside_the_link() { + let dir = std::env::temp_dir().join(format!("rustgis-repair-linked-observation-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let target = dir.join("target.mbtiles"); + std::fs::write(&target, b"main database").unwrap(); + let link = dir.join("link.mbtiles"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + let link = link.display().to_string(); + let observed = ArchiveObservation::read(&link).unwrap(); + observed.unchanged(&link).expect("nothing was committed"); + // SQLite on unix commits beside the target; on Windows beside the path as given. + for beside in [target.display().to_string(), link.clone()] { + for sidecar in ["-wal", "-journal"] { + let sidecar = format!("{beside}{sidecar}"); + std::fs::write(&sidecar, b"committed frame").unwrap(); + assert!(observed.unchanged(&link).is_err(), "a commit to {sidecar} escaped the freshness check"); + std::fs::remove_file(&sidecar).unwrap(); + observed.unchanged(&link).unwrap(); + } + } + std::fs::remove_dir_all(dir).unwrap(); + } } From 287d4902807d06588d45f2d1f35ba5b4c6151623 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Mon, 9 Nov 2026 21:57:43 +0100 Subject: [PATCH 14/15] Answer and end relay archive batches when reads stop at exit --- crates/assistant/src/mcp.rs | 9 +-- crates/assistant/src/relay.rs | 19 +++++- crates/ui-egui/src/readjob.rs | 119 ++++++++++++++++++++++++++++++++-- 3 files changed, 135 insertions(+), 12 deletions(-) diff --git a/crates/assistant/src/mcp.rs b/crates/assistant/src/mcp.rs index 3839ad221c..2f809613d0 100644 --- a/crates/assistant/src/mcp.rs +++ b/crates/assistant/src/mcp.rs @@ -129,14 +129,15 @@ impl ArchiveBatch { self.finish_pending(s, result); self.advance(s) } - /// A changed project or map invalidates the suffix, not the command's actual result. - /// Complete every remaining request without dispatching even a read-only tool. - pub(crate) fn cancel_remaining(mut self, s: &mut Session, result: Result) -> ArchiveStep { + /// A changed project or map invalidates the suffix, not the command's actual result; so does + /// the app closing. Complete every remaining request with `remaining` as its error, without + /// dispatching even a read-only tool. + pub(crate) fn cancel_remaining(mut self, s: &mut Session, result: Result, remaining: &str) -> ArchiveStep { self.finish_pending(s, result); for msg in self.messages.by_ref() { match request_identity(&msg) { Ok(Some((_, id))) => { - self.replies.push(error_value(&id, -32000, "archive batch context changed; remaining request cancelled")); + self.replies.push(error_value(&id, -32000, remaining)); } Ok(None) => {} Err(reply) => self.replies.push(reply), diff --git a/crates/assistant/src/relay.rs b/crates/assistant/src/relay.rs index 9c2f1b4740..7ed789e261 100644 --- a/crates/assistant/src/relay.rs +++ b/crates/assistant/src/relay.rs @@ -748,14 +748,29 @@ impl Relay { /// Completes the real archive command before running its batch suffix or issuing its reply /// and audit. A suffix may resume only in the project and map selected at its latest pause. pub fn finish_archive_read(&self, s: &mut Session, result: Result) { + self.complete_archive_read(s, result, None); + } + + /// Ends the admitted archive command for good because RustGIS is closing: `why` answers it + /// and every remaining request of its message, none of which runs, and the slot is cleared. + pub fn abandon_archive_read(&self, s: &mut Session, why: String) { + self.complete_archive_read(s, Err(why.clone()), Some(&why)); + } + + /// [`Relay::finish_archive_read`], or with `abandon` set the suffix is cancelled with that error. + fn complete_archive_read(&self, s: &mut Session, result: Result, abandon: Option<&str>) { let Some(archive) = lock(&self.owner.shared.archive).take() else { return }; let Archive { req, batch, started, opened, map } = archive; // Check before entering the agent: its remembered layer ID is only map-local, and a // stale continuation must not change the current actor, selection or stored agent state. - let same_context = s.opened == opened && s.map().instance_id() == map; + let same_context = abandon.is_none() && s.opened == opened && s.map().instance_id() == map; let layers = same_context.then(|| self.enter_agent(s, &req.token)); let step = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - if same_context { (*batch).finish(s, result) } else { (*batch).cancel_remaining(s, result) } + if same_context { + (*batch).finish(s, result) + } else { + (*batch).cancel_remaining(s, result, abandon.unwrap_or("archive batch context changed; remaining request cancelled")) + } })) .unwrap_or_else(|_| crate::mcp::ArchiveStep::Done(Some(internal_error(&req.line)), false)); let (out, picked) = match step { diff --git a/crates/ui-egui/src/readjob.rs b/crates/ui-egui/src/readjob.rs index 7c6a44b6c1..bce1107b97 100644 --- a/crates/ui-egui/src/readjob.rs +++ b/crates/ui-egui/src/readjob.rs @@ -470,13 +470,32 @@ pub fn land(app: &mut RustGisApp) -> bool { } /// Cancels the read under way, and what waits, and waits up to `wait` for its thread to stop. +/// Each one that never ran is told so, the one under way first and then those waiting in the +/// order asked: its `then`, and a relay's agent, whose batch is ended unrun rather than resumed. pub fn stop(app: &mut RustGisApp, wait: std::time::Duration) { - app.reads.waiting.clear(); - let Some(job) = app.reads.job.take() else { return }; - job.control.cancel(); - let end = std::time::Instant::now() + wait; - while job.slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner).is_none() && std::time::Instant::now() < end { - std::thread::sleep(std::time::Duration::from_millis(10)); + let why = tl!("RustGIS is closing").to_string(); + let job = app.reads.job.take(); + let waiting: Vec = app.reads.waiting.drain(..).collect(); + if let Some(job) = job { + job.control.cancel(); + let end = std::time::Instant::now() + wait; + while job.slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner).is_none() && std::time::Instant::now() < end { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + abandon_asked(app, &job.asked, why.clone()); + } + for asked in &waiting { + abandon_asked(app, asked, why.clone()); + } +} + +/// `asked` will never run: its `then` is told, and a relay's agent is answered with `why` for the +/// command and everything after it in its message, none of which runs. +fn abandon_asked(app: &mut RustGisApp, asked: &Asked, why: String) { + not_run(app, asked.then, why.clone()); + if let Some(relay) = &asked.relay { + relay.begin_archive_landing(&app.session); + relay.abandon_archive_read(&mut app.session, why); } } @@ -870,6 +889,94 @@ mod tests { let _ = std::fs::remove_dir_all(dir); } + thread_local! { + static TOLD: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; + } + + fn record(tag: &str, landed: Landed) { + TOLD.with(|told| { + told.borrow_mut().push(match landed { + Landed::NotRun(why) => format!("{tag}: {why}"), + Landed::Ran(_) => format!("{tag}: ran"), + }); + }); + } + + fn told_running(_: &mut RustGisApp, landed: Landed) { + record("running", landed); + } + + fn told_waiting(_: &mut RustGisApp, landed: Landed) { + record("waiting", landed); + } + + #[test] + fn stopping_ends_the_relay_batch_unrun_and_tells_the_reads_that_never_ran_in_order() { + use std::io::{BufRead, BufReader, Write}; + let _turn = crate::fetching::turn(); + TOLD.with(|told| told.borrow_mut().clear()); + let dir = std::env::temp_dir().join(format!("rustgis-relay-read-stop-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let original = dir.join("original.pmtiles"); + let replacement = dir.join("replacement.pmtiles"); + let csv = dir.join("first.csv"); + archive(&original, "Original"); + archive(&replacement, "Replacement"); + std::fs::write(&csv, "name,x,y\nA,8,47\n").unwrap(); + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + app.run("map.addTiles", json!({"url":original,"name":"Linked","overlay":true})).unwrap(); + app.reads_off_frame = true; + // One read under way and one waiting, each with a callback. + assert_eq!(run_then(&mut app, "layer.addData", json!({"path":csv}), Some(told_running)).unwrap()["started"], "Add Data"); + assert_eq!(run_then(&mut app, "layer.addData", json!({"path":csv}), Some(told_waiting)).unwrap()["waiting"], 1); + // The agent's batch: two archive commands, then a command that changes the map. + let batch = json!([ + relay_command(1, "map.repairTiles", json!({"index":0,"path":replacement})), + relay_command(2, "map.reloadTiles", json!({"index":0})), + relay_command(3, "layer.template", json!({"template":"points","name":"Mutated"})), + ]); + let (relay, _grant, client, ahead) = paused_relay(&mut app, batch); + admitted(&mut app, ahead, relay.clone()); + assert_eq!(app.reads.waiting.len(), 2); + let layers = app.session.map().layers.len(); + // User work said after admission belongs to the user, not to the agent's closing error. + app.session.notify("Unrelated user notice after admission".to_string(), false); + stop(&mut app, Duration::from_secs(10)); + assert!(!busy(&app)); + let why = tl!("RustGIS is closing").to_string(); + assert_eq!(TOLD.with(|told| told.borrow().clone()), [format!("running: {why}"), format!("waiting: {why}")]); + let mut client = BufReader::new(client); + let mut line = String::new(); + client.read_line(&mut line).unwrap(); + let reply: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(reply[0]["result"]["isError"], true, "{reply}"); + assert!(!reply[0].to_string().contains("Unrelated user notice"), "the agent's error carries only its own notices: {reply}"); + for rest in [1, 2] { + assert_eq!(reply[rest]["error"]["message"], why.as_str(), "{reply}"); + } + assert_eq!(app.session.map().tiles[0].url, original.to_str().unwrap(), "the repair never ran"); + assert_eq!(app.session.map().layers.len(), layers, "the batch's later commands never ran"); + assert!(app.session.map().layer_by_name("Mutated").is_none()); + assert!(relay.take_archive_read().is_none(), "the second archive command was never admitted"); + // The relay is not left holding the batch: the agent's next call is served. + writeln!(client.get_mut(), "{}", relay_command(4, "map.tiles", json!({}))).unwrap(); + client.get_mut().flush().unwrap(); + let end = std::time::Instant::now() + Duration::from_secs(10); + // Short waits, so each wait for the reply is followed by another poll. + client.get_ref().set_read_timeout(Some(Duration::from_millis(20))).unwrap(); + line.clear(); + let next = loop { + relay.poll(&mut app.session); + match client.read_line(&mut line) { + Ok(n) if n > 0 => break serde_json::from_str::(&line).unwrap(), + _ => assert!(std::time::Instant::now() < end, "the relay never answered the next call"), + } + }; + assert_eq!(next["result"]["isError"], false, "{next}"); + relay.stop(); + let _ = std::fs::remove_dir_all(dir); + } + #[test] fn archive_reload_lands_once_and_stale_or_cancelled_reads_keep_retained_metadata() { let _turn = crate::fetching::turn(); From 33e17dcbbe05b780ac9079bc48ef2f45bdabe0f4 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Tue, 10 Nov 2026 06:28:17 +0100 Subject: [PATCH 15/15] Say where archive reload and repair validate inline --- crates/engine/src/cmd/repair.rs | 10 ++++++++++ docs/mcp.md | 2 ++ docs/user-guide/maps.md | 5 +++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/engine/src/cmd/repair.rs b/crates/engine/src/cmd/repair.rs index 1702c48fa2..e36009ddc0 100644 --- a/crates/engine/src/cmd/repair.rs +++ b/crates/engine/src/cmd/repair.rs @@ -481,6 +481,11 @@ fn compatible( Ok(()) } +/// `map.repairTiles`: relinks a local PMTiles or MBTiles archive after validating the new file. +/// The desktop's menus and direct attached MCP calls validate it on the read queue's worker +/// ([`super::reads`]) and land the prepared result here. Called any other way (a notebook cell, a +/// task step, a script, the CLI), it validates inline on the caller's thread, like Add Data and +/// Reload Data, because those callers run their commands in order and the next may depend on it. fn repair_tiles(s: &mut Session, p: &Value) -> CmdResult { #[cfg(target_arch = "wasm32")] { @@ -687,6 +692,11 @@ fn archive_candidate_after_read( Ok(candidate.layer) } +/// `map.reloadTiles`: rereads a local PMTiles or MBTiles archive at its linked path. The desktop's +/// menus and direct attached MCP calls validate it on the read queue's worker ([`super::reads`]) +/// and land the prepared result here. Called any other way (a notebook cell, a task step, a +/// script, the CLI), it validates inline on the caller's thread, like Add Data and Reload Data, +/// because those callers run their commands in order and the next may depend on it. fn reload_tiles(s: &mut Session, p: &Value) -> CmdResult { #[cfg(target_arch = "wasm32")] { diff --git a/docs/mcp.md b/docs/mcp.md index 297a58f153..c6fd0cf8e8 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -137,6 +137,8 @@ later entries in the same JSON-RPC batch and later relay requests wait too. Canc the caller or changing the original archive target prevents installation. The original agent still owns the resulting undo step. Arbitrary `run` scripts and file-bridge scripts retain their existing synchronous execution contract; this does not move their child commands to the worker. +Notebook cells, task steps and Script pane scripts validate reload and repair inline on the window +thread, without reload status or cancellation during validation. File and network commands need `--yes`, unless **Let your Agents Control RustGIS** is ticked. Without **Full access**, an attached agent cannot replace the open project: diff --git a/docs/user-guide/maps.md b/docs/user-guide/maps.md index ad244561c0..6762b615a0 100644 --- a/docs/user-guide/maps.md +++ b/docs/user-guide/maps.md @@ -362,8 +362,9 @@ On the desktop, menu actions and direct attached MCP `run_command` calls validat repair on a background worker; the status bar shows the work and **Cancel**. Only one read runs at a time. If its project, map, archive or presentation changes before it lands, the result is left unapplied: retry on the intended target. A paused attached batch also rejects its remaining -commands if the project or selected map changes. Attached MCP `run` scripts and file-bridge -scripts validate inline, without reload status or cancellation during validation. +commands if the project or selected map changes. Notebook cells, task steps, scripts (the Script +pane and attached MCP `run`) and file-bridge scripts validate inline on the window thread, without +reload status or cancellation during validation. Cancelling MBTiles validation also interrupts SQLite queries used to infer missing zoom metadata. It cannot preempt a filesystem operation blocked inside the operating system.