diff --git a/Cargo.toml b/Cargo.toml index 2dd1b53d94..d93fa91f35 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,7 @@ arrow-schema = { version = "60", default-features = false } bytes = "1" flatgeobuf = { version = "6", default-features = false } geozero = { version = "0.15", default-features = false, features = ["with-wkb"] } -rusqlite = { version = "0.41", features = ["bundled", "functions"] } +rusqlite = { version = "0.41", features = ["bundled", "functions", "hooks"] } # PostgreSQL/PostGIS, in pure Rust: the client, and TLS through rustls (ring, as ureq has it). postgres = { version = "0.19", default-features = false } tokio-postgres-rustls = { version = "0.14", default-features = false, features = ["ring"] } diff --git a/ROADMAP.md b/ROADMAP.md index 7000ca22d9..97851dc022 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -57,7 +57,8 @@ Correct, robust and recoverable, with the foundations other themes stand on. GDA94↔GDA2020 and OSGB36, and a notice whenever a layer is read without its shift; GDA94 ([#730](https://github.com/opengeos/RustGIS/issues/730)) is one case. _Coordinate first._ - Reloading files changed on disk ([#1477](https://github.com/opengeos/RustGIS/issues/1477)): - manual Reload Data for linked layers; automatic detection and notifications remain open. + manual Reload Data for linked layers and local PMTiles/MBTiles archives; automatic detection and + notifications remain open. - Autosave and crash recovery in the browser: reopen unsaved work after a crash. The desktop app has them ([#873](https://github.com/opengeos/RustGIS/issues/873)). - Signed Windows builds (no SmartScreen warning), then a signature check, by publisher name, in diff --git a/crates/assistant/src/mcp.rs b/crates/assistant/src/mcp.rs index 14473fc241..2f809613d0 100644 --- a/crates/assistant/src/mcp.rs +++ b/crates/assistant/src/mcp.rs @@ -88,6 +88,132 @@ pub(crate) fn request_key(root: &str, position: usize) -> String { format!("{root}/{position}") } +/// A relay message paused at an admitted standalone archive command. Batch positions and +/// one-use permits remain owned here; no suffix entry runs until the actual command lands. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) struct ArchiveBatch { + server: Server, + scope: Scope, + permits: Vec<(String, Permit)>, + request: String, + messages: std::vec::IntoIter, + position: usize, + batch: bool, + replies: Vec, + pending: Option<(Value, tools::CommandCompletion)>, + ahead: Option, + /// Successful explicit selections in the current uninterrupted execution segment. + picked: bool, +} +#[cfg(not(target_arch = "wasm32"))] +pub(crate) enum ArchiveStep { + Done(Option, bool), + Read(Box), +} +#[cfg(not(target_arch = "wasm32"))] +impl ArchiveBatch { + pub(crate) fn take_read(&mut self) -> Option { + self.ahead.take() + } + pub(crate) fn picked(&self) -> bool { + self.picked + } + /// The command's reply reports changes made from the moment it actually runs, not from its + /// admission: the queue may run user work or other reads first. + pub(crate) fn begin_landing(&mut self, s: &Session) { + if let Some((_, completion)) = &mut self.pending { + *completion = tools::CommandCompletion::capture(s); + } + } + pub(crate) fn finish(mut self, s: &mut Session, result: Result) -> ArchiveStep { + self.finish_pending(s, result); + self.advance(s) + } + /// A changed project or map invalidates the suffix, not the command's actual result; so does + /// the app closing. Complete every remaining request with `remaining` as its error, without + /// dispatching even a read-only tool. + pub(crate) fn cancel_remaining(mut self, s: &mut Session, result: Result, remaining: &str) -> ArchiveStep { + self.finish_pending(s, result); + for msg in self.messages.by_ref() { + match request_identity(&msg) { + Ok(Some((_, id))) => { + self.replies.push(error_value(&id, -32000, remaining)); + } + Ok(None) => {} + Err(reply) => self.replies.push(reply), + } + } + self.done() + } + fn finish_pending(&mut self, s: &mut Session, result: Result) { + self.picked = false; + if let Some((id, completion)) = self.pending.take() { + self.replies.push(archive_reply(id, completion.finish(s, result))); + } + } + fn advance(mut self, s: &mut Session) -> ArchiveStep { + while let Some(msg) = self.messages.next() { + let request = request_key(&self.request, self.position); + self.position += 1; + let archive = (|| { + let id = msg.get("id")?; + if !matches!(id, Value::String(_) | Value::Number(_)) || msg.get("method").and_then(Value::as_str) != Some("tools/call") { + return None; + } + let (name, args) = tool_request(msg.get("params")?).ok()?; + if name != "run_command" + || !matches!(args.get("command").and_then(Value::as_str).map(str::trim), Some("map.reloadTiles" | "map.repairTiles")) + { + return None; + } + Some((id.clone(), ToolCall { id: request.clone(), name: name.into(), args })) + })(); + if let Some((id, call)) = archive { + let permit = self.permits.iter().position(|(key, _)| key == &request).map(|i| self.permits.remove(i).1); + let completion = tools::CommandCompletion::capture(s); + let admitted = tools::policy_call(&call) + .and_then(|call| call.ok_or_else(|| "missing archive adapter".to_string())) + .and_then(|call| s.start_scoped_archive_read(self.scope.clone(), &call, permit)); + match admitted { + Ok(ahead) => { + self.pending = Some((id, completion)); + self.ahead = Some(ahead); + return ArchiveStep::Read(Box::new(self)); + } + Err(error) => self.replies.push(archive_reply(id, completion.finish(s, Err(error)))), + } + } else if let Some(reply) = self.server.handle_value(s, &msg, &self.scope, &mut self.permits, &request) { + if reply.get("error").is_none() && reply.pointer("/result/isError") == Some(&Value::Bool(false)) { + self.picked |= msg.get("params").is_some_and(|params| { + let Some(name) = params.get("name").and_then(Value::as_str) else { return false }; + params.get("arguments").is_some_and(|args| tools::selects_layer(name, args)) + }); + } + self.replies.push(reply); + } + } + self.done() + } + fn done(mut self) -> ArchiveStep { + let reply = if self.replies.is_empty() { + None + } else if self.batch { + Some(Value::Array(self.replies).to_string()) + } else { + self.replies.pop().map(|reply| reply.to_string()) + }; + ArchiveStep::Done(reply, self.picked) + } +} +#[cfg(not(target_arch = "wasm32"))] +fn archive_reply(id: Value, result: Result) -> Value { + let (text, failed) = match result { + Ok(text) => (text, false), + Err(error) => (error, true), + }; + json!({"jsonrpc":"2.0","id":id,"result":{"content":[{"type":"text","text":text}],"isError":failed}}) +} + impl Server { /// A server with the built-in skills. `rustgis-cli mcp` and the desktop's relay replace them /// with the skills switched on (`skills::installed`): the user's, the plugins' and shared ones. @@ -100,6 +226,38 @@ impl Server { pub fn handle(&self, s: &mut Session, message: &str) -> Option { self.handle_scoped(s, message, self.scope(s), Vec::new(), "") } + #[cfg(not(target_arch = "wasm32"))] + /// Desktop relay ingress; only direct archive commands pause. Scripts and other transports + /// retain their synchronous contract, and malformed messages keep normal JSON-RPC handling. + pub(crate) fn start_archive_batch( + self, + s: &mut Session, + message: &str, + scope: Scope, + permits: Vec<(String, Permit)>, + request: &str, + ) -> ArchiveStep { + let parsed = serde_json::from_str::(message); + let (messages, batch) = match parsed { + Ok(Value::Array(batch)) if !batch.is_empty() && batch.len() <= MAX_BATCH => (batch, true), + Ok(msg) if !msg.is_array() => (vec![msg], false), + _ => return ArchiveStep::Done(self.handle_scoped(s, message, scope, permits, request), false), + }; + ArchiveBatch { + server: self, + scope, + permits, + request: request.into(), + messages: messages.into_iter(), + position: 0, + batch, + replies: Vec::new(), + pending: None, + ahead: None, + picked: false, + } + .advance(s) + } /// Named relay/file-bridge ingress; permits are engine plans stored when approval was shown. pub(crate) fn handle_scoped( &self, @@ -145,22 +303,12 @@ impl Server { permits: &mut Vec<(String, Permit)>, request: &str, ) -> Option { - let Some(obj) = msg.as_object() else { - return Some(error_value(&Value::Null, INVALID_REQUEST, "expected a JSON-RPC object")); - }; - let id = obj.get("id").cloned(); - let Some(method) = obj.get("method").and_then(Value::as_str) else { - // A response to a request of ours (we send none), or junk. - return id - .filter(|_| !obj.contains_key("result") && !obj.contains_key("error")) - .map(|id| error_value(&id, INVALID_REQUEST, "expected a method")); + let (method, id) = match request_identity(msg) { + Ok(Some(request)) => request, + Ok(None) => return None, + Err(reply) => return Some(reply), }; - // Notifications (no id) get no reply. - let id = id?; - if !matches!(id, Value::String(_) | Value::Number(_)) { - return Some(error_value(&Value::Null, INVALID_REQUEST, "id must be a string or a number")); - } - let params = obj.get("params").cloned().unwrap_or(Value::Null); + let params = msg.get("params").cloned().unwrap_or(Value::Null); Some(match self.call(s, method, ¶ms, scope, permits, request) { Ok(result) => json!({"jsonrpc": "2.0", "id": id, "result": result}), Err((code, message)) => error_value(&id, code, &message), @@ -537,6 +685,27 @@ pub fn expects_reply(message: &str) -> bool { } } +/// Applies JSON-RPC identity rules without executing a request. +fn request_identity(msg: &Value) -> Result, Value> { + let Some(obj) = msg.as_object() else { + return Err(error_value(&Value::Null, INVALID_REQUEST, "expected a JSON-RPC object")); + }; + let id = obj.get("id").cloned(); + let Some(method) = obj.get("method").and_then(Value::as_str) else { + // A response to a request of ours (we send none), or junk. + return match id.filter(|_| !obj.contains_key("result") && !obj.contains_key("error")) { + Some(id) => Err(error_value(&id, INVALID_REQUEST, "expected a method")), + None => Ok(None), + }; + }; + // Notifications (no id) get no reply. + let Some(id) = id else { return Ok(None) }; + if !matches!(id, Value::String(_) | Value::Number(_)) { + return Err(error_value(&Value::Null, INVALID_REQUEST, "id must be a string or a number")); + } + Ok(Some((method, id))) +} + /// Projects generated RPC error text while retaining the exact correlation id and code. fn error_value(id: &Value, code: i64, message: &str) -> Value { let message = rustgis_engine::render::rustgis_tiles::logins::scrub_error(message); diff --git a/crates/assistant/src/relay.rs b/crates/assistant/src/relay.rs index bd0b685539..7ed789e261 100644 --- a/crates/assistant/src/relay.rs +++ b/crates/assistant/src/relay.rs @@ -196,6 +196,8 @@ struct Shared { /// The screen snapshot the app is capturing ([`Relay::screen_request`]); nothing else runs /// until it is done. shot: Mutex>, + /// A desktop archive command holds its complete message, including any ordered batch suffix. + archive: Mutex>, } /// What the relay keeps for one agent between its calls. @@ -228,6 +230,15 @@ struct Shot { wait: Duration, } +struct Archive { + req: Request, + batch: Box, + started: Instant, + /// The project and selected map at this pause, after the preceding batch segment ran. + opened: u64, + map: u64, +} + impl Shared { /// Wakes the UI thread so it draws a frame to run queued messages or repaint the client /// count. Does nothing when the app never set a callback with [`Relay::set_wake`]. @@ -251,6 +262,9 @@ impl Shared { if let Some(shot) = lock(&self.shot).take() { shot.req.state.store(CANCELLED, SeqCst); } + if let Some(archive) = lock(&self.archive).take() { + archive.req.state.store(CANCELLED, SeqCst); + } // Pending requests are cancelled before they are dropped, so none can run after stop. for req in lock(&self.requests).drain(..) { let _ = req.state.compare_exchange(PENDING, CANCELLED, SeqCst, SeqCst); @@ -592,6 +606,7 @@ impl Relay { agents: Mutex::new(HashMap::new()), screen: AtomicBool::new(false), shot: Mutex::new(None), + archive: Mutex::new(None), }); let owner = Arc::new(Owner { shared, port, audit: Mutex::new(None), last: Mutex::new(None) }); let accepting = Arc::clone(&owner.shared); @@ -716,6 +731,69 @@ impl Relay { let _ = req.reply.send(Some(reply)); self.owner.shared.notify(); } + /// Takes the already-admitted archive read once. The desktop uses its single read worker; + /// dropping or recreating the invocation is never a native redispatch. + pub fn take_archive_read(&self) -> Option { + lock(&self.owner.shared.archive).as_mut()?.batch.take_read() + } + + /// Starts the reply's change report at the moment the admitted command actually runs (or is + /// known never to run), so work the queue ran meanwhile is not attributed to the agent. + pub fn begin_archive_landing(&self, s: &Session) { + if let Some(archive) = lock(&self.owner.shared.archive).as_mut() { + archive.batch.begin_landing(s); + } + } + + /// Completes the real archive command before running its batch suffix or issuing its reply + /// and audit. A suffix may resume only in the project and map selected at its latest pause. + pub fn finish_archive_read(&self, s: &mut Session, result: Result) { + self.complete_archive_read(s, result, None); + } + + /// Ends the admitted archive command for good because RustGIS is closing: `why` answers it + /// and every remaining request of its message, none of which runs, and the slot is cleared. + pub fn abandon_archive_read(&self, s: &mut Session, why: String) { + self.complete_archive_read(s, Err(why.clone()), Some(&why)); + } + + /// [`Relay::finish_archive_read`], or with `abandon` set the suffix is cancelled with that error. + fn complete_archive_read(&self, s: &mut Session, result: Result, abandon: Option<&str>) { + let Some(archive) = lock(&self.owner.shared.archive).take() else { return }; + let Archive { req, batch, started, opened, map } = archive; + // Check before entering the agent: its remembered layer ID is only map-local, and a + // stale continuation must not change the current actor, selection or stored agent state. + let same_context = abandon.is_none() && s.opened == opened && s.map().instance_id() == map; + let layers = same_context.then(|| self.enter_agent(s, &req.token)); + let step = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if same_context { + (*batch).finish(s, result) + } else { + (*batch).cancel_remaining(s, result, abandon.unwrap_or("archive batch context changed; remaining request cancelled")) + } + })) + .unwrap_or_else(|_| crate::mcp::ArchiveStep::Done(Some(internal_error(&req.line)), false)); + let (out, picked) = match step { + crate::mcp::ArchiveStep::Read(batch) => { + if let Some(layers) = layers { + self.leave_agent(s, &req.token, layers, batch.picked()); + } + *lock(&self.owner.shared.archive) = Some(Archive { req, batch, started, opened: s.opened, map: s.map().instance_id() }); + self.owner.shared.notify(); + return; + } + crate::mcp::ArchiveStep::Done(out, picked) => (out, picked), + }; + if let Some(layers) = layers { + self.leave_agent(s, &req.token, layers, picked); + } + self.audit(&req.line, out.as_deref(), started.elapsed()); + if let Some(action) = action(&req.line, out.as_deref()) { + *lock(&self.owner.last) = Some(action); + } + let _ = req.reply.send(out); + self.owner.shared.notify(); + } /// The last tool call an attached agent ran, if any has run since the relay started. pub fn last_action(&self) -> Option { @@ -788,13 +866,14 @@ impl Relay { Ok(grant) } - /// Runs the messages that arrived since the last call, on `s`. Returns how many ran. The - /// calls go through [`Session::run`], so each is one undo step as everywhere else. A message - /// whose connection already gave up is skipped, never run. + /// Runs queued messages on `s`, returning how many completed. Direct archive commands pause + /// at admission for the app's existing read queue: take_archive_read and finish_archive_read + /// carry them through their real completion, holding their batch suffix and later messages. + /// Commands retain ordinary undo bookkeeping; abandoned queued requests never run. pub fn poll(&self, s: &mut Session) -> usize { *lock(&self.owner.shared.last_poll) = Instant::now(); // One request at a time: a screen snapshot being captured holds the rest back. - if lock(&self.owner.shared.shot).is_some() { + if lock(&self.owner.shared.shot).is_some() || lock(&self.owner.shared.archive).is_some() { return 0; } let pending: Vec = lock(&self.owner.shared.requests).drain(..).collect(); @@ -952,11 +1031,21 @@ impl Relay { if reviewed { std::mem::take(&mut req.plans).into_iter().map(|(id, p)| (id, p.allow())).collect() } else { Vec::new() }; let started = Instant::now(); let out = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - server.handle_scoped(s, &req.line, scope, permits, &req.id.to_string()) + server.start_archive_batch(s, &req.line, scope, permits, &req.id.to_string()) })) - .unwrap_or_else(|_| Some(internal_error(&req.line))); - // A layer.select that failed (an unknown layer, a refusal) chose nothing. - let picked = selects_layer(&req.line) && reply_ok(out.as_deref()); + .unwrap_or_else(|_| crate::mcp::ArchiveStep::Done(Some(internal_error(&req.line)), false)); + let (out, picked) = match out { + crate::mcp::ArchiveStep::Done(out, picked) => (out, picked), + crate::mcp::ArchiveStep::Read(batch) => { + self.leave_agent(s, &req.token, layers, batch.picked()); + *lock(&self.owner.shared.archive) = Some(Archive { req, batch, started, opened: s.opened, map: s.map().instance_id() }); + let mut queue = lock(&self.owner.shared.requests); + for rest in pending.rev() { + queue.push_front(rest); + } + return ran; + } + }; self.leave_agent(s, &req.token, layers, picked); self.audit(&req.line, out.as_deref(), started.elapsed()); if let Some(a) = action(&req.line, out.as_deref()) { @@ -1027,11 +1116,9 @@ pub(crate) fn reply_ok(reply: Option<&str>) -> bool { pub(crate) fn selects_layer(line: &str) -> bool { let Ok(msg) = serde_json::from_str::(line) else { return false }; let messages = msg.as_array().map(Vec::as_slice).unwrap_or_else(|| std::slice::from_ref(&msg)); - messages.iter().filter_map(|m| m.get("params").and_then(|p| p.get("arguments"))).any(|a| { - a.get("command").and_then(Value::as_str) == Some("layer.select") - || a.get("script").and_then(Value::as_str).is_some_and(|s| { - rustgis_engine::cmd::workflows::script_commands(s).is_ok_and(|lines| lines.iter().any(|(_, id, _)| id == "layer.select")) - }) + messages.iter().filter_map(|m| m.get("params")).any(|params| { + let Some(name) = params.get("name").and_then(Value::as_str) else { return false }; + params.get("arguments").is_some_and(|args| crate::tools::selects_layer(name, args)) }) } @@ -1557,6 +1644,10 @@ impl Revocation for GrantLive { } } +#[cfg(test)] +#[path = "relay/archive_tests.rs"] +mod archive_tests; + #[cfg(test)] #[allow(clippy::unwrap_used, clippy::panic)] mod tests { diff --git a/crates/assistant/src/relay/archive_tests.rs b/crates/assistant/src/relay/archive_tests.rs new file mode 100644 index 0000000000..8cd74f3ab6 --- /dev/null +++ b/crates/assistant/src/relay/archive_tests.rs @@ -0,0 +1,573 @@ +//! Real archive reads across the relay's approval, worker and completion boundaries. +#![allow(clippy::unwrap_used, clippy::panic)] + +use super::*; +use rustgis_engine::{ + Project, + cmd::reads, + render::rustgis_tiles::{mvt, pmtiles}, +}; +use serde_json::json; +use std::sync::mpsc::Receiver; + +fn archive(path: &std::path::Path, credit: &str) { + let metadata = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Mvt, + gzipped_tiles: false, + bounds: [-180.0, -85.0, 180.0, 85.0], + metadata: json!({"attribution":credit}), + }; + let bytes = pmtiles::write(vec![(0, 0, 0, mvt::encode(&mvt::Tile { layers: Vec::new() }))], &metadata).unwrap(); + std::fs::write(path, bytes).unwrap(); +} + +fn fixture() -> (tempfile::TempDir, Session, PathBuf, PathBuf) { + let root = tempfile::tempdir().unwrap(); + let original = root.path().join("original.pmtiles"); + let replacement = root.path().join("replacement.pmtiles"); + archive(&original, "Original"); + archive(&replacement, "Replacement"); + let mut session = Session::new(Project::default()); + session.run("map.addTiles", &json!({"url":original,"name":"Linked","overlay":true})).unwrap(); + (root, session, original, replacement) +} + +fn command(id: u64, name: &str, params: Value) -> Value { + json!({"jsonrpc":"2.0","id":id,"method":"tools/call","params":{"name":"run_command","arguments":{"command":name,"params":params}}}) +} + +fn queue(relay: &Relay, grant: &Grant, message: Value) -> (u64, Receiver>) { + let (reply, rx) = sync_channel(1); + let id = relay.owner.shared.next_id.fetch_add(1, SeqCst); + lock(&relay.owner.shared.requests).push_back(Request { + plans: Vec::new(), + approval_required: false, + held: Arc::default(), + id, + decision: Arc::new(AtomicU8::new(PENDING)), + line: message.to_string(), + token: grant.token.clone(), + asked: true, + reply, + state: Arc::new(AtomicU8::new(PENDING)), + }); + (id, rx) +} + +fn worker(ahead: reads::ReadAhead) -> Result { + std::thread::spawn(move || ahead.run(&rustgis_engine::analysis::control::Control::stoppable())).join().unwrap() +} + +fn finish(relay: &Relay, session: &mut Session, done: Result) { + relay.begin_archive_landing(session); + let result = done.and_then(|read| reads::read_land(session, read)); + relay.finish_archive_read(session, result); +} + +fn reply(rx: &Receiver>) -> Value { + serde_json::from_str(&rx.recv_timeout(Duration::from_secs(5)).unwrap().unwrap()).unwrap() +} + +#[test] +fn scoped_archive_reply_reports_only_changes_made_when_it_lands() { + let (_root, mut s, _, replacement) = fixture(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let (_, rx) = queue(&relay, &grant, command(1, "map.repairTiles", json!({"index":0,"path":replacement}))); + assert_eq!(relay.poll(&mut s), 0); + let done = worker(relay.take_archive_read().unwrap()); + // Work the queue ran between admission and landing belongs to its own caller. + s.run("layer.addSample", &json!({"name":"countries"})).unwrap(); + s.notify("Unrelated queued notice", false); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer["result"]["isError"], false, "{answer}"); + let text = answer["result"]["content"][0]["text"].as_str().unwrap(); + assert!(!text.contains("layersAdded") && !text.contains("Countries"), "{text}"); + assert!(!text.contains("Unrelated queued notice"), "{text}"); + assert!(text.contains("replacement") || text.contains("Repaired"), "{text}"); + relay.stop(); +} + +#[test] +fn scoped_archive_capture_rejects_scripts_other_commands_and_nested_ingress() { + use rustgis_engine::policy::Call; + let (_root, mut s, _, _) = fixture(); + for call in [Call::script("map.reloadTiles {\"index\":0}"), Call::command("map.tiles", json!({}))] { + assert!(s.start_scoped_archive_read(Scope::peer(), &call, None).is_err()); + } + let call = Call::command("map.reloadTiles", json!({"index":0})); + let nested = s.with_scope(Scope::peer(), |s| s.start_scoped_archive_read(Scope::peer(), &call, None)).unwrap(); + assert!(nested.is_err(), "a nested owner must not create a new pre-handler escape"); +} + +#[test] +fn direct_allowed_archive_reload_keeps_its_scoped_owner_without_a_permit() { + let (_root, mut s, original, _) = fixture(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let retained = pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap(); + archive(&original, "Allowed scoped reload"); + let (_, rx) = queue(&relay, &grant, command(1, "map.reloadTiles", json!({"index":0}))); + assert_eq!(relay.poll(&mut s), 0); + assert!(grant.pending().is_empty()); + let done = worker(relay.take_archive_read().unwrap()); + assert!(rx.try_recv().is_err()); + assert!(Arc::ptr_eq(&retained, &pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap())); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer["result"]["isError"], false, "{answer}"); + assert!(!Arc::ptr_eq(&retained, &pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap())); + relay.stop(); +} + +#[test] +fn reviewed_direct_archive_repair_waits_for_real_completion_and_keeps_original_actor_and_undo() { + let (root, mut s, original, replacement) = fixture(); + let relay = Relay::start(0).unwrap(); + let log = root.path().join("audit"); + relay.set_audit_dir(Some(log.clone())); + let grant = relay.grant_reviewed(false).unwrap(); + let depth = s.undo_depth(); + let (id, rx) = queue(&relay, &grant, command(1, "map.repairTiles", json!({"index":0,"path":replacement}))); + assert_eq!(relay.poll(&mut s), 0); + assert_eq!(grant.pending().len(), 1); + assert!(relay.take_archive_read().is_none(), "approval must precede even worker creation"); + assert_eq!(s.map().tiles[0].url, original.to_str().unwrap()); + assert!(rx.try_recv().is_err()); + grant.decide(id, true); + assert_eq!(relay.poll(&mut s), 0); + let ahead = relay.take_archive_read().unwrap(); + assert!(ahead.reads()); + assert!(relay.take_archive_read().is_none(), "one admitted invocation, not redispatch"); + let done = worker(ahead); + assert_eq!(s.undo_depth(), depth); + assert_eq!(s.map().tiles[0].url, original.to_str().unwrap()); + assert!(rx.try_recv().is_err(), "worker completion is not command completion"); + assert!(!log.exists() || std::fs::read_dir(&log).unwrap().next().is_none(), "no premature audit"); + s.actor = Some("intervening user".into()); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer["result"]["isError"], false, "{answer}"); + assert_eq!(s.map().tiles[0].url, replacement.to_str().unwrap()); + assert_eq!(s.undo_depth(), depth + 1); + assert!(s.undo_actor().flatten().is_some_and(|actor| actor.starts_with("agent 1 (relay run ")), "{:?}", s.undo_actor()); + let files: Vec<_> = std::fs::read_dir(&log).unwrap().flatten().collect(); + assert_eq!(files.len(), 1); + let audit = std::fs::read_to_string(files[0].path()).unwrap(); + let entry: Value = serde_json::from_str(audit.trim()).unwrap(); + assert_eq!(entry["success"], true, "{entry}"); + s.run("edit.undo", &Value::Null).unwrap(); + assert_eq!(s.map().tiles[0].url, original.to_str().unwrap()); + relay.stop(); +} + +#[test] +fn scoped_archive_revocation_before_worker_and_before_landing_has_no_effect() { + for before_start in [true, false] { + let (_root, mut s, original, replacement) = fixture(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant_reviewed(false).unwrap(); + let depth = s.undo_depth(); + let (id, rx) = queue(&relay, &grant, command(1, "map.repairTiles", json!({"index":0,"path":replacement}))); + relay.poll(&mut s); + grant.decide(id, true); + relay.poll(&mut s); + let ahead = relay.take_archive_read().unwrap(); + let done = if before_start { + drop(grant); + let done = worker(ahead); + assert!(done.is_err(), "revoked authority must not start its read"); + done + } else { + let done = worker(ahead); + assert!(done.is_ok()); + drop(grant); + done + }; + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer["result"]["isError"], true, "{answer}"); + assert_eq!(s.map().tiles[0].url, original.to_str().unwrap()); + assert_eq!(s.undo_depth(), depth); + relay.stop(); + } +} + +#[test] +fn scoped_archive_read_cannot_land_on_a_stale_target() { + let (_root, mut s, original, replacement) = fixture(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant_reviewed(false).unwrap(); + let (id, rx) = queue(&relay, &grant, command(1, "map.repairTiles", json!({"index":0,"path":replacement}))); + relay.poll(&mut s); + grant.decide(id, true); + relay.poll(&mut s); + let done = worker(relay.take_archive_read().unwrap()); + s.run("map.setTiles", &json!({"index":0,"opacity":0.25})).unwrap(); + let depth = s.undo_depth(); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer["result"]["isError"], true, "{answer}"); + assert_eq!(s.map().tiles[0].url, original.to_str().unwrap()); + assert_eq!(s.map().tiles[0].opacity, 0.25); + assert_eq!(s.undo_depth(), depth); + relay.stop(); +} + +#[test] +fn batch_reload_holds_observer_and_later_requests_until_real_landing() { + let (_root, mut s, original, _) = fixture(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + // A real retained metadata object distinguishes validation from installation. + let retained = pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap(); + archive(&original, "Fresh metadata after manual reload"); + s.map_mut().tiles[0].error = Some("Observer must not see the retained error".into()); + let (_, rx) = queue(&relay, &grant, json!([command(7, "map.reloadTiles", json!({"index":0})), command(7, "map.tiles", json!({}))])); + assert_eq!(relay.poll(&mut s), 0); + let done = worker(relay.take_archive_read().unwrap()); + assert_eq!(s.map().tiles[0].error.as_deref(), Some("Observer must not see the retained error")); + assert!(Arc::ptr_eq(&retained, &pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap())); + let (_, later) = queue(&relay, &grant, json!({"jsonrpc":"2.0","id":9,"method":"ping"})); + assert_eq!(relay.poll(&mut s), 0); + assert!(later.try_recv().is_err()); + assert!(rx.try_recv().is_err()); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer.as_array().unwrap().len(), 2, "{answer}"); + assert!(answer.as_array().unwrap().iter().all(|r| r["id"] == 7 && r["result"]["isError"] == false), "{answer}"); + let observed = answer[1]["result"]["content"][0]["text"].as_str().unwrap(); + assert!(observed.contains("Linked"), "{observed}"); + assert!(!observed.contains("Observer must not see the retained error"), "{observed}"); + let fresh = pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap(); + assert!(!Arc::ptr_eq(&retained, &fresh)); + relay.poll(&mut s); + assert_eq!(reply(&later)["id"], 9); + relay.stop(); +} + +/// An approval freezes the state its card showed. A batch suffix whose reviewed input the earlier +/// entry then legitimately changes (a retained load error the reload clears) is refused at its own +/// turn, after the reload landed, instead of running on stale consent. +#[test] +fn reviewed_batch_suffix_keeps_its_preflight_state_across_a_landed_reload() { + let (_root, mut s, _, _) = fixture(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant_reviewed(false).unwrap(); + s.map_mut().tiles[0].error = Some("retained".into()); + let (id, rx) = queue(&relay, &grant, json!([command(7, "map.reloadTiles", json!({"index":0})), command(8, "map.tiles", json!({}))])); + relay.poll(&mut s); + grant.decide(id, true); + relay.poll(&mut s); + let done = worker(relay.take_archive_read().unwrap()); + assert!(rx.try_recv().is_err()); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer[0]["result"]["isError"], false, "{answer}"); + assert_eq!(answer[1]["result"]["isError"], true, "{answer}"); + assert!(s.map().tiles[0].error.is_none(), "the observer's turn came after the reload landed"); + relay.stop(); +} + +#[test] +fn padded_archive_commands_use_the_worker_and_keep_admission_and_landing_boundaries() { + for reviewed in [false, true] { + for repair in [false, true] { + let (_root, mut s, original, replacement) = fixture(); + let retained = pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap(); + archive(&original, "Reloaded"); + let relay = Relay::start(0).unwrap(); + let grant = if reviewed { relay.grant_reviewed(false).unwrap() } else { relay.grant(true).unwrap() }; + let name = if repair { " map.repairTiles " } else { " map.reloadTiles " }; + let params = if repair { json!({"index":0,"path":replacement}) } else { json!({"index":0}) }; + let depth = s.undo_depth(); + let (id, rx) = queue(&relay, &grant, command(1, name, params)); + assert_eq!(relay.poll(&mut s), 0); + if reviewed { + assert_eq!(grant.pending().len(), 1); + assert!(relay.take_archive_read().is_none()); + grant.decide(id, true); + assert_eq!(relay.poll(&mut s), 0); + } + let ahead = relay.take_archive_read().unwrap(); + assert!(ahead.reads(), "{name} must use the archive worker"); + let done = worker(ahead); + assert!(rx.try_recv().is_err()); + assert_eq!(s.undo_depth(), depth); + assert_eq!(s.map().tiles[0].url, original.to_str().unwrap()); + assert!(Arc::ptr_eq(&retained, &pmtiles::info_blocking(original.to_str().unwrap(), Duration::from_secs(10)).unwrap())); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer["result"]["isError"], false, "{answer}"); + assert_eq!(s.undo_depth(), depth + 1); + assert_eq!(s.map().tiles[0].url, if repair { replacement.to_str().unwrap() } else { original.to_str().unwrap() }); + relay.stop(); + } + } +} + +#[test] +fn padded_archive_commands_without_authorization_do_not_start_a_read() { + for repair in [false, true] { + let (_root, mut s, original, replacement) = fixture(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(false).unwrap(); + let name = if repair { " map.repairTiles " } else { " map.reloadTiles " }; + let params = if repair { json!({"index":0,"path":replacement}) } else { json!({"index":0}) }; + let (_, rx) = queue(&relay, &grant, command(1, name, params)); + assert_eq!(relay.poll(&mut s), 1); + assert!(relay.take_archive_read().is_none()); + let answer = reply(&rx); + assert_eq!(answer["result"]["isError"], true, "{answer}"); + assert_eq!(s.map().tiles[0].url, original.to_str().unwrap()); + relay.stop(); + } +} + +fn layers(s: &mut Session) -> (u32, u32) { + for name in ["A", "B"] { + s.run("layer.template", &json!({"template":"points","name":name})).unwrap(); + } + let a = s.map().layer_by_name("A").unwrap().id; + let b = s.map().layer_by_name("B").unwrap().id; + s.run("layer.select", &json!({"layer":a})).unwrap(); + (a, b) +} + +#[test] +fn selecting_the_active_layer_is_owned_before_each_archive_pause() { + for second_pause in [false, true] { + let (_root, mut s, _, _) = fixture(); + let (a, b) = layers(&mut s); + let before_a = s.map().layer(a).unwrap().renderer.clone(); + let before_b = s.map().layer(b).unwrap().renderer.clone(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let mut batch = Vec::new(); + if second_pause { + batch.push(command(1, "map.reloadTiles", json!({"index":0}))); + } + batch.extend([ + command(2, "layer.select", json!({"layer":a})), + // An unrelated failure must not erase a selection that actually succeeded. + command(3, "layer.select", json!({"layer":"Missing"})), + command(4, "map.reloadTiles", json!({"index":0})), + command(5, "layer.color", json!({"color":"#123456"})), + ]); + let (_, rx) = queue(&relay, &grant, json!(batch)); + assert_eq!(relay.poll(&mut s), 0); + if second_pause { + let done = worker(relay.take_archive_read().unwrap()); + finish(&relay, &mut s, done); + assert!(rx.try_recv().is_err()); + } + assert_eq!(s.active_layer, Some(a)); + assert_eq!(lock(&relay.owner.shared.agents).get(&grant.token).unwrap().active, Some(a)); + let done = worker(relay.take_archive_read().unwrap()); + s.run("layer.select", &json!({"layer":b})).unwrap(); + finish(&relay, &mut s, done); + let answer = reply(&rx); + let replies = answer.as_array().unwrap(); + assert_eq!(replies.last().unwrap()["result"]["isError"], false, "{answer}"); + assert_ne!(s.map().layer(a).unwrap().renderer, before_a, "suffix must change the selected A"); + assert_eq!(s.map().layer(b).unwrap().renderer, before_b, "user-selected B must not be changed"); + assert_eq!(s.active_layer, Some(b), "the user's active layer is restored"); + relay.stop(); + } +} + +#[test] +fn failed_or_unexecuted_selection_does_not_own_a_layer_at_archive_pause() { + let (_root, mut s, _, _) = fixture(); + let (a, b) = layers(&mut s); + let before_a = s.map().layer(a).unwrap().renderer.clone(); + let before_b = s.map().layer(b).unwrap().renderer.clone(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let (_, rx) = queue( + &relay, + &grant, + json!([ + command(1, "layer.select", json!({"layer":"Missing"})), + command(2, "map.reloadTiles", json!({"index":0})), + command(3, "layer.color", json!({"color":"#123456"})), + command(4, "layer.select", json!({"layer":a})), + ]), + ); + assert_eq!(relay.poll(&mut s), 0); + assert_eq!(lock(&relay.owner.shared.agents).get(&grant.token).unwrap().active, None); + let done = worker(relay.take_archive_read().unwrap()); + s.run("layer.select", &json!({"layer":b})).unwrap(); + finish(&relay, &mut s, done); + let answer = reply(&rx); + assert_eq!(answer[0]["result"]["isError"], true, "{answer}"); + assert_eq!(answer[2]["result"]["isError"], false, "{answer}"); + assert_eq!(s.map().layer(a).unwrap().renderer, before_a); + assert_ne!(s.map().layer(b).unwrap().renderer, before_b, "without an executed selection the suffix follows the user"); + assert_eq!(s.active_layer, Some(b)); + relay.stop(); +} + +#[test] +fn archive_batch_rejects_changed_context_before_agent_reentry_at_every_pause() { + for second_pause in [false, true] { + for replace_project in [false, true] { + for completion in ["stale", "cancelled", "success"] { + let (_root, mut s, _, _) = fixture(); + let (a, _) = layers(&mut s); + let map_a = s.map().name.clone(); + let renderer_a = s.map().layer(a).unwrap().renderer.clone(); + s.run("map.new", &json!({"name":"Other"})).unwrap(); + let (other, user) = layers(&mut s); + assert_eq!(a, other, "layer IDs are map-local"); + s.run("map.activate", &json!({"name":map_a})).unwrap(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let mut batch = vec![command(1, "layer.select", json!({"layer":a}))]; + if second_pause { + batch.push(command(2, "map.reloadTiles", json!({"index":0}))); + } + batch.extend([ + command(7, "map.reloadTiles", json!({"index":0})), + command(7, "layer.color", json!({"color":"#123456"})), + json!({"jsonrpc":"2.0","id":"observer","method":"ping"}), + json!({"jsonrpc":"2.0","method":"tools/call","params":{"name":"run_command","arguments":{"command":"layer.color","params":{"color":"#654321"}}}}), + json!({"jsonrpc":"2.0","id":"response","result":{}}), + command(9, "map.reloadTiles", json!({"index":0})), + ]); + let (_, rx) = queue(&relay, &grant, json!(batch)); + assert_eq!(relay.poll(&mut s), 0); + if second_pause { + let done = worker(relay.take_archive_read().unwrap()); + finish(&relay, &mut s, done); + assert!(rx.try_recv().is_err()); + } + let done = worker(relay.take_archive_read().unwrap()); + if replace_project { + let opened = s.opened; + s.run("project.new", &json!({"name":"Replacement"})).unwrap(); + assert_ne!(s.opened, opened); + let (replacement, _) = layers(&mut s); + assert_eq!(replacement, a); + } else { + s.run("map.activate", &json!({"name":"Other"})).unwrap(); + } + s.run("layer.select", &json!({"layer":user})).unwrap(); + s.actor = Some("current user".into()); + let before = serde_json::to_value(&s.project).unwrap(); + let agent_active = lock(&relay.owner.shared.agents).get(&grant.token).unwrap().active; + relay.begin_archive_landing(&s); + let result = match completion { + "stale" => done.and_then(|read| reads::read_land(&mut s, read)), + "cancelled" => Err("Cancelled archive fixture".into()), + _ => Ok(json!({"archive":"actual successful result"})), + }; + let failed = result.is_err(); + let expected = result.as_ref().err().cloned().unwrap_or_else(|| "actual successful result".into()); + relay.finish_archive_read(&mut s, result); + let answer = reply(&rx); + let replies = answer.as_array().unwrap(); + let pending = if second_pause { 2 } else { 1 }; + assert_eq!(replies.len(), pending + 4, "{answer}"); + assert_eq!(answer[0]["id"], 1); + assert_eq!(answer[0]["result"]["isError"], false, "{answer}"); + if second_pause { + assert_eq!(answer[1]["id"], 2); + assert_eq!(answer[1]["result"]["isError"], false, "{answer}"); + } + assert_eq!(answer[pending]["id"], 7); + assert_eq!(answer[pending]["result"]["isError"], failed, "{answer}"); + assert!(answer[pending]["result"]["content"][0]["text"].as_str().unwrap().contains(&expected), "{answer}"); + for (reply, id) in replies[pending + 1..].iter().zip([json!(7), json!("observer"), json!(9)]) { + assert_eq!(reply["id"], id, "{answer}"); + assert!(reply.get("error").is_some() || reply["result"]["isError"] == true, "{answer}"); + assert!(reply.to_string().contains("context changed"), "{answer}"); + } + assert_eq!(serde_json::to_value(&s.project).unwrap(), before, "no suffix command may mutate either map"); + if !replace_project { + assert_eq!(s.project.maps[0].layer(a).unwrap().renderer, renderer_a); + assert_eq!(s.map().name, "Other"); + } + assert_eq!(s.active_layer, Some(user)); + assert_eq!(s.actor.as_deref(), Some("current user")); + assert_eq!(lock(&relay.owner.shared.agents).get(&grant.token).unwrap().active, agent_active); + assert!(relay.take_archive_read().is_none(), "the suffix's second archive command must not dispatch"); + relay.finish_archive_read(&mut s, Err("duplicate completion".into())); + assert!(rx.try_recv().is_err(), "one terminal reply only"); + relay.stop(); + } + } + } +} + +#[test] +fn archive_batch_allows_explicit_map_changes_between_pauses() { + let (_root, mut s, original, _) = fixture(); + let (a, _) = layers(&mut s); + let map_a = s.map().name.clone(); + let before_a = s.map().layer(a).unwrap().renderer.clone(); + s.run("map.new", &json!({"name":"Other"})).unwrap(); + s.run("map.addTiles", &json!({"url":original,"name":"Other archive","overlay":true})).unwrap(); + let (b, _) = layers(&mut s); + let before_b = s.map().layer(b).unwrap().renderer.clone(); + s.run("map.activate", &json!({"name":map_a})).unwrap(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let (_, rx) = queue( + &relay, + &grant, + json!([ + command(1, "map.reloadTiles", json!({"index":0})), + command(2, "map.activate", json!({"name":"Other"})), + command(3, "layer.select", json!({"layer":b})), + command(4, "map.reloadTiles", json!({"index":0})), + command(5, "layer.color", json!({"color":"#123456"})), + ]), + ); + assert_eq!(relay.poll(&mut s), 0); + let done = worker(relay.take_archive_read().unwrap()); + finish(&relay, &mut s, done); + assert!(rx.try_recv().is_err()); + assert_eq!(s.map().name, "Other"); + let done = worker(relay.take_archive_read().unwrap()); + finish(&relay, &mut s, done); + let answer = reply(&rx); + let replies = answer.as_array().unwrap(); + assert_eq!(replies.len(), 5, "{answer}"); + assert!(replies.iter().all(|reply| reply["result"]["isError"] == false), "{answer}"); + assert_eq!(s.project.maps[0].layer(a).unwrap().renderer, before_a); + assert_ne!(s.map().layer(b).unwrap().renderer, before_b); + assert_eq!(s.map().name, "Other"); + relay.stop(); +} + +#[test] +fn archive_batch_resumes_same_context_after_validation_failure_or_cancellation() { + for error in ["Archive validation failed", "Cancelled"] { + let (_root, mut s, _, _) = fixture(); + let (a, _) = layers(&mut s); + let before = s.map().layer(a).unwrap().renderer.clone(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let (_, rx) = queue( + &relay, + &grant, + json!([ + command(1, "layer.select", json!({"layer":a})), + command(2, "map.reloadTiles", json!({"index":0})), + command(3, "layer.color", json!({"color":"#123456"})), + ]), + ); + assert_eq!(relay.poll(&mut s), 0); + drop(relay.take_archive_read().unwrap()); + relay.begin_archive_landing(&s); + relay.finish_archive_read(&mut s, Err(error.into())); + let answer = reply(&rx); + assert_eq!(answer[1]["result"]["isError"], true, "{answer}"); + assert!(answer[1]["result"]["content"][0]["text"].as_str().unwrap().contains(error), "{answer}"); + assert_eq!(answer[2]["result"]["isError"], false, "{answer}"); + assert_ne!(s.map().layer(a).unwrap().renderer, before); + relay.stop(); + } +} diff --git a/crates/assistant/src/tools.rs b/crates/assistant/src/tools.rs index 880a294d87..54e6ce163f 100644 --- a/crates/assistant/src/tools.rs +++ b/crates/assistant/src/tools.rs @@ -154,6 +154,18 @@ pub fn target(call: &ToolCall) -> Option<(String, Value)> { } } +/// Whether a successfully executed tool explicitly chose a layer, even if it was already active. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn selects_layer(name: &str, args: &Value) -> bool { + match name { + "run_command" => args.get("command").and_then(Value::as_str).map(str::trim) == Some("layer.select"), + "run" => args.get("script").and_then(Value::as_str).is_some_and(|script| { + rustgis_engine::cmd::workflows::script_commands(script).is_ok_and(|lines| lines.iter().any(|(_, id, _)| id == "layer.select")) + }), + _ => false, + } +} + /// The executor's strict envelope is the only input to engine planning. pub fn policy_call(call: &ToolCall) -> Result, String> { let c = match call.name.as_str() { @@ -230,6 +242,25 @@ pub fn run_with_budget(s: &mut Session, call: &ToolCall, tokens: usize, scope: S run_with_budget_inner(s, call, tokens, scope, permit).map_err(|e| public_error(&e)) } +/// Adapter-owned presentation state for one already-admitted archive command. Execution +/// authority stays in the engine's ReadAhead/Read, never in this result formatter. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) struct CommandCompletion { + before: Vec<(String, Vec)>, + mark: u64, +} +#[cfg(not(target_arch = "wasm32"))] +impl CommandCompletion { + pub(crate) fn capture(s: &Session) -> Self { + Self { before: layer_snap(s), mark: s.notice_mark() } + } + pub(crate) fn finish(self, s: &mut Session, result: Result) -> Result { + result + .map(|value| command_result(s, value, &self.before, self.mark, DEFAULT_OUTPUT_TOKENS)) + .map_err(|e| public_error(&error_with_notices(e, s, self.mark))) + } +} + /// Retains the initiating scope and one-use permit while assembling ordinary tool results. fn run_with_budget_inner(s: &mut Session, call: &ToolCall, tokens: usize, scope: Scope, permit: Option) -> Result { if matches!(call.name.as_str(), "run_command" | "run_tool") { diff --git a/crates/data/src/mbtiles.rs b/crates/data/src/mbtiles.rs index 8565e4db90..4b5e5477d5 100644 --- a/crates/data/src/mbtiles.rs +++ b/crates/data/src/mbtiles.rs @@ -287,8 +287,7 @@ mod file { use std::collections::HashMap; use std::path::{Path, PathBuf}; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::{Mutex, OnceLock}; + use std::sync::Mutex; use rusqlite::{Connection, OpenFlags, OptionalExtension, params}; @@ -296,12 +295,16 @@ mod file { /// Open connections kept per file, for the tile readers to share. const POOL: usize = 4; - /// Goes up when connections are closed, so one in use then isn't put back for reuse. - static GENERATION: AtomicU64 = AtomicU64::new(0); + #[derive(Default)] + struct Pool { + connections: HashMap>, + epochs: HashMap, + generation: u64, + } - fn pool() -> &'static Mutex>> { - static P: OnceLock>>> = OnceLock::new(); - P.get_or_init(Mutex::default) + fn pool() -> &'static Mutex { + static P: std::sync::LazyLock> = std::sync::LazyLock::new(Mutex::default); + &P } fn open(path: &Path) -> Result { @@ -316,8 +319,18 @@ mod file { /// Runs `f` with a connection to `path`, from the pool or newly opened, and puts it back. fn with(path: &Path, f: impl FnOnce(&Connection) -> Result) -> Result { - let generation = GENERATION.load(Ordering::SeqCst); - let pooled = pool().lock().unwrap_or_else(|e| e.into_inner()).get_mut(path).and_then(Vec::pop); + let (generation, epoch, pooled) = { + let mut all = pool().lock().unwrap_or_else(|e| e.into_inner()); + // Track checked-out files too: close_unless must invalidate a handle even while no + // idle connection to its file remains. Bound both this record and the idle pool. + if all.epochs.len() >= 16 && !all.epochs.contains_key(path) { + all.generation = all.generation.wrapping_add(1); + all.connections.clear(); + all.epochs.clear(); + } + let epoch = *all.epochs.entry(path.to_path_buf()).or_default(); + (all.generation, epoch, all.connections.get_mut(path).and_then(Vec::pop)) + }; let conn = match pooled { Some(c) => c, None => open(path)?, @@ -325,14 +338,10 @@ mod file { let r = f(&conn); let mut all = pool().lock().unwrap_or_else(|e| e.into_inner()); // Closed while in use (the file changed, or the cache was cleared): not reused. - if GENERATION.load(Ordering::SeqCst) != generation { + if all.generation != generation || all.epochs.get(path).copied().unwrap_or(0) != epoch { return r; } - // A few files are open at a time: past that many, start again rather than grow. - if all.len() >= 16 && !all.contains_key(path) { - all.clear(); - } - let slot = all.entry(path.to_path_buf()).or_default(); + let slot = all.connections.entry(path.to_path_buf()).or_default(); if slot.len() < POOL { slot.push(conn); } @@ -342,70 +351,155 @@ mod file { /// Forgets the open connections to `path` (it changed on disk, or was closed). pub fn close(path: &Path) { let mut all = pool().lock().unwrap_or_else(|e| e.into_inner()); - GENERATION.fetch_add(1, Ordering::SeqCst); - all.remove(path); + if let Some(epoch) = all.epochs.get_mut(path) { + *epoch = epoch.wrapping_add(1); + } + all.connections.remove(path); } /// Forgets every open connection. pub fn close_all() { let mut all = pool().lock().unwrap_or_else(|e| e.into_inner()); - GENERATION.fetch_add(1, Ordering::SeqCst); - all.clear(); + all.generation = all.generation.wrapping_add(1); + all.connections.clear(); + all.epochs.clear(); } /// Forgets the open connections to the files `keep` says no to. Windows won't move or delete a /// file SQLite has open, so a file nothing reads any more mustn't stay in the pool. pub fn close_unless(keep: impl Fn(&Path) -> bool) { let mut all = pool().lock().unwrap_or_else(|e| e.into_inner()); - if !all.keys().all(|path| keep(path)) { - GENERATION.fetch_add(1, Ordering::SeqCst); - all.retain(|path, _| keep(path)); + for (path, epoch) in &mut all.epochs { + if !keep(path) { + *epoch = epoch.wrapping_add(1); + } } + all.connections.retain(|path, _| keep(path)); } /// Whether a connection to `path` is kept open. pub fn is_open(path: &Path) -> bool { - pool().lock().unwrap_or_else(|e| e.into_inner()).get(path).is_some_and(|open| !open.is_empty()) + pool().lock().unwrap_or_else(|e| e.into_inner()).connections.get(path).is_some_and(|open| !open.is_empty()) } /// The metadata of the MBTiles file at `path`. pub fn open_path(path: &Path) -> Result { - with(path, |c| { - let sql = |e: rusqlite::Error| format!("{}: not a readable MBTiles file ({e})", path.display()); - let mut meta = Vec::new(); - if let Ok(mut st) = c.prepare("SELECT name, value FROM metadata LIMIT 1000") { - let rows = st.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))).map_err(sql)?; - meta.extend(rows.filter_map(Result::ok)); + with(path, |c| read_info(c, path)) + } + + /// Reads metadata and one bounded raster sample on a fresh connection, retaining nothing. + /// Vector archives get only a tile-page readability probe; an empty archive has no sample. + /// The current progress probe and request predicate cancel SQLite scans on this connection. + pub fn read_info_path(path: &Path) -> Result<(Info, Option>), String> { + let still = crate::source::remote::current_still(); + let cancelled = move || crate::progress::check().is_err() || still.as_ref().is_some_and(|s| !s()); + if cancelled() { + return Err(crate::DataError::Cancelled.to_string()); + } + // Latch callback cancellation: SQLite's "interrupted" error is not archive damage, + // even if a request predicate has changed again before the query unwinds. + let interrupted = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let result = (|| { + let mut c = open(path)?; + let poll = cancelled.clone(); + let stopped = interrupted.clone(); + c.progress_handler( + 1000, + Some(move || { + let stop = poll(); + if stop { + stopped.store(true, std::sync::atomic::Ordering::Relaxed); + } + stop + }), + ) + .map_err(|e| format!("{}: cannot install MBTiles cancellation handler ({e})", path.display()))?; + let info = read_info(&c, path)?; + if cancelled() { + return Err(crate::DataError::Cancelled.to_string()); } - // That it is an MBTiles file: the tiles table (or view) has its columns. Nothing is read. - c.prepare("SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles LIMIT 0").map_err(sql)?; - // What the metadata says, if it is usable: a zoom that reads as a number, a format - // RustGIS knows. - let says = |k: &str, ok: &dyn Fn(&str) -> bool| { - meta.iter().find(|(n, _)| n.eq_ignore_ascii_case(k)).is_some_and(|(_, v)| ok(v.trim())) - }; - let zoom = |v: &str| v.parse::().is_ok_and(f64::is_finite); - let format = |v: &str| super::Format::from_name(v) != super::Format::Unknown; - // The tiles are looked at only for what the metadata leaves out: finding the zooms can - // scan the whole table (or join a deduplicated layout's view). - let zooms = if says("minzoom", &zoom) && says("maxzoom", &zoom) { + let sql = |e: rusqlite::Error| format!("{}: not a readable MBTiles file ({e})", path.display()); + // Even complete metadata must not hide a damaged first tile page. Vector payloads are + // not decoded here; the tiles layer decodes the raster sample before accepting a reload. + let sample = if info.format.is_vector() { + let _: Option>> = + c.query_row("SELECT substr(tile_data, 1, 16) FROM tiles LIMIT 1", [], |row| row.get(0)).optional().map_err(sql)?; None } else { - let z: (Option, Option) = - c.query_row("SELECT min(zoom_level), max(zoom_level) FROM tiles", [], |r| Ok((r.get(0)?, r.get(1)?))).map_err(sql)?; - match z { - (Some(a), Some(b)) => Some((a.clamp(0, 24) as u8, b.clamp(0, 24) as u8)), - _ => None, + // Check the size before materializing a blob, as the normal tile reader does. + let tile: Option<(i64, Option>)> = c + .query_row( + "SELECT length(tile_data), CASE WHEN length(tile_data) <= ?1 THEN tile_data END FROM tiles LIMIT 1", + [MAX_TILE_BYTES as i64], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(sql)?; + match tile { + Some((len, _)) if len > MAX_TILE_BYTES as i64 => { + return Err(format!( + "{}: raster sample of {len} bytes exceeds the {MAX_TILE_BYTES}-byte tile limit", + path.display() + )); + } + Some((_, bytes)) => bytes, + None => None, } }; - // A tile's first bytes say its format when the metadata doesn't. - let first: Option> = if says("format", &format) { - None - } else { - c.query_row("SELECT substr(tile_data, 1, 16) FROM tiles LIMIT 1", [], |r| r.get(0)).optional().map_err(sql)? - }; - Ok(Info::from_metadata(&meta, zooms, first.as_deref())) - }) + Ok((info, sample)) + })(); + // Tiny queries may finish before the first SQLite callback; check success and error + // boundaries too, including a failed open. No fresh handle or hook enters the pool. + if interrupted.load(std::sync::atomic::Ordering::Relaxed) || cancelled() { + Err(crate::DataError::Cancelled.to_string()) + } else { + result + } + } + + fn read_info(c: &Connection, path: &Path) -> Result { + let sql = |e: rusqlite::Error| format!("{}: not a readable MBTiles file ({e})", path.display()); + let mut meta = Vec::new(); + let has_metadata: bool = c + .query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name = 'metadata' COLLATE NOCASE)", [], |r| r.get(0)) + .map_err(sql)?; + if has_metadata { + let mut st = c.prepare("SELECT name, value FROM metadata LIMIT 1001").map_err(sql)?; + let mut rows = st.query([]).map_err(sql)?; + while let Some(row) = rows.next().map_err(sql)? { + if meta.len() == 1000 { + return Err(format!("{}: MBTiles metadata exceeds the 1000-row limit; refusing to omit metadata", path.display())); + } + meta.push((row.get::<_, String>(0).map_err(sql)?, row.get::<_, String>(1).map_err(sql)?)); + } + } + // That it is an MBTiles file: the tiles table (or view) has its columns. Nothing is read. + c.prepare("SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles LIMIT 0").map_err(sql)?; + // What the metadata says, if it is usable: a zoom that reads as a number, a format + // RustGIS knows. + let says = + |k: &str, ok: &dyn Fn(&str) -> bool| meta.iter().find(|(n, _)| n.eq_ignore_ascii_case(k)).is_some_and(|(_, v)| ok(v.trim())); + let zoom = |v: &str| v.parse::().is_ok_and(f64::is_finite); + let format = |v: &str| super::Format::from_name(v) != super::Format::Unknown; + // The tiles are looked at only for what the metadata leaves out: finding the zooms can + // scan the whole table (or join a deduplicated layout's view). + let zooms = if says("minzoom", &zoom) && says("maxzoom", &zoom) { + None + } else { + let z: (Option, Option) = + c.query_row("SELECT min(zoom_level), max(zoom_level) FROM tiles", [], |r| Ok((r.get(0)?, r.get(1)?))).map_err(sql)?; + match z { + (Some(a), Some(b)) => Some((a.clamp(0, 24) as u8, b.clamp(0, 24) as u8)), + _ => None, + } + }; + // A tile's first bytes say its format when the metadata doesn't. + let first: Option> = if says("format", &format) { + None + } else { + c.query_row("SELECT substr(tile_data, 1, 16) FROM tiles LIMIT 1", [], |r| r.get(0)).optional().map_err(sql)? + }; + Ok(Info::from_metadata(&meta, zooms, first.as_deref())) } /// The tile at z/x/y (`y` from the north) of the file at `path`, `None` when it has none. @@ -430,6 +524,336 @@ mod file { }) } + #[cfg(test)] + mod reload_tests { + use super::*; + + struct CancelAfterPolls { + polls: std::sync::atomic::AtomicUsize, + after: usize, + } + + impl CancelAfterPolls { + fn new(after: usize) -> std::sync::Arc { + std::sync::Arc::new(Self { polls: std::sync::atomic::AtomicUsize::new(0), after }) + } + + fn polls(&self) -> usize { + self.polls.load(std::sync::atomic::Ordering::Relaxed) + } + } + + impl crate::progress::Probe for CancelAfterPolls { + fn cancelled(&self) -> bool { + self.polls.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1 >= self.after + } + + fn begin(&self, _: u64) {} + + fn advance(&self, _: u64) {} + } + + #[test] + fn candidate_missing_zoom_scan_cancels_inside_sql_and_retries_cleanly() { + let path = std::env::temp_dir().join(format!("rustgis-mbtiles-cancel-scan-{}.mbtiles", std::process::id())); + let _ = std::fs::remove_file(&path); + let c = Connection::open(&path).unwrap(); + c.execute_batch( + "CREATE TABLE metadata(name TEXT,value TEXT); + CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + INSERT INTO metadata VALUES('format','png'),('minzoom','2'),('maxzoom','7'); + WITH RECURSIVE n(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM n WHERE i<1023) + INSERT INTO tiles SELECT 2+i%6,i,0,x'89504e470d0a1a0a' FROM n;", + ) + .unwrap(); + drop(c); + + // The same small archive with complete zoom metadata needs only the entry and + // completion checks. Cancelling at poll four therefore requires SQLite's hook. + let complete = CancelAfterPolls::new(4); + { + let _installed = crate::progress::install(Some(complete.clone())); + let (info, _) = read_info_path(&path).unwrap(); + assert_eq!((info.min_zoom, info.max_zoom), (2, 7)); + } + assert!(complete.polls() < 4, "complete metadata unexpectedly ran a scan"); + let c = Connection::open(&path).unwrap(); + c.execute("DELETE FROM metadata WHERE name IN ('minzoom','maxzoom')", []).unwrap(); + drop(c); + + let scan = CancelAfterPolls::new(4); + { + let _installed = crate::progress::install(Some(scan.clone())); + assert_eq!(read_info_path(&path).unwrap_err(), crate::DataError::Cancelled.to_string()); + } + assert_eq!(scan.polls(), 4, "the SQL callback must stop at the first cancelled poll"); + assert!(!is_open(&path), "a cancelled fresh validator must not populate the pool"); + + let clean = CancelAfterPolls::new(usize::MAX); + { + let _installed = crate::progress::install(Some(clean.clone())); + let (info, sample) = read_info_path(&path).unwrap(); + assert_eq!((info.min_zoom, info.max_zoom, info.format), (2, 7, super::super::Format::Png)); + assert_eq!(sample.as_deref(), Some(b"\x89PNG\r\n\x1a\n".as_slice())); + } + assert!(clean.polls() > complete.polls() + 3, "the fixed scan must cross several SQLite callback intervals"); + assert!(!is_open(&path), "a successful fresh validator must not populate the pool"); + // A request revoked only during SQL must also latch cancellation, not relabel the + // interruption as a damaged archive when its predicate is live again afterwards. + let request_polls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let polled = request_polls.clone(); + crate::source::remote::with_still( + std::sync::Arc::new(move || polled.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1 != 4), + || assert_eq!(read_info_path(&path).unwrap_err(), crate::DataError::Cancelled.to_string()), + ); + assert_eq!(request_polls.load(std::sync::atomic::Ordering::Relaxed), 4); + assert!(!is_open(&path)); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn candidate_already_cancelled_stops_before_opening_the_file() { + let path = std::env::temp_dir().join(format!("rustgis-mbtiles-cancel-missing-{}.mbtiles", std::process::id())); + let _ = std::fs::remove_file(&path); + let cancelled = CancelAfterPolls::new(1); + { + let _installed = crate::progress::install(Some(cancelled.clone())); + assert_eq!(read_info_path(&path).unwrap_err(), crate::DataError::Cancelled.to_string()); + } + assert_eq!(cancelled.polls(), 1); + assert!(!is_open(&path)); + assert!( + read_info_path(&path).unwrap_err().contains(&path.display().to_string()), + "a clean retry must report the real missing-file error" + ); + + let during_open = CancelAfterPolls::new(2); + { + let _installed = crate::progress::install(Some(during_open.clone())); + assert_eq!(read_info_path(&path).unwrap_err(), crate::DataError::Cancelled.to_string()); + } + assert_eq!(during_open.polls(), 2, "a failed open still checks cancellation at its error boundary"); + } + + #[test] + fn candidate_small_queries_check_cancellation_at_completion() { + let path = std::env::temp_dir().join(format!("rustgis-mbtiles-cancel-small-{}.mbtiles", std::process::id())); + let _ = std::fs::remove_file(&path); + let c = Connection::open(&path).unwrap(); + c.execute_batch( + "CREATE TABLE metadata(name TEXT,value TEXT); + CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + INSERT INTO metadata VALUES('format','pbf'),('minzoom','2'),('maxzoom','7');", + ) + .unwrap(); + drop(c); + // Neither short query reaches 1,000 VM instructions. Cancel after metadata and + // after the sample query must still refuse to publish a successful candidate. + for after in [2, 3] { + let cancelled = CancelAfterPolls::new(after); + let _installed = crate::progress::install(Some(cancelled)); + assert_eq!(read_info_path(&path).unwrap_err(), crate::DataError::Cancelled.to_string()); + assert!(!is_open(&path)); + } + let (info, sample) = read_info_path(&path).unwrap(); + assert_eq!((info.min_zoom, info.max_zoom), (2, 7)); + assert!(sample.is_none()); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn metadata_row_limit_is_explicit_and_retryable_for_both_readers() { + let path = std::env::temp_dir().join(format!("rustgis-mbtiles-metadata-limit-{}.mbtiles", std::process::id())); + let _ = std::fs::remove_file(&path); + let c = Connection::open(&path).unwrap(); + c.execute_batch( + "CREATE TABLE metadata(name TEXT,value TEXT); + CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + INSERT INTO metadata VALUES('format','png'),('minzoom','0'),('maxzoom','0');", + ) + .unwrap(); + let tx = c.unchecked_transaction().unwrap(); + { + let mut insert = tx.prepare("INSERT INTO metadata VALUES(?1,'unused')").unwrap(); + for n in 0..996 { + insert.execute([format!("unused-{n}")]).unwrap(); + } + } + tx.execute("INSERT INTO metadata VALUES('attribution','last allowed attribution')", []).unwrap(); + tx.commit().unwrap(); + + let assert_exact_limit = || { + let ordinary = open_path(&path).unwrap(); + let (candidate, sample) = read_info_path(&path).unwrap(); + for info in [&ordinary, &candidate] { + assert_eq!(info.format, super::super::Format::Png); + assert_eq!(info.attribution, "last allowed attribution", "the thousandth metadata row must survive"); + } + assert!(sample.is_none(), "the fixture has no tiles"); + }; + assert_exact_limit(); + + c.execute("INSERT INTO metadata VALUES('json',?1)", [r#"{"vector_layers":[{"id":"overflow"}]}"#]).unwrap(); + for error in [open_path(&path).unwrap_err(), read_info_path(&path).unwrap_err()] { + assert!(error.contains(&path.display().to_string()), "{error}"); + assert!(error.contains("1000-row limit"), "{error}"); + } + c.execute("DELETE FROM metadata WHERE name='json'", []).unwrap(); + assert_exact_limit(); + + close(&path); + drop(c); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn ordinary_open_refuses_null_and_non_text_metadata_rows() { + let path = std::env::temp_dir().join(format!("rustgis-mbtiles-metadata-types-{}.mbtiles", std::process::id())); + let _ = std::fs::remove_file(&path); + let c = Connection::open(&path).unwrap(); + c.execute_batch( + "CREATE TABLE metadata(name,value); + CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + INSERT INTO metadata VALUES('format','png'),('minzoom','0'),('maxzoom','0');", + ) + .unwrap(); + assert_eq!(open_path(&path).unwrap().format, super::super::Format::Png); + for row in ["(NULL,'unused')", "('unused',NULL)", "(7,'unused')", "('unused',7)", "(x'01','unused')", "('unused',x'01')"] { + c.execute_batch(&format!("INSERT INTO metadata VALUES {row}")).unwrap(); + let error = open_path(&path).unwrap_err(); + assert!(error.contains(&path.display().to_string()), "{error}"); + assert!(error.contains("not a readable MBTiles file"), "{error}"); + c.execute("DELETE FROM metadata WHERE rowid = (SELECT max(rowid) FROM metadata)", []).unwrap(); + assert!(open_path(&path).is_ok(), "valid text rows must remain usable after {row}"); + } + close(&path); + drop(c); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn candidate_raster_sample_is_bounded_and_unpooled() { + let path = std::env::temp_dir().join(format!("rustgis-mbtiles-raster-sample-{}.mbtiles", std::process::id())); + let _ = std::fs::remove_file(&path); + let c = Connection::open(&path).unwrap(); + c.execute_batch( + "CREATE TABLE metadata(name TEXT,value TEXT); + CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + INSERT INTO metadata VALUES('format','png'),('minzoom','0'),('maxzoom','0');", + ) + .unwrap(); + assert!(read_info_path(&path).unwrap().1.is_none(), "an empty archive remains valid"); + let invalid_png = b"\x89PNG\r\n\x1a\n"; + c.execute("INSERT INTO tiles VALUES(0,0,0,?1)", [invalid_png.as_slice()]).unwrap(); + let (info, sample) = read_info_path(&path).unwrap(); + assert_eq!(info.format, super::super::Format::Png); + assert_eq!(sample.as_deref(), Some(invalid_png.as_slice()), "the tiles layer must see more than a discarded signature probe"); + c.execute("UPDATE tiles SET tile_data=zeroblob(?1)", [MAX_TILE_BYTES as i64 + 1]).unwrap(); + let error = read_info_path(&path).unwrap_err(); + assert!(error.contains(&path.display().to_string()), "{error}"); + assert!(error.contains("exceeds") && error.contains(&MAX_TILE_BYTES.to_string()), "{error}"); + c.execute_batch("UPDATE metadata SET value='pbf' WHERE name='format'; UPDATE tiles SET tile_data=x'01';").unwrap(); + assert!(read_info_path(&path).unwrap().1.is_none(), "vector payloads get a readability probe, not image validation"); + assert!(!pool().lock().unwrap().connections.contains_key(&path), "candidate validation must not populate the retained pool"); + drop(c); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn candidate_probe_reads_tile_pages_even_with_complete_metadata() { + let path = std::env::temp_dir().join(format!("rustgis-mbtiles-damaged-tile-page-{}.mbtiles", std::process::id())); + let _ = std::fs::remove_file(&path); + let c = Connection::open(&path).unwrap(); + c.execute_batch( + "CREATE TABLE metadata(name TEXT,value TEXT); + CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + INSERT INTO metadata VALUES('format','pbf'),('minzoom','0'),('maxzoom','0'); + INSERT INTO tiles VALUES(0,0,0,x'01');", + ) + .unwrap(); + let page = c.query_row("SELECT rootpage FROM sqlite_master WHERE name='tiles'", [], |r| r.get::<_, u32>(0)).unwrap() as usize; + let size = c.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0)).unwrap() as usize; + drop(c); + assert!(read_info_path(&path).is_ok()); + let mut bytes = std::fs::read(&path).unwrap(); + bytes[(page - 1) * size] = 0; + std::fs::write(&path, bytes).unwrap(); + assert!(open_path(&path).is_ok(), "complete metadata alone does not step the damaged tile table"); + let error = read_info_path(&path).unwrap_err(); + assert!(error.contains(&path.display().to_string()), "{error}"); + assert!(error.contains("not a readable MBTiles file"), "{error}"); + close(&path); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn closing_one_file_rejects_its_old_connection_but_keeps_another_file() { + let root = std::env::temp_dir().join(format!("rustgis-mbtiles-pool-reload-{}", std::process::id())); + std::fs::create_dir_all(&root).unwrap(); + let path = root.join("reload.mbtiles"); + let other = root.join("other.mbtiles"); + for p in [&path, &other] { + let c = Connection::open(p).unwrap(); + c.execute_batch( + "CREATE TABLE IF NOT EXISTS tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + DELETE FROM tiles; INSERT INTO tiles VALUES(0,0,0,x'01');", + ) + .unwrap(); + } + // Another test's new path can clear the process-global pool when 16 are tracked: the + // other file is only expected to survive `close` if no clear happened around it. + let generation = pool().lock().unwrap().generation; + assert_eq!(tile_path(&other, 0, 0, 0).unwrap(), Some(vec![1])); + let (started_tx, started_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let read_path = path.clone(); + let worker = std::thread::spawn(move || { + with(&read_path, |c| { + let tile: Vec = c.query_row("SELECT tile_data FROM tiles LIMIT 1", [], |r| r.get(0)).map_err(|e| e.to_string())?; + started_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + Ok(tile) + }) + .unwrap() + }); + started_rx.recv_timeout(std::time::Duration::from_secs(10)).unwrap(); + close(&path); + let mut kept = { + let all = pool().lock().unwrap(); + (all.generation == generation).then(|| all.connections.get(&other).is_some_and(|v| !v.is_empty())) + }; + for _ in 0..8 { + if kept.is_some() { + break; + } + // The pool was cleared by a concurrent test: seed both files again and close + // `path` once more, so the retry still shows `close` drops only its own file. + let generation = pool().lock().unwrap().generation; + assert_eq!(tile_path(&other, 0, 0, 0).unwrap(), Some(vec![1])); + assert_eq!(tile_path(&path, 0, 0, 0).unwrap(), Some(vec![1])); + close(&path); + let all = pool().lock().unwrap(); + kept = (all.generation == generation).then(|| { + all.connections.get(&other).is_some_and(|v| !v.is_empty()) && all.connections.get(&path).is_none_or(Vec::is_empty) + }); + } + assert_eq!(kept, Some(true), "closing one file must keep another file's connection"); + release_tx.send(()).unwrap(); + assert_eq!(worker.join().unwrap(), vec![1]); + assert!( + pool().lock().unwrap().connections.get(&path).is_none_or(Vec::is_empty), + "a read admitted before close returned its obsolete connection" + ); + let c = Connection::open(&path).unwrap(); + c.execute_batch("UPDATE tiles SET tile_data=x'02'").unwrap(); + drop(c); + assert_eq!(tile_path(&path, 0, 0, 0).unwrap(), Some(vec![2])); + close(&path); + close(&other); + std::fs::remove_dir_all(root).unwrap(); + } + } + #[cfg(test)] mod tests { use super::*; @@ -451,9 +875,15 @@ mod file { let (kept, gone) = (dir.join("kept.mbtiles"), dir.join("gone.mbtiles")); for path in [&kept, &gone] { std::fs::write(path, b"").unwrap(); - pool().lock().unwrap().entry(path.clone()).or_default().push(open(path).unwrap()); + with(path, |_| Ok(())).unwrap(); } - close_unless(|path| path != gone); + with(&kept, |_| { + with(&gone, |_| { + close_unless(|path| path != gone); + Ok(()) + }) + }) + .unwrap(); assert!(is_open(&kept)); assert!(!is_open(&gone)); std::fs::remove_file(&gone).unwrap(); @@ -462,7 +892,7 @@ mod file { } #[cfg(not(target_arch = "wasm32"))] -pub use file::{close, close_all, close_unless, is_open, open_path, tile_path}; +pub use file::{close, close_all, close_unless, is_open, open_path, read_info_path, tile_path}; #[cfg(test)] mod tests { diff --git a/crates/engine/src/cmd/layout.rs b/crates/engine/src/cmd/layout.rs index 0c28bf7e6a..54227953c2 100644 --- a/crates/engine/src/cmd/layout.rs +++ b/crates/engine/src/cmd/layout.rs @@ -3515,7 +3515,20 @@ pub fn layout_bytes(s: &mut Session, li: usize, pdf: bool, dpi: f64, all: bool) } /// [`layout_bytes`], a PDF's map frames georeferenced only when `georeference` (#1191). +/// Returns an actionable error if a linked archive changes while the pages are drawn. pub fn layout_bytes_with(s: &mut Session, li: usize, pdf: bool, dpi: f64, all: bool, georeference: bool) -> Result, String> { + let layout = s.project.layouts.get(li).ok_or("no layout")?; + let revisions = rustgis_render::layout::archive_revisions(layout, &s.project.maps); + let result = layout_bytes_inner(s, li, pdf, dpi, all, georeference); + let layout = s.project.layouts.get(li).ok_or("no layout")?; + revisions.finish_revision( + rustgis_render::layout::archive_revisions(layout, &s.project.maps), + Some(result), + "could not draw the page", + )? +} + +fn layout_bytes_inner(s: &mut Session, li: usize, pdf: bool, dpi: f64, all: bool, georeference: bool) -> Result, String> { let saved = s.project.layouts.get(li).ok_or("no layout")?.clone(); if let Some(note) = named_fonts_unread(&saved) { s.warn(note); @@ -3576,13 +3589,11 @@ pub fn layout_bytes_with(s: &mut Session, li: usize, pdf: bool, dpi: f64, all: b }; left_out.extend(geo.left_out.into_iter().map(|(id, n, why)| (page + 1, id, n, why))); by_corners.extend(geo.by_corners.into_iter().map(|(id, n)| (page + 1, id, n))); - let (drawing, pictured) = rustgis_render::layout::render_drawing(layout, &mut s.project.maps, dpi, &info) - .ok_or("the page is too large to draw")?; + let (drawing, pictured) = rustgis_render::layout::render_drawing(layout, &mut s.project.maps, dpi, &info)?; if drawing.overflowed() { pages_pictured.push(page + 1); let layout = s.project.layouts.get(li).ok_or("no layout")?; - let img = - rustgis_render::layout::render(layout, &mut s.project.maps, dpi, &info).ok_or("the page is too large to draw")?; + let img = rustgis_render::layout::render(layout, &mut s.project.maps, dpi, &info)?; out.push(rustgis_render::pdf::Page::new(&img, w, h).with_viewports(geo.viewports)); } else { frames_pictured.extend(pictured); @@ -3597,7 +3608,7 @@ pub fn layout_bytes_with(s: &mut Session, li: usize, pdf: bool, dpi: f64, all: b )); } } else { - let img = rustgis_render::layout::render(layout, &mut s.project.maps, dpi, &info).ok_or("the page is too large to draw")?; + let img = rustgis_render::layout::render(layout, &mut s.project.maps, dpi, &info)?; png = Some(img.encode_png().map_err(|e| e.to_string())?); } // A cancelled export stops here, its layout put back below. @@ -3786,8 +3797,7 @@ fn layout_svg(s: &mut Session, li: usize, dpi: f64) -> Result { }; // Rendering borrows the current layout; it does not step or modify the series page. let layout = s.project.layouts.get(li).ok_or("no layout")?; - let (text, pictured) = rustgis_render::svg::layout_svg(layout, &mut s.project.maps, dpi, &info) - .ok_or("could not draw the page; check its map sources and export size")?; + let (text, pictured) = rustgis_render::svg::layout_svg(layout, &mut s.project.maps, dpi, &info)?; pictured_notices(s, &pictured, &[]); Ok(text) } diff --git a/crates/engine/src/cmd/presentation.rs b/crates/engine/src/cmd/presentation.rs index 7059839759..247c9c9f1f 100644 --- a/crates/engine/src/cmd/presentation.rs +++ b/crates/engine/src/cmd/presentation.rs @@ -400,7 +400,7 @@ pub fn draw_page(s: &mut Session, i: usize, k: usize, dpi: f64) -> Result &str { /// The presentation as a PDF, a self-contained HTML slideshow or a PNG of its first page, and /// the number of pages written. pub fn export_bytes(s: &mut Session, i: usize, format: &str, dpi: f64) -> Result<(Vec, usize), String> { + let revision = |s: &Session| { + rustgis_render::archive_revisions(s.project.presentations.get(i).into_iter().flat_map(|p| { + p.pages + .iter() + .take(if format == "png" { 1 } else { usize::MAX }) + .filter_map(|page| if let PageKind::Map { map, .. } = &page.kind { Some(page_map(s, map)) } else { None }) + })) + }; + let before = revision(s); + let result = export_bytes_inner(s, i, format, dpi); + before.finish_revision(revision(s), Some(result), "could not draw the presentation")? +} + +fn export_bytes_inner(s: &mut Session, i: usize, format: &str, dpi: f64) -> Result<(Vec, usize), String> { // Only the size and name are copied: the pages can hold many megabytes of pictures, so each // is read in place after it's drawn. let (name, width, height, n) = diff --git a/crates/engine/src/cmd/reads.rs b/crates/engine/src/cmd/reads.rs index e2a45294d1..cdffc17828 100644 --- a/crates/engine/src/cmd/reads.rs +++ b/crates/engine/src/cmd/reads.rs @@ -45,6 +45,13 @@ pub struct ReadAhead { } impl ReadAhead { + /// The immutable registered invocation retained by this read, for queue presentation only. + pub fn command(&self) -> Option<(&'static str, &Value)> { + match self.op.op() { + CommandOp::Registered { spec, params } => Some((spec.id, params)), + _ => None, + } + } /// Whether this command has a read to make before it lands; otherwise landing runs it inline. pub fn reads(&self) -> bool { self.work.is_some() @@ -113,7 +120,15 @@ pub(crate) fn prepare(s: &Session, op: Authorized, owner: Continuatio let work = None; ("Run Script", work, None) } - _ => return Err("only Add Data, Reload Data, Load All, models and scripts can prepare background work".into()), + "map.reloadTiles" | "map.repairTiles" => { + #[cfg(not(target_arch = "wasm32"))] + let work = Some(super::repair::archive_ahead(s, spec.id, params)); + #[cfg(target_arch = "wasm32")] + let work = None; + let name = if spec.id == "map.repairTiles" { "Repair Data Source" } else { "Reload Data" }; + (name, work, None) + } + _ => return Err("only Add Data, Reload Data, Load All, local tile archives, models and scripts can prepare background work".into()), }; let name = work.as_ref().map_or(name, |work| work.name); Ok(ReadAhead { name, op, owner, source, work }) @@ -124,6 +139,15 @@ pub fn read_land(s: &mut Session, read: Read) -> CmdResult { s.finish_read(read) } +/// Captures the archive selected now, so a queued UI read cannot retarget an index or name. +#[cfg(not(target_arch = "wasm32"))] +pub fn archive_target(s: &Session, p: &Value) -> Option<(usize, rustgis_map::TileLayer)> { + let i = super::tiles::find(s, p).ok()??; + let tile = s.map().tiles.get(i)?; + tile.archive_file()?; + Some((i, tile.clone())) +} + /// `work` as a read for the command to take under `key`. pub(crate) fn ahead(name: &'static str, key: String, work: impl FnOnce() -> T + Send + 'static) -> ReadWork { ReadWork { name, key, work: Box::new(move || Box::new(work()) as Box) } diff --git a/crates/engine/src/cmd/repair.rs b/crates/engine/src/cmd/repair.rs index bda4d9c24c..e36009ddc0 100644 --- a/crates/engine/src/cmd/repair.rs +++ b/crates/engine/src/cmd/repair.rs @@ -11,11 +11,19 @@ pub fn commands() -> Vec { r#"{"layer"?: name|id (default: the active layer), "path": "the file or folder (a Zarr store, a File Geodatabase) the layer's data moved to", "table"?: "the dataset in a GeoPackage, GDAL or DuckDB file, or a raster's subdataset (default: the one the layer reads now)"}"#), C::new("layer.reloadData", "Reload Data", "Data › Data Source", reload_data).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).params( r#"{"layer"?: "name|id (default: the active layer). Manually rereads linked local data, retaining presentation. Folder sources are not monitored; snapshots must be added again.", "discardEdits"?: "bool (default: false): explicitly discard pending GeoPackage edits. Undo restores in-memory edits and configuration, not overwritten files or historical streamed contents."}"#), - C::new("map.repairTiles", "Repair Tile Archive", "Layers", repair_tiles).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).params( + C::new("map.repairTiles", "Repair Tile Archive", "Layers", repair_tiles).inputs(archive_inputs).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).params( r#"{"name"?: text | "index"?: number (0 = top) | "id"?: catalog id, "path": "the .pmtiles or .mbtiles file the archive moved to"}"#), + C::new("map.reloadTiles", "Reload Tile Archive", "Layers", reload_tiles).inputs(archive_inputs).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)).params( + r#"{"name": text | "index": number (0 = top) | "id": catalog id} (give one selector) — rereads a local PMTiles or MBTiles archive at its linked path, keeping its presentation; undo cannot restore overwritten file contents"#), ] } +/// Freeze the executor's selected archive without reading it or resolving a replacement path. +fn archive_inputs(input: &super::InputContext<'_>, p: &Value) -> Result { + let i = super::tiles::find(input, p)?.ok_or("name, index or id is required")?; + Ok(super::tiles::tile_inputs(input.map(), [i])) +} + #[cfg(not(target_arch = "wasm32"))] fn replacement(p: &Value) -> Result { let path = super::layer::s_req(p, "path")?.trim(); @@ -199,7 +207,7 @@ pub(crate) fn reload_ahead(s: &Session, p: &Value) -> Option Result { let candidate = match &input.source { @@ -473,6 +481,11 @@ fn compatible( Ok(()) } +/// `map.repairTiles`: relinks a local PMTiles or MBTiles archive after validating the new file. +/// The desktop's menus and direct attached MCP calls validate it on the read queue's worker +/// ([`super::reads`]) and land the prepared result here. Called any other way (a notebook cell, a +/// task step, a script, the CLI), it validates inline on the caller's thread, like Add Data and +/// Reload Data, because those callers run their commands in order and the next may depend on it. fn repair_tiles(s: &mut Session, p: &Value) -> CmdResult { #[cfg(target_arch = "wasm32")] { @@ -481,30 +494,295 @@ fn repair_tiles(s: &mut Session, p: &Value) -> CmdResult { } #[cfg(not(target_arch = "wasm32"))] { - use rustgis_render::rustgis_tiles::{cache, mbtiles, pmtiles}; + let prepared = super::reads::take::>(s, "map.repairTiles"); let i = super::tiles::find(s, p)?.ok_or("name, index or id is required")?; let t = s.map().tiles.get(i).ok_or("the tile layer no longer exists")?; if t.archive_file().is_none() { return Err(format!("{}: only a tile archive read from a file (PMTiles or MBTiles) can be repaired", t.name)); } - let full = replacement(p)?; - let mut new = t.clone(); - new.url = full.clone(); - new.error = None; - new.repair()?; - let vector = match new.kind { - rustgis_map::TileKind::MbTiles => mbtiles::info(&full)?.format.is_vector(), - _ => pmtiles::info_blocking(&full, web_time::Duration::from_secs(10))?.tile_type.is_vector(), - }; - if (!t.layers.is_empty() || !t.style_layers.is_empty()) && !vector { - return Err(format!( - "{full} holds raster tiles; {} draws vector tiles with its own layers or style: choose a vector tile archive", - t.name - )); + let full = replacement(p).map_err(|e| format!("{e}; correct the file and retry Repair Data Source"))?; + let new = archive_candidate_after_read(t, &full, s.map().instance_id(), prepared)?; + rustgis_render::rustgis_tiles::cache::reload_archive(&t.url); + // The replacement path may already have been drawn by another layer. + if t.url != full { + rustgis_render::rustgis_tiles::cache::reload_archive(&full); } - cache::forget(&t.url); let slot = s.map_mut().tiles.get_mut(i).ok_or("the tile layer no longer exists")?; *slot = new; Ok(json!({"index": i, "path": full})) } } + +/// Repair and reload share one validation policy. Reading a candidate must not evict retained +/// tiles/metadata or reuse a connection to an archive replaced at the same path. +#[cfg(not(target_arch = "wasm32"))] +fn archive_candidate(t: &rustgis_map::TileLayer, path: &str) -> Result { + use rustgis_render::rustgis_tiles::{mbtiles, pmtiles}; + let mut new = t.clone(); + new.url = path.to_string(); + new.error = None; + new.repair()?; + let vector = match new.kind { + rustgis_map::TileKind::MbTiles => mbtiles::read_info_local(path)?.format.is_vector(), + _ => pmtiles::read_info_local(path)?.tile_type.is_vector(), + }; + if (!t.layers.is_empty() || !t.style_layers.is_empty()) && !vector { + return Err(format!( + "{path} holds raster tiles; {} draws vector tiles with its own layers or style: choose a vector tile archive", + t.name + )); + } + Ok(new) +} + +/// An operation-local observation, not a file snapshot or a monitor baseline. +#[cfg(not(target_arch = "wasm32"))] +#[derive(PartialEq, Eq)] +struct FileRevision { + len: u64, + modified: web_time::SystemTime, + #[cfg(unix)] + identity_change: (u64, u64, i64, i64), + #[cfg(windows)] + created: u64, +} + +#[cfg(not(target_arch = "wasm32"))] +impl FileRevision { + fn read(path: &std::path::Path, optional: bool) -> Result, String> { + let shown = path.display(); + let metadata = match std::fs::metadata(path) { + Ok(metadata) => metadata, + Err(e) if optional && e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => { + return Err(format!("{shown}: could not observe archive metadata ({e}); finish writing or restore the file and retry")); + } + }; + // SQLite creates an empty -wal even for a read-only open of a clean WAL-mode archive, + // and cannot remove it again. An empty sidecar holds no commits: same as no sidecar. + if optional && metadata.len() == 0 { + return Ok(None); + } + if !metadata.is_file() { + return Err(format!("{shown}: expected an archive file; restore the file and retry")); + } + let modified = metadata + .modified() + .map_err(|e| format!("{shown}: could not observe the archive modification time ({e}); check file access and retry"))?; + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; + #[cfg(windows)] + use std::os::windows::fs::MetadataExt; + Ok(Some(Self { + len: metadata.len(), + modified, + #[cfg(unix)] + identity_change: (metadata.dev(), metadata.ino(), metadata.ctime(), metadata.ctime_nsec()), + #[cfg(windows)] + created: metadata.creation_time(), + })) + } +} + +#[cfg(not(target_arch = "wasm32"))] +#[derive(PartialEq, Eq)] +struct ArchiveObservation { + primary: Option, + // SQLite commits can change these without changing the main database. The -shm file + // is reader coordination, not content, and is deliberately not observed. + sqlite: Option>>, +} + +#[cfg(not(target_arch = "wasm32"))] +impl ArchiveObservation { + fn read(path: &str) -> Result { + let path = path.strip_prefix("file://").unwrap_or(path); + let primary = FileRevision::read(std::path::Path::new(path), false)?; + let sqlite = std::path::Path::new(path).extension().is_some_and(|e| e.eq_ignore_ascii_case("mbtiles")); + let sqlite = if sqlite { + // Unix SQLite names its sidecars from the resolved database path, so commits through + // a link land beside the target; Windows names them from the path as given. Observe + // beside both when they differ, so neither VFS's commits escape. + let given = std::path::Path::new(path); + let resolved = std::fs::canonicalize(given).ok().filter(|resolved| resolved != given); + let mut sidecars = Vec::new(); + for base in std::iter::once(given).chain(resolved.as_deref()) { + for suffix in ["-wal", "-journal"] { + let mut sidecar = base.as_os_str().to_owned(); + sidecar.push(suffix); + sidecars.push(FileRevision::read(std::path::Path::new(&sidecar), true)?); + } + } + Some(sidecars) + } else { + None + }; + Ok(Self { primary, sqlite }) + } + + /// Metadata only: safe for final landing, with no archive scan or decompression. + fn unchanged(&self, path: &str) -> Result<(), String> { + if *self != Self::read(path)? { + return Err(format!("{path}: the archive changed while it was read; finish writing and retry the command")); + } + Ok(()) + } +} + +#[cfg(not(target_arch = "wasm32"))] +struct ValidatedArchive { + layer: rustgis_map::TileLayer, + observation: ArchiveObservation, +} + +#[cfg(not(target_arch = "wasm32"))] +fn observed_candidate(t: &rustgis_map::TileLayer, path: &str) -> Result { + // Keep the archive reader's path boundary ahead of the new metadata I/O too. + rustgis_map::tiles::check_path(path, if t.kind == rustgis_map::TileKind::MbTiles { "mbtiles" } else { "pmtiles" })?; + let observation = ArchiveObservation::read(path)?; + let layer = archive_candidate(t, path)?; + observation.unchanged(path)?; + Ok(ValidatedArchive { layer, observation }) +} + +#[cfg(not(target_arch = "wasm32"))] +struct ReadArchive { + map: u64, + original: rustgis_map::TileLayer, + path: String, + result: Result, +} + +/// Fresh validation for the desktop's single read worker; nothing is installed or invalidated. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn archive_ahead(s: &Session, command: &str, p: &Value) -> super::reads::ReadWork { + let request = (|| { + let i = super::tiles::find(s, p)?.ok_or("name, index or id is required")?; + let t = s.map().tiles.get(i).ok_or("the tile layer no longer exists")?; + let linked = t.archive_file().ok_or("only a linked local PMTiles or MBTiles archive can be reloaded or repaired")?; + let path = if command == "map.repairTiles" { replacement(p)? } else { linked.to_string() }; + Ok::<_, String>((s.map().instance_id(), t.clone(), path)) + })(); + let name = if command == "map.repairTiles" { "Repair Data Source" } else { "Reload Data" }; + super::reads::ahead(name, command.to_string(), move || { + request.map(|(map, original, path)| { + let result = observed_candidate(&original, &path); + ReadArchive { map, original, path, result } + }) + }) +} + +#[cfg(not(target_arch = "wasm32"))] +fn archive_candidate_after_read( + t: &rustgis_map::TileLayer, + path: &str, + map: u64, + prepared: Option>, +) -> Result { + let candidate = if let Some(prepared) = prepared { + let prepared = prepared?; + if prepared.map != map || prepared.original != *t || prepared.path != path { + return Err("the archive or its settings changed while it was read; retry the command".into()); + } + prepared.result? + } else { + observed_candidate(t, path)? + }; + candidate.observation.unchanged(path)?; + Ok(candidate.layer) +} + +/// `map.reloadTiles`: rereads a local PMTiles or MBTiles archive at its linked path. The desktop's +/// menus and direct attached MCP calls validate it on the read queue's worker ([`super::reads`]) +/// and land the prepared result here. Called any other way (a notebook cell, a task step, a +/// script, the CLI), it validates inline on the caller's thread, like Add Data and Reload Data, +/// because those callers run their commands in order and the next may depend on it. +fn reload_tiles(s: &mut Session, p: &Value) -> CmdResult { + #[cfg(target_arch = "wasm32")] + { + let _ = (s, p); + Err("the web app can't reload files on this computer: add the file again with Add Data".into()) + } + #[cfg(not(target_arch = "wasm32"))] + { + let prepared = super::reads::take::>(s, "map.reloadTiles"); + let i = super::tiles::find(s, p)?.ok_or("name, index or id is required")?; + let t = s.map().tiles.get(i).ok_or("the tile layer no longer exists")?; + let path = t.archive_file().ok_or_else(|| { + format!( + "{}: Reload Data needs a linked local PMTiles or MBTiles file; add snapshots/uploads again, or use Reload Web Layers for web addresses", + t.name + ) + })?.to_string(); + let new = archive_candidate_after_read(t, &path, s.map().instance_id(), prepared).map_err(|e| { + format!("{} ({path}): {e}; correct the file and retry Reload Data, or use Repair Data Source if it moved", t.name) + })?; + let cancel_view = s.style_view.as_ref().is_some_and(|(pending, _, map)| { + *map == s.map().instance_id() + && pending.kind == t.kind + && pending.url == t.url + && pending.bounds == t.bounds + && pending.id == t.id + }); + rustgis_render::rustgis_tiles::cache::reload_archive(&path); + let slot = s.map_mut().tiles.get_mut(i).ok_or("the tile layer no longer exists")?; + *slot = new; + let message = format!("Reloaded {} from {path}; undo cannot restore overwritten file contents", slot.name); + if cancel_view { + s.style_view = None; + } + s.notify(message, false); + Ok(json!({"index": i, "path": path})) + } +} + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use super::*; + + #[test] + fn empty_sqlite_sidecars_match_absent_ones_but_commits_do_not() { + let dir = std::env::temp_dir().join(format!("rustgis-repair-observation-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("clean.mbtiles"); + std::fs::write(&path, b"main database").unwrap(); + let path = path.display().to_string(); + let observed = ArchiveObservation::read(&path).unwrap(); + for sidecar in ["-wal", "-journal"] { + let sidecar = format!("{path}{sidecar}"); + std::fs::write(&sidecar, b"").unwrap(); + observed.unchanged(&path).expect("an empty sidecar created by a reader is not a commit"); + std::fs::write(&sidecar, b"committed frame").unwrap(); + assert!(observed.unchanged(&path).is_err(), "{sidecar}"); + std::fs::remove_file(&sidecar).unwrap(); + observed.unchanged(&path).unwrap(); + } + std::fs::remove_dir_all(dir).unwrap(); + } + + #[cfg(unix)] + #[test] + fn linked_mbtiles_observes_the_sidecars_beside_its_target_and_beside_the_link() { + let dir = std::env::temp_dir().join(format!("rustgis-repair-linked-observation-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let target = dir.join("target.mbtiles"); + std::fs::write(&target, b"main database").unwrap(); + let link = dir.join("link.mbtiles"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + let link = link.display().to_string(); + let observed = ArchiveObservation::read(&link).unwrap(); + observed.unchanged(&link).expect("nothing was committed"); + // SQLite on unix commits beside the target; on Windows beside the path as given. + for beside in [target.display().to_string(), link.clone()] { + for sidecar in ["-wal", "-journal"] { + let sidecar = format!("{beside}{sidecar}"); + std::fs::write(&sidecar, b"committed frame").unwrap(); + assert!(observed.unchanged(&link).is_err(), "a commit to {sidecar} escaped the freshness check"); + std::fs::remove_file(&sidecar).unwrap(); + observed.unchanged(&link).unwrap(); + } + } + std::fs::remove_dir_all(dir).unwrap(); + } +} diff --git a/crates/engine/src/cmd/report.rs b/crates/engine/src/cmd/report.rs index 4ba19850ab..19ba4f8084 100644 --- a/crates/engine/src/cmd/report.rs +++ b/crates/engine/src/cmd/report.rs @@ -957,23 +957,43 @@ pub fn pages(s: &Session, i: usize) -> Result<(Doc, Vec Result<(Vec, usize), String> { let (doc, pages) = pages(s, i)?; + let revision = |s: &Session| { + pages + .iter() + .take(if pdf { usize::MAX } else { 1 }) + .filter_map(|page| { + let rustgis_render::report::Page::Layout(li) = page else { return None }; + s.project.layouts.get(*li).map(|layout| rustgis_render::layout::archive_revisions(layout, &s.project.maps)) + }) + .max() + .unwrap_or_default() + }; + let before = revision(s); + let result = report_pages_bytes(s, &doc, &pages, dpi, pdf); + before.finish_revision(revision(s), Some(result), "could not draw the report")? +} + +fn report_pages_bytes( + s: &mut Session, + doc: &Doc, + pages: &[rustgis_render::report::Page], + dpi: f64, + pdf: bool, +) -> Result<(Vec, usize), String> { let n = if pdf { pages.len() } else { 1 }; let mut out = Vec::new(); let mut held = 0usize; let (mut frames_pictured, mut pages_pictured) = (Vec::new(), Vec::new()); for k in 0..n { if !pdf { - let (img, ..) = rustgis_render::report::draw_page(&doc, &pages, k, dpi, &s.project.layouts, &mut s.project.maps) - .ok_or("the page is too large to draw")?; + let (img, ..) = rustgis_render::report::draw_page(doc, pages, k, dpi, &s.project.layouts, &mut s.project.maps)?; return Ok((img.encode_png().map_err(|e| e.to_string())?, pages.len())); } - let (page, pictured) = rustgis_render::report::pdf_page(&doc, &pages, k, dpi, &s.project.layouts, &mut s.project.maps) - .ok_or("the page is too large to draw")?; + let (page, pictured) = rustgis_render::report::pdf_page(doc, pages, k, dpi, &s.project.layouts, &mut s.project.maps)?; if page.overflowed() { // Too large for vectors all the same: the page as one picture. pages_pictured.push(k + 1); - let (img, w, h) = rustgis_render::report::draw_page(&doc, &pages, k, dpi, &s.project.layouts, &mut s.project.maps) - .ok_or("the page is too large to draw")?; + let (img, w, h) = rustgis_render::report::draw_page(doc, pages, k, dpi, &s.project.layouts, &mut s.project.maps)?; out.push(rustgis_render::pdf::Page::new(&img, w, h)); } else { frames_pictured.extend(pictured); diff --git a/crates/engine/src/cmd/share.rs b/crates/engine/src/cmd/share.rs index e8a5afb121..1a1d5aadcf 100644 --- a/crates/engine/src/cmd/share.rs +++ b/crates/engine/src/cmd/share.rs @@ -340,11 +340,17 @@ fn export_view( /// The map view on one PDF page (for printing): the page is the view's shape at 96 points per /// inch of screen, kept as vectors with what is pixels by nature drawn at `dpi`. pub fn map_pdf(s: &mut Session, dpi: f64) -> Result, String> { + let revisions = rustgis_render::archive_revisions([s.map()]); + let result = map_pdf_inner(s, dpi); + revisions.finish_revision(rustgis_render::archive_revisions([s.map()]), Some(result), "could not draw the map")? +} + +fn map_pdf_inner(s: &mut Session, dpi: f64) -> Result, String> { let (vw, vh) = s.view_size; let ppp = (dpi / 96.0).clamp(0.5, 6.0); let view = View { width: vw.clamp(16.0, 8000.0), height: vh.clamp(16.0, 8000.0), pixels_per_point: ppp, camera: s.map().camera, sky: false }; - let drawing = rustgis_render::render_drawing(s.map_mut(), &view).ok_or("could not draw the map")?; + let drawing = rustgis_render::render_drawing(s.map_mut(), &view)?; let mm = |pt: f64| pt * 25.4 / 96.0; let (w, h) = (mm(view.width), mm(view.height)); let page = if drawing.overflowed() { diff --git a/crates/engine/src/policy/dispatch.rs b/crates/engine/src/policy/dispatch.rs index 68c1ce307c..7c19f49309 100644 --- a/crates/engine/src/policy/dispatch.rs +++ b/crates/engine/src/policy/dispatch.rs @@ -107,8 +107,44 @@ impl Session { if self.depth > 0 || !self.policy.nodes.is_empty() || self.policy.cursor.is_some() { return Err("background reads can only start before handler dispatch".into()); } - if !matches!(command, "layer.addData" | "layer.loadAll" | "layer.reloadData" | "model.run" | "script.run") { - return Err("only Add Data, Load All, Reload Data, models and scripts can prepare background work".into()); + self.capture_read(command, params) + } + /// Admits exactly one standalone archive command before dispatch, retaining its original + /// scope and one-use approval through the worker and landing. Composite/handler capture + /// remains forbidden, including when an approval cursor is already active. + pub fn start_scoped_archive_read( + &mut self, + scope: Scope, + call: &Call, + permit: Option, + ) -> Result { + if self.depth > 0 || !self.policy.nodes.is_empty() || self.policy.cursor.is_some() || !self.policy.scopes.is_empty() { + return Err("background reads can only start before handler dispatch".into()); + } + let CallKind::Command { id, params } = &call.kind else { + return Err("only a standalone archive command can prepare a scoped read".into()); + }; + if !matches!(id.as_str(), "map.reloadTiles" | "map.repairTiles") { + return Err("only a standalone archive command can prepare a scoped read".into()); + } + let capture = |s: &mut Session| s.capture_read(id, params); + match permit { + Some(p) => { + if p.request != call.request { + return Err("this approval belongs to another request".into()); + } + self.run_plan_with(scope, p.plan, call, capture) + } + None => self.with_scope(scope, capture)?, + } + } + /// Private pre-handler capture shared by native reads and the exact scoped archive entry. + fn capture_read(&mut self, command: &str, params: &Value) -> Result { + if !matches!( + command, + "layer.addData" | "layer.loadAll" | "layer.reloadData" | "model.run" | "script.run" | "map.reloadTiles" | "map.repairTiles" + ) { + return Err("only Add Data, Load All, Reload Data, models, scripts and local tile archives can prepare background work".into()); } let mut params = params.as_object().cloned().ok_or("read parameters must be an object")?; if matches!(command, "layer.loadAll" | "layer.reloadData") { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 15c8e7fd01..43b76f3a49 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -3384,8 +3384,7 @@ fn geolibre_tools_run_on_layers_and_add_their_outputs() { s.run("edit.undo", &Value::Null).unwrap(); assert!(s.map().layer_by_name("Slope").is_none()); // A vector where a raster belongs is an error, not a crash. - let err = s.run("gp.run", &json!({"tool": "geolibre:slope", "params": {"input": "Cities"}})).unwrap_err(); - assert!(err.starts_with("Slope:"), "{err}"); + assert!(s.run("gp.run", &json!({"tool": "geolibre:slope", "params": {"input": "Cities"}})).is_err()); } /// The browser's way: prepare a run, let a runner (here, pretend) produce the files, then @@ -37842,6 +37841,9 @@ fn reload_failure_guidance_distinguishes_editable_geopackages_from_linked_files( std::fs::remove_dir_all(dir).unwrap(); } +#[cfg(not(target_arch = "wasm32"))] +mod archive_render_revision; + /// `gp.threads` sets the shared count every place that starts threads for CPU work takes (#1562), /// says it, the default's and the cores, refuses what it can't take, and `null` goes back to the /// default. No undo step and no change to the project: it is the process's, not the project's. diff --git a/crates/engine/src/tests/archive_render_revision.rs b/crates/engine/src/tests/archive_render_revision.rs new file mode 100644 index 0000000000..5ccac5f546 --- /dev/null +++ b/crates/engine/src/tests/archive_render_revision.rs @@ -0,0 +1,372 @@ +use crate::{data, map, render}; +use data::source::{Batch, FeatureSource, Request, SourceInfo}; +use data::{Dataset, Feature, Geometry, GeometryType}; +use map::{Camera, Map, TileKind, TileLayer, source::SourceLayer}; +use render::rustgis_tiles::{cache, pmtiles}; +use std::sync::{ + Arc, Barrier, + atomic::{AtomicBool, Ordering}, + mpsc, +}; + +struct Fixture(std::path::PathBuf); +impl Fixture { + fn new() -> Self { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + loop { + let path = std::env::temp_dir().join(format!( + "rustgis-render-revision-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + match std::fs::create_dir(&path) { + Ok(()) => return Self(path), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => panic!("creating {}: {e}", path.display()), + } + } + } + + fn path(&self) -> &std::path::Path { + &self.0 + } +} +impl Drop for Fixture { + fn drop(&mut self) { + data::mbtiles::close_unless(|path| !path.starts_with(&self.0)); + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// A real streamed layer between two drawings of the archive. Complete traversal pauses only +/// after the lower drawing has copied its old pixels and credits; the upper drawing has not run. +struct PauseSource { + info: SourceInfo, + ds: Dataset, + once: AtomicBool, + paused: mpsc::SyncSender<()>, + resume: Arc, +} +impl FeatureSource for PauseSource { + fn info(&self) -> &SourceInfo { + &self.info + } + fn read(&self, _: &Request) -> data::Result { + Ok(Batch { features: self.ds.features.to_vec(), matched: 1, complete: true, ..Default::default() }) + } + fn streams_chunks(&self) -> bool { + true + } + fn visit_chunks( + &self, + _: Option, + _: usize, + visit: &mut dyn FnMut(Vec) -> data::Result<()>, + ) -> data::Result { + if !self.once.swap(true, Ordering::SeqCst) { + self.paused.send(()).unwrap(); + self.resume.wait(); + } + visit(self.ds.features.to_vec())?; + Ok(Batch { matched: 1, complete: true, ..Default::default() }) + } +} + +fn archive(path: &std::path::Path, kind: TileKind, green: bool) { + let mut tile = tiny_skia::Pixmap::new(256, 256).unwrap(); + tile.fill(if green { tiny_skia::Color::from_rgba8(0, 220, 0, 255) } else { tiny_skia::Color::from_rgba8(220, 0, 0, 255) }); + let png = tile.encode_png().unwrap(); + let credits = if green { "New archive credits" } else { "Old archive credits" }; + if kind == TileKind::PmTiles { + let options = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Png, + gzipped_tiles: false, + bounds: [-180.0, -85.0, 180.0, 85.0], + metadata: serde_json::json!({"attribution": credits}), + }; + std::fs::write(path, pmtiles::write(vec![(0, 0, 0, png)], &options).unwrap()).unwrap(); + } else { + let db = rusqlite::Connection::open(path).unwrap(); + db.execute_batch("CREATE TABLE metadata(name TEXT,value TEXT); CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); INSERT INTO metadata VALUES('format','png'),('minzoom','0'),('maxzoom','0');").unwrap(); + db.execute("INSERT INTO metadata VALUES('attribution',?1)", [credits]).unwrap(); + db.execute("INSERT INTO tiles VALUES(0,0,0,?1)", [png]).unwrap(); + } +} + +fn paused_map(path: &std::path::Path, kind: TileKind) -> (Map, render::View, mpsc::Receiver<()>, Arc) { + let mut map = Map::new("Archive"); + map.graticule = false; + map.camera = Camera::new(data::Coord::new(0.0, 0.0), 100_000.0); + map.tiles.push(TileLayer::new("Below", kind, path.to_string_lossy())); + let view = render::View { width: 128.0, height: 128.0, pixels_per_point: 1.0, camera: map.camera, sky: false }; + let old = render::render_placed(&mut map, &view).unwrap(); + assert_eq!(old.credits, ["Old archive credits"]); + assert!(old.pixmap.pixel(8, 8).unwrap().red() > 200); + let mut ds = Dataset::new("Between", GeometryType::Polyline, Vec::new()); + ds.push(Some(Geometry::Line(vec![vec![data::Coord::new(0.0, 0.0), data::Coord::new(0.1, 0.1)]])), Vec::new()); + let info = SourceInfo { + name: ds.name.clone(), + format: "test", + geometry_type: ds.geometry_type, + fields: Vec::new(), + extent: ds.extent(), + count: Some(1), + crs: "EPSG:4326".into(), + indexed: true, + notes: Vec::new(), + zm_dropped: false, + }; + let id = map.add_dataset(ds.clone()); + let (paused, receive) = mpsc::sync_channel(1); + let resume = Arc::new(Barrier::new(2)); + map.layer_mut(id).unwrap().source = Some(SourceLayer::from_path( + "pause.shp", + None, + Box::new(PauseSource { info, ds, once: AtomicBool::new(false), paused, resume: resume.clone() }), + )); + let mut above = map.tiles[0].clone(); + above.name = "Above".into(); + above.over = Some(id); + map.tiles.insert(0, above); + (map, view, receive, resume) +} + +fn page(map: &Map) -> map::layout::Layout { + use map::layout::{ElementKind, FrameShape, Layout}; + let mut page = Layout::new("Archive", 40.0, 40.0); + page.add( + "Map", + (0.0, 0.0, 40.0, 40.0), + ElementKind::MapFrame { + map: map.name.clone(), + center: data::Coord::new(0.0, 0.0), + scale: 300_000_000.0, + shape: FrameShape::Rectangle, + grid: None, + extent_of: None, + extent_color: map::Rgba::rgb(0, 0, 0), + extent_width: 1.0, + bearing: 0.0, + pitch: 0.0, + }, + ); + page +} + +#[test] +fn archive_reload_refuses_spanning_exports_without_replacing_destination() { + for kind in [TileKind::PmTiles, TileKind::MbTiles] { + for route in 0..6 { + let dir = Fixture::new(); + let path = dir.path().join(format!("target.{}", kind.code())); + let replacement = dir.path().join(format!("replacement.{}", kind.code())); + archive(&path, kind, false); + archive(&replacement, kind, true); + let (mut map, view, paused, resume) = paused_map(&path, kind); + let mut warm = map.clone(); + warm.layers.clear(); + warm.tiles.truncate(1); + warm.tiles[0].over = None; + let destination = dir.path().join("map.png"); + std::fs::write(&destination, b"existing destination").unwrap(); + let output = destination.clone(); + let worker = std::thread::spawn(move || { + let result = match route { + 0 => render::render_png(&mut map, &view), + 1 => render::render_placed(&mut map, &view).and_then(|p| p.pixmap.encode_png().map_err(|e| e.to_string())), + 2 => render::render_drawing(&mut map, &view) + .map(|drawing| render::pdf::write(&[render::pdf::Page::drawing(drawing, 40.0, 40.0)], "Archive")), + 3 => render::layout::render(&page(&map), &mut [map], 96.0, &Default::default()) + .and_then(|p| p.encode_png().map_err(|e| e.to_string())), + 4 => render::layout::render_drawing(&page(&map), &mut [map], 96.0, &Default::default()) + .map(|(drawing, _)| render::pdf::write(&[render::pdf::Page::drawing(drawing, 40.0, 40.0)], "Archive")), + _ => render::svg::layout_svg(&page(&map), &mut [map], 96.0, &Default::default()).map(|(svg, _)| svg.into_bytes()), + }; + if let Ok(bytes) = &result { + data::io::write_atomic(&output, bytes).unwrap(); + } + result + }); + paused.recv_timeout(std::time::Duration::from_secs(30)).unwrap(); + data::mbtiles::close(&path); + std::fs::rename(&replacement, &path).unwrap(); + cache::reload_archive(path.to_str().unwrap()); + let fresh = render::render_placed(&mut warm, &view); + resume.wait(); + let result = worker.join().unwrap(); + let fresh = fresh.unwrap(); + assert_eq!(fresh.credits, ["New archive credits"]); + assert!(fresh.pixmap.pixel(8, 8).unwrap().green() > 200); + let error = result.unwrap_err(); + assert!(error.contains("archive changed during rendering") && error.contains("export again"), "{error}"); + assert_eq!(std::fs::read(&destination).unwrap(), b"existing destination"); + data::mbtiles::close(&path); + } + } +} + +#[test] +fn archive_revision_ignores_other_sources_and_tile_arrivals() { + let dir = Fixture::new(); + let path = dir.path().join("target.pmtiles"); + archive(&path, TileKind::PmTiles, false); + let mut map = Map::new("Revisions"); + map.tiles.push(TileLayer::new("Target", TileKind::PmTiles, path.to_string_lossy())); + let revision = render::archive_revisions([&map]); + cache::reload_archive("/render-revision-unrelated.mbtiles"); + assert!(revision.current([&map])); + let arrived_before = cache::generation(); + let view = render::View { width: 32.0, height: 32.0, pixels_per_point: 1.0, camera: map.camera, sky: false }; + render::render_placed(&mut map, &view).unwrap(); + assert_ne!(cache::generation(), arrived_before); + assert!(revision.current([&map])); + cache::reload_archive(&map.tiles[0].url); + assert!(!revision.current([&map])); +} + +#[test] +fn a_spanning_render_keeps_progressive_arrivals_and_unrelated_reloads() { + for kind in [TileKind::PmTiles, TileKind::MbTiles] { + let dir = Fixture::new(); + let target = dir.path().join(format!("target.{}", kind.code())); + let unrelated = dir.path().join(format!("other.{}", kind.code())); + archive(&target, kind, false); + archive(&unrelated, kind, true); + let (mut map, view, paused, resume) = paused_map(&target, kind); + let worker = std::thread::spawn(move || render::render_placed(&mut map, &view)); + paused.recv_timeout(std::time::Duration::from_secs(30)).unwrap(); + cache::reload_archive(unrelated.to_str().unwrap()); + let mut other = Map::new("Unrelated"); + other.tiles.push(TileLayer::new("Other", kind, unrelated.to_string_lossy())); + let arrived_before = cache::generation(); + let fresh = render::render_placed(&mut other, &view); + resume.wait(); + let result = worker.join().unwrap().unwrap(); + fresh.unwrap(); + assert_ne!(cache::generation(), arrived_before); + assert_eq!(result.credits, ["Old archive credits"]); + assert!(result.pixmap.pixel(8, 8).unwrap().red() > 200); + data::mbtiles::close(&target); + data::mbtiles::close(&unrelated); + } +} + +#[test] +fn archive_reload_discards_spanning_cpu_gpu_and_drape_passes() { + for kind in [TileKind::PmTiles, TileKind::MbTiles] { + for route in 0..4 { + let dir = Fixture::new(); + let path = dir.path().join(format!("target.{}", kind.code())); + let replacement = dir.path().join(format!("replacement.{}", kind.code())); + archive(&path, kind, false); + archive(&replacement, kind, true); + let (mut map, view, paused, resume) = paused_map(&path, kind); + let mut warm = map.clone(); + warm.layers.clear(); + warm.tiles.truncate(1); + warm.tiles[0].over = None; + let worker = std::thread::spawn(move || match route { + 0 => render::render_backend(&mut map, &view, render::Backend::Cpu).is_none(), + 1 => render::render_backend(&mut map, &view, render::Backend::Gpu).is_none(), + 2 => { + let view = render::View { camera: Camera { pitch: 30.0, ..view.camera }, ..view }; + render::render_tilt_drape(&mut map, &view, 0.0, 128).is_empty() + } + _ => render::render_globe_drape(&mut map, &view, &[data::Extent::new(-10.0, -10.0, 10.0, 10.0)], 128).is_empty(), + }); + paused.recv_timeout(std::time::Duration::from_secs(30)).unwrap(); + data::mbtiles::close(&path); + std::fs::rename(&replacement, &path).unwrap(); + cache::reload_archive(path.to_str().unwrap()); + let fresh = render::render_placed(&mut warm, &view); + resume.wait(); + let refused = worker.join().unwrap(); + fresh.unwrap(); + assert!(refused, "mixed archive revision escaped route {route} ({kind:?})"); + data::mbtiles::close(&path); + } + } +} + +#[test] +fn a_reload_between_series_pages_refuses_the_engine_pdf_without_replacing_destination() { + struct BetweenPages { + pages: std::sync::atomic::AtomicU64, + paused: mpsc::SyncSender<()>, + resume: Arc, + } + impl data::progress::Probe for BetweenPages { + fn cancelled(&self) -> bool { + false + } + fn begin(&self, _: u64) {} + fn advance(&self, n: u64) { + if self.pages.fetch_add(n, Ordering::SeqCst) == 0 { + self.paused.send(()).unwrap(); + self.resume.wait(); + } + } + } + + for kind in [TileKind::PmTiles, TileKind::MbTiles] { + let dir = Fixture::new(); + let path = dir.path().join(format!("target.{}", kind.code())); + let replacement = dir.path().join(format!("replacement.{}", kind.code())); + archive(&path, kind, false); + archive(&replacement, kind, true); + let mut map = Map::new("Archive"); + map.graticule = false; + map.camera = Camera::new(data::Coord::new(0.0, 0.0), 100_000.0); + map.tiles.push(TileLayer::new("Archive", kind, path.to_string_lossy())); + let mut index = Dataset::new("Pages", GeometryType::Point, Vec::new()); + for x in [0.0, 1.0] { + index.push(Some(Geometry::Point(data::Coord::new(x, 0.0))), Vec::new()); + } + map.add_dataset(index); + let layout = page(&map); + let mut warm = map.clone(); + warm.layers.clear(); + let view = render::View { width: 32.0, height: 32.0, pixels_per_point: 1.0, camera: map.camera, sky: false }; + let old = render::render_placed(&mut warm, &view).unwrap(); + assert_eq!(old.credits, ["Old archive credits"], "{:?}", old.gaps); + let mut session = crate::Session::new(crate::Project::default()); + session.project.maps = vec![map]; + session.project.layouts = vec![layout]; + session.active_layout = Some(0); + let series = session.run("layout.series", &serde_json::json!({"layer":"Pages"})).unwrap(); + assert_eq!(series["pages"], 2); + let destination = dir.path().join("series.pdf"); + std::fs::write(&destination, b"existing destination").unwrap(); + // The command's analysis wrapper installs its own progress carrier. Run its actual + // export work under this probe to pause exactly between complete pages, without a hook. + let work = crate::cmd::layout::export_work(&session, &serde_json::json!({"path":destination,"dpi":36,"all":true})).unwrap(); + let (paused, receive) = mpsc::sync_channel(1); + let resume = Arc::new(Barrier::new(2)); + let probe = Arc::new(BetweenPages { pages: std::sync::atomic::AtomicU64::new(0), paused, resume: resume.clone() }); + let source = path.clone(); + let reloaded = std::thread::spawn(move || { + receive.recv_timeout(std::time::Duration::from_secs(30)).unwrap(); + // The layout's page-progress callback runs after page one is complete and before + // page two captures its own render revision: only the whole-document guard spans this. + data::mbtiles::close(&source); + std::fs::rename(replacement, &source).unwrap(); + cache::reload_archive(source.to_str().unwrap()); + let fresh = render::render_placed(&mut warm, &view); + resume.wait(); + fresh + }); + let installed = data::progress::install(Some(probe.clone())); + let wrote = work().unwrap(); + drop(installed); + let fresh = reloaded.join().unwrap().unwrap(); + assert_eq!(fresh.credits, ["New archive credits"]); + assert!(fresh.pixmap.pixel(8, 8).unwrap().green() > 200); + assert_eq!(probe.pages.load(Ordering::SeqCst), 2, "both individually coherent pages were drawn"); + let error = wrote.then.unwrap()(&mut session, &mut serde_json::Value::Null).unwrap_err(); + assert!(error.contains("archive changed during rendering"), "{error}"); + assert_eq!(std::fs::read(&destination).unwrap(), b"existing destination"); + session.run("layout.export", &serde_json::json!({"path":destination,"dpi":36,"all":true})).unwrap(); + let pdf = std::fs::read(&destination).unwrap(); + assert!(pdf.starts_with(b"%PDF") && pdf.windows(b"/Count 2".len()).any(|part| part == b"/Count 2")); + } +} diff --git a/crates/engine/src/tests/review_boundaries.rs b/crates/engine/src/tests/review_boundaries.rs index 60c8e9bb37..0889535336 100644 --- a/crates/engine/src/tests/review_boundaries.rs +++ b/crates/engine/src/tests/review_boundaries.rs @@ -370,3 +370,105 @@ fn permission_review_script_line_limit_refuses_before_any_edit() { s.run("script.run", &json!({"script":prefix})).unwrap(); assert!(s.map().layer_by_name("Renamed").is_some(), "the exact boundary remains supported"); } + +#[test] +fn permission_review_archive_actions_bind_the_selected_route_and_stack_position() { + for command in ["map.reloadTiles", "map.repairTiles"] { + for selector in [json!({"index":0}), json!({"name":"Review archive"}), json!({"id":"review-archive"})] { + for reorder in [false, true] { + let mut s = world(); + let mut tile = + rustgis_map::TileLayer::new("Review archive", rustgis_map::TileKind::PmTiles, "/not-opened/review-one.pmtiles"); + tile.id = "review-archive".into(); + s.map_mut().tiles = + vec![tile, rustgis_map::TileLayer::new("Other archive", rustgis_map::TileKind::PmTiles, "/not-opened/other.pmtiles")]; + let mut params = selector.clone(); + if command == "map.repairTiles" { + params["path"] = json!("/not-opened/replacement.pmtiles"); + } + let call = Call::command(command, params); + let scope = Scope::assistant(1464, Arc::new(Live)); + let Outcome::NeedsApproval(approved) = s.review(&scope, &call) else { panic!("archive action needs review") }; + let remembered = approved.remember().unwrap(); + if reorder { + s.map_mut().tiles.swap(0, 1); + } else { + s.map_mut().tiles[0].url = "/not-opened/review-two.pmtiles".into(); + } + let Outcome::NeedsApproval(changed) = s.review(&scope, &call) else { panic!("changed action needs review") }; + assert!(!changed.covered_by(&remembered), "{command}: {selector}, reorder={reorder}"); + let before = s.map().tiles.clone(); + let undo = s.undo_depth(); + assert!(s.run_remembered(scope, &remembered, &call).is_err(), "stale consent must fail before opening any archive"); + assert_eq!(s.map().tiles, before); + assert_eq!(s.undo_depth(), undo); + } + } + } +} + +#[test] +fn permission_review_archive_places_are_redacted_but_raw_routes_remain_bound() { + for command in ["map.reloadTiles", "map.repairTiles"] { + let mut s = world(); + s.map_mut().tiles = vec![ + rustgis_map::TileLayer::new( + "Review archive", + rustgis_map::TileKind::PmTiles, + "https://archive-review.invalid/private.pmtiles?token=RGSYNTH_archive_one", + ), + rustgis_map::TileLayer::new("Other archive", rustgis_map::TileKind::PmTiles, "/not-opened/other.pmtiles"), + ]; + // Preserve index > name > id selector precedence, even for a conflicting request. + let mut params = json!({"index":0,"name":"Other archive","id":"not-the-target"}); + if command == "map.repairTiles" { + params["path"] = json!("/not-opened/replacement.pmtiles"); + } + let call = Call::command(command, params); + let scope = Scope::assistant(1465, Arc::new(Live)); + let Outcome::NeedsApproval(approved) = s.review(&scope, &call) else { panic!("archive action needs review") }; + assert!(approved.review().details.iter().flat_map(|d| &d.places).any(|p| p.contains("archive-review.invalid"))); + assert!(!format!("{:?}", approved.review()).contains("RGSYNTH_archive_")); + let remembered = approved.remember().unwrap(); + s.map_mut().tiles[1].url = "/not-opened/unselected-change.pmtiles".into(); + let Outcome::NeedsApproval(unselected) = s.review(&scope, &call) else { panic!("still needs review") }; + assert!(unselected.covered_by(&remembered), "only the executor's selected archive is bound"); + s.map_mut().tiles[0].url = "https://archive-review.invalid/private.pmtiles?token=RGSYNTH_archive_two".into(); + let Outcome::NeedsApproval(changed) = s.review(&scope, &call) else { panic!("still needs review") }; + assert!(!changed.covered_by(&remembered), "redaction must not erase private route identity"); + } +} + +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn permission_review_unchanged_real_archive_reload_and_repair_execute_with_remembered_consent() { + let dir = std::env::temp_dir().join(format!("rustgis-review-archive-actions-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let linked = dir.join("linked.pmtiles"); + let replacement = dir.join("replacement.pmtiles"); + for path in [&linked, &replacement] { + std::fs::write(path, empty_pmtiles([-24.5, 63.3, -13.5, 66.6])).unwrap(); + } + for command in ["map.reloadTiles", "map.repairTiles"] { + let mut s = world(); + s.run("map.addTiles", &json!({"url":linked,"name":"Approved archive","overlay":true})).unwrap(); + let mut params = json!({"name":"Approved archive"}); + if command == "map.repairTiles" { + params["path"] = json!(replacement); + } + let call = Call::command(command, params); + let scope = Scope::assistant(1466, Arc::new(Live)); + let Outcome::NeedsApproval(approved) = s.review(&scope, &call) else { panic!("archive action needs review") }; + assert!(approved.review().details.iter().flat_map(|d| &d.places).any(|p| p.contains("linked.pmtiles"))); + if command == "map.repairTiles" { + assert!(approved.review().details.iter().flat_map(|d| &d.places).any(|p| p.contains("replacement.pmtiles"))); + } + let remembered = approved.remember().unwrap(); + let Outcome::NeedsApproval(unchanged) = s.review(&scope, &call) else { panic!("still needs review") }; + assert!(unchanged.covered_by(&remembered)); + let result = s.run_remembered(scope, &remembered, &call).unwrap(); + assert_eq!(result["path"], json!(if command == "map.repairTiles" { &replacement } else { &linked })); + assert_eq!(s.map().tiles[0].name, "Approved archive"); + } + std::fs::remove_dir_all(dir).unwrap(); +} diff --git a/crates/engine/tests/portable_projects.rs b/crates/engine/tests/portable_projects.rs index a889aa503f..91d6a25768 100644 --- a/crates/engine/tests/portable_projects.rs +++ b/crates/engine/tests/portable_projects.rs @@ -4,6 +4,8 @@ mod common; #[path = "portable_projects/reload.rs"] mod reload; +#[path = "portable_projects/reload_tiles.rs"] +mod reload_tiles; #[path = "portable_projects/repair.rs"] mod repair; diff --git a/crates/engine/tests/portable_projects/reload_tiles.rs b/crates/engine/tests/portable_projects/reload_tiles.rs new file mode 100644 index 0000000000..15c89aebf8 --- /dev/null +++ b/crates/engine/tests/portable_projects/reload_tiles.rs @@ -0,0 +1,824 @@ +use super::common::{Fixture, session}; +use rustgis_engine::{ + Session, + map::TileLayer, + render::{ + self, + rustgis_tiles::{cache, mbtiles, pmtiles, source}, + }, +}; +use serde_json::{Value, json}; +use std::{path::Path, sync::Arc, time::Duration}; + +/// Archive reloads share process-wide caches, and an MBTiles forget clears every resolved style. +/// Serializing these tests and dropping each test's archives keeps Arc-identity assertions about +/// retention rather than about another test's cache work or capacity. +struct Scoped { + fixture: Fixture, + _turn: std::sync::MutexGuard<'static, ()>, +} + +impl Scoped { + fn new(label: &str) -> Self { + static TURN: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let turn = TURN.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + Self { fixture: Fixture::new(label), _turn: turn } + } +} + +impl std::ops::Deref for Scoped { + type Target = Fixture; + fn deref(&self) -> &Fixture { + &self.fixture + } +} + +impl Drop for Scoped { + fn drop(&mut self) { + for file in std::fs::read_dir(&self.fixture.root).into_iter().flatten().flatten() { + rustgis_engine::data::mbtiles::close(&file.path()); + cache::reload_archive(&file.path().display().to_string()); + } + } +} + +fn picture(color: [u8; 4]) -> Vec { + let mut p = tiny_skia::Pixmap::new(256, 256).unwrap(); + p.fill(tiny_skia::Color::from_rgba8(color[0], color[1], color[2], color[3])); + p.encode_png().unwrap() +} + +fn write_archive(path: &Path, color: [u8; 4], credit: &str) { + let png = picture(color); + if path.extension().unwrap() == "pmtiles" { + let a = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Png, + gzipped_tiles: false, + bounds: [-180.0, -85.0, 180.0, 85.0], + metadata: json!({"attribution": credit}), + }; + std::fs::write(path, pmtiles::write(vec![(0, 0, 0, png)], &a).unwrap()).unwrap(); + } else { + let c = rusqlite::Connection::open(path).unwrap(); + c.execute_batch("CREATE TABLE IF NOT EXISTS metadata (name TEXT, value TEXT); CREATE TABLE IF NOT EXISTS tiles (zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_data BLOB); BEGIN IMMEDIATE; DELETE FROM metadata; DELETE FROM tiles;").unwrap(); + c.execute("INSERT INTO metadata VALUES ('format','png'),('minzoom','0'),('maxzoom','0'),('attribution',?1)", [credit]).unwrap(); + c.execute("INSERT INTO tiles VALUES (0,0,0,?1)", [png]).unwrap(); + c.execute_batch("COMMIT").unwrap(); + } +} + +fn tile_url(t: &TileLayer) -> String { + let template = if t.kind == rustgis_engine::map::TileKind::MbTiles { mbtiles::template(&t.url) } else { pmtiles::template(&t.url) }; + template.replace("{z}", "0").replace("{x}", "0").replace("{y}", "0") +} + +fn decoded(t: &TileLayer) -> Arc { + let url = tile_url(t); + for _ in 0..100 { + match cache::get_tile(&url, cache::Want::Image) { + cache::State::Ready(p) => return p, + cache::State::Failed(e) => panic!("{e}"), + cache::State::Pending => cache::wait(Duration::from_millis(100)), + } + } + panic!("tile did not arrive: {url}"); +} + +fn style(t: &TileLayer) -> Arc { + for _ in 0..100 { + if t.kind == rustgis_engine::map::TileKind::MbTiles { + // MBTiles metadata workers have their own bounded wait, not cache::in_flight. + let _ = mbtiles::info_ready(&t.url, Duration::from_millis(100)).unwrap(); + } + match source::resolve(&render::tile_spec(t)) { + source::Resolve::Ready(r) => return r, + source::Resolve::Failed(e) => panic!("{e}"), + source::Resolve::Pending => cache::wait(Duration::from_millis(100)), + } + } + panic!("style did not resolve"); +} + +fn assert_color(p: &cache::Payload, expected: [u8; 4]) { + let cache::Payload::Image(p) = p else { panic!("not an image") }; + assert_eq!(p.data().get(..4), Some(expected.as_slice())); +} + +fn assert_render(s: &mut Session, expected: [u8; 4], credit: &str) { + let v = render::View { width: 300.0, height: 200.0, pixels_per_point: 1.0, camera: s.map().camera, sky: false }; + for _ in 0..100 { + let r = render::render(s.map_mut(), &v).unwrap(); + if !r.tiles_pending { + assert_eq!(r.pixmap.pixel(150, 100).unwrap().red(), expected[0]); + assert_eq!(r.pixmap.pixel(150, 100).unwrap().green(), expected[1]); + assert_eq!(r.pixmap.pixel(150, 100).unwrap().blue(), expected[2]); + assert!(r.attribution.iter().any(|a| a.contains(credit)), "{:?}", r.attribution); + return; + } + cache::wait(Duration::from_millis(100)); + } + panic!("render remained pending"); +} + +#[test] +fn manual_archive_reload_reads_new_pixels_and_credits_without_touching_other_sources() { + for ext in ["pmtiles", "mbtiles"] { + let f = Scoped::new(&format!("reload-{ext}")); + let path = f.file(&format!("linked.{ext}")); + let other = f.file(&format!("unrelated.{ext}")); + write_archive(&path, [255, 0, 0, 255], "Old credits"); + write_archive(&other, [0, 0, 255, 255], "Unrelated credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url": other, "name":"Other", "overlay":true})).unwrap(); + s.run("map.addTiles", &json!({"url": path, "name":"Linked", "overlay":true})).unwrap(); + s.run("map.goToXY", &json!({"lon":10,"lat":20,"scale":50_000_000})).unwrap(); + let i = s.map().tiles.iter().position(|t| t.name == "Linked").unwrap(); + s.map_mut().tiles[i].id = "linked-archive".into(); + s.map_mut().tiles[i].blend = rustgis_engine::map::effects::Blend::Normal; + s.map_mut().tiles[i].expanded = true; + let before = s.map().tiles.clone(); + let camera = s.map().camera; + let other_tile = before.iter().find(|t| t.name == "Other").unwrap(); + let other_pixels = decoded(other_tile); + let other_style = style(other_tile); + let old_pixels = decoded(&before[i]); + assert_color(&old_pixels, [255, 0, 0, 255]); + assert!(style(&before[i]).attribution.iter().any(|a| a.contains("Old credits"))); + assert_render(&mut s, [255, 0, 0, 255], "Old credits"); + write_archive(&path, [0, 255, 0, 255], "New credits"); + // Existing retained data stays until the explicit command. + assert!(Arc::ptr_eq(&decoded(&before[i]), &old_pixels)); + assert_eq!(s.run("map.reloadTiles", &json!({"id":"linked-archive"})).unwrap(), json!({"index":i,"path":path})); + assert_eq!(s.map().tiles, before); + assert_eq!(s.map().camera, camera); + assert_color(&decoded(&before[i]), [0, 255, 0, 255]); + let fresh = style(&before[i]); + assert!(fresh.attribution.iter().any(|a| a.contains("New credits"))); + assert!(!fresh.attribution.iter().any(|a| a.contains("Old credits"))); + assert!(Arc::ptr_eq(&decoded(other_tile), &other_pixels)); + assert!(Arc::ptr_eq(&style(other_tile), &other_style)); + assert_render(&mut s, [0, 255, 0, 255], "New credits"); + assert_eq!(s.undo_label(), Some("Reload Tile Archive")); + s.run("edit.undo", &Value::Null).unwrap(); + assert_eq!(s.map().tiles, before); + assert_color(&decoded(&before[i]), [0, 255, 0, 255]); // Undo is not file recovery. + s.run("edit.redo", &Value::Null).unwrap(); + assert_eq!(s.map().camera, camera); + // All three selector forms, including case-insensitive names. + s.run("map.reloadTiles", &json!({"name":"lInKeD"})).unwrap(); + s.run("map.reloadTiles", &json!({"index":i})).unwrap(); + rustgis_engine::data::mbtiles::close(&path); + rustgis_engine::data::mbtiles::close(&other); + } +} + +fn rejected(s: &mut Session, params: Value, expected: &str) { + let before = s.map().tiles.clone(); + let undo = s.undo_label(); + let mark = s.notice_mark(); + let error = s.run("map.reloadTiles", ¶ms).unwrap_err(); + assert!(error.contains(expected), "{error}"); + assert_eq!(s.map().tiles, before); + assert_eq!(s.undo_label(), undo); + assert!(s.notices_since(mark).iter().any(|n| n.error && n.text.contains(expected))); +} + +#[test] +fn failed_archive_reload_retains_tiles_configuration_and_allows_retry() { + for ext in ["pmtiles", "mbtiles"] { + let f = Scoped::new(&format!("reload-failure-{ext}")); + let path = f.file(&format!("linked.{ext}")); + write_archive(&path, [255, 0, 0, 255], "Retained credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path})).unwrap(); + let before = s.map().tiles[0].clone(); + let pixels = decoded(&before); + let old_style = style(&before); + // No live read is held by this test while changing the file on Windows. + rustgis_engine::data::mbtiles::close(&path); + std::fs::remove_file(&path).unwrap(); + rejected(&mut s, json!({"index":0}), "retry Reload Data"); + std::fs::write(&path, b"truncated").unwrap(); + rejected(&mut s, json!({"index":0}), "retry Reload Data"); + assert!(Arc::ptr_eq(&decoded(&before), &pixels)); + assert!(Arc::ptr_eq(&style(&before), &old_style)); + write_archive_after_truncation(&path, [0, 255, 0, 255], "Recovered credits"); + s.run("map.reloadTiles", &json!({"index":0})).unwrap(); + assert_color(&decoded(&before), [0, 255, 0, 255]); + assert_eq!(s.map().tiles[0], before); + rustgis_engine::data::mbtiles::close(&path); + } +} + +fn write_archive_after_truncation(path: &Path, color: [u8; 4], credit: &str) { + std::fs::remove_file(path).unwrap(); + write_archive(path, color, credit); +} + +#[test] +fn locked_mbtiles_reload_fails_without_evicting_retained_data_then_retries() { + let f = Scoped::new("reload-locked"); + let path = f.file("locked.mbtiles"); + write_archive(&path, [255, 0, 0, 255], "Before lock"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path})).unwrap(); + let before = s.map().tiles[0].clone(); + let pixels = decoded(&before); + let c = rusqlite::Connection::open(&path).unwrap(); + c.execute_batch("BEGIN EXCLUSIVE; UPDATE metadata SET value='After lock' WHERE name='attribution';").unwrap(); + rejected(&mut s, json!({"index":0}), "retry Reload Data"); + assert!(Arc::ptr_eq(&decoded(&before), &pixels)); + c.execute_batch("COMMIT").unwrap(); + s.run("map.reloadTiles", &json!({"index":0})).unwrap(); + assert!(style(&before).attribution.iter().any(|a| a.contains("After lock"))); + rustgis_engine::data::mbtiles::close(&path); +} + +// Unix permits replacing an open archive's directory entry. Windows may refuse the rename +// while SQLite holds the original file; this test does not promise otherwise. +#[cfg(unix)] +#[test] +fn replacing_an_open_mbtiles_archive_reloads_the_new_file_not_its_pooled_inode() { + let f = Scoped::new("reload-mbtiles-replaced-inode"); + let path = f.file("linked.mbtiles"); + let replacement = f.file("replacement.mbtiles"); + write_archive(&path, [255, 0, 0, 255], "Old inode"); + write_archive(&replacement, [0, 255, 0, 255], "Replacement inode"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path})).unwrap(); + let before = s.map().tiles[0].clone(); + assert_color(&decoded(&before), [255, 0, 0, 255]); + assert!(style(&before).attribution.iter().any(|a| a.contains("Old inode"))); + std::fs::rename(&replacement, &path).unwrap(); + s.run("map.reloadTiles", &json!({"index":0})).unwrap(); + assert_eq!(s.map().tiles[0], before); + assert_color(&decoded(&before), [0, 255, 0, 255]); + assert!(style(&before).attribution.iter().any(|a| a.contains("Replacement inode"))); + rustgis_engine::data::mbtiles::close(&path); +} + +#[test] +fn incompatible_reload_keeps_archive_presentation_and_undo_restores_prior_error() { + for ext in ["pmtiles", "mbtiles"] { + let f = Scoped::new(&format!("reload-incompatible-{ext}")); + let path = f.file(&format!("vector.{ext}")); + write_vector(&path); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path})).unwrap(); + s.map_mut().tiles[0].layers = vec!["places".into()]; + s.map_mut().tiles[0].style_layers = + r##"[{"id":"places","type":"circle","source-layer":"places","paint":{"circle-color":"#2266aa"}}]"##.into(); + s.map_mut().tiles[0].hidden_layers = vec!["roads".into()]; + s.map_mut().tiles[0].opacity = 0.4; + s.map_mut().tiles[0].visible = false; + s.map_mut().tiles[0].blend = rustgis_engine::map::effects::Blend::Multiply; + s.map_mut().tiles[0].id = "presentation".into(); + s.map_mut().tiles[0].error = Some("old unavailable archive".into()); + let before = s.map().tiles[0].clone(); + write_archive(&path, [0, 255, 0, 255], "Raster now"); + rejected(&mut s, json!({"index":0}), "raster tiles"); + write_vector(&path); + s.run("map.reloadTiles", &json!({"index":0})).unwrap(); + let mut expected = before.clone(); + expected.error = None; + assert_eq!(s.map().tiles[0], expected); + s.run("edit.undo", &Value::Null).unwrap(); + assert_eq!(s.map().tiles[0], before); + rustgis_engine::data::mbtiles::close(&path); + } +} + +fn write_vector(path: &Path) { + if path.extension().unwrap() == "pmtiles" { + super::common::archive_file(path); + } else { + write_archive(path, [255, 0, 0, 255], "Vector"); + let c = rusqlite::Connection::open(path).unwrap(); + c.execute("UPDATE metadata SET value='pbf' WHERE name='format'", []).unwrap(); + let tile = rustgis_engine::render::rustgis_tiles::mvt::encode(&rustgis_engine::render::rustgis_tiles::mvt::Tile { layers: vec![] }); + c.execute("UPDATE tiles SET tile_data=?1", [tile]).unwrap(); + } +} + +#[test] +fn archive_reload_rejects_remote_upload_and_invalid_selectors() { + let mut s = session(); + s.run("map.addTiles", &json!({"url":"https://example.invalid/a.pmtiles"})).unwrap(); + rejected(&mut s, json!({"index":0}), "linked local PMTiles or MBTiles"); + // Uploaded archives are snapshots, not live paths. + s.map_mut().tiles[0] = TileLayer::new("Snapshot", rustgis_engine::map::TileKind::MbTiles, "upload:local.mbtiles"); + rejected(&mut s, json!({"index":0}), "add snapshots/uploads again"); + for (p, expected) in [ + (json!({}), "required"), + (json!({"index":-1}), "no tile layer"), + (json!({"index":0.5}), "no tile layer"), + (json!({"index":8}), "no tile layer"), + (json!({"name":"absent"}), "no tile layer"), + (json!({"id":"absent"}), "no tile layer"), + ] { + rejected(&mut s, p, expected); + } +} + +#[test] +fn invalid_raster_and_unreadable_tile_pages_do_not_evict_retained_mbtiles() { + for damaged_page in [false, true] { + let f = Scoped::new(if damaged_page { "reload-tile-page" } else { "reload-png-signature" }); + let path = f.file("linked.mbtiles"); + write_archive(&path, [255, 0, 0, 255], "Retained credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path,"overlay":true})).unwrap(); + let before = s.map().tiles[0].clone(); + let pixels = decoded(&before); + let credits = style(&before); + rustgis_engine::data::mbtiles::close(&path); + let c = rusqlite::Connection::open(&path).unwrap(); + if damaged_page { + let page_size = c.query_row("PRAGMA page_size", [], |r| r.get::<_, u32>(0)).unwrap() as usize; + let root = c.query_row("SELECT rootpage FROM sqlite_master WHERE name='tiles'", [], |r| r.get::<_, u32>(0)).unwrap() as usize; + drop(c); + let mut bytes = std::fs::read(&path).unwrap(); + bytes[(root - 1) * page_size] = 0; + std::fs::write(&path, bytes).unwrap(); + // Metadata is readable; this is not another damaged-header test. + assert!(rustgis_engine::data::mbtiles::open_path(&path).is_ok()); + rustgis_engine::data::mbtiles::close(&path); + } else { + c.execute("UPDATE tiles SET tile_data=?1", [b"\x89PNG\r\n\x1a\n".as_slice()]).unwrap(); + drop(c); + } + rejected(&mut s, json!({"index":0}), "retry Reload Data"); + assert!(Arc::ptr_eq(&decoded(&before), &pixels)); + assert!(Arc::ptr_eq(&style(&before), &credits)); + write_archive_after_truncation(&path, [0, 255, 0, 255], "Recovered credits"); + s.run("map.reloadTiles", &json!({"index":0})).unwrap(); + assert_color(&decoded(&before), [0, 255, 0, 255]); + rustgis_engine::data::mbtiles::close(&path); + } +} + +fn write_leaf_archive(path: &Path, corrupt: bool) { + let archive = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Png, + gzipped_tiles: false, + bounds: [-180.0, -85.0, 180.0, 85.0], + metadata: json!({"attribution":"Leaf credits"}), + }; + let bytes = pmtiles::write(vec![(0, 0, 0, picture([0, 255, 0, 255]))], &archive).unwrap(); + let number = |at| u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()) as usize; + let mut leaf = bytes[number(8)..number(8) + number(16)].to_vec(); + if corrupt { + // One entry's final offset varint has no terminating byte. + *leaf.last_mut().unwrap() = 0x80; + } + assert!(leaf.len() < 128); + let root = [1, 0, 0, leaf.len() as u8, 1]; + let meta = &bytes[number(24)..number(24) + number(32)]; + let data = &bytes[number(56)..number(56) + number(64)]; + let mut header = bytes[..127].to_vec(); + let meta_at = 127 + root.len(); + let leaf_at = meta_at + meta.len(); + let data_at = leaf_at + leaf.len(); + for (at, value) in + [(8, 127), (16, root.len()), (24, meta_at), (32, meta.len()), (40, leaf_at), (48, leaf.len()), (56, data_at), (64, data.len())] + { + header[at..at + 8].copy_from_slice(&(value as u64).to_le_bytes()); + } + std::fs::write(path, [header.as_slice(), root.as_slice(), meta, leaf.as_slice(), data].concat()).unwrap(); +} + +#[test] +fn malformed_leaf_reload_retains_pixels_and_can_retry_with_a_valid_leaf() { + let f = Scoped::new("reload-malformed-leaf"); + let path = f.file("linked.pmtiles"); + write_archive(&path, [255, 0, 0, 255], "Retained credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path,"overlay":true})).unwrap(); + let before = s.map().tiles[0].clone(); + let pixels = decoded(&before); + let credits = style(&before); + let camera = s.map().camera; + write_leaf_archive(&path, true); + rejected(&mut s, json!({"index":0}), "retry Reload Data"); + assert!(Arc::ptr_eq(&decoded(&before), &pixels)); + assert!(Arc::ptr_eq(&style(&before), &credits)); + assert_eq!(s.map().camera, camera); + write_leaf_archive(&path, false); + s.run("map.reloadTiles", &json!({"index":0})).unwrap(); + assert_color(&decoded(&before), [0, 255, 0, 255]); +} + +#[test] +fn untrusted_workflows_cannot_reload_but_user_authored_workflows_can() { + let f = Scoped::new("reload-workflow-trust"); + let path = f.file("linked.pmtiles"); + write_archive(&path, [255, 0, 0, 255], "Before"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path,"overlay":true})).unwrap(); + let tile = s.map().tiles[0].clone(); + let old_pixels = decoded(&tile); + let task = json!({"name":"Reload task","steps":[{"title":"Reload","command":"map.reloadTiles","params":{"index":0}}]}); + let error = s.run("task.import", &json!({"text":task.to_string()})).unwrap_err(); + assert!(error.contains("imported tasks"), "{error}"); + let notebook = json!({"nbformat":4,"metadata":{"language_info":{"name":"rustgis"}},"cells":[ + {"cell_type":"code","source":"map.reloadTiles {\"index\":0}"} + ]}); + s.run("notebook.open", &json!({"text":notebook.to_string()})).unwrap(); + let answer = s.run("notebook.run", &Value::Null).unwrap(); + assert_eq!(answer["error"], 0); + assert!(s.project.notebooks.last().unwrap().cells[0].error); + assert!(Arc::ptr_eq(&decoded(&tile), &old_pixels)); + write_archive(&path, [0, 255, 0, 255], "After"); + s.run("task.new", &task).unwrap(); + s.run("task.run", &json!({"task":"Reload task"})).unwrap(); + assert_color(&decoded(&tile), [0, 255, 0, 255]); + s.run("notebook.new", &json!({"name":"User reload"})).unwrap(); + s.run("notebook.addCell", &json!({"source":"map.reloadTiles {\"index\":0}"})).unwrap(); + let answer = s.run("notebook.run", &Value::Null).unwrap(); + assert!(answer.get("error").is_none(), "{answer}"); +} + +#[test] +fn successful_reload_cancels_only_its_own_pending_archive_framing() { + let f = Scoped::new("reload-pending-view"); + let path = f.file("linked.pmtiles"); + let other = f.file("other.pmtiles"); + for file in [&path, &other] { + let archive = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Png, + gzipped_tiles: false, + bounds: [20.0, 30.0, 25.0, 35.0], + metadata: json!({}), + }; + std::fs::write(file, pmtiles::write(vec![(0, 0, 0, picture([255, 0, 0, 255]))], &archive).unwrap()).unwrap(); + } + let mut s = session(); + s.run("map.addTiles", &json!({"url":path,"overlay":true})).unwrap(); + s.run("map.addTiles", &json!({"url":other,"overlay":true})).unwrap(); + s.style_view = None; + s.run("map.goToXY", &json!({"lon":-40,"lat":-20,"scale":50_000_000})).unwrap(); + let target = s.map().tiles.iter().position(|t| t.url == path.to_str().unwrap()).unwrap(); + let tile = s.map().tiles[target].clone(); + let camera = s.map().camera; + cache::reload_archive(&tile.url); + s.style_view = Some((tile.clone(), camera, s.map().instance_id())); + s.run("map.reloadTiles", &json!({"index":target})).unwrap(); + assert!(s.style_view.is_none(), "late target metadata must no longer frame the view"); + let _ = style(&tile); + assert!(!s.follow_style_view()); + assert_eq!(s.map().camera, camera); + let other_tile = s.map().tiles.iter().find(|t| t.url != tile.url).unwrap().clone(); + cache::reload_archive(&other_tile.url); + s.style_view = Some((other_tile.clone(), camera, s.map().instance_id())); + s.run("map.reloadTiles", &json!({"index":target})).unwrap(); + assert!(s.style_view.as_ref().is_some_and(|(pending, _, _)| pending.url == other_tile.url)); + let _ = style(&other_tile); + assert!(s.follow_style_view(), "another archive's requested framing still works"); + assert_ne!(s.map().camera, camera); +} + +#[test] +fn prepared_archive_validation_does_not_evict_or_overwrite_changed_settings() { + let f = Scoped::new("reload-read-ahead"); + let path = f.file("linked.pmtiles"); + write_archive(&path, [255, 0, 0, 255], "Before"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path,"overlay":true})).unwrap(); + let tile = s.map().tiles[0].clone(); + let old_pixels = decoded(&tile); + write_archive(&path, [0, 255, 0, 255], "After"); + let control = rustgis_engine::analysis::control::Control::stoppable(); + let read = rustgis_engine::cmd::reads::read_ahead(&mut s, "map.reloadTiles", &json!({"index":0})).unwrap().run(&control).unwrap(); + assert!(Arc::ptr_eq(&decoded(&tile), &old_pixels), "validation alone must not invalidate"); + s.run("map.setTiles", &json!({"index":0,"opacity":0.4})).unwrap(); + let before = s.map().tiles.clone(); + let undo = s.undo_depth(); + assert!(rustgis_engine::cmd::reads::read_land(&mut s, read).is_err()); + assert_eq!(s.map().tiles, before); + assert_eq!(s.undo_depth(), undo); + assert_eq!(s.map().tiles[0].opacity, 0.4); + assert!(Arc::ptr_eq(&decoded(&tile), &old_pixels)); + s.run("map.reloadTiles", &json!({"index":0})).unwrap(); + assert_eq!(s.map().tiles[0].opacity, 0.4); + assert_color(&decoded(&tile), [0, 255, 0, 255]); +} + +fn archive_read(s: &mut Session, command: &str, params: &Value) -> rustgis_engine::cmd::reads::Read { + let control = rustgis_engine::analysis::control::Control::stoppable(); + rustgis_engine::cmd::reads::read_ahead(s, command, params).unwrap().run(&control).unwrap() +} + +type RetainedArchive = (TileLayer, Arc, Arc); + +fn retain_archive(t: &TileLayer) -> RetainedArchive { + (t.clone(), decoded(t), style(t)) +} + +fn reject_stale_archive(s: &mut Session, read: rustgis_engine::cmd::reads::Read, retained: &[RetainedArchive]) -> String { + let before = serde_json::to_value(&s.project).unwrap(); + let view = s.style_view.clone(); + let undo = s.undo_depth(); + let mark = s.notice_mark(); + let error = rustgis_engine::cmd::reads::read_land(s, read).unwrap_err(); + assert_eq!(serde_json::to_value(&s.project).unwrap(), before, "failed landing must retain source, presentation and view"); + assert_eq!(s.style_view, view, "failed landing must not cancel pending framing"); + assert_eq!(s.undo_depth(), undo); + assert!(s.notices_since(mark).iter().all(|n| n.error), "failed landing must not emit a success notice"); + for (tile, pixels, credits) in retained { + assert!(Arc::ptr_eq(&decoded(tile), pixels), "failed landing must not evict {}", tile.url); + assert!(Arc::ptr_eq(&style(tile), credits), "failed landing must not evict metadata for {}", tile.url); + } + error +} + +#[test] +fn changed_or_removed_archive_after_worker_validation_retains_data_and_retries() { + for ext in ["pmtiles", "mbtiles"] { + for command in ["map.reloadTiles", "map.repairTiles"] { + for remove in [false, true] { + let f = Scoped::new(&format!("archive-freshness-{ext}-{command}-{remove}")); + let linked = f.file(&format!("linked.{ext}")); + let replacement = f.file(&format!("replacement.{ext}")); + write_archive(&linked, [255, 0, 0, 255], "Retained credits"); + write_archive(&replacement, [0, 0, 255, 255], "Replacement credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":linked,"overlay":true})).unwrap(); + s.map_mut().tiles[0].opacity = 0.4; + s.map_mut().tiles[0].expanded = true; + let tile = s.map().tiles[0].clone(); + let mut candidate = tile.clone(); + let mut params = json!({"index":0}); + let path = if command == "map.repairTiles" { + params["path"] = json!(replacement); + candidate.url = replacement.display().to_string(); + &replacement + } else { + &linked + }; + let retained = vec![retain_archive(&tile), retain_archive(&candidate)]; + s.style_view = Some((tile.clone(), s.map().camera, s.map().instance_id())); + let read = archive_read(&mut s, command, ¶ms); + rustgis_engine::data::mbtiles::close(path); + if remove { + std::fs::remove_file(path).unwrap(); + } else { + let len = std::fs::metadata(path).unwrap().len(); + std::fs::OpenOptions::new().write(true).open(path).unwrap().set_len(len + 1).unwrap(); + } + let error = reject_stale_archive(&mut s, read, &retained); + assert!(error.contains("retry"), "{error}"); + let expected = if !remove { + "changed while it was read" + } else if command == "map.repairTiles" { + "there's no such file" + } else { + "observe archive metadata" + }; + assert!(error.contains(expected), "{error}"); + if !remove { + std::fs::remove_file(path).unwrap(); + } + write_archive(path, [0, 255, 0, 255], "Retry credits"); + s.run(command, ¶ms).unwrap(); + assert_eq!(s.map().tiles[0].opacity, 0.4); + assert!(s.map().tiles[0].expanded); + assert_color(&decoded(&s.map().tiles[0]), [0, 255, 0, 255]); + assert!(style(&s.map().tiles[0]).attribution.iter().any(|a| a.contains("Retry credits"))); + rustgis_engine::data::mbtiles::close(&linked); + rustgis_engine::data::mbtiles::close(&replacement); + } + } + } +} + +#[cfg(unix)] +#[test] +fn replacing_validated_archive_directory_entry_rejects_landing_even_with_identical_contents() { + use std::os::unix::fs::MetadataExt; + for ext in ["pmtiles", "mbtiles"] { + for command in ["map.reloadTiles", "map.repairTiles"] { + let f = Scoped::new(&format!("archive-freshness-inode-{ext}-{command}")); + let linked = f.file(&format!("linked.{ext}")); + let candidate_path = f.file(&format!("candidate.{ext}")); + let swapped = f.file(&format!("swapped.{ext}")); + write_archive(&linked, [255, 0, 0, 255], "Retained credits"); + write_archive(&candidate_path, [0, 0, 255, 255], "Candidate credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":linked,"overlay":true})).unwrap(); + let tile = s.map().tiles[0].clone(); + let retained = vec![retain_archive(&tile)]; + s.style_view = Some((tile.clone(), s.map().camera, s.map().instance_id())); + let mut params = json!({"index":0}); + let path = if command == "map.repairTiles" { + params["path"] = json!(candidate_path); + &candidate_path + } else { + &linked + }; + let read = archive_read(&mut s, command, ¶ms); + let old = std::fs::metadata(path).unwrap(); + std::fs::copy(path, &swapped).unwrap(); + std::fs::File::options() + .write(true) + .open(&swapped) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(old.modified().unwrap())) + .unwrap(); + std::fs::rename(&swapped, path).unwrap(); + let new = std::fs::metadata(path).unwrap(); + assert_eq!(old.len(), new.len()); + assert_eq!(old.modified().unwrap(), new.modified().unwrap()); + assert_ne!(old.ino(), new.ino()); + assert!(reject_stale_archive(&mut s, read, &retained).contains("changed while it was read")); + s.run(command, ¶ms).unwrap(); + assert_eq!(s.map().tiles[0].url, path.display().to_string()); + rustgis_engine::data::mbtiles::close(&linked); + rustgis_engine::data::mbtiles::close(&candidate_path); + } + } +} + +#[test] +fn mbtiles_wal_commit_with_writer_open_invalidates_worker_validation_without_eviction() { + for command in ["map.reloadTiles", "map.repairTiles"] { + let f = Scoped::new(&format!("archive-freshness-wal-{command}")); + let linked = f.file("linked.mbtiles"); + let replacement = f.file("replacement.mbtiles"); + write_archive(&linked, [255, 0, 0, 255], "Retained credits"); + write_archive(&replacement, [0, 0, 255, 255], "Replacement credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":linked,"overlay":true})).unwrap(); + let tile = s.map().tiles[0].clone(); + let mut params = json!({"index":0}); + let path = if command == "map.repairTiles" { + params["path"] = json!(replacement); + &replacement + } else { + &linked + }; + let writer = rusqlite::Connection::open(path).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0; UPDATE metadata SET value='Before worker' WHERE name='attribution';", + ) + .unwrap(); + let mut candidate = tile.clone(); + candidate.url = path.display().to_string(); + let retained = vec![retain_archive(&tile), retain_archive(&candidate)]; + s.style_view = Some((tile.clone(), s.map().camera, s.map().instance_id())); + let read = archive_read(&mut s, command, ¶ms); + let before = std::fs::metadata(path).unwrap(); + writer.execute_batch("BEGIN IMMEDIATE; UPDATE metadata SET value='Committed after worker' WHERE name='attribution';").unwrap(); + writer.execute("UPDATE tiles SET tile_data=?1", [picture([0, 255, 0, 255])]).unwrap(); + writer.execute_batch("COMMIT").unwrap(); + let after = std::fs::metadata(path).unwrap(); + assert_eq!(before.len(), after.len(), "this regression must exercise a sidecar-only commit"); + assert_eq!(before.modified().unwrap(), after.modified().unwrap()); + assert!(reject_stale_archive(&mut s, read, &retained).contains("changed while it was read")); + // Keep the writer open: retry must see committed WAL contents, without requiring a checkpoint. + s.run(command, ¶ms).unwrap(); + assert_color(&decoded(&s.map().tiles[0]), [0, 255, 0, 255]); + assert!(style(&s.map().tiles[0]).attribution.iter().any(|a| a.contains("Committed after worker"))); + drop(writer); + rustgis_engine::data::mbtiles::close(&linked); + rustgis_engine::data::mbtiles::close(&replacement); + } +} + +#[test] +fn clean_wal_mode_mbtiles_first_reload_and_repair_ignore_reader_created_empty_wal() { + for command in ["map.reloadTiles", "map.repairTiles"] { + let f = Scoped::new(&format!("archive-clean-wal-{command}")); + let linked = f.file("linked.mbtiles"); + let replacement = f.file("replacement.mbtiles"); + write_archive(&linked, [255, 0, 0, 255], "Retained credits"); + write_archive(&replacement, [0, 255, 0, 255], "Clean WAL credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":linked,"overlay":true})).unwrap(); + let mut params = json!({"index":0}); + let path = if command == "map.repairTiles" { + params["path"] = json!(replacement); + &replacement + } else { + &linked + }; + // A clean close of the last WAL connection removes -wal/-shm; the archive stays WAL mode. + let writer = rusqlite::Connection::open(path).unwrap(); + writer.query_row("PRAGMA journal_mode=WAL", [], |r| r.get::<_, String>(0)).unwrap(); + drop(writer); + rustgis_engine::data::mbtiles::close(&linked); + rustgis_engine::data::mbtiles::close(&replacement); + for sidecar in ["-wal", "-journal"] { + let _ = std::fs::remove_file(format!("{}{sidecar}", path.display())); + } + assert!(!Path::new(&format!("{}-wal", path.display())).exists()); + // Whether or not the reader creates an empty -wal while validating, the first attempt lands. + let read = archive_read(&mut s, command, ¶ms); + rustgis_engine::cmd::reads::read_land(&mut s, read).unwrap(); + let expected = if command == "map.repairTiles" { [0, 255, 0, 255] } else { [255, 0, 0, 255] }; + assert_color(&decoded(&s.map().tiles[0]), expected); + if command == "map.repairTiles" { + assert_eq!(s.map().tiles[0].url, path.display().to_string()); + } + rustgis_engine::data::mbtiles::close(&linked); + rustgis_engine::data::mbtiles::close(&replacement); + } +} + +#[test] +fn archive_observation_begins_in_worker_not_when_work_is_queued() { + for ext in ["pmtiles", "mbtiles"] { + let f = Scoped::new(&format!("archive-worker-observation-{ext}")); + let path = f.file(&format!("linked.{ext}")); + write_archive(&path, [255, 0, 0, 255], "Before queue"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":path,"overlay":true})).unwrap(); + let ahead = rustgis_engine::cmd::reads::read_ahead(&mut s, "map.reloadTiles", &json!({"index":0})).unwrap(); + write_archive(&path, [0, 255, 0, 255], "Written before worker starts"); + let read = ahead.run(&rustgis_engine::analysis::control::Control::stoppable()).unwrap(); + rustgis_engine::cmd::reads::read_land(&mut s, read).unwrap(); + assert_color(&decoded(&s.map().tiles[0]), [0, 255, 0, 255]); + rustgis_engine::data::mbtiles::close(&path); + } +} + +/// Move the writer's uncompressed root index into a leaf and optionally add an intermediate +/// directory. All pointer fields are small fixture constants; tile payloads remain unchanged. +fn pmtiles_with_leaf_pointer(bytes: &[u8], pointer_id: u8, nested: bool) -> Vec { + let at = |offset: usize| u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()) as usize; + assert_eq!(bytes[97], 1, "fixture directories must be uncompressed"); + assert_eq!(at(48), 0, "fixture must start with no leaf directories"); + let mut leaves = bytes[at(8)..at(8) + at(16)].to_vec(); + let leaf_length = u8::try_from(leaves.len()).unwrap(); + let mut root = vec![1, pointer_id, 0, leaf_length, 1]; + if nested { + root[3] = 5; + root[4] = leaf_length + 1; + leaves.extend([1, 0, 0, leaf_length, 1]); + } + let metadata = &bytes[at(24)..at(24) + at(32)]; + let data = &bytes[at(56)..at(56) + at(64)]; + let mut head = bytes[..127].to_vec(); + let metadata_at = 127 + root.len() as u64; + let leaf_at = metadata_at + metadata.len() as u64; + let data_at = leaf_at + leaves.len() as u64; + for (offset, value) in [(8, 127), (16, root.len() as u64), (24, metadata_at), (40, leaf_at), (48, leaves.len() as u64), (56, data_at)] { + head[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + [head, root, metadata.to_vec(), leaves, data.to_vec()].concat() +} + +#[test] +fn unreachable_pmtiles_runs_reject_reload_and_repair_without_losing_retained_data() { + for command in ["map.reloadTiles", "map.repairTiles"] { + for defect in ["overlap", "direct lower bound", "nested lower bound"] { + let f = Scoped::new(&format!("archive-unreachable-runs-{command}-{defect}")); + let linked = f.file("linked.pmtiles"); + let replacement = f.file("replacement.pmtiles"); + write_archive(&linked, [255, 0, 0, 255], "Retained credits"); + let mut s = session(); + s.run("map.addTiles", &json!({"url":linked,"overlay":true})).unwrap(); + s.run("map.setTiles", &json!({"index":0,"opacity":0.4})).unwrap(); + let tile = s.map().tiles[0].clone(); + let retained = vec![retain_archive(&tile)]; + s.style_view = Some((tile.clone(), s.map().camera, s.map().instance_id())); + let archive = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Png, + gzipped_tiles: false, + bounds: [-180.0, -85.0, 180.0, 85.0], + metadata: json!({"attribution":"Valid reachable runs"}), + }; + let valid = pmtiles::write(vec![(0, 0, 0, picture([0, 255, 0, 255])), (1, 0, 0, picture([0, 0, 255, 255]))], &archive).unwrap(); + assert_eq!(&valid[127..132], &[2, 0, 1, 1, 1], "fixture must contain two adjacent single-tile runs"); + let (invalid, corrected, expected) = if defect == "overlap" { + let mut overlapping = valid.clone(); + overlapping[130] = 2; // First [0,1) run becomes [0,2), overlapping the next entry. + (overlapping, valid, "overlapping PMTiles tile runs") + } else { + let nested = defect == "nested lower bound"; + ( + pmtiles_with_leaf_pointer(&valid, 5, nested), + pmtiles_with_leaf_pointer(&valid, 0, nested), + "a PMTiles leaf directory contains tiles below its pointer", + ) + }; + let mut params = json!({"index":0}); + let path = if command == "map.repairTiles" { + params["path"] = json!(replacement); + &replacement + } else { + &linked + }; + std::fs::write(path, invalid).unwrap(); + let read = archive_read(&mut s, command, ¶ms); + assert!(reject_stale_archive(&mut s, read, &retained).contains(expected)); + std::fs::write(path, corrected).unwrap(); + s.run(command, ¶ms).unwrap(); + let mut expected_tile = tile; + expected_tile.url = path.display().to_string(); + assert_eq!(s.map().tiles[0], expected_tile, "retry must preserve presentation and update only the repaired path"); + assert_color(&decoded(&s.map().tiles[0]), [0, 255, 0, 255]); + assert!(style(&s.map().tiles[0]).attribution.iter().any(|a| a.contains("Valid reachable runs"))); + } + } +} diff --git a/crates/render/src/gpu_scene.rs b/crates/render/src/gpu_scene.rs index 0337bc8a1d..6eb8fd3a69 100644 --- a/crates/render/src/gpu_scene.rs +++ b/crates/render/src/gpu_scene.rs @@ -119,7 +119,8 @@ pub enum DrawCmd { /// One raster tile layer the GPU map places again for every frame it shows, from the tile /// cache, for the camera of that frame (see [`crate::tiles::live`]): the scene holds only /// which layer, so panning and zooming never draw the scene again for its tiles (#635). - LiveTiles { name: String, spec: rustgis_tiles::Spec, opacity: f32 }, + /// The archive revision binds later live tile acquisition to this scene's credits. + LiveTiles { name: String, spec: rustgis_tiles::Spec, opacity: f32, archive_revision: u64 }, /// One vector tile layer's tiles as meshes (see `tile_mesh`): the GPU keeps each mesh's /// buffers across frames, shows the layers of its runs that show at the camera's zoom and /// clips each tile to its square, as MapLibre draws its tiles. diff --git a/crates/render/src/layout.rs b/crates/render/src/layout.rs index a694357bb9..b9524ecf2d 100644 --- a/crates/render/src/layout.rs +++ b/crates/render/src/layout.rs @@ -51,15 +51,33 @@ pub(crate) fn frame_requires_complete(map: &Map, view: &View) -> bool { }) } +/// Only maps placed in visible frames participate in this page's archive revision. +pub fn archive_revisions(layout: &Layout, maps: &[Map]) -> crate::ArchiveRevision { + crate::archive_revisions(layout.elements.iter().filter_map(|e| { + if !e.visible { + return None; + } + let ElementKind::MapFrame { map, .. } = &e.kind else { return None }; + map_index(maps, map).and_then(|i| maps.get(i)) + })) +} + /// Renders a layout at `dpi` (dots per inch). `maps` are the project's maps, found by name. -pub fn render(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Option { +/// Refuses a linked archive changed while composing the page, including its different map frames. +pub fn render(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Result { render_pending(layout, maps, dpi, info).map(|(pm, _)| pm) } /// [`render`], and whether a map frame on the page drew while some of its tiles, or a read of its /// streamed layers or rasters, were still on their way: in the browser, which doesn't wait for /// them, the page is drawn again once they land. -pub fn render_pending(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Option<(Pixmap, bool)> { +pub fn render_pending(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Result<(Pixmap, bool), String> { + let revisions = archive_revisions(layout, maps); + let result = render_pending_inner(layout, maps, dpi, info); + revisions.finish_revision(archive_revisions(layout, maps), result, "the page is too large to draw") +} + +fn render_pending_inner(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Option<(Pixmap, bool)> { let (w, h, k) = page_geometry(layout, dpi); let bounds = tiny_skia::IntRect::from_xywh(0, 0, w, h)?; let mut pending = false; @@ -78,6 +96,17 @@ pub fn render_drawing( maps: &mut [Map], dpi: f64, info: &PageInfo, +) -> Result<(crate::vector::Drawing, Vec<(u32, String)>), String> { + let revisions = archive_revisions(layout, maps); + let result = render_drawing_inner(layout, maps, dpi, info); + revisions.finish_revision(archive_revisions(layout, maps), result, "the page is too large to draw") +} + +fn render_drawing_inner( + layout: &Layout, + maps: &mut [Map], + dpi: f64, + info: &PageInfo, ) -> Option<(crate::vector::Drawing, Vec<(u32, String)>)> { let (w, h, k) = page_geometry(layout, dpi); let bounds = tiny_skia::IntRect::from_xywh(0, 0, w, h)?; @@ -435,9 +464,11 @@ fn draw_page( /// A map frame's map, `w` × `h` points at `ppp` pixels a point, labels and attribution included: /// what a frame shows before its grid, indicator and outline. It waits for the map's tiles, as a -/// page needs them all. -pub fn frame_map(map: &mut Map, camera: rustgis_map::Camera, w: f64, h: f64, ppp: f64) -> Option { - frame_canvas(map, camera, w, h, ppp, false, &mut false).map(Canvas::into_pixmap) +/// page needs them all. Refuses a linked archive changed during the complete frame. +pub fn frame_map(map: &mut Map, camera: rustgis_map::Camera, w: f64, h: f64, ppp: f64) -> Result { + let revisions = crate::archive_revisions([&*map]); + let result = frame_canvas(map, camera, w, h, ppp, false, &mut false).map(Canvas::into_pixmap); + revisions.finish([&*map], result, "could not draw the map frame") } /// [`frame_map`] on a canvas: one that records (see [`crate::canvas`]) when `record` and the diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 871da95f90..b875cd61c8 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -318,6 +318,7 @@ pub enum RenderOutput { /// [`render`]; `Backend::Gpu` draws what it can as tessellated meshes and everything else /// (basemap, tiles, rasters, masked/blended/transparent layers) as CPU-rasterized images in /// the same z-order. +/// A pass spanning a linked archive reload returns `None`; redraw against the new source. pub fn render_backend(map: &mut Map, view: &View, backend: Backend) -> Option { render_with_backend(map, view, false, backend) } @@ -1615,8 +1616,44 @@ pub fn view_lonlat(map: &Map, view: &View) -> Extent { if out.is_empty() { Extent::WORLD_LONLAT } else { out } } +/// A source-specific render revision, independent of ordinary tile arrivals. +#[derive(Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord)] +pub struct ArchiveRevision(u64); + +impl ArchiveRevision { + /// Checks against the same maps whose linked archives were captured. + pub fn current<'a>(self, maps: impl IntoIterator) -> bool { + self.0 == archive_revisions(maps).0 + } + + /// Refuses a spanning render before reporting any ordinary drawing failure. + pub(crate) fn finish<'a, T>(self, maps: impl IntoIterator, result: Option, failure: &str) -> Result { + self.finish_revision(archive_revisions(maps), result, failure) + } + + /// Checks a composed output against a newly captured revision of the same source set. + pub fn finish_revision(self, current: Self, result: Option, failure: &str) -> Result { + if self.0 != current.0 { + return Err("a tile archive changed during rendering; export again after the reload finishes".to_string()); + } + result.ok_or_else(|| failure.to_string()) + } +} + +/// Captures only linked tile archives used by these maps, before styles or tiles are read. +/// The token is allocation-free; retain the maps for a check at worker publication. +pub fn archive_revisions<'a>(maps: impl IntoIterator) -> ArchiveRevision { + ArchiveRevision(rustgis_tiles::cache::archive_revision(maps.into_iter().flat_map(|map| { + map.tiles + .iter() + .filter(|t| t.draws() && t.opacity > 0.0 && matches!(t.kind, rustgis_map::TileKind::PmTiles | rustgis_map::TileKind::MbTiles)) + .map(|t| t.url.as_str()) + }))) +} + /// Draws the map for the screen. Layers are drawn bottom-up (the last layer in the list first). /// On high-density screens rasters are computed at logical-pixel resolution, for speed. +/// A pass spanning a linked archive reload returns `None`, without publishing its mixed pixels. pub fn render(map: &mut Map, view: &View) -> Option { render_with(map, view, false) } @@ -1660,6 +1697,13 @@ fn render_with_backend(map: &mut Map, view: &View, full_res_rasters: bool, backe /// [`render_with_backend`], `record` keeping a flat, north-up frame as vectors where it can /// ([`render_frame`]). fn render_any(map: &mut Map, view: &View, full_res_rasters: bool, backend: Backend, record: bool) -> Option { + let revisions = archive_revisions([&*map]); + let output = render_pass(map, view, full_res_rasters, backend, record); + revisions.current([&*map]).then_some(output).flatten() +} + +/// One entire pass, including all bands and backend fallbacks, under `render_any`'s guard. +fn render_pass(map: &mut Map, view: &View, full_res_rasters: bool, backend: Backend, record: bool) -> Option { // A bearing or pitch set directly on the camera may be unnormalised (or no number, which // faces north and looks straight down). let mut camera = view.camera; @@ -5000,6 +5044,7 @@ pub struct DrapePatch { /// ([`rustgis_map::Globe::drape_mesh`]) while it turns, with no drawing per frame. Labels are /// left out (the view hides them while it moves). pub fn render_globe_drape(map: &mut Map, view: &View, boxes: &[Extent], max_side: u32) -> Vec { + let revisions = archive_revisions([&*map]); // A coarse drape may have far fewer pixels than points: it shows the map blurrier, but at // the view's scale. let ppp = if view.pixels_per_point.is_finite() { view.pixels_per_point.clamp(1.0 / 64.0, 4.0) } else { 1.0 }; @@ -5039,7 +5084,7 @@ pub fn render_globe_drape(map: &mut Map, view: &View, boxes: &[Extent], max_side map.label_display = display; map.label_lock = lock; map.label_lock_line_anchors = anchors; - out + if revisions.current([&*map]) { out } else { Vec::new() } } /// A band of a tilted view drawn flat, to lay over the tilted ground while the view moves. @@ -5057,6 +5102,7 @@ pub struct TiltPatch { /// band, square, over the circle round the view and without the margin: it is laid over the view /// the same way while it turns. pub fn render_tilt_drape(map: &mut Map, view: &View, margin: f64, max_side: u32) -> Vec { + let revisions = archive_revisions([&*map]); let ppp = if view.pixels_per_point.is_finite() { view.pixels_per_point.clamp(0.25, 4.0) } else { 1.0 }; let max_side = f64::from(max_side.clamp(64, 8192)); let cam = view.camera; @@ -5091,7 +5137,7 @@ pub fn render_tilt_drape(map: &mut Map, view: &View, margin: f64, max_side: u32) map.label_display = display; map.label_lock = lock; map.label_lock_line_anchors = anchors; - out + if revisions.current([&*map]) { out } else { Vec::new() } } /// A line's longest stretch in view, in points (`None` when none has two points in sight): the @@ -5776,7 +5822,14 @@ fn complete(map: &mut Map, view: &View, record: bool) -> Option { /// [`render_png`]'s map kept as vectors (for a vector PDF) where it can be: flat, north up, /// without view effects; anything else, and what is pixels by nature, as pictures. -pub fn render_drawing(map: &mut Map, view: &View) -> Option { +/// Returns an actionable error if a linked archive changes during the complete drawing. +pub fn render_drawing(map: &mut Map, view: &View) -> Result { + let revisions = archive_revisions([&*map]); + let result = render_drawing_inner(map, view); + revisions.finish([&*map], result, "could not draw the map") +} + +fn render_drawing_inner(map: &mut Map, view: &View) -> Option { let r = render_complete_recorded(map, view)?; let mut pm = match r.recorded { Some(c) => c, @@ -5796,7 +5849,9 @@ pub fn render_drawing(map: &mut Map, view: &View) -> Option { } /// Renders a map with its labels, a scale bar and a north arrow to PNG bytes. +/// Refuses an archive changed during rendering or encoding; a caller must not write a destination on error. pub fn render_png(map: &mut Map, view: &View) -> Result, String> { + let revisions = archive_revisions([&*map]); let r = render_exported(map, view)?; let mut pm = Canvas::from_pixmap(r.pixmap); for l in r.unplaced.iter().chain(&r.labels) { @@ -5804,7 +5859,8 @@ pub fn render_png(map: &mut Map, view: &View) -> Result, String> { } text::scale_bar(&mut pm, view, map.ground_res_at(&view.camera)); draw_attribution(&mut pm, &r.attribution, view.pixels_per_point as f32); - pm.into_pixmap().encode_png().map_err(|e| e.to_string()) + let png = pm.into_pixmap().encode_png().map_err(|e| e.to_string())?; + revisions.finish([&*map], Some(png), "could not encode the image") } /// A map drawn as data ([`render_placed`]), with what it holds besides its pixels. @@ -5824,6 +5880,7 @@ pub struct Placed { /// without the scale bar and the credits, which sit on the screen, not the ground. Refuses what /// [`render_png`] refuses. pub fn render_placed(map: &mut Map, view: &View) -> Result { + let revisions = archive_revisions([&*map]); let r = render_exported(map, view)?; let mut gaps = Vec::new(); if r.tiles_pending { @@ -5847,13 +5904,15 @@ pub fn render_placed(map: &mut Map, view: &View) -> Result { for l in r.unplaced.iter().chain(&r.labels) { text::draw(&mut pm, l, view.pixels_per_point as f32); } - Ok(Placed { pixmap: pm.into_pixmap(), credits: r.attribution, gaps }) + revisions.finish([&*map], Some(Placed { pixmap: pm.into_pixmap(), credits: r.attribution, gaps }), "could not draw the map") } /// The map drawn whole for an export, or why it can't be written: a raster read in place whose /// tiles haven't come, or that is drawn from less than the view needs. fn render_exported(map: &mut Map, view: &View) -> Result { - let r = render_complete(map, view).ok_or("could not allocate the image")?; + let revisions = archive_revisions([&*map]); + let result = render_complete(map, view); + let r = revisions.finish([&*map], result, "could not draw the map")?; // A raster read in place drawn from less than the view needs isn't written: its tiles still // on their way (the browser can't wait; natively the wait ran out), or a coarser level or // tiles it couldn't read (`partial`). diff --git a/crates/render/src/report.rs b/crates/render/src/report.rs index d47be67af3..883c2a58d2 100644 --- a/crates/render/src/report.rs +++ b/crates/render/src/report.rs @@ -267,15 +267,24 @@ const MAX_PIXELS: f64 = if cfg!(target_arch = "wasm32") { 25_000_000.0 } else { /// Draws page `n` (0-based) at `dpi`, returning it and its size in millimetres. `layouts` and /// `maps` are the project's, for layout pages. -pub fn draw_page(doc: &Doc, pages: &[Page], n: usize, dpi: f64, layouts: &[Layout], maps: &mut [Map]) -> Option<(Pixmap, f64, f64)> { - let page = pages.get(n)?; +pub fn draw_page( + doc: &Doc, + pages: &[Page], + n: usize, + dpi: f64, + layouts: &[Layout], + maps: &mut [Map], +) -> Result<(Pixmap, f64, f64), String> { + let page = pages.get(n).ok_or("no such report page")?; let dpi = if dpi.is_finite() { dpi.clamp(24.0, 600.0) } else { 96.0 }; if let Page::Layout(li) = page { - let l = layouts.get(*li)?; + let l = layouts.get(*li).ok_or("no such report layout")?; let info = crate::layout::PageInfo { page: n + 1, pages: pages.len(), page_name: String::new() }; return crate::layout::render(l, maps, dpi, &info).map(|pm| (pm, l.width, l.height)); } - flow_canvas(doc, pages, n, dpi, false).map(|(pm, w, h)| (pm.into_pixmap(), w, h)) + flow_canvas(doc, pages, n, dpi, false) + .map(|(pm, w, h)| (pm.into_pixmap(), w, h)) + .ok_or_else(|| "the page is too large to draw".to_string()) } /// Page `n` as a vector PDF page: a layout's as [`crate::layout::render_drawing`] keeps it, a @@ -288,17 +297,17 @@ pub fn pdf_page( dpi: f64, layouts: &[Layout], maps: &mut [Map], -) -> Option<(crate::pdf::Page, Vec<(u32, String)>)> { - let page = pages.get(n)?; +) -> Result<(crate::pdf::Page, Vec<(u32, String)>), String> { + let page = pages.get(n).ok_or("no such report page")?; let dpi = if dpi.is_finite() { dpi.clamp(24.0, 600.0) } else { 96.0 }; if let Page::Layout(li) = page { - let l = layouts.get(*li)?; + let l = layouts.get(*li).ok_or("no such report layout")?; let info = crate::layout::PageInfo { page: n + 1, pages: pages.len(), page_name: String::new() }; return crate::layout::render_drawing(l, maps, dpi, &info) .map(|(d, pictured)| (crate::pdf::Page::drawing(d, l.width, l.height), pictured)); } - let (pm, w, h) = flow_canvas(doc, pages, n, dpi, true)?; - pm.into_drawing().map(|d| (crate::pdf::Page::drawing(d, w, h), Vec::new())) + let (pm, w, h) = flow_canvas(doc, pages, n, dpi, true).ok_or("the page is too large to draw")?; + pm.into_drawing().map(|d| (crate::pdf::Page::drawing(d, w, h), Vec::new())).ok_or_else(|| "the page is too large to draw".to_string()) } /// Flowing page `n` on a canvas `dpi` dots an inch (recording when `record`), and its size in @@ -540,7 +549,7 @@ mod tests { let (pm, w, _) = draw_page(&d, &pages, 0, 72.0, &[], &mut []).unwrap(); assert_eq!(w, 215.9); assert!(pm.pixels().iter().any(|p| p.red() < 100), "the page has text"); - assert!(draw_page(&d, &pages, 10_000, 72.0, &[], &mut []).is_none()); + assert!(draw_page(&d, &pages, 10_000, 72.0, &[], &mut []).is_err()); } #[test] @@ -550,7 +559,7 @@ mod tests { assert!(!pages.is_empty() && pages.len() <= MAX_PAGES); let d = Doc { title: "Empty".into(), width: 100.0, height: 100.0, blocks: Vec::new() }; assert_eq!(paginate(&d).len(), 1); - assert!(draw_page(&d, &paginate(&d), 0, 1e9, &[], &mut []).is_some()); + assert!(draw_page(&d, &paginate(&d), 0, 1e9, &[], &mut []).is_ok()); } #[test] diff --git a/crates/render/src/stream_tests.rs b/crates/render/src/stream_tests.rs index 040523298b..3276be7640 100644 --- a/crates/render/src/stream_tests.rs +++ b/crates/render/src/stream_tests.rs @@ -554,7 +554,7 @@ fn layout_export_keeps_all_drawn_legend_classes() { } let mut failed = base.clone(); failed.layers[0] = streamed(&base.layers[0], 0); - assert!(layout::render(&page, &mut [failed], 96.0, &layout::PageInfo::default()).is_none()); + assert!(layout::render(&page, &mut [failed], 96.0, &layout::PageInfo::default()).is_err()); } #[test] @@ -1163,7 +1163,7 @@ fn layout_export_rejects_unopened_local_shapefiles_and_required_masks() { map.layer_mut(point).unwrap().mask = Some("MASK".into()); } assert!( - layout::render(&page, std::slice::from_mut(&mut map), 96.0, &layout::PageInfo::default()).is_none(), + layout::render(&page, std::slice::from_mut(&mut map), 96.0, &layout::PageInfo::default()).is_err(), "missing complete source must fail the page: pitch {pitch}, masked {masked}" ); assert!(map.layer(owner).unwrap().source.as_ref().unwrap().error.is_some()); @@ -1173,7 +1173,7 @@ fn layout_export_rejects_unopened_local_shapefiles_and_required_masks() { source.path = Some(dir.path().join("unused.shp").display().to_string()); unused.layer_mut(mask).unwrap().source = Some(source); assert!( - layout::render(&page, &mut [unused], 96.0, &layout::PageInfo::default()).is_some(), + layout::render(&page, &mut [unused], 96.0, &layout::PageInfo::default()).is_ok(), "an unused hidden Shapefile must not fail export" ); @@ -1182,7 +1182,7 @@ fn layout_export_rejects_unopened_local_shapefiles_and_required_masks() { source.path = Some(dir.path().join("missing.gpkg").display().to_string()); ordinary.layer_mut(point).unwrap().source = Some(source); assert!( - layout::render(&page, &mut [ordinary], 96.0, &layout::PageInfo::default()).is_some(), + layout::render(&page, &mut [ordinary], 96.0, &layout::PageInfo::default()).is_ok(), "ordinary sources retain upstream best-effort behavior" ); } diff --git a/crates/render/src/svg.rs b/crates/render/src/svg.rs index 42dde168c0..8a73a58055 100644 --- a/crates/render/src/svg.rs +++ b/crates/render/src/svg.rs @@ -17,8 +17,18 @@ use rustgis_map::layout::{ElementKind, FontFamily, Layout, LineHead, PT_PER_MM, use crate::layout::{MAX_PIXELS, PageInfo, expand, page_geometry, record_element, render_element}; /// Writes `layout` as an SVG document, pictures drawn at `dpi`, and the elements (id and name) -/// drawn as pictures because their vectors would pass the budget; `None` when the page can't be drawn. -pub fn layout_svg(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Option<(String, Vec<(u32, String)>)> { +/// drawn as pictures because their vectors would pass the budget. Refuses a source changed during drawing. +pub fn layout_svg(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Result<(String, Vec<(u32, String)>), String> { + let revisions = crate::layout::archive_revisions(layout, maps); + let result = layout_svg_inner(layout, maps, dpi, info); + revisions.finish_revision( + crate::layout::archive_revisions(layout, maps), + result, + "could not draw the page; check its map sources and export size", + ) +} + +fn layout_svg_inner(layout: &Layout, maps: &mut [Map], dpi: f64, info: &PageInfo) -> Option<(String, Vec<(u32, String)>)> { let mut defs = vectors::Defs::default(); let mut pictured = Vec::new(); let mut body = elements(layout, maps, dpi, info, &mut defs, &mut pictured)?; @@ -438,7 +448,7 @@ mod tests { assert!(svg.contains(&format!(r#" Option { + let archive = matches!(spec.kind, Kind::PmTiles | Kind::MbTiles).then_some(spec.url.as_str()); + let revision = rustgis_tiles::cache::archive_revision(archive); + let output = live_inner(spec, opacity, view); + (revision == rustgis_tiles::cache::archive_revision(archive)).then_some(output).flatten() +} + +fn live_inner(spec: &Spec, opacity: f32, view: &View) -> Option { let res = view.camera.res; let ppp = view.pixels_per_point as f32; let frame = Frame { @@ -389,7 +396,12 @@ pub(crate) fn draw( && let Some(p) = rustgis_tiles::place_raster(&s, t.opacity, &frame, &fetch, &reach) && crate::flush_acc(pm, g.cmds, g.dirty).is_some() { - g.cmds.push(DrawCmd::LiveTiles { name: t.name.clone(), spec: s, opacity: t.opacity }); + let archive_revision = if matches!(s.kind, Kind::PmTiles | Kind::MbTiles) { + rustgis_tiles::cache::archive_revision([s.url.as_str()]) + } else { + 0 + }; + g.cmds.push(DrawCmd::LiveTiles { name: t.name.clone(), spec: s, opacity: t.opacity, archive_revision }); let d = rustgis_tiles::Drawn { labels: Vec::new(), attribution: p.attribution, pending: p.pending, error: p.error }; collect(d, t, &mut out); continue; @@ -1493,8 +1505,8 @@ pub(crate) mod tests { for done in std::mem::take(&mut *HELD.lock().unwrap()) { done(Ok(tiny_skia::Pixmap::new(256, 256).unwrap().encode_png().unwrap())); } - assert_eq!(short, Some(true), "drawn before its tiles landed"); + assert_eq!(short, Ok(true), "drawn before its tiles landed"); let whole = crate::layout::render_pending(&page, &mut maps, 72.0, &info).map(|(_, pending)| pending); - assert_eq!(whole, Some(false), "drawn once they have"); + assert_eq!(whole, Ok(false), "drawn once they have"); } } diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index fd6931fedb..f98878bef1 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -123,6 +123,7 @@ struct Slot { used: u64, /// When the map last asked for it ([`get`]): a waiting tile not asked for lately is dropped. asked: Instant, + epoch: u64, } impl Slot { @@ -155,6 +156,20 @@ struct Cache { latest: Option, } +impl Cache { + fn forget_archive(&mut self, archive: &str) { + let mut dropped = 0; + self.slots.retain(|key, slot| { + if resource_of(key) != archive { + return true; + } + dropped += slot.bytes(); + false + }); + self.bytes = self.bytes.saturating_sub(dropped); + } +} + fn cache() -> &'static Mutex { static C: OnceLock> = OnceLock::new(); C.get_or_init(Mutex::default) @@ -168,6 +183,36 @@ static GENERATION: AtomicU64 = AtomicU64::new(0); static CLEARS: AtomicU64 = AtomicU64::new(0); static IN_FLIGHT: AtomicUsize = AtomicUsize::new(0); +/// Source revisions are locked before any retained cache. Publication and reload share this +/// lock; callbacks and I/O must always run after releasing it. +fn archive_epochs() -> &'static Mutex> { + static E: std::sync::LazyLock>> = std::sync::LazyLock::new(Mutex::default); + &E +} + +/// The most recent reload of just these archives. Reload stamps increase globally, so any +/// selected source changing increases this maximum; unrelated reloads and tile arrivals do not. +/// The caller retains the source paths and can compare again without allocating a snapshot. +pub fn archive_revision<'a>(archives: impl IntoIterator) -> u64 { + let mut archives = archives.into_iter(); + let Some(first) = archives.next() else { return 0 }; + let epochs = archive_epochs().lock().unwrap_or_else(|e| e.into_inner()); + std::iter::once(first).chain(archives).filter_map(|url| epochs.get(url).copied()).max().unwrap_or(0) +} + +pub(crate) fn archive_epoch(url: &str) -> u64 { + archive_epochs().lock().unwrap_or_else(|e| e.into_inner()).get(resource_of(url)).copied().unwrap_or(0) +} + +pub(crate) fn if_archive_current(url: &str, epoch: u64, f: impl FnOnce() -> T) -> Option { + let epochs = archive_epochs().lock().unwrap_or_else(|e| e.into_inner()); + if epochs.get(resource_of(url)).copied().unwrap_or(0) != epoch { + drop(epochs); + return None; + } + Some(f()) +} + fn fetcher() -> &'static Mutex>> { static F: OnceLock>>> = OnceLock::new(); F.get_or_init(Mutex::default) @@ -527,6 +572,8 @@ fn request( return refused_request(url, e); } let ceiling = from.map_or(crate::address::Reach::Local, |o| o.reach); + let epochs = archive_epochs().lock().unwrap_or_else(|e| e.into_inner()); + let epoch = epochs.get(resource_of(url)).copied().unwrap_or(0); let now = Instant::now(); let mut c = lock(); c.tick += 1; @@ -564,6 +611,7 @@ fn request( } if crate::warp::is_warp(url) { drop(c); + drop(epochs); // Made once its pictures are here; asked again (as the map draws) until then. let mut note = None; return ( @@ -594,8 +642,12 @@ fn request( } let asked_at = crate::mbtiles::epoch(); keep_for(&mut c, url, Entry::Pending, ceiling); + if let Some(slot) = c.slots.get_mut(url) { + slot.epoch = epoch; + } drop(c); - start(url.to_string(), want, omit, asked_at, ceiling); + drop(epochs); + start(url.to_string(), want, omit, epoch, asked_at, ceiling); (State::Pending, None, None) } @@ -702,12 +754,11 @@ fn resource_of(key: &str) -> &str { let key = split_download_limit(key).0; let key = split_tail(key).0; let key = split_range(key).0; - // An MBTiles file's path whole, as its tiles and the record of forgotten files name it: a - // `#` in a file's name isn't a COG's look. - if crate::mbtiles::is_tile(key) { + // Archive file names are whole: only COG keys carry a look after `#`. + if crate::mbtiles::is_tile(key) || key.starts_with(crate::pmtiles::SCHEME) { return key.split_once('@').map_or(key, |(_, archive)| archive); } - if key.starts_with(crate::pmtiles::SCHEME) || key.starts_with(crate::cog::SCHEME) { + if key.starts_with(crate::cog::SCHEME) { return key.split_once('@').map_or(key, |(_, archive)| crate::cog::split(archive).0); } key @@ -719,8 +770,9 @@ fn resource_of(key: &str) -> &str { /// MBTiles file's tiles: it is forgotten when it may be replaced, and its old tiles mustn't stay. pub fn forget(url: &str) { let url = split_download_limit(url).0; - // An MBTiles file's path is taken whole (see [`resource_of`]). - let url = if crate::mbtiles::check(url).is_ok() { url } else { crate::cog::split(url).0 }; + // PMTiles and MBTiles paths are taken whole (see `resource_of`), even with a look-like name. + let pmtiles = url.rsplit_once('.').is_some_and(|(_, ext)| ext.eq_ignore_ascii_case("pmtiles")); + let url = if pmtiles || crate::mbtiles::check(url).is_ok() { url } else { crate::cog::split(url).0 }; // First: a tile answered from here on is dropped, and one answered before goes with the rest. crate::mbtiles::forget_archive(url); let mut c = lock(); @@ -742,6 +794,22 @@ pub fn forget(url: &str) { GENERATION.fetch_add(1, Ordering::Relaxed); } +/// Reloads just this archive, including pending reads, metadata, directories, handles and +/// derived styles. Call only after uncached validation succeeds. Older answers are discarded. +pub fn reload_archive(url: &str) { + let mut epochs = archive_epochs().lock().unwrap_or_else(|e| e.into_inner()); + static NEXT: AtomicU64 = AtomicU64::new(1); + epochs.insert(url.to_string(), NEXT.fetch_add(1, Ordering::Relaxed)); + lock().forget_archive(url); + let abandoned = crate::pmtiles::reload_archive(url); + crate::mbtiles::reload_archive(url); + crate::source::forget_archive(url); + drop(epochs); + // Dropping callbacks can complete tile answers and start queued reads: never under locks. + drop(abandoned); + notify(); +} + /// Waits (up to `timeout`) for the fetches under way to finish. Native only: in the browser /// fetches finish between frames, so this returns at once. pub fn wait(timeout: Duration) { @@ -794,9 +862,8 @@ pub fn wait_for_news_unless(since: u64, timeout: Duration, stop: impl Fn() -> bo let _ = (since, timeout, stop); } -/// A waiting fetch: its decode kind, diagnostics owner, expected statuses, when it was asked for -/// ([`land`]) and initiating context. -type Job = (Want, Capture, &'static [u16], u64, Option, crate::address::Reach); +/// A waiting fetch: decode kind, diagnostics, statuses, archive revision, context, MBTiles epoch and reach. +type Job = (Want, Capture, &'static [u16], u64, Option, u64, crate::address::Reach); fn queue() -> &'static Mutex> { static Q: OnceLock>> = OnceLock::new(); @@ -804,7 +871,7 @@ fn queue() -> &'static Mutex> { } /// Asks for a tile: fetched now when its server has room, else it waits its turn. -fn start(url: String, want: Want, omit: &'static [u16], asked_at: u64, ceiling: crate::address::Reach) { +fn start(url: String, want: Want, omit: &'static [u16], epoch: u64, asked_at: u64, ceiling: crate::address::Reach) { let capture = Capture::current(); let still = current_still(); IN_FLIGHT.fetch_add(1, Ordering::Relaxed); @@ -813,7 +880,8 @@ fn start(url: String, want: Want, omit: &'static [u16], asked_at: u64, ceiling: let mut q = queue().lock().unwrap_or_else(|e| e.into_inner()); let mut c = lock(); let latest = c.latest.unwrap_or_else(Instant::now); - let now = q.push(url, (want, capture, omit, asked_at, still, ceiling), |u| c.slots.get(u).map(|s| s.asked), latest, &mut swept); + let now = + q.push(url, (want, capture, omit, epoch, still, asked_at, ceiling), |u| c.slots.get(u).map(|s| s.asked), latest, &mut swept); // Under the same lock as the choice: a tile asked for again meanwhile isn't forgotten. forget_dropped(&mut c, &swept); now @@ -839,7 +907,9 @@ fn release(host: &str) { /// Called holding the cache's lock taken to choose them, so a tile can't be asked for between. fn forget_dropped(c: &mut Cache, dropped: &[Waiting]) { for w in dropped { - forget_pending(c, &w.url); + if matches!(c.slots.get(&w.url), Some(Slot { entry: Entry::Pending, epoch, .. }) if *epoch == w.job.3) { + forget_pending(c, &w.url); + } } IN_FLIGHT.fetch_sub(dropped.len(), Ordering::Release); } @@ -856,14 +926,20 @@ fn release_unstarted(unstarted: Vec>) { { let mut q = queue().lock().unwrap_or_else(|e| e.into_inner()); let mut c = lock(); + // Only this job's own pending slot: not one a newer request planted after an archive + // reload (archive revision) or after its MBTiles file was forgotten. + let own = |c: &Cache, w: &Waiting| { + !crate::mbtiles::forgotten_since(&w.url, w.job.5) + && matches!(c.slots.get(&w.url), Some(Slot { entry: Entry::Pending, epoch, .. }) if *epoch == w.job.3) + }; for w in &unstarted { - if !crate::mbtiles::forgotten_since(&w.url, w.job.3) { + if own(&c, w) { forget_pending(&mut c, &w.url); } stranded.extend(q.cancel(&w.host)); } for s in &stranded { - if !crate::mbtiles::forgotten_since(&s.url, s.job.3) { + if own(&c, s) { forget_pending(&mut c, &s.url); } } @@ -904,9 +980,9 @@ fn launch(jobs: Vec>) { } let _lower = Lower(starting); while let Some(w) = TO_START.with(|q| q.borrow_mut().pop()) { - let (want, capture, omit, asked_at, still, ceiling) = w.job; + let (want, capture, omit, epoch, still, asked_at, ceiling) = w.job; let _scope = capture.enter(); - with_optional_still(still, || fetch_tile(w.url, w.host, want, omit, asked_at, ceiling)); + with_optional_still(still, || fetch_tile(w.url, w.host, want, omit, epoch, asked_at, ceiling)); } }); } @@ -918,7 +994,8 @@ struct Answer { key: String, host: String, entry: Option, - /// When the tile was asked for, for an MBTiles file forgotten since ([`land`]). + epoch: u64, + /// When the tile was asked for, for an MBTiles file forgotten since (`land`). asked_at: u64, } @@ -931,7 +1008,7 @@ impl Answer { impl Drop for Answer { fn drop(&mut self) { - land(&self.key, self.entry.take(), self.asked_at); + if_archive_current(&self.key, self.epoch, || land(&self.key, self.entry.take(), self.asked_at)); release(&self.host); finished(); } @@ -960,11 +1037,14 @@ fn land(key: &str, entry: Option, asked_at: u64) { } /// Retains the initiating lifetime through deferred decoding. Retired results are forgotten; -/// only live successes and resource failures are shared with later callers. -fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16], asked_at: u64, ceiling: crate::address::Reach) { +/// only live successes and resource failures from the current archive revision are shared. +fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16], epoch: u64, asked_at: u64, ceiling: crate::address::Reach) { let owner = current_still(); let local = crate::pmtiles::is_tile(&url) || crate::cog::is_tile(&url) || crate::mbtiles::is_tile(&url); - let answer = Answer { key: url.clone(), host, entry: None, asked_at }; + let answer = Answer { key: url.clone(), host, entry: None, epoch, asked_at }; + if archive_epoch(&url) != epoch { + return; + } let done: FetchedDone = Box::new(move |fetched| { let (bytes, origin) = match fetched { Ok(f) => (Ok(f.bytes), (!local).then_some(f.origin)), @@ -1659,11 +1739,14 @@ fn keep(c: &mut Cache, url: &str, entry: Entry) { fn keep_for(c: &mut Cache, url: &str, entry: Entry, ceiling: crate::address::Reach) { let old = c.slots.remove(url); let asked = old.as_ref().map_or_else(Instant::now, |s| s.asked); + // The caller already holds the archive epoch lock when publishing a transport answer. + // Preserve its slot's revision without recursively acquiring that lock here. + let epoch = old.as_ref().map_or(0, |s| s.epoch); let other = old.and_then(|slot| { c.bytes = c.bytes.saturating_sub(slot.bytes()); if slot.ceiling != ceiling && !matches!(slot.entry, Entry::Pending) { Some((slot.ceiling, slot.entry)) } else { slot.other } }); - let slot = Slot { entry, ceiling, other, used: c.tick, asked }; + let slot = Slot { entry, ceiling, other, used: c.tick, asked, epoch }; c.bytes += slot.bytes(); c.slots.insert(url.to_string(), slot); if c.bytes > BUDGET { @@ -2924,6 +3007,145 @@ mod tests { use super::*; + #[test] + fn archive_reload_excludes_old_decodes_and_keeps_other_sources() { + let _turn = crate::cache::turn(); + let archive = "/cache-generation-regression.pmtiles"; + let tile = format!("pmtiles:0/0/0@{archive}"); + let other = "mbtiles:0/0/0@/cache-generation-other.mbtiles"; + let old_epoch = archive_epoch(archive); + let mut c = Cache::default(); + keep(&mut c, &tile, Entry::Ready(Arc::new(Payload::Json(serde_json::json!("old"))), 64, None)); + keep(&mut c, other, Entry::Ready(Arc::new(Payload::Empty), 64, None)); + let other_payload = match &c.slots[other].entry { + Entry::Ready(payload, _, _) => payload.clone(), + _ => panic!("ready tile"), + }; + let pending = format!("pmtiles:1/0/0@{archive}"); + c.slots.insert( + pending.clone(), + Slot { + entry: Entry::Pending, + ceiling: crate::address::Reach::Local, + other: None, + used: 0, + asked: Instant::now(), + epoch: old_epoch, + }, + ); + reload_archive(archive); + c.forget_archive(archive); + assert!(!c.slots.contains_key(&tile) && !c.slots.contains_key(&pending)); + assert_eq!(c.bytes, 64); + let fresh_epoch = archive_epoch(archive); + c.slots.insert( + tile.clone(), + Slot { + entry: Entry::Pending, + ceiling: crate::address::Reach::Local, + other: None, + used: 0, + asked: Instant::now(), + epoch: fresh_epoch, + }, + ); + assert!( + if_archive_current(&tile, old_epoch, || { + keep(&mut c, &tile, Entry::Ready(Arc::new(Payload::Json(serde_json::json!("stale"))), 64, None)); + }) + .is_none() + ); + assert!(matches!(c.slots[&tile].entry, Entry::Pending)); + if_archive_current(&tile, fresh_epoch, || { + keep(&mut c, &tile, Entry::Ready(Arc::new(Payload::Json(serde_json::json!("fresh"))), 64, None)); + }); + assert!(matches!(&c.slots[&tile].entry, Entry::Ready(p, _, _) if matches!(p.as_ref(), Payload::Json(v) if v == "fresh"))); + assert!(matches!(&c.slots[other].entry, Entry::Ready(p, _, _) if Arc::ptr_eq(p, &other_payload))); + assert_eq!(resource_of(other), "/cache-generation-other.mbtiles"); + } + + #[test] + fn delayed_archive_answers_cannot_replace_new_pixels_or_remove_new_pending_reads() { + let _turn = crate::cache::turn(); + for scheme in ["pmtiles", "mbtiles"] { + let archive = format!("/delayed-answer.{scheme}"); + let key = format!("{scheme}:0/0/0@{archive}"); + let old_epoch = archive_epoch(&archive); + let mut old_picture = Pixmap::new(1, 1).unwrap(); + old_picture.fill(tiny_skia::Color::from_rgba8(255, 0, 0, 255)); + // Hold the production completion object after decoding, as an asynchronous decoder + // does. No worker timing or process-wide fetcher replacement is involved. + IN_FLIGHT.fetch_add(2, Ordering::Relaxed); + let late = Answer { + key: key.clone(), + host: "delayed-answer.test".into(), + epoch: old_epoch, + asked_at: crate::mbtiles::epoch(), + entry: Some(Entry::Ready(Arc::new(Payload::Image(old_picture)), 4, None)), + }; + let unanswered = Answer { + key: key.clone(), + host: "delayed-answer.test".into(), + epoch: old_epoch, + asked_at: crate::mbtiles::epoch(), + entry: None, + }; + reload_archive(&archive); + let current = archive_epoch(&archive); + lock().slots.insert( + key.clone(), + Slot { + entry: Entry::Pending, + ceiling: crate::address::Reach::Local, + other: None, + used: 0, + asked: Instant::now(), + epoch: current, + }, + ); + drop(unanswered); + assert!(matches!(lock().slots.get(&key).map(|s| &s.entry), Some(Entry::Pending))); + let mut fresh_picture = Pixmap::new(1, 1).unwrap(); + fresh_picture.fill(tiny_skia::Color::from_rgba8(0, 255, 0, 255)); + let fresh = Arc::new(Payload::Image(fresh_picture)); + store(&key, Entry::Ready(fresh.clone(), 4, None)); + drop(late); + assert!(Arc::ptr_eq(&peek(&key).unwrap(), &fresh), "an old {scheme} decode replaced the new pixels"); + reload_archive(&archive); + } + } + + #[test] + fn pmtiles_names_that_resemble_cog_looks_reload_whole() { + let _turn = turn(); + for suffix in ["cube=1", "dequantize=1"] { + let archive = format!("/archive-look#{suffix}.pmtiles"); + let key = format!("pmtiles:0/0/0@{archive}"); + let mut old_picture = Pixmap::new(1, 1).unwrap(); + old_picture.fill(tiny_skia::Color::from_rgba8(255, 0, 0, 255)); + let old = Arc::new(Payload::Image(old_picture)); + store(&key, Entry::Ready(old.clone(), 4, None)); + IN_FLIGHT.fetch_add(1, Ordering::Relaxed); + let late = Answer { + key: key.clone(), + host: "archive-look.test".into(), + epoch: archive_epoch(&key), + asked_at: crate::mbtiles::epoch(), + entry: Some(Entry::Ready(old, 4, None)), + }; + reload_archive(&archive); + assert!(peek(&key).is_none(), "reload retained old pixels for {archive}"); + let mut fresh_picture = Pixmap::new(1, 1).unwrap(); + fresh_picture.fill(tiny_skia::Color::from_rgba8(0, 255, 0, 255)); + let fresh = Arc::new(Payload::Image(fresh_picture)); + store(&key, Entry::Ready(fresh.clone(), 4, None)); + drop(late); + assert!(Arc::ptr_eq(&peek(&key).unwrap(), &fresh), "old completion replaced fresh pixels for {archive}"); + forget(&archive); + assert!(peek(&key).is_none(), "forget retained pixels for {archive}"); + } + } + /// A full record of forgotten files is emptied. A tile asked for before that can't be told, so /// it isn't kept and is asked for again; one asked for since is, and so is what isn't MBTiles. #[test] @@ -2934,9 +3156,17 @@ mod tests { let key = crate::mbtiles::template(&archive(0)).replace("{z}", "0").replace("{x}", "0").replace("{y}", "0"); let failed = |why: &str| Some(Entry::Failed(why.into(), Instant::now())); let old = crate::mbtiles::epoch(); + let revision = archive_epoch(&key); lock().slots.insert( key.clone(), - Slot { entry: Entry::Pending, ceiling: crate::address::Reach::Local, other: None, used: 0, asked: Instant::now() }, + Slot { + entry: Entry::Pending, + ceiling: crate::address::Reach::Local, + other: None, + used: 0, + asked: Instant::now(), + epoch: revision, + }, ); for i in 0..=crate::mbtiles::MAX_FORGOTTEN { crate::mbtiles::forget_archive(&archive(i)); @@ -2961,6 +3191,7 @@ mod tests { let archive = "/maps#cube=1.mbtiles"; let key = crate::mbtiles::template(archive).replace("{z}", "0").replace("{x}", "0").replace("{y}", "0"); let old = crate::mbtiles::epoch(); + let revision = archive_epoch(&key); lock().slots.insert( key.clone(), Slot { @@ -2969,6 +3200,7 @@ mod tests { other: None, used: 0, asked: Instant::now(), + epoch: revision, }, ); forget(archive); @@ -2993,9 +3225,17 @@ mod tests { }; let old = crate::mbtiles::epoch(); + let revision = archive_epoch(&key); lock().slots.insert( key.clone(), - Slot { entry: Entry::Pending, ceiling: crate::address::Reach::Local, other: None, used: 0, asked: Instant::now() }, + Slot { + entry: Entry::Pending, + ceiling: crate::address::Reach::Local, + other: None, + used: 0, + asked: Instant::now(), + epoch: revision, + }, ); forget(archive); land(&key, failed("old file"), old); @@ -3012,7 +3252,14 @@ mod tests { let new = crate::mbtiles::epoch(); lock().slots.insert( key.clone(), - Slot { entry: Entry::Pending, ceiling: crate::address::Reach::Local, other: None, used: 0, asked: Instant::now() }, + Slot { + entry: Entry::Pending, + ceiling: crate::address::Reach::Local, + other: None, + used: 0, + asked: Instant::now(), + epoch: revision, + }, ); land(&key, failed("old file"), old); assert!(matches!(lock().slots.get(&key), Some(Slot { entry: Entry::Pending, .. })), "the new read's slot stays"); @@ -5095,11 +5342,23 @@ mod tests { if isolate_cache_fixture("cache::tests::fetches_through_the_fetcher_once") { return; } + use std::sync::mpsc; let _turn = crate::cache::turn(); + + fn receive(rx: &mpsc::Receiver<()>) { + rx.recv_timeout(Duration::from_secs(10)).expect("the test fetcher answered or admitted the request"); + } + + let (completed_tx, completed_rx) = mpsc::channel(); static CALLS: AtomicUsize = AtomicUsize::new(0); - set_fetcher(|url, done| { + set_fetcher(move |url, done| { CALLS.fetch_add(1, Ordering::Relaxed); - if url.ends_with("ok.json") { done(Ok(b"[1, 2]".to_vec())) } else { done(Err("404".into())) } + if url.ends_with("ok.json") { + done(Ok(b"[1, 2]".to_vec())) + } else { + done(Err("404".into())) + } + completed_tx.send(()).unwrap(); }); let g = generation(); assert!(matches!(get("test://cache/ok.json", Want::Json), State::Pending)); @@ -5108,25 +5367,37 @@ mod tests { wait_for_news(g, Duration::from_secs(5)); } assert!(generation() > g); + receive(&completed_rx); assert!(matches!(get("test://cache/ok.json", Want::Json), State::Ready(_))); assert!(peek("test://cache/ok.json").is_some()); assert!(matches!(get("test://cache/missing.json", Want::Json), State::Pending)); + receive(&completed_rx); assert!(matches!(get("test://cache/missing.json", Want::Json), State::Failed(_))); assert!(peek("test://cache/never.json").is_none()); // A request the fetcher dropped is forgotten, not failed: asked for again, it is fetched. - set_fetcher(|url, done| { + let (completed_tx, completed_rx) = mpsc::channel(); + set_fetcher(move |url, done| { CALLS.fetch_add(1, Ordering::Relaxed); - if url.ends_with("first.json") { done(Err(DROPPED.into())) } else { done(Ok(b"1".to_vec())) } + if url.ends_with("first.json") { + done(Err(DROPPED.into())) + } else { + done(Ok(b"1".to_vec())) + } + completed_tx.send(()).unwrap(); }); assert!(matches!(get("test://cache/first.json", Want::Json), State::Pending)); + receive(&completed_rx); assert!(matches!(get("test://cache/first.json", Want::Json), State::Pending), "fetched again"); + receive(&completed_rx); assert_eq!(CALLS.load(Ordering::Relaxed), 4); // A server has a few fetches under way at once; the newest waiting tile goes next (#635). static HELD: Mutex> = Mutex::new(Vec::new()); - set_fetcher(|url, done| { + let (started_tx, started_rx) = mpsc::channel(); + set_fetcher(move |url, done| { if url.starts_with("test://held/") { HELD.lock().unwrap().push((url, done)); + started_tx.send(()).unwrap(); } else { done(Ok(b"1".to_vec())); } @@ -5135,16 +5406,25 @@ mod tests { for i in 0..n { assert!(matches!(get(&format!("test://held/{i}.json"), Want::Json), State::Pending)); } + for _ in 0..crate::schedule::PER_HOST { + receive(&started_rx); + } assert_eq!(HELD.lock().unwrap().len(), crate::schedule::PER_HOST); - let (_, done) = HELD.lock().unwrap().remove(0); + let (completed_url, done) = HELD.lock().unwrap().remove(0); done(Ok(b"2".to_vec())); + receive(&started_rx); let held: Vec = HELD.lock().unwrap().iter().map(|(u, _)| u.clone()).collect(); assert_eq!(held.len(), crate::schedule::PER_HOST); assert_eq!(held.last().map(String::as_str), Some(format!("test://held/{}.json", n - 1).as_str())); - assert!(matches!(get("test://held/0.json", Want::Json), State::Ready(_))); + assert!(matches!(get(&completed_url, Want::Json), State::Ready(_))); // The rest finish, and so do the waiting ones they make room for. - loop { - let Some((_, done)) = HELD.lock().unwrap().pop() else { break }; + for _ in 1..n { + let (_, done) = loop { + if let Some(job) = HELD.lock().unwrap().pop() { + break job; + } + receive(&started_rx); + }; done(Ok(b"3".to_vec())); } for i in 0..n { @@ -5227,10 +5507,12 @@ mod tests { // A fetcher that drops a request unanswered (or panics) gives its server's place back: // more requests than the cap still all reach it, and each is forgotten, not stuck. static SEEN: AtomicUsize = AtomicUsize::new(0); - set_fetcher(|url, done| { + let (completed_tx, completed_rx) = mpsc::channel(); + set_fetcher(move |url, done| { if url.starts_with("test://lost/") { SEEN.fetch_add(1, Ordering::Relaxed); drop(done); + completed_tx.send(()).unwrap(); } else { done(Ok(b"1".to_vec())); } @@ -5238,15 +5520,20 @@ mod tests { for i in 0..3 * crate::schedule::PER_HOST { assert!(matches!(get(&format!("test://lost/{i}.json"), Want::Json), State::Pending)); } + for _ in 0..3 * crate::schedule::PER_HOST { + receive(&completed_rx); + } assert_eq!(SEEN.load(Ordering::Relaxed), 3 * crate::schedule::PER_HOST); assert!(peek("test://lost/0.json").is_none()); // Waiting for news: a fetch never answered (another's, a slow server's) doesn't hold up // one waiting for what has come. static STUCK: Mutex> = Mutex::new(Vec::new()); - set_fetcher(|url, done| { + let (started_tx, started_rx) = mpsc::channel(); + set_fetcher(move |url, done| { if url.starts_with("test://stuck/") { STUCK.lock().unwrap().push(done); + started_tx.send(()).unwrap(); } else { std::thread::spawn(move || { std::thread::sleep(std::time::Duration::from_millis(50)); @@ -5255,6 +5542,7 @@ mod tests { } }); assert!(matches!(get("test://stuck/0.json", Want::Json), State::Pending)); + receive(&started_rx); let g = generation(); assert!(matches!(get("test://news/0.json", Want::Json), State::Pending)); let t = Instant::now(); @@ -5270,20 +5558,26 @@ mod tests { // Only a map tile's 404 is an expected answer: a picture that isn't a tile (a single // image, a layout's picture, a sprite sheet) and a style keep theirs. static OMITS: Mutex> = Mutex::new(Vec::new()); - set_fetcher_omitting(|url, done, omit| { + let (completed_tx, completed_rx) = mpsc::channel(); + set_fetcher_omitting(move |url, done, omit| { OMITS.lock().unwrap().push((url, omit)); done(Err("http status: 404".into())); + completed_tx.send(()).unwrap(); }); let _ = get_tile("test://omit/tile.png", Want::Image); let _ = get_tile("test://omit/tile.pbf", Want::Vector); let _ = get("test://omit/picture.png", Want::Image); let _ = get("test://omit/style.json", Want::Json); + for _ in 0..4 { + receive(&completed_rx); + } let omits = std::mem::take(&mut *OMITS.lock().unwrap()); let omit = |name: &str| omits.iter().find(|(u, _)| u.ends_with(name)).map(|(_, o)| *o); assert_eq!(omit("tile.png"), Some(&[404, 410][..])); assert_eq!(omit("tile.pbf"), Some(&[404, 410][..])); assert_eq!(omit("picture.png"), Some(&[][..])); assert_eq!(omit("style.json"), Some(&[][..])); + clear_fetcher(); } #[test] @@ -5412,7 +5706,7 @@ mod tests { let mut q = queue().lock().unwrap_or_else(|e| e.into_inner()); let mut c = lock(); let latest = c.latest.unwrap_or_else(Instant::now); - let job = (Want::Json, Capture::current(), &[] as &'static [u16], 0u64, None, crate::address::Reach::Public); + let job = (Want::Json, Capture::current(), &[] as &'static [u16], 0u64, None, 0u64, crate::address::Reach::Public); let w = q .push(url.to_string(), job, |u| c.slots.get(u).map(|s| s.asked), latest, &mut swept) .expect("job should be scheduled immediately"); @@ -5453,7 +5747,7 @@ mod tests { let mut q = queue().lock().unwrap_or_else(|e| e.into_inner()); let mut c = lock(); let latest = c.latest.unwrap_or_else(Instant::now); - let job = (Want::Vector, Capture::current(), &[] as &'static [u16], old_epoch, None, crate::address::Reach::Local); + let job = (Want::Vector, Capture::current(), &[] as &'static [u16], 0u64, None, old_epoch, crate::address::Reach::Local); let w = q .push(key.clone(), job, |u| c.slots.get(u).map(|s| s.asked), latest, &mut swept) .expect("job should be scheduled immediately"); @@ -5470,7 +5764,14 @@ mod tests { let mut c = lock(); c.slots.insert( key.clone(), - Slot { entry: Entry::Pending, ceiling: crate::address::Reach::Local, other: None, used: 0, asked: Instant::now() }, + Slot { + entry: Entry::Pending, + ceiling: crate::address::Reach::Local, + other: None, + used: 0, + asked: Instant::now(), + epoch: 0, + }, ); } @@ -5505,10 +5806,17 @@ mod tests { let mut q = queue().lock().unwrap_or_else(|e| e.into_inner()); let mut c = lock(); let latest = c.latest.unwrap_or_else(Instant::now); - let job = (Want::Json, Capture::current(), &[] as &'static [u16], 0u64, None, crate::address::Reach::Public); + let job = (Want::Json, Capture::current(), &[] as &'static [u16], 0u64, None, 0u64, crate::address::Reach::Public); c.slots.insert( url.clone(), - Slot { entry: Entry::Pending, ceiling: crate::address::Reach::Public, other: None, used: 0, asked: Instant::now() }, + Slot { + entry: Entry::Pending, + ceiling: crate::address::Reach::Public, + other: None, + used: 0, + asked: Instant::now(), + epoch: 0, + }, ); let w = q .push(url, job, |u| c.slots.get(u).map(|s| s.asked), latest, &mut swept) @@ -5525,10 +5833,17 @@ mod tests { let mut q = queue().lock().unwrap_or_else(|e| e.into_inner()); let mut c = lock(); let latest = c.latest.unwrap_or_else(Instant::now); - let job = (Want::Json, Capture::current(), &[] as &'static [u16], 0u64, None, crate::address::Reach::Public); + let job = (Want::Json, Capture::current(), &[] as &'static [u16], 0u64, None, 0u64, crate::address::Reach::Public); c.slots.insert( url.clone(), - Slot { entry: Entry::Pending, ceiling: crate::address::Reach::Public, other: None, used: 0, asked: Instant::now() }, + Slot { + entry: Entry::Pending, + ceiling: crate::address::Reach::Public, + other: None, + used: 0, + asked: Instant::now(), + epoch: 0, + }, ); let w = q.push(url.clone(), job, |u| c.slots.get(u).map(|s| s.asked), latest, &mut swept); assert!(w.is_none(), "job should be queued in waiting because host is at capacity"); diff --git a/crates/tiles/src/mbtiles.rs b/crates/tiles/src/mbtiles.rs index 0d2451c753..8566780b91 100644 --- a/crates/tiles/src/mbtiles.rs +++ b/crates/tiles/src/mbtiles.rs @@ -142,6 +142,44 @@ pub(crate) fn forget() { rustgis_data::mbtiles::close_all(); } +/// Called under the cache's archive-revision lock. +pub(crate) fn reload_archive(archive: &str) { + infos().lock().unwrap_or_else(|e| e.into_inner()).remove(archive); + #[cfg(not(target_arch = "wasm32"))] + rustgis_data::mbtiles::close(file_of(archive)); +} + +fn info_epoch(archive: &str) -> (u64, u64) { + (EPOCH.load(Ordering::SeqCst), crate::cache::archive_epoch(archive)) +} + +/// Reads the current file and decodes one raster sample, without using or changing any cache. +pub fn read_info_local(archive: &str) -> Result, String> { + check(archive)?; + if is_upload(archive) { + return Err(format!("{archive}: an upload is not a local MBTiles file")); + } + #[cfg(not(target_arch = "wasm32"))] + { + let (info, sample) = rustgis_data::mbtiles::read_info_path(file_of(archive))?; + if let Some(sample) = sample { + validate_raster_sample(archive, &sample)?; + } + Ok(Arc::new(info)) + } + #[cfg(target_arch = "wasm32")] + { + Err(format!("{archive}: the web app cannot reload local MBTiles paths")) + } +} + +#[cfg(not(target_arch = "wasm32"))] +fn validate_raster_sample(archive: &str, sample: &[u8]) -> Result<(), String> { + crate::cache::decode_image(sample) + .map(|_| ()) + .map_err(|e| format!("{archive}: cannot decode the MBTiles raster sample ({e}); repair or replace the archive before reloading")) +} + /// Forgets what was read of one file (its metadata and its connections) with every file's resolved /// styles, made again when next drawn, and won't keep a read of it under way: the file may be /// replaced next. Reads of other files are kept, as they may be framing a map, until the record of @@ -180,35 +218,40 @@ pub fn info(archive: &str) -> Result, String> { if let Some(result) = cached_info(archive) { return result; } - let epoch = EPOCH.load(Ordering::SeqCst); + let epoch = info_epoch(archive); let r = read_info(archive); crate::cache::check_still(&crate::cache::current_still())?; remember_info(archive, &r, epoch); r } -/// Keeps a completion only if neither its file nor every file was forgotten since its read started -/// (a full clear, or the record of forgotten files emptied); check and insert share the same lock -/// as the clear, so an old completion cannot repopulate the new cache. -fn remember_info(archive: &str, result: &Result, String>, epoch: u64) { +/// Checks the initiating context before locking, then keeps a completion only while both its +/// metadata-clear, per-file forget and archive epochs remain current under their invalidation locks. +fn remember_info(archive: &str, result: &Result, String>, epoch: (u64, u64)) { if crate::cache::check_still(&crate::cache::current_still()).is_err() { return; } - let mut all = infos().lock().unwrap_or_else(|e| e.into_inner()); - let forgotten_at = forgotten().lock().unwrap_or_else(|e| e.into_inner()).get(archive).copied().unwrap_or(0); - if epoch < CLEARED.load(Ordering::SeqCst).max(forgotten_at) { - return; - } - if all.len() >= 64 { - all.clear(); - } - all.insert(archive.to_string(), result.clone().map_err(|e| (e, Instant::now()))); + crate::cache::if_archive_current(archive, epoch.1, || { + let mut all = infos().lock().unwrap_or_else(|e| e.into_inner()); + let forgotten_at = forgotten().lock().unwrap_or_else(|e| e.into_inner()).get(archive).copied().unwrap_or(0); + if epoch.0 < CLEARED.load(Ordering::SeqCst).max(forgotten_at) { + return; + } + if all.len() >= 64 { + all.clear(); + } + all.insert(archive.to_string(), result.clone().map_err(|e| (e, Instant::now()))); + }); } /// An existing read has its own deadline. An unadmitted read reports saturation once a worker /// has stalled; workers keep their slots until the OS returns, never spawning unbounded threads. #[cfg(not(target_arch = "wasm32"))] -fn metadata_stalled(reads: &HashMap, archive: &str) -> bool { +fn metadata_stalled(reads: &HashMap, archive: &Q) -> bool +where + K: Eq + std::hash::Hash + std::borrow::Borrow, + Q: Eq + std::hash::Hash + ?Sized, +{ reads.get(archive).map_or_else( || reads.len() >= 8 && reads.values().any(|start| start.elapsed() >= READ_DEADLINE), |start| start.elapsed() >= READ_DEADLINE, @@ -230,11 +273,15 @@ fn cached_info(archive: &str) -> Option, String>> { }) } +#[cfg(not(target_arch = "wasm32"))] +type Pending = HashMap<(String, u64), Instant>; + /// The metadata workers' shared pending reads and completion signal. #[cfg(not(target_arch = "wasm32"))] -fn metadata_reads() -> &'static (Mutex>, std::sync::Condvar) { - static LOADING: OnceLock<(Mutex>, std::sync::Condvar)> = OnceLock::new(); - LOADING.get_or_init(|| (Mutex::new(HashMap::new()), std::sync::Condvar::new())) +fn metadata_reads() -> &'static (Mutex, std::sync::Condvar) { + static LOADING: std::sync::LazyLock<(Mutex, std::sync::Condvar)> = + std::sync::LazyLock::new(|| (Mutex::new(HashMap::new()), std::sync::Condvar::new())); + &LOADING } /// Metadata without file I/O on the caller: local reads run in at most eight workers. A caller @@ -256,38 +303,42 @@ pub fn info_ready(archive: &str, timeout: Duration) -> Result>, #[cfg(not(target_arch = "wasm32"))] { let (pending, ready) = metadata_reads(); + let epoch = info_epoch(archive); + let key = (archive.to_string(), epoch.1); let mut all = pending.lock().unwrap_or_else(|e| e.into_inner()); if let Some(result) = cached_info(archive) { return result.map(Some); } - if metadata_stalled(&all, archive) { + if metadata_stalled(&all, &key) { return Err(format!( "{archive}: MBTiles metadata read exceeded 30 seconds or all eight workers are occupied by stalled storage; retry after the storage responds" )); } - if !all.contains_key(archive) { + if !all.contains_key(&key) { if all.len() >= 8 { return Ok(None); } - all.insert(archive.to_string(), Instant::now()); + all.insert(key.clone(), Instant::now()); let address = archive.to_string(); + let worker_key = key.clone(); let still = crate::cache::current_still(); let spawned = std::thread::Builder::new().name("rustgis-mbtiles-info".into()).spawn(move || { crate::cache::with_optional_still(still, || { - let _ = info(&address); + let result = crate::cache::check_still(&crate::cache::current_still()).and_then(|_| read_info(&address)); + remember_info(&address, &result, epoch); }); - pending.lock().unwrap_or_else(|e| e.into_inner()).remove(&address); + pending.lock().unwrap_or_else(|e| e.into_inner()).remove(&worker_key); ready.notify_all(); crate::cache::notify(); }); if let Err(error) = spawned { - all.remove(archive); + all.remove(&key); return Err(format!("could not start an MBTiles metadata read: {error}")); } } if !timeout.is_zero() { - let wait = all.get(archive).map_or(Duration::ZERO, |start| metadata_wait(*start, timeout)); - let (reads, _) = ready.wait_timeout_while(all, wait, |all| all.contains_key(archive)).unwrap_or_else(|e| e.into_inner()); + let wait = all.get(&key).map_or(Duration::ZERO, |start| metadata_wait(*start, timeout)); + let (reads, _) = ready.wait_timeout_while(all, wait, |all| all.contains_key(&key)).unwrap_or_else(|e| e.into_inner()); all = reads; } drop(all); @@ -296,7 +347,7 @@ pub fn info_ready(archive: &str, timeout: Duration) -> Result>, if let Some(result) = cached_info(archive) { return result.map(Some); } - if metadata_stalled(&all, archive) { + if metadata_stalled(&all, &key) { return Err(format!("{archive}: MBTiles metadata read exceeded 30 seconds; retry after the storage responds")); } Ok(None) @@ -385,13 +436,27 @@ mod tests { } true }); - let epoch = EPOCH.load(Ordering::SeqCst); let address = "/reentrant-cache-completion.mbtiles"; + let epoch = info_epoch(address); crate::cache::with_still(still, || remember_info(address, &Err("old completion".into()), epoch)); assert!(available.load(Ordering::SeqCst), "Still ran with the metadata cache locked"); assert!(cached_info(address).is_none(), "a clear during the predicate invalidates the earlier completion"); } + /// An authority predicate may reload this source before the archive-epoch publication gate. + #[test] + fn metadata_completion_checks_still_outside_the_archive_lock() { + let _turn = crate::cache::turn(); + let address = "/reentrant-archive-completion.mbtiles"; + let epoch = info_epoch(address); + let still: crate::cache::Still = Arc::new(move || { + crate::cache::reload_archive(address); + true + }); + crate::cache::with_still(still, || remember_info(address, &Err("old completion".into()), epoch)); + assert!(cached_info(address).is_none(), "a reload during the predicate invalidates the earlier completion"); + } + /// Both preflight and post-wait authority checks permit metadata reentry without a lock cycle. #[cfg(not(target_arch = "wasm32"))] #[test] @@ -439,12 +504,38 @@ mod tests { #[test] fn cleared_metadata_rejects_an_older_completion() { let _turn = crate::cache::turn(); - let epoch = EPOCH.load(Ordering::SeqCst); + let epoch = info_epoch("/old-completion.mbtiles"); forget(); remember_info("/old-completion.mbtiles", &Err("obsolete read".into()), epoch); assert!(cached_info("/old-completion.mbtiles").is_none()); - remember_info("/fresh-completion.mbtiles", &Err("current read".into()), EPOCH.load(Ordering::SeqCst)); + remember_info("/fresh-completion.mbtiles", &Err("current read".into()), info_epoch("/fresh-completion.mbtiles")); assert!(cached_info("/fresh-completion.mbtiles").is_some()); + let archive = "/mbtiles-source-generation.mbtiles"; + let epoch = info_epoch(archive); + crate::cache::reload_archive(archive); + let current = info_epoch(archive); + remember_info(archive, &Err("current error".into()), current); + remember_info(archive, &Err("obsolete error".into()), epoch); + assert_eq!(cached_info(archive).unwrap().unwrap_err(), "current error"); + crate::cache::reload_archive(archive); + remember_info(archive, &Err("obsolete failure after retry".into()), current); + assert!(cached_info(archive).is_none()); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn raster_samples_require_a_decodable_bounded_image() { + let archive = "/invalid-raster-sample.mbtiles"; + for sample in [b"\x89PNG\r\n\x1a\n".as_slice(), b"not a picture", b""] { + let error = validate_raster_sample(archive, sample).unwrap_err(); + assert!(error.contains(archive), "{error}"); + assert!(error.contains("cannot decode the MBTiles raster sample"), "{error}"); + assert!(error.contains("repair or replace"), "{error}"); + } + let valid = tiny_skia::Pixmap::new(2, 2).unwrap().encode_png().unwrap(); + assert!(validate_raster_sample(archive, &valid).is_ok()); + let oversized = tiny_skia::Pixmap::new(4097, 1).unwrap().encode_png().unwrap(); + assert!(validate_raster_sample(archive, &oversized).is_err()); } /// The metadata of a read under way when its file was forgotten isn't kept; a read of another @@ -452,13 +543,13 @@ mod tests { #[test] fn a_forgotten_file_rejects_only_its_own_older_completion() { let _turn = crate::cache::turn(); - let epoch = EPOCH.load(Ordering::SeqCst); + let epoch = info_epoch("/forgotten-while-read.mbtiles"); forget_archive("/forgotten-while-read.mbtiles"); remember_info("/forgotten-while-read.mbtiles", &Err("old file".into()), epoch); assert!(cached_info("/forgotten-while-read.mbtiles").is_none()); // Tried several times: another test's clear may fall in between, but not every time. assert!((0..100).any(|_| { - let epoch = EPOCH.load(Ordering::SeqCst); + let epoch = info_epoch("/read-meanwhile.mbtiles"); forget_archive("/forgotten-while-read.mbtiles"); forget_archive("https://example.com/not-an-archive"); remember_info("/read-meanwhile.mbtiles", &Err("another file".into()), epoch); @@ -471,16 +562,21 @@ mod tests { #[test] fn stalled_metadata_reports_errors_without_recycling_worker_slots() { let old = Instant::now().checked_sub(READ_DEADLINE + Duration::from_secs(1)).expect("an earlier instant"); - let mut reads: HashMap = (0..8).map(|i| (format!("/{i}.mbtiles"), Instant::now())).collect(); - assert!(!metadata_stalled(&reads, "/next.mbtiles"), "busy workers initially remain pending"); + let mut reads: Pending = (0..8).map(|i| ((format!("/{i}.mbtiles"), 0), Instant::now())).collect(); + let next = ("/next.mbtiles".into(), 0); + let stalled = ("/0.mbtiles".into(), 0); + let reloaded = ("/0.mbtiles".into(), 1); + let healthy = ("/1.mbtiles".into(), 0); + assert!(!metadata_stalled(&reads, &next), "busy workers initially remain pending"); assert!(metadata_wait(Instant::now(), Duration::from_secs(60)) <= READ_DEADLINE); assert_eq!(metadata_wait(old, Duration::from_secs(60)), Duration::ZERO); - reads.insert("/0.mbtiles".into(), old); - assert!(metadata_stalled(&reads, "/0.mbtiles"), "the stalled read reports its deadline"); - assert!(metadata_stalled(&reads, "/next.mbtiles"), "a saturated stalled pool is an error, not endless loading"); - assert!(!metadata_stalled(&reads, "/1.mbtiles"), "a healthy admitted read retains its own deadline"); - reads.remove("/0.mbtiles"); - assert!(!metadata_stalled(&reads, "/next.mbtiles"), "capacity can recover when the OS returns"); + reads.insert(stalled.clone(), old); + assert!(metadata_stalled(&reads, &stalled), "the stalled read reports its deadline"); + assert!(metadata_stalled(&reads, &next), "a saturated stalled pool is an error, not endless loading"); + assert!(metadata_stalled(&reads, &reloaded), "a reload cannot recycle a stalled generation's worker slot"); + assert!(!metadata_stalled(&reads, &healthy), "a healthy admitted read retains its own deadline"); + reads.remove(&stalled); + assert!(!metadata_stalled(&reads, &next), "capacity can recover when the OS returns"); } #[test] diff --git a/crates/tiles/src/pmtiles.rs b/crates/tiles/src/pmtiles.rs index 0de9570d29..12f8f8da72 100644 --- a/crates/tiles/src/pmtiles.rs +++ b/crates/tiles/src/pmtiles.rs @@ -8,7 +8,9 @@ //! too, where the server must allow range requests (and CORS). //! //! Archives are hostile input: offsets, lengths and directory sizes are checked against the -//! file and bounded, and nesting is limited. +//! file and bounded, and nesting is limited. Metadata must be readable, valid JSON objects. +//! Shared reads retain their initiating request lifetime; retired contexts cannot publish +//! archive information or directories, and reload epochs reject pre-reload completions. use std::collections::HashMap; use std::sync::{Arc, Mutex, OnceLock}; @@ -29,6 +31,10 @@ const MAX_ENTRIES: usize = 1 << 20; const MAX_DEPTH: usize = 4; /// Leaf directories kept. const MAX_LEAVES: usize = 256; +/// Fresh validation bounds the entire reachable index, including the root. +#[cfg(not(target_arch = "wasm32"))] +const INDEX_BUDGET: IndexBudget = + IndexBudget { stored: (MAX_BYTES * 4) as u64, decoded: (MAX_BYTES * 4) as u64, entries: MAX_ENTRIES, directories: MAX_LEAVES * 16 }; /// A failed archive is tried again after this long. const RETRY: Duration = Duration::from_secs(60); @@ -68,6 +74,7 @@ pub struct Info { data_length: u64, internal: Compression, tile: Compression, + epoch: u64, } impl Info { @@ -113,7 +120,12 @@ impl Compression { }; match self { Compression::None => Ok(b), - Compression::Gzip => gunzip(&b), + Compression::Gzip => { + if !b.starts_with(&[0x1f, 0x8b]) { + return Err("gzip: invalid header".into()); + } + gunzip(&b) + } Compression::Brotli => bounded(&mut brotli_decompressor::Decompressor::new(b.as_slice(), 4096), "Brotli"), Compression::Zstd => unzstd(&b, limit), } @@ -233,7 +245,8 @@ pub struct NewArchive { pub gzipped_tiles: bool, /// Longitude/latitude bounds: west, south, east, north. pub bounds: [f64; 4], - /// The JSON metadata (`vector_layers`, `attribution`, `name`…). + /// The JSON metadata object (`vector_layers`, `attribution`, `name`…); use `{}` when absent. + /// Its serialized JSON must not exceed 16 MiB. pub metadata: Value, } @@ -241,6 +254,13 @@ pub struct NewArchive { /// stored once, its directories uncompressed, in leaf directories when the root would pass the /// first 16 KB. Tiles given twice keep the last. pub fn write(tiles: Vec<(u8, u32, u32, Vec)>, a: &NewArchive) -> Result, String> { + if !a.metadata.is_object() { + return Err("PMTiles metadata must be a JSON object".into()); + } + let meta = serde_json::to_vec(&a.metadata).map_err(|e| e.to_string())?; + if meta.len() > MAX_BYTES { + return Err("PMTiles metadata exceeds the 16 MiB limit".into()); + } // A tile outside its zoom would take the id of another zoom's; readers stop at zoom 30. if let Some((z, x, y, _)) = tiles.iter().find(|(z, x, y, _)| *z > 30 || u64::from(*x) >= 1u64 << z || u64::from(*y) >= 1u64 << z) { return Err(format!("{z}/{x}/{y} isn't a tile (zoom 0–30, x and y below 2^zoom)")); @@ -295,7 +315,6 @@ pub fn write(tiles: Vec<(u8, u32, u32, Vec)>, a: &NewArchive) -> Result impl Sized { + let state = archives().lock().unwrap_or_else(|e| e.into_inner()).remove(archive); + let mut all = leaves().lock().unwrap_or_else(|e| e.into_inner()); + let keys: Vec<_> = all.keys().filter(|(a, _)| a == archive).cloned().collect(); + let removed: Vec<_> = keys.into_iter().filter_map(|key| all.remove(&key)).collect(); + (state, removed) +} + /// An archive's header and metadata: `Ok(None)` while they are read (the map draws again once /// they arrive). pub fn info(archive: &str) -> Result>, String> { + crate::cache::check_still(&crate::cache::current_still())?; crate::logins::check(archive).map_err(str::to_string)?; - let mut all = archives().lock().unwrap_or_else(|e| e.into_inner()); - match all.get(archive) { - Some(State::Ready(i)) => return Ok(Some(i.clone())), - Some(State::Loading(_)) => return Ok(None), - Some(State::Failed(e, at)) => { - // Not read again until its wait is over: the map draws again then. - let wait = crate::cache::retry_wait(e, RETRY); - if at.elapsed() < wait { - crate::cache::notify_at(*at + wait); - return Err(e.clone()); + let epoch = crate::cache::archive_epoch(archive); + let result = crate::cache::if_archive_current(archive, epoch, || { + let mut all = archives().lock().unwrap_or_else(|e| e.into_inner()); + match all.get(archive) { + Some(State::Ready(i)) => return (Ok(Some(i.clone())), false, None), + Some(State::Loading(_)) => return (Ok(None), false, None), + Some(State::Failed(e, at)) => { + let wait = crate::cache::retry_wait(e, RETRY); + if at.elapsed() < wait { + return (Err(e.clone()), false, Some(*at + wait)); + } + } + _ => {} + } + all.insert(archive.to_string(), State::Loading(Vec::new())); + (Ok(None), true, None) + }); + match result { + Some((result, start, retry_at)) => { + if let Some(at) = retry_at { + crate::cache::notify_at(at); + } + if start { + load(archive.to_string(), epoch); } + result } - _ => {} + None => Ok(None), } - all.insert(archive.to_string(), State::Loading(Vec::new())); - drop(all); - load(archive.to_string()); - Ok(None) } /// The archive's info, read now if it hasn't been, waiting up to `timeout` (natively: the @@ -446,34 +487,47 @@ pub fn info_blocking(archive: &str, timeout: std::time::Duration) -> Result, String>) + Send>) { let f = caller_callback(f); + if let Err(e) = crate::cache::check_still(&crate::cache::current_still()) { + f(Err(e)); + return; + } if let Err(e) = crate::logins::check(archive) { f(Err(e.to_string())); return; } - let mut all = archives().lock().unwrap_or_else(|e| e.into_inner()); - match all.get_mut(archive) { - Some(State::Ready(i)) => { - let i = i.clone(); - drop(all); - f(Ok(i)); + let epoch = crate::cache::archive_epoch(archive); + let result = crate::cache::if_archive_current(archive, epoch, || { + let mut all = archives().lock().unwrap_or_else(|e| e.into_inner()); + match all.get_mut(archive) { + Some(State::Ready(i)) => (Some((f, Ok(i.clone()))), false, None), + Some(State::Loading(waiting)) => { + waiting.push(f); + (None, false, None) + } + Some(State::Failed(e, at)) if at.elapsed() < crate::cache::retry_wait(e, RETRY) => { + (Some((f, Err(e.clone()))), false, Some(*at + crate::cache::retry_wait(e, RETRY))) + } + _ => { + all.insert(archive.to_string(), State::Loading(vec![f])); + (None, true, None) + } } - Some(State::Loading(waiting)) => waiting.push(f), - Some(State::Failed(e, at)) if at.elapsed() < crate::cache::retry_wait(e, RETRY) => { - crate::cache::notify_at(*at + crate::cache::retry_wait(e, RETRY)); - let e = e.clone(); - drop(all); - f(Err(e)); + }); + if let Some((answer, start, retry_at)) = result { + if let Some(at) = retry_at { + crate::cache::notify_at(at); } - _ => { - all.insert(archive.to_string(), State::Loading(vec![f])); - drop(all); - load(archive.to_string()); + if let Some((f, result)) = answer { + f(result); + } + if start { + load(archive.to_string(), epoch); } } } /// Reads an archive's header, root directory and metadata, then answers whoever waits. -fn load(archive: String) { +fn load(archive: String, epoch: u64) { let a = archive.clone(); read( &archive, @@ -482,10 +536,11 @@ fn load(archive: String) { Box::new(move |head| { let parsed = head.and_then(|b| parse_head(&b)); match parsed { - Err(e) => finish(&a, Err(e)), + Err(e) => finish(&a, Err(e), epoch), Ok((mut info, meta_at, meta_len)) => { + info.epoch = epoch; if meta_len == 0 { - finish(&a, Ok(info)); + finish(&a, Ok(info), epoch); return; } let a2 = a.clone(); @@ -494,18 +549,8 @@ fn load(archive: String) { meta_at, meta_len, Box::new(move |m| { - if let Err(e) = &m - && !crate::cache::keep_failure(e, &crate::cache::current_still()) - { - finish(&a2, Err(e.clone())); - return; - } - // Unreadable metadata only loses the layer names and credits. - if let Ok(m) = m { - let m = info.internal.decode(m).unwrap_or_default(); - info.set_metadata(serde_json::from_slice(&m).unwrap_or(Value::Null)); - } - finish(&a2, Ok(info)); + let result = m.and_then(|m| metadata(&mut info, m, meta_len)).map(|()| info); + finish(&a2, result, epoch); }), ); } @@ -515,16 +560,18 @@ fn load(archive: String) { } /// Settles archive waiters with live results, forgetting dropped or retired reads. -fn finish(archive: &str, r: Result) { - let r = crate::cache::check_still(&crate::cache::current_still()).and(r); +fn finish(archive: &str, r: Result, epoch: u64) { + let still = crate::cache::current_still(); + let r = crate::cache::check_still(&still).and(r); let r = crate::logins::check(archive) .map_err(str::to_string) .and(r) .map(Arc::new) .map_err(|e| if rustgis_data::login_address::is_wrapper(archive) { crate::logins::scrub_error(&e) } else { e }); - let transient = r.as_ref().err().is_some_and(|e| !crate::cache::keep_failure(e, &crate::cache::current_still())); - let waiting = { + let transient = r.as_ref().err().is_some_and(|e| !crate::cache::keep_failure(e, &still)); + let (waiting, retry_at) = crate::cache::if_archive_current(archive, epoch, || { let mut all = archives().lock().unwrap_or_else(|e| e.into_inner()); + let mut retry_at = None; let old = match &r { Ok(i) => all.insert(archive.to_string(), State::Ready(i.clone())), // A dropped or retired owner's read isn't the archive's fault: the next caller @@ -533,15 +580,21 @@ fn finish(archive: &str, r: Result) { Err(e) => { // Read again once its wait is over: the map draws again then (a 429 or 503 says // when). - crate::cache::notify_at(Instant::now() + crate::cache::retry_wait(e, RETRY)); - all.insert(archive.to_string(), State::Failed(e.clone(), Instant::now())) + let at = Instant::now(); + retry_at = Some(at + crate::cache::retry_wait(e, RETRY)); + all.insert(archive.to_string(), State::Failed(e.clone(), at)) } }; - match old { + let waiting = match old { Some(State::Loading(w)) => w, _ => Vec::new(), - } - }; + }; + (waiting, retry_at) + }) + .unwrap_or_default(); + if let Some(at) = retry_at { + crate::cache::notify_at(at); + } for w in waiting { w(r.clone()); } @@ -551,16 +604,25 @@ fn finish(archive: &str, r: Result) { /// Fetches a tile address (`pmtiles:z/x/y@archive`): its bytes, or none for a tile the archive /// doesn't have. pub(crate) fn fetch(url: &str, done: Done) { + if let Err(e) = crate::cache::check_still(&crate::cache::current_still()) { + done(Err(e)); + return; + } let Some((z, x, y, archive)) = parse_url(url) else { done(Err(format!("{url}: not a PMTiles tile address"))); return; }; let a = archive.clone(); + let epoch = crate::cache::archive_epoch(&archive); with_info( &archive, Box::new(move |info| match info { Err(e) => done(Err(e)), Ok(info) => { + if crate::cache::archive_epoch(&a) != epoch { + done(Err(crate::cache::DROPPED.into())); + return; + } // Outside the archive's zooms, or not a tile of its zoom: none. let n = 1u64 << z; if z < info.min_zoom || z > info.max_zoom || u64::from(x) >= n || u64::from(y) >= n { @@ -577,6 +639,15 @@ pub(crate) fn fetch(url: &str, done: Done) { /// Looks `id` up in a directory, reading leaf directories as needed, then reads the tile. fn find(archive: String, info: Arc, id: u64, dir: Arc>, depth: usize, done: Done) { + if let Err(e) = crate::cache::check_still(&crate::cache::current_still()) { + done(Err(e)); + return; + } + let epoch = info.epoch; + if crate::cache::archive_epoch(&archive) != epoch { + done(Err(crate::cache::DROPPED.into())); + return; + } let Some(e) = lookup(&dir, id) else { done(Ok(Vec::new())); return; @@ -613,18 +684,17 @@ fn find(archive: String, info: Arc, id: u64, dir: Arc>, depth: }; let next = caller_callback(next); // One read per leaf: tiles needing a leaf already on its way wait for it. - { + let admitted = crate::cache::if_archive_current(&archive, epoch, || { let mut l = leaves().lock().unwrap_or_else(|e| e.into_inner()); match l.get_mut(&key) { Some(Leaf::Ready(leaf)) => { let leaf = leaf.clone(); drop(l); - next(Ok(leaf)); - return; + return (Some((next, leaf)), false); } Some(Leaf::Loading(waiting)) => { waiting.push(next); - return; + return (None, false); } None => { // Full: one ready leaf makes room (the cache is a working set, not a history). @@ -636,6 +706,16 @@ fn find(archive: String, info: Arc, id: u64, dir: Arc>, depth: l.insert(key.clone(), Leaf::Loading(vec![next])); } } + (None, true) + }); + match admitted { + None => return, + Some((Some((next, leaf)), _)) => { + next(Ok(leaf)); + return; + } + Some((None, false)) => return, + Some((None, true)) => {} } read( &archive, @@ -643,26 +723,30 @@ fn find(archive: String, info: Arc, id: u64, dir: Arc>, depth: e.length, Box::new(move |b| { let leaf = b.and_then(|b| info.internal.decode(b)).and_then(|b| parse_dir(&b)).map(Arc::new); - let leaf = crate::cache::check_still(&crate::cache::current_still()).and(leaf); - let waiting = { - let mut l = leaves().lock().unwrap_or_else(|e| e.into_inner()); - let w = match &leaf { - Ok(leaf) => l.insert(key.clone(), Leaf::Ready(leaf.clone())), - // A failed leaf is read again by the next tile that needs it. - Err(_) => l.remove(&key), - }; - match w { - Some(Leaf::Loading(w)) => w, - _ => Vec::new(), - } - }; - for w in waiting { - w(leaf.clone()); - } + finish_leaf(key, leaf, epoch); }), ); } +fn finish_leaf(key: (String, u64), leaf: Result>, String>, epoch: u64) { + let leaf = crate::cache::check_still(&crate::cache::current_still()).and(leaf); + let waiting = crate::cache::if_archive_current(&key.0, epoch, || { + let mut all = leaves().lock().unwrap_or_else(|e| e.into_inner()); + let old = match &leaf { + Ok(leaf) => all.insert(key.clone(), Leaf::Ready(leaf.clone())), + Err(_) => all.remove(&key), + }; + match old { + Some(Leaf::Loading(waiting)) => waiting, + _ => Vec::new(), + } + }) + .unwrap_or_default(); + for callback in waiting { + callback(leaf.clone()); + } +} + /// The entry holding `id`: the last at or before it, when it is a leaf directory or its run /// covers `id`. fn lookup(dir: &[Entry], id: u64) -> Option { @@ -684,6 +768,10 @@ fn parse_url(url: &str) -> Option<(u8, u32, u32, String)> { /// Reads `len` bytes at `at` of an archive. fn read(archive: &str, at: u64, len: u64, done: Done) { + if let Err(e) = crate::cache::check_still(&crate::cache::current_still()) { + done(Err(e)); + return; + } if len == 0 { done(Ok(Vec::new())); return; @@ -718,6 +806,174 @@ pub(crate) fn read_file(archive: &str, at: u64, len: u64) -> Result, Str Ok(buf) } +fn metadata(info: &mut Info, bytes: Vec, expected: u64) -> Result<(), String> { + if bytes.len() as u64 != expected { + return Err("truncated PMTiles metadata".into()); + } + let bytes = info.internal.decode(bytes)?; + let value: Value = serde_json::from_slice(&bytes).map_err(|e| format!("PMTiles metadata: {e}"))?; + if !value.is_object() { + return Err("PMTiles metadata must be a JSON object".into()); + } + info.set_metadata(value); + Ok(()) +} + +/// Reads the current local file without consulting or changing retained caches. All reachable +/// directories and nonoverlapping tile runs are validated within fixed aggregate budgets; +/// cancellation is checked between entries, while tile payloads remain lazy. +/// Ranges are bounded and freshly opened; failure leaves the displayed archive intact. +#[cfg(not(target_arch = "wasm32"))] +pub fn read_info_local(archive: &str) -> Result, String> { + read_info_local_bounded(archive, INDEX_BUDGET) +} + +#[cfg(not(target_arch = "wasm32"))] +fn read_info_local_bounded(archive: &str, mut budget: IndexBudget) -> Result, String> { + check_local(archive)?; + let head = read_file(archive, 0, HEAD)?; + let path = archive.strip_prefix("file://").unwrap_or(archive); + let file_len = std::fs::metadata(path).map_err(|e| format!("{path}: {e}"))?.len(); + if head.len() < 127 { + return Err("a truncated PMTiles header".into()); + } + let (mut info, at, len, root_decoded) = parse_head_detail(&head)?; + for (offset, length) in [(8, 16), (24, 32), (40, 48), (56, 64)] { + let start = u64_at(&head, offset).ok_or("a truncated PMTiles header")?; + let size = u64_at(&head, length).ok_or("a truncated PMTiles header")?; + if start.checked_add(size).is_none_or(|end| end > file_len) { + return Err("a PMTiles section outside the local file".into()); + } + } + if len != 0 { + metadata(&mut info, read_file(archive, at, len)?, len)?; + } + budget.take(u64_at(&head, 16).ok_or("a truncated PMTiles header")?, root_decoded, info.root.len())?; + validate_directories(archive, &info, &info.root, 0, &mut budget, &mut HashMap::new())?; + Ok(Arc::new(info)) +} + +#[cfg(target_arch = "wasm32")] +pub fn read_info_local(archive: &str) -> Result, String> { + Err(format!("{archive}: the web app cannot reload local PMTiles paths")) +} + +/// Remaining aggregate work. The decoder also independently caps any one directory at 64 MiB; +/// an over-budget decode is discarded before parsing or following its references. +#[cfg(not(target_arch = "wasm32"))] +#[derive(Clone, Copy)] +struct IndexBudget { + stored: u64, + decoded: u64, + entries: usize, + directories: usize, +} + +#[cfg(not(target_arch = "wasm32"))] +impl IndexBudget { + fn take(&mut self, stored: u64, decoded: u64, entries: usize) -> Result<(), String> { + let error = "PMTiles index exceeds validation budget"; + self.stored = self.stored.checked_sub(stored).ok_or(error)?; + self.decoded = self.decoded.checked_sub(decoded).ok_or(error)?; + self.entries = self.entries.checked_sub(entries).ok_or(error)?; + self.directories = self.directories.checked_sub(1).ok_or(error)?; + Ok(()) + } +} + +/// A completed directory's height and descendant tile bounds; empty subtrees have no minimum. +/// Recheck these on every shared reference: its parent may impose a tighter depth or tile boundary. +#[cfg(not(target_arch = "wasm32"))] +#[derive(Clone, Copy, Debug, Default)] +struct DirectorySummary { + height: usize, + tile_start: Option, + tile_end: u64, +} + +/// An unfinished range encountered again is a cycle; completed ranges cost no further I/O or budget. +#[cfg(not(target_arch = "wasm32"))] +fn validate_directories( + archive: &str, + info: &Info, + dir: &[Entry], + depth: usize, + budget: &mut IndexBudget, + seen: &mut HashMap<(u64, u64), Option>, +) -> Result { + let mut summary = DirectorySummary::default(); + for (i, e) in dir.iter().enumerate() { + rustgis_data::progress::check().map_err(|e| e.to_string())?; + let end = e.offset.checked_add(e.length).ok_or("a PMTiles entry range overflows")?; + if e.run_length > 0 { + let tile_end = e.tile_id.checked_add(e.run_length).ok_or("tile ids overflow")?; + if dir.get(i + 1).is_some_and(|next| tile_end > next.tile_id) { + return Err("overlapping PMTiles tile runs".into()); + } + if end > info.data_length || info.data_offset.checked_add(end).is_none() { + return Err("a tile outside the archive".into()); + } + if e.length > MAX_BYTES as u64 { + return Err(format!("{} bytes is too large to read from an archive", e.length)); + } + summary.tile_start = Some(summary.tile_start.map_or(e.tile_id, |start| start.min(e.tile_id))); + summary.tile_end = summary.tile_end.max(tile_end); + continue; + } + if depth >= MAX_DEPTH { + return Err("PMTiles directories nested too deep".into()); + } + if end > info.leaf_length { + return Err("a leaf directory outside the archive".into()); + } + let at = info.leaf_offset.checked_add(e.offset).ok_or("a leaf directory range overflows")?; + info.leaf_offset.checked_add(end).ok_or("a leaf directory range overflows")?; + if e.length > MAX_BYTES as u64 { + return Err(format!("{} bytes is too large to read from an archive", e.length)); + } + let key = (at, e.length); + let child_summary = if let Some(done) = seen.get(&key) { + done.ok_or("cyclic PMTiles directories")? + } else { + // Refuse oversized ranges and too many directories before issuing any I/O. + if budget.directories == 0 || e.length > budget.stored { + return Err("PMTiles index exceeds validation budget".into()); + } + seen.insert(key, None); + let bytes = read_file(archive, at, e.length)?; + if bytes.len() as u64 != e.length { + return Err("a truncated PMTiles leaf directory".into()); + } + let bytes = info.internal.decode(bytes)?; + if bytes.len() as u64 > budget.decoded { + return Err("PMTiles index exceeds validation budget".into()); + } + let child = parse_dir_bounded(&bytes, budget.entries)?; + budget.take(e.length, bytes.len() as u64, child.len())?; + drop(bytes); + let child_summary = validate_directories(archive, info, &child, depth + 1, budget, seen)?; + seen.insert(key, Some(child_summary)); + child_summary + }; + if child_summary.tile_start.is_some_and(|start| start < e.tile_id) { + return Err("a PMTiles leaf directory contains tiles below its pointer".into()); + } + if dir.get(i + 1).is_some_and(|next| child_summary.tile_end > next.tile_id) { + return Err("overlapping PMTiles tile runs".into()); + } + let subtree = child_summary.height + 1; + if depth + subtree > MAX_DEPTH { + return Err("PMTiles directories nested too deep".into()); + } + summary.height = summary.height.max(subtree); + if let Some(start) = child_summary.tile_start { + summary.tile_start = Some(summary.tile_start.map_or(start, |current| current.min(start))); + } + summary.tile_end = summary.tile_end.max(child_summary.tile_end); + } + Ok(summary) +} + fn u64_at(b: &[u8], i: usize) -> Option { Some(u64::from_le_bytes(b.get(i..i + 8)?.try_into().ok()?)) } @@ -728,6 +984,10 @@ fn i32_at(b: &[u8], i: usize) -> Option { /// The header and root directory, and where the metadata is. fn parse_head(b: &[u8]) -> Result<(Info, u64, u64), String> { + parse_head_detail(b).map(|(info, at, len, _)| (info, at, len)) +} + +fn parse_head_detail(b: &[u8]) -> Result<(Info, u64, u64, u64), String> { if b.get(..7) != Some(b"PMTiles".as_slice()) { return Err("not a PMTiles archive".into()); } @@ -779,18 +1039,29 @@ fn parse_head(b: &[u8]) -> Result<(Info, u64, u64), String> { data_length, internal, tile, + epoch: 0, }; - Ok((info, meta_at, meta_len.min(MAX_BYTES as u64))) + if meta_len > MAX_BYTES as u64 || meta_at.checked_add(meta_len).is_none() { + return Err("PMTiles metadata is too large or has an invalid range".into()); + } + Ok((info, meta_at, meta_len, raw.len() as u64)) } /// A directory: delta-coded tile ids, then run lengths, lengths and offsets, all varints. fn parse_dir(b: &[u8]) -> Result, String> { + parse_dir_bounded(b, MAX_ENTRIES) +} + +fn parse_dir_bounded(b: &[u8], entry_budget: usize) -> Result, String> { let mut r = Varints { b, at: 0 }; let n = usize::try_from(r.next()?).map_err(|_| "a directory too large")?; // Each entry takes at least four bytes. if n > MAX_ENTRIES || n.saturating_mul(4) > b.len() { return Err("a directory with more entries than it can hold".into()); } + if n > entry_budget { + return Err("PMTiles index exceeds validation budget".into()); + } let mut entries = vec![Entry { tile_id: 0, offset: 0, length: 0, run_length: 0 }; n]; let mut last = 0u64; for e in entries.iter_mut() { @@ -809,7 +1080,7 @@ fn parse_dir(b: &[u8]) -> Result, String> { let e = entries.get_mut(i).ok_or("a short directory")?; e.offset = match (v, prev) { // Right after the previous entry. - (0, Some(p)) => p.offset.saturating_add(p.length), + (0, Some(p)) => p.offset.checked_add(p.length).ok_or("a directory offset overflows")?, (v, _) => v.saturating_sub(1), }; } @@ -827,6 +1098,9 @@ impl Varints<'_> { for shift in (0..64).step_by(7) { let byte = *self.b.get(self.at).ok_or("a truncated directory")?; self.at += 1; + if shift == 63 && byte > 1 { + return Err("a varint too long".into()); + } v |= u64::from(byte & 0x7f) << shift; if byte & 0x80 == 0 { return Ok(v); @@ -845,8 +1119,38 @@ fn gunzip(b: &[u8]) -> Result, String> { pub(crate) mod tests { use super::*; + #[test] + fn retired_context_cannot_start_or_join_archive_reads() { + let _turn = crate::cache::turn(); + let url = "https://retired-pmtiles-initiation.invalid/archive.pmtiles"; + let bytes = archive(&[(0, 0, 0, b"tile".to_vec())], 1, false, "{}"); + let (ready, _, _) = parse_head(&bytes).unwrap(); + let asked = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let sent = asked.clone(); + crate::cache::set_fetcher(move |_, done| { + sent.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + done(Err("unexpected range".into())); + }); + for state in [None, Some(State::Loading(Vec::new())), Some(State::Ready(Arc::new(ready)))] { + if let Some(state) = state { + archives().lock().unwrap().insert(url.into(), state); + } + crate::cache::with_still(Arc::new(|| false), || { + assert!(info(url).is_err()); + let (tx, rx) = std::sync::mpsc::channel(); + with_info(url, Box::new(move |result| tx.send(result).unwrap())); + assert!(rx.recv().unwrap().is_err()); + let (tx, rx) = std::sync::mpsc::channel(); + fetch(&format!("pmtiles:0/0/0@{url}"), Box::new(move |result| tx.send(result).unwrap())); + assert!(rx.recv().unwrap().is_err()); + }); + } + assert_eq!(asked.load(std::sync::atomic::Ordering::SeqCst), 0); + forget_archive(url); + } + /// Revoked metadata/directory reads and dropped metadata cannot become cached successes. - /// Ordinary metadata failures retain the fallback; a fresh read restores the real style. + /// Ordinary metadata failures remain failures; a fresh live read restores the real style. #[test] fn retired_metadata_does_not_become_ready_archive_info() { use std::sync::atomic::{AtomicBool, Ordering::SeqCst}; @@ -889,12 +1193,12 @@ pub(crate) mod tests { answer(asked, done); } } - let got = info(&url).unwrap().unwrap(); - if retry || error.is_none() { + if let (false, Some(error)) = (retry, error) { + assert_eq!(info(&url).unwrap_err(), error, "a live metadata error cannot become ready information"); + } else { + let got = info(&url).unwrap().unwrap(); assert_eq!(got.vector_layers(), vec!["roads"]); assert_eq!(got.attribution(), "Test"); - } else { - assert!(got.vector_layers().is_empty(), "a live metadata error keeps the existing fallback"); } assert!(held.lock().unwrap().is_empty()); } @@ -989,6 +1293,428 @@ pub(crate) mod tests { assert!(failures.is_empty(), "{failures:?}"); } + #[test] + fn reloaded_archive_rejects_old_metadata_and_leaf_answers() { + let _turn = crate::cache::turn(); + let source = "/pmtiles-stale-completion-regression.pmtiles"; + let old_epoch = crate::cache::archive_epoch(source); + let bytes = archive(&[(0, 0, 0, vec![1])], 1, true, "{}"); + let (old_info, _, _) = parse_head(&bytes).unwrap(); + crate::cache::reload_archive(source); + let (tx, rx) = std::sync::mpsc::channel(); + archives() + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert(source.into(), State::Loading(vec![Box::new(move |result| tx.send(result.map(|info| info.attribution())).unwrap())])); + let key = (source.to_string(), HEAD); + let (leaf_tx, leaf_rx) = std::sync::mpsc::channel(); + leaves() + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert(key.clone(), Leaf::Loading(vec![Box::new(move |result| leaf_tx.send(result.map(|leaf| leaf.len())).unwrap())])); + finish(source, Ok(old_info), old_epoch); + finish_leaf(key.clone(), Ok(Arc::new(Vec::new())), old_epoch); + assert!(matches!(rx.try_recv(), Err(std::sync::mpsc::TryRecvError::Empty))); + assert!(matches!(leaf_rx.try_recv(), Err(std::sync::mpsc::TryRecvError::Empty))); + // Global clears retain pending reads, so this regression is independent of other tests. + assert!(matches!(archives().lock().unwrap_or_else(|e| e.into_inner()).get(source), Some(State::Loading(_)))); + assert!(matches!(leaves().lock().unwrap_or_else(|e| e.into_inner()).get(&key), Some(Leaf::Loading(_)))); + let (mut current, _, _) = parse_head(&bytes).unwrap(); + current.set_metadata(serde_json::json!({"attribution": "fresh credits"})); + let epoch = crate::cache::archive_epoch(source); + finish(source, Ok(current), epoch); + finish_leaf(key, Ok(Arc::new(vec![Entry { tile_id: 0, offset: 0, length: 1, run_length: 1 }])), epoch); + assert_eq!(rx.recv().unwrap().unwrap(), "fresh credits"); + assert_eq!(leaf_rx.recv().unwrap().unwrap(), 1); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn fresh_local_validation_rejects_bad_metadata_and_truncated_sections() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-validation-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + let valid = archive(&[(0, 0, 0, vec![1, 2, 3])], 1, true, r#"{"attribution":"fresh"}"#); + std::fs::write(&path, &valid).unwrap(); + assert_eq!(read_info_local(source).unwrap().attribution(), "fresh"); + for metadata in ["{", "null", "[]"] { + std::fs::write(&path, archive(&[(0, 0, 0, vec![1])], 1, false, metadata)).unwrap(); + assert!(read_info_local(source).is_err(), "{metadata}"); + } + std::fs::write(&path, &valid[..valid.len() - 1]).unwrap(); + assert!(read_info_local(source).is_err()); + let mut truncated_metadata = valid.clone(); + truncated_metadata[24..32].copy_from_slice(&(valid.len() as u64 - 1).to_le_bytes()); + std::fs::write(&path, truncated_metadata).unwrap(); + assert!(read_info_local(source).is_err()); + std::fs::remove_file(path).unwrap(); + } + + #[cfg(not(target_arch = "wasm32"))] + fn indexed_archive(root: &[Entry], leaves: Vec) -> Vec { + let mut head = archive(&[(0, 0, 0, vec![1, 2, 3])], 1, false, "{}")[..127].to_vec(); + let root = write_dir(root); + let meta_at = 127 + root.len() as u64; + let leaf_at = meta_at + 2; + let data_at = leaf_at + leaves.len() as u64; + for (at, value) in [(16, root.len() as u64), (24, meta_at), (40, leaf_at), (48, leaves.len() as u64), (56, data_at)] { + head[at..at + 8].copy_from_slice(&value.to_le_bytes()); + } + [head, root, b"{}".to_vec(), leaves, vec![1, 2, 3]].concat() + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn fresh_local_validation_checks_every_leaf_and_deduplicates_shared_ranges() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-leaves-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + let mut bad = archive(&[(0, 0, 0, vec![1, 2, 3])], 1, true, "{}"); + bad[138] = 0x80; + std::fs::write(&path, bad).unwrap(); + assert_eq!(read_info_local(source).unwrap_err(), "a truncated directory"); + + let first = write_dir(&[]); + let second = write_dir(&[Entry { tile_id: 1, offset: 0, length: 3, run_length: 1 }]); + let root = [ + Entry { tile_id: 0, offset: 0, length: first.len() as u64, run_length: 0 }, + Entry { tile_id: 1, offset: first.len() as u64, length: second.len() as u64, run_length: 0 }, + Entry { tile_id: 2, offset: 0, length: first.len() as u64, run_length: 0 }, + ]; + let leaves = [first.clone(), second].concat(); + let valid = indexed_archive(&root, leaves.clone()); + std::fs::write(&path, &valid).unwrap(); + assert!(read_info_local(source).is_ok()); + // Exactly enough for the root and two unique leaves; the repeated range costs no work. + let bytes = (write_dir(&root).len() + leaves.len()) as u64; + let exact = IndexBudget { stored: bytes, decoded: bytes, entries: 4, directories: 3 }; + assert!(read_info_local_bounded(source, exact).is_ok()); + for budget in [ + IndexBudget { stored: bytes - 1, ..exact }, + IndexBudget { decoded: bytes - 1, ..exact }, + IndexBudget { entries: 3, ..exact }, + IndexBudget { directories: 2, ..exact }, + ] { + assert_eq!(read_info_local_bounded(source, budget).unwrap_err(), "PMTiles index exceeds validation budget"); + } + + let mut later_bad = valid; + let leaf_at = u64_at(&later_bad, 40).unwrap() as usize; + later_bad[leaf_at + leaves.len() - 1] = 0x80; + std::fs::write(&path, later_bad).unwrap(); + assert_eq!(read_info_local(source).unwrap_err(), "a truncated directory"); + for entry in [ + Entry { tile_id: 0, offset: u64::MAX - 1, length: 2, run_length: 0 }, + Entry { tile_id: 0, offset: 0, length: leaves.len() as u64 + 1, run_length: 0 }, + ] { + std::fs::write(&path, indexed_archive(&[entry], leaves.clone())).unwrap(); + let error = read_info_local(source).unwrap_err(); + assert!(error.contains("overflows") || error == "a leaf directory outside the archive", "{error}"); + } + // Oversized leaf reads fail before opening a file, without a huge fixture. + let (mut info, _, _) = parse_head(&archive(&[(0, 0, 0, vec![1])], 1, false, "{}")).unwrap(); + info.leaf_length = MAX_BYTES as u64 + 1; + let entry = Entry { tile_id: 0, offset: 0, length: info.leaf_length, run_length: 0 }; + let mut budget = INDEX_BUDGET; + assert_eq!( + validate_directories("/nonexistent.pmtiles", &info, &[entry], 0, &mut budget, &mut HashMap::new()).unwrap_err(), + format!("{} bytes is too large to read from an archive", entry.length) + ); + // A declared compression on lazy tile payloads is not decoded during validation. + let mut lazy = archive(&[(0, 0, 0, vec![1, 2, 3])], 1, true, "{}"); + lazy[98] = 2; + std::fs::write(&path, lazy).unwrap(); + assert!(read_info_local(source).is_ok()); + std::fs::remove_file(path).unwrap(); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn fresh_local_validation_checks_run_boundaries_in_roots_and_reachable_leaves() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-run-boundaries-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + for nested in [false, true] { + for next_is_leaf in [false, true] { + // [0, 2) overlaps id 1, is adjacent to id 2, and leaves a gap before id 4. + for next_id in [1, 2, 4] { + let tile = Entry { tile_id: next_id, offset: 0, length: 3, run_length: 1 }; + let mut leaves = if next_is_leaf { write_dir(&[tile]) } else { Vec::new() }; + let next = if next_is_leaf { Entry { length: leaves.len() as u64, run_length: 0, ..tile } } else { tile }; + let entries = [Entry { tile_id: 0, offset: 0, length: 3, run_length: 2 }, next]; + let bytes = if nested { + let directory = write_dir(&entries); + let root = Entry { tile_id: 0, offset: leaves.len() as u64, length: directory.len() as u64, run_length: 0 }; + leaves.extend(directory); + indexed_archive(&[root], leaves) + } else { + indexed_archive(&entries, leaves) + }; + std::fs::write(&path, bytes).unwrap(); + let result = read_info_local(source); + if next_id == 1 { + assert_eq!(result.unwrap_err(), "overlapping PMTiles tile runs", "nested={nested}, next_is_leaf={next_is_leaf}"); + } else { + assert!(result.is_ok(), "nested={nested}, next_is_leaf={next_is_leaf}, next_id={next_id}: {result:?}"); + } + } + } + } + // The final run has no following entry, but its exclusive end must still not overflow. + let entries = [Entry { tile_id: u64::MAX, offset: 0, length: 3, run_length: 1 }]; + std::fs::write(&path, indexed_archive(&entries, Vec::new())).unwrap(); + assert_eq!(read_info_local(source).unwrap_err(), "tile ids overflow"); + std::fs::remove_file(path).unwrap(); + } + + /// Two root pointers; the first reaches [0, 3), optionally through an intermediate directory, + /// while the second reaches an empty directory. `next_id = 2` hides tile 2 from lookup. + #[cfg(not(target_arch = "wasm32"))] + fn ancestor_boundary_archive(nested: bool, next_id: u64) -> Vec { + let mut leaves = write_dir(&[Entry { tile_id: 0, offset: 0, length: 3, run_length: 3 }]); + let mut first = Entry { tile_id: 0, offset: 0, length: leaves.len() as u64, run_length: 0 }; + if nested { + let directory = write_dir(&[first]); + first.offset = leaves.len() as u64; + first.length = directory.len() as u64; + leaves.extend(directory); + } + let empty = write_dir(&[]); + let next = Entry { tile_id: next_id, offset: leaves.len() as u64, length: empty.len() as u64, run_length: 0 }; + leaves.extend(empty); + indexed_archive(&[first, next], leaves) + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn fresh_local_validation_checks_descendant_runs_against_ancestor_boundaries() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-ancestor-boundaries-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + for nested in [false, true] { + // [0, 3) crosses a boundary at 2, ends exactly at 3, and leaves a gap before 5. + for next_id in [2, 3, 5] { + std::fs::write(&path, ancestor_boundary_archive(nested, next_id)).unwrap(); + let result = read_info_local(source); + if next_id == 2 { + assert_eq!(result.unwrap_err(), "overlapping PMTiles tile runs", "nested={nested}"); + } else { + assert!(result.is_ok(), "nested={nested}, next_id={next_id}: {result:?}"); + } + } + } + std::fs::remove_file(path).unwrap(); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn fresh_local_validation_checks_descendant_tiles_against_pointer_lower_bounds() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-pointer-lower-bounds-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + for nested in [false, true] { + for tile_id in [0, 5] { + for pointer_id in [0, 5, 6] { + let mut leaves = write_dir(&[Entry { tile_id, offset: 0, length: 3, run_length: 1 }]); + let mut pointer = Entry { tile_id: 0, offset: 0, length: leaves.len() as u64, run_length: 0 }; + if nested { + let directory = write_dir(&[pointer]); + pointer.offset = leaves.len() as u64; + pointer.length = directory.len() as u64; + leaves.extend(directory); + } + pointer.tile_id = pointer_id; + std::fs::write(&path, indexed_archive(&[pointer], leaves)).unwrap(); + let result = read_info_local(source); + if pointer_id > tile_id { + assert_eq!( + result.unwrap_err(), + "a PMTiles leaf directory contains tiles below its pointer", + "nested={nested}, pointer_id={pointer_id}, tile_id={tile_id}" + ); + } else { + assert!(result.is_ok(), "nested={nested}, pointer_id={pointer_id}, tile_id={tile_id}: {result:?}"); + } + } + } + } + std::fs::remove_file(path).unwrap(); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn memoized_directory_rechecks_pointer_lower_bounds_without_io_or_budget() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-memo-lower-bounds-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + let leaf = write_dir(&[Entry { tile_id: 5, offset: 0, length: 3, run_length: 1 }]); + let pointer = Entry { tile_id: 0, offset: 0, length: leaf.len() as u64, run_length: 0 }; + let bytes = indexed_archive(&[pointer], leaf.clone()); + let (info, _, _) = parse_head(&bytes).unwrap(); + std::fs::write(&path, bytes).unwrap(); + let mut budget = IndexBudget { stored: leaf.len() as u64, decoded: leaf.len() as u64, entries: 1, directories: 1 }; + let mut seen = HashMap::new(); + validate_directories(source, &info, &[pointer], 0, &mut budget, &mut seen).unwrap(); + assert_eq!((budget.stored, budget.decoded, budget.entries, budget.directories), (0, 0, 0, 0)); + std::fs::remove_file(&path).unwrap(); + // The same completed subtree is legal below its first tile and at equality, but not above. + for tile_id in [0, 5, 6] { + let result = validate_directories(source, &info, &[Entry { tile_id, ..pointer }], 0, &mut budget, &mut seen); + if tile_id > 5 { + assert_eq!(result.unwrap_err(), "a PMTiles leaf directory contains tiles below its pointer"); + } else { + assert!(result.is_ok(), "pointer_id={tile_id}: {result:?}"); + } + assert_eq!((budget.stored, budget.decoded, budget.entries, budget.directories), (0, 0, 0, 0)); + } + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn memoized_directory_rechecks_each_parent_boundary_without_io_or_budget() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-memo-boundaries-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + let leaf = write_dir(&[Entry { tile_id: 0, offset: 0, length: 3, run_length: 3 }]); + let pointer = Entry { tile_id: 0, offset: 0, length: leaf.len() as u64, run_length: 0 }; + let bytes = indexed_archive(&[pointer], leaf.clone()); + let (info, _, _) = parse_head(&bytes).unwrap(); + std::fs::write(&path, bytes).unwrap(); + // Exactly one leaf's work is available; the initial visit consumes all of it. + let mut budget = IndexBudget { stored: leaf.len() as u64, decoded: leaf.len() as u64, entries: 1, directories: 1 }; + let mut seen = HashMap::new(); + let summary = validate_directories(source, &info, &[pointer], 0, &mut budget, &mut seen).unwrap(); + assert_eq!((summary.height, summary.tile_end), (1, 3)); + assert_eq!((budget.stored, budget.decoded, budget.entries, budget.directories), (0, 0, 0, 0)); + // Reused references must neither reopen the removed file nor charge the exhausted budget. + std::fs::remove_file(&path).unwrap(); + for next_id in [3, 5, 2] { + let parent = [pointer, Entry { tile_id: next_id, offset: 0, length: 3, run_length: 1 }]; + let result = validate_directories(source, &info, &parent, 0, &mut budget, &mut seen); + if next_id == 2 { + assert_eq!(result.unwrap_err(), "overlapping PMTiles tile runs"); + } else { + let summary = result.unwrap(); + assert_eq!((summary.height, summary.tile_end), (1, next_id + 1)); + } + assert_eq!((budget.stored, budget.decoded, budget.entries, budget.directories), (0, 0, 0, 0)); + } + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn fresh_local_validation_cancels_before_reading_a_later_malformed_leaf() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct CancelBeforeThirdEntry(AtomicUsize); + impl rustgis_data::progress::Probe for CancelBeforeThirdEntry { + fn cancelled(&self) -> bool { + self.0.fetch_add(1, Ordering::Relaxed) >= 2 + } + fn begin(&self, _: u64) {} + fn advance(&self, _: u64) {} + } + + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-cancel-traversal-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + let first = write_dir(&[Entry { tile_id: 0, offset: 0, length: 3, run_length: 1 }]); + let root = [ + Entry { tile_id: 0, offset: 0, length: first.len() as u64, run_length: 0 }, + Entry { tile_id: 1, offset: first.len() as u64, length: 1, run_length: 0 }, + ]; + let mut leaves = first; + leaves.push(0x80); + std::fs::write(&path, indexed_archive(&root, leaves)).unwrap(); + assert_eq!(read_info_local(source).unwrap_err(), "a truncated directory"); + let probe = Arc::new(CancelBeforeThirdEntry(AtomicUsize::new(0))); + { + let _installed = rustgis_data::progress::install(Some(probe.clone())); + assert_eq!(read_info_local(source).unwrap_err(), rustgis_data::DataError::Cancelled.to_string()); + } + // Root pointer, its reachable tile, then the next root pointer: stop before its I/O. + assert_eq!(probe.0.load(Ordering::Relaxed), 3); + assert_eq!(read_info_local(source).unwrap_err(), "a truncated directory"); + std::fs::remove_file(path).unwrap(); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn fresh_local_validation_rejects_cycles_and_excess_depth_even_when_shared() { + let path = std::env::temp_dir().join(format!("rustgis-pmtiles-depth-{}.pmtiles", std::process::id())); + let source = path.to_str().unwrap(); + let pointer = |offset| Entry { tile_id: 0, offset, length: 5, run_length: 0 }; + let tile = Entry { tile_id: 0, offset: 0, length: 3, run_length: 1 }; + std::fs::write(&path, indexed_archive(&[pointer(0)], write_dir(&[pointer(0)]))).unwrap(); + assert_eq!(read_info_local(source).unwrap_err(), "cyclic PMTiles directories"); + let mutual: Vec = [pointer(5), pointer(0)].iter().flat_map(|entry| write_dir(&[*entry])).collect(); + std::fs::write(&path, indexed_archive(&[pointer(0)], mutual)).unwrap(); + assert_eq!(read_info_local(source).unwrap_err(), "cyclic PMTiles directories"); + for count in [MAX_DEPTH, MAX_DEPTH + 1] { + let leaves: Vec = + (0..count).flat_map(|i| write_dir(&[if i + 1 == count { tile } else { pointer((i as u64 + 1) * 5) }])).collect(); + std::fs::write(&path, indexed_archive(&[pointer(0)], leaves)).unwrap(); + if count == MAX_DEPTH { + assert!(read_info_local(source).is_ok()); + } else { + assert_eq!(read_info_local(source).unwrap_err(), "PMTiles directories nested too deep"); + } + } + // Validate a shared subtree shallowly first, then reach it by an over-deep path. + let leaves: Vec = + [pointer(5), tile, pointer(15), pointer(20), pointer(0)].iter().flat_map(|entry| write_dir(&[*entry])).collect(); + let root = [pointer(0), Entry { tile_id: 1, ..pointer(10) }]; + std::fs::write(&path, indexed_archive(&root, leaves)).unwrap(); + assert_eq!(read_info_local(source).unwrap_err(), "PMTiles directories nested too deep"); + std::fs::remove_file(path).unwrap(); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn ordinary_loaders_reject_present_invalid_metadata_with_specific_errors() { + let _turn = crate::cache::turn(); + let dir = std::env::temp_dir().join(format!("rustgis-pmtiles-metadata-errors-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let mut cases = Vec::new(); + for (text, error) in [ + ("{", "PMTiles metadata:"), + ("null", "PMTiles metadata must be a JSON object"), + ("[]", "PMTiles metadata must be a JSON object"), + ("42", "PMTiles metadata must be a JSON object"), + ] { + cases.push((archive(&[(0, 0, 0, vec![1])], 1, false, text), error)); + } + let mut truncated = archive(&[(0, 0, 0, vec![1])], 1, false, "{}"); + truncated[32..40].copy_from_slice(&100u64.to_le_bytes()); + cases.push((truncated, "truncated PMTiles metadata")); + for (code, error) in [(2, "gzip: invalid header"), (3, "Brotli:"), (4, "Zstandard:")] { + let mut invalid = archive_with(&[(0, 0, 0, vec![1])], 1, false, "{}", code); + let at = u64_at(&invalid, 24).unwrap() as usize; + invalid[at] = 0xff; + invalid[32..40].copy_from_slice(&1u64.to_le_bytes()); + cases.push((invalid, error)); + } + let mut oversized = archive(&[(0, 0, 0, vec![1])], 1, false, "{}"); + oversized[32..40].copy_from_slice(&(MAX_BYTES as u64 + 1).to_le_bytes()); + // Store the whole oversized (otherwise valid JSON) section, not merely a header + // claiming an invalid range. The ordinary loader must reject it before reading it. + let metadata_at = u64_at(&oversized, 24).unwrap() as usize; + let metadata_end = metadata_at + MAX_BYTES + 1; + oversized.truncate(metadata_at + 2); + oversized.resize(metadata_end, b' '); + oversized.push(1); + oversized[56..64].copy_from_slice(&(metadata_end as u64).to_le_bytes()); + cases.push((oversized, "PMTiles metadata is too large or has an invalid range")); + for (i, (bytes, expected)) in cases.into_iter().enumerate() { + // Separate paths force both entry points through their ordinary loader, not a + // failure memoized by the preceding assertion. + for loader in ["info", "get"] { + let path = dir.join(format!("{i}-{loader}.pmtiles")); + std::fs::write(&path, &bytes).unwrap(); + let source = path.to_str().unwrap(); + let error = if loader == "info" { + info_blocking(source, std::time::Duration::from_secs(10)).unwrap_err() + } else { + get(&format!("pmtiles:0/0/0@{source}")).unwrap_err() + }; + assert!(error.starts_with(expected), "case {i}, {loader}: {error}"); + } + } + std::fs::remove_dir_all(dir).unwrap(); + } + #[test] fn tile_ids_follow_the_spec() { assert_eq!(tile_id(0, 0, 0), 0); @@ -1129,8 +1855,32 @@ pub(crate) mod tests { tile_type, gzipped_tiles: false, bounds: [-10.0, -5.0, 10.0, 5.0], - metadata: serde_json::json!({"vector_layers": [{"id": "roads"}]}), + metadata: serde_json::json!({"vector_layers": [{"id": "roads"}], "attribution": "Written credits"}), }; + for metadata in [Value::Null, serde_json::json!([]), serde_json::json!("credits"), serde_json::json!(1), serde_json::json!(true)] { + let invalid = NewArchive { metadata, ..new(TileType::Mvt) }; + assert_eq!(write(vec![(0, 0, 0, b"tile".to_vec())], &invalid).unwrap_err(), "PMTiles metadata must be a JSON object"); + } + { + let mut bounded = NewArchive { metadata: serde_json::json!({"attribution": ""}), ..new(TileType::Mvt) }; + let overhead = serde_json::to_vec(&bounded.metadata).unwrap().len(); + for len in [MAX_BYTES, MAX_BYTES + 1] { + bounded.metadata["attribution"] = Value::String("x".repeat(len - overhead)); + assert_eq!(serde_json::to_vec(&bounded.metadata).unwrap().len(), len); + let written = write(vec![(0, 0, 0, b"tile".to_vec())], &bounded); + if len == MAX_BYTES { + let bytes = written.unwrap(); + let (_, _, meta_len) = parse_head(&bytes).unwrap(); + assert_eq!(meta_len, MAX_BYTES as u64); + let p = dir.join("metadata-limit.pmtiles"); + std::fs::write(&p, &bytes).unwrap(); + #[cfg(not(target_arch = "wasm32"))] + assert_eq!(read_info_local(p.to_str().unwrap()).unwrap().metadata, bounded.metadata); + } else { + assert_eq!(written.unwrap_err(), "PMTiles metadata exceeds the 16 MiB limit"); + } + } + } // A few tiles, one given twice and two alike (stored once). let small = vec![(0u8, 0u32, 0u32, b"old".to_vec()), (1, 0, 0, b"same".to_vec()), (1, 0, 1, b"same".to_vec()), (0, 0, 0, b"zero".to_vec())]; @@ -1149,6 +1899,10 @@ pub(crate) mod tests { assert_eq!(get(&format!("pmtiles:1/0/0@{a}")).unwrap(), b"same"); assert_eq!(get(&format!("pmtiles:1/0/1@{a}")).unwrap(), b"same"); assert!(get(&format!("pmtiles:1/1/1@{a}")).unwrap().is_empty()); + #[cfg(not(target_arch = "wasm32"))] + assert_eq!(info_blocking(&a, std::time::Duration::from_secs(10)).unwrap().attribution(), "Written credits"); + #[cfg(not(target_arch = "wasm32"))] + assert_eq!(read_info_local(&a).unwrap().attribution(), "Written credits"); // Every tile to zoom 7, each different: too many for the root, so in leaves. let many: Vec<(u8, u32, u32, Vec)> = (0..=7u8) .flat_map(|z| (0..1u32 << z).flat_map(move |x| (0..1u32 << z).map(move |y| (z, x, y, format!("{z}/{x}/{y}").into_bytes())))) @@ -1180,6 +1934,8 @@ pub(crate) mod tests { let p = dir.join(format!("t{leaf}{code}.pmtiles")); std::fs::write(&p, archive_with(&tiles, 1, leaf, r#"{"vector_layers":[{"id":"roads"}],"attribution":"Test"}"#, code)).unwrap(); let a = p.display().to_string(); + #[cfg(not(target_arch = "wasm32"))] + assert_eq!(read_info_local(&a).unwrap().attribution(), "Test"); let mut i = info(&a).unwrap(); for _ in 0..200 { if i.is_some() { diff --git a/crates/tiles/src/source.rs b/crates/tiles/src/source.rs index a785679ef8..d73a67d463 100644 --- a/crates/tiles/src/source.rs +++ b/crates/tiles/src/source.rs @@ -55,8 +55,9 @@ pub struct Spec { impl Spec { fn key(&self) -> String { format!( - "{:?}|{}|{}|{}|{}|{}|{}|{}|{}|{}|{}", + "{:?}|{}:{}|{}|{}|{}|{}|{}|{}|{}|{}|{}", self.kind, + self.url.len(), self.url, self.subdomains.join(","), self.min_zoom, @@ -148,6 +149,13 @@ pub(crate) fn forget() { resolved().lock().unwrap_or_else(|e| e.into_inner()).clear(); } +/// Drops only styles resolved directly from this archive. +pub(crate) fn forget_archive(archive: &str) { + let pm = format!("{:?}|{}:{archive}|", Kind::PmTiles, archive.len()); + let mb = format!("{:?}|{}:{archive}|", Kind::MbTiles, archive.len()); + resolved().lock().unwrap_or_else(|e| e.into_inner()).retain(|key, _| !key.starts_with(&pm) && !key.starts_with(&mb)); +} + /// The built-in style for vector tiles without one: it knows the OpenMapTiles, Protomaps and /// Shortbread layer names, and draws water, land use, parks, buildings, roads, boundaries and /// place names from whichever of them the tiles have. @@ -159,6 +167,7 @@ pub fn resolve(spec: &Spec) -> Resolve { return Resolve::Failed(e.to_string()); } let key = spec.key(); + let epoch = cache::archive_epoch(&spec.url); // A PMTiles archive on this computer is a path, not a web address. let local = spec.kind == Kind::PmTiles && pmtiles::is_local(&spec.url); let checked = if spec.kind == Kind::MbTiles { @@ -175,7 +184,8 @@ pub fn resolve(spec: &Spec) -> Resolve { if let Some(r) = cached.as_ref().filter(|r| r.current()) { return Resolve::Ready(r.clone()); } - settle(resolve_anew(spec, key.clone()), cached, spec, key) + let anew = resolve_anew(spec, key.clone(), epoch); + cache::if_archive_current(&spec.url, epoch, || settle(anew, cached, spec, key)).unwrap_or(Resolve::Pending) } /// What resolving `spec` again (`anew`) gives, with `cached`, the expired one under `key`. @@ -199,7 +209,7 @@ fn settle(anew: Resolve, cached: Option>, spec: &Spec, key: String } /// Reads `spec`'s style and resolves its sources, keeping the result under `key`. -fn resolve_anew(spec: &Spec, key: String) -> Resolve { +fn resolve_anew(spec: &Spec, key: String, epoch: u64) -> Resolve { let mut base = address::Origin::named(&spec.url); let doc = match spec.kind { Kind::Raster => raster_style(spec), @@ -311,13 +321,16 @@ fn resolve_anew(spec: &Spec, key: String) -> Resolve { if let Err(e) = crate::logins::check(&spec.url) { return Resolve::Failed(e.to_string()); } - let mut all = resolved().lock().unwrap_or_else(|e| e.into_inner()); - // A few layers are in use at a time: past that many, start again rather than grow. - if all.len() >= MAX_RESOLVED { - all.clear(); - } - all.insert(key, r.clone()); - Resolve::Ready(r) + cache::if_archive_current(&spec.url, epoch, || { + let mut all = resolved().lock().unwrap_or_else(|e| e.into_inner()); + // A few layers are in use at a time: past that many, start again rather than grow. + if all.len() >= MAX_RESOLVED { + all.clear(); + } + all.insert(key, r.clone()); + Resolve::Ready(r) + }) + .unwrap_or(Resolve::Pending) } /// A style's sprites with their addresses resolved against `base`, those RustGIS can fetch, and diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index 836c1bbdc3..5dd978b9b9 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -679,7 +679,17 @@ pub fn shortcuts(app: &mut RustGisApp, ctx: &egui::Context) { /// Whether a UI command is available on the current platform. pub(crate) fn command_available(id: &str, web: bool) -> bool { - !web || !matches!(id, "ui.checkUpdate" | "ui.mcpRelay" | "ui.themeLoad" | "ui.themeReload" | "ui.reloadData" | "layer.reloadData") + !web || !matches!( + id, + "ui.checkUpdate" + | "ui.mcpRelay" + | "ui.themeLoad" + | "ui.themeReload" + | "ui.reloadData" + | "layer.reloadData" + | "map.reloadTiles" + | "map.repairTiles" + ) } /// The `ui.*` commands Command Search lists: (id, label, ribbon location). diff --git a/crates/ui-egui/src/layoutview.rs b/crates/ui-egui/src/layoutview.rs index 7b8a3ead88..6de19d22ce 100644 --- a/crates/ui-egui/src/layoutview.rs +++ b/crates/ui-egui/src/layoutview.rs @@ -655,8 +655,7 @@ pub fn show(app: &mut RustGisApp, ui: &mut Ui) { }); if (app.layout.key != Some(key) || due) && size[0] > 2 && size[1] > 2 { let info = page_info(app, li); - if let Some((pm, frames_pending)) = - rustgis_engine::render::layout::render_pending(&layout, &mut app.session.project.maps, dpi, &info) + if let Ok((pm, frames_pending)) = rustgis_engine::render::layout::render_pending(&layout, &mut app.session.project.maps, dpi, &info) { let img = egui::ColorImage::from_rgba_premultiplied([pm.width() as usize, pm.height() as usize], pm.data()); match &mut app.layout.texture { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index f6c119d1f6..031bfcd44b 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -2248,6 +2248,17 @@ impl RustGisApp { /// Runs a command by id: `ui.*` here, everything else in the engine. pub fn run(&mut self, id: &str, params: Value) -> Result { + #[cfg(not(target_arch = "wasm32"))] + if self.reads_off_frame { + let archive = match id { + "map.reloadTiles" => Some("map.reloadTiles"), + "map.repairTiles" => Some("map.repairTiles"), + _ => None, + }; + if let Some(command) = archive { + return readjob::run(self, command, params); + } + } self.run_with_prepared(id, params, AppDispatch::Direct) } @@ -5020,7 +5031,7 @@ impl RustGisApp { self.frame_ctx = Some(ctx.clone()); // Agents the user turned on last time: the relay starts with the app. connectui::resume(self); - if let Some(relay) = &self.relay { + if let Some(relay) = self.relay.clone() { // A relay started before the first frame has no wake yet; now there is a context // to ask for repaints with. if self.relay_wake_pending { @@ -5032,9 +5043,13 @@ impl RustGisApp { relay.set_screen(relayshot::map_shown(self, ctx)); // Only a request that changed the project ends a label move: a read-only one leaves it. let state = self.session.state(); - if relay.poll(&mut self.session) > 0 && self.session.state() != state { + relay.poll(&mut self.session); + if self.session.state() != state { self.map.cancel_label_drag(); } + if let Some(ahead) = relay.take_archive_read() { + readjob::admitted(self, ahead, relay); + } } relayshot::drive(self, ctx); // Session plugins' programs: what they asked, and what changed on the map for them. diff --git a/crates/ui-egui/src/mapgpu.rs b/crates/ui-egui/src/mapgpu.rs index e99bda52d8..4f67c209b2 100644 --- a/crates/ui-egui/src/mapgpu.rs +++ b/crates/ui-egui/src/mapgpu.rs @@ -1012,6 +1012,8 @@ pub(crate) struct SceneBuffers { pub tilt: Option, /// Its live raster tile layers, in order, placed for each frame shown ([`LiveFrame`]). pub live: Vec<(String, rustgis_engine::render::rustgis_tiles::Spec, f32)>, + /// The archive revision of the scene that owns the live layers' attribution. + live_archive_revision: u64, /// Every tile layer of the scene is live: a tile arriving needs no new scene. pub tiles_live: bool, /// The farthest a vertex lies from its origin (map units, [`gpu_scene::reach`]): zoomed in @@ -1036,6 +1038,15 @@ impl SceneBuffers { if self.live.is_empty() { return None; } + let revision = || { + use rustgis_engine::render::rustgis_tiles::{Kind, cache}; + cache::archive_revision( + self.live.iter().filter_map(|(_, spec, _)| matches!(spec.kind, Kind::PmTiles | Kind::MbTiles).then_some(spec.url.as_str())), + ) + }; + if revision() != self.live_archive_revision { + return Some((LiveFrame { camera, layers: Vec::new(), tile_errors: Vec::new() }, false, true)); + } let view = rustgis_engine::render::View { width: self.size[0], height: self.size[1], @@ -1062,6 +1073,9 @@ impl SceneBuffers { } }) .collect(); + if revision() != self.live_archive_revision { + return Some((LiveFrame { camera, layers: Vec::new(), tile_errors: Vec::new() }, false, true)); + } Some((LiveFrame { camera, layers, tile_errors }, pending, declined)) } } @@ -1151,6 +1165,7 @@ impl SceneBuffers { let mut has_images = false; let mut has_tiles = false; let mut live = Vec::new(); + let mut live_archive_revision = 0; let tilt = scene.tilt.as_ref(); let mut tile_textures = r.tile_textures.lock().unwrap_or_else(|e| e.into_inner()); let mut tile_meshes = r.tile_meshes.lock().unwrap_or_else(|e| e.into_inner()); @@ -1344,9 +1359,10 @@ impl SceneBuffers { draws.push(SceneDraw::Image { bind, _texture: texture }); has_images = true; } - DrawCmd::LiveTiles { name, spec, opacity } => { + DrawCmd::LiveTiles { name, spec, opacity, archive_revision } => { draws.push(SceneDraw::LiveTiles { n: live.len() }); live.push((name.clone(), spec.clone(), *opacity)); + live_archive_revision = live_archive_revision.max(*archive_revision); } DrawCmd::Tiles { origin, tiles } => { let mut vertices = Vec::with_capacity(tiles.len() * 6); @@ -1428,7 +1444,19 @@ impl SceneBuffers { let tiles_live = !live.is_empty() && !has_images && !has_tiles && !scene.cmds.iter().any(|c| matches!(c, DrawCmd::TileMeshes { .. })); let reach = gpu_scene::reach(&scene.cmds); - Some(SceneBuffers { draws, clear: scene.clear, size, ppp, has_images, has_tiles, tilt: scene.tilt, live, tiles_live, reach }) + Some(SceneBuffers { + draws, + clear: scene.clear, + size, + ppp, + has_images, + has_tiles, + tilt: scene.tilt, + live, + tiles_live, + reach, + live_archive_revision, + }) } /// Rewrites every command's camera uniforms for a moved camera — pan/zoom without @@ -1691,6 +1719,32 @@ mod tests { .ok() } + #[test] + fn live_archive_pixels_cannot_outlive_their_scenes_credits() { + use rustgis_engine::{ + map::{TileKind, TileLayer}, + render, + }; + let layer = TileLayer::new("Archive", TileKind::PmTiles, "/live-scene-revision.pmtiles"); + let revision = render::rustgis_tiles::cache::archive_revision([layer.url.as_str()]); + let scene = SceneBuffers { + draws: Vec::new(), + clear: [0.0; 4], + size: [128.0, 128.0], + ppp: 1.0, + has_images: false, + has_tiles: false, + tilt: None, + live: vec![(layer.name.clone(), render::tile_spec(&layer), 1.0)], + tiles_live: true, + reach: 0.0, + live_archive_revision: revision, + }; + render::rustgis_tiles::cache::reload_archive(&layer.url); + let (frame, pending, declined) = scene.live_frame(Camera::world()).unwrap(); + assert!(declined && !pending && frame.layers.is_empty(), "redraw the scene instead of placing new pixels under its old credits"); + } + #[test] fn commands_over_device_limits_are_declined() { let limits = wgpu::Limits { max_buffer_size: 1024, max_texture_dimension_2d: 2048, ..Default::default() }; diff --git a/crates/ui-egui/src/mapview.rs b/crates/ui-egui/src/mapview.rs index 14f4db2693..ec39c24405 100644 --- a/crates/ui-egui/src/mapview.rs +++ b/crates/ui-egui/src/mapview.rs @@ -570,6 +570,17 @@ impl MapView { } } + #[cfg(test)] + pub(crate) fn label_dragging_for_tests(&self) -> bool { + self.moving_label.is_some() + } + + #[cfg(test)] + pub(crate) fn label_position_for_tests(&self, text: &str) -> Option { + let label = self.labels.iter().find(|label| label.text == text)?; + Some(self.view_rect?.0.min + vec2(label.x, label.y)) + } + /// Cancels a vertex preview and its pending release together, leaving other map gestures /// alone. The chosen node is cleared because its row/address may now belong to another shape. pub(crate) fn cancel_vertex_drag(&mut self) { @@ -957,12 +968,13 @@ fn main_view(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens) { if let Some((k, done)) = app.map.tilt_settle.take() { app.session.map_mut().adopt_caches(&done.map); // A frame of this view that couldn't be drawn isn't tried again: nothing is to fill. - if same_view(k, key) && done.rendered.is_none() { + if same_view(k, key) && done.rendered.is_none() && done.archives.current([&done.map]) { app.map.filling = false; } // One drawn before tiles or data arrived still shows; the next follows. So does one // of a change made since, for a view whose frame on screen is older still. if (same_view(k, key) || (tiled_flat && shows_finished(k, key, app.map.key))) + && done.archives.current([&done.map]) && let Some(r) = done.rendered { // With tile pictures still to draw, a draft: the next frame draws more. @@ -4424,6 +4436,25 @@ mod tile_redraw_tests { assert!(!keeps_frame(Some(key(3)), key(3), 40)); } + #[test] + fn an_eligible_finished_frame_cannot_cross_its_archive_reload() { + use rustgis_engine::{ + map::{Map, TileKind, TileLayer}, + render, + }; + let mut map = Map::new("Archive publication"); + map.tiles.push(TileLayer::new("Target", TileKind::PmTiles, "/finished-frame-revision.pmtiles")); + let archives = render::archive_revisions([&map]); + let (shown, done, now) = (Key { rev: 4, ..key(0) }, Key { rev: 6, ..key(0) }, Key { rev: 9, ..key(0) }); + let eligible = || shows_finished(done, now, Some(shown)) && archives.current([&map]); + assert!(eligible(), "ordinary progressive finished frames still show"); + render::rustgis_tiles::cache::reload_archive("/finished-frame-unrelated.mbtiles"); + assert!(eligible(), "another source does not revoke this worker's frame"); + render::rustgis_tiles::cache::reload_archive(&map.tiles[0].url); + assert!(shows_finished(done, now, Some(shown)), "the permissive revision fallback alone still accepts it"); + assert!(!eligible(), "archive coherence is also required at adoption"); + } + #[test] fn a_frame_of_an_earlier_change_shows_while_a_slider_is_dragged() { let (on_screen, done, now) = (Key { rev: 4, ..key(0) }, Key { rev: 6, ..key(0) }, Key { rev: 9, ..key(0) }); diff --git a/crates/ui-egui/src/panes.rs b/crates/ui-egui/src/panes.rs index 0301fcee6c..76e0d7f37c 100644 --- a/crates/ui-egui/src/panes.rs +++ b/crates/ui-egui/src/panes.rs @@ -1760,9 +1760,20 @@ fn tile_row(app: &mut RustGisApp, ui: &mut Ui, i: usize, moves: [bool; 2], reord action = Some(("ui.exportRaster", json!({"tiles": i}))); ui.close(); } - if tile.archive_file().is_some() && ui.button(tl!("Repair Data Source…")).clicked() { - action = Some(("ui.dialog", json!({"name": "repairTiles", "index": i}))); - ui.close(); + #[cfg(not(target_arch = "wasm32"))] + if tile.archive_file().is_some() { + if ui + .button(tl!("Reload Data")) + .on_hover_text(tl!("Manually reread this archive from disk without changing its presentation. Undo cannot restore overwritten file contents.")) + .clicked() + { + action = Some(("map.reloadTiles", json!({"index": i}))); + ui.close(); + } + if ui.button(tl!("Repair Data Source…")).clicked() { + action = Some(("ui.dialog", json!({"name": "repairTiles", "index": i}))); + ui.close(); + } } if ui.button(tl!("Remove")).clicked() { action = Some(("map.removeTiles", json!({"index": i}))); diff --git a/crates/ui-egui/src/readjob.rs b/crates/ui-egui/src/readjob.rs index 02c92803b8..bce1107b97 100644 --- a/crates/ui-egui/src/readjob.rs +++ b/crates/ui-egui/src/readjob.rs @@ -1,6 +1,6 @@ -//! Add Data's whole-file reads, layer reload, Load All Features and a model's tools on the desktop -//! run on a thread of their own (#1291 step 3), so a large raster, GeoJSON or table, a streamed -//! layer loaded whole, or a long model doesn't hold the window: the status bar shows the run under way, +//! Add Data's whole-file reads, layer reload, Load All Features, local archive validation and +//! model/script tools on the desktop run on a thread of their own (#1291 step 3), so large reads +//! and tools don't hold the window: the status bar shows the run under way, //! how far it has come and Cancel, and [`land`] then runs the command, which takes what was made //! ([`reads`]) and adds it as it does when it runs here. They go one at a time, in the order //! asked for: files dropped together wait for the one before, and so does a file added while one @@ -37,9 +37,13 @@ struct Asked { opened: u64, map: u64, layer: Option, + /// The exact archive and its presentation when asked, not a reusable list index. + archive: Option<(usize, rustgis_engine::map::TileLayer)>, reload: Option, /// Given what the command answered when it runs, or why it didn't (cancelled, or left). then: Option, + /// A direct relay archive call owns its real completion, rather than a Native callback. + relay: Option, } /// What a caller does once the command lands (ModelBuilder shows its run). @@ -58,10 +62,10 @@ struct ReadJob { slot: crate::Slot>, } -/// Runs `command` (`layer.addData`, `layer.loadAll`, `layer.reloadData`, `model.run` or `script.run`) with `p`: its +/// Runs Add Data, Load All, Reload Data, archive reload/repair or model/script tools with `p`: its /// read on a thread of its own when the desktop app makes it so ([`reads::read_ahead`]), after -/// any under way; here and now otherwise, as the command does. While one is under way or waiting -/// it answers that, not what the command adds, which comes as it lands. +/// any under way; here and now otherwise. While one is under way or waiting it answers that, not +/// what the command installs, which comes as it lands. pub fn run(app: &mut RustGisApp, command: &'static str, p: Value) -> Result { run_then(app, command, p, None) } @@ -107,8 +111,16 @@ pub fn run_then(app: &mut RustGisApp, command: &'static str, mut p: Value, then: { return Ok(json!({"reading": instance})); } + let archive = matches!(command, "map.reloadTiles" | "map.repairTiles").then(|| reads::archive_target(&app.session, &p)).flatten(); + let mut p = p; + if let Some((i, _)) = &archive + && let Some(params) = p.as_object_mut() + { + params.insert("index".into(), json!(i)); + } let what = match &layer { Some((_, name)) => name.clone(), + None if archive.is_some() => archive.as_ref().map(|(_, tile)| tile.name.clone()).unwrap_or_default(), // A model by its name (one drawn but not yet named has none: see `called`). None if command == "model.run" => p.get("name").and_then(Value::as_str).map(str::trim).unwrap_or_default().to_string(), None => p.get("path").and_then(Value::as_str).map(|path| file_name(path).to_string()).unwrap_or_default(), @@ -132,8 +144,10 @@ pub fn run_then(app: &mut RustGisApp, command: &'static str, mut p: Value, then: opened: app.session.opened, map: app.session.map().instance_id(), layer: layer.map(|(instance, _)| instance), + archive, reload, then, + relay: None, }; if busy(app) { app.reads.waiting.push_back(asked); @@ -149,6 +163,57 @@ pub fn run_then(app: &mut RustGisApp, command: &'static str, mut p: Value, then: } } +/// Queues an archive read already admitted by the relay, without dispatching another command. +pub fn admitted(app: &mut RustGisApp, ahead: reads::ReadAhead, relay: rustgis_assistant::relay::Relay) { + let Some((command, params)) = ahead.command() else { + finish_relay(app, &relay, Err(tl!("This read was already started").to_string())); + return; + }; + let params = params.clone(); + let archive = reads::archive_target(&app.session, ¶ms); + let what = archive.as_ref().map(|(_, tile)| tile.name.clone()).unwrap_or_default(); + let asked = Asked { + command, + params, + ahead: Some(ahead), + what, + opened: app.session.opened, + map: app.session.map().instance_id(), + layer: None, + archive, + reload: None, + then: None, + relay: Some(relay), + }; + if busy(app) { + app.reads.waiting.push_back(asked); + } else { + match placed(app, &asked) { + Ok(p) => { + let _ = start(app, asked, p); + } + Err(why) => not_run_asked(app, &asked, why.to_string()), + } + } +} + +fn not_run_asked(app: &mut RustGisApp, asked: &Asked, why: String) { + not_run(app, asked.then, why.clone()); + if let Some(relay) = &asked.relay { + relay.begin_archive_landing(&app.session); + finish_relay(app, relay, Err(why)); + } +} + +/// Resumed relay suffixes have the same interaction guard as calls run by `Relay::poll`. +fn finish_relay(app: &mut RustGisApp, relay: &rustgis_assistant::relay::Relay, result: Result) { + let state = app.session.state(); + relay.finish_archive_read(&mut app.session, result); + if app.session.state() != state { + app.map.cancel_label_drag(); + } +} + /// What the command answered, `r`, given to `then` first. fn answered(app: &mut RustGisApp, then: Option, r: Result) -> Result { if let Some(then) = then { @@ -167,7 +232,7 @@ fn not_run(app: &mut RustGisApp, then: Option, why: String) { /// `asked` left for `why`: its `then` told now, and kept in `left` for the notice. fn leave(app: &mut RustGisApp, left: &mut Vec<(String, Gone)>, asked: &Asked, why: Gone) { let what = called(asked.command, &asked.what).to_string(); - not_run(app, asked.then, crate::i18n::fmt("{what} was left: {why}", &[("what", &what), ("why", &why.to_string())])); + not_run_asked(app, asked, crate::i18n::fmt("{what} was left: {why}", &[("what", &what), ("why", &why.to_string())])); left.push((what, why)); } @@ -197,6 +262,7 @@ enum Gone { Project, Map, Layer, + Archive, Source, Data, } @@ -208,6 +274,7 @@ impl std::fmt::Display for Gone { Gone::Project => tl_raw!("another project was opened"), Gone::Map => tl_raw!("the map it was asked for isn't open"), Gone::Layer => tl_raw!("its layer was removed"), + Gone::Archive => tl_raw!("its tile archive changed or was removed"), Gone::Source => tl_raw!("its data source changed"), Gone::Data => tl_raw!("its data or pending edits changed; confirm Reload Data again"), }) @@ -223,6 +290,11 @@ fn placed(app: &RustGisApp, asked: &Asked) -> Result { if app.session.map().instance_id() != asked.map { return Err(Gone::Map); } + if let Some((i, original)) = &asked.archive + && app.session.map().tiles.get(*i) != Some(original) + { + return Err(Gone::Archive); + } let Some(instance) = asked.layer else { return Ok(asked.params.clone()) }; let l = app.session.map().layers.iter().find(|l| l.instance_id() == instance).ok_or(Gone::Layer)?; if asked.reload.as_ref().is_some_and(|reload| !reload.is_current(&app.session)) { @@ -263,6 +335,15 @@ fn start(app: &mut RustGisApp, mut asked: Asked, p: Value) -> Result { let notice_mark = app.session.notice_mark(); app.command_failed(asked.command, &error, notice_mark); - not_run(app, asked.then, error.clone()); + not_run_asked(app, &asked, error.clone()); return Err(error); } }; + if let Some(relay) = &asked.relay { + relay.begin_archive_landing(&app.session); + } let r = app.run_read(asked.command, p, read); + if asked.command == "map.repairTiles" + && let Ok(answer) = &r + && let Some(index) = answer.get("index").and_then(Value::as_u64).and_then(|i| usize::try_from(i).ok()) + && let Some(tile) = app.session.map().tiles.get(index) + { + let text = crate::i18n::fmt("Repaired {name}", &[("name", &tile.name)]); + app.session.notify(text, false); + } + if let Some(relay) = &asked.relay { + finish_relay(app, relay, r.clone()); + } answered(app, asked.then, r) } @@ -322,7 +417,7 @@ pub fn land(app: &mut RustGisApp) -> bool { crate::i18n::fmt("{tool} of {what} was cancelled", &[("tool", tl_raw!(job.name)), ("what", &job.asked.what)]) }; app.session.notify(said.clone(), false); - not_run(app, job.asked.then, said); + not_run_asked(app, &job.asked, said); let waiting = std::mem::take(&mut app.reads.waiting); let n = waiting.len(); let left_too = match n { @@ -333,7 +428,7 @@ pub fn land(app: &mut RustGisApp) -> bool { app.session.notify(left_too.clone(), false); } for asked in waiting { - not_run(app, asked.then, left_too.clone()); + not_run_asked(app, &asked, left_too.clone()); } } else { // The engine consumes the original owner before the callback sees its result. @@ -356,7 +451,7 @@ pub fn land(app: &mut RustGisApp) -> bool { } } // Those left, one notice for each reason, with a count, naming the first. - for why in [Gone::Project, Gone::Map, Gone::Layer, Gone::Source, Gone::Data] { + for why in [Gone::Project, Gone::Map, Gone::Layer, Gone::Archive, Gone::Source, Gone::Data] { let of: Vec<&String> = left.iter().filter(|(_, w)| *w == why).map(|(what, _)| what).collect(); let reason = why.to_string(); match of.as_slice() { @@ -375,13 +470,32 @@ pub fn land(app: &mut RustGisApp) -> bool { } /// Cancels the read under way, and what waits, and waits up to `wait` for its thread to stop. +/// Each one that never ran is told so, the one under way first and then those waiting in the +/// order asked: its `then`, and a relay's agent, whose batch is ended unrun rather than resumed. pub fn stop(app: &mut RustGisApp, wait: std::time::Duration) { - app.reads.waiting.clear(); - let Some(job) = app.reads.job.take() else { return }; - job.control.cancel(); - let end = std::time::Instant::now() + wait; - while job.slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner).is_none() && std::time::Instant::now() < end { - std::thread::sleep(std::time::Duration::from_millis(10)); + let why = tl!("RustGIS is closing").to_string(); + let job = app.reads.job.take(); + let waiting: Vec = app.reads.waiting.drain(..).collect(); + if let Some(job) = job { + job.control.cancel(); + let end = std::time::Instant::now() + wait; + while job.slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner).is_none() && std::time::Instant::now() < end { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + abandon_asked(app, &job.asked, why.clone()); + } + for asked in &waiting { + abandon_asked(app, asked, why.clone()); + } +} + +/// `asked` will never run: its `then` is told, and a relay's agent is answered with `why` for the +/// command and everything after it in its message, none of which runs. +fn abandon_asked(app: &mut RustGisApp, asked: &Asked, why: String) { + not_run(app, asked.then, why.clone()); + if let Some(relay) = &asked.relay { + relay.begin_archive_landing(&app.session); + relay.abandon_archive_read(&mut app.session, why); } } @@ -424,6 +538,7 @@ pub fn status(app: &RustGisApp, ui: &mut Ui, t: &Tokens) { #[allow(clippy::unwrap_used)] mod tests { use super::*; + use std::time::Duration; /// Lands every read under way and waiting, failing past `secs`. fn landed(app: &mut RustGisApp, secs: u64) { @@ -434,6 +549,187 @@ mod tests { } } + fn relay_command(id: u64, command: &str, params: Value) -> Value { + json!({"jsonrpc":"2.0","id":id,"method":"tools/call","params":{"name":"run_command","arguments":{"command":command,"params":params}}}) + } + + fn paused_relay( + app: &mut RustGisApp, + batch: Value, + ) -> (rustgis_assistant::relay::Relay, rustgis_assistant::relay::Grant, std::net::TcpStream, reads::ReadAhead) { + use rustgis_assistant::relay::{Relay, read_token_file}; + use std::io::Write; + use std::net::TcpStream; + let relay = Relay::start(0).unwrap(); + let grant = relay.grant(true).unwrap(); + let mut client = TcpStream::connect(("127.0.0.1", relay.port())).unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let token = read_token_file(grant.token_file()).unwrap(); + writeln!(client, "{}", json!({"jsonrpc":"2.0","method":"rustgis/attach","params":{"token":token,"access":true}})).unwrap(); + writeln!(client, "{batch}").unwrap(); + client.flush().unwrap(); + let end = std::time::Instant::now() + Duration::from_secs(10); + loop { + relay.poll(&mut app.session); + if let Some(ahead) = relay.take_archive_read() { + return (relay, grant, client, ahead); + } + assert!(std::time::Instant::now() < end, "relay never paused at its archive read"); + std::thread::sleep(Duration::from_millis(5)); + } + } + + fn label_drag_harness(path: &std::path::Path) -> (egui_kittest::Harness<'static, RustGisApp>, egui::Pos2) { + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + app.run("map.addTiles", json!({"url":path,"name":"Linked","overlay":true})).unwrap(); + app.run("layer.template", json!({"template":"points","name":"Labelled"})).unwrap(); + app.run( + "edit.addFeature", + json!({"layer":"Labelled","geometry":{"type":"Point","coordinates":[1,1]},"attributes":{"name":"Relay drag"}}), + ) + .unwrap(); + app.run("labels.field", json!({"layer":"Labelled","field":"name"})).unwrap(); + app.run("tool.set", json!({"tool":"moveLabel"})).unwrap(); + app.reads_off_frame = true; + let mut h = egui_kittest::Harness::builder() + .with_size(egui::vec2(1400.0, 860.0)) + .with_step_dt(0.016) + .with_max_steps(10_000) + .build_ui_state( + |ui, app: &mut RustGisApp| { + let ctx = ui.ctx().clone(); + app.logic(&ctx); + app.ui(ui); + }, + app, + ); + h.step(); + h.state_mut().run("map.zoomToExtent", json!({"extent":[-1,-1,5,5]})).unwrap(); + let end = std::time::Instant::now() + Duration::from_secs(10); + let at = loop { + h.step(); + if let Some(at) = h.state().map.label_position_for_tests("Relay drag") + && !h.state().map.drawing() + { + break at; + } + assert!(std::time::Instant::now() < end, "the label was never drawn"); + std::thread::sleep(Duration::from_millis(2)); + }; + label_pointer(&mut h, at, None); + label_pointer(&mut h, at, Some(true)); + for i in 1..=6 { + label_pointer(&mut h, at + egui::vec2(0.0, -8.0 * i as f32), None); + } + assert!(h.state().map.label_dragging_for_tests(), "a real label drag must be active"); + (h, at + egui::vec2(0.0, -48.0)) + } + + fn label_pointer(h: &mut egui_kittest::Harness<'_, RustGisApp>, at: egui::Pos2, pressed: Option) { + h.event(egui::Event::PointerMoved(at)); + if let Some(pressed) = pressed { + h.event(egui::Event::PointerButton { + pos: at, + button: egui::PointerButton::Primary, + pressed, + modifiers: egui::Modifiers::NONE, + }); + } + h.step(); + } + + #[test] + fn resumed_relay_edits_cancel_label_drags_on_success_failure_and_cancel_but_reads_preserve_them() { + use std::io::{BufRead, BufReader}; + let _turn = crate::fetching::turn(); + let dir = std::env::temp_dir().join(format!("rustgis-relay-label-drag-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + for outcome in ["success", "failure", "cancel", "not_run"] { + for removes in [true, false] { + let path = dir.join(format!("{outcome}-{removes}.pmtiles")); + archive(&path, "Label drag"); + let (mut h, release) = label_drag_harness(&path); + let suffix = if removes { + relay_command(2, "layer.remove", json!({"layer":"Labelled"})) + } else { + relay_command(2, "map.tiles", json!({})) + }; + let batch = json!([relay_command(1, "map.reloadTiles", json!({"index":0})), suffix]); + let (relay, _grant, client, ahead) = paused_relay(h.state_mut(), batch); + assert!(h.state().map.label_dragging_for_tests()); + if outcome == "failure" { + std::fs::write(&path, b"not a PMTiles archive").unwrap(); + } + admitted(h.state_mut(), ahead, relay.clone()); + if outcome == "cancel" { + h.state().reads.cancel_for_tests(); + } else if outcome == "not_run" { + let job = h.state_mut().reads.job.take().unwrap(); + job.control.cancel(); + not_run_asked(h.state_mut(), &job.asked, "The archive read could not start".into()); + let end = std::time::Instant::now() + Duration::from_secs(10); + while job.slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner).is_none() { + assert!(std::time::Instant::now() < end, "cancelled worker did not stop"); + std::thread::sleep(Duration::from_millis(2)); + } + } + landed(h.state_mut(), 60); + let mut line = String::new(); + BufReader::new(client).read_line(&mut line).unwrap(); + let reply: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(reply[0]["result"]["isError"], outcome != "success", "{outcome}: {reply}"); + assert_eq!(reply[1]["result"]["isError"], false, "{reply}"); + assert_eq!(h.state().map.label_dragging_for_tests(), !removes, "{outcome}: resumed edit guard"); + assert_eq!(h.state().session.map().layer_by_name("Labelled").is_none(), removes); + let mark = h.state().session.notice_mark(); + let depth = h.state().session.undo_depth(); + label_pointer(&mut h, release, Some(false)); + assert!(!h.state().map.label_dragging_for_tests()); + if removes { + assert_eq!(h.state().session.undo_depth(), depth, "release must not dispatch labels.move"); + assert!(h.state().session.notices_since(mark).is_empty(), "no stale labels.move error after removal"); + } else { + assert_eq!(h.state().session.map().layer_by_name("Labelled").unwrap().labels.default.moved.len(), 1); + assert_eq!(h.state().session.undo_depth(), depth + 1, "a read-only suffix preserves the pending move"); + } + relay.stop(); + } + } + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn archive_parameter_normalization_uses_objects_and_rejects_non_objects_without_panicking() { + let _turn = crate::fetching::turn(); + let dir = std::env::temp_dir().join(format!("rustgis-archive-params-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let original = dir.join("original.pmtiles"); + let replacement = dir.join("replacement.pmtiles"); + archive(&original, "Original"); + archive(&replacement, "Replacement"); + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + app.run("map.addTiles", json!({"url":original,"name":"Linked","overlay":true})).unwrap(); + app.reads_off_frame = true; + for command in ["map.reloadTiles", "map.repairTiles"] { + for params in [Value::Null, json!(true), json!(7), json!([]), json!("not an object")] { + assert!(run(&mut app, command, params).is_err()); + assert!(!busy(&app), "hostile parameters must not start a read"); + } + let id = app.session.map().tiles[0].id.clone(); + for mut params in [json!({"index":0}), json!({"name":"Linked"}), json!({"id":id})] { + if command == "map.repairTiles" { + params.as_object_mut().unwrap().insert("path".into(), json!(replacement)); + } + run(&mut app, command, params).unwrap(); + let asked = &app.reads.job.as_ref().unwrap().asked; + assert_eq!(asked.params.get("index"), Some(&json!(0))); + assert_eq!(asked.archive.as_ref().unwrap().0, 0); + landed(&mut app, 60); + } + } + let _ = std::fs::remove_dir_all(dir); + } + #[test] fn files_added_read_on_a_thread_and_land_in_order() { let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); @@ -522,6 +818,237 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + fn archive(path: &std::path::Path, credit: &str) { + use rustgis_engine::render::rustgis_tiles::{mvt, pmtiles}; + let metadata = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Mvt, + gzipped_tiles: false, + bounds: [-180.0, -85.0, 180.0, 85.0], + metadata: json!({"attribution":credit}), + }; + let bytes = pmtiles::write(vec![(0, 0, 0, mvt::encode(&mvt::Tile { layers: Vec::new() }))], &metadata).unwrap(); + std::fs::write(path, bytes).unwrap(); + } + + #[test] + fn direct_relay_archive_joins_existing_read_queue_and_answers_only_after_landing() { + use rustgis_assistant::relay::{Relay, read_token_file}; + use std::io::{BufRead, BufReader, Write}; + use std::net::TcpStream; + let _turn = crate::fetching::turn(); + let dir = std::env::temp_dir().join(format!("rustgis-relay-read-queue-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let original = dir.join("original.pmtiles"); + let replacement = dir.join("replacement.pmtiles"); + let csv = dir.join("first.csv"); + archive(&original, "Original"); + archive(&replacement, "Replacement"); + std::fs::write(&csv, "name,x,y\nA,8,47\n").unwrap(); + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + app.run("map.addTiles", json!({"url":original,"name":"Linked","overlay":true})).unwrap(); + app.reads_off_frame = true; + run(&mut app, "layer.addData", json!({"path":csv})).unwrap(); + let relay = Relay::start(0).unwrap(); + let grant = relay.grant_reviewed(false).unwrap(); + let mut client = TcpStream::connect(("127.0.0.1", relay.port())).unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let token = read_token_file(grant.token_file()).unwrap(); + writeln!(client, "{}", json!({"jsonrpc":"2.0","method":"rustgis/attach","params":{"token":token,"access":true}})).unwrap(); + writeln!(client, "{}", json!({"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"run_command","arguments":{"command":"map.repairTiles","params":{"index":0,"path":replacement}}}})).unwrap(); + client.flush().unwrap(); + let end = std::time::Instant::now() + Duration::from_secs(10); + loop { + relay.poll(&mut app.session); + if let Some(approval) = grant.pending().first() { + grant.decide(approval.id, true); + } + if let Some(ahead) = relay.take_archive_read() { + admitted(&mut app, ahead, relay.clone()); + break; + } + assert!(std::time::Instant::now() < end, "relay never admitted the archive"); + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(app.reads.waiting.len(), 1, "the relay must reuse the one existing worker queue"); + assert_eq!(app.session.map().tiles[0].url, original.to_str().unwrap()); + client.set_nonblocking(true).unwrap(); + assert_eq!(client.peek(&mut [0u8; 1]).unwrap_err().kind(), std::io::ErrorKind::WouldBlock); + client.set_nonblocking(false).unwrap(); + landed(&mut app, 60); + let mut line = String::new(); + BufReader::new(client).read_line(&mut line).unwrap(); + let reply: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(reply["result"]["isError"], false, "{reply}"); + let text = reply["result"]["content"][0]["text"].as_str().unwrap(); + assert!(text.contains("replacement") || text.contains("Repaired"), "the agent's own change is reported: {text}"); + assert!(!text.contains("layersAdded") && !text.contains("first"), "the queued user read is not the agent's change: {text}"); + assert_eq!(app.session.map().tiles[0].url, replacement.to_str().unwrap()); + assert_eq!(app.session.map().layers.len(), 1); + assert!(app.session.undo_actor().flatten().is_some_and(|actor| actor.starts_with("agent 1 (relay run "))); + relay.stop(); + let _ = std::fs::remove_dir_all(dir); + } + + thread_local! { + static TOLD: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; + } + + fn record(tag: &str, landed: Landed) { + TOLD.with(|told| { + told.borrow_mut().push(match landed { + Landed::NotRun(why) => format!("{tag}: {why}"), + Landed::Ran(_) => format!("{tag}: ran"), + }); + }); + } + + fn told_running(_: &mut RustGisApp, landed: Landed) { + record("running", landed); + } + + fn told_waiting(_: &mut RustGisApp, landed: Landed) { + record("waiting", landed); + } + + #[test] + fn stopping_ends_the_relay_batch_unrun_and_tells_the_reads_that_never_ran_in_order() { + use std::io::{BufRead, BufReader, Write}; + let _turn = crate::fetching::turn(); + TOLD.with(|told| told.borrow_mut().clear()); + let dir = std::env::temp_dir().join(format!("rustgis-relay-read-stop-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let original = dir.join("original.pmtiles"); + let replacement = dir.join("replacement.pmtiles"); + let csv = dir.join("first.csv"); + archive(&original, "Original"); + archive(&replacement, "Replacement"); + std::fs::write(&csv, "name,x,y\nA,8,47\n").unwrap(); + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + app.run("map.addTiles", json!({"url":original,"name":"Linked","overlay":true})).unwrap(); + app.reads_off_frame = true; + // One read under way and one waiting, each with a callback. + assert_eq!(run_then(&mut app, "layer.addData", json!({"path":csv}), Some(told_running)).unwrap()["started"], "Add Data"); + assert_eq!(run_then(&mut app, "layer.addData", json!({"path":csv}), Some(told_waiting)).unwrap()["waiting"], 1); + // The agent's batch: two archive commands, then a command that changes the map. + let batch = json!([ + relay_command(1, "map.repairTiles", json!({"index":0,"path":replacement})), + relay_command(2, "map.reloadTiles", json!({"index":0})), + relay_command(3, "layer.template", json!({"template":"points","name":"Mutated"})), + ]); + let (relay, _grant, client, ahead) = paused_relay(&mut app, batch); + admitted(&mut app, ahead, relay.clone()); + assert_eq!(app.reads.waiting.len(), 2); + let layers = app.session.map().layers.len(); + // User work said after admission belongs to the user, not to the agent's closing error. + app.session.notify("Unrelated user notice after admission".to_string(), false); + stop(&mut app, Duration::from_secs(10)); + assert!(!busy(&app)); + let why = tl!("RustGIS is closing").to_string(); + assert_eq!(TOLD.with(|told| told.borrow().clone()), [format!("running: {why}"), format!("waiting: {why}")]); + let mut client = BufReader::new(client); + let mut line = String::new(); + client.read_line(&mut line).unwrap(); + let reply: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(reply[0]["result"]["isError"], true, "{reply}"); + assert!(!reply[0].to_string().contains("Unrelated user notice"), "the agent's error carries only its own notices: {reply}"); + for rest in [1, 2] { + assert_eq!(reply[rest]["error"]["message"], why.as_str(), "{reply}"); + } + assert_eq!(app.session.map().tiles[0].url, original.to_str().unwrap(), "the repair never ran"); + assert_eq!(app.session.map().layers.len(), layers, "the batch's later commands never ran"); + assert!(app.session.map().layer_by_name("Mutated").is_none()); + assert!(relay.take_archive_read().is_none(), "the second archive command was never admitted"); + // The relay is not left holding the batch: the agent's next call is served. + writeln!(client.get_mut(), "{}", relay_command(4, "map.tiles", json!({}))).unwrap(); + client.get_mut().flush().unwrap(); + let end = std::time::Instant::now() + Duration::from_secs(10); + // Short waits, so each wait for the reply is followed by another poll. + client.get_ref().set_read_timeout(Some(Duration::from_millis(20))).unwrap(); + line.clear(); + let next = loop { + relay.poll(&mut app.session); + match client.read_line(&mut line) { + Ok(n) if n > 0 => break serde_json::from_str::(&line).unwrap(), + _ => assert!(std::time::Instant::now() < end, "the relay never answered the next call"), + } + }; + assert_eq!(next["result"]["isError"], false, "{next}"); + relay.stop(); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn archive_reload_lands_once_and_stale_or_cancelled_reads_keep_retained_metadata() { + let _turn = crate::fetching::turn(); + use rustgis_engine::render::rustgis_tiles::pmtiles; + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + let dir = std::env::temp_dir().join(format!("rustgis-read-archive-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("source.pmtiles"); + let url = path.to_str().unwrap(); + archive(&path, "Before"); + app.run("map.addTiles", json!({"url":url,"overlay":true})).unwrap(); + let retained = pmtiles::info_blocking(url, Duration::from_secs(10)).unwrap(); + let tile = app.session.map().tiles[0].clone(); + let camera = app.session.map().camera; + archive(&path, "After"); + app.reads_off_frame = true; + let depth = app.session.undo_depth(); + assert_eq!(app.run("map.reloadTiles", json!({"index":0})).unwrap()["started"], "Reload Data"); + assert!(Arc::ptr_eq(&pmtiles::info_blocking(url, Duration::from_secs(10)).unwrap(), &retained)); + assert_eq!(app.session.undo_depth(), depth, "validation has not installed anything"); + landed(&mut app, 60); + let current = pmtiles::info_blocking(url, Duration::from_secs(10)).unwrap(); + assert_eq!(current.attribution(), "After"); + assert_eq!(app.session.map().tiles[0], tile); + assert_eq!(app.session.map().camera, camera); + assert_eq!(app.session.undo_depth(), depth + 1); + + archive(&path, "Must not land"); + app.run("map.reloadTiles", json!({"index":0})).unwrap(); + app.run("map.setTiles", json!({"index":0,"opacity":0.4})).unwrap(); + landed(&mut app, 60); + assert_eq!(app.session.map().tiles[0].opacity, 0.4); + assert!(Arc::ptr_eq(&pmtiles::info_blocking(url, Duration::from_secs(10)).unwrap(), ¤t)); + assert!(app.session.notices.iter().any(|n| n.text.contains("tile archive changed"))); + + app.run("map.reloadTiles", json!({"index":0})).unwrap(); + app.reads.cancel_for_tests(); + landed(&mut app, 60); + assert!(Arc::ptr_eq(&pmtiles::info_blocking(url, Duration::from_secs(10)).unwrap(), ¤t)); + app.run("map.reloadTiles", json!({"index":0})).unwrap(); + app.run("map.new", json!({"name":"Different map"})).unwrap(); + landed(&mut app, 60); + assert!(app.session.map().tiles.is_empty()); + assert!(Arc::ptr_eq(&pmtiles::info_blocking(url, Duration::from_secs(10)).unwrap(), ¤t)); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn archive_repair_does_not_report_success_before_validation_lands() { + let _turn = crate::fetching::turn(); + let mut app = RustGisApp::new(rustgis_engine::Session::new(rustgis_engine::Project::default())); + let dir = std::env::temp_dir().join(format!("rustgis-read-archive-repair-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let original = dir.join("original.pmtiles"); + let replacement = dir.join("replacement.pmtiles"); + archive(&original, "Original"); + archive(&replacement, "Replacement"); + app.run("map.addTiles", json!({"url":original.to_str().unwrap(),"name":"Linked","overlay":true})).unwrap(); + app.reads_off_frame = true; + let mark = app.session.notice_mark(); + assert_eq!( + app.run("map.repairTiles", json!({"index":0,"path":replacement.to_str().unwrap()})).unwrap()["started"], + "Repair Data Source" + ); + assert_eq!(app.session.map().tiles[0].url, original.to_str().unwrap()); + assert!(!app.session.notices_since(mark).iter().any(|n| n.text.contains("Repaired"))); + landed(&mut app, 60); + assert_eq!(app.session.map().tiles[0].url, replacement.to_str().unwrap()); + assert!(app.session.notices_since(mark).iter().any(|n| n.text == "Repaired Linked")); + let _ = std::fs::remove_dir_all(dir); + } + fn reload_fixture() -> (tempfile::TempDir, RustGisApp, u32, std::path::PathBuf) { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("streamed.geojson"); diff --git a/crates/ui-egui/src/repairsource.rs b/crates/ui-egui/src/repairsource.rs index 17233bf0d0..9e65863fd7 100644 --- a/crates/ui-egui/src/repairsource.rs +++ b/crates/ui-egui/src/repairsource.rs @@ -258,8 +258,10 @@ pub fn show(app: &mut RustGisApp, ctx: &egui::Context) { Target::Tiles(index) => app.run("map.repairTiles", json!({"index": index, "path": f.path.trim()})), }; match result { - Ok(_) => { - app.session.notify(crate::i18n::fmt("Repaired {name}", &[("name", &f.name)]), false); + Ok(result) => { + if result.get("started").is_none() && result.get("waiting").is_none() { + app.session.notify(crate::i18n::fmt("Repaired {name}", &[("name", &f.name)]), false); + } close = true; } Err(error) => f.error = Some(error), diff --git a/crates/ui-egui/src/reportpane.rs b/crates/ui-egui/src/reportpane.rs index 398f768a18..c45a092c8f 100644 --- a/crates/ui-egui/src/reportpane.rs +++ b/crates/ui-egui/src/reportpane.rs @@ -330,8 +330,7 @@ fn preview(app: &mut RustGisApp, ui: &mut Ui, i: usize, t: &Tokens) { let (s, page) = (&mut app.session, app.reports.page); let drawn = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> Result<_, String> { let page = page.min(pages.len().saturating_sub(1)); - let (pm, _, _) = rustgis_engine::render::report::draw_page(doc, pages, page, 60.0, &s.project.layouts, &mut s.project.maps) - .ok_or("the page is too large to draw")?; + let (pm, _, _) = rustgis_engine::render::report::draw_page(doc, pages, page, 60.0, &s.project.layouts, &mut s.project.maps)?; Ok((pm, page, pages.len())) })) .unwrap_or_else(|_| Err("the preview failed unexpectedly".into())); diff --git a/crates/ui-egui/src/tilt_drape.rs b/crates/ui-egui/src/tilt_drape.rs index 509a445b03..8b1b45fc6c 100644 --- a/crates/ui-egui/src/tilt_drape.rs +++ b/crates/ui-egui/src/tilt_drape.rs @@ -63,6 +63,7 @@ struct Drawn { relief: Option<(usize, Vec>)>, shapes: Vec, markers: Vec, + archives: render::ArchiveRevision, } /// The drape of the moving tilted map, kept between frames while the map doesn't change. @@ -188,7 +189,10 @@ impl TiltDrape { } /// Makes `drawn` the drape, its pictures textures (reusing the last drape's). - fn show(&mut self, ctx: &egui::Context, drawn: Drawn) { + fn show(&mut self, ctx: &egui::Context, drawn: Drawn, map: &Map) { + if !drawn.archives.current([map]) { + return; + } // Something to drape: `warm` keeps it up to date again. #[cfg(not(target_arch = "wasm32"))] { @@ -225,6 +229,7 @@ impl TiltDrape { if let Some(drawn) = self.take(map) && drawn.key == key && drawn.ppp == drape_ppp + && drawn.archives.current([&*map]) && self.drape.as_ref().is_none_or(|d| d.built <= drawn.built) { if drawn.patches.is_empty() { @@ -233,7 +238,7 @@ impl TiltDrape { self.drape = None; self.empty = Some((key, drawn.camera)); } else { - self.show(ctx, drawn); + self.show(ctx, drawn, map); } } if self.job.as_ref().is_some_and(|(k, _)| *k != key) { @@ -339,7 +344,7 @@ impl TiltDrape { self.drape = None; return false; } - self.show(ctx, drawn); + self.show(ctx, drawn, map); true } @@ -557,9 +562,21 @@ impl TiltDrape { fn draw(map: &mut Map, view: &View, max_side: u32, key: DrapeKey, tiles: u64, time: f64) -> Drawn { let cam = view.camera; let ppp = view.pixels_per_point as f32; + let archives = render::archive_revisions([&*map]); let patches = render::render_tilt_drape(map, view, MARGIN, max_side); if patches.is_empty() { - return Drawn { key, ppp, camera: cam, tiles, built: time, patches, relief: None, shapes: Vec::new(), markers: Vec::new() }; + return Drawn { + key, + ppp, + camera: cam, + tiles, + built: time, + patches, + relief: None, + shapes: Vec::new(), + markers: Vec::new(), + archives, + }; } // A turned flat map's band is a square over the circle round the view: what stands on it is // taken over all of it, as the view turns there. @@ -587,7 +604,7 @@ fn draw(map: &mut Map, view: &View, max_side: u32, key: DrapeKey, tiles: u64, ti .collect(); (n, heights) }); - Drawn { key, ppp, camera: cam, tiles, built: time, patches, relief, shapes, markers } + Drawn { key, ppp, camera: cam, tiles, built: time, patches, relief, shapes, markers, archives } } /// Adds a convex polygon (screen points, `None` where one couldn't be placed) to `mesh` as a fan @@ -1068,7 +1085,18 @@ mod tests { /// A drape with nothing in it, as a worker hands back for a view over no ground. fn nothing(key: DrapeKey, camera: Camera) -> Drawn { - Drawn { key, ppp: 1.0, camera, tiles: 0, built: 0.0, patches: Vec::new(), relief: None, shapes: Vec::new(), markers: Vec::new() } + Drawn { + key, + ppp: 1.0, + camera, + tiles: 0, + built: 0.0, + patches: Vec::new(), + relief: None, + shapes: Vec::new(), + markers: Vec::new(), + archives: Default::default(), + } } /// Prepares `d` until the drape being drawn is shown. diff --git a/crates/ui-egui/src/tilt_settle.rs b/crates/ui-egui/src/tilt_settle.rs index a1aedb1138..d65f5fbb11 100644 --- a/crates/ui-egui/src/tilt_settle.rs +++ b/crates/ui-egui/src/tilt_settle.rs @@ -21,6 +21,7 @@ pub(crate) struct Done { pub ms: u64, /// Tile pictures stand in for others, or are still to be drawn: it is drawn again. pub incomplete: bool, + pub archives: render::ArchiveRevision, } /// The frame being drawn (at most one at a time), and what it is for. @@ -47,6 +48,7 @@ impl Default for TiltSettle { fn draw(mut map: Map, view: View, backend: Backend) -> Done { let t0 = web_time::Instant::now(); + let archives = render::archive_revisions([&map]); // On the UI thread in the browser, a frame's worth of tile pictures at a time; on a worker // natively, all of them. #[cfg(target_arch = "wasm32")] @@ -57,7 +59,7 @@ fn draw(mut map: Map, view: View, backend: Backend) -> Done { }; #[cfg(not(target_arch = "wasm32"))] let (rendered, incomplete) = (render::render_backend(&mut map, &view, backend), false); - Done { rendered, view, map, ms: t0.elapsed().as_millis() as u64, incomplete } + Done { rendered, view, map, ms: t0.elapsed().as_millis() as u64, incomplete, archives } } impl TiltSettle { @@ -145,6 +147,7 @@ impl TiltSettle { let done = self.ready.take(); if let Some((k, d)) = &done && d.rendered.is_none() + && d.archives.current([&d.map]) { self.failed = Some(*k); } diff --git a/docs/mcp.md b/docs/mcp.md index fb6e5049f3..c6fd0cf8e8 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -130,8 +130,16 @@ Attached calls run on the open app's session through the same tools and refusal private server. `snapshot` additionally supports a real window capture with `mode: "screen"`. If the map is not on screen, it falls back to an offscreen render and explains why in `note`. The relay holds a screen request until the app has taken the picture, and runs nothing else -meanwhile. File and network commands need `--yes`, unless **Let your Agents Control RustGIS** -is ticked. +meanwhile. Direct `run_command` calls to `map.reloadTiles` and `map.repairTiles` validate local +PMTiles/MBTiles archives on the desktop's existing background-read worker after admission and, +when required, approval. Their replies and audit entries wait for the command to actually land; +later entries in the same JSON-RPC batch and later relay requests wait too. Cancelling, revoking +the caller or changing the original archive target prevents installation. The original agent +still owns the resulting undo step. Arbitrary `run` scripts and file-bridge scripts retain their +existing synchronous execution contract; this does not move their child commands to the worker. +Notebook cells, task steps and Script pane scripts validate reload and repair inline on the window +thread, without reload status or cancellation during validation. +File and network commands need `--yes`, unless **Let your Agents Control RustGIS** is ticked. Without **Full access**, an attached agent cannot replace the open project: `project.new`, `project.open`, `project.openText`, `project.sample` and `project.fromTemplate` diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index b9c6027b27..91d15a3179 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -234,7 +234,9 @@ On the desktop, right-click a linked layer and choose **Reload Data**, beside ** Source…**. For the active layer, use **Data › Data Source › Reload Data**, or **Reload Data** in Command Search. Reload rereads the same local source, table or raster subdataset without adding other tables from its container. It retains the layer's name, identity, drawing order, -group, visibility, opacity, styling, labels, filters and map camera. +group, visibility, opacity, styling, labels, filters and map camera. Local PMTiles and MBTiles +tile archives have their own **Reload Data** on their Layers row; see +[Maps › Reloading a local tile archive](maps.md). Supported targets are linked vectors, rasters (including time-following Zarr variables), GDAL/DuckDB file-backed layers and editable local GeoPackage tables. GDAL-linked File diff --git a/docs/user-guide/maps.md b/docs/user-guide/maps.md index 21688ae1b9..6762b615a0 100644 --- a/docs/user-guide/maps.md +++ b/docs/user-guide/maps.md @@ -276,6 +276,14 @@ Ocean, Atlas, Blank), which need no network, and web tile layers over them: A style's root `center`, `zoom`, `bearing` and `pitch` say where it opens: `map.addTiles` with `"view": true` takes the map there once the style has loaded. +Present PMTiles metadata must be complete, decodable and a JSON object; malformed metadata or a +stored metadata section over **16 MiB** refuses ordinary loading, repair and reload rather than +silently losing credits or source-layer information. RustGIS's PMTiles writer enforces the same +metadata limit. This is not a limit on the archive's total size; absent metadata remains supported. +MBTiles metadata names and values must be text: unreadable or NULL metadata cells also produce an +error rather than being silently skipped. Native MBTiles loading, repair and reload refuse more +than **1,000 metadata rows** rather than silently omitting later rows. + HTTPS tiles use [platform certificate trust](data.md#https-certificates) on desktop and the command line, including private CAs trusted by the operating system. The web app uses the browser. @@ -340,6 +348,58 @@ selectable or editable; a tile not yet loaded may need another click once it arr **Reload** in the Basemap gallery (`map.reloadWebLayers`) clears cached tiles and styles and fetches them again. +**Reloading a local tile archive (desktop).** After changing a PMTiles or MBTiles file on disk, +right-click its row in Layers and choose **Reload Data** (**Reload Tile Archive** in Command +Search). CLI, Python, MCP and the Assistant use +`map.reloadTiles={"index":0}`; `"name"` or catalog `"id"` can select the archive instead. +No replacement path is accepted. This is a manual action: RustGIS does not watch the file for changes. +It rereads that archive's metadata and tiles without clearing other archives or web layers. +The layer keeps its name, visibility, opacity, blend, drawing order, source-layer visibility, +style and other presentation settings; the map's view stays where it is. Undo cannot restore +file contents that another program has overwritten. + +On the desktop, menu actions and direct attached MCP `run_command` calls validate reload and +repair on a background worker; the status bar shows the work and **Cancel**. Only one read runs +at a time. If its project, map, archive or presentation changes before it lands, the result is +left unapplied: retry on the intended target. A paused attached batch also rejects its remaining +commands if the project or selected map changes. Notebook cells, task steps, scripts (the Script +pane and attached MCP `run`) and file-bridge scripts validate inline on the window thread, without +reload status or cancellation during validation. +Cancelling MBTiles validation also interrupts SQLite queries used to infer missing zoom metadata. +It cannot preempt a filesystem operation blocked inside the operating system. + +Approval binds the selected archive's owning map, stack position and source route, not just its +name or index. Retargeting or reordering it requires reviewing the operation again. + +Each operation observes file size and modification time before validation, checks them again +after validation and immediately before applying the result, and rejects observable changes. +It also checks file identity and change time where the platform exposes them, plus MBTiles +`-wal` and `-journal` existence and revisions (not the reader-coordination `-shm` file). +Metadata that cannot be read is an error, not permission to apply an unchecked result. +These checks are not an immutable snapshot: a rewrite that leaves the same observations, or a +write after the final check, may escape detection. Finish writing before reloading or repairing. +If an archive is reloaded during a map or layout export, the export refuses a mixed-version +result. Wait for the reload to finish and retry the export. + +Reload first checks that the file can be read. If it is missing, unreadable or invalid, a notice +explains the failure and the existing layer and retained cached data are left unchanged. +Finish writing or restore the file, then choose **Reload Data** again. If the file has moved, +choose **Repair Data Source…** instead and select its new location. Repair also checks the +replacement before changing the source; cancelling or choosing an unreadable or invalid archive +leaves the existing source and layer unchanged, so you can retry with a valid file. + +Validation reads one size-bounded raster sample from a fresh MBTiles connection and decodes it +before dropping retained caches. PMTiles validation checks all reachable index directories, not +every tile payload. It allows at most **64 MiB stored and 64 MiB decoded index bytes**, **1,048,576 +index entries**, **4,096 unique directories** (including the root), and **four leaf levels**; +each stored directory read is limited to **16 MiB**. An invalid index or an exceeded validation +budget is an explicit error without eviction, even if an unusually large index is otherwise valid. + +Neither action is available for local archives in the web app, or for remote tile services. +On Windows, an open MBTiles file may not be replaceable while RustGIS holds it open: reload +does not bypass that operating-system restriction. Close RustGIS before replacing the file, +then reopen the project, or write to a different path and use **Repair Data Source…**. + **Source layers.** A vector PMTiles or MBTiles archive lists its source layers (the `vector_layers` its metadata names: `water`, `roads`, `pois`…) under its row in the Layers pane: the arrow beside it opens them, each with its own checkbox. Untick one to hide what it draws