diff --git a/apps/rustgis-web/src/request.rs b/apps/rustgis-web/src/request.rs index bb34998c7c..8c094ddb8d 100644 --- a/apps/rustgis-web/src/request.rs +++ b/apps/rustgis-web/src/request.rs @@ -35,6 +35,25 @@ pub(crate) async fn prepare_and_send> Ok(send(prepared)) } +/// Formats a fetch failure without exposing the authenticated request's credentials. +pub(crate) fn fetch_error(url: &str, error: String, authenticated: bool) -> String { + use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::logins; + if authenticated && cache::http_status(&error).is_none() { logins::scrub_error(&format!("{url}: {error}")) } else { error } +} + +/// Checks the browser's normalized destination under the caller's inherited ceiling. +/// Normalization may reveal a local host; its original spelling cannot narrow an explicit +/// local choice or widen a public document's permission. The normalizer performs no I/O. +pub(crate) fn destination( + url: &str, + ceiling: rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::address::Reach, + normalize: impl FnOnce(&str) -> Result, +) -> Result { + let normalized = normalize(url)?; + rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::address::check_reach(&normalized, ceiling)?; + Ok(normalized) +} + #[cfg(test)] mod tests { use super::*; @@ -49,6 +68,39 @@ mod tests { task::{Context, Poll, Waker}, }; + /// The browser parser's output is checked at the same preparation boundary used by GET, + /// HEAD and POST: an explicit local choice survives normalization, but public input does not. + #[test] + fn normalized_destinations_keep_the_request_ceiling() { + use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::address::Reach; + for ceiling in [Reach::Local, Reach::Public] { + for (spelling, normalized, permitted) in [ + ("http://local%68ost:3000/style.json", "http://localhost:3000/style.json", ceiling == Reach::Local), + ("http://%31%36%39.254.169.254/tile", "http://169.254.169.254/tile", false), + ("https://public.example/tile", "https://public.example/tile", true), + ] { + let result = destination(spelling, ceiling, |_| Ok(normalized.into())); + assert_eq!(result.is_ok(), permitted, "{spelling} under {ceiling:?}"); + if permitted { + assert_eq!(result.unwrap(), normalized); + } + } + } + } + + /// Status-only authenticated failures retain their machine-readable routine gap/backoff signal. + #[test] + fn authenticated_fetch_errors_preserve_http_statuses() { + for authenticated in [false, true] { + for error in ["HTTP 404", "HTTP 410", "HTTP 429 (retry after 30 s)"] { + assert_eq!(fetch_error("https://status.example/tile", error.into(), authenticated), error); + } + } + let error = fetch_error("https://status.example/tile?token=synthetic", "couldn't reach it".into(), true); + assert!(!error.contains("synthetic")); + assert!(cache::http_status(&error).is_none()); + } + fn poll(f: Pin<&mut F>) -> Poll { f.poll(&mut Context::from_waker(Waker::noop())) } diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index ab41fdc11e..c89bf81b73 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -597,11 +597,24 @@ async fn fetch_bytes( still: Option, omit: &'static [u16], ) -> Result, String> { + use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::address::Reach; + fetch_reached(url, Reach::Local, capture, still, omit).await.map(|fetched| fetched.bytes) +} + +/// GET carries its inherited reach into final URL preparation and returns the normalized +/// credential-free destination with its bytes. Fetch never follows an opaque redirect. +async fn fetch_reached( + url: &str, + ceiling: rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::address::Reach, + capture: Capture, + still: Option, + omit: &'static [u16], +) -> Result { use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{cache::RequestDiagnostics, failure::Failure}; let mut diagnostics = RequestDiagnostics::with_capture(url, "GET", capture); diagnostics.omit_statuses(omit); let answer = async { - use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{cache, logins}; + use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{address, cache, logins}; let (promise, abort, range, tail, max, url, bound, authenticated) = crate::request::prepare_and_send( &still, || async { @@ -617,6 +630,7 @@ async fn fetch_bytes( let bound = url; let prepared = rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::s3::prepare(logins::inner(url))?; let url = prepared.url.as_str(); + let destination = check_destination(url, ceiling)?; // An ArcGIS sign-in's token, for the portal and the servers it covers, renewed first when it // is about to expire. It goes as the request's `token` parameter, not a header: ArcGIS's // query endpoints answer the header's CORS preflight with an error. @@ -631,6 +645,10 @@ async fn fetch_bytes( if prepared.signed { None } else { rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::arcgis_auth::with_token(login_url) }; // Only the request carries credentials: messages and cache keys keep the original address. let request = arcgis.as_deref().unwrap_or(login_url); + // URL parsing in a browser can normalize a numeric or escaped hostname. Judge + // that actual destination against the already checked, prepared address. + let normalized = web_sys::Url::new(request).map_err(|_| "not a valid request address")?; + address::check_reach(&normalized.href(), ceiling)?; let authenticated = prepared.signed || arcgis.is_some() || login.is_some(); let err = |e: wasm_bindgen::JsValue| { if authenticated { "the browser couldn't process the authenticated request".to_string() } else { format!("{e:?}") } @@ -662,16 +680,15 @@ async fn fetch_bytes( } init.set_headers(&headers); } - // A signed or authorized request goes only where its credentials are for. - if authenticated { - init.set_redirect(web_sys::RequestRedirect::Error); - } + // Fetch hides a manual redirect's Location, so the app cannot check its next + // destination. Refuse it instead of letting the browser follow it unchecked. + init.set_redirect(if authenticated { web_sys::RequestRedirect::Error } else { web_sys::RequestRedirect::Manual }); // A credential-bearing address goes nowhere else: no referrer from it, no cookies with it. if arcgis.is_some() || login.is_some() { init.set_referrer_policy(web_sys::ReferrerPolicy::NoReferrer); init.set_credentials(web_sys::RequestCredentials::Omit); } - Ok((window, request.to_string(), init, abort, range, tail, max, prepared.url, bound.to_string(), authenticated)) + Ok((window, normalized.href(), init, abort, range, tail, max, destination, bound.to_string(), authenticated)) }, |(window, request, init, abort, range, tail, max, url, bound, authenticated)| { (window.fetch_with_str_and_init(&request, &init), abort, range, tail, max, url, bound, authenticated) @@ -704,6 +721,7 @@ async fn fetch_bytes( .dyn_into() .map_err(err)?; diagnostics.answered(resp.status()); + check_redirect(&resp)?; if !resp.ok() { return Err(refused(&resp)); } @@ -771,12 +789,12 @@ async fn fetch_bytes( if whole_tail.is_some_and(|n| bytes.len() as u64 > n) { return Err(format!("{url}: {}", cache::NO_RANGES)); } - Ok(bytes) + Ok(cache::Fetched { bytes, origin: address::Origin::named(&url).retain_binding(&bound)? }) } .await; cache::check_still(&still)?; logins::check(&bound).map_err(str::to_string)?; - result.map_err(|error| if authenticated { logins::scrub_error(&format!("{url}: {error}")) } else { error }) + result.map_err(|error| crate::request::fetch_error(&url, error, authenticated)) } .await; diagnostics.record(answer.map_err(|e| Failure::new(url, e))).map_err(|failure| failure.message) @@ -793,7 +811,7 @@ async fn post_json( use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{cache::RequestDiagnostics, failure::Failure}; let mut diagnostics = RequestDiagnostics::with_capture(url, "POST", capture); let answer = async { - use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{cache, logins, s3}; + use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{address, cache, logins, s3}; let (promise, authenticated) = crate::request::prepare_and_send( &still, || async { @@ -807,6 +825,7 @@ async fn post_json( let target = logins::inherit(url, &prepared.url); let login = logins::apply(&target).map_err(str::to_string)?; let request = prepared.url.as_str(); + check_destination(request, address::Reach::Local)?; let err = |e: wasm_bindgen::JsValue| { if login.is_some() { "the browser couldn't process the authenticated request".to_string() } else { format!("{e:?}") } }; @@ -857,7 +876,7 @@ async fn post_json( .await; cache::check_still(&still)?; logins::check(url).map_err(str::to_string)?; - result.map_err(|error| if authenticated { logins::scrub_error(&format!("{url}: {error}")) } else { error }) + result.map_err(|error| crate::request::fetch_error(url, error, authenticated)) } .await; diagnostics.record(answer.map_err(|e| Failure::new(url, e))).map_err(|failure| failure.message) @@ -874,7 +893,7 @@ async fn head( use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{cache::RequestDiagnostics, failure::Failure}; let mut diagnostics = RequestDiagnostics::with_capture(url, "HEAD", capture); let answer = async { - use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{cache, logins, s3}; + use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{address, cache, logins, s3}; let sent = crate::request::prepare_and_send( &still, || async { @@ -887,6 +906,7 @@ async fn head( return Ok(None); } let url = prepared.url.as_str(); + check_destination(url, address::Reach::Local)?; let target = logins::inherit(bound, url); let login = logins::apply(&target).map_err(str::to_string)?; let request = login.as_ref().map_or(url, |applied| applied.url.as_str()); @@ -895,6 +915,7 @@ async fn head( }; let init = web_sys::RequestInit::new(); init.set_method("HEAD"); + init.set_redirect(if login.is_some() { web_sys::RequestRedirect::Error } else { web_sys::RequestRedirect::Manual }); let login_header = login.as_ref().and_then(|applied| applied.header.as_ref()); if !prepared.headers.is_empty() || login_header.is_some() { let headers = web_sys::Headers::new().map_err(err)?; @@ -909,7 +930,6 @@ async fn head( init.set_headers(&headers); } if login.is_some() { - init.set_redirect(web_sys::RequestRedirect::Error); init.set_referrer_policy(web_sys::ReferrerPolicy::NoReferrer); init.set_credentials(web_sys::RequestCredentials::Omit); } @@ -933,6 +953,7 @@ async fn head( .dyn_into() .map_err(err)?; diagnostics.answered(resp.status()); + check_redirect(&resp)?; if !resp.ok() { return Err(refused(&resp)); } @@ -947,12 +968,29 @@ async fn head( .await; cache::check_still(&still)?; logins::check(url).map_err(str::to_string)?; - result.map_err(|error| if authenticated { logins::scrub_error(&format!("{reached}: {error}")) } else { error }) + result.map_err(|error| crate::request::fetch_error(&reached, error, authenticated)) } .await; diagnostics.record(answer.map_err(|e| Failure::new(url, e))).map_err(|failure| failure.message) } +/// Checks the URL as the browser parses it, before any request or login renewal is sent. +fn check_destination(url: &str, ceiling: rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::address::Reach) -> Result { + crate::request::destination(url, ceiling, |url| { + web_sys::Url::new(url).map(|normalized| normalized.href()).map_err(|_| "not a valid request address".into()) + }) +} + +/// A manual redirect is an opaque, zero-status response: Fetch exposes no Location to check. +/// Other zero-status responses are unusable too; none is treated as a successful download. +fn check_redirect(response: &web_sys::Response) -> Result<(), String> { + if response.status() == 0 { + Err(rustgis_ui_egui::tl!("The browser cannot verify this redirect's destination. Use the final service address.").into()) + } else { + Ok(()) + } +} + /// Makes an HTTP request (the AI assistant's calls to its model), returning the response of any /// status. async fn http( @@ -1023,9 +1061,9 @@ fn clipboard() -> Option { /// the app must not read what the user saved on the site. fn services(inbox: Inbox, font_inbox: Inbox, notes: Notes, ctx: egui::Context, embedded: bool) -> Services { // Map tiles come through the browser's fetch (the app repaints when they arrive). - rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::cache::set_fetcher_omitting(|url, done, omit| { + rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::cache::set_fetcher_with_origin(|url, ceiling, done, omit| { wasm_bindgen_futures::spawn_local(crate::request::handoff( - move |capture, still| async move { fetch_bytes(&url, capture, still, omit).await }, + move |capture, still| async move { fetch_reached(&url, ceiling, capture, still, omit).await }, done, )); }); diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index 7da31f998e..8e573532e9 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -344,11 +344,29 @@ impl Capture { /// request's own facts (address, method, status, duration) and its failure's message: the /// transport's, or the browser's. `details` may hold the few response headers a refusal /// keeps. A response body is never passed. Everything is redacted and bounded here. - pub fn network_failure(&self, source: &str, message: &str, details: &str, mut network: Network) { + pub fn network_failure(&self, source: &str, message: &str, details: &str, network: Network) { + self.keep_network_failure(source, message, details, network, false); + } + + /// Keeps repeated policy refusals once while their sanitized facts remain in this bounded + /// history. Clearing or evicting the record permits it to be reported again. + pub fn network_failure_once(&self, source: &str, message: &str, details: &str, network: Network) { + self.keep_network_failure(source, message, details, network, true); + } + + /// Compares sanitized facts under the retention lock, so Clear cannot lose a refusal. + fn keep_network_failure(&self, source: &str, message: &str, details: &str, mut network: Network, once: bool) { let Some(state) = self.0.upgrade() else { return }; network.url = clean_address(&network.url); + if once { + // Rejected tile paths on one server are one policy refusal, not distinct attempts. + network.url = url::Url::parse(&network.url) + .ok() + .filter(|u| u.origin().is_tuple()) + .map_or(network.url, |u| u.origin().ascii_serialization()); + } network.method = known_method(&network.method); - History(state).retain(Record { + let record = Record { id: 0, timestamp_ms: timestamp_ms(), severity: Severity::Error, @@ -356,7 +374,16 @@ impl Capture { message: bounded(message, MAX_MESSAGE_BYTES), details: bounded(details, MAX_DETAILS_BYTES), network: Some(network), - }); + }; + let mut state = lock(&state); + if once + && state.records.iter().any(|r| { + r.source == record.source && r.message == record.message && r.details == record.details && r.network == record.network + }) + { + return; + } + store(&mut state, record); } } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 7a7c1477a1..f89afef8ad 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -29707,24 +29707,23 @@ fn session_log_entries_name_their_project_and_say_when_their_params_are_partial( let ran = last(&s); assert_eq!((ran.params["script"].as_str(), ran.partial), (Some(script), false), "{ran:?}"); // A secret's key written twice is read as its last value: the first isn't kept as written. - let _ = - s.run("script.run", &json!({"script": "map.new {\"name\": \"M\", \"secretAccessKey\": \"S3CRET\", \"secretAccessKey\": \"\"}"})); + let _ = s.run("script.run", &json!({"script": "map.new {\"name\": \"M\", \"secretAccessKey\": \"zzzz\", \"secretAccessKey\": \"\"}"})); let twice = last(&s); - assert!(!twice.params.to_string().contains("S3CRET") && twice.partial, "{twice:?}"); + assert!(!twice.params.to_string().contains("zzzz") && twice.partial, "{twice:?}"); // Nor with its name escaped, nor a key a plain name blanks only for its value, nor deeper in; // and a key only logs leave out (`passphrase`) isn't kept either, nor one past eight levels. for script in [ - r#"map.new {"name": "M", "passphrase": "S3CRET"}"#, - r#"map.new {"name": "M", "a": [[[[[[[[[{"passphrase": "S3CRET"}]]]]]]]]]}"#, - r#"map.new {"name": "M", "\u0074oken": "S3CRET", "\u0074oken": ""}"#, - r#"map.new {"name": "M", "key": "AKIAIOSFODNN7S3CRET", "key": ""}"#, - r#"map.new {"name": "M", "style": {"token": "S3CRET", "token": ""}}"#, + r#"map.new {"name": "M", "passphrase": "zzzz"}"#, + r#"map.new {"name": "M", "a": [[[[[[[[[{"passphrase": "zzzz"}]]]]]]]]]}"#, + r#"map.new {"name": "M", "\u0074oken": "zzzz", "\u0074oken": ""}"#, + r#"map.new {"name": "M", "key": "AKIAIOSFODNN7zzzz", "key": ""}"#, + r#"map.new {"name": "M", "style": {"token": "zzzz", "token": ""}}"#, ] { let before = s.log.len(); let _ = s.run("script.run", &json!({"script": script})); let e = last(&s); assert_eq!((s.log.len(), e.command.as_str()), (before + 1, "script.run"), "{script}: its own entry"); - assert!(!e.params.to_string().contains("S3CRET") && e.partial, "{script}: {e:?}"); + assert!(!e.params.to_string().contains("zzzz") && e.partial, "{script}: {e:?}"); } // A name past the bound is cut, and says so. s.project.name = "n".repeat(crate::cmd::MAX_NAME + 5); @@ -29736,10 +29735,10 @@ fn session_log_entries_name_their_project_and_say_when_their_params_are_partial( s.run("script.run", &json!({"script": long})).unwrap(); assert!(last(&s).partial); // An address without its token: partial too. - let _ = s.run("service.layers", &json!({"url": "https://x.org/wms?token=S3CRET&layers=a", "text": "not a capabilities document"})); + let _ = s.run("service.layers", &json!({"url": "https://x.org/wms?token=zzzz&layers=a", "text": "not a capabilities document"})); let service = last(&s); assert_eq!(service.command, "service.layers", "{service:?}"); - assert!(!service.params.to_string().contains("S3CRET") && service.partial, "{service:?}"); + assert!(!service.params.to_string().contains("zzzz") && service.partial, "{service:?}"); // And `project.log` says so: `partial` is in its answer. let answer = s.run("project.log", &json!({"command": "service.layers", "limit": 1})).unwrap(); assert_eq!(answer["entries"][0]["partial"], json!(true), "{answer}"); diff --git a/crates/engine/tests/earthdata_logout.rs b/crates/engine/tests/earthdata_logout.rs index 324c14f61a..6b1eed1d64 100644 --- a/crates/engine/tests/earthdata_logout.rs +++ b/crates/engine/tests/earthdata_logout.rs @@ -7,6 +7,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; +use base64::Engine as _; use rustgis_engine::render::rustgis_tiles::cache::{self, State, Want}; use rustgis_engine::{Project, Session}; use serde_json::{Value, json}; @@ -39,7 +40,9 @@ fn earthdata_logout_forgets_what_was_read_with_the_token() { // A token kept until the logout, and one forgotten on expiry before it (its JWT says it // expired in 1970): what either read goes at the logout. - for (round, token) in ["edl.token.value", "eyJhbGciOiJub25lIn0.eyJleHAiOjF9.c2ln"].into_iter().enumerate() { + // An unsigned placeholder with an explicit expiry claim, never a usable credential. + let expired = format!("unsigned.{}.unsigned", base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(br#"{"exp":0}"#)); + for (round, token) in ["placeholder", expired.as_str()].into_iter().enumerate() { let status = s.run("earthdata.login", &json!({"token": token})).unwrap_or_else(|e| panic!("{e}")); assert_eq!(status["token"], Value::Bool(true)); // Addresses of the round's own, so nothing read before it answers from the cache. diff --git a/crates/render/src/tile_mesh.rs b/crates/render/src/tile_mesh.rs index 707dde7d52..57c28a4a8d 100644 --- a/crates/render/src/tile_mesh.rs +++ b/crates/render/src/tile_mesh.rs @@ -393,7 +393,7 @@ pub fn meshes( // Whether the style's sprite is still on its way, asked before the store is locked (a // lookup in the sprites the tile cache keeps). let (sprite_pending, sprites_read) = if rustgis_tiles::shapes::draws_patterns(style, level) { - match rustgis_tiles::sprite::load(&style.sprites, false) { + match style.load_sprites(false) { rustgis_tiles::sprite::Load::Pending => (true, Vec::new()), rustgis_tiles::sprite::Load::Ready(s) => (false, s.sheet_ids()), rustgis_tiles::sprite::Load::Failed => (false, Vec::new()), @@ -683,6 +683,7 @@ mod tests { let layer = serde_json::json!({"id": "p", "type": "fill", "source": "s", "source-layer": "l", "paint": paint}); let sprites = vec![("default".to_owned(), address.clone())]; Arc::new(Resolved { + from: None, style: Style { name: String::new(), sources: vec![("s".into(), Source::parse(&serde_json::json!({"type": "vector", "tiles": ["x/{z}/{x}/{y}"]})))], @@ -703,6 +704,22 @@ mod tests { assert!(meshes(&resolved(serde_json::json!({"fill-pattern": "dots"})), "s", &[], 3, None).1); // Without pattern fills, a mesh doesn't wait for sprites. assert!(!meshes(&resolved(serde_json::json!({"fill-color": "#f00"})), "s", &[], 3, None).1); + // A public style's private sprite must be refused synchronously, not queued by + // this preliminary GPU lookup. `serve` answers without opening any sockets. + let mut private = resolved(serde_json::json!({"fill-pattern":"dots"})); + let style = Arc::get_mut(&mut private).unwrap(); + style.from = Some(rustgis_tiles::address::Origin::named("https://mesh-style.example/style.json")); + style.sprites = vec![("default".into(), "http://127.0.0.1/mesh-private-only".into())]; + let id = TileId { z: 3, x: 0, y: 0 }; + let tiles = [DataTile { payload: Arc::new(Payload::Vector(tile())), at: id.extent(), z: id.z, clip: id.extent() }]; + let (made, waiting) = meshes(&private, "s", &tiles, 3, None); + assert!(!waiting, "a refused sprite must never become a pending fetch"); + assert!(made.first().and_then(Option::as_ref).is_some_and(|m| !m.waiting)); + assert_eq!( + crate::tiles::tests::PRIVATE_SPRITE_REQUESTS.load(std::sync::atomic::Ordering::Relaxed), + 0, + "neither sprite JSON nor PNG may reach the fetcher" + ); } #[test] @@ -713,6 +730,7 @@ mod tests { use rustgis_tiles::style::{Source, Style, StyleLayer}; let layer = serde_json::json!({"id": "p", "type": "fill", "source": "s", "source-layer": "parks", "paint": {"fill-color": "#f00"}}); let style = Arc::new(Resolved { + from: None, style: Style { name: String::new(), sources: vec![("s".into(), Source::parse(&serde_json::json!({"type": "vector", "tiles": ["x/{z}/{x}/{y}"]})))], diff --git a/crates/render/src/tiles.rs b/crates/render/src/tiles.rs index 2c0487b1a2..2556d0d8af 100644 --- a/crates/render/src/tiles.rs +++ b/crates/render/src/tiles.rs @@ -868,10 +868,15 @@ pub(crate) mod tests { static HOLD: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); static HELD: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + pub(crate) static PRIVATE_SPRITE_REQUESTS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + /// Serves tiles and documents for `https://tiles.test/…` without a network. Under `split/`, /// zoom 11 is red, zoom 13 blue, and zoom 12 never arrives. pub(crate) fn serve() { rustgis_tiles::cache::set_fetcher(|url, done| { + if url.starts_with("http://127.0.0.1/mesh-private-only") { + PRIVATE_SPRITE_REQUESTS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } let color = |r, g, b| { let mut pm = tiny_skia::Pixmap::new(256, 256).unwrap(); pm.fill(tiny_skia::Color::from_rgba8(r, g, b, 255)); diff --git a/crates/tiles/src/address.rs b/crates/tiles/src/address.rs index 2da6a9d26e..f9159bd20f 100644 --- a/crates/tiles/src/address.rs +++ b/crates/tiles/src/address.rs @@ -7,8 +7,11 @@ //! can't make the app probe local services. Link-local addresses, where cloud metadata services //! answer, are never fetched. //! -//! Hosts are judged by name or literal address; names are not resolved, so a public name that -//! resolves to a local address is not caught here. +//! These spelling checks also run after template expansion. Native fetchers additionally +//! filter the resolved IPs before connecting: a public document cannot select a local +//! connection, and no request can connect to a forbidden address. The browser controls its +//! own DNS and connections; its adapter checks the normalized URL and refuses redirects whose +//! destinations it cannot inspect. use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; @@ -23,6 +26,61 @@ pub enum Reach { Forbidden, } +/// Where a fetched document actually came from. Runtime only: a document cannot grant +/// itself a local-network allowance by including a field in its JSON. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Origin { + pub(crate) url: String, + pub(crate) reach: Reach, +} + +impl Origin { + /// Origin for a trusted transport's normalized final destination when DNS is opaque, + /// such as the browser. This does not perform normalization or grant credentials. + pub fn named(url: &str) -> Self { + Self { url: url.to_owned(), reach: reach(crate::logins::inner(url)) } + } + + /// Provenance for a trusted native transport's actual peer, never its DNS candidates. + /// Forbidden peers and non-web destinations are rejected before their bytes are accepted. + pub fn connected(url: &str, peer: IpAddr) -> Result { + let observed = ip_reach(peer); + check_reach(url, Reach::Local)?; + if observed == Reach::Forbidden { + return Err("that peer address is never fetched".into()); + } + Ok(Self { url: url.into(), reach: observed }) + } + + /// Retains a selected binding after the transport verifies it remained authorized through + /// every redirect. Omit this when a hop dropped the login, even on returning to its server. + /// The normalized destination and observed network reach remain unchanged. + pub fn retain_binding(mut self, selected: &str) -> Result { + self.url = inherit(selected, &self.url)?; + Ok(self) + } + + /// Checks a descendant without replacing the document's observed reach with host spelling. + pub(crate) fn check(&self, url: &str) -> Result<(), String> { + // Expansion can introduce a selector that was hidden from document joining. + // Only the supplying document's own selection may survive that boundary. + let (plain, _) = crate::s3::split_anonymous(url); + if let Some(child) = rustgis_data::login_address::parse(plain).map_err(str::to_string)? { + let parent = rustgis_data::login_address::parse(&self.url).map_err(str::to_string)?; + if parent.is_none_or(|p| p.binding != child.binding || p.origin != child.origin) { + return Err("a document cannot select another service login".into()); + } + } + if let Some(child) = rustgis_data::s3_address::parse(plain).map_err(str::to_string)? { + let parent = rustgis_data::s3_address::parse(&self.url).map_err(str::to_string)?; + if parent.is_none_or(|p| p.binding != child.binding || p.endpoint != child.endpoint) { + return Err("a document cannot select another S3 account".into()); + } + } + check_reach(url, self.reach) + } +} + /// The host of an HTTP(S) address, without user info, port or IPv6 brackets. fn host(url: &str) -> Option { let rest = url.split_once("://")?.1; @@ -40,6 +98,11 @@ pub fn ip_reach(ip: IpAddr) -> Reach { if let Some(v4) = v6.to_ipv4_mapped() { return v4_reach(v4); } + // RFC 6052's well-known /96 translates to this IPv4 destination. Its IPv6 + // spelling must not widen a local or forbidden peer's network reach. + if let [0x64, 0xff9b, 0, 0, 0, 0, hi, lo] = v6.segments() { + return v4_reach(Ipv4Addr::from((u32::from(hi) << 16) | u32::from(lo))); + } let seg0 = v6.segments().first().copied().unwrap_or(0); if v6.is_unspecified() || (seg0 & 0xffc0) == 0xfe80 { Reach::Forbidden @@ -119,26 +182,53 @@ pub fn is_https(url: &str) -> bool { /// Whether `url` may be fetched. `from` is where it was found (a document, or the address that /// redirected to it), or `None` when the user gave it. pub fn check(url: &str, from: Option<&str>) -> Result<(), String> { + check_reach(url, from.map_or(Reach::Local, |u| reach(crate::logins::inner(u)))) +} + +/// Keeps a selected account only on its own effective server. Explicit anonymous reads stay +/// anonymous for descendants; neither wrapper can grant a wider network reach. +pub(crate) fn inherit(from: &str, to: &str) -> Result { + let (plain, anonymous) = crate::s3::split_anonymous(from); + if anonymous { + return Ok(crate::s3::anonymous(to)); + } + if let Some(bound) = rustgis_data::s3_address::parse(plain).map_err(str::to_string)? { + let old = crate::s3::bound_destination(plain)?.ok_or("not a saved S3 address")?; + let candidate = rustgis_data::s3_address::bind(bound.binding, bound.endpoint, to).map_err(str::to_string)?; + let new = crate::s3::bound_destination(&candidate)?.ok_or("not a saved S3 address")?; + if crate::logins::origin(&old).is_some_and(|origin| Some(origin) == crate::logins::origin(&new)) { + return Ok(candidate); + } + return Ok(to.to_string()); + } + Ok(crate::logins::inherit(from, to)) +} + +/// Checks a transport's normalized destination under its inherited runtime ceiling, unwrapping +/// account markers without widening it. Call before sending; Forbidden is always refused. +pub fn check_reach(url: &str, ceiling: Reach) -> Result<(), String> { + if ceiling == Reach::Forbidden { + return Err("a forbidden document cannot supply request destinations".into()); + } crate::logins::check(url).map_err(str::to_string)?; if let Some(bound) = rustgis_data::login_address::parse(url).map_err(str::to_string)? { - return check(bound.url, from.map(crate::logins::inner)); + return check_reach(bound.url, ceiling); } - let from = from.map(crate::logins::inner); // An address read through a saved S3 connection is judged by where it is fetched, which its // own endpoint decides (no login is needed to keep it). if let Some(destination) = crate::s3::bound_destination(url)? { - return check(&destination, from); + return check_reach(&destination, ceiling); } // An `s3://` address is judged by the web address it is fetched from. // An address read anonymously is judged as the address it marks. if let (inner, true) = crate::s3::split_anonymous(url) { - if rustgis_data::s3_address::is_wrapper(inner) { + if rustgis_data::s3_address::is_wrapper(inner) || crate::s3::split_anonymous(inner).1 { return Err("not a saved S3 address".into()); } - return if crate::s3::is_s3(inner) { check(&crate::s3::https_url(url)?, from) } else { check(inner, from) }; + return if crate::s3::is_s3(inner) { check_reach(&crate::s3::https_url(url)?, ceiling) } else { check_reach(inner, ceiling) }; } if crate::s3::is_s3(url) { - return check(&crate::s3::https_url(url)?, from); + return check_reach(&crate::s3::https_url(url)?, ceiling); } let lower = url.to_ascii_lowercase(); if !(lower.starts_with("https://") || lower.starts_with("http://")) { @@ -147,11 +237,9 @@ pub fn check(url: &str, from: Option<&str>) -> Result<(), String> { if url.chars().any(|c| c.is_whitespace() || c.is_control()) { return Err(format!("{url}: the address has spaces in it")); } - match (reach(url), from) { - (Reach::Forbidden, _) => Err(format!("{url}: that address is never fetched")), - (Reach::Local, Some(origin)) if reach(origin) == Reach::Public => { - Err(format!("{url}: a document on the web can't point into the local network")) - } + match reach(url) { + Reach::Forbidden => Err(format!("{url}: that address is never fetched")), + Reach::Local if ceiling == Reach::Public => Err(format!("{url}: a document on the web can't point into the local network")), _ => Ok(()), } } @@ -160,6 +248,40 @@ pub fn check(url: &str, from: Option<&str>) -> Result<(), String> { mod tests { use super::*; + #[test] + fn connected_destinations_separate_names_from_observed_peers() { + let origin = Origin::connected("https://selected.internal/style.json", "203.0.113.1".parse().unwrap()).unwrap(); + assert_eq!(origin.reach, Reach::Public); + assert!(origin.check("http://127.0.0.1/tile").is_err()); + assert!(Origin::connected("http://metadata.google.internal/", "203.0.113.1".parse().unwrap()).is_err()); + assert!(Origin::connected("https://selected.internal/", "169.254.169.254".parse().unwrap()).is_err()); + } + + /// Translation addresses retain the IPv4 destination's policy, including on native peers. + #[test] + fn nat64_destinations_keep_the_embedded_ipv4_reach() { + for (v4, expected) in [ + ("169.254.169.254", Reach::Forbidden), + ("0.0.0.0", Reach::Forbidden), + ("127.0.0.1", Reach::Local), + ("10.0.0.1", Reach::Local), + ("203.0.113.1", Reach::Public), + ] { + let peer: IpAddr = format!("64:ff9b::{v4}").parse().unwrap(); + let url = format!("http://[{peer}]/tile"); + assert_eq!(ip_reach(peer), expected, "translated {v4}"); + assert_eq!(reach(&url), expected); + assert_eq!(check(&url, None).is_ok(), expected != Reach::Forbidden); + assert_eq!(check(&url, Some("https://public.example/style.json")).is_ok(), expected == Reach::Public); + let observed = Origin::connected("https://dns.example/tile", peer); + if expected == Reach::Forbidden { + assert!(observed.is_err(), "DNS cannot hide a forbidden translated peer"); + } else { + assert_eq!(observed.unwrap().reach, expected); + } + } + } + #[test] fn judges_addresses() { assert_eq!(reach("https://tile.openstreetmap.org/1/0/0.png"), Reach::Public); diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 4560fca965..cc9fe728b3 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -52,6 +52,15 @@ pub enum State { /// Called once with a fetch's bytes or error. pub type Done = Box, String>) + Send>; + +/// Bytes and the actual effective origin observed by a replacement transport. +#[derive(Debug)] +pub struct Fetched { + pub bytes: Vec, + pub origin: crate::address::Origin, +} +/// A policy-aware fetch completes once with bytes and their final provenance, or an error. +pub type FetchedDone = Box) + Send>; /// Called once with a download's bytes or what is known of its failure (#509). pub type DoneDetailed = Box, Failure>) + Send>; // The carrier lives with source I/O so lower-layer workers inherit the same context. @@ -79,7 +88,7 @@ pub(crate) fn caller_callback( }) } -type Fetcher = Box; +type Fetcher = Box; /// Hands back a decoded picture: [`RAW`] pixels, or the encoded bytes it was given when it /// couldn't decode them (they are decoded here then). pub type Decoded = Box, Vec>) + Send>; @@ -99,18 +108,43 @@ pub const DROPPED: &str = "dropped: the view moved on"; pub const NO_RANGES: &str = "the server doesn't answer byte-range requests"; enum Entry { - Ready(Arc, usize), + Ready(Arc, usize, Option), Pending, Failed(String, Instant), } struct Slot { entry: Entry, + /// A restrictive request's refusal must not poison a later explicit local request. + ceiling: crate::address::Reach, + /// Completed Public and Local outcomes coexist; only one fetch per URL runs at a time. + other: Option<(crate::address::Reach, Entry)>, used: u64, /// When the map last asked for it ([`get`]): a waiting tile not asked for lately is dropped. asked: Instant, } +impl Slot { + /// Prefer this caller's completed outcome; otherwise a narrower ready answer can be shared. + fn for_reach(&self, ceiling: crate::address::Reach) -> (&Entry, crate::address::Reach) { + if let Some((reach, entry)) = &self.other + && *reach == ceiling + { + return (entry, *reach); + } + (&self.entry, self.ceiling) + } + + /// Both resident outcomes count towards the cache's single memory budget. + fn bytes(&self) -> usize { + let bytes = |entry: &Entry| match entry { + Entry::Ready(_, n, _) => *n, + _ => 0, + }; + bytes(&self.entry) + self.other.as_ref().map_or(0, |(_, entry)| bytes(entry)) + } +} + #[derive(Default)] struct Cache { slots: HashMap, @@ -149,6 +183,9 @@ fn waker() -> &'static Mutex>> { /// Replaces how addresses are fetched (the browser's `fetch`). `done` must be called once. /// Decode download markers with [`split_download_limit`] and enforce their budget while reading. +/// The legacy byte-only hook is a trusted embedding override responsible for its own +/// destination policy. New transports should use set_fetcher_with_origin to receive the +/// inherited ceiling and report actual provenance; this hook cannot report redirects or peers. pub fn set_fetcher(f: impl Fn(String, Done) + Send + Sync + 'static) { set_fetcher_omitting(move |url, done, _| f(url, done)); } @@ -157,7 +194,38 @@ pub fn set_fetcher(f: impl Fn(String, Done) + Send + Sync + 'static) { /// map's edge): a transport keeping diagnostics passes them to /// [`RequestDiagnostics::omit_statuses`], so it records what the native transport records. pub fn set_fetcher_omitting(f: impl Fn(String, Done, &'static [u16]) + Send + Sync + 'static) { - *fetcher().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(Box::new(f))); + *fetcher().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(Box::new(move |url, _, done, omit| { + let origin = crate::address::Origin::named(&crate::s3::bound_destination(&url).ok().flatten().unwrap_or_else(|| url.clone())); + f(url, Box::new(move |r| done(r.map(|bytes| Fetched { bytes, origin }))), omit); + }))); +} + +/// Installs a trusted replacement transport. The transport must check normalized URLs, +/// redirects and (when available) actual peers against the supplied ceiling before sending. +/// This wrapper validates the reported final origin before accepting bytes; it cannot undo +/// a request already sent. Completion reports provenance atomically with the bytes. +/// Browser transports use Origin::named on a normalized +/// destination; replacement native transports can use Origin::connected with their actual +/// socket peer. The built-in native connector records its peer reach separately. +/// The reported origin also owns login provenance: use Origin::retain_binding only if every +/// hop retained the selected login. Completion never restores a binding the transport dropped. +pub fn set_fetcher_with_origin(f: impl Fn(String, crate::address::Reach, FetchedDone, &'static [u16]) + Send + Sync + 'static) { + *fetcher().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(Box::new(move |url, ceiling, done, omit| { + f( + url, + ceiling, + Box::new(move |result| { + done(result.and_then(|fetched| { + crate::address::check_reach(&fetched.origin.url, ceiling)?; + if fetched.origin.reach > ceiling { + return Err("the fetched answer exceeds its request's network reach".into()); + } + Ok(fetched) + })); + }), + omit, + ); + }))); } /// Undoes [`set_fetcher`]: addresses are fetched natively again (a test done with its own). @@ -384,7 +452,7 @@ pub fn in_flight() -> usize { /// to try again). Every failed fetch is recorded in the asking session's diagnostics: use /// [`get_tile`] for a map tile, whose 404 is routine. pub fn get(url: &str, want: Want) -> State { - request(url, want, &[]) + request(url, want, &[], None).0 } /// Like [`get`], for a map tile: a server's 404 (no tile there) or 410 (no longer there) for it @@ -394,22 +462,62 @@ pub fn get(url: &str, want: Want) -> State { /// a tile (a single-image layer, a layout's picture, a style's sprite sheet) goes through /// [`get`], so its 404 is recorded. pub fn get_tile(url: &str, want: Want) -> State { - request(url, want, TILE_OMIT) + get_tile_from(url, want, None) +} + +/// Requests a tile under its supplying document's reach, with routine missing-tile statuses. +pub(crate) fn get_tile_from(url: &str, want: Want, from: Option<&crate::address::Origin>) -> State { + request(url, want, TILE_OMIT, from).0 +} + +/// Requests a document-selected asset without granting it an explicit local allowance. +pub(crate) fn get_from(url: &str, want: Want, from: Option<&crate::address::Origin>) -> State { + request(url, want, &[], from).0 +} + +/// Returns JSON and its effective origin from the same cache entry and lock acquisition. +pub(crate) fn document(url: &str, from: Option<&crate::address::Origin>) -> (State, Option) { + let (state, origin, _) = request(url, Want::Json, &[], from); + (state, origin) +} + +/// A drawable tile and any omissions in a usable partial composite, from the same lookup. +pub(crate) fn tile_noted(url: &str, want: Want, from: Option<&crate::address::Origin>) -> (State, Option) { + let (state, _, note) = request(url, want, TILE_OMIT, from); + (state, note) +} + +/// Missing tiles and a view's unfetched dropped work are ordinary gaps, not drawing errors. +pub(crate) fn routine_tile_failure(error: &str) -> bool { + error == DROPPED || http_status(error).is_some_and(|status| TILE_OMIT.contains(&status)) } /// The statuses a map tile's request omits ([`get_tile`]). const TILE_OMIT: &[u16] = &[404, 410]; /// [`get`] with the statuses that are expected answers for this fetch. -fn request(url: &str, want: Want, omit: &'static [u16]) -> State { +fn request( + url: &str, + want: Want, + omit: &'static [u16], + from: Option<&crate::address::Origin>, +) -> (State, Option, Option) { if let Err(e) = crate::logins::check(resource_of(url)) { - return State::Failed(e.to_string()); + return (State::Failed(e.to_string()), None, None); } // A map tile of a layer in another system: its service's picture as it comes, or made from // its pictures (#769). if let Some(base) = crate::warp::direct(url) { - return request(base, want, omit); + return request(base, want, omit, from); + } + // This shared entry point also serves documents, sprites and archive readers without + // drawing. Drawing checks separately so it can report refusals in unfetched margin tiles. + if let Some(from) = from + && let Err(e) = from.check(url) + { + return refused_request(url, e); } + let ceiling = from.map_or(crate::address::Reach::Local, |o| o.reach); let now = Instant::now(); let mut c = lock(); c.tick += 1; @@ -418,60 +526,120 @@ fn request(url: &str, want: Want, omit: &'static [u16]) -> State { if let Some(slot) = c.slots.get_mut(url) { slot.used = tick; slot.asked = now; - match &slot.entry { - Entry::Ready(p, _) => return State::Ready(p.clone()), - Entry::Pending => return State::Pending, - Entry::Failed(e, at) if at.elapsed() < retry_wait(e, RETRY) => { + let (entry, stored_ceiling) = slot.for_reach(ceiling); + match entry { + Entry::Ready(p, _, origin) => { + if origin.as_ref().is_none_or(|o| o.reach <= ceiling) { + return (State::Ready(p.clone()), origin.clone(), None); + } + // The same name can resolve both locally and publicly. Retry through the + // restrictive pool rather than reusing local bytes or hiding the public answer. + } + Entry::Pending => return (State::Pending, None, None), + Entry::Failed(e, at) if ceiling == stored_ceiling && at.elapsed() < retry_wait(e, RETRY) => { // A server that said when to come back (429 or 503 with `Retry-After`) is asked // again then: the map draws again at that time, though nothing else draws it. // One that didn't is asked again by the next draw after a minute, as before. if let Some(wait) = retry_after(e) { notify_at(*at + wait); } - return State::Failed(e.clone()); + return (State::Failed(e.clone()), None, None); } Entry::Failed(..) => {} } + // Another reach is fetching this URL. Its completion leaves this caller's retained + // outcome intact; wait before retrying an expired or absent outcome of our own. + if matches!(slot.entry, Entry::Pending) { + return (State::Pending, None, None); + } } if crate::warp::is_warp(url) { drop(c); // Made once its pictures are here; asked again (as the map draws) until then. - return match crate::warp::make(url) { - crate::warp::Made::Waiting => State::Pending, - crate::warp::Made::Done(payload) => { - let payload = Arc::new(payload); - let n = payload_bytes(&payload); - store(url, Entry::Ready(payload.clone(), n)); - State::Ready(payload) - } - crate::warp::Made::ForNow(payload) => State::Ready(payload), - crate::warp::Made::Failed(e) => { - store(url, Entry::Failed(e.clone(), Instant::now())); - State::Failed(e) - } - }; + let mut note = None; + return ( + match crate::warp::make(url, from) { + crate::warp::Made::Waiting => State::Pending, + crate::warp::Made::Done(payload) => { + let payload = Arc::new(payload); + let n = payload_bytes(&payload); + store_from( + url, + Entry::Ready(payload.clone(), n, Some(crate::address::Origin { url: url.into(), reach: ceiling })), + ceiling, + ); + State::Ready(payload) + } + crate::warp::Made::ForNow(payload, omitted) => { + note = Some(omitted); + State::Ready(payload) + } + crate::warp::Made::Failed(e) => { + store_from(url, Entry::Failed(e.clone(), Instant::now()), ceiling); + State::Failed(e) + } + }, + None, + note, + ); } let asked_at = crate::mbtiles::epoch(); - c.slots.insert(url.to_string(), Slot { entry: Entry::Pending, used: tick, asked: now }); + keep_for(&mut c, url, Entry::Pending, ceiling); drop(c); - start(url.to_string(), want, omit, asked_at); - State::Pending + start(url.to_string(), want, omit, asked_at, ceiling); + (State::Pending, None, None) +} + +/// A cache-policy refusal is a failed request too: record it like a transport refusal, +/// after releasing the cache lock, so it cannot become unexplained missing tiles. +fn refused_request(url: &str, message: String) -> (State, Option, Option) { + note_refusal(url, &message); + (State::Failed(message), None, None) +} + +/// A policy refusal is retried by drawing each frame; keep its network facts once per history. +pub(crate) fn note_refusal(url: &str, message: &str) { + // Validation reports the effective web destination, without request-only account markers. + // Resolve S3 endpoints without credentials, signing or I/O, just as destination checks do. + let plain = crate::logins::inner(crate::s3::split_anonymous(url).0); + let destination = match crate::s3::bound_destination(plain) { + Ok(Some(at)) => Some(at), + _ if crate::s3::is_s3(plain) => crate::s3::https_url(url).ok(), + _ => None, + }; + let effective = destination.as_deref().unwrap_or(plain); + let mut diagnostics = RequestDiagnostics::new(effective, "GET"); + diagnostics.once = true; + // Keep the reason independent of expanded tile paths; the history groups by server. + let reason = + message.strip_prefix(url).or_else(|| message.strip_prefix(effective)).and_then(|s| s.strip_prefix(": ")).unwrap_or(message); + diagnostics.failed(&Failure::new(effective, reason)); } /// What the cache holds for `url`, without fetching it. pub fn peek(url: &str) -> Option> { + peek_from(url, None) +} + +/// A cache-only lookup cannot reuse a local answer for a public document. +pub(crate) fn peek_from(url: &str, from: Option<&crate::address::Origin>) -> Option> { crate::logins::check(resource_of(url)).ok()?; if let Some(base) = crate::warp::direct(url) { - return peek(base); + return peek_from(base, from); } let mut c = lock(); c.tick += 1; let tick = c.tick; let slot = c.slots.get_mut(url)?; - match &slot.entry { - Entry::Ready(p, _) => { + let ceiling = from.map_or(crate::address::Reach::Local, |o| o.reach); + match slot.for_reach(ceiling).0 { + Entry::Ready(p, _, origin) => { + if from.is_some_and(|f| f.check(url).is_err() || origin.as_ref().is_some_and(|o| o.reach > f.reach)) { + return None; + } + let payload = p.clone(); slot.used = tick; - Some(p.clone()) + Some(payload) } _ => None, } @@ -497,7 +665,10 @@ pub fn peek_failure(url: &str) -> Option { /// Forgets everything (and failures, so they are tried again). pub fn clear() { let mut c = lock(); - c.slots.retain(|_, s| matches!(s.entry, Entry::Pending)); + c.slots.retain(|_, s| { + s.other = None; + matches!(s.entry, Entry::Pending) + }); c.bytes = 0; drop(c); // Styles, TileJSON documents and PMTiles archives are read again too. @@ -541,17 +712,14 @@ pub fn forget(url: &str) { // First: a tile answered from here on is dropped, and one answered before goes with the rest. crate::mbtiles::forget_archive(url); let mut c = lock(); - let dropped: usize = c - .slots - .iter() - .filter(|(k, s)| !matches!(s.entry, Entry::Pending) && resource_of(k) == url) - .map(|(_, s)| match &s.entry { - Entry::Ready(_, n) => *n, - _ => 0, - }) - .sum(); - c.slots.retain(|k, s| resource_of(k) != url || (matches!(s.entry, Entry::Pending) && !crate::mbtiles::is_tile(k))); - c.bytes = c.bytes.saturating_sub(dropped); + c.slots.retain(|k, s| { + if resource_of(k) != url { + return true; + } + s.other = None; + matches!(s.entry, Entry::Pending) && !crate::mbtiles::is_tile(k) + }); + c.bytes = c.slots.values().map(Slot::bytes).sum(); drop(c); crate::pmtiles::forget_archive(url); crate::cog::forget_file(url); @@ -616,7 +784,7 @@ pub fn wait_for_news_unless(since: u64, timeout: Duration, stop: impl Fn() -> bo /// A waiting fetch: its decode kind, diagnostics owner, expected statuses, when it was asked for /// ([`land`]) and initiating context. -type Job = (Want, Capture, &'static [u16], u64, Option); +type Job = (Want, Capture, &'static [u16], u64, Option, crate::address::Reach); fn queue() -> &'static Mutex> { static Q: OnceLock>> = OnceLock::new(); @@ -624,7 +792,7 @@ fn queue() -> &'static Mutex> { } /// Asks for a tile: fetched now when its server has room, else it waits its turn. -fn start(url: String, want: Want, omit: &'static [u16], asked_at: u64) { +fn start(url: String, want: Want, omit: &'static [u16], asked_at: u64, ceiling: crate::address::Reach) { let capture = Capture::current(); let still = current_still(); IN_FLIGHT.fetch_add(1, Ordering::Relaxed); @@ -633,7 +801,7 @@ fn start(url: String, want: Want, omit: &'static [u16], asked_at: u64) { let mut q = queue().lock().unwrap_or_else(|e| e.into_inner()); let mut c = lock(); let latest = c.latest.unwrap_or_else(Instant::now); - let now = q.push(url, (want, capture, omit, asked_at, still), |u| c.slots.get(u).map(|s| s.asked), latest, &mut swept); + let now = q.push(url, (want, capture, omit, asked_at, still, ceiling), |u| c.slots.get(u).map(|s| s.asked), latest, &mut swept); // Under the same lock as the choice: a tile asked for again meanwhile isn't forgotten. forget_dropped(&mut c, &swept); now @@ -659,9 +827,7 @@ fn release(host: &str) { /// Called holding the cache's lock taken to choose them, so a tile can't be asked for between. fn forget_dropped(c: &mut Cache, dropped: &[Waiting]) { for w in dropped { - if matches!(c.slots.get(&w.url), Some(Slot { entry: Entry::Pending, .. })) { - c.slots.remove(&w.url); - } + forget_pending(c, &w.url); } IN_FLIGHT.fetch_sub(dropped.len(), Ordering::Release); } @@ -692,9 +858,9 @@ fn launch(jobs: Vec>) { } let _lower = Lower; while let Some(w) = pop() { - let (want, capture, omit, asked_at, still) = w.job; + let (want, capture, omit, asked_at, still, ceiling) = w.job; let _scope = capture.enter(); - with_optional_still(still, || fetch_tile(w.url, w.host, want, omit, asked_at)); + with_optional_still(still, || fetch_tile(w.url, w.host, want, omit, asked_at, ceiling)); } } // A tile queued after the last `pop` and before the flag went down is started here. @@ -747,19 +913,22 @@ fn land(key: &str, entry: Option, asked_at: u64) { Some(entry) => keep(&mut c, key, entry), // Never answered (or dropped by the fetcher): forgotten, fetched again when asked for. None => { - if matches!(c.slots.get(key), Some(Slot { entry: Entry::Pending, .. })) { - c.slots.remove(key); - } + forget_pending(&mut c, key); } } } /// Retains the initiating lifetime through deferred decoding. Retired results are forgotten; /// only live successes and resource failures are shared with later callers. -fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16], asked_at: u64) { +fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16], asked_at: u64, ceiling: crate::address::Reach) { let owner = current_still(); + let local = crate::pmtiles::is_tile(&url) || crate::cog::is_tile(&url) || crate::mbtiles::is_tile(&url); let answer = Answer { key: url.clone(), host, entry: None, asked_at }; - let done: Done = Box::new(move |bytes| { + let done: FetchedDone = Box::new(move |fetched| { + let (bytes, origin) = match fetched { + Ok(f) => (Ok(f.bytes), (!local).then_some(f.origin)), + Err(e) => (Err(e), None), + }; // Shared archive reads can answer under another loader's narrower lifetime. Retain // that floor with this tile's owner before a browser decoder suspends completion. let still = with_optional_still(owner, current_still); @@ -774,7 +943,7 @@ fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16], asked Ok(_) if check_still(&still).is_err() => return, Ok(p) => { let size = payload_bytes(&p); - Entry::Ready(Arc::new(p), size) + Entry::Ready(Arc::new(p), size, origin) } // The browser's decoder may answer after the initiating owner was retired. Err(e) if !keep_failure(&e, &still) => return, @@ -791,6 +960,12 @@ fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16], asked let decoder = image_decoder().lock().unwrap_or_else(|e| e.into_inner()).clone(); decode_with(bytes, want, decoder.as_deref(), settle); }); + if !local { + fetch_reached(url, ceiling, done, omit); + return; + } + let origin = crate::address::Origin::named(&url); + let done: Done = Box::new(move |r| done(r.map(|bytes| Fetched { bytes, origin }))); // A tile from a PMTiles archive is read from the archive, by range. if crate::pmtiles::is_tile(&url) { crate::pmtiles::fetch(&url, done); @@ -905,6 +1080,8 @@ pub struct RequestDiagnostics<'a> { started: Instant, status: Option, timed_out: bool, + /// Unsent policy refusals are repeated by redraw, not distinct transport attempts. + once: bool, /// Statuses the caller expects (a tile past a map's edge, a store's absent chunk): their /// failures aren't recorded one by one, as each would flood the history. One note per server /// says that they're omitted ([`rustgis_data::diagnostics::Capture::note_absence`]). @@ -918,7 +1095,7 @@ impl<'a> RequestDiagnostics<'a> { /// Keeps the request's initiating owner across a queue hop or an async browser task. pub fn with_capture(url: &'a str, method: &'static str, capture: rustgis_data::diagnostics::Capture) -> Self { - Self { url, method, capture, started: Instant::now(), status: None, timed_out: false, omit: &[] } + Self { url, method, capture, started: Instant::now(), status: None, timed_out: false, once: false, omit: &[] } } /// Statuses whose failures are expected answers, noted once per server, not recorded each @@ -1000,18 +1177,18 @@ impl<'a> RequestDiagnostics<'a> { _ => NetworkKind::Other, } }; - self.capture.network_failure( - "Network", - &clean(&failure.message), - &clean(&details), - Network { - url: url.to_string(), - method: self.method.to_string(), - status: self.status, - duration_ms: Some(self.started.elapsed().as_millis().min(u128::from(u64::MAX)) as u64), - kind, - }, - ); + let network = Network { + url: url.to_string(), + method: self.method.to_string(), + status: self.status, + duration_ms: (!self.once).then(|| self.started.elapsed().as_millis().min(u128::from(u64::MAX)) as u64), + kind, + }; + if self.once { + self.capture.network_failure_once("Network", &clean(&failure.message), &clean(&details), network); + } else { + self.capture.network_failure("Network", &clean(&failure.message), &clean(&details), network); + } } } @@ -1085,7 +1262,12 @@ fn download_detailed_within(url: &str, max: usize, timeout: std::time::Duration, }); let f = fetcher().lock().unwrap_or_else(|e| e.into_inner()).clone(); match f { - Some(f) => f(marked.clone(), Box::new(move |r| done(r.map_err(|e| Failure::new(&marked, e)))), omit), + Some(f) => f( + marked.clone(), + crate::address::Reach::Local, + Box::new(move |r| done(r.map(|fetched| fetched.bytes).map_err(|e| Failure::new(&marked, e)))), + omit, + ), None => native::download_within(marked, timeout, omit, done), } } @@ -1275,22 +1457,27 @@ pub fn head(url: &str, done: HeadDone) { /// A status in `omit` is an expected answer: its failure isn't recorded, only noted once per /// server (a tile past a map's edge). pub(crate) fn fetch_bytes(url: String, done: Done, omit: &'static [u16]) { + fetch_reached(url, crate::address::Reach::Local, Box::new(move |r| done(r.map(|f| f.bytes))), omit); +} + +/// Carries destination policy into the transport and returns provenance with the bytes. +fn fetch_reached(url: String, ceiling: crate::address::Reach, done: FetchedDone, omit: &'static [u16]) { if let Err(e) = check_current() { done(Err(e)); return; } let still = current_still(); - let done: Done = Box::new(move |r| with_optional_still(still, || done(r))); + let done: FetchedDone = Box::new(move |r| with_optional_still(still, || done(r))); if let Err(e) = crate::logins::check(resource_of(&url)) { done(Err(e.to_string())); return; } let key = url.clone(); - let done: Done = Box::new(move |r| done(crate::logins::check(resource_of(&key)).map_err(str::to_string).and(r))); + let done: FetchedDone = Box::new(move |r| done(crate::logins::check(resource_of(&key)).map_err(str::to_string).and(r))); let f = fetcher().lock().unwrap_or_else(|e| e.into_inner()).clone(); match f { - Some(f) => f(url, done, omit), - None => native::fetch(url, done, omit), + Some(f) => f(url, ceiling, done, omit), + None => native::fetch_reached(url, ceiling, done, omit), } } @@ -1347,11 +1534,29 @@ fn finished() { } } +/// Stores an explicitly fetched answer; composed answers use their inherited ceiling. +#[cfg(test)] fn store(url: &str, entry: Entry) { + store_from(url, entry, crate::address::Reach::Local); +} + +/// Moves a cached failure past its original retry deadline without fetching or changing clocks. +#[cfg(test)] +pub(crate) fn expire_failure(url: &str) -> bool { + let mut c = lock(); + let Some(slot) = c.slots.get_mut(url) else { return false }; + let Entry::Failed(error, at) = &mut slot.entry else { return false }; + *at = Instant::now() - retry_wait(error, RETRY) - Duration::from_secs(1); + true +} + +/// Composed cache entries retain the ceiling their constituent requests used. +fn store_from(url: &str, entry: Entry, ceiling: crate::address::Reach) { if login_gone(url) { return; } - keep(&mut lock(), url, entry); + let mut c = lock(); + keep_for(&mut c, url, entry, ceiling); } /// Whether the login `url` was fetched with is gone, in which case nothing of it is kept. @@ -1360,45 +1565,68 @@ fn login_gone(url: &str) -> bool { return false; } let mut c = lock(); - if let Some(Slot { entry: Entry::Ready(_, n), .. }) = c.slots.remove(url) { - c.bytes = c.bytes.saturating_sub(n); + if let Some(slot) = c.slots.remove(url) { + c.bytes = c.bytes.saturating_sub(slot.bytes()); } true } -/// [`store`], holding the cache's lock. +/// Keeps a transport completion under the ceiling of its still-pending slot. fn keep(c: &mut Cache, url: &str, entry: Entry) { - let added = if let Entry::Ready(_, n) = &entry { *n } else { 0 }; - // Dropped while it was fetched (cleared, or evicted): kept anyway, it was asked for. - let tick = c.tick; - let asked = c.slots.get(url).map_or_else(Instant::now, |s| s.asked); - let old = c.slots.insert(url.to_string(), Slot { entry, used: tick, asked }); - if let Some(Slot { entry: Entry::Ready(_, n), .. }) = old { - c.bytes = c.bytes.saturating_sub(n); - } - c.bytes += added; + let ceiling = c.slots.get(url).map_or(crate::address::Reach::Local, |s| s.ceiling); + keep_for(c, url, entry, ceiling); +} + +/// Replaces one reach's outcome without evicting the other's completed answer or retry time. +fn keep_for(c: &mut Cache, url: &str, entry: Entry, ceiling: crate::address::Reach) { + let old = c.slots.remove(url); + let asked = old.as_ref().map_or_else(Instant::now, |s| s.asked); + let other = old.and_then(|slot| { + c.bytes = c.bytes.saturating_sub(slot.bytes()); + if slot.ceiling != ceiling && !matches!(slot.entry, Entry::Pending) { Some((slot.ceiling, slot.entry)) } else { slot.other } + }); + let slot = Slot { entry, ceiling, other, used: c.tick, asked }; + c.bytes += slot.bytes(); + c.slots.insert(url.to_string(), slot); if c.bytes > BUDGET { evict(c); } } +/// Dropping a fetch leaves any completed result for the other reach available. +fn forget_pending(c: &mut Cache, url: &str) { + if !matches!(c.slots.get(url), Some(Slot { entry: Entry::Pending, .. })) { + return; + } + if let Some(mut slot) = c.slots.remove(url) + && let Some((reach, entry)) = slot.other.take() + { + slot.entry = entry; + slot.ceiling = reach; + c.slots.insert(url.to_string(), slot); + } +} + /// Drops the least recently used entries until the cache is at three quarters of its budget. fn evict(c: &mut Cache) { let mut ready: Vec<(u64, String, usize)> = c .slots .iter() - .filter_map(|(k, s)| match s.entry { - Entry::Ready(_, n) => Some((s.used, k.clone(), n)), - Entry::Failed(..) => Some((s.used, k.clone(), 0)), - Entry::Pending => None, - }) + .filter(|(_, s)| !matches!(s.entry, Entry::Pending) || s.other.is_some()) + .map(|(k, s)| (s.used, k.clone(), s.bytes())) .collect(); ready.sort_unstable_by_key(|r| r.0); for (_, k, n) in ready { if c.bytes <= BUDGET / 4 * 3 { break; } - c.slots.remove(&k); + if let Some(slot) = c.slots.get_mut(&k) + && matches!(slot.entry, Entry::Pending) + { + slot.other = None; + } else { + c.slots.remove(&k); + } c.bytes = c.bytes.saturating_sub(n); } } @@ -1494,7 +1722,8 @@ mod native { use std::collections::HashMap; use std::sync::{Condvar, Mutex, OnceLock}; - use super::{Capture, DROPPED, Done, DoneDetailed, MAX_BYTES}; + use super::{Capture, DROPPED, Done, DoneDetailed, Fetched, FetchedDone, MAX_BYTES}; + use crate::address::{Origin, Reach}; use crate::failure::{Failure, Reason}; /// Fetches under way at once, over all servers: two servers at their cap. @@ -1510,7 +1739,7 @@ mod native { struct Jobs { /// Waiting, oldest first: the address, its server, callback, initiating owner, and the statuses /// whose failures the caller expects. - waiting: Vec<(String, String, Done, Capture, &'static [u16], Option)>, + waiting: Vec<(String, String, FetchedDone, Capture, &'static [u16], Option, Reach)>, /// Fetches under way, by server. active: HashMap, } @@ -1573,19 +1802,25 @@ mod native { /// Marks a queued job as a local file read. const LOCAL: &str = "local-file:"; + /// Queues an explicitly selected address, keeping the byte-only public callback stable. pub(super) fn fetch(url: String, done: Done, omit: &'static [u16]) { + fetch_reached(url, Reach::Local, Box::new(move |r| done(r.map(|f| f.bytes))), omit); + } + + /// Keeps the destination ceiling with queued work; the web build requires its own fetcher. + pub(super) fn fetch_reached(url: String, ceiling: Reach, done: FetchedDone, omit: &'static [u16]) { let capture = Capture::current(); let q = queue(); let dropped = { let mut jobs = q.jobs.lock().unwrap_or_else(|e| e.into_inner()); let host = host(&url); - jobs.waiting.push((url, host, done, capture, omit, super::current_still())); + jobs.waiting.push((url, host, done, capture, omit, super::current_still(), ceiling)); (jobs.waiting.len() > QUEUE).then(|| jobs.waiting.remove(0)) }; // Every worker may be waiting for a server at its cap, and only some can take this one. q.ready.notify_all(); // Answered once the queue is unlocked: `done` takes the cache's lock and wakes the app. - if let Some((_, _, done, _, _, _)) = dropped { + if let Some((_, _, done, _, _, _, _)) = dropped { done(Err(DROPPED.into())); } } @@ -1599,18 +1834,21 @@ mod native { .http_status_as_error(false) .timeout_global(Some(timeout)) .user_agent(USER_AGENT) + // A proxy could resolve the target itself, past the checked resolver. + .proxy(None) // Redirects are followed by `get`, which checks where each one leads. .max_redirects(0) } + /// Workers share this agent's checked connections and bounded idle pool. pub(super) fn agent() -> ureq::Agent { - config_builder(std::time::Duration::from_secs(30)) + let config = config_builder(std::time::Duration::from_secs(30)) // The workers share the agent: keep a connection open for each fetch a server // may have under way, so the next tile doesn't connect again. .max_idle_connections_per_host(PER_HOST) .max_idle_connections(WORKERS) - .build() - .into() + .build(); + checked_agent(config, ureq::unversioned::resolver::DefaultResolver::default(), false) } /// The agent the workers share, and with it their open connections. @@ -1621,42 +1859,344 @@ mod native { /// An agent for downloads, which may take minutes. pub(super) fn download_agent(timeout: std::time::Duration) -> ureq::Agent { - config_builder(timeout).build().into() + checked_agent(config_builder(timeout).build(), ureq::unversioned::resolver::DefaultResolver::default(), false) } - /// Resolves names as ureq does, keeping only the internet addresses (checked where the - /// connection is made, so a name that resolves elsewhere on another lookup is caught too). - #[derive(Debug, Default)] - struct PublicOnly(ureq::unversioned::resolver::DefaultResolver); + /// Applies the same IP boundary to every native connection, including a new lookup after + /// DNS changes. Document requests keep their inherited public or local ceiling, + /// and forbidden addresses never leave the resolver. Plugin/POST agents narrow + /// this further to internet addresses only. + #[derive(Debug)] + struct CheckedResolver { + inner: R, + public_only: bool, + } - impl ureq::unversioned::resolver::Resolver for PublicOnly { + impl ureq::unversioned::resolver::Resolver for CheckedResolver { + /// Filters the exact socket addresses passed to the connector, without a second lookup. fn resolve( &self, uri: &ureq::http::Uri, config: &ureq::config::Config, timeout: ureq::unversioned::transport::NextTimeout, ) -> Result { - let all = self.0.resolve(uri, config, timeout)?; + let url = uri.to_string(); + crate::address::check(&url, None).map_err(|e| ureq::Error::Io(std::io::Error::new(std::io::ErrorKind::PermissionDenied, e)))?; + let ceiling = if self.public_only { Reach::Public } else { Reach::Local }; + let all = self.inner.resolve(uri, config, timeout)?; let mut kept = self.empty(); - for a in all.iter().filter(|a| crate::address::ip_reach(a.ip()) == crate::address::Reach::Public) { + for a in all.iter().filter(|a| { + let reach = crate::address::ip_reach(a.ip()); + reach != crate::address::Reach::Forbidden && ceiling != crate::address::Reach::Forbidden && reach <= ceiling + }) { kept.push(*a); } if kept.is_empty() { let host = uri.host().unwrap_or_default(); - let msg = format!("{host} resolves to this computer or the local network, which isn't fetched from here"); + let msg = + format!("{host} resolves to this computer or the local network, or a forbidden address, which isn't fetched from here"); return Err(ureq::Error::Io(std::io::Error::new(std::io::ErrorKind::PermissionDenied, msg))); } Ok(kept) } } - /// A download agent that connects only to internet addresses. + /// Resolver filtering retains explicit local fallback while removing forbidden peers. + /// Public-document requests never receive local candidates. This fixture opens no sockets. + #[cfg(test)] + #[test] + fn mixed_dns_destinations_keep_local_only_when_allowed() { + use ureq::unversioned::resolver::{ResolvedSocketAddrs, Resolver}; + #[derive(Debug)] + struct Answer(bool); + impl Resolver for Answer { + fn resolve( + &self, + _: &ureq::http::Uri, + _: &ureq::config::Config, + _: ureq::unversioned::transport::NextTimeout, + ) -> Result { + let mut all = self.empty(); + all.push("127.0.0.1:80".parse().unwrap()); + all.push("169.254.169.254:80".parse().unwrap()); + all.push("[64:ff9b::a9fe:a9fe]:80".parse().unwrap()); + if self.0 { + all.push("203.0.113.1:80".parse().unwrap()); + } + Ok(all) + } + } + let uri = "http://mixed-dns.example/".parse().unwrap(); + let config = config_builder(std::time::Duration::from_secs(1)).build(); + for mixed in [true, false] { + for public_only in [true, false] { + let result = CheckedResolver { inner: Answer(mixed), public_only }.resolve(&uri, &config, Default::default()); + if !mixed && public_only { + assert!(result.unwrap_err().to_string().contains("resolves to")); + } else { + let kept = result.unwrap(); + assert_eq!(kept.len(), if mixed && !public_only { 2 } else { 1 }); + assert!(kept.iter().all(|a| crate::address::ip_reach(a.ip()) != Reach::Forbidden)); + if public_only { + assert!(kept.iter().all(|a| crate::address::ip_reach(a.ip()) == Reach::Public)); + } + } + } + } + } + + thread_local! { + static SENT_TO: std::cell::Cell> = const { std::cell::Cell::new(None) }; + } + + /// Reads the reach of this synchronous request, including a pooled connection. This is + /// observed provenance, never an ambient permission; each agent has a fixed IP ceiling. + fn connected(f: impl FnOnce() -> Result) -> Result<(T, Reach), ureq::Error> { + struct Restore(Option); + impl Drop for Restore { + fn drop(&mut self) { + SENT_TO.with(|r| r.set(self.0)); + } + } + let _restore = Restore(SENT_TO.with(|r| r.replace(None))); + f().map(|v| (v, SENT_TO.with(|r| r.get()).unwrap_or(Reach::Public))) + } + + #[derive(Debug, Default)] + struct ReachConnector(C); + + impl>> + ureq::unversioned::transport::Connector<()> for ReachConnector + { + type Out = Box; + fn connect( + &self, + details: &ureq::unversioned::transport::ConnectionDetails, + chained: Option<()>, + ) -> Result, ureq::Error> { + use ureq::unversioned::transport::time::Duration; + let first = details.addrs.first().map_or(Reach::Public, |a| crate::address::ip_reach(a.ip())); + let mixed = details.addrs.iter().any(|a| crate::address::ip_reach(a.ip()) != first); + if !mixed { + return Ok(self.0.connect(details, chained)?.map(|inner| Box::new(ReachedTransport { inner, reach: first }) as Self::Out)); + } + // The transport API does not expose its peer. Try at most two homogeneous + // groups under one deadline, so success identifies the actual reach without + // losing an explicitly selected server's fallback or changing its TLS name. + let other = if first == Reach::Public { Reach::Local } else { Reach::Public }; + let began = web_time::Instant::now(); + let mut last = None; + for (index, reach) in [first, other].into_iter().enumerate() { + let mut addrs = details.resolver.empty(); + for address in details.addrs.iter().filter(|a| crate::address::ip_reach(a.ip()) == reach) { + addrs.push(*address); + } + let mut attempt = connection_at(details, addrs); + attempt.now = (details.current_time)(); + if !details.timeout.after.is_not_happening() { + let left = details.timeout.after.saturating_sub(began.elapsed()); + if left.is_zero() { + return Err(ureq::Error::Timeout(details.timeout.reason)); + } + attempt.timeout.after = Duration::Exact(left / (2 - index) as u32); + } + match self.0.connect(&attempt, chained) { + Ok(Some(inner)) => return Ok(Some(Box::new(ReachedTransport { inner, reach }) as Self::Out)), + Ok(None) => {} + Err(e) => last = Some(e), + } + } + match last { + Some(error) => Err(error), + None => Ok(None), + } + } + } + + /// Reuses URI, TLS, cancellation and resolver settings while narrowing socket candidates. + fn connection_at<'a>( + d: &ureq::unversioned::transport::ConnectionDetails<'a>, + addrs: ureq::unversioned::resolver::ResolvedSocketAddrs, + ) -> ureq::unversioned::transport::ConnectionDetails<'a> { + ureq::unversioned::transport::ConnectionDetails { + uri: d.uri, + addrs, + config: d.config, + request_level: d.request_level, + resolver: d.resolver, + now: d.now, + timeout: d.timeout, + current_time: d.current_time.clone(), + run_connector: d.run_connector.clone(), + register_shutdown: d.register_shutdown.clone(), + } + } + + #[derive(Debug)] + struct ReachedTransport { + inner: Box, + reach: Reach, + } + + impl ureq::unversioned::transport::Transport for ReachedTransport { + fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers { + self.inner.buffers() + } + fn transmit_output(&mut self, amount: usize, timeout: ureq::unversioned::transport::NextTimeout) -> Result<(), ureq::Error> { + self.inner.transmit_output(amount, timeout)?; + SENT_TO.with(|r| r.set(Some(self.reach))); + Ok(()) + } + fn await_input(&mut self, timeout: ureq::unversioned::transport::NextTimeout) -> Result { + self.inner.await_input(timeout) + } + fn is_open(&mut self) -> bool { + self.inner.is_open() + } + fn is_tls(&self) -> bool { + self.inner.is_tls() + } + } + + /// Public-document requests cannot reuse an explicitly local connection from another pool. + fn shared_public_agent() -> &'static ureq::Agent { + static A: OnceLock = OnceLock::new(); + A.get_or_init(|| public_agent(std::time::Duration::from_secs(30))) + } + + /// Builds the ordinary and internet-only agents through one connection boundary. The + /// supplied resolver also lets controlled fixtures exercise DNS changes without network DNS. + pub(super) fn checked_agent( + config: ureq::config::Config, + resolver: impl ureq::unversioned::resolver::Resolver, + public_only: bool, + ) -> ureq::Agent { + ureq::Agent::with_parts( + config, + ReachConnector::::default(), + CheckedResolver { inner: resolver, public_only }, + ) + } + + /// Keeps the normal DNS filtering and peer recording around a controlled test transport. + #[cfg(test)] + pub(super) fn fixture_transport_agent( + connector: impl ureq::unversioned::transport::Connector<(), Out = Box>, + ) -> ureq::Agent { + ureq::Agent::with_parts( + config_builder(std::time::Duration::from_secs(10)).build(), + ReachConnector(connector), + CheckedResolver { inner: ureq::unversioned::resolver::DefaultResolver::default(), public_only: false }, + ) + } + + #[cfg(test)] + /// Exercises classified DNS and real HTTP while routing every connection to an owned listener. + pub(super) fn fixture_agent(answer: std::net::SocketAddr, listener: std::net::SocketAddr, public_only: bool) -> ureq::Agent { + fixture_answers(vec![answer], listener, public_only, false) + } + + /// Mixed-DNS fixtures fail the synthetic public connection without opening a socket; + /// all other fixture connections go only to the listener owned by the test. + #[cfg(test)] + fn fixture_answers( + answers: Vec, + listener: std::net::SocketAddr, + public_only: bool, + fail_public: bool, + ) -> ureq::Agent { + use ureq::unversioned::{ + resolver::Resolver, + transport::{ConnectionDetails, Connector, DefaultConnector, Transport}, + }; + #[derive(Debug)] + struct Dns(Vec); + impl Resolver for Dns { + fn resolve( + &self, + uri: &ureq::http::Uri, + _: &ureq::config::Config, + _: ureq::unversioned::transport::NextTimeout, + ) -> Result { + let mut addresses = self.empty(); + let literal = uri.host().and_then(|h| h.parse::().ok()); + if let Some(ip) = literal { + addresses.push(std::net::SocketAddr::new(ip, uri.port_u16().unwrap_or(80))); + } else { + for answer in &self.0 { + addresses.push(*answer); + } + } + Ok(addresses) + } + } + #[derive(Debug)] + struct Route(std::net::SocketAddr, bool); + impl Connector<()> for Route { + type Out = Box; + fn connect(&self, d: &ConnectionDetails, _: Option<()>) -> Result, ureq::Error> { + if self.1 && d.addrs.first().is_some_and(|a| crate::address::ip_reach(a.ip()) == Reach::Public) { + return Err(std::io::Error::from(std::io::ErrorKind::ConnectionRefused).into()); + } + let mut addrs = d.addrs.clone(); + for address in addrs.iter_mut() { + *address = self.0; + } + let fixture = connection_at(d, addrs); + DefaultConnector::default().connect(&fixture, None) + } + } + ureq::Agent::with_parts( + config_builder(std::time::Duration::from_secs(2)).build(), + ReachConnector(Route(listener, fail_public)), + CheckedResolver { inner: Dns(answers), public_only }, + ) + } + + /// Actual requests exercise resolver filtering, connector fallback and retained provenance. + #[cfg(test)] + #[test] + fn mixed_dns_destinations_preserve_explicit_local_fallback() { + let _turn = super::turn(); + let base = super::tests::serve(vec!["HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntile".into()]); + let listener = base.strip_prefix("http://").unwrap().parse().unwrap(); + let answers = vec!["203.0.113.1:80".parse().unwrap(), "127.0.0.1:80".parse().unwrap()]; + let public = fixture_answers(answers.clone(), listener, true, true); + assert!(fetch_answer(&public, "http://mixed-fixture.example/style.json", Reach::Public, false, &[]).is_err()); + let explicit = fixture_answers(answers, listener, false, true); + let got = fetch_answer(&explicit, "http://mixed-fixture.example/style.json", Reach::Local, false, &[]).unwrap(); + assert_eq!(got.bytes, b"tile"); + assert_eq!(got.origin.reach, Reach::Local); + } + + /// A public hop narrows both GET and HEAD before redirected local DNS can connect. + #[cfg(test)] + #[test] + fn public_redirects_filter_the_next_hosts_dns() { + let _turn = super::turn(); + let target = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + target.set_nonblocking(true).unwrap(); + let address = target.local_addr().unwrap(); + let redirect = format!( + "HTTP/1.1 302 Found\r\nLocation: http://redirect-local.example:{}/tile\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + address.port() + ); + let first = super::tests::serve(vec![redirect.clone(), redirect]); + let listener = first.strip_prefix("http://").unwrap().parse().unwrap(); + let selected = fixture_agent("203.0.113.1:80".parse().unwrap(), listener, false); + let public = checked_agent(config_builder(std::time::Duration::from_secs(2)).build(), super::tests::LocalAnswer(address), true); + let url = "http://first-public.example/style.json"; + assert!( + fetch_answer_with_public(&selected, &public, url, Reach::Local, false, &[]).unwrap_err().to_string().contains("resolves to") + ); + assert!(head_with_public(&selected, &public, url).unwrap_err().contains("resolves to")); + assert_eq!(target.accept().unwrap_err().kind(), std::io::ErrorKind::WouldBlock); + } + + /// An internet-only agent with the same idle capacity as the tile workers' ordinary pool. pub(super) fn public_agent(timeout: std::time::Duration) -> ureq::Agent { - let config = config_builder(timeout) - // A proxy would resolve the address itself, past `PublicOnly`. - .proxy(None) - .build(); - ureq::Agent::with_parts(config, ureq::unversioned::transport::DefaultConnector::default(), PublicOnly::default()) + let config = config_builder(timeout).max_idle_connections_per_host(PER_HOST).max_idle_connections(WORKERS).build(); + checked_agent(config, ureq::unversioned::resolver::DefaultResolver::default(), true) } /// Asks for an address's headers on a thread of its own. @@ -1684,26 +2224,34 @@ mod native { }); } - /// Asks for `url`'s headers, following up to 5 redirects, each checked as [`get`] checks - /// them. + /// Asks for a caller-selected archive's headers, following up to 5 redirects, each checked + /// as [`get`] checks them. Document-derived reads use the GET queue's inherited ceiling. pub(super) fn head_with(agent: &ureq::Agent, url: &str) -> Result { + head_with_public(agent, shared_public_agent(), url) + } + + /// Uses a separate internet-only pool after a public hop; fixtures supply both pools. + fn head_with_public(agent: &ureq::Agent, public: &ureq::Agent, url: &str) -> Result { let mut diagnostics = super::RequestDiagnostics::new(url, "HEAD"); let mut carried_login = false; let answer = (|| -> Result { crate::logins::check(url).map_err(str::to_string)?; let prepared = crate::s3::prepare(crate::logins::inner(url))?; + crate::address::check(&prepared.url, None)?; if prepared.signed { return Ok(super::Head::default()); } let mut at = prepared.url.clone(); let first_origin = crate::logins::origin(&at); let mut same_origin = true; + let mut ceiling = Reach::Local; for redirects in 0..=5 { // The login goes to its own origin only: once a redirect leaves it, never again. let bound = crate::logins::inherit(url, &at); let login = if same_origin { crate::logins::apply(&bound)? } else { None }; carried_login |= login.is_some(); - let mut req = agent.head(login.as_ref().map_or(at.as_str(), |a| a.url.as_str())); + let sending = if ceiling == Reach::Public { public } else { agent }; + let mut req = sending.head(login.as_ref().map_or(at.as_str(), |a| a.url.as_str())); if let Some((name, value)) = login.as_ref().and_then(|a| a.header.as_ref()) { req = req.header(name.as_str(), value.as_str()); } @@ -1714,12 +2262,18 @@ mod native { } diagnostics.sending(); super::check_current()?; - let resp = req.call().map_err(unanswered)?; + let req = if sending.config().timeouts().global == agent.config().timeouts().global { + req + } else { + req.config().timeout_global(agent.config().timeouts().global).build() + }; + let (resp, reached) = connected(|| req.call()).map_err(unanswered)?; + ceiling = ceiling.min(reached); diagnostics.answered(resp.status().as_u16()); let header = |k: &str| resp.headers().get(k).and_then(|v| v.to_str().ok()); if resp.status().is_redirection() { let next = crate::template::join(&at, header("location").ok_or("a redirect with no Location")?); - crate::address::check(&next, Some(&at))?; + Origin { url: at.clone(), reach: ceiling }.check(&next)?; if crate::address::is_https(&at) && !crate::address::is_https(&next) { return Err(format!("{next}: a redirect from https to http").into()); } @@ -1839,18 +2393,19 @@ mod native { }); } + /// Dispatches each queued read through the pool matching its inherited IP ceiling. fn work() { let agent = shared_agent(); let q = queue(); loop { - let (url, done, host, capture, omit, still) = { + let (url, done, host, capture, omit, still, ceiling) = { let mut jobs = q.jobs.lock().unwrap_or_else(|e| e.into_inner()); loop { - let hosts: Vec<&str> = jobs.waiting.iter().map(|(_, h, _, _, _, _)| h.as_str()).collect(); + let hosts: Vec<&str> = jobs.waiting.iter().map(|(_, h, _, _, _, _, _)| h.as_str()).collect(); if let Some(i) = pick(&hosts, &jobs.active) { - let (url, host, done, capture, omit, still) = jobs.waiting.remove(i); + let (url, host, done, capture, omit, still, ceiling) = jobs.waiting.remove(i); *jobs.active.entry(host.clone()).or_insert(0) += 1; - break (url, done, host, capture, omit, still); + break (url, done, host, capture, omit, still, ceiling); } jobs = q.ready.wait(jobs).unwrap_or_else(|e| e.into_inner()); } @@ -1858,7 +2413,8 @@ mod native { let _scope = capture.enter(); let result = super::with_optional_still(still.clone(), || { super::check_current()?; - get(agent, &url, omit) + fetch_answer(if ceiling == Reach::Public { shared_public_agent() } else { agent }, &url, ceiling, false, omit) + .map_err(|f| f.message) }); { let mut jobs = q.jobs.lock().unwrap_or_else(|e| e.into_inner()); @@ -1875,11 +2431,11 @@ mod native { } } - /// Fetches `url`, following up to 5 redirects. `url` itself passed `address::check` when - /// it was made (the layer's address, or one its style or TileJSON chose, checked in - /// `source::resolve`); each redirect is checked here. + /// Fetches `url`, following up to 5 redirects. Checks the final network destination here, + /// even when the address was already checked before expansion or signing; every redirect + /// and every resolved connection is checked too. pub(super) fn get(agent: &ureq::Agent, url: &str, omit: &'static [u16]) -> Result, String> { - fetch_answer(agent, url, false, omit).map_err(|f| f.message) + fetch_answer(agent, url, Reach::Local, false, omit).map(|f| f.bytes).map_err(|f| f.message) } /// The User-Agent every request goes with. @@ -1927,7 +2483,7 @@ mod native { /// [`get_detailed`], a status in `omit` not recorded as a failure. pub(super) fn get_detailed_omitting(agent: &ureq::Agent, url: &str, omit: &'static [u16]) -> Result, Failure> { - fetch_answer(agent, url, true, omit).map_err(|f| Failure { + fetch_answer(agent, url, Reach::Local, true, omit).map(|f| f.bytes).map_err(|f| Failure { url: if url.contains(rustgis_data::login_address::PREFIX) { crate::logins::scrub_error(url) } else { url.to_string() }, ..f }) @@ -1944,23 +2500,44 @@ mod native { /// as tiles a server doesn't have are routine and only their status is read. A status in /// `omit` is an expected answer: its failure isn't recorded, only noted once per server. A /// request carrying a login is recorded by its plain address and scrubbed text only - /// ([`RequestDiagnostics::failed`]). - fn fetch_answer(agent: &ureq::Agent, url: &str, detail: bool, omit: &'static [u16]) -> Result, Failure> { + /// ([`RequestDiagnostics::failed`]). `agent` must already enforce `initial` in its resolver; + /// the worker selects that pool before dispatch. Redirects can only narrow it further. + pub(super) fn fetch_answer( + agent: &ureq::Agent, + url: &str, + initial: Reach, + detail: bool, + omit: &'static [u16], + ) -> Result { + fetch_answer_with_public(agent, shared_public_agent(), url, initial, detail, omit) + } + + /// The request path with both pools explicit, so redirect DNS is tested without live DNS. + fn fetch_answer_with_public( + agent: &ureq::Agent, + public: &ureq::Agent, + url: &str, + initial: Reach, + detail: bool, + omit: &'static [u16], + ) -> Result { use std::io::Read; let mut carried_login = false; // Local file failures are not network traffic. let local = crate::mbtiles::is_tile(url) || url.starts_with(LOCAL); let mut diagnostics = super::RequestDiagnostics::new(url, "GET"); diagnostics.omit_statuses(omit); - let answer = (|| -> Result, Failure> { + let answer = (|| -> Result { super::check_current()?; if crate::mbtiles::is_tile(url) { - return crate::mbtiles::read_local(url).map_err(Failure::from); + return crate::mbtiles::read_local(url).map(|bytes| Fetched { bytes, origin: Origin::named(url) }).map_err(Failure::from); } if let Some(local) = url.strip_prefix(LOCAL) { let (path, range) = super::split_range(local); let (a, b) = range.ok_or("a local read without a range")?; - return crate::pmtiles::read_file(path, a, b - a + 1).map_err(Failure::from); + return crate::pmtiles::read_file(path, a, b - a + 1) + .map(|bytes| Fetched { bytes, origin: Origin::named(url) }) + .map_err(Failure::from); } let (url, limit) = super::split_download_limit(url); let max = limit.unwrap_or(MAX_BYTES); @@ -1972,7 +2549,9 @@ mod native { // endpoint is signed. crate::logins::check(url)?; let prepared = crate::s3::prepare(crate::logins::inner(url))?; + Origin { url: prepared.url.clone(), reach: initial }.check(&prepared.url)?; let mut at = prepared.url.clone(); + let mut ceiling = initial; let first_origin = crate::logins::origin(&at); let mut same_origin = true; let mut redirects = 0; @@ -1981,7 +2560,8 @@ mod native { let bound = crate::logins::inherit(url, &at); let login = if !prepared.signed && same_origin { crate::logins::apply(&bound)? } else { None }; carried_login |= login.is_some(); - let mut req = agent.get(login.as_ref().map_or(at.as_str(), |a| a.url.as_str())); + let sending = if ceiling == Reach::Public && initial != Reach::Public { public } else { agent }; + let mut req = sending.get(login.as_ref().map_or(at.as_str(), |a| a.url.as_str())); if let Some((name, value)) = login.as_ref().and_then(|a| a.header.as_ref()) { req = req.header(name.as_str(), value.as_str()); } @@ -2016,7 +2596,14 @@ mod native { } diagnostics.sending(); super::check_current()?; - let mut resp = req.call().map_err(unanswered)?; + // Narrowing the connection pool must retain the caller's download timeout. + let req = if sending.config().timeouts().global == agent.config().timeouts().global { + req + } else { + req.config().timeout_global(agent.config().timeouts().global).build() + }; + let (mut resp, reached) = connected(|| req.call()).map_err(unanswered)?; + ceiling = ceiling.min(reached); let status = resp.status().as_u16(); diagnostics.answered(status); // A data host refusing the request (or the token): say what to do. @@ -2078,7 +2665,7 @@ mod native { return Err(Failure::without_answer("", crate::earthdata::refused_message(url), Reason::Status(401))); } // Where a redirect leads is the server's choice, not the user's. - crate::address::check(&next, Some(&at))?; + Origin { url: at.clone(), reach: ceiling }.check(&next)?; // Never from HTTPS down to plain HTTP, however either scheme is written. if crate::address::is_https(&at) && !crate::address::is_https(&next) { return Err(format!("{next}: a redirect from https to http").into()); @@ -2090,8 +2677,10 @@ mod native { same_origin &= crate::logins::origin(&next) == first_origin; at = next; }; + let retained = if same_origin || crate::s3::split_anonymous(url).1 { crate::address::inherit(url, &at)? } else { at }; + let origin = Origin { url: retained, reach: ceiling }; if resp.status().as_u16() == 204 { - return Ok(Vec::new()); + return Ok(Fetched { bytes: Vec::new(), origin }); } // A range asked for and the whole file answered (200): fine for a range from the start // that holds the whole file (a small archive's header read), cut to the range; otherwise @@ -2124,7 +2713,7 @@ mod native { if buf.len() > max { return Err(format!("larger than {max} bytes").into()); } - return Ok(buf); + return Ok(Fetched { bytes: buf, origin }); } // ureq limits the wire bytes; `take` also caps the decoded bytes when gzip expands them. resp.body_mut() @@ -2143,7 +2732,7 @@ mod native { } buf.truncate(usize::try_from(b - a + 1).unwrap_or(usize::MAX)); } - Ok(buf) + Ok(Fetched { bytes: buf, origin }) })(); let result = if local { answer } else { diagnostics.record(answer) }; let result = crate::logins::check(super::resource_of(url)).map_err(Failure::from).and(result); @@ -2161,6 +2750,11 @@ mod native { mod native { use super::{Done, DoneDetailed}; + /// Keeps the destination ceiling with queued work; the web build requires its own fetcher. + pub(super) fn fetch_reached(url: String, _ceiling: crate::address::Reach, done: super::FetchedDone, _omit: &'static [u16]) { + done(Err(format!("{url}: this build has no way to fetch tiles"))); + } + pub(super) fn fetch(url: String, done: Done, _omit: &'static [u16]) { done(Err(format!("{url}: this build has no way to fetch tiles"))); } @@ -2186,6 +2780,33 @@ mod native { #[cfg(test)] mod tests { + #[test] + fn policy_refusals_group_paths_without_evicting_other_diagnostics() { + let _turn = turn(); + let history = rustgis_data::diagnostics::History::default(); + let _capture = history.enter(); + for i in 0..20 { + let url = format!("http://127.0.0.1/{i}/0/0"); + for asked in [ + url.clone(), + crate::s3::anonymous(&url), + rustgis_data::s3_address::bind("0123456789abcdef", "http://127.0.0.1", &url).unwrap(), + rustgis_data::s3_address::bind("0123456789abcdef", "http://127.0.0.1", &format!("s3://fixture-bucket/{i}/0/0")).unwrap(), + ] { + let reason = crate::address::check_reach(&asked, crate::address::Reach::Public).unwrap_err(); + note_refusal(&asked, &reason); + } + } + assert_eq!(history.records().len(), 1, "expanded paths must share a policy diagnostic"); + assert_eq!(history.records()[0].network.as_ref().unwrap().url, "http://127.0.0.1"); + assert_eq!(history.records()[0].message, "a document on the web can't point into the local network"); + note_refusal("http://127.0.0.1/other", "different refusal"); + note_refusal("http://127.0.0.2/0", "different refusal"); + assert_eq!(history.records().len(), 3); + history.clear(); + note_refusal("http://127.0.0.1/0", "different refusal"); + assert_eq!(history.records().len(), 1); + } /// Runs a fixture that needs stable process-wide cache entries without unrelated login /// revocation clearing them. The exact named child must execute its existing assertions. #[cfg(not(target_arch = "wasm32"))] @@ -2217,7 +2838,10 @@ mod tests { let key = crate::mbtiles::template(&archive(0)).replace("{z}", "0").replace("{x}", "0").replace("{y}", "0"); let failed = |why: &str| Some(Entry::Failed(why.into(), Instant::now())); let old = crate::mbtiles::epoch(); - lock().slots.insert(key.clone(), Slot { entry: Entry::Pending, used: 0, asked: Instant::now() }); + lock().slots.insert( + key.clone(), + Slot { entry: Entry::Pending, ceiling: crate::address::Reach::Local, other: None, used: 0, asked: Instant::now() }, + ); for i in 0..=crate::mbtiles::MAX_FORGOTTEN { crate::mbtiles::forget_archive(&archive(i)); } @@ -2241,7 +2865,16 @@ mod tests { let archive = "/maps#cube=1.mbtiles"; let key = crate::mbtiles::template(archive).replace("{z}", "0").replace("{x}", "0").replace("{y}", "0"); let old = crate::mbtiles::epoch(); - lock().slots.insert(key.clone(), Slot { entry: Entry::Failed("old file".into(), Instant::now()), used: 0, asked: Instant::now() }); + lock().slots.insert( + key.clone(), + Slot { + entry: Entry::Failed("old file".into(), Instant::now()), + ceiling: crate::address::Reach::Local, + other: None, + used: 0, + asked: Instant::now(), + }, + ); forget(archive); assert!(!lock().slots.contains_key(&key)); assert!(crate::mbtiles::forgotten_since(&key, old), "the file is recorded under the path its tiles name"); @@ -2264,7 +2897,10 @@ mod tests { }; let old = crate::mbtiles::epoch(); - lock().slots.insert(key.clone(), Slot { entry: Entry::Pending, used: 0, asked: Instant::now() }); + lock().slots.insert( + key.clone(), + Slot { entry: Entry::Pending, ceiling: crate::address::Reach::Local, other: None, used: 0, asked: Instant::now() }, + ); forget(archive); land(&key, failed("old file"), old); assert_eq!(kept(), None); @@ -2278,7 +2914,10 @@ mod tests { // Or the old one lands while the new one is pending: the new one's answer is what stays. let new = crate::mbtiles::epoch(); - lock().slots.insert(key.clone(), Slot { entry: Entry::Pending, used: 0, asked: Instant::now() }); + lock().slots.insert( + key.clone(), + Slot { entry: Entry::Pending, ceiling: crate::address::Reach::Local, other: None, used: 0, asked: Instant::now() }, + ); land(&key, failed("old file"), old); assert!(matches!(lock().slots.get(&key), Some(Slot { entry: Entry::Pending, .. })), "the new read's slot stays"); land(&key, None, old); @@ -2922,10 +3561,40 @@ mod tests { /// A server on this computer answering each request with the next of `answers`. #[cfg(not(target_arch = "wasm32"))] - fn serve(answers: Vec) -> String { + pub(super) fn serve(answers: Vec) -> String { serve_bytes(answers.into_iter().map(String::into_bytes).collect()) } + /// Both responses on one owned keep-alive socket retain its actual local provenance, + /// including when the URL's hostname alone would classify it as public. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn reused_native_connection_keeps_its_origin() { + use std::io::Write; + for named in [false, true] { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + for i in 0..2 { + assert!(read_request(&mut stream).starts_with("GET /"), "both requests must use the accepted socket"); + let connection = if i == 0 { "keep-alive" } else { "close" }; + write!(stream, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: {connection}\r\n\r\n{{}}").unwrap(); + } + }); + let agent = native::checked_agent(native::agent().config().clone(), LocalAnswer(address), false); + let host = if named { format!("reuse.public.example:{}", address.port()) } else { address.to_string() }; + for path in ["first", "second"] { + let url = format!("http://{host}/{path}"); + let answer = native::fetch_answer(&agent, &url, crate::address::Reach::Local, false, &[]).unwrap(); + assert_eq!(answer.origin.reach, crate::address::Reach::Local); + assert_eq!(answer.origin.url, url); + assert_eq!(answer.bytes, b"{}"); + } + server.join().unwrap(); + } + } + #[cfg(not(target_arch = "wasm32"))] fn serve_bytes(answers: Vec>) -> String { serve_recorded(answers).0 @@ -3191,7 +3860,7 @@ mod tests { assert!(requests[0].contains("x-key: a\r\n")); assert!(requests[1].contains("x-key: b\r\n")); assert!(!requests[2].contains("x-key:")); - store(&ua, Entry::Ready(Arc::new(Payload::Json(serde_json::json!({"a": 1}))), 64)); + store(&ua, Entry::Ready(Arc::new(Payload::Json(serde_json::json!({"a": 1}))), 64, None)); assert!(peek(&ua).is_some()); assert!(peek(&ub).is_none() && peek(&raw).is_none()); assert!(logins::deactivate(&a)); @@ -3236,7 +3905,7 @@ mod tests { assert_ne!(url, replacement_url); release_tx.send(()).unwrap(); assert!(worker.join().unwrap().is_err()); - store(&url, Entry::Ready(Arc::new(Payload::Json(serde_json::json!({"old": true}))), 64)); + store(&url, Entry::Ready(Arc::new(Payload::Json(serde_json::json!({"old": true}))), 64, None)); assert!(peek(&url).is_none()); assert!(matches!(get(&url, Want::Json), State::Failed(_))); assert!(peek(&replacement_url).is_none()); @@ -3303,7 +3972,7 @@ mod tests { let mut resolved = None; for _ in 0..200 { for (address, json, len) in &documents { - store(address, Entry::Ready(Arc::new(Payload::Json(json.clone())), *len)); + store(address, Entry::Ready(Arc::new(Payload::Json(json.clone())), *len, None)); } if let source::Resolve::Ready(r) = source::resolve(&spec) { resolved = Some(r); @@ -3458,6 +4127,561 @@ mod tests { assert!(native::get(&agent, &format!("{base}/x"), &[]).unwrap_err().contains("too many")); } + /// URL text inside a redirect query remains data on the same document and server. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn query_only_redirects_keep_the_document_origin() { + let (base, asked) = serve_recorded( + [ + "HTTP/1.1 302 Found\r\nLocation: ?next=https://other.example/tiles\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + "HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntile", + ] + .into_iter() + .map(|s| s.as_bytes().to_vec()) + .collect(), + ); + let answer = + native::fetch_answer(&native::agent(), &format!("{base}/style.json?old=1"), crate::address::Reach::Local, false, &[]).unwrap(); + assert_eq!(answer.bytes, b"tile"); + assert_eq!(answer.origin.url, format!("{base}/style.json?next=https://other.example/tiles")); + assert_eq!(answer.origin.reach, crate::address::Reach::Local); + assert!(asked.recv_timeout(std::time::Duration::from_secs(2)).unwrap().starts_with("GET /style.json?old=1 ")); + assert!( + asked.recv_timeout(std::time::Duration::from_secs(2)).unwrap().starts_with("GET /style.json?next=https://other.example/tiles ") + ); + } + + /// A redirected style/TileJSON must use its effective public origin, including from + /// cache. The fixture routes synthetic public IPs only to its own loopback listener. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn redirected_documents_cannot_keep_a_local_allowance() { + let _turn = turn(); + use crate::{ + address::Reach, + source::{Kind, Resolve, Spec}, + }; + for kind in [Kind::Style, Kind::Vector] { + let tiles = "http://{switch:127.0.0.1}/redirected-document/{z}/{x}/{y}"; + let doc = if kind == Kind::Style { + serde_json::json!({"version": 8, "sources": { + "bad": {"type":"vector", "tiles":[tiles]}, + "relative": {"type":"vector", "tiles":["tiles/{z}/{x}/{y}"]} + }, "layers":[{"id":"bad", "type":"fill", "source":"bad", "source-layer":"l"}]}) + } else { + serde_json::json!({"tiles":[tiles]}) + }; + let destination = "http://document.public.example/final/style.json"; + let base = serve(vec![ + format!("HTTP/1.1 302 Found\r\nLocation: {destination}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"), + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{doc}", doc.to_string().len()), + ]); + let listener = base.strip_prefix("http://").unwrap().parse().unwrap(); + let agent = native::fixture_agent("203.0.113.1:80".parse().unwrap(), listener, false); + let asked = format!("{base}/original.json"); + let fetched = native::fetch_answer(&agent, &asked, Reach::Local, false, &[]).unwrap(); + assert_eq!((&*fetched.origin.url, fetched.origin.reach), (destination, Reach::Public)); + let payload = decode(&fetched.bytes, Want::Json).unwrap(); + store(&asked, Entry::Ready(Arc::new(payload), 1024, Some(fetched.origin))); + let spec = Spec { + kind, + url: asked, + subdomains: Vec::new(), + min_zoom: 0, + max_zoom: 14, + tile_size: 256, + tms: false, + attribution: String::new(), + layers: Vec::new(), + style_layers: String::new(), + hidden_layers: Vec::new(), + }; + let Resolve::Ready(resolved) = crate::source::resolve(&spec) else { panic!("the cached document must resolve") }; + if kind == Kind::Style { + let relative = resolved.style.sources.iter().find(|(id, _)| id == "relative").unwrap(); + assert!( + matches!(&relative.1, crate::style::Source::Vector { tiles, .. } if tiles == &["http://document.public.example/final/tiles/{z}/{x}/{y}"]) + ); + } + let sent = Arc::new(AtomicUsize::new(0)); + let observed = sent.clone(); + set_fetcher(move |_, done| { + observed.fetch_add(1, Ordering::SeqCst); + done(Ok(Vec::new())); + }); + let mut picture = Pixmap::new(64, 64).unwrap(); + let drawn = crate::draw::draw(&mut picture, &spec, 1.0, &crate::draw::tests::world_frame(), true); + assert!(drawn.error.as_deref().is_some_and(|e| e.contains("local network")), "{:?}", drawn.error); + assert_eq!(sent.load(Ordering::SeqCst), 0, "the redirected document must send no local tiles"); + clear_fetcher(); + } + // Resolved forbidden IPs never reach even the fixture connector. A regression can + // only contact this test's own listener, not a real metadata or local service. + let target = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + target.set_nonblocking(true).unwrap(); + for ip in ["169.254.169.254:80", "0.0.0.0:80", "[fe80::1]:80", "[::]:80"] { + let agent = native::fixture_agent(ip.parse().unwrap(), target.local_addr().unwrap(), false); + assert!(native::get(&agent, "http://blocked.example/", &[]).unwrap_err().contains("resolves to")); + } + assert_eq!(target.accept().unwrap_err().kind(), std::io::ErrorKind::WouldBlock); + } + + /// Draws the actual raster selection path against seeded cache entries, without I/O. + fn draw_cached_raster(url: &str, from: crate::address::Origin) -> (Pixmap, crate::Drawn) { + let mut style = crate::style::Style::parse(&serde_json::json!({"version":8, + "sources":{"s":{"type":"raster","tiles":[url],"maxzoom":0}}, + "layers":[{"id":"r","type":"raster","source":"s"}]})) + .unwrap(); + if let Some((_, crate::style::Source::Raster { from: origin, .. })) = style.sources.first_mut() { + *origin = Some(from.clone()); + } + let resolved = crate::source::Resolved { + style, + from: Some(from), + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }; + let frame = crate::draw::tests::world_frame(); + let mut image = Pixmap::new(64, 64).unwrap(); + let mut drawn = crate::Drawn::default(); + crate::draw::draw_layers(Some(&mut image), (64, 64), &frame.extent(64, 64), &resolved, &frame, crate::draw::Pass::All, &mut drawn); + (image, drawn) + } + + /// Browser-normalized metadata reaches the document consumer atomically with its bytes. + /// Descendants carry Public through the real tile queue to the replacement transport, + /// which refuses a normalized local address before its simulated send. + #[test] + fn replacement_transport_destinations_keep_normalized_origins_and_ceiling() { + let _turn = turn(); + let sent = Arc::new(Mutex::new(Vec::new())); + let attempts = Arc::new(Mutex::new(Vec::new())); + let attempted = attempts.clone(); + let observed = sent.clone(); + set_fetcher_with_origin(move |url, ceiling, done, _| { + attempted.lock().unwrap().push((url.clone(), ceiling)); + let normalized = if url == "https://public.example\\@localhost/style.json" { + "https://public.example/@localhost/style.json" + } else if url == "http://local%68ost/tile.png" { + "http://127.0.0.1/tile.png" + } else { + &url + }; + let check = crate::address::check_reach(normalized, ceiling); + done(check.map(|_| { + observed.lock().unwrap().push((normalized.to_string(), ceiling)); + Fetched { bytes: b"{}".to_vec(), origin: crate::address::Origin::named(normalized) } + })); + }); + let root = "https://public.example\\@localhost/style.json"; + document(root, None); + wait(std::time::Duration::from_secs(2)); + let (state, origin) = document(root, None); + assert!(matches!(state, State::Ready(_))); + let origin = origin.unwrap(); + assert_eq!(origin.url, "https://public.example/@localhost/style.json"); + assert_eq!(origin.reach, crate::address::Reach::Public); + let child = "http://local%68ost/tile.png"; + get_tile_from(child, Want::Image, Some(&origin)); + wait(std::time::Duration::from_secs(2)); + assert!(matches!(get_tile_from(child, Want::Image, Some(&origin)), State::Failed(_))); + assert_eq!(attempts.lock().unwrap().last().unwrap(), &(child.to_string(), crate::address::Reach::Public)); + assert_eq!(attempts.lock().unwrap().len(), 2, "the child reaches the normalization boundary"); + assert_eq!(sent.lock().unwrap().len(), 1, "normalized local child never reaches send"); + assert!(origin.check("http://127.0.0.1/tile.png").is_err()); + } + + /// The replacement transport, which sees every redirect, owns the retained binding. + /// Returning anonymously to the original server must not restore a dropped login. + #[test] + fn replacement_destinations_do_not_restore_dropped_logins() { + let _turn = turn(); + let plain = "https://redirect-binding.example/style.json"; + let binding = crate::logins::activate( + "https://redirect-binding.example", + "redirect-fixture", + crate::logins::Method::Header("X-Key".into()), + "fixture", + false, + ) + .unwrap(); + let bound = crate::logins::bind(&binding, plain).unwrap(); + for retained in [false, true] { + let origin = crate::address::Origin::named(plain); + let origin = if retained { origin.retain_binding(&bound).unwrap() } else { origin }; + set_fetcher_with_origin(move |_, _, done, _| { + done(Ok(Fetched { bytes: Vec::new(), origin: origin.clone() })); + }); + let (tx, rx) = std::sync::mpsc::channel(); + fetch_reached(bound.clone(), crate::address::Reach::Local, Box::new(move |r| tx.send(r).unwrap()), &[]); + let fetched = rx.recv_timeout(std::time::Duration::from_secs(2)).unwrap().unwrap(); + let child = crate::template::join(&fetched.origin.url, "tile.json"); + assert_eq!(rustgis_data::login_address::parse(&child).unwrap().is_some(), retained); + } + crate::logins::deactivate(&binding); + } + + /// Local bytes and local failures cannot prevent a public-ceiling fetch of the same name. + #[test] + fn cached_destinations_refetch_under_the_requested_reach() { + use crate::address::{Origin, Reach}; + let _turn = turn(); + let calls = Arc::new(Mutex::new(Vec::new())); + let sent = calls.clone(); + set_fetcher_with_origin(move |_, ceiling, done, _| { + sent.lock().unwrap().push(ceiling); + done(Err("fixture destination refused".into())); + }); + let from = Origin::named("https://public-document.example/style.json"); + for ready in [true, false] { + let url = format!("https://split-cache.example/{ready}"); + let entry = if ready { + Entry::Ready(Arc::new(Payload::Empty), 64, Some(Origin { url: url.clone(), reach: Reach::Local })) + } else { + Entry::Failed("http status: 404".into(), Instant::now()) + }; + store(&url, entry); + assert!(matches!(get_tile_from(&url, Want::Image, Some(&from)), State::Pending), "a restricted fetch must start"); + wait(std::time::Duration::from_secs(2)); + assert!(matches!(get_tile_from(&url, Want::Image, Some(&from)), State::Failed(e) if e == "fixture destination refused")); + } + assert_eq!(*calls.lock().unwrap(), [Reach::Public, Reach::Public]); + let url = "https://split-cache.example/reverse"; + store_from(url, Entry::Failed("restricted refusal".into(), Instant::now()), Reach::Public); + assert!(matches!(get_tile(url, Want::Image), State::Pending), "explicit requests retry restrictive failures too"); + wait(std::time::Duration::from_secs(2)); + assert_eq!(calls.lock().unwrap().last(), Some(&Reach::Local)); + let url = "https://split-cache.example/reusable"; + store(url, Entry::Ready(Arc::new(Payload::Empty), 0, Some(Origin::named(url)))); + assert!(matches!(get_tile_from(url, Want::Image, Some(&from)), State::Ready(_))); + assert_eq!(calls.lock().unwrap().len(), 3, "a permitted ready answer is reused"); + } + + /// Concurrent layers with different reach must not evict each other's ready answer or + /// retry failure on every frame. The transport spy never sends a network request. + #[test] + fn shared_destination_reaches_do_not_refetch_each_other() { + use crate::address::{Origin, Reach}; + let _turn = turn(); + let _alarm = alarm_turn(); + let calls = Arc::new(Mutex::new(Vec::new())); + let sent = calls.clone(); + set_fetcher_with_origin(move |url, ceiling, done, _| { + sent.lock().unwrap().push((url.clone(), ceiling)); + done(if url.ends_with("/public") || ceiling == Reach::Local && url.ends_with("/ready") { + let bytes: &[u8] = if ceiling == Reach::Public { br#"{"reach":"public"}"# } else { br#"{"reach":"local"}"# }; + Ok(Fetched { bytes: bytes.to_vec(), origin: Origin { url, reach: ceiling } }) + } else if url.ends_with("/retry") { + let seconds = if ceiling == Reach::Local { "120" } else { "30" }; + Err(crate::failure::with_retry_after("http status: 429".into(), 429, Some(seconds))) + } else { + Err("fixture destination refused".into()) + }); + }); + let from = Origin::named("https://public-style.example/style.json"); + for case in ["ready", "failures", "retry", "public"] { + let url = format!("https://shared-reach.example/{case}"); + let mut retry_times = None; + get(&url, Want::Json); + wait(std::time::Duration::from_secs(2)); + for _ in 0..3 { + get_from(&url, Want::Json, Some(&from)); + wait(std::time::Duration::from_secs(2)); + let local = get(&url, Want::Json); + wait(std::time::Duration::from_secs(2)); + assert_eq!(matches!(local, State::Ready(_)), matches!(case, "ready" | "public"), "the explicit layer keeps its answer"); + let (public, origin) = document(&url, Some(&from)); + if case == "public" { + assert_eq!(origin.unwrap().reach, Reach::Public); + assert!(matches!(public, State::Ready(p) if matches!(p.as_ref(), Payload::Json(v) if v["reach"] == "public"))); + let (local, origin) = document(&url, None); + assert_eq!(origin.unwrap().reach, Reach::Local); + assert!(matches!(local, State::Ready(p) if matches!(p.as_ref(), Payload::Json(v) if v["reach"] == "local"))); + } else { + let State::Failed(error) = public else { panic!("the public style must retain its refusal") }; + if case == "retry" { + assert_eq!(retry_after(&error), Some(Duration::from_secs(30))); + let State::Failed(error) = get(&url, Want::Json) else { panic!("the explicit layer keeps its cooldown") }; + assert_eq!(retry_after(&error), Some(Duration::from_secs(120))); + let c = lock(); + let slot = c.slots.get(&url).unwrap(); + let times = [Reach::Local, Reach::Public].map(|reach| match slot.for_reach(reach) { + (Entry::Failed(_, at), stored) if reach == stored => *at, + _ => panic!("each reach keeps its own failed-at time"), + }); + if let Some(previous) = retry_times { + assert_eq!(times, previous, "alternating reads must not postpone either retry"); + } + retry_times = Some(times); + } + } + } + let sent = calls.lock().unwrap(); + assert_eq!(sent.iter().filter(|(u, _)| u == &url).count(), 2, "one fetch under each reach, not one per frame: {sent:?}"); + drop(sent); + if case == "retry" { + // Move each recorded failure just past its own original wait, without sleeping + // or changing production clocks. The other reach's cooldown remains usable. + for (reach, count) in [(Reach::Public, 3), (Reach::Local, 4)] { + let from = (reach == Reach::Public).then_some(&from); + let State::Failed(error) = get_from(&url, Want::Json, from) else { panic!("the cooldown is still active") }; + let expired = Instant::now() - retry_wait(&error, RETRY) - Duration::from_secs(1); + store_from(&url, Entry::Failed(error, expired), reach); + assert!(matches!(get_from(&url, Want::Json, from), State::Pending), "retry at this reach's deadline"); + wait(std::time::Duration::from_secs(2)); + assert_eq!(calls.lock().unwrap().iter().filter(|(u, _)| u == &url).count(), count); + } + } + } + } + + /// A pending or dropped restricted fetch cannot blank a completed explicit layer. + /// Forgetting a resource or clearing the cache removes both retained outcomes. + #[test] + fn a_restricted_read_preserves_the_explicit_answer_while_pending_or_dropped() { + use crate::address::{Origin, Reach}; + let _turn = turn(); + let (tx, rx) = std::sync::mpsc::channel(); + set_fetcher_with_origin(move |url, ceiling, done, _| { + if ceiling == Reach::Public { + tx.send(done).unwrap(); + } else { + done(Ok(Fetched { bytes: b"{}".to_vec(), origin: Origin { url, reach: Reach::Local } })); + } + }); + let from = Origin::named("https://public-style.example/style.json"); + for clear_all in [false, true] { + let url = format!("https://pending-reach.example/{clear_all}"); + get(&url, Want::Json); + wait(std::time::Duration::from_secs(2)); + assert!(matches!(get_from(&url, Want::Json, Some(&from)), State::Pending)); + let pending = rx.recv_timeout(std::time::Duration::from_secs(2)).unwrap(); + assert!(matches!(get(&url, Want::Json), State::Ready(_))); + assert!(peek(&url).is_some()); + assert!(peek_from(&url, Some(&from)).is_none(), "public readers cannot see retained local bytes"); + drop(pending); + wait(std::time::Duration::from_secs(2)); + assert!(matches!(get(&url, Want::Json), State::Ready(_)), "dropping the public fetch preserves the local layer"); + assert!(matches!(get_from(&url, Want::Json, Some(&from)), State::Pending)); + rx.recv_timeout(std::time::Duration::from_secs(2)).unwrap()(Err("fixture destination refused".into())); + wait(std::time::Duration::from_secs(2)); + assert!(matches!(get_from(&url, Want::Json, Some(&from)), State::Failed(_))); + if clear_all { + clear(); + } else { + forget(&url); + } + assert!(peek(&url).is_none(), "the retained local answer was forgotten"); + assert!(matches!(get_from(&url, Want::Json, Some(&from)), State::Pending), "the retained public failure was forgotten"); + rx.recv_timeout(std::time::Duration::from_secs(2)).unwrap()(Err("fixture destination refused".into())); + wait(std::time::Duration::from_secs(2)); + } + } + + /// The native response's effective origin keeps the selected saved S3 account for + /// relative documents, without transferring that binding to another server. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn native_destination_origins_keep_saved_s3_selection() { + let _turn = turn(); + let base = serve(vec!["HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}".into()]); + let binding = "0123456789abcdef"; + let name = "destination-origin-binding"; + crate::s3::connect_saved( + crate::s3::Connection { + name: name.into(), + endpoint: Some(base.clone()), + region: "us-east-1".into(), + buckets: vec!["bucket".into()], + credentials: None, + }, + binding, + "destination-account", + ) + .unwrap(); + let bound = rustgis_data::s3_address::bind(binding, &base, "s3://bucket/styles/style.json").unwrap(); + let got = native::fetch_answer(&native::agent(), &bound, crate::address::Reach::Local, false, &[]).unwrap(); + let reported = crate::address::Origin::named(&crate::s3::prepare(&bound).unwrap().url).retain_binding(&bound).unwrap(); + assert_eq!(reported, got.origin, "the no-redirect replacement adapter retains the same S3 selection"); + let child = crate::template::join(&got.origin.url, "tiles.json"); + let selected = rustgis_data::s3_address::parse(&child).unwrap(); + assert!(selected.is_some(), "a relative document must retain the selected S3 binding"); + assert_eq!(selected.unwrap().binding, binding); + assert_eq!(crate::s3::prepare(&child).unwrap().url, format!("{base}/bucket/styles/tiles.json")); + let external = crate::template::join(&got.origin.url, "https://other-origin.example/tiles.json"); + assert!(rustgis_data::s3_address::parse(&external).unwrap().is_none()); + crate::s3::disconnect(name); + // Joining is independent of the live login registry. A removed selection keeps the + // child's address and fails preparation, rather than substituting the parent. + assert_eq!(crate::template::join(&got.origin.url, "tiles.json"), child); + assert!(crate::s3::prepare(&child).is_err()); + } + + /// A cached DNS refusal reaches the drawing, while ordinary missing tiles stay quiet. + #[test] + fn cached_destination_failures_reach_raster_drawing() { + let _turn = turn(); + let from = crate::address::Origin::named("https://public-document.example/style.json"); + set_fetcher_with_origin(|_, _, done, _| done(Err("fixture resolves to the local network".into()))); + for (case, reach) in + [("local", crate::address::Reach::Local), ("public", crate::address::Reach::Public), ("missing", crate::address::Reach::Public)] + { + let url = format!("http://cached-draw.example/{case}"); + let entry = if case == "missing" { + Entry::Failed("http status: 404".into(), Instant::now()) + } else { + let mut image = Pixmap::new(1, 1).unwrap(); + image.fill(tiny_skia::Color::from_rgba8(255, 0, 0, 255)); + Entry::Ready(Arc::new(Payload::Image(image)), 4, Some(crate::address::Origin { url: url.clone(), reach })) + }; + let ceiling = if case == "local" { crate::address::Reach::Local } else { crate::address::Reach::Public }; + store_from(&url, entry, ceiling); + draw_cached_raster(&url, from.clone()); + wait(std::time::Duration::from_secs(2)); + let (image, drawn) = draw_cached_raster(&url, from.clone()); + assert_eq!(drawn.error.is_some(), case == "local", "{case}: {:?}", drawn.error); + assert_eq!(image.pixel(32, 32).unwrap().alpha() > 0, case == "public"); + } + } + + /// Partial WMTS composites retain both their useful pixels and the refused-component + /// warning, including the short-lived composite cache hit. + #[test] + fn partial_warp_destinations_report_refused_components() { + use rustgis_data::ogc::warp::{Grid, Matrix, Warp}; + let _turn = turn(); + set_fetcher_with_origin(|_, _, done, _| done(Err("fixture resolves to the local network".into()))); + let warp = Warp { + epsg: 4326, + grid: Grid::Tiles(vec![Matrix { + id: "m".into(), + left: -180.0, + top: 90.0, + cell: 180.0 / 256.0, + tile_w: 256, + tile_h: 256, + cols: 2, + rows: 1, + }]), + area: None, + systems: Vec::new(), + }; + let template = warp.address("http://partial-warp.example/{nm}/{ncol}/{nrow}"); + let url = crate::template::expand(&template, crate::template::TileId { z: 0, x: 0, y: 0 }, 256, false, &[]); + let (base, _, spec) = Warp::parse(&url).unwrap(); + let half = crate::template::HALF_WORLD; + let plan = spec.plan(base, [-half, -half, half, half], 256, &crate::warp::transformer(4326).unwrap()); + assert_eq!(plan.parts.len(), 2); + for (i, part) in plan.parts.iter().enumerate() { + let mut image = Pixmap::new(1, 1).unwrap(); + image.fill(tiny_skia::Color::from_rgba8(255, 0, 0, 255)); + let reach = if i == 0 { crate::address::Reach::Public } else { crate::address::Reach::Local }; + store( + &part.url, + Entry::Ready(Arc::new(Payload::Image(image)), 4, Some(crate::address::Origin { url: part.url.clone(), reach })), + ); + } + let from = crate::address::Origin::named("https://public-document.example/style.json"); + draw_cached_raster(&url, from.clone()); + wait(std::time::Duration::from_secs(2)); + for _ in 0..2 { + let (image, drawn) = draw_cached_raster(&url, from.clone()); + assert!(image.pixels().iter().any(|p| p.alpha() > 0), "valid component still draws"); + assert!(drawn.error.as_deref().is_some_and(|e| e.contains("local network")), "{:?}", drawn.error); + } + } + + /// A public style's WMS composite must not regain a local allowance for its parts or + /// reuse a composite made explicitly from local answers. Cache seeding performs no I/O. + #[test] + fn warped_destinations_keep_the_document_boundary() { + let _turn = turn(); + let url = "http://warped-local-destination.example/wms#rustgis-warp=0/0/0;crs=4326;map=east"; + let (base, (z, x, y), warp) = rustgis_data::ogc::warp::Warp::parse(url).unwrap(); + let extent = crate::template::TileId { z, x, y }.extent(); + let transformer = crate::warp::transformer(warp.epsg).unwrap(); + let plan = warp.plan(base, [extent.min_x, extent.min_y, extent.max_x, extent.max_y], 256, &transformer); + assert!(!plan.parts.is_empty(), "the fixture must require source pictures"); + for part in plan.parts { + let origin = crate::address::Origin { url: part.url.clone(), reach: crate::address::Reach::Local }; + store(&part.url, Entry::Ready(Arc::new(Payload::Image(Pixmap::new(1, 1).unwrap())), 4, Some(origin))); + } + let from = crate::address::Origin::named("https://public-document.example/style.json"); + set_fetcher_with_origin(|_, ceiling, done, _| { + assert_eq!(ceiling, crate::address::Reach::Public); + done(Err("fixture resolves to the local network".into())); + }); + assert!(matches!(get_tile(url, Want::Image), State::Ready(_)), "explicit local composition still works"); + assert!(matches!(get_tile_from(url, Want::Image, Some(&from)), State::Pending), "a cached local composite needs restricted parts"); + wait(std::time::Duration::from_secs(2)); + assert!( + matches!(get_tile_from(url, Want::Image, Some(&from)), State::Failed(e) if e.contains("local network")), + "composition must inherit public reach and report its failed restricted fetch" + ); + } + + /// Replaces only the fixture's public DNS names; all connections, if a check regresses, + /// can reach only the listener this test just created. + #[cfg(not(target_arch = "wasm32"))] + #[derive(Debug)] + pub(super) struct LocalAnswer(pub(super) std::net::SocketAddr); + + #[cfg(not(target_arch = "wasm32"))] + impl ureq::unversioned::resolver::Resolver for LocalAnswer { + fn resolve( + &self, + uri: &ureq::http::Uri, + config: &ureq::config::Config, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> Result { + if uri.host().is_some_and(|host| host.ends_with(".example")) { + let mut answer = self.empty(); + answer.push(self.0); + Ok(answer) + } else { + ureq::unversioned::resolver::DefaultResolver::default().resolve(uri, config, timeout) + } + } + } + + /// GET, HEAD and downloads use the same checked resolver and recheck redirected names. + /// A refusal must occur before connecting, not after a response or timeout. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn native_destinations_filter_dns_before_connecting() { + let _turn = turn(); + let target = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + target.set_nonblocking(true).unwrap(); + let answer = target.local_addr().unwrap(); + let public = format!("http://destinations.example:{}/tile", answer.port()); + for config in [native::agent().config().clone(), native::download_agent(std::time::Duration::from_secs(1)).config().clone()] { + assert!(config.proxy().is_none(), "a proxy must not bypass checked DNS"); + let agent = native::checked_agent(config, LocalAnswer(answer), true); + assert!(native::get(&agent, &public, &[]).unwrap_err().contains("resolves to")); + assert!(native::head_with(&agent, &public).unwrap_err().contains("resolves to")); + } + assert_eq!(target.accept().unwrap_err().kind(), std::io::ErrorKind::WouldBlock, "no forbidden connection was attempted"); + + let split = serve(vec!["HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntile".into()]); + let listener: std::net::SocketAddr = split.strip_prefix("http://").unwrap().parse().unwrap(); + let explicit = format!("http://self-hosted.example:{}/tiles", listener.port()); + let agent = native::checked_agent(native::agent().config().clone(), LocalAnswer(listener), false); + let fetched = native::fetch_answer(&agent, &explicit, crate::address::Reach::Local, false, &[]).unwrap(); + assert_eq!(fetched.bytes, b"tile"); + assert_eq!(fetched.origin.url, explicit, "preserve the hostname used for TLS and relative references"); + assert_eq!(fetched.origin.reach, crate::address::Reach::Local); + + // Explicit local services still work, including a checked same-origin redirect. + let local = serve(vec![ + "HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".into(), + "HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntile".into(), + ]); + assert_eq!(download_blocking_limited(&local, 4).unwrap(), b"tile"); + } + /// An agent that fetches `https://` addresses over plain TCP, as if TLS were there, so a test /// server on this computer can stand for an HTTPS server. #[cfg(not(target_arch = "wasm32"))] @@ -3468,9 +4692,9 @@ mod tests { #[derive(Debug)] struct Claimed(T); impl Connector for ClaimsTls { - type Out = Claimed; + type Out = Box; fn connect(&self, _: &ConnectionDetails, chained: Option) -> Result, ureq::Error> { - Ok(chained.map(Claimed)) + Ok(chained.map(|transport| Claimed(transport).boxed())) } } impl Transport for Claimed { @@ -3490,12 +4714,7 @@ mod tests { true } } - let config = ureq::Agent::config_builder() - .http_status_as_error(false) - .max_redirects(0) - .timeout_global(Some(std::time::Duration::from_secs(10))) - .build(); - ureq::Agent::with_parts(config, TcpConnector::default().chain(ClaimsTls), ureq::unversioned::resolver::DefaultResolver::default()) + native::fixture_transport_agent(TcpConnector::default().chain(ClaimsTls)) } /// Never from HTTPS down to plain HTTP, however the scheme is written: the HTTP client reads diff --git a/crates/tiles/src/draw.rs b/crates/tiles/src/draw.rs index 92326ced6c..2576eca554 100644 --- a/crates/tiles/src/draw.rs +++ b/crates/tiles/src/draw.rs @@ -52,6 +52,20 @@ pub struct Drawn { pub error: Option, } +/// Retains each distinct refusal alongside any earlier source or sprite warning. +pub(crate) fn add_error(to: &mut Option, error: String) { + for message in error.split("; ") { + match to { + Some(previous) if !previous.split("; ").any(|kept| kept == message) => { + previous.push_str("; "); + previous.push_str(message); + } + None => *to = Some(message.to_string()), + _ => {} + } + } +} + /// One tile's worth of drawing: the tile data, where that tile is, and the square it stands in for. #[derive(Clone)] pub(crate) struct Item { @@ -61,17 +75,36 @@ pub(crate) struct Item { pub(crate) clip: Extent, } +/// Selected tiles and whether their vector data leaves part of the view without a mesh. +/// A visible style background needs ordinary drawing when mesh squares cannot cover it. +pub(crate) struct TileSelection { + pub(crate) items: Vec, + pub(crate) mesh_gaps: bool, + /// A failed or partial read may supply more pixels after the tile cache changes. + pub(crate) retry: bool, +} + /// What of a resolved style was left out, as a drawing's error (the style still draws): what /// resolving it left out, and any sprites that couldn't be read, for the images shown at `frame`. /// A drawing takes its images at the frame's density; a mesh (`mesh`) its pattern fills at 1×. pub(crate) fn left_out(r: &Resolved, frame: &Frame, mesh: bool) -> Option { + left_out_at(r, frame, frame, mesh) +} + +/// Reports icons at their live frame and patterns and relief at their prepared shape frame. +pub(crate) fn left_out_at(r: &Resolved, frame: &Frame, shapes: &Frame, mesh: bool) -> Option { let zoom = template::style_zoom(frame.m_per_px * f64::from(frame.ppp)); - let shown = |draws: fn(&StyleLayer) -> bool| r.style.layers.iter().any(|l| l.visible_at(zoom) && draws(l)); + let shape_zoom = template::style_zoom(shapes.m_per_px * f64::from(shapes.ppp)); + let shown = |zoom, draws: fn(&StyleLayer) -> bool| r.style.layers.iter().any(|l| l.visible_at(zoom) && draws(l)); let hi_dpi = frame.ppp >= 1.5; - let unread = sprite_notes(shown(StyleLayer::draws_icons), shown(StyleLayer::draws_patterns), hi_dpi, hi_dpi && !mesh, |hi, what| { - sprite::unread_note(&r.sprites, r.style.sprites_listed, &sprite::load(&r.sprites, hi), what) - }); - let relief = relief_note(&r.style.layers, zoom); + let unread = sprite_notes( + shown(zoom, StyleLayer::draws_icons), + shown(shape_zoom, StyleLayer::draws_patterns), + hi_dpi, + shapes.ppp >= 1.5 && !mesh, + |hi, what| sprite::unread_note(&r.sprites, r.style.sprites_listed, &r.load_sprites(hi), what), + ); + let relief = relief_note(&r.style.layers, shape_zoom); let all: Vec<&str> = r.left_out.iter().map(String::as_str).chain(unread.iter().map(String::as_str)).chain(relief.as_deref()).collect(); (!all.is_empty()).then(|| all.join("; ")) } @@ -131,15 +164,15 @@ pub fn draw(pm: &mut Pixmap, spec: &Spec, opacity: f32, frame: &Frame, labels: b } }; out.attribution = r.attribution.clone(); - out.error = left_out(&r, frame, false); out.pending |= r.waiting; if !(frame.m_per_px.is_finite() && frame.m_per_px > 0.0) || opacity <= 0.0 { + out.error = left_out(&r, frame, false); return out; } let (w, h) = (pm.width(), pm.height()); // A style drawing vector shapes draws them from tile pictures drawn once (see `picture`): // a map moving across them only puts them in place. - if let Some(pictures) = crate::picture::cover(&r, frame, &frame.extent(w, h), None, &mut out.pending) { + if let Some(pictures) = crate::picture::cover(&r, frame, &frame.extent(w, h), None, &mut out.pending, &mut out.error) { for it in &pictures { draw_raster(pm, it, frame, opacity); } @@ -148,6 +181,7 @@ pub fn draw(pm: &mut Pixmap, spec: &Spec, opacity: f32, frame: &Frame, labels: b } return out; } + out.error = left_out(&r, frame, false); let mut layer_pm = if opacity < 0.999 { Pixmap::new(w, h) } else { None }; let target: &mut Pixmap = match layer_pm.as_mut() { Some(p) => p, @@ -172,7 +206,9 @@ pub(crate) enum Pass { } /// Draws the layers of a resolved style for `pass` into `target` (a `w × h` pixmap at `frame`) -/// from the tiles over `view`, collecting labels into `out`. +/// from the tiles over `view`, collecting labels into `out`. Shape-only selection without a +/// target includes neighbouring vector tiles whose paint can reach the viewport. +/// Returns whether a failed or partial tile read could add pixels when the cache changes. pub(crate) fn draw_layers( mut target: Option<&mut Pixmap>, (w, h): (u32, u32), @@ -181,7 +217,8 @@ pub(crate) fn draw_layers( frame: &Frame, pass: Pass, out: &mut Drawn, -) { +) -> bool { + let mut retry = false; let zoom = template::style_zoom(frame.m_per_px * f64::from(frame.ppp)); let mut items: HashMap> = HashMap::new(); let mut boxes: Boxes = HashMap::new(); @@ -191,7 +228,7 @@ pub(crate) fn draw_layers( let mut sprite_state: Option>> = None; let mut sprite = |pending: &mut bool| -> Option> { sprite_state - .get_or_insert_with(|| match sprite::load(&r.sprites, frame.ppp >= 1.5) { + .get_or_insert_with(|| match r.load_sprites(frame.ppp >= 1.5) { Load::Ready(s) => Some(s), Load::Pending => { *pending = true; @@ -207,7 +244,7 @@ pub(crate) fn draw_layers( Pass::Shapes => layer.kind != LayerKind::Symbol, Pass::Labels => layer.kind == LayerKind::Symbol, }; - if !wanted || !layer.visible_at(zoom) { + if !wanted || !layer.visible_at(zoom) || !drawable(layer) { continue; } if layer.kind == LayerKind::Background { @@ -227,11 +264,28 @@ pub(crate) fn draw_layers( } if !items.contains_key(src_id) { let Some(src) = r.style.sources.iter().find(|s| s.0 == src_id).map(|s| &s.1) else { continue }; - // Hillshade reads the heights a pixel past the view's edges: those tiles too. - let pad = 2.0 * frame.m_per_px; + // Hillshade needs neighbouring heights. Picture diagnostics also need vector + // neighbours whose wide shapes can reach the current viewport, not all of a + // prepared picture's padding. Raster layers keep the exact viewport. + let pixels = if target.is_none() && pass == Pass::Shapes && matches!(src, Source::Vector { .. }) { + r.style + .layers + .iter() + .filter(|l| l.source.as_deref() == Some(src_id) && l.visible_at(zoom) && drawable(l)) + .map(|l| match l.kind { + LayerKind::Line | LayerKind::Circle => overhang(l, zoom) * frame.ppp, + LayerKind::Fill | LayerKind::FillExtrusion => 2.0 * frame.ppp.max(1.0), + _ => 0.0, + }) + .fold(0.0f32, f32::max) + } else { + 0.0 + }; + let pad = if matches!(src, Source::RasterDem { .. }) { 2.0 * frame.m_per_px } else { f64::from(pixels) * frame.m_per_px }; let reach = Extent::new(view.min_x - pad, view.min_y - pad, view.max_x + pad, view.max_y + pad); - let list = tiles_for(src, if matches!(src, Source::RasterDem { .. }) { &reach } else { view }, frame, zoom, &mut out.pending); - items.insert(src_id.to_string(), list); + let selected = tiles_in(src, &reach, None, frame, zoom, &mut out.pending, &mut out.error); + retry |= selected.retry; + items.insert(src_id.to_string(), selected.items); } let Some(list) = items.get(src_id) else { continue }; match layer.kind { @@ -252,9 +306,6 @@ pub(crate) fn draw_layers( } LayerKind::Fill | LayerKind::FillExtrusion | LayerKind::Line | LayerKind::Circle => { let Some(t) = target.as_deref_mut() else { continue }; - if !drawable(layer) { - continue; - } // A pattern fill without its sprite isn't drawn: its fallback colour is black. let sheet = if pattern(layer).is_some() { match sprite(&mut out.pending) { @@ -289,6 +340,7 @@ pub(crate) fn draw_layers( } // The cap keeps the labels placed first, whichever layers they came from. crate::symbols::keep_first(&mut out.labels, crate::symbols::MAX_LABELS); + retry } /// One raster tile placed on the map, for a renderer that draws each tile itself (the GPU map @@ -332,6 +384,7 @@ pub fn place_raster(spec: &Spec, opacity: f32, frame: &Frame, view: &Extent, rea place_as(spec, opacity, frame, view, reach, None, false) } +/// Places raster or prepared vector pictures and retains both shape and label refusals. fn place_as( spec: &Spec, opacity: f32, @@ -356,23 +409,29 @@ fn place_as( } }; out.attribution = r.attribution.clone(); - out.error = left_out(&r, frame, false); out.pending |= r.waiting; if !(frame.m_per_px.is_finite() && frame.m_per_px > 0.0) || opacity <= 0.0 { + out.error = left_out(&r, frame, false); return Some(out); } // Without pictures, a style drawing shapes is left to the check below (no raster layers only). - if let Some(pictures) = pictures.then(|| crate::picture::cover(&r, frame, view, Some(reach), &mut out.pending)).flatten() { + if let Some(pictures) = + pictures.then(|| crate::picture::cover(&r, frame, view, Some(reach), &mut out.pending, &mut out.error)).flatten() + { let opacity = opacity.clamp(0.0, 1.0); out.tiles = pictures.into_iter().map(|it| Placement { picture: it.payload, at: it.at, clip: it.clip, opacity }).collect(); if let Some(size) = labels { let mut d = Drawn::default(); draw_layers(None, size, &frame.extent(size.0, size.1), &r, frame, Pass::Labels, &mut d); + if let Some(refused) = d.error { + add_error(&mut out.error, refused); + } out.labels = d.labels; out.pending |= d.pending; } return Some(out); } + out.error = left_out(&r, frame, false); let zoom = template::style_zoom(frame.m_per_px * f64::from(frame.ppp)); let shown: Vec<&StyleLayer> = r.style.layers.iter().filter(|l| l.visible_at(zoom) && l.kind != LayerKind::Other).collect(); // Raster layers only; several are composited together before the layer's opacity applies. @@ -384,7 +443,8 @@ fn place_as( let Some(src_id) = layer.source.as_deref() else { continue }; if !items.contains_key(src_id) { let Some(src) = r.style.sources.iter().find(|s| s.0 == src_id).map(|s| &s.1) else { continue }; - items.insert(src_id, tiles_in(src, view, Some(reach), frame, zoom, &mut out.pending)); + let list = tiles_in(src, view, Some(reach), frame, zoom, &mut out.pending, &mut out.error); + items.insert(src_id, list.items); } let o = (layer.raster_opacity(zoom) * opacity).clamp(0.0, 1.0); for it in items.get(src_id).into_iter().flatten() { @@ -450,7 +510,7 @@ pub fn pick(spec: &Spec, x: f64, y: f64, m_per_px: f64, tol_px: f32, limit: usiz if !matches!(src, Source::Vector { .. }) { continue; } - items.insert(src_id.to_string(), tiles_for(src, &view, &frame, zoom, &mut pending)); + items.insert(src_id.to_string(), tiles_for(src, &view, &frame, zoom, &mut pending, &mut None)); } let Some(list) = items.get(src_id) else { continue }; // Every tile within reach (a feature near a tile's edge may be in its neighbour), the real @@ -544,65 +604,122 @@ fn hits(f: &crate::mvt::Feature, kind: LayerKind, p: [f32; 2], reach: f32) -> bo } /// The tiles of a source in view, with stand-ins for those not here yet. -fn tiles_for(src: &Source, view: &Extent, frame: &Frame, zoom: f32, pending: &mut bool) -> Vec { - tiles_in(src, view, None, frame, zoom, pending) +fn tiles_for(src: &Source, view: &Extent, frame: &Frame, zoom: f32, pending: &mut bool, error: &mut Option) -> Vec { + tiles_in(src, view, None, frame, zoom, pending, error).items } /// The tiles of a source in view and out to `reach` (those in the cache only), with stand-ins -/// for those not here yet. -pub(crate) fn tiles_in(src: &Source, view: &Extent, reach: Option<&Extent>, frame: &Frame, zoom: f32, pending: &mut bool) -> Vec { - let (tiles, min_z, max_z, want, z) = match src { - Source::Raster { tiles, tile_size, min_zoom, max_zoom, .. } | Source::RasterDem { tiles, tile_size, min_zoom, max_zoom, .. } => { +/// for those not here yet. Expanded destinations retain the supplying document's reach; +/// refused tiles are counted in `error` and network history without requesting or reusing bytes. Valid +/// tiles still draw, even when another shard of the same source is refused. +pub(crate) fn tiles_in( + src: &Source, + view: &Extent, + reach: Option<&Extent>, + frame: &Frame, + zoom: f32, + pending: &mut bool, + error: &mut Option, +) -> TileSelection { + let (tiles, from, min_z, max_z, want, z) = match src { + Source::Raster { tiles, from, tile_size, min_zoom, max_zoom, .. } + | Source::RasterDem { tiles, from, tile_size, min_zoom, max_zoom, .. } => { let z = template::zoom_for(frame.m_per_px * f64::from(frame.ppp), *tile_size, *max_zoom); - (tiles, *min_zoom, *max_zoom, Want::Image, z) + (tiles, from, *min_zoom, *max_zoom, Want::Image, z) } - Source::Vector { tiles, min_zoom, max_zoom, .. } => { - (tiles, *min_zoom, *max_zoom, Want::Vector, (zoom.floor() as u8).min(*max_zoom)) + Source::Vector { tiles, from, min_zoom, max_zoom, .. } => { + (tiles, from, *min_zoom, *max_zoom, Want::Vector, (zoom.floor() as u8).min(*max_zoom)) } - Source::Other => return Vec::new(), + Source::Other => return TileSelection { items: Vec::new(), mesh_gaps: true, retry: false }, }; if tiles.is_empty() || z < min_z || zoom + 0.5 < f32::from(min_z) { - return Vec::new(); + return TileSelection { items: Vec::new(), mesh_gaps: true, retry: false }; } // A mosaic over half the world wide (the globe's, a world projection's) takes the zoom whose // tiles all fit, as its far edges are in view; a flat view keeps those nearest its centre. let z = if view.width() > template::HALF_WORLD { template::fitting_zoom(view, z, min_z) } else { z }; - let url = |t: TileId| { + let url = |t: TileId| -> Result, (String, String)> { let i = (t.x as usize).wrapping_add(t.y as usize) % tiles.len(); - tiles.get(i).map(|tpl| template::expand(tpl, t, 256, false, &[])) + let Some(tpl) = tiles.get(i) else { return Ok(None) }; + let expanded = template::expand(tpl, t, 256, false, &[]); + // Archive tile keys are made here from the archive the layer explicitly names. They + // aren't network destinations; their HTTP range reads pass the transport checks. + let checked = if let Some(from) = from { + // A document cannot supply internal archive keys, even with local reach. + from.check(&expanded) + } else if !crate::pmtiles::is_tile(&expanded) && !crate::mbtiles::is_tile(&expanded) && !crate::cog::is_tile(&expanded) { + crate::address::check(&expanded, None) + } else { + Ok(()) + }; + checked.map_err(|e| (expanded.clone(), e))?; + Ok(Some(expanded)) }; + // Ancestors and children are cache-only, but an invalid destination must not read a tile + // previously cached by an explicitly local layer either. + let cached_url = |t| url(t).ok().flatten(); let mut exact = Vec::new(); let mut stand_ins = Vec::new(); + // Like prepared pictures, a mesh cannot cover a view larger than selection's tile cap. + let span = TileId::span(z); + let mut mesh_gaps = ((view.width() / span).ceil() + 1.0) * ((view.height() / span).ceil() + 1.0) > template::MAX_TILES as f64; + let mut omitted = 0; + let mut first_error = None; + let mut retry = false; + let mut partial = 0; + let mut first_partial = None; // A missing tile is stood in for by its nearest ancestor in the cache and, for raster tiles, // by its children in the cache, the ancestor only in the quarters they leave: zooming in, the // tiles of the zoom being passed through are often missing while those of the zoom it ends // at are already here, and they are sharper. Nothing is drawn twice, so a see-through layer // stays even. let stand_in = |out: &mut Vec, p: &Placed| { - let parent = ancestor(p, min_z, &url); + let parent = ancestor(p, min_z, &cached_url, from.as_ref()); if want != Want::Image || p.id.z >= max_z { out.extend(parent); return; } for c in p.id.children() { let cp = Placed { id: c, wx: p.wx * 2 + i64::from(c.x & 1) }; - match url(c).and_then(|u| cache::peek(&u)) { + match cached_url(c).and_then(|u| cache::peek_from(&u, from.as_ref())) { Some(payload) => out.push(Item { payload, at: cp.extent(), z: c.z, clip: cp.extent() }), None => out.extend(parent.as_ref().map(|a| Item { clip: cp.extent(), ..a.clone() })), } } }; let mut placed = reach.map(|e| template::covering(e, z)).unwrap_or_default(); - if placed.is_empty() { + if placed.is_empty() || placed.len() >= template::MAX_TILES { placed = template::covering(view, z); } + mesh_gaps |= placed.is_empty(); for p in placed { - let Some(u) = url(p.id) else { continue }; + let area = p.extent(); + // A tile merely touching the view's edge has no visible pixels. + let visible = area.min_x < view.max_x && area.max_x > view.min_x && area.min_y < view.max_y && area.max_y > view.min_y; + let u = match url(p.id) { + Ok(Some(u)) => u, + Ok(None) => continue, + Err((url, e)) => { + if visible { + cache::note_refusal(&url, &e); + mesh_gaps = true; + omitted += 1; + first_error.get_or_insert(e); + } + continue; + } + }; // Past the view, only what the cache already has. - let state = if p.extent().intersects(view) { - cache::get_tile(&u, want) + let state = if visible { + let (state, note) = cache::tile_noted(&u, want, from.as_ref()); + if let Some(note) = note { + retry = true; + partial += 1; + first_partial.get_or_insert(note); + } + state } else { - match cache::peek(&u) { + match cache::peek_from(&u, from.as_ref()) { Some(payload) => State::Ready(payload), None => { stand_in(&mut stand_ins, &p); @@ -611,24 +728,43 @@ pub(crate) fn tiles_in(src: &Source, view: &Extent, reach: Option<&Extent>, fram } }; match state { - State::Ready(payload) => exact.push(Item { payload, at: p.extent(), z: p.id.z, clip: p.extent() }), - State::Failed(_) => {} + State::Ready(payload) => { + mesh_gaps |= visible && !matches!(payload.as_ref(), Payload::Vector(_)); + exact.push(Item { payload, at: area, z: p.id.z, clip: area }); + } + State::Failed(e) => { + retry |= visible; + mesh_gaps |= visible; + if !cache::routine_tile_failure(&e) { + omitted += 1; + first_error.get_or_insert(e); + } + } State::Pending => { *pending = true; + let first = stand_ins.len(); stand_in(&mut stand_ins, &p); + // Vector stand-ins are a single ancestor covering this tile's whole square. + mesh_gaps |= visible && !stand_ins.get(first).is_some_and(|it| matches!(it.payload.as_ref(), Payload::Vector(_))); } } } // Stand-ins first, so the real tiles draw over their edges. stand_ins.extend(exact); - stand_ins + if let Some(reason) = first_error { + add_error(error, format!("{omitted} tiles left out: {reason}")); + } + if let Some(reason) = first_partial { + add_error(error, format!("{partial} tiles are partial (first: {reason})")); + } + TileSelection { items: stand_ins, mesh_gaps, retry } } /// The nearest ancestor of a missing tile already in the cache, clipped to the missing tile. -fn ancestor(p: &Placed, min_z: u8, url: &dyn Fn(TileId) -> Option) -> Option { +fn ancestor(p: &Placed, min_z: u8, url: &dyn Fn(TileId) -> Option, from: Option<&crate::address::Origin>) -> Option { for lv in 1..=p.id.z.saturating_sub(min_z).min(8) { let a = p.id.ancestor(lv)?; - if let Some(payload) = url(a).and_then(|u| cache::peek(&u)) { + if let Some(payload) = url(a).and_then(|u| cache::peek_from(&u, from)) { let ap = Placed { id: a, wx: p.wx.div_euclid(1 << lv) }; return Some(Item { payload, at: ap.extent(), z: a.z, clip: p.extent() }); } @@ -788,19 +924,27 @@ fn boxes_of(known: &mut Boxes, it: &Item, layer: &StyleLayer) -> Arc f32 { let ctx = FeatureCtx::zoom_only(zoom); let reach = match layer.kind { LayerKind::Line => { - let k = if layer.line_join(&ctx) == crate::style::LineJoin::Miter { 2.0 } else { 0.75 }; - layer.line_width(&ctx) * k + let miter = layer.layout_uses_features("line-join") || layer.line_join(&ctx) == crate::style::LineJoin::Miter; + let Some(width) = layer.paint_width_bound("line-width", zoom, 1.0) else { return 512.0 }; + width * if miter { 2.0 } else { 0.75 } + } + LayerKind::Circle => { + let (Some(radius), Some(stroke)) = + (layer.paint_width_bound("circle-radius", zoom, 5.0), layer.paint_width_bound("circle-stroke-width", zoom, 0.0)) + else { + return 512.0; + }; + radius + stroke } - LayerKind::Circle => layer.circle_radius(&ctx) + layer.circle_stroke_width(&ctx), _ => 0.0, }; - if reach.is_finite() { reach.clamp(OVERHANG, 512.0) } else { OVERHANG } + if reach.is_finite() { reach.clamp(OVERHANG, 512.0) } else { 512.0 } } fn draw_vector(pm: &mut Pixmap, it: &Item, layer: &StyleLayer, frame: &Frame, zoom: f32, sheet: Option<&Sprite>, boxes: &[[f32; 4]]) { @@ -1174,6 +1318,18 @@ fn area(r: &[(f32, f32)]) -> f32 { #[cfg(test)] pub(crate) mod tests { + #[test] + fn refusal_counts_are_not_substring_duplicates() { + let mut error = Some("12 tiles left out: refused".to_string()); + super::add_error(&mut error, "2 tiles left out: refused".into()); + assert_eq!(error.as_deref(), Some("12 tiles left out: refused; 2 tiles left out: refused")); + let combined = error.unwrap(); + let mut error = Some("sprite left out".to_string()); + super::add_error(&mut error, combined.clone()); + super::add_error(&mut error, combined); + assert_eq!(error.as_deref(), Some("sprite left out; 12 tiles left out: refused; 2 tiles left out: refused")); + } + #[test] fn a_relief_layer_rustgis_cant_colour_is_said_the_same_at_every_zoom() { let layer = |id: &str, extra: serde_json::Value| { @@ -1531,6 +1687,72 @@ pub(crate) mod tests { let _ = std::fs::remove_dir_all(dir); } + #[test] + fn bounded_shape_widths_keep_their_conservative_reach() { + for (radius, expected) in [ + (serde_json::json!(["min", 2, ["get", "radius"]]), 16.0), + (serde_json::json!(["max", 0, ["min", 30, ["get", "radius"]]]), 30.0), + (serde_json::json!(["case", ["has", "radius"], 30, 40]), 40.0), + (serde_json::json!(["step", ["get", "radius"], 20, 10, 30]), 30.0), + (serde_json::json!(["interpolate", ["linear"], ["get", "radius"], 0, 20, 100, 30]), 512.0), + (serde_json::json!(["get", "radius"]), 512.0), + // NaN is ignored by min: it must not masquerade as a finite cap on the other input. + (serde_json::json!(["min", ["sqrt", -1], ["get", "radius"]]), 512.0), + ] { + let layer = StyleLayer::parse(&serde_json::json!({"id":"p", "type":"circle", "paint":{"circle-radius":radius}})).unwrap(); + assert_eq!(overhang(&layer, 4.0), expected); + for value in [ + None, + Some(crate::mvt::Value::Number(-1.0)), + Some(crate::mvt::Value::Number(500.0)), + Some(crate::mvt::Value::String("invalid".into())), + ] { + let props = value.map(|v| vec![("radius".into(), v)]).unwrap_or_default(); + let f = FeatureCtx { props: &props, ..FeatureCtx::zoom_only(4.0) }; + assert!(overhang(&layer, 4.0) >= layer.circle_radius(&f).min(512.0)); + } + } + let line = StyleLayer::parse(&serde_json::json!({"id":"r", "type":"line", "paint":{"line-width":["min", 20, ["get","width"]]}, + "layout":{"line-join":["case", ["has","width"], "miter", "round"]}})) + .unwrap(); + assert_eq!(overhang(&line, 4.0), 40.0, "a feature may choose a mitred corner"); + } + + #[test] + fn large_shape_caps_keep_the_full_margin() { + for (kind, paint) in [ + ("line", serde_json::json!({"line-width":["min",3e38,["get","width"]]})), + ( + "circle", + serde_json::json!({"circle-radius":["min",3e38,["get","radius"]],"circle-stroke-width":["min",3e38,["get","stroke"]]}), + ), + ] { + let layer = StyleLayer::parse(&serde_json::json!({"id":"p","type":kind,"paint":paint})).unwrap(); + assert_eq!(overhang(&layer, 4.0), 512.0, "combining large caps must not shrink the margin"); + } + } + + #[test] + fn interpolated_width_caps_do_not_hide_nonfinite_outputs() { + let stop = serde_json::json!(["min", 2, ["/", -1, ["get", "zero"]]]); + let radius = serde_json::json!(["min", ["interpolate", ["linear"], ["get", "t"], 0, stop, 1, stop], ["get", "radius"]]); + let layer = StyleLayer::parse(&serde_json::json!({"id":"p","type":"circle","paint":{"circle-radius":radius}})).unwrap(); + let props = vec![ + ("zero".into(), crate::mvt::Value::Number(0.0)), + ("t".into(), crate::mvt::Value::Number(0.5)), + ("radius".into(), crate::mvt::Value::Number(100.0)), + ]; + let f = FeatureCtx { props: &props, ..FeatureCtx::zoom_only(4.0) }; + assert_eq!(layer.circle_radius(&f), 100.0, "min ignores the interpolated NaN"); + assert!(overhang(&layer, 4.0) >= layer.circle_radius(&f), "the inferred cap must contain the actual painted radius"); + let radius = serde_json::json!(["interpolate", ["linear"], ["get", "t"], -2e18, -2e18, 0, 150]); + let layer = StyleLayer::parse(&serde_json::json!({"id":"p","type":"circle","paint":{"circle-radius":radius}})).unwrap(); + let props = vec![("t".into(), crate::mvt::Value::Number(-1.0))]; + let f = FeatureCtx { props: &props, ..FeatureCtx::zoom_only(4.0) }; + assert_eq!(layer.circle_radius(&f), 256.0, "finite interpolation may lose precision through cancellation"); + assert!(overhang(&layer, 4.0) >= layer.circle_radius(&f), "finite stop values alone do not prove a conservative cap"); + } + #[test] fn wide_lines_reach_in_from_past_the_pixmap() { // The top-left 64 pixels of a tile drawn 256 across; a road 120 points wide runs 100 diff --git a/crates/tiles/src/picture.rs b/crates/tiles/src/picture.rs index 3e0deb9902..fb1c96b6e9 100644 --- a/crates/tiles/src/picture.rs +++ b/crates/tiles/src/picture.rs @@ -68,6 +68,9 @@ struct Entry { /// Drawn while some of it was on its way, at this cache generation: drawn again once the /// cache has more. waiting: Option, + /// A failed or partial constituent read is reconsidered when the tile cache changes, + /// without keeping an export pending while its server is refusing it. + failed: Option, /// For a style with pattern fills, the ids of the sprites it was drawn from. sheets: Option>, } @@ -78,7 +81,7 @@ fn sprites_read(r: &Resolved, zoom: f32, ppp: f32) -> (bool, Option> if !r.style.layers.iter().any(|l| l.visible_at(zoom) && l.draws_patterns()) { return (false, None); } - match sprite::load(&r.sprites, ppp >= 1.5) { + match sprite::load_from(&r.sprites, ppp >= 1.5, r.from.as_ref()) { Load::Pending => (true, None), Load::Ready(s) => (false, Some(s.sheet_ids())), Load::Failed => (false, Some(Vec::new())), @@ -212,15 +215,19 @@ fn draws_shapes(r: &Resolved, zoom: f32) -> bool { /// The pictures of `r`'s shapes over `view`, drawing those missing, and out to `reach` (only /// those already drawn), bottom first; `None` when the style draws no vector shapes here, or the /// view is too wide for pictures (the whole world on the globe). `frame` gives the scale. -pub(crate) fn cover(r: &Arc, frame: &Frame, view: &Extent, reach: Option<&Extent>, pending: &mut bool) -> Option> { +pub(crate) fn cover( + r: &Arc, + frame: &Frame, + view: &Extent, + reach: Option<&Extent>, + pending: &mut bool, + error: &mut Option, +) -> Option> { let ppp = frame.ppp; if !(ppp.is_finite() && ppp > 0.0 && frame.m_per_px.is_finite() && frame.m_per_px > 0.0) || view.width() > template::HALF_WORLD { return None; } let zoom = template::style_zoom(frame.m_per_px * f64::from(ppp)); - if !draws_shapes(r, zoom) { - return None; - } let step = (zoom * STEPS).round(); let level = (step / STEPS).floor(); // Squares of 256 to 512 pixels: those the size of a 256-point tile at one pixel a point, of @@ -230,15 +237,33 @@ pub(crate) fn cover(r: &Arc, frame: &Frame, view: &Extent, reach: Opti let z = (level + down) as u8; let side = (512.0 * ppp * (step / STEPS - level - down).exp2()).round().clamp(16.0, MAX_SIDE) as u32; let m_per_px = TileId::span(z) / f64::from(side); + // Tile selection, visibility and retries must describe the pixels actually prepared, + // including rounding across a source zoom boundary and fractional pixel densities. + let picture_frame = Frame { left: frame.left, top: frame.top, m_per_px, ppp }; + let zoom = template::style_zoom(m_per_px * f64::from(ppp)); + if !draws_shapes(r, zoom) { + return None; + } let key = |tile| Key { style: Arc::as_ptr(r) as usize, tile, step: step as i32, ppp: ppp.to_bits() }; let generation = cache::generation(); - let (sprites_pending, read) = sprites_read(r, zoom, ppp); - // A view of more squares than a covering holds (a huge window) is drawn straight instead; - // a reach of more keeps to the view. + // Decline before starting diagnostics or reads if this view needs straight drawing. let span = TileId::span(z); if ((view.width() / span).ceil() + 1.0) * ((view.height() / span).ceil() + 1.0) > template::MAX_TILES as f64 { return None; } + if let Some(reason) = crate::draw::left_out_at(r, frame, &picture_frame, false) { + crate::draw::add_error(error, reason); + } + // Pictures include padding and may stand in at another zoom. Their preparation errors + // do not describe this viewport: gather current diagnostics without repainting shapes. + let mut visible = Drawn::default(); + draw_layers(None, (0, 0), view, r, &picture_frame, Pass::Shapes, &mut visible); + *pending |= visible.pending; + if let Some(reason) = visible.error { + crate::draw::add_error(error, reason); + } + let (sprites_pending, read) = sprites_read(r, zoom, ppp); + // A reach of more squares than a covering holds keeps to the view. let squares = reach .map(|e| template::covering(e, z)) .filter(|s| !s.is_empty() && s.len() < template::MAX_TILES) @@ -256,6 +281,7 @@ pub(crate) fn cover(r: &Arc, frame: &Frame, view: &Extent, reach: Opti Some(e) if !in_view || (e.waiting.is_none_or(|g| g == generation) + && e.failed.is_none_or(|g| g == generation) && sprites_serve(e.sheets.as_deref(), sprites_pending, read.as_deref())) => { e.used = tick; @@ -311,7 +337,8 @@ pub(crate) fn cover(r: &Arc, frame: &Frame, view: &Extent, reach: Opti let tick = s.tick; let mut added = false; for (&i, d) in todo.iter().zip(drawn) { - let (Some(slot), Some((pm, waiting))) = (found.get_mut(i), d) else { continue }; + let (Some(slot), Some(d)) = (found.get_mut(i), d) else { continue }; + let DrawnPicture { image: pm, pending: waiting, retry } = d; *pending |= waiting; let bytes = pm.data().len(); let picture = Arc::new(Payload::Image(pm)); @@ -321,6 +348,7 @@ pub(crate) fn cover(r: &Arc, frame: &Frame, view: &Extent, reach: Opti bytes, used: tick, waiting: waiting.then_some(generation), + failed: retry.then_some(generation), sheets: read.clone(), }; s.bytes += bytes; @@ -426,6 +454,13 @@ fn evict(s: &mut Store) { } } +/// One prepared picture and the pending or incomplete reads that produced it. +struct DrawnPicture { + image: Pixmap, + pending: bool, + retry: bool, +} + /// Draws the pictures of `squares`: each a `side`-pixel pixmap and whether some of it is still /// on its way, `None` for those not drawn. Natively the squares are shared among threads. With /// `until`, drawing stops once it has passed, after a picture (a round of them natively). @@ -437,7 +472,7 @@ fn draw_all( ppp: f32, reach: f64, until: Option, -) -> Vec> { +) -> Vec> { let Some(until) = until else { return draw_some(r, squares, side, m_per_px, ppp, reach) }; let round = if cfg!(target_arch = "wasm32") { 1 } else { std::thread::available_parallelism().map_or(1, |n| n.get()).min(16) }; let mut out = Vec::with_capacity(squares.len()); @@ -452,14 +487,14 @@ fn draw_all( } /// Draws the pictures of all of `squares` (see [`draw_all`]). -fn draw_some(r: &Resolved, squares: &[Placed], side: u32, m_per_px: f64, ppp: f32, reach: f64) -> Vec> { +fn draw_some(r: &Resolved, squares: &[Placed], side: u32, m_per_px: f64, ppp: f32, reach: f64) -> Vec> { let one = |p: &Placed| draw_one(r, p, side, m_per_px, ppp, reach); #[cfg(not(target_arch = "wasm32"))] { let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(squares.len()).min(16); if threads > 1 { let next = std::sync::atomic::AtomicUsize::new(0); - let out: Mutex>> = Mutex::new((0..squares.len()).map(|_| None).collect()); + let out: Mutex>> = Mutex::new((0..squares.len()).map(|_| None).collect()); std::thread::scope(|scope| { for _ in 0..threads { scope.spawn(|| { @@ -481,7 +516,7 @@ fn draw_some(r: &Resolved, squares: &[Placed], side: u32, m_per_px: f64, ppp: f3 } /// Draws the shapes of one square, with those of the tiles around it out to `reach` points. -fn draw_one(r: &Resolved, p: &Placed, side: u32, m_per_px: f64, ppp: f32, reach: f64) -> Option<(Pixmap, bool)> { +fn draw_one(r: &Resolved, p: &Placed, side: u32, m_per_px: f64, ppp: f32, reach: f64) -> Option { let mut pm = Pixmap::new(side, side)?; let e = p.extent(); let frame = Frame { left: e.min_x, top: e.max_y, m_per_px, ppp }; @@ -489,12 +524,346 @@ fn draw_one(r: &Resolved, p: &Placed, side: u32, m_per_px: f64, ppp: f32, reach: let pad = reach * f64::from(ppp) * m_per_px; let view = Extent::new(e.min_x - pad, e.min_y - pad, e.max_x + pad, e.max_y + pad); let mut d = Drawn::default(); - draw_layers(Some(&mut pm), (side, side), &view, r, &frame, Pass::Shapes, &mut d); - Some((pm, d.pending)) + let retry = draw_layers(Some(&mut pm), (side, side), &view, r, &frame, Pass::Shapes, &mut d); + Some(DrawnPicture { image: pm, pending: d.pending, retry }) } #[cfg(test)] mod tests { + /// Pattern failures follow prepared pixels, while absent patterns leave no sprite warning. + #[test] + fn rounded_picture_patterns_report_sprite_failures() { + let _turn = cache::turn(); + for min in [3, 4] { + let base = format!("https://rounded-pattern.example/{min}"); + let style = serde_json::to_vec(&serde_json::json!({"version":8,"sprite":format!("{base}/sprite"), + "sources":{"s":{"type":"vector","tiles":[format!("{base}/tile")],"maxzoom":0}}, + "layers":[{"id":"pattern","type":"fill","source":"s","source-layer":"l","minzoom":min,"paint":{"fill-pattern":"a"}}]})) + .unwrap(); + let bytes = crate::mvt::encode(&Tile::default()); + cache::set_fetcher(move |url, done| { + if url.ends_with("/style.json") { + done(Ok(style.clone())); + } else if url.contains("/sprite") { + done(Err("HTTP 503".into())); + } else { + done(Ok(bytes.clone())); + } + }); + let spec = Spec { + kind: Kind::Style, + url: format!("{base}/style.json"), + subdomains: Vec::new(), + min_zoom: 0, + max_zoom: 14, + tile_size: 512, + tms: false, + attribution: String::new(), + layers: Vec::new(), + style_layers: String::new(), + hidden_layers: Vec::new(), + }; + let f = frame(2.999, 1.0); + let view = f.extent(64, 64); + resolved(&spec); + let mut pm = Pixmap::new(64, 64).unwrap(); + draw(&mut pm, &spec, 1.0, &f, false); + cache::wait(Duration::from_secs(2)); + let d = draw(&mut pm, &spec, 1.0, &f, false); + assert_eq!(d.error.as_deref().is_some_and(|e| e.contains("pattern")), min == 3, "draw at pattern minzoom {min}: {:?}", d.error); + let placed = place(&spec, 1.0, &f, &view, &view, None).unwrap(); + assert_eq!( + placed.error.as_deref().is_some_and(|e| e.contains("pattern")), + min == 3, + "placement at pattern minzoom {min}: {:?}", + placed.error + ); + } + } + + /// A rounded picture zoom must report and retry the tiles that supplied its pixels. + #[test] + fn picture_failures_at_rounded_zoom_retry_through_cover() { + let _turn = cache::turn(); + let tile = Tile { + layers: vec![Layer { + name: "l".into(), + extent: 4096, + features: vec![Feature { + id: None, + kind: GeomKind::Polygon, + props: Vec::new(), + geometry: vec![vec![[0.0, 0.0], [4096.0, 0.0], [4096.0, 4096.0], [0.0, 4096.0], [0.0, 0.0]]], + }], + }], + }; + for ppp in [1.0, 1.5] { + let root = format!("https://picture-rounded.example/{ppp}"); + let failed_url = format!("{root}/3"); + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let attempts = calls.clone(); + let bytes = crate::mvt::encode(&tile); + cache::set_fetcher(move |url, done| { + if url.ends_with("/3") && attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 { + done(Err("HTTP 503".into())); + } else { + done(Ok(bytes.clone())); + } + }); + let style = crate::style::Style::parse(&serde_json::json!({"version":8, + "sources":{"s":{"type":"vector","tiles":[format!("{root}/{{z}}")]}}, + "layers":[{"id":"fill","type":"fill","source":"s","source-layer":"l"}]})) + .unwrap(); + let r = Arc::new(Resolved { + style, + from: None, + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }); + let f = frame(2.999, ppp); + let view = f.extent(64, 64); + cover(&r, &f, &view, None, &mut false, &mut None); + cache::wait(Duration::from_secs(2)); + let mut error = None; + let failed = cover(&r, &f, &view, None, &mut false, &mut error).unwrap(); + assert!(error.is_some(), "the failed picture input must be reported at {ppp} pixels per point"); + assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1); + assert!(cache::expire_failure(&failed_url)); + cover(&r, &f, &view, None, &mut false, &mut None); + cache::wait(Duration::from_secs(2)); + assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 2, "cover must retry its rounded-zoom input"); + let recovered = cover(&r, &f, &view, None, &mut false, &mut None).unwrap(); + assert!(failed.iter().zip(&recovered).all(|(a, b)| !Arc::ptr_eq(&a.payload, &b.payload))); + assert!( + recovered + .iter() + .any(|it| matches!(it.payload.as_ref(), Payload::Image(pm) if pm.data().as_chunks::<4>().0.iter().any(|p| p[3]>0))) + ); + } + } + + /// A layer the renderer cannot draw must not select or report its separate source. + #[test] + fn undrawn_line_patterns_do_not_report_refused_tiles() { + let _turn = cache::turn(); + let bytes = crate::mvt::encode(&Tile::default()); + cache::set_fetcher(move |url, done| { + assert!(url.starts_with("https://pattern-control.example/")); + done(Ok(bytes.clone())); + }); + let mut style = crate::style::Style::parse(&serde_json::json!({"version":8, + "sources":{"hidden":{"type":"vector","tiles":["http://127.0.0.1/{z}"]},"shown":{"type":"vector","tiles":["https://pattern-control.example/{z}"]}}, + "layers":[{"id":"pattern","type":"line","source":"hidden","source-layer":"l","paint":{"line-pattern":"rail"}}, + {"id":"fill","type":"fill","source":"shown","source-layer":"l"}]})).unwrap(); + for (_, src) in &mut style.sources { + if let Source::Vector { from, .. } = src { + *from = Some(crate::address::Origin::named("https://pattern-control.example/style.json")); + } + } + let r = Resolved { + style, + from: None, + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }; + let f = frame(2.2, 1.0); + let mut drawn = Drawn::default(); + draw_layers(None, (64, 64), &f.extent(64, 64), &r, &f, Pass::Shapes, &mut drawn); + assert!(drawn.error.is_none(), "an unsupported layer cannot omit drawable tiles: {:?}", drawn.error); + } + + /// Padded preparation and later cache reuse must report the current viewport only. + #[test] + fn picture_refusals_follow_the_viewport_not_preparation_padding() { + let _turn = cache::turn(); + let bytes = crate::mvt::encode(&Tile::default()); + cache::set_fetcher(move |_, done| done(Ok(bytes.clone()))); + let mut style = crate::style::Style::parse(&serde_json::json!({"version":8, + "sources":{"s":{"type":"vector","tiles":["http://127.0.0.1/{z}/{x}/{y}","https://picture-view.example/{z}/{x}/{y}"]}}, + "layers":[{"id":"line","type":"line","source":"s","source-layer":"l","paint":{"line-width":20}}]})) + .unwrap(); + let from = crate::address::Origin::named("https://picture-view.example/style.json"); + if let Source::Vector { from: selected, .. } = &mut style.sources.first_mut().unwrap().1 { + *selected = Some(from.clone()); + } + let r = Arc::new(Resolved { + style, + from: Some(from), + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }); + let square = TileId { z: 3, x: 6, y: 4 }.extent(); + let f = Frame { left: square.min_x, top: square.max_y, m_per_px: template::HALF_WORLD / (256.0 * 2.2f64.exp2()), ppp: 1.0 }; + let view = Extent::new( + square.min_x + square.width() * 0.3, + square.min_y + square.height() * 0.3, + square.min_x + square.width() * 0.6, + square.min_y + square.height() * 0.6, + ); + let mut kept = Vec::new(); + for _ in 0..2 { + let mut error = None; + kept = cover(&r, &f, &view, None, &mut false, &mut error).unwrap(); + cache::wait(Duration::from_secs(2)); + assert!(error.is_none(), "only padding crosses a refused shard: {error:?}"); + } + // A deterministic destination refusal does not make these pixels retryable. + cache::get_tile("https://picture-view.example/unrelated", cache::Want::Vector); + cache::wait(Duration::from_secs(2)); + let mut error = None; + let again = cover(&r, &f, &view, None, &mut false, &mut error).unwrap(); + assert!(error.is_none()); + assert_eq!(kept.len(), again.len()); + assert!(kept.iter().zip(&again).all(|(old, new)| Arc::ptr_eq(&old.payload, &new.payload))); + } + + /// A neighbouring tile can paint a circle into view even when its own square is outside. + #[test] + fn picture_reports_refused_symbols_reaching_in_from_a_neighbour() { + let _turn = cache::turn(); + let bytes = crate::mvt::encode(&Tile { + layers: vec![Layer { + name: "l".into(), + extent: 4096, + features: vec![Feature { + id: None, + kind: GeomKind::Point, + props: vec![("radius".into(), crate::mvt::Value::Number(40.0))], + geometry: vec![vec![[4090.0, 1024.0]]], + }], + }], + }); + cache::set_fetcher(move |_, done| done(Ok(bytes.clone()))); + for (radius, inset, reaches) in [ + (serde_json::json!(20), 4.0, true), + (serde_json::json!(["get", "radius"]), 24.0, true), + (serde_json::json!(["min", 2, ["get", "radius"]]), 300.0, false), + ] { + let tile = TileId { z: 2, x: 3, y: 2 }.extent(); + let m_per_px = frame(2.2, 1.0).m_per_px; + let f = + Frame { left: tile.min_x + inset * m_per_px, top: tile.max_y - tile.height() * 0.25 + 32.0 * m_per_px, m_per_px, ppp: 1.0 }; + let view = f.extent(64, 64); + for restricted in [false, true] { + let mut style = crate::style::Style::parse(&serde_json::json!({"version":8, + "sources":{"s":{"type":"vector","tiles":["http://127.0.0.1/{z}/{x}/{y}","https://picture-neighbour.example/{z}/{x}/{y}"]}}, + "layers":[{"id":"point","type":"circle","source":"s","source-layer":"l","paint":{"circle-radius":radius,"circle-color":"#ff0000"}}]})) + .unwrap(); + let from = restricted.then(|| crate::address::Origin::named("https://picture-neighbour.example/style.json")); + if let Source::Vector { from: selected, .. } = &mut style.sources.first_mut().unwrap().1 { + *selected = from.clone(); + } + let r = Arc::new(Resolved { + style, + from, + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }); + cover(&r, &f, &view, None, &mut false, &mut None); + cache::wait(Duration::from_secs(2)); + let mut error = None; + let pictures = cover(&r, &f, &view, None, &mut false, &mut error).unwrap(); + let painted = pictures.iter().any(|it| { + let Payload::Image(pm) = it.payload.as_ref() else { return false }; + let x = ((view.min_x + 2.0 * m_per_px - it.at.min_x) / it.at.width() * f64::from(pm.width())).floor(); + let y = ((it.at.max_y - (view.max_y - 32.0 * m_per_px)) / it.at.height() * f64::from(pm.height())).floor(); + x >= 0.0 && y >= 0.0 && pm.pixel(x as u32, y as u32).is_some_and(|px| px.alpha() > 0) + }); + assert_eq!( + painted, + reaches && !restricted, + "the permitted neighbour paints only when its radius reaches the actual viewport" + ); + assert_eq!(error.is_some(), reaches && restricted, "only visible missing neighbour geometry must be reported: {error:?}"); + } + } + } + + /// Routine missing tiles and reported failures both replace their pixels after recovery. + #[test] + fn picture_failures_refresh_after_recovery() { + let _turn = cache::turn(); + let tile = Tile { + layers: vec![Layer { + name: "l".into(), + extent: 4096, + features: vec![Feature { + id: None, + kind: GeomKind::Polygon, + props: Vec::new(), + geometry: vec![vec![[0.0, 0.0], [4096.0, 0.0], [4096.0, 4096.0], [0.0, 4096.0], [0.0, 0.0]]], + }], + }], + }; + let mut stale = Vec::new(); + for status in [503, 404, 410] { + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let attempts = calls.clone(); + let bytes = crate::mvt::encode(&tile); + cache::set_fetcher(move |_, done| { + if attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 { + done(Err(format!("http status: {status}"))); + } else { + done(Ok(bytes.clone())); + } + }); + let url = format!("https://picture-retry.example/{status}"); + let style = crate::style::Style::parse(&serde_json::json!({"version":8, + "sources":{"s":{"type":"vector","tiles":[url],"maxzoom":0}}, + "layers":[{"id":"fill","type":"fill","source":"s","source-layer":"l","paint":{"fill-color":"#ff0000"}}]})) + .unwrap(); + let r = Arc::new(Resolved { + style, + from: None, + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }); + let f = frame(2.2, 1.0); + let view = f.extent(64, 64); + cover(&r, &f, &view, None, &mut false, &mut None); + cache::wait(Duration::from_secs(2)); + let mut error = None; + let mut pending = false; + let failed = cover(&r, &f, &view, None, &mut pending, &mut error).unwrap(); + assert!(!pending, "an unavailable tile must not keep export pending"); + assert_eq!(error.is_some(), status == 503, "routine status {status}: {error:?}"); + assert!(!failed.is_empty()); + assert!(failed.iter().all(|it| matches!(it.payload.as_ref(), Payload::Image(pm) if pm.data().iter().all(|b| *b == 0)))); + assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1); + assert!(cache::expire_failure(&url)); + cover(&r, &f, &view, None, &mut false, &mut None); + cache::wait(Duration::from_secs(2)); + assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 2, "cover must initiate the expired tile's retry"); + assert!(matches!(cache::peek(&url).as_deref(), Some(Payload::Vector(t)) if !t.layers[0].features.is_empty())); + let mut error = None; + let recovered = cover(&r, &f, &view, None, &mut false, &mut error).unwrap(); + assert!(error.is_none(), "recovered tile still has an old picture warning: {error:?}"); + let replaced = failed.len() == recovered.len() + && failed.iter().zip(&recovered).all(|(old, new)| old.at == new.at && !Arc::ptr_eq(&old.payload, &new.payload)); + let painted = recovered + .iter() + .any(|it| matches!(it.payload.as_ref(), Payload::Image(pm) if pm.data().as_chunks::<4>().0.iter().any(|px| px[3] > 0))); + if !replaced || !painted { + stale.push(status); + } + } + assert!(stale.is_empty(), "recovered geometry left old blank pictures cached for HTTP {stale:?}"); + } use super::*; use crate::draw::{draw, place}; use crate::mvt::{Feature, GeomKind, Layer, Tile}; @@ -662,33 +1031,34 @@ mod tests { drawn(&spec, &f, 1.0); let view = f.extent(300, 200); let mut pending = false; - let held: Vec> = cover(&r, &f, &view, None, &mut pending).unwrap().into_iter().map(|i| i.payload).collect(); + let held: Vec> = cover(&r, &f, &view, None, &mut pending, &mut None).unwrap().into_iter().map(|i| i.payload).collect(); let theirs = |items: &[Item]| items.iter().all(|i| held.iter().any(|h| Arc::ptr_eq(h, &i.payload))); // A little further in, or coarser (as frames drawn mid-motion are): those drawn stand in. // Over the same ground: the view's squares are those drawn, or above or below them. for g in [frame(2.3, 1.0), frame(2.9, 1.0), frame(2.2, 0.6), frame(1.6, 1.0)] { - let (items, lacks) = paced(Pace::Moving, || cover(&r, &g, &view, None, &mut pending).unwrap()); + let (items, lacks) = paced(Pace::Moving, || cover(&r, &g, &view, None, &mut pending, &mut None).unwrap()); assert!(lacks == Some(Unfinished::Later) && !items.is_empty() && theirs(&items), "stood in at {}", g.m_per_px); // Each covering the view's squares, where it is. assert!(items.iter().all(|i| i.at.intersects(&i.clip) && i.clip.width() > 0.0)); } // Still, or over two levels away: drawn. let g = frame(2.3, 1.0); - let still = cover(&r, &g, &view, None, &mut pending).unwrap(); + let still = cover(&r, &g, &view, None, &mut pending, &mut None).unwrap(); assert!(!theirs(&still)); let far = frame(5.5, 1.0); - let (items, lacks) = paced(Pace::Moving, || cover(&r, &far, &view, None, &mut pending).unwrap()); + let (items, lacks) = paced(Pace::Moving, || cover(&r, &far, &view, None, &mut pending, &mut None).unwrap()); // Nothing stands in that far away (a slow machine may leave some for the next frame). assert!(lacks != Some(Unfinished::Later) && !theirs(&items)); // Settling, the exact pictures are drawn while there's time (a round at least). let g = frame(2.35, 1.0); - let (items, lacks) = paced(Pace::Settling, || cover(&r, &g, &view, None, &mut pending).unwrap()); + let (items, lacks) = paced(Pace::Settling, || cover(&r, &g, &view, None, &mut pending, &mut None).unwrap()); // A round at least is new; complete, nothing held stands in. let held_in = |i: &Item| held.iter().any(|h| Arc::ptr_eq(h, &i.payload)); assert!(lacks != Some(Unfinished::Later) && items.iter().any(|i| !held_in(i))); assert!(lacks.is_some() || !items.iter().any(held_in)); // Inside nested calls, the outer one hears of pictures standing in. - let ((_, inner), outer) = paced(Pace::Settling, || paced(Pace::Moving, || cover(&r, &frame(2.25, 1.0), &view, None, &mut pending))); + let ((_, inner), outer) = + paced(Pace::Settling, || paced(Pace::Moving, || cover(&r, &frame(2.25, 1.0), &view, None, &mut pending, &mut None))); assert!(inner == Some(Unfinished::Later) && outer == inner); assert!(PACE.with(|p| p.get()).is_none()); // One deadline for the frame, whoever asks; an inner call keeps within it. @@ -742,16 +1112,16 @@ mod tests { let r = resolved(&spec); let mut pending = false; let world = Extent::new(-template::HALF_WORLD * 1.5, -1e6, template::HALF_WORLD * 1.5, 1e6); - assert!(cover(&r, &f, &world, None, &mut pending).is_none()); + assert!(cover(&r, &f, &world, None, &mut pending, &mut None).is_none()); // More squares than a covering holds: drawn straight too, not left blank. let near = frame(12.0, 1.0); let huge = near.extent(20_000, 10_000); assert!(huge.width() < template::HALF_WORLD); - assert!(cover(&r, &near, &huge, None, &mut pending).is_none()); + assert!(cover(&r, &near, &huge, None, &mut pending, &mut None).is_none()); for m_per_px in [0.0, f64::NAN, f64::INFINITY] { - assert!(cover(&r, &Frame { m_per_px, ..f }, &view, None, &mut pending).is_none()); + assert!(cover(&r, &Frame { m_per_px, ..f }, &view, None, &mut pending, &mut None).is_none()); } - assert!(cover(&r, &Frame { ppp: f32::NAN, ..f }, &view, None, &mut pending).is_none()); + assert!(cover(&r, &Frame { ppp: f32::NAN, ..f }, &view, None, &mut pending, &mut None).is_none()); let _ = std::fs::remove_dir_all(dir); } } diff --git a/crates/tiles/src/shapes.rs b/crates/tiles/src/shapes.rs index a105200388..22b4626be4 100644 --- a/crates/tiles/src/shapes.rs +++ b/crates/tiles/src/shapes.rs @@ -18,7 +18,7 @@ use crate::cache::Payload; use crate::draw::{Drawn, Frame, Label, Pass, draw_layers, tiles_in}; use crate::mvt::{GeomKind, Tile}; use crate::source::{Resolve, Resolved, Spec, resolve}; -use crate::sprite::{self, Load, Sprite}; +use crate::sprite::{Load, Sprite}; use crate::style::{FeatureCtx, LayerKind, LineCap, LineJoin, PATTERNS, Rgba, Source, StyleLayer}; use crate::template; @@ -76,16 +76,28 @@ pub fn vector_tiles(spec: &Spec, frame: &Frame, view: &Extent, reach: Option<&Ex let source = drawn_source(&r, level)?; let src = r.style.sources.iter().find(|(id, _)| *id == source).map(|(_, s)| s)?; let mut pending = false; - let tiles = tiles_in(src, view, reach, frame, zoom, &mut pending) - .into_iter() - .map(|it| DataTile { payload: it.payload, at: it.at, z: it.z, clip: it.clip }) - .collect(); + let mut error = crate::draw::left_out(&r, frame, true); + let mut selection_error = None; + let selection = tiles_in(src, view, reach, frame, zoom, &mut pending, &mut selection_error); + // Mesh backgrounds cover only returned data squares. When selection leaves holes, + // ordinary drawing paints the whole background and the remaining geometry together. + if selection.mesh_gaps + && r.style.layers.iter().any(|l| l.kind == LayerKind::Background && shown_in(l, f32::from(level), f32::from(level) + 1.0)) + { + return None; + } + if let Some(note) = selection_error { + crate::draw::add_error(&mut error, note); + } + let tiles = selection.items.into_iter().map(|it| DataTile { payload: it.payload, at: it.at, z: it.z, clip: it.clip }).collect(); let mut d = Drawn::default(); if let Some(size) = labels { draw_layers(None, size, &frame.extent(size.0, size.1), &r, frame, Pass::Labels, &mut d); } + if let Some(refused) = d.error { + crate::draw::add_error(&mut error, refused); + } let attribution = r.attribution.clone(); - let error = crate::draw::left_out(&r, frame, true); Some(VectorTiles { style: r, source, tiles, level, labels: d.labels, attribution, pending: pending || d.pending, error }) } @@ -226,7 +238,7 @@ fn average(img: &tiny_skia::Pixmap) -> Option<[f32; 4]> { /// the next level, in its order. Its background, when it has one, is the first run, filling the /// square. pub fn shapes(style: &Resolved, source: &str, tile: &Tile, level: u8) -> TileShapes { - shapes_with(style, source, tile, level, &|| sprite::load(&style.sprites, false)) + shapes_with(style, source, tile, level, &|| style.load_sprites(false)) } /// [`shapes`], with `load` to load the style's sprites. @@ -632,25 +644,20 @@ mod tests { // A raster layer from the middle of the level: its meshes would serve where it shows. let style = STYLE.replacen('[', r#"[{"id": "r", "type": "raster", "source": "x", "minzoom": 2.8},"#, 1); let (spec, dir) = archive("level", &style); - ready(&spec); - let f = frame(2.5); - assert!(vector_tiles(&spec, &f, &f.extent(300, 200), None, None).is_none()); + assert!(drawn_source(&ready(&spec), 2).is_none()); // Shown from the next level only: these meshes are fine. let style = STYLE.replacen('[', r#"[{"id": "r", "type": "raster", "source": "x", "minzoom": 3},"#, 1); let (spec, dir2) = archive("next-level", &style); - ready(&spec); - assert!(vector_tiles(&spec, &f, &f.extent(300, 200), None, None).is_some()); + assert!(drawn_source(&ready(&spec), 2).is_some()); // Caps changing within the level. let style = STYLE.replace(r#""line-cap": "round""#, r#""line-cap": ["step", ["zoom"], "butt", 2.5, "round"]"#); let (spec, dir4) = archive("caps", &style); - ready(&spec); - assert!(vector_tiles(&spec, &f, &f.extent(300, 200), None, None).is_none()); + assert!(drawn_source(&ready(&spec), 2).is_none()); let _ = std::fs::remove_dir_all(dir4); // Caps changing at the next level: not within this one. let style = STYLE.replace(r#""line-cap": "round""#, r#""line-cap": ["step", ["zoom"], "butt", 3, "round"]"#); let (spec, dir5) = archive("next-caps", &style); - ready(&spec); - assert!(vector_tiles(&spec, &f, &f.extent(300, 200), None, None).is_some()); + assert!(drawn_source(&ready(&spec), 2).is_some()); let _ = std::fs::remove_dir_all(dir5); // A long dash pattern from the end of the level. let style = STYLE.replace( @@ -658,22 +665,41 @@ mod tests { r#""line-dasharray": ["step", ["zoom"], ["literal", [3, 2]], 2.9, ["literal", [3, 2, 1]]]"#, ); let (spec, dir3) = archive("late-dashes", &style); - ready(&spec); - assert!(vector_tiles(&spec, &f, &f.extent(300, 200), None, None).is_none()); + assert!(drawn_source(&ready(&spec), 2).is_none()); for d in [dir, dir2, dir3] { let _ = std::fs::remove_dir_all(d); } } + /// Fully loaded geometry reaches the entry point for eligible styles only; no missing tile + /// or background gap can make an unsupported style pass this rejection test accidentally. + #[test] + fn unsupported_styles_stay_out_of_the_mesh_entrypoint() { + let _turn = crate::cache::turn(); + let (spec, dir) = archive("entrypoint", STYLE); + let loaded = tile(&spec, 2.5); + assert!(!loaded.pending && !loaded.tiles.is_empty()); + let frame = frame(2.5); + let view = frame.extent(300, 200); + assert!(vector_tiles(&spec, &frame, &view, None, None).is_some()); + for style in [ + STYLE.replacen('[', r#"[{"id":"r","type":"raster","source":"x"},"#, 1), + STYLE.replace(r#""line-cap": "round""#, r#""line-cap": ["step", ["zoom"], "butt", 2.5, "round"]"#), + ] { + let unsupported = Spec { style_layers: style, ..spec.clone() }; + assert!(drawn_source(&ready(&unsupported), 2).is_none()); + assert!(vector_tiles(&unsupported, &frame, &view, None, None).is_none()); + } + let _ = std::fs::remove_dir_all(dir); + } + #[test] fn styles_with_long_dash_patterns_are_declined() { let _turn = crate::cache::turn(); // Three lengths repeat to six: more than the meshes hold. let style = STYLE.replace(r#""line-dasharray": [3, 2]"#, r#""line-dasharray": [3, 2, 1]"#); let (spec, dir) = archive("dashes", &style); - ready(&spec); - let f = frame(2.5); - assert!(vector_tiles(&spec, &f, &f.extent(300, 200), None, None).is_none()); + assert!(drawn_source(&ready(&spec), 2).is_none()); let _ = std::fs::remove_dir_all(dir); } @@ -693,13 +719,11 @@ mod tests { // A raster layer is a picture of its own. let style = STYLE.replacen('[', r#"[{"id": "r", "type": "raster", "source": "x"},"#, 1); let (spec, dir) = archive("raster", &style); - ready(&spec); - let f = frame(2.5); - assert!(vector_tiles(&spec, &f, &f.extent(300, 200), None, None).is_none()); + assert!(drawn_source(&ready(&spec), 2).is_none()); let _ = std::fs::remove_dir_all(dir); // Nor a broken scale. let (spec, dir) = archive("scale", STYLE); - ready(&spec); + tile(&spec, 2.5); let f = Frame { m_per_px: f64::NAN, ..frame(2.5) }; assert!(vector_tiles(&spec, &f, &frame(2.5).extent(300, 200), None, None).is_none()); let _ = std::fs::remove_dir_all(dir); diff --git a/crates/tiles/src/source.rs b/crates/tiles/src/source.rs index 1f778def24..c7c55d776c 100644 --- a/crates/tiles/src/source.rs +++ b/crates/tiles/src/source.rs @@ -12,7 +12,7 @@ use std::sync::{Arc, Mutex, OnceLock}; use serde_json::{Value, json}; use web_time::Instant; -use crate::cache::{self, Payload, State, Want}; +use crate::cache::{self, Payload, State}; use crate::style::{Source, Style}; use crate::{address, cog, mbtiles, pmtiles, template}; @@ -75,6 +75,8 @@ impl Spec { /// A style whose sources all have tile addresses. pub struct Resolved { pub style: Style, + /// The effective style document, kept through redirects and cache hits. + pub from: Option, /// The style's sprites as (id, address without `.json`/`.png`), those RustGIS can /// fetch. pub sprites: Vec<(String, String)>, @@ -91,6 +93,11 @@ pub struct Resolved { } impl Resolved { + /// Loads every rendering path's sprite sheets under this style's actual origin. + pub fn load_sprites(&self, hi_dpi: bool) -> crate::sprite::Load { + crate::sprite::load_from(&self.sprites, hi_dpi, self.from.as_ref()) + } + /// Whether it may still be used as it is. fn current(&self) -> bool { !self.waiting && (self.left_out.is_empty() || self.made.lock().unwrap_or_else(|e| e.into_inner()).elapsed() < cache::RETRY) @@ -106,6 +113,7 @@ fn kept(old: Arc, new: Arc) -> Arc { || old.style != new.style || old.sprites != new.sprites || old.attribution != new.attribution + || old.from != new.from { return new; } @@ -189,14 +197,18 @@ fn settle(anew: Resolve, cached: Option>, spec: &Spec, key: String /// Reads `spec`'s style and resolves its sources, keeping the result under `key`. fn resolve_anew(spec: &Spec, key: String) -> Resolve { + let mut base = address::Origin::named(&spec.url); let doc = match spec.kind { Kind::Raster => raster_style(spec), Kind::Vector => match vector_style(spec) { Some(d) => d, None => return Resolve::Failed("the built-in vector style could not be read".into()), }, - Kind::Style => match fetch_json(&spec.url) { - Ok(Some(d)) => d, + Kind::Style => match fetch_json(&spec.url, None) { + Ok(Some((d, reached))) => { + base = reached; + d + } Ok(None) => return Resolve::Pending, Err(e) => return Resolve::Failed(e), }, @@ -263,7 +275,6 @@ fn resolve_anew(spec: &Spec, key: String) -> Resolve { style.layers.retain(|l| l.source_layer.as_ref().is_none_or(|s| !spec.hidden_layers.contains(s))); } // Sources given by a TileJSON address. - let base = spec.url.clone(); let Sources { mut left_out, waiting } = match resolve_sources(&mut style, spec, &base) { Ok(Some(sources)) => sources, Ok(None) => return Resolve::Pending, @@ -284,7 +295,15 @@ fn resolve_anew(spec: &Spec, key: String) -> Resolve { } let (sprites, sprites_left_out) = resolve_sprites(&style, &base); left_out.extend(sprites_left_out); - let r = Arc::new(Resolved { style, attribution, sprites, left_out, made: Mutex::new(Instant::now()), waiting }); + let r = Arc::new(Resolved { + style, + from: (spec.kind == Kind::Style).then_some(base), + attribution, + sprites, + left_out, + made: Mutex::new(Instant::now()), + waiting, + }); if let Err(e) = crate::logins::check(&spec.url) { return Resolve::Failed(e.to_string()); } @@ -300,15 +319,15 @@ fn resolve_anew(spec: &Spec, key: String) -> Resolve { /// A style's sprites with their addresses resolved against `base`, those RustGIS can fetch, and /// what was left out of them, and why, if a layer not hidden draws images and so misses them: the /// sprites are the style's choice, and one that can't be fetched only loses its own icons. -fn resolve_sprites(style: &Style, base: &str) -> (Vec<(String, String)>, Vec) { +fn resolve_sprites(style: &Style, base: &address::Origin) -> (Vec<(String, String)>, Vec) { let mut sprites = Vec::new(); let mut left_out = style.sprites_left_out.clone(); let (mut mapbox, mut refused) = (Vec::new(), Vec::new()); for (id, s) in &style.sprites { - let address = template::join(base, s); + let address = template::join(&base.url, s); if s.starts_with("mapbox://") { mapbox.push(id.as_str()); - } else if address::check(&address, Some(base)).is_err() { + } else if base.check(&address).is_err() { refused.push(id.as_str()); } else { sprites.push((id.clone(), address)); @@ -340,7 +359,7 @@ struct Sources { /// Resolves the sources of `style` in place: what was left out of it, or `None` while some the /// style can't be drawn without are still on their way. -fn resolve_sources(style: &mut Style, spec: &Spec, base: &str) -> Result, String> { +fn resolve_sources(style: &mut Style, spec: &Spec, base: &address::Origin) -> Result, String> { let mut pending = false; let mut left_out = Vec::new(); let mut waiting = false; @@ -380,19 +399,24 @@ fn resolve_sources(style: &mut Style, spec: &Spec, base: &str) -> Result Result { - let (tiles, url) = match src { - Source::Vector { tiles, url, .. } | Source::Raster { tiles, url, .. } | Source::RasterDem { tiles, url, .. } => (tiles, url), +/// (`Ok(true)` while that is on its way), resolved against `base`. Tile selection checks each +/// expanded address, preserving shard positions when one is refused. +fn resolve_source(src: &mut Source, spec: &Spec, base: &address::Origin) -> Result { + let (tiles, url, from) = match src { + Source::Vector { tiles, url, from, .. } | Source::Raster { tiles, url, from, .. } | Source::RasterDem { tiles, url, from, .. } => { + (tiles, url, from) + } Source::Other => return Ok(false), }; + // Kept only in the resolved style, never accepted from a document or saved in the project. + *from = (spec.kind == Kind::Style).then(|| base.clone()); if tiles.is_empty() && let Some(u) = url.clone() { if u.starts_with("mapbox://") { return Err(format!("{u}: mapbox:// addresses need Mapbox's own library")); } - let u = template::join(base, &u); + let u = template::join(&base.url, &u); // An ArcGIS vector tile service answers its TileJSON-like description as JSON only // when asked (`?f=json`); its styles point at the service itself. let path = u.split(['?', '#']).next().unwrap_or(&u).trim_end_matches('/').to_string(); @@ -413,15 +437,20 @@ fn resolve_source(src: &mut Source, spec: &Spec, base: &str) -> Result { + if spec.kind == Kind::Style { + base.check(&u)?; + } else { + address::check(&u, None)?; + } + match fetch_json(&u, (spec.kind == Kind::Style).then_some(base))? { + Some((tj, reached)) => { src.apply_tilejson(&tj); // Tile addresses relative to the TileJSON document, which chose them. - if let Source::Vector { tiles, .. } | Source::Raster { tiles, .. } | Source::RasterDem { tiles, .. } = src { + if let Source::Vector { tiles, from, .. } | Source::Raster { tiles, from, .. } | Source::RasterDem { tiles, from, .. } = src + { + *from = Some(reached.clone()); for t in tiles.iter_mut() { - *t = template::join(&u, t); - address::check(t, Some(&u))?; + *t = template::join(&reached.url, t); } } } @@ -436,10 +465,7 @@ fn resolve_source(src: &mut Source, spec: &Spec, base: &str) -> Result Option> { } /// The document at `url`: `Ok(None)` while it is on its way. -fn fetch_json(url: &str) -> Result, String> { - match cache::get(url, Want::Json) { - State::Ready(p) => match &*p { - Payload::Json(v) => Ok(Some(v.clone())), +fn fetch_json(url: &str, from: Option<&address::Origin>) -> Result, String> { + match cache::document(url, from) { + (State::Ready(p), origin) => match &*p { + Payload::Json(v) => Ok(Some((v.clone(), origin.unwrap_or_else(|| address::Origin::named(url))))), _ => Err(format!("{}: not a JSON document", crate::logins::inner(url))), }, - State::Pending => Ok(None), - State::Failed(e) => Err(format!("{}: {e}", crate::logins::inner(url))), + (State::Pending, _) => Ok(None), + (State::Failed(e), _) => Err(format!("{}: {e}", crate::logins::inner(url))), } } @@ -651,6 +677,81 @@ pub fn plain_text(html: &str) -> String { #[cfg(test)] mod tests { + /// An expanded selector must not let a public style choose another saved account. + #[test] + fn expanded_selectors_cannot_choose_a_documents_account() { + let _turn = cache::turn(); + let endpoint = "https://expanded-account.example"; + let binding = crate::s3::new_binding().unwrap(); + let name = "expanded-account-fixture"; + crate::s3::connect_saved( + crate::s3::Connection { + name: name.into(), + endpoint: Some(endpoint.into()), + region: "us-east-1".into(), + buckets: vec!["bucket".into()], + credentials: None, + }, + &binding, + "expanded-account", + ) + .unwrap(); + let selected = rustgis_data::s3_address::bind(&binding, endpoint, &format!("{endpoint}/bucket/tile.pbf")).unwrap(); + assert!( + address::Origin { url: selected.clone(), reach: address::Reach::Public }.check(&selected).is_ok(), + "the document may retain its own selected account" + ); + let login = crate::logins::activate( + endpoint, + "expanded-service-fixture", + crate::logins::Method::Header("X-Fixture".into()), + "fixture", + false, + ) + .unwrap(); + let service = crate::logins::bind(&login, &format!("{endpoint}/tile.pbf")).unwrap(); + assert!(address::Origin { url: service.clone(), reach: address::Reach::Public }.check(&service).is_ok()); + let asked = Arc::new(Mutex::new(Vec::new())); + let seen = asked.clone(); + cache::set_fetcher(move |url, done| { + seen.lock().unwrap().push(url); + done(Ok(crate::mvt::encode(&crate::mvt::Tile::default()))); + }); + for selected in [selected, service] { + let mut style = Style::parse( + &json!({"version":8,"sources":{"s":{"type":"vector","tiles":[format!("{{switch:{selected}}}")]}},"layers":[]}), + ) + .unwrap(); + let base = address::Origin::named("https://expanded-style.example/style.json"); + let spec = spec(Kind::Style, &base.url); + resolve_sources(&mut style, &spec, &base).unwrap(); + let src = &style.sources.first().unwrap().1; + let frame = crate::draw::tests::world_frame(); + let mut error = None; + crate::draw::tiles_in(src, &frame.extent(64, 64), None, &frame, 0.0, &mut false, &mut error); + cache::wait(std::time::Duration::from_secs(2)); + assert!(error.is_some()); + } + crate::logins::deactivate(&login); + crate::s3::disconnect(name); + assert!(asked.lock().unwrap().is_empty(), "the style selected an account after expansion"); + } + + #[test] + fn forbidden_provenance_cannot_queue_sprites() { + let _turn = cache::turn(); + let asked = Arc::new(Mutex::new(Vec::new())); + let seen = asked.clone(); + cache::set_fetcher(move |url, done| { + seen.lock().unwrap().push(url); + done(Err("fixture".into())); + }); + let from = address::Origin::named("http://metadata/style.json"); + let state = crate::sprite::load_from(&[("default".into(), "http://127.0.0.1/forbidden-sprite".into())], false, Some(&from)); + cache::wait(std::time::Duration::from_secs(2)); + assert!(asked.lock().unwrap().is_empty(), "forbidden provenance reached transport"); + assert!(matches!(state, crate::sprite::Load::Failed)); + } use super::*; fn spec(kind: Kind, url: &str) -> Spec { @@ -763,6 +864,544 @@ mod tests { assert!(matches!(resolve(&spec(Kind::Raster, "http://localhost:8080/{z}/{x}/{y}.png")), Resolve::Ready(_))); } + /// Resolving then drawing exercises the actual source/template/cache chain for raster, + /// vector and elevation tiles. The fetcher is a spy: no fixture address is contacted. + #[test] + fn style_tile_destinations_are_checked_after_expansion() { + let _turn = cache::turn(); + let sent = Arc::new(Mutex::new(Vec::new())); + let observed = sent.clone(); + cache::set_fetcher(move |url, done| { + observed.lock().unwrap().push(url); + done(Ok(Vec::new())); + }); + let frame = crate::draw::tests::world_frame(); + for kind in ["raster", "vector", "raster-dem"] { + for (case, base, host, refusal) in [ + ("forbidden", "https://destinations.example/style.json", "0.0.0.0", Some("never fetched")), + ("local", "https://destinations.example/style.json", "127.0.0.1", Some("local network")), + ("public", "https://destinations.example/style.json", "tiles.example", None), + ("self-hosted", "http://localhost/style.json", "127.0.0.1", None), + ] { + let url = format!("http://{{switch:{host}}}/destination-regression/{kind}/{case}/{{z}}/{{x}}/{{y}}"); + let layer_kind = match kind { + "raster-dem" => "hillshade", + "vector" => "fill", + _ => kind, + }; + let doc = json!({"version": 8, "sources": { + // A document cannot supply the runtime provenance field itself. + "tiles": {"type": kind, "tiles": [url], "from": "http://localhost/"} + }, "layers": [{"id": "tiles", "type": layer_kind, "source": "tiles", "source-layer": "l"}]}); + let mut style = Style::parse(&doc).unwrap(); + let spec = spec(Kind::Style, &format!("{base}?{kind}-{case}")); + let sources = resolve_sources(&mut style, &spec, &address::Origin::named(base)).unwrap().unwrap(); + let r = Resolved { + style, + sprites: Vec::new(), + from: None, + attribution: Vec::new(), + left_out: sources.left_out, + made: Mutex::new(Instant::now()), + waiting: false, + }; + resolved().lock().unwrap().insert(spec.key(), Arc::new(r)); + let before = sent.lock().unwrap().len(); + let mut pm = tiny_skia::Pixmap::new(64, 64).unwrap(); + let drawn = crate::draw::draw(&mut pm, &spec, 1.0, &frame, true); + cache::wait(std::time::Duration::from_secs(2)); + if let Some(why) = refusal { + assert_eq!(sent.lock().unwrap().len(), before, "{kind}/{case} reached the fetcher"); + assert!(drawn.error.as_deref().is_some_and(|e| e.contains(why)), "{kind}/{case}: {:?}", drawn.error); + } else { + assert!(sent.lock().unwrap().len() > before, "{kind}/{case} must still fetch"); + assert!(drawn.error.is_none(), "{kind}/{case}: {:?}", drawn.error); + } + if kind == "vector" { + // A closer view uses prepared pictures. Its refusal must survive a cache + // hit, and the GPU's data-tile path must report it too. + let near = crate::draw::Frame { left: 0.0, top: 64000.0, m_per_px: 1000.0, ppp: 1.0 }; + for _ in 0..2 { + let d = crate::draw::draw(&mut pm, &spec, 1.0, &near, false); + assert_eq!(d.error.is_some(), refusal.is_some(), "prepared picture: {case}"); + } + let view = near.extent(64, 64); + let gpu = crate::shapes::vector_tiles(&spec, &near, &view, None, None).unwrap(); + assert_eq!(gpu.error.is_some(), refusal.is_some(), "GPU data tiles: {case}"); + let moved = crate::draw::Frame { m_per_px: 1100.0, ..near }; + let (d, _) = + crate::picture::paced(crate::picture::Pace::Moving, || crate::draw::draw(&mut pm, &spec, 1.0, &moved, false)); + assert_eq!(d.error.is_some(), refusal.is_some(), "moving stand-in: {case}"); + } + } + } + } + + /// A refused shard leaves its squares empty while the same source's valid squares draw. + /// Both a template list and a host switch are exercised without contacting a server. + #[test] + fn refused_tile_shards_leave_the_other_tiles_drawn() { + let _turn = cache::turn(); + let sent = Arc::new(Mutex::new(Vec::new())); + let observed = sent.clone(); + let mut tile = tiny_skia::Pixmap::new(1, 1).unwrap(); + tile.fill(tiny_skia::Color::from_rgba8(255, 0, 0, 255)); + let bytes = tile.encode_png().unwrap(); + cache::set_fetcher(move |url, done| { + observed.lock().unwrap().push(url); + done(Ok(bytes.clone())); + }); + let public = "http://partial-tiles.example/{z}/{x}/{y}"; + let from = address::Origin::named("https://partial-style.example/style.json"); + for x in 0..2 { + for y in 0..2 { + cache::get_tile_from(&format!("http://partial-tiles.example/1/{x}/{y}"), cache::Want::Image, Some(&from)); + } + } + cache::wait(std::time::Duration::from_secs(2)); + for tiles in [ + vec!["http://127.0.0.1/{z}/{x}/{y}", public], + vec!["pmtiles:{z}/{x}/{y}@https://archive-key.example/fixture.pmtiles", public], + vec!["http://{switch:127.0.0.1}/{z}/{x}/{y}", public], + vec!["http://{switch:127.0.0.1,partial-tiles.example}/{z}/{x}/{y}"], + vec![public], + ] { + let clean = tiles == [public]; + let doc = json!({"version":8,"sources":{"s":{"type":"raster","tileSize":32,"maxzoom":1,"tiles":tiles}}, + "layers":[{"id":"r","type":"raster","source":"s"}]}); + let mut style = Style::parse(&doc).unwrap(); + let spec = spec(Kind::Style, &format!("{}?{}", from.url, tiles.len())); + let sources = resolve_sources(&mut style, &spec, &from).unwrap().unwrap(); + let r = Resolved { + style, + from: Some(from.clone()), + sprites: Vec::new(), + attribution: Vec::new(), + left_out: sources.left_out, + made: Mutex::new(Instant::now()), + waiting: false, + }; + let frame = crate::draw::tests::world_frame(); + let mut pm = tiny_skia::Pixmap::new(64, 64).unwrap(); + let mut drawn = crate::draw::Drawn::default(); + crate::draw::draw_layers(Some(&mut pm), (64, 64), &frame.extent(64, 64), &r, &frame, crate::draw::Pass::All, &mut drawn); + assert_eq!(pm.pixel(48, 16).unwrap().red(), 255, "a refused shard must not hide a valid tile"); + assert_eq!(pm.pixel(16, 16).unwrap().alpha(), if clean { 255 } else { 0 }); + assert_eq!(drawn.error.is_none(), clean, "{:?}", drawn.error); + if let Some(error) = drawn.error { + assert!(error.contains("tiles left out"), "{error}"); + assert!(error.contains("local network") || error.contains("http://"), "{error}"); + } + assert!(!drawn.pending); + } + assert!(sent.lock().unwrap().iter().all(|url| url.starts_with("http://partial-tiles.example/"))); + } + + /// Refused expanded destinations enter network history once until cleared, with clean controls. + #[test] + fn expanded_refusals_enter_network_history_once() { + let _turn = cache::turn(); + cache::set_fetcher(|_, done| done(Ok(Vec::new()))); + for bad in [true, false] { + let history = rustgis_data::diagnostics::History::default(); + let _capture = history.enter(); + let host = if bad { "{switch:127.0.0.1}" } else { "diagnostic-public.example" }; + let mut style = Style::parse( + &json!({"version":8,"sources":{"s":{"type":"vector","tiles":[format!("http://{host}/{{z}}/{{x}}/{{y}}")]}},"layers":[]}), + ) + .unwrap(); + let from = address::Origin::named("https://diagnostic-style.example/style.json"); + let spec = spec(Kind::Style, &from.url); + resolve_sources(&mut style, &spec, &from).unwrap().unwrap(); + let src = &style.sources[0].1; + let frame = crate::draw::tests::world_frame(); + let view = frame.extent(64, 64); + for _ in 0..3 { + let (mut pending, mut error) = (false, None); + crate::draw::tiles_in(src, &view, None, &frame, 0.0, &mut pending, &mut error); + assert_eq!(error.is_some(), bad); + } + assert_eq!(history.records().iter().filter(|r| r.network.is_some()).count(), usize::from(bad)); + history.clear(); + let (mut pending, mut error) = (false, None); + crate::draw::tiles_in(src, &view, None, &frame, 0.0, &mut pending, &mut error); + assert_eq!( + history.records().iter().filter(|r| r.network.is_some()).count(), + usize::from(bad), + "clear allows the refusal to be reported again" + ); + } + cache::wait(std::time::Duration::from_secs(2)); + } + + /// Refused cache-only shards outside the view do not count as visible holes. + #[test] + fn cache_only_destinations_do_not_report_visible_omissions() { + let _turn = cache::turn(); + cache::set_fetcher(|_, done| done(Ok(Vec::new()))); + let mut style = Style::parse(&json!({"version":8,"sources":{"s":{"type":"vector","tiles":[ + "http://{switch:127.0.0.1}/{z}/{x}/{y}", "https://cache-only.example/{z}/{x}/{y}"]}},"layers":[]})) + .unwrap(); + let (_, src) = style.sources.first_mut().unwrap(); + if let Source::Vector { from, .. } = src { + *from = Some(address::Origin::named("https://cache-only.example/style.json")); + } + let frame = crate::draw::tests::world_frame(); + let reach = crate::template::TileId { z: 0, x: 0, y: 0 }.extent(); + for (x, refused, padding) in [(1, false, 1.0), (1, false, 0.0), (1, true, -1.0), (0, true, 1.0)] { + let mut view = crate::template::TileId { z: 1, x, y: 0 }.extent(); + view.min_x += padding; + view.max_x -= padding; + view.min_y += padding; + view.max_y -= padding; + let mut pending = false; + let mut error = None; + crate::draw::tiles_in(src, &view, Some(&reach), &frame, 1.0, &mut pending, &mut error); + assert_eq!(error.is_some(), refused, "{error:?}"); + } + cache::wait(std::time::Duration::from_secs(2)); + } + + /// A refused vector shard must not remove the style background over that square. + /// Falling back to the ordinary drawing path covers both all-refused and mixed shards. + #[test] + fn refused_vector_destinations_keep_the_style_background() { + let _turn = cache::turn(); + let tile = crate::mvt::encode(&crate::mvt::Tile { + layers: vec![crate::mvt::Layer { + name: "l".into(), + extent: 4096, + features: vec![crate::mvt::Feature { + id: None, + kind: crate::mvt::GeomKind::Polygon, + props: Vec::new(), + geometry: vec![vec![[0.0, 0.0], [4096.0, 0.0], [4096.0, 4096.0], [0.0, 4096.0], [0.0, 0.0]]], + }], + }], + }); + cache::set_fetcher(move |url, done| { + assert!(url.starts_with("http://background-public.example/"), "refused destinations never reach transport"); + done(Ok(tile.clone())); + }); + let scale = crate::template::HALF_WORLD / 512.0; + let frame = crate::draw::Frame { left: -100.0 * scale, top: 100.0 * scale, m_per_px: scale, ppp: 1.0 }; + let public = "http://background-public.example/{z}/{x}/{y}"; + let from = address::Origin::named("https://background-style.example/style.json"); + for x in 0..2 { + for y in 0..2 { + cache::get_tile_from(&format!("http://background-public.example/1/{x}/{y}"), cache::Want::Vector, Some(&from)); + } + } + cache::wait(std::time::Duration::from_secs(2)); + for (case, tiles) in [ + vec!["http://{switch:127.0.0.1}/{z}/{x}/{y}"], + vec!["http://{switch:127.0.0.1}/{z}/{x}/{y}", public], + vec!["http://127.0.0.1/{z}/{x}/{y}", public], + ] + .into_iter() + .enumerate() + { + let mixed = tiles.len() > 1; + let doc = json!({"version":8,"sources":{"s":{"type":"vector","maxzoom":1,"tiles":tiles}},"layers":[ + {"id":"bg","type":"background","paint":{"background-color":"#f00"}}, + {"id":"fill","type":"fill","source":"s","source-layer":"l","paint":{"fill-color":"#0f0"}}]}); + let mut style = Style::parse(&doc).unwrap(); + let spec = spec(Kind::Style, &format!("https://background-style.example/{case}")); + let sources = resolve_sources(&mut style, &spec, &from).unwrap().unwrap(); + let ready = Arc::new(Resolved { + style, + from: Some(from.clone()), + sprites: Vec::new(), + attribution: Vec::new(), + left_out: sources.left_out, + made: Mutex::new(Instant::now()), + waiting: false, + }); + resolved().lock().unwrap().insert(spec.key(), ready); + let view = frame.extent(200, 200); + assert!(crate::shapes::vector_tiles(&spec, &frame, &view, None, None).is_none(), "background needs complete fallback drawing"); + let mut image = tiny_skia::Pixmap::new(200, 200).unwrap(); + let drawn = crate::draw::draw(&mut image, &spec, 1.0, &frame, false); + assert_eq!(image.pixel(16, 16).unwrap().red(), 255); + if mixed { + assert_eq!(image.pixel(150, 16).unwrap().green(), 255, "permitted vector geometry still draws"); + } + assert!(drawn.error.is_some()); + } + } + + /// Pending vector tiles, including a partly cached view, leave the full background + /// visible without using a disallowed stand-in. Completed tiles can use meshes again. + #[test] + fn pending_vector_tiles_keep_the_style_background() { + let _turn = cache::turn(); + let frame = crate::draw::Frame { left: -100_000.0, top: 100_000.0, m_per_px: crate::template::HALF_WORLD / 512.0, ppp: 1.0 }; + let tile = crate::mvt::encode(&crate::mvt::Tile { + layers: vec![crate::mvt::Layer { name: "l".into(), extent: 4096, features: Vec::new() }], + }); + for (partial, ancestor) in [(false, false), (true, false), (false, true)] { + let host = format!("https://pending-background-{partial}-{ancestor}.example"); + let doc = json!({"version":8,"sources":{"s":{"type":"vector","tiles":[format!("{host}/{{z}}/{{x}}/{{y}}")],"maxzoom":1}}, + "layers":[{"id":"bg","type":"background","paint":{"background-color":"#f00"}}, + {"id":"fill","type":"fill","source":"s","source-layer":"l"}]}); + let mut style = Style::parse(&doc).unwrap(); + let spec = spec(Kind::Style, &format!("{host}/style.json")); + let from = address::Origin::named(&spec.url); + resolve_sources(&mut style, &spec, &from).unwrap().unwrap(); + resolved().lock().unwrap().insert( + spec.key(), + Arc::new(Resolved { + style, + from: Some(from), + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }), + ); + if partial || ancestor { + let ready = tile.clone(); + cache::set_fetcher(move |_, done| done(Ok(ready.clone()))); + let key = if ancestor { "0/0/0" } else { "1/1/1" }; + cache::get_tile(&format!("{host}/{key}"), cache::Want::Vector); + cache::wait(std::time::Duration::from_secs(2)); + } + let held = Arc::new(Mutex::new(Vec::new())); + let hold = held.clone(); + cache::set_fetcher(move |_, done| hold.lock().unwrap().push(done)); + let view = frame.extent(200, 200); + assert_eq!( + crate::shapes::vector_tiles(&spec, &frame, &view, None, None).is_some(), + ancestor, + "only a complete permitted stand-in can cover the background while pending" + ); + let mut image = tiny_skia::Pixmap::new(200, 200).unwrap(); + let drawn = crate::draw::draw(&mut image, &spec, 1.0, &frame, false); + assert_eq!(image.pixel(16, 16).unwrap().red(), 255); + assert!(drawn.pending && drawn.error.is_none()); + for done in std::mem::take(&mut *held.lock().unwrap()) { + done(Ok(tile.clone())); + } + cache::wait(std::time::Duration::from_secs(2)); + assert!(crate::shapes::vector_tiles(&spec, &frame, &view, None, None).is_some()); + } + } + + /// Bounded tile selection cannot turn a large view into a partial background mesh. + /// No pixmap or real network is needed to exercise the selection limit. + #[test] + fn oversized_vector_views_keep_the_style_background() { + let _turn = cache::turn(); + let tile = crate::mvt::encode(&crate::mvt::Tile { + layers: vec![crate::mvt::Layer { name: "l".into(), extent: 4096, features: Vec::new() }], + }); + for (w, h) in [(20_000.0, 10_000.0), (40_000.0, 20_000.0)] { + let host = format!("https://oversized-background-{w}.example"); + let document = json!({"version":8,"sources":{"s":{"type":"vector","tiles":[format!("{host}/{{z}}/{{x}}/{{y}}")]}}, + "layers":[{"id":"bg","type":"background"},{"id":"fill","type":"fill","source":"s","source-layer":"l"}]}); + let ready = tile.clone(); + cache::set_fetcher(move |url, done| { + done(Ok(if url.ends_with("style.json") { document.to_string().into_bytes() } else { ready.clone() })) + }); + let spec = spec(Kind::Style, &format!("{host}/style.json")); + resolve(&spec); + cache::wait(std::time::Duration::from_secs(2)); + let frame = crate::draw::Frame { left: 0.0, top: h, m_per_px: 1.0, ppp: 1.0 }; + let view = frame.extent(w as u32, h as u32); + crate::shapes::vector_tiles(&spec, &frame, &view, None, None); + cache::wait(std::time::Duration::from_secs(2)); + assert!( + crate::shapes::vector_tiles(&spec, &frame, &view, None, None).is_none(), + "bounded selection must not leave background holes in a {w} by {h} view" + ); + } + } + + /// A sprite notice does not remove otherwise usable vector tiles from the mesh path. + #[test] + fn sprite_notices_do_not_disable_vector_meshes() { + let _turn = cache::turn(); + let document = json!({"version":8,"sprite":"unsupported://sheet", + "sources":{"s":{"type":"vector","tiles":["https://notice-tiles.example/{z}/{x}/{y}"],"maxzoom":0}}, + "layers":[{"id":"bg","type":"background"},{"id":"fill","type":"fill","source":"s","source-layer":"l"}, + {"id":"icons","type":"symbol","source":"s","source-layer":"l","layout":{"icon-image":"pin"}}]}); + let tile = crate::mvt::encode(&crate::mvt::Tile { + layers: vec![crate::mvt::Layer { name: "l".into(), extent: 4096, features: Vec::new() }], + }); + cache::set_fetcher(move |url, done| { + done(Ok(if url.ends_with("style.json") { document.to_string().into_bytes() } else { tile.clone() })) + }); + let spec = spec(Kind::Style, "https://notice-style.example/style.json"); + resolve(&spec); + cache::wait(std::time::Duration::from_secs(2)); + let Resolve::Ready(ready) = resolve(&spec) else { panic!("the style must be ready") }; + assert!(!ready.left_out.is_empty()); + let frame = crate::draw::tests::world_frame(); + crate::shapes::vector_tiles(&spec, &frame, &frame.extent(64, 64), None, None); + cache::wait(std::time::Duration::from_secs(2)); + let tiles = crate::shapes::vector_tiles(&spec, &frame, &frame.extent(64, 64), None, None) + .expect("a sprite notice must not force otherwise usable geometry through CPU drawing"); + assert!(tiles.error.is_some(), "the sprite notice is retained"); + } + + /// Resolving a child beyond the saved-address limit must report that child as unusable, + /// rather than asking the cache for the supplying document as though it were a sprite. + #[test] + fn oversized_saved_s3_children_do_not_reuse_the_document() { + let href = "x".repeat(rustgis_data::s3_address::MAX_ADDRESS); + let style = Style::parse(&json!({"version":8,"sources":{},"sprite":href, + "layers":[{"id":"icons","type":"symbol","layout":{"icon-image":"pin"}}]})) + .unwrap(); + let url = + rustgis_data::s3_address::bind("0123456789abcdef", "https://s3-join.example", "https://s3-join.example/bucket/style.json") + .unwrap(); + let base = address::Origin { url, reach: address::Reach::Public }; + let (sprites, notes) = resolve_sprites(&style, &base); + assert!(sprites.is_empty(), "the document is not its own sprite: {sprites:?}"); + assert_eq!(notes.len(), 1); + assert!(notes[0].contains("won't fetch"), "{notes:?}"); + } + + /// The generated raster style names the user's selected address as both base and tile; + /// resolving it must retain one wrapper rather than nesting the same selection. + #[test] + fn direct_saved_s3_rasters_keep_one_binding() { + let _turn = cache::turn(); + let selected = rustgis_data::s3_address::bind( + "0123456789abcdef", + "https://s3-direct.example", + "https://s3-direct.example/bucket/{z}/{x}/{y}.png", + ) + .unwrap(); + let spec = spec(Kind::Raster, &selected); + let Resolve::Ready(resolved) = resolve(&spec) else { panic!("a direct raster needs no document fetch") }; + let source = &resolved.style.sources.first().unwrap().1; + let Source::Raster { tiles, .. } = source else { panic!("the generated raster source is retained") }; + assert_eq!(tiles, &[selected]); + } + + /// A direct TileJSON selection reaches the cache once with its chosen account, and + /// relative tile addresses inherit that same account from the returned document. + #[test] + fn direct_saved_s3_tilejson_keeps_its_binding() { + let _turn = cache::turn(); + let endpoint = "https://s3-direct-vector.example"; + let binding = crate::s3::new_binding().unwrap(); + let name = "direct TileJSON fixture"; + crate::s3::connect_saved( + crate::s3::Connection { + name: name.into(), + endpoint: Some(endpoint.into()), + region: "us-east-1".into(), + buckets: vec!["bucket".into()], + credentials: None, + }, + &binding, + "direct-vector", + ) + .unwrap(); + let selected = rustgis_data::s3_address::bind(&binding, endpoint, &format!("{endpoint}/bucket/tiles.json")).unwrap(); + let asked = selected.clone(); + cache::set_fetcher_with_origin(move |url, _, done, _| { + assert_eq!(url, asked, "the TileJSON request keeps exactly one selected wrapper"); + let plain = crate::s3::bound_destination(&url).unwrap().unwrap(); + let origin = address::Origin::named(&plain).retain_binding(&url).unwrap(); + done(Ok(cache::Fetched { bytes: br#"{"tiles":["{z}/{x}/{y}.pbf"]}"#.to_vec(), origin })); + }); + let spec = spec(Kind::Vector, &selected); + assert!(matches!(resolve(&spec), Resolve::Pending)); + cache::wait(std::time::Duration::from_secs(2)); + let Resolve::Ready(resolved) = resolve(&spec) else { panic!("the selected TileJSON must load") }; + let Source::Vector { tiles, .. } = &resolved.style.sources.first().unwrap().1 else { panic!("the vector source is retained") }; + let child = rustgis_data::s3_address::parse(&tiles[0]).unwrap().unwrap(); + assert_eq!((child.binding, child.url), (binding.as_str(), &*format!("{endpoint}/bucket/{{z}}/{{x}}/{{y}}.pbf"))); + crate::s3::disconnect(name); + } + + /// Label-only sources are checked too, even when the GPU or prepared pictures draw + /// a different source's shapes. Neither renderer may discard a label refusal. + #[test] + fn label_only_destinations_report_refusals_in_both_renderers() { + let _turn = cache::turn(); + cache::set_fetcher(|_, done| done(Ok(Vec::new()))); + let frame = crate::draw::Frame { left: 0.0, top: 64000.0, m_per_px: 1000.0, ppp: 1.0 }; + let view = frame.extent(64, 64); + for (host, refused) in [("0.0.0.0", true), ("labels.example", false)] { + let base = format!("https://label-destinations.example/{host}/style.json"); + let doc = json!({"version":8,"sources":{ + "shapes":{"type":"vector","tiles":["https://shapes.example/{z}/{x}/{y}"]}, + "labels":{"type":"vector","tiles":[format!("http://{{switch:{host}}}/{{z}}/{{x}}/{{y}}")]}}, + "layers":[{"id":"shapes","type":"fill","source":"shapes","source-layer":"l"}, + {"id":"labels","type":"symbol","source":"labels","source-layer":"l","layout":{"text-field":"name"}}]}); + let mut style = Style::parse(&doc).unwrap(); + let spec = spec(Kind::Style, &base); + resolve_sources(&mut style, &spec, &address::Origin::named(&base)).unwrap().unwrap(); + resolved().lock().unwrap().insert( + spec.key(), + Arc::new(Resolved { + style, + from: None, + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }), + ); + let gpu = crate::shapes::vector_tiles(&spec, &frame, &view, None, Some((64, 64))).unwrap(); + assert_eq!(gpu.error.is_some(), refused, "label-only GPU: {host}"); + let placed = crate::draw::place(&spec, 1.0, &frame, &view, &view, Some((64, 64))).unwrap(); + assert_eq!(placed.error.is_some(), refused, "label-only placement: {host}"); + } + cache::wait(std::time::Duration::from_secs(2)); + } + + /// A local style can use a public TileJSON, but that document cannot choose local tiles. + /// The metadata is read through the actual cache; only the resulting tile send is refused. + #[test] + fn tilejson_destinations_keep_the_supplying_documents_reach() { + let _turn = cache::turn(); + let sent = Arc::new(Mutex::new(Vec::new())); + let observed = sent.clone(); + let metadata = "https://tilejson-destinations.example/tiles.json"; + cache::set_fetcher(move |url, done| { + observed.lock().unwrap().push(url.clone()); + done(if url == metadata { + Ok(json!({"tiles": ["http://{switch:127.0.0.1}/tilejson-destinations/{z}/{x}/{y}"]}).to_string().into_bytes()) + } else { + Ok(Vec::new()) + }); + }); + let base = "http://localhost/tilejson-destinations/style.json"; + let s = spec(Kind::Style, base); + let doc = json!({"version": 8, "sources": {"tiles": {"type": "raster", "url": metadata}}, + "layers": [{"id": "tiles", "type": "raster", "source": "tiles"}]}); + let mut ready = None; + for _ in 0..3 { + let mut style = Style::parse(&doc).unwrap(); + if let Some(sources) = resolve_sources(&mut style, &s, &address::Origin::named(base)).unwrap() { + ready = Some(Resolved { + style, + sprites: Vec::new(), + from: None, + attribution: Vec::new(), + left_out: sources.left_out, + made: Mutex::new(Instant::now()), + waiting: false, + }); + break; + } + cache::wait(std::time::Duration::from_secs(2)); + } + resolved().lock().unwrap().insert(s.key(), Arc::new(ready.expect("metadata must resolve"))); + let mut pm = tiny_skia::Pixmap::new(64, 64).unwrap(); + let drawn = crate::draw::draw(&mut pm, &s, 1.0, &crate::draw::tests::world_frame(), false); + cache::wait(std::time::Duration::from_secs(2)); + assert!(drawn.error.as_deref().is_some_and(|e| e.contains("local network")), "{:?}", drawn.error); + assert_eq!(*sent.lock().unwrap(), [metadata], "only the metadata, not its chosen local tile, was fetched"); + } + #[test] fn an_unreadable_elevation_source_is_left_out_and_said() { let doc = serde_json::json!({"version": 8, "sources": { @@ -771,22 +1410,22 @@ mod tests { "layers": [{"id": "shade", "type": "hillshade", "source": "dem"}, {"id": "sat", "type": "raster", "source": "sat"}]}); let mut style = Style::parse(&doc).unwrap(); let base = "https://t.example.com/style.json"; - let left_out = resolve_sources(&mut style, &spec(Kind::Style, base), base).unwrap().unwrap().left_out; + let left_out = resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)).unwrap().unwrap().left_out; assert_eq!(left_out.len(), 1); assert!(left_out[0].starts_with("elevation source dem left out"), "{left_out:?}"); assert!(matches!(style.sources.iter().find(|s| s.0 == "dem").map(|s| &s.1), Some(Source::Other))); // A source the style can't draw without still fails it. let doc = serde_json::json!({"version": 8, "sources": {"sat": {"type": "raster", "url": "http://169.254.169.254/sat.json"}}, "layers": []}); let mut style = Style::parse(&doc).unwrap(); - assert!(resolve_sources(&mut style, &spec(Kind::Style, base), base).is_err()); + assert!(resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)).is_err()); // Clean sources leave nothing out. let doc = serde_json::json!({"version": 8, "sources": {"sat": {"type": "raster", "tiles": ["https://t.example.com/{z}/{x}/{y}.png"]}}, "layers": []}); let mut style = Style::parse(&doc).unwrap(); - assert_eq!(resolve_sources(&mut style, &spec(Kind::Style, base), base), Ok(Some(Sources::default()))); + assert_eq!(resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)), Ok(Some(Sources::default()))); // Only a source a layer draws from is missed; the others are dropped without a word. let left_out_of = |doc: serde_json::Value| { let mut style = Style::parse(&doc).unwrap(); - resolve_sources(&mut style, &spec(Kind::Style, base), base).unwrap().unwrap().left_out + resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)).unwrap().unwrap().left_out }; let unused = serde_json::json!({"version": 8, "sources": {"dem": {"type": "raster-dem", "url": "http://169.254.169.254/dem.json"}}, "layers": []}); assert!(left_out_of(unused).is_empty()); @@ -806,7 +1445,7 @@ mod tests { "layers": [{"id": "shade", "type": "hillshade", "source": "dem"}]}); let mut style = Style::parse(&doc).unwrap(); let base = "https://t.example.com/style.json"; - assert_eq!(resolve_sources(&mut style, &spec(Kind::Style, base), base), Ok(Some(Sources::default()))); + assert_eq!(resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)), Ok(Some(Sources::default()))); assert!( matches!(style.source("dem"), Some(Source::RasterDem { tiles, .. }) if tiles == &["https://t.example.com/{z}/{x}/{y}.png"]) ); @@ -822,14 +1461,16 @@ mod tests { "layers": [{"id": "shade", "type": "hillshade", "source": "dem"}, {"id": "sat", "type": "raster", "source": "sat"}]}); let mut style = Style::parse(&doc).unwrap(); let base = "https://t.example.com/style.json"; - let sources = resolve_sources(&mut style, &spec(Kind::Style, base), base).unwrap().expect("drawn without it meanwhile"); + let sources = resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)) + .unwrap() + .expect("drawn without it meanwhile"); assert_eq!(sources, Sources { left_out: Vec::new(), waiting: true }, "nothing lost, only waited for"); assert!(matches!(style.source("dem"), Some(Source::Other))); assert!(matches!(style.source("sat"), Some(Source::Raster { .. }))); // One no layer draws from isn't waited for. let unused = serde_json::json!({"version": 8, "sources": {"dem": {"type": "raster-dem", "url": "https://unused-on-its-way.example.com/dem.json"}}, "layers": []}); let mut style = Style::parse(&unused).unwrap(); - assert_eq!(resolve_sources(&mut style, &spec(Kind::Style, base), base), Ok(Some(Sources::default()))); + assert_eq!(resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)), Ok(Some(Sources::default()))); assert!(matches!(style.source("dem"), Some(Source::Other))); // Drawn meanwhile, pending (so it is drawn again, and an export waits), without a notice; // resolved again on the next drawing. @@ -837,6 +1478,7 @@ mod tests { let r = Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: Vec::new(), made: Mutex::new(Instant::now()), @@ -864,12 +1506,13 @@ mod tests { "paint": {"color-relief-color": ["interpolate", ["linear"], ["elevation"], 0, "#00f", 10, "#00f"]}}]}); let mut style = Style::parse(&doc).unwrap(); let base = "https://t.example.com/style.json"; - let sources = resolve_sources(&mut style, &spec(Kind::Style, base), base).unwrap().unwrap(); + let sources = resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)).unwrap().unwrap(); assert_eq!(sources.left_out.len(), 1); assert!(matches!(style.source("dem"), Some(Source::Other))); let r = Resolved { style, sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: sources.left_out, made: Mutex::new(Instant::now()), @@ -893,6 +1536,7 @@ mod tests { "layers": [{"id": "relief", "type": "color-relief", "source": "dem", "paint": {"color-relief-color": colour}}]}); let r = Resolved { style: Style::parse(&doc).unwrap(), + from: None, sprites: Vec::new(), attribution: Vec::new(), left_out: Vec::new(), @@ -921,6 +1565,7 @@ mod tests { let r = Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out, made: Mutex::new(Instant::now()), @@ -939,6 +1584,7 @@ mod tests { let stale = Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: vec!["x".into()], made: Mutex::new(long_ago), @@ -948,6 +1594,7 @@ mod tests { let complete = Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: Vec::new(), made: Mutex::new(long_ago), @@ -959,6 +1606,7 @@ mod tests { let expired = Arc::new(Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: vec!["x".into()], made: Mutex::new(long_ago), @@ -991,6 +1639,7 @@ mod tests { Arc::new(Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: left_out.iter().map(|s| s.to_string()).collect(), made: Mutex::new(at), @@ -1002,12 +1651,21 @@ mod tests { assert!(!old.current()); let r = kept(old.clone(), made(&["dem: gone"], Instant::now())); assert!(Arc::ptr_eq(&r, &old) && r.current()); + // Equal content from another origin must not retain the old root's authority. + let mut moved = made(&["dem: gone"], Instant::now()); + Arc::get_mut(&mut moved).unwrap().from = Some(address::Origin::named("https://refreshed-style.example/style.json")); + assert!(Arc::ptr_eq(&kept(old.clone(), moved.clone()), &moved)); + let mut local = made(&["dem: gone"], Instant::now()); + Arc::get_mut(&mut local).unwrap().from = + Some(address::Origin { url: "https://refreshed-style.example/style.json".into(), reach: address::Reach::Local }); + assert!(Arc::ptr_eq(&kept(local, moved.clone()), &moved), "a public refresh of the same URL drops local authority"); // Something else (or nothing) left out, or another style: the new one. let new = made(&[], Instant::now()); assert!(Arc::ptr_eq(&kept(old.clone(), new.clone()), &new)); let other = Arc::new(Resolved { style: Style::parse(&serde_json::json!({"version": 8, "sources": {}, "layers": [{"id": "bg", "type": "background"}]})).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: vec!["dem: gone".into()], made: Mutex::new(Instant::now()), @@ -1018,6 +1676,7 @@ mod tests { let arrived = Arc::new(Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: vec!["dem: gone".into()], made: Mutex::new(Instant::now()), @@ -1028,6 +1687,7 @@ mod tests { let recredited = Arc::new(Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: vec!["© Example".into()], left_out: vec!["dem: gone".into()], made: Mutex::new(Instant::now()), @@ -1045,7 +1705,7 @@ mod tests { {"id": "box", "url": "mapbox://sprites/x"}, {"id": "meta", "url": "http://169.254.169.254/s"}, {"url": "no-id"}]}); - let (sprites, said) = resolve_sprites(&Style::parse(&doc).unwrap(), base); + let (sprites, said) = resolve_sprites(&Style::parse(&doc).unwrap(), &address::Origin::named(base)); assert_eq!(sprites, [("good".to_owned(), "https://t.example.com/sprites/a".to_owned())]); assert_eq!(said.len(), 3, "{said:?}"); assert!(said[0].starts_with("1 of 4 sprites left out"), "{said:?}"); @@ -1055,7 +1715,7 @@ mod tests { let two = |layers: &serde_json::Value| { let doc = serde_json::json!({"version": 8, "sources": {}, "layers": layers, "sprite": [ {"id": "a", "url": "mapbox://sprites/a"}, {"id": "b", "url": "mapbox://sprites/b"}]}); - resolve_sprites(&Style::parse(&doc).unwrap(), base).1 + resolve_sprites(&Style::parse(&doc).unwrap(), &address::Origin::named(base)).1 }; // Said only of a style that misses them: not with no layer drawing images, or one hidden. assert!(two(&serde_json::json!([])).is_empty()); @@ -1066,7 +1726,7 @@ mod tests { assert!(said[0].starts_with("2 of 2 sprites left out") && said[0].ends_with(": a, b"), "{said:?}"); // Clean: nothing to say. let doc = serde_json::json!({"version": 8, "sources": {}, "layers": [], "sprite": "https://t.example.com/s"}); - assert!(resolve_sprites(&Style::parse(&doc).unwrap(), base).1.is_empty()); + assert!(resolve_sprites(&Style::parse(&doc).unwrap(), &address::Origin::named(base)).1.is_empty()); } /// Every sprite of a style fetched with a login (not only the first): one at the style's own @@ -1083,7 +1743,7 @@ mod tests { {"id": "near", "url": "sprites/a"}, {"id": "also", "url": format!("{origin}/other/b")}, {"id": "far", "url": "https://elsewhere.example.com/c"}]}); - let (sprites, said) = resolve_sprites(&Style::parse(&doc).unwrap(), &base); + let (sprites, said) = resolve_sprites(&Style::parse(&doc).unwrap(), &address::Origin::named(&base)); assert_eq!(sprites.len(), 3, "{said:?}"); let plain = [format!("{origin}/sprites/a"), format!("{origin}/other/b"), "https://elsewhere.example.com/c".to_string()]; for ((id, address), plain) in sprites.iter().zip(&plain) { diff --git a/crates/tiles/src/sprite.rs b/crates/tiles/src/sprite.rs index 6d2605411d..d9b0c47192 100644 --- a/crates/tiles/src/sprite.rs +++ b/crates/tiles/src/sprite.rs @@ -266,7 +266,12 @@ enum SheetLoad { /// and a sprite has it, else at 1×. Ready once none is on its way, with those that could be /// read; one that can't only loses its images. pub fn load(sprites: &[(String, String)], hi_dpi: bool) -> Load { - load_with(sprites, |base| load_sheet(base, hi_dpi)) + load_from(sprites, hi_dpi, None) +} + +/// Loads style-chosen sheets with the same reach as their supplying document. +pub(crate) fn load_from(sprites: &[(String, String)], hi_dpi: bool, from: Option<&crate::address::Origin>) -> Load { + load_with(sprites, |base| load_sheet(base, hi_dpi, from)) } fn load_with(sprites: &[(String, String)], sheet: impl Fn(&str) -> SheetLoad) -> Load { @@ -301,27 +306,29 @@ fn keep(all: &mut HashMap>, key: String, sheet: Arc, b all.insert(key, sheet); } -fn load_sheet(base: &str, hi_dpi: bool) -> SheetLoad { +/// Tries high density then normal density without changing the document's authority. +fn load_sheet(base: &str, hi_dpi: bool, from: Option<&crate::address::Origin>) -> SheetLoad { // A login revoked or not covering this address: before any cached sheet is served. if crate::logins::check(base).is_err() { return SheetLoad::Failed; } if hi_dpi { - match load_one(base, "@2x") { + match load_one(base, "@2x", from) { SheetLoad::Failed => {} other => return other, } } - load_one(base, "") + load_one(base, "", from) } -fn load_one(base: &str, density: &str) -> SheetLoad { - let key = format!("{base}|{density}"); +/// Caches decoded sheets separately by reach; both index and picture use that ceiling. +fn load_one(base: &str, density: &str, from: Option<&crate::address::Origin>) -> SheetLoad { + let key = format!("{:?}|{base}|{density}", from.map(|o| o.reach)); if let Some(s) = sprites().lock().unwrap_or_else(|p| p.into_inner()).get(&key) { return SheetLoad::Ready(s.clone()); } - let index = cache::get(&with_suffix(base, &format!("{density}.json")), Want::Json); - let sheet = cache::get(&with_suffix(base, &format!("{density}.png")), Want::Image); + let index = cache::get_from(&with_suffix(base, &format!("{density}.json")), Want::Json, from); + let sheet = cache::get_from(&with_suffix(base, &format!("{density}.png")), Want::Image, from); match (index, sheet) { (State::Ready(i), State::Ready(p)) => match (&*i, &*p) { (Payload::Json(index), Payload::Image(sheet)) => { diff --git a/crates/tiles/src/style.rs b/crates/tiles/src/style.rs index 509db2e9b5..f7bfbf6e2f 100644 --- a/crates/tiles/src/style.rs +++ b/crates/tiles/src/style.rs @@ -544,6 +544,31 @@ fn calls(n: &Node, op: Op) -> bool { } } +/// A finite upper bound for numeric results at this zoom. Unknown arithmetic stays unbounded; +/// a cap or a finite set of possible outputs can still bound a feature-dependent expression. +fn numeric_upper(n: &Node, f: &FeatureCtx) -> Option { + if free(n) { + return eval(n, f, &mut Vec::new()).as_f64().filter(|v| v.is_finite()); + } + let bound = |n| numeric_upper(n, f); + match n { + Node::Call(Op::Min, args) => args.iter().filter_map(bound).reduce(f64::min), + Node::Call(Op::Max | Op::Coalesce | Op::IsNumber, args) => numeric_upper_all(args.iter(), f), + Node::Call(Op::Case, args) => { + numeric_upper_all(args.iter().skip(1).step_by(2).chain(args.last().filter(|_| args.len() % 2 == 1)), f) + } + Node::Match(_, arms, default) => numeric_upper_all(arms.iter().map(|(_, n)| n).chain(std::iter::once(default.as_ref())), f), + Node::Step(_, first, stops) => numeric_upper_all(std::iter::once(first.as_ref()).chain(stops.iter().map(|(_, n)| n)), f), + // Interpolation and other arithmetic may create NaN or lose precision even from + // bounded inputs. Only an enclosing proven cap may narrow their full margin. + _ => None, + } +} + +fn numeric_upper_all<'a>(nodes: impl Iterator, f: &FeatureCtx) -> Option { + nodes.map(|n| numeric_upper(n, f)).try_fold(f64::NEG_INFINITY, |upper, n| Some(upper.max(n?))).filter(|v| v.is_finite()) +} + /// See [`Expr::feature_free`]. fn free(n: &Node) -> bool { match n { @@ -1258,6 +1283,8 @@ impl StyleView { pub enum Source { Vector { tiles: Vec, + /// Runtime provenance of the tile templates; `None` for a user-selected template. + from: Option, url: Option, min_zoom: u8, max_zoom: u8, @@ -1265,6 +1292,8 @@ pub enum Source { }, Raster { tiles: Vec, + /// Runtime provenance of the tile templates; `None` for a user-selected template. + from: Option, url: Option, tile_size: u32, min_zoom: u8, @@ -1274,6 +1303,8 @@ pub enum Source { /// Elevation tiles, for `hillshade` and `color-relief` layers. RasterDem { tiles: Vec, + /// Runtime provenance of the tile templates; `None` for a user-selected template. + from: Option, url: Option, tile_size: u32, min_zoom: u8, @@ -1355,14 +1386,14 @@ impl Source { let (tiles, url, attribution) = (strings(get("tiles")), str_of(get("url")), str_of(get("attribution")).unwrap_or_default()); let (min_zoom, max_zoom) = (zoom_u8(get("minzoom"), 0), zoom_u8(get("maxzoom"), 22)); match get("type").and_then(Json::as_str) { - Some("vector") => Source::Vector { tiles, url, min_zoom, max_zoom, attribution }, + Some("vector") => Source::Vector { tiles, from: None, url, min_zoom, max_zoom, attribution }, Some("raster") => { let tile_size = get("tileSize").and_then(Json::as_u64).map_or(512, |t| t.clamp(1, 4096) as u32); - Source::Raster { tiles, url, tile_size, min_zoom, max_zoom, attribution } + Source::Raster { tiles, from: None, url, tile_size, min_zoom, max_zoom, attribution } } Some("raster-dem") => { let tile_size = get("tileSize").and_then(Json::as_u64).map_or(512, |t| t.clamp(1, 4096) as u32); - Source::RasterDem { tiles, url, tile_size, min_zoom, max_zoom, attribution, encoding: DemEncoding::parse(j) } + Source::RasterDem { tiles, from: None, url, tile_size, min_zoom, max_zoom, attribution, encoding: DemEncoding::parse(j) } } _ => Source::Other, } @@ -1611,6 +1642,22 @@ impl StyleLayer { self.paint.get(name).is_some_and(Expr::uses_zoom) } + /// A conservative width at this zoom, including the property's default when evaluation + /// fails. `None` leaves an unbounded expression to the drawing's full margin. + pub(crate) fn paint_width_bound(&self, name: &str, zoom: f32, default: f32) -> Option { + let f = FeatureCtx::zoom_only(zoom); + let Some(expr) = self.paint.get(name) else { return Some(default) }; + if expr.feature_free() { + return Some(self.width(name, &f, default)); + } + numeric_upper(&expr.0, &f).map(|v| (v as f32).max(default)).filter(|v| v.is_finite()) + } + + /// Whether a layout property can differ between features at the same zoom. + pub(crate) fn layout_uses_features(&self, name: &str) -> bool { + self.layout.get(name).is_some_and(|expr| !expr.feature_free()) + } + /// Whether the paint property `name` asks the sprite for an image. pub fn paint_uses_image(&self, name: &str) -> bool { self.paint.get(name).is_some_and(Expr::uses_image) @@ -2390,6 +2437,7 @@ mod tests { st.source("r"), Some(&Source::Raster { tiles: vec!["https://r/{z}/{x}/{y}.png".into()], + from: None, url: None, tile_size: 256, min_zoom: 0, @@ -2404,6 +2452,7 @@ mod tests { v, Source::Vector { tiles: vec!["https://x/{z}/{x}/{y}.pbf".into()], + from: None, url: Some("https://x/tiles.json".into()), min_zoom: 0, max_zoom: 14, diff --git a/crates/tiles/src/template.rs b/crates/tiles/src/template.rs index 279915067a..e644599714 100644 --- a/crates/tiles/src/template.rs +++ b/crates/tiles/src/template.rs @@ -199,31 +199,79 @@ pub fn expand(template: &str, t: TileId, tile_size: u32, tms: bool, subdomains: } /// Resolves `href` against the document at `base` (for addresses inside a style or TileJSON). +/// An invalid bound base stays refused by request validation rather than resolving anonymously. pub fn join(base: &str, href: &str) -> String { - let href = crate::logins::inner(href); + let (plain_base, anonymous_base) = crate::s3::split_anonymous(base); + let (plain_href, anonymous_href) = crate::s3::split_anonymous(href); + if anonymous_base || anonymous_href { + if crate::s3::split_anonymous(plain_base).1 { + return base.to_string(); + } + if crate::s3::split_anonymous(plain_href).1 { + return href.to_string(); + } + // An anonymous child narrows authority, including when its parent chose an account. + let base = crate::logins::inner(rustgis_data::s3_address::inner(plain_base)); + return crate::s3::anonymous(&join(base, plain_href)); + } + // Only the supplying document's base carries authority. A link may name a resource, + // but neither a service-login nor an S3 selector in it can choose an account. + let href = match rustgis_data::login_address::parse(href) { + Ok(Some(bound)) => bound.url, + Err(_) => return href.to_string(), + Ok(None) => href, + }; + let href = match rustgis_data::s3_address::parse(href) { + Ok(Some(bound)) => bound.url, + Err(_) => return href.to_string(), + Ok(None) => href, + }; + match rustgis_data::s3_address::parse(base) { + Ok(Some(bound)) => { + let joined = join(bound.url, href); + return crate::address::inherit(base, &joined).unwrap_or_else(|_| { + // As for bound service logins, keep the actual derived address even when + // it exceeds the parser's limit. Request validation then refuses it; never + // substitute the parent document or drop the selected account. + format!("{}{}:{}:{}:{joined}", rustgis_data::s3_address::PREFIX, bound.binding, bound.endpoint.len(), bound.endpoint) + }); + } + Err(_) => return base.to_string(), + Ok(None) => {} + } match rustgis_data::login_address::parse(base) { Ok(Some(bound)) => return crate::logins::inherit(base, &join(bound.url, href)), Err(_) => return base.to_string(), Ok(None) => {} } - if href.contains("://") { + // URL-like text in the query or fragment is data, not an absolute reference. + let (href_path, tail) = match href.find(['?', '#']) { + Some(i) => (href.get(..i).unwrap_or(href), href.get(i..).unwrap_or("")), + None => (href, ""), + }; + if href_path.contains("://") { return href.to_string(); } let Some((scheme, rest)) = base.split_once("://") else { return href.to_string() }; - let host_end = rest.find('/').unwrap_or(rest.len()); + let host_end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); let host = rest.get(..host_end).unwrap_or(rest); if let Some(path) = href.strip_prefix("//") { return format!("{scheme}://{path}"); } - // The path (with `.` and `..` resolved, RFC 3986) and what follows it. - let (href_path, tail) = match href.find(['?', '#']) { - Some(i) => (href.get(..i).unwrap_or(href), href.get(i..).unwrap_or("")), - None => (href, ""), - }; if href.starts_with('/') { return format!("{scheme}://{host}{}{tail}", remove_dots(href_path)); } let path = rest.get(host_end..).unwrap_or("").split(['?', '#']).next().unwrap_or(""); + if href_path.is_empty() { + // RFC 3986 §5.2.2: an empty reference path keeps the document, and keeps its + // query only when the reference supplies none. A base fragment is never inherited. + let query = if href.starts_with('?') { + "" + } else { + rest.split('#').next().and_then(|r| r.find('?').and_then(|i| r.get(i..))).unwrap_or("") + }; + return format!("{scheme}://{host}{path}{query}{tail}"); + } let dir = path.rfind('/').map_or("/", |i| path.get(..=i).unwrap_or("/")); format!("{scheme}://{host}{}{tail}", remove_dots(&format!("{dir}{href_path}"))) } @@ -256,6 +304,40 @@ fn remove_dots(path: &str) -> String { mod tests { use super::*; + /// Empty-path references keep the document path and inherit only an absent query. + #[test] + fn empty_path_links_keep_the_bound_document_path() { + let url = "https://s3.example/dir/style.json?old=1#old"; + for base in [ + url.to_string(), + rustgis_data::s3_address::bind("0123456789abcdef", "https://s3.example", url).unwrap(), + rustgis_data::login_address::bind("0123456789abcdef", "https://s3.example", url).unwrap(), + crate::s3::anonymous(url), + ] { + for (href, expected) in [ + ("?f=json", "https://s3.example/dir/style.json?f=json"), + ("?next=https://other.example/tiles", "https://s3.example/dir/style.json?next=https://other.example/tiles"), + ("#https://other.example/tiles", "https://s3.example/dir/style.json?old=1#https://other.example/tiles"), + ("#new", "https://s3.example/dir/style.json?old=1#new"), + ("", "https://s3.example/dir/style.json?old=1"), + ("?", "https://s3.example/dir/style.json?"), + ("#", "https://s3.example/dir/style.json?old=1#"), + ] { + let joined = join(&base, href); + let expected = if let Some(bound) = rustgis_data::s3_address::parse(&base).unwrap() { + rustgis_data::s3_address::bind(bound.binding, bound.endpoint, expected).unwrap() + } else if let Some(bound) = rustgis_data::login_address::parse(&base).unwrap() { + rustgis_data::login_address::bind(bound.binding, bound.origin, expected).unwrap() + } else if crate::s3::split_anonymous(&base).1 { + crate::s3::anonymous(expected) + } else { + expected.to_string() + }; + assert_eq!(joined, expected, "reference {href:?} keeps the document's authority wrapper"); + } + } + } + #[test] fn derived_links_keep_only_the_document_binding() { let base = rustgis_data::login_address::bind("0123456789abcdef", "https://a.test", "https://a.test/style.json").unwrap(); @@ -272,6 +354,29 @@ mod tests { assert_eq!(join("rustgis-login:broken", "/tile"), "rustgis-login:broken"); } + /// A document may name a resource, but cannot choose another saved S3 account for it. + #[test] + fn document_s3_selectors_do_not_choose_an_account() { + let endpoint = "https://s3-links.example"; + let base = rustgis_data::s3_address::bind("0123456789abcdef", endpoint, &format!("{endpoint}/bucket/style.json")).unwrap(); + for url in [format!("{endpoint}/bucket/tile"), "https://other.example/tile".into()] { + let supplied = rustgis_data::s3_address::bind("fedcba9876543210", endpoint, &url).unwrap(); + assert_eq!(join("https://public.example/style.json", &supplied), url, "the supplied selector is removed"); + let joined = join(&base, &supplied); + let selected = rustgis_data::s3_address::parse(&joined).unwrap(); + if url.starts_with(endpoint) { + assert_eq!(selected.unwrap().binding, "0123456789abcdef", "only the document's own account is inherited"); + } else { + assert!(selected.is_none(), "a different server has no inherited account"); + } + } + for malformed in ["rustgis-s3:broken", "rustgis-login:broken"] { + let joined = join(&base, malformed); + assert_eq!(joined, malformed, "malformed selectors remain refused, not relative paths"); + assert!(crate::address::check(&joined, None).is_err()); + } + } + #[test] fn joins_relative_addresses() { let base = "https://x.org/a/b/styles/root.json"; @@ -345,5 +450,23 @@ mod tests { assert_eq!(join("https://a.com/styles/x", "/planet"), "https://a.com/planet"); assert_eq!(join("https://a.com/x", "//b.com/y"), "https://b.com/y"); assert_eq!(join("https://a.com/x", "https://c.com/z"), "https://c.com/z"); + assert_eq!(join("https://a.com/x", "tiles.json?next=https://b.com"), "https://a.com/tiles.json?next=https://b.com"); + assert_eq!(join("https://a.com/x", "/tiles.json#https://b.com"), "https://a.com/tiles.json#https://b.com"); + assert_eq!(join("https://tiles.example?token=x", "/tile/0"), "https://tiles.example/tile/0"); + assert_eq!(join("https://tiles.example#view", "tile/0"), "https://tiles.example/tile/0"); + } + + #[test] + fn direct_anonymous_sources_keep_one_marker() { + let anonymous = crate::s3::anonymous("https://tiles.example/{z}/{x}/{y}"); + assert_eq!(join(&anonymous, &anonymous), anonymous); + assert_eq!(join("https://tiles.example/style.json", &anonymous), anonymous); + assert_eq!(join(&anonymous, "tile/0"), crate::s3::anonymous("https://tiles.example/{z}/{x}/tile/0")); + let malformed_child = crate::s3::anonymous(&anonymous); + let joined = join(&crate::s3::anonymous("https://tiles.example/style.json"), &malformed_child); + assert!(crate::address::check(&joined, None).is_err(), "a malformed child was replaced with {joined}"); + assert_eq!(joined, malformed_child); + let malformed_base = crate::s3::anonymous(&crate::s3::anonymous("https://tiles.example/style.json")); + assert!(crate::address::check(&join(&malformed_base, "tile/0"), None).is_err()); } } diff --git a/crates/tiles/src/warp.rs b/crates/tiles/src/warp.rs index fa31acbbb0..b55fb10e9f 100644 --- a/crates/tiles/src/warp.rs +++ b/crates/tiles/src/warp.rs @@ -50,8 +50,8 @@ pub(crate) fn forget() { } /// The map tiles made without a picture that failed: when each was made, and it. -fn made_for_now() -> &'static Mutex)>> { - static KEPT: OnceLock)>>> = OnceLock::new(); +fn made_for_now() -> &'static Mutex, String)>> { + static KEPT: OnceLock, String)>>> = OnceLock::new(); KEPT.get_or_init(Mutex::default) } @@ -72,7 +72,7 @@ pub(crate) enum Made { Done(Payload), /// Made without a picture that failed (the service may answer when asked again): shown, not /// kept in the cache, and made again once the picture is asked for again ([`FOR_NOW`]). - ForNow(Arc), + ForNow(Arc, String), Failed(String), } @@ -81,11 +81,14 @@ pub(crate) enum Made { /// [`missing`]) is left clear, and one that failed otherwise too, the tile made again once it is /// asked for again; with none there at all, the tile is empty when they were all missing, else /// it fails. -pub(crate) fn make(url: &str) -> Made { - if let Some((at, tile)) = made_for_now().lock().unwrap_or_else(|e| e.into_inner()).get(url) +pub(crate) fn make(url: &str, from: Option<&crate::address::Origin>) -> Made { + // Partial composites are shared only with callers holding the same reach. The plan is + // geometry alone and can be shared; its pictures always pass the inherited boundary. + let key = format!("{:?}|{url}", from.map_or(crate::address::Reach::Local, |o| o.reach)); + if let Some((at, tile, note)) = made_for_now().lock().unwrap_or_else(|e| e.into_inner()).get(&key) && at.elapsed() < FOR_NOW { - return Made::ForNow(tile.clone()); + return Made::ForNow(tile.clone(), note.clone()); } let known = plans().lock().unwrap_or_else(|e| e.into_inner()).get(url).cloned(); let plan = match known { @@ -104,9 +107,11 @@ pub(crate) fn make(url: &str) -> Made { } let mut pictures = Vec::with_capacity(plan.parts.len()); let (mut waiting, mut error) = (false, None); + let mut failed = 0; for p in &plan.parts { // A matrix tile past the service's coverage is routine; a WMS picture (any area asked) isn't. - let state = if p.tile.is_some() { cache::get_tile(&p.url, Want::Image) } else { cache::get(&p.url, Want::Image) }; + let state = + if p.tile.is_some() { cache::get_tile_from(&p.url, Want::Image, from) } else { cache::get_from(&p.url, Want::Image, from) }; match state { State::Ready(payload) => pictures.push(Some(payload)), State::Pending => waiting = true, @@ -117,7 +122,8 @@ pub(crate) fn make(url: &str) -> Made { // for any raster tile, the map's layer list doesn't name it: a layer drawn in its // own system does, see `native_tiles`.) if p.tile.is_none() || !missing(&e) { - error = Some(e); + failed += 1; + error.get_or_insert(e); } pictures.push(None); } @@ -150,17 +156,18 @@ pub(crate) fn make(url: &str) -> Made { let tile = compose(&plan, &pictures, SIZE).map_or(Payload::Empty, Payload::Image); let mut kept = made_for_now().lock().unwrap_or_else(|e| e.into_inner()); if !for_now { - kept.remove(url); + kept.remove(&key); return Made::Done(tile); } // Those past their while first, then all if it is still full. - kept.retain(|_, (at, _)| at.elapsed() < FOR_NOW); + kept.retain(|_, (at, _, _)| at.elapsed() < FOR_NOW); if kept.len() >= MAX_FOR_NOW { kept.clear(); } let tile = Arc::new(tile); - kept.insert(url.to_string(), (web_time::Instant::now(), tile.clone())); - Made::ForNow(tile) + let note = format!("{failed} warped tile components left out: {}", error.unwrap_or_default()); + kept.insert(key, (web_time::Instant::now(), tile.clone(), note.clone())); + Made::ForNow(tile, note) } /// Whether a WMTS tile's failure means the service doesn't have it (past its coverage): a 4xx diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 6af4a9d50b..20120fa937 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -627,6 +627,21 @@ database is kept without its password. The commands `sources.list`, `sources.sav `sources.favourite` and `sources.remove` read and change the lists, for the command line and the [AI Assistant](assistant.md). On the command line they last for the run. +### Network downloads + +On desktop and the command line, web tiles, service documents and file downloads through +**From URL…** connect directly, including `layer.addUrl`. These requests do not use +`HTTP_PROXY`, `HTTPS_PROXY` or `ALL_PROXY`: a proxy could resolve a destination beyond the +app's IP checks. There is no proxy override for these requests. If your network requires a +proxy, download a file with a trusted external tool and open the local copy in RustGIS. +Live tile and service access requires direct connectivity. The web app uses the browser's +network settings. Browser GET and HEAD requests refuse redirects, including HTTP-to-HTTPS upgrades +and CDN hops, because the app cannot inspect their destinations. This includes **From URL…** +file downloads, fonts, service documents, tiles and sprites, and PMTiles/COG archive range +reads. Supply the final address; for a redirecting file download, download it with a trusted +external tool and open the local copy. The browser also controls DNS and local-network access; +see [tile request limits](maps.md#basemaps). + ### HTTPS certificates On desktop and the command line, RustGIS's HTTPS clients use platform certificate verification: diff --git a/docs/user-guide/maps.md b/docs/user-guide/maps.md index a5d166c3f2..21688ae1b9 100644 --- a/docs/user-guide/maps.md +++ b/docs/user-guide/maps.md @@ -279,6 +279,33 @@ Ocean, Atlas, Blank), which need no network, and web tile layers over them: HTTPS tiles use [platform certificate trust](data.md#https-certificates) on desktop and the command line, including private CAs trusted by the operating system. The web app uses the browser. +Tile addresses are checked after filling their placeholders, before cached bytes or network +requests are used. In the native app, a document on the internet cannot select a local +destination IP. Explicitly selected self-hosted servers remain supported, including ordinary +hostnames resolved by local DNS, with local fallback when the same hostname also returns +public addresses. Native fetchers filter the resolved IPs before connecting, +using the supplying document's actual connection reach. Native requests refuse +link-local and unspecified addresses, including those represented by IPv4-mapped IPv6 +addresses or the well-known NAT64 prefix. Public IPv4 destinations under that prefix can pass. +Document redirects also change the base for relative references. +IPv4 decoding covers mapped addresses and the well-known `64:ff9b::/96` prefix. It does not +inspect destinations behind 6to4 tunnels or custom translation gateways, including the +[RFC 8215](https://www.rfc-editor.org/rfc/rfc8215#section-5) `64:ff9b:1::/48` prefix, whose +translation is network-specific. Such routes can hide a private or link-local IPv4 destination +behind a public-looking IPv6 address; restrictions on those routes require network firewall policy. +Native redirects are checked at each hop. In the web app, use the final service address: +tile, style, TileJSON and sprite redirects, including HTTP-to-HTTPS upgrades and CDN hops, +are refused because the browser hides their destinations from the app. The same browser +transport refuses redirects for downloads, fonts, archive range reads and HEAD requests for +file metadata; see +[network downloads](data.md#network-downloads). +The web app checks the URL's spelling and normalized address, but cannot inspect resolved IPs. +A public-looking hostname can reach a private or link-local server if the browser permits it; +DNS resolution, local-network permissions and certificate trust are controlled by the browser. +Native tile and document requests, and file downloads through **From URL…**, +connect directly without an environment HTTP proxy, so a proxy cannot resolve destinations +beyond these checks. See [network downloads](data.md#network-downloads) if your network requires a proxy. + The gallery's **Web Basemaps** need no API key: the OpenFreeMap vector maps (Liberty, Bright, Positron, Dark, Fiord) and USGS Imagery. Choosing one replaces the other web basemaps; tick **Stack on the current basemaps** above them to add it over the others instead (clicking one