From 9ea3911af1990adaaf26067aa85a2b0a3d861c80 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 7 Nov 2026 12:39:47 -0600 Subject: [PATCH 01/10] fix(tiles): enforce request destination provenance --- apps/rustgis-web/src/web.rs | 37 +- crates/render/src/tile_mesh.rs | 2 + crates/tiles/src/address.rs | 49 ++- crates/tiles/src/cache.rs | 654 ++++++++++++++++++++++++++++----- crates/tiles/src/draw.rs | 109 ++++-- crates/tiles/src/picture.rs | 64 +++- crates/tiles/src/shapes.rs | 19 +- crates/tiles/src/source.rs | 273 ++++++++++++-- crates/tiles/src/sprite.rs | 23 +- crates/tiles/src/style.rs | 14 +- crates/tiles/src/warp.rs | 14 +- docs/user-guide/maps.md | 12 + 12 files changed, 1047 insertions(+), 223 deletions(-) diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index ab41fdc11e..ffa648a257 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -601,7 +601,7 @@ async fn fetch_bytes( let mut diagnostics = RequestDiagnostics::with_capture(url, "GET", capture); diagnostics.omit_statuses(omit); let answer = async { - use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{cache, logins}; + use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{address, cache, logins}; let (promise, abort, range, tail, max, url, bound, authenticated) = crate::request::prepare_and_send( &still, || async { @@ -617,6 +617,7 @@ async fn fetch_bytes( let bound = url; let prepared = rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::s3::prepare(logins::inner(url))?; let url = prepared.url.as_str(); + check_destination(url)?; // An ArcGIS sign-in's token, for the portal and the servers it covers, renewed first when it // is about to expire. It goes as the request's `token` parameter, not a header: ArcGIS's // query endpoints answer the header's CORS preflight with an error. @@ -631,6 +632,10 @@ async fn fetch_bytes( if prepared.signed { None } else { rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::arcgis_auth::with_token(login_url) }; // Only the request carries credentials: messages and cache keys keep the original address. let request = arcgis.as_deref().unwrap_or(login_url); + // URL parsing in a browser can normalize a numeric or escaped hostname. Judge + // that actual destination against the already checked, prepared address. + let normalized = web_sys::Url::new(request).map_err(|_| "not a valid request address")?; + address::check(&normalized.href(), Some(url))?; let authenticated = prepared.signed || arcgis.is_some() || login.is_some(); let err = |e: wasm_bindgen::JsValue| { if authenticated { "the browser couldn't process the authenticated request".to_string() } else { format!("{e:?}") } @@ -662,10 +667,9 @@ async fn fetch_bytes( } init.set_headers(&headers); } - // A signed or authorized request goes only where its credentials are for. - if authenticated { - init.set_redirect(web_sys::RequestRedirect::Error); - } + // Fetch hides a manual redirect's Location, so the app cannot check its next + // destination. Refuse it instead of letting the browser follow it unchecked. + init.set_redirect(if authenticated { web_sys::RequestRedirect::Error } else { web_sys::RequestRedirect::Manual }); // A credential-bearing address goes nowhere else: no referrer from it, no cookies with it. if arcgis.is_some() || login.is_some() { init.set_referrer_policy(web_sys::ReferrerPolicy::NoReferrer); @@ -704,6 +708,7 @@ async fn fetch_bytes( .dyn_into() .map_err(err)?; diagnostics.answered(resp.status()); + check_redirect(&resp)?; if !resp.ok() { return Err(refused(&resp)); } @@ -807,6 +812,7 @@ async fn post_json( let target = logins::inherit(url, &prepared.url); let login = logins::apply(&target).map_err(str::to_string)?; let request = prepared.url.as_str(); + check_destination(request)?; let err = |e: wasm_bindgen::JsValue| { if login.is_some() { "the browser couldn't process the authenticated request".to_string() } else { format!("{e:?}") } }; @@ -887,6 +893,7 @@ async fn head( return Ok(None); } let url = prepared.url.as_str(); + check_destination(url)?; let target = logins::inherit(bound, url); let login = logins::apply(&target).map_err(str::to_string)?; let request = login.as_ref().map_or(url, |applied| applied.url.as_str()); @@ -895,6 +902,7 @@ async fn head( }; let init = web_sys::RequestInit::new(); init.set_method("HEAD"); + init.set_redirect(if login.is_some() { web_sys::RequestRedirect::Error } else { web_sys::RequestRedirect::Manual }); let login_header = login.as_ref().and_then(|applied| applied.header.as_ref()); if !prepared.headers.is_empty() || login_header.is_some() { let headers = web_sys::Headers::new().map_err(err)?; @@ -909,7 +917,6 @@ async fn head( init.set_headers(&headers); } if login.is_some() { - init.set_redirect(web_sys::RequestRedirect::Error); init.set_referrer_policy(web_sys::ReferrerPolicy::NoReferrer); init.set_credentials(web_sys::RequestCredentials::Omit); } @@ -933,6 +940,7 @@ async fn head( .dyn_into() .map_err(err)?; diagnostics.answered(resp.status()); + check_redirect(&resp)?; if !resp.ok() { return Err(refused(&resp)); } @@ -953,6 +961,23 @@ async fn head( diagnostics.record(answer.map_err(|e| Failure::new(url, e))).map_err(|failure| failure.message) } +/// Checks the URL as the browser parses it, before any request or login renewal is sent. +fn check_destination(url: &str) -> Result<(), String> { + use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::address; + let normalized = web_sys::Url::new(url).map_err(|_| "not a valid request address")?; + address::check(&normalized.href(), Some(url)) +} + +/// A manual redirect is an opaque, zero-status response: Fetch exposes no Location to check. +/// Other zero-status responses are unusable too; none is treated as a successful download. +fn check_redirect(response: &web_sys::Response) -> Result<(), String> { + if response.status() == 0 { + Err("The browser cannot verify this redirect's destination. Use the final service address.".into()) + } else { + Ok(()) + } +} + /// Makes an HTTP request (the AI assistant's calls to its model), returning the response of any /// status. async fn http( diff --git a/crates/render/src/tile_mesh.rs b/crates/render/src/tile_mesh.rs index 707dde7d52..28c3fa1627 100644 --- a/crates/render/src/tile_mesh.rs +++ b/crates/render/src/tile_mesh.rs @@ -683,6 +683,7 @@ mod tests { let layer = serde_json::json!({"id": "p", "type": "fill", "source": "s", "source-layer": "l", "paint": paint}); let sprites = vec![("default".to_owned(), address.clone())]; Arc::new(Resolved { + from: None, style: Style { name: String::new(), sources: vec![("s".into(), Source::parse(&serde_json::json!({"type": "vector", "tiles": ["x/{z}/{x}/{y}"]})))], @@ -713,6 +714,7 @@ mod tests { use rustgis_tiles::style::{Source, Style, StyleLayer}; let layer = serde_json::json!({"id": "p", "type": "fill", "source": "s", "source-layer": "parks", "paint": {"fill-color": "#f00"}}); let style = Arc::new(Resolved { + from: None, style: Style { name: String::new(), sources: vec![("s".into(), Source::parse(&serde_json::json!({"type": "vector", "tiles": ["x/{z}/{x}/{y}"]})))], diff --git a/crates/tiles/src/address.rs b/crates/tiles/src/address.rs index 753dc6d1fb..1ed8c7eaaa 100644 --- a/crates/tiles/src/address.rs +++ b/crates/tiles/src/address.rs @@ -7,8 +7,11 @@ //! can't make the app probe local services. Link-local addresses, where cloud metadata services //! answer, are never fetched. //! -//! Hosts are judged by name or literal address; names are not resolved, so a public name that -//! resolves to a local address is not caught here. +//! These spelling checks also run after template expansion. Native fetchers additionally +//! filter the resolved IPs before connecting: a public document cannot select a local +//! connection, and no request can connect to a forbidden address. The browser controls its +//! own DNS and connections; its adapter checks the normalized URL and refuses redirects whose +//! destinations it cannot inspect. use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; @@ -23,6 +26,26 @@ pub enum Reach { Forbidden, } +/// Where a fetched document actually came from. Runtime only: a document cannot grant +/// itself a local-network allowance by including a field in its JSON. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Origin { + pub(crate) url: String, + pub(crate) reach: Reach, +} + +impl Origin { + /// Origin for transports without redirects or DNS visibility, such as the browser. + pub(crate) fn named(url: &str) -> Self { + Self { url: url.to_owned(), reach: reach(crate::logins::inner(url)) } + } + + /// Checks a descendant without replacing the document's observed reach with host spelling. + pub(crate) fn check(&self, url: &str) -> Result<(), String> { + check_reach(url, self.reach) + } +} + /// The host of an HTTP(S) address, without user info, port or IPv6 brackets. fn host(url: &str) -> Option { let rest = url.split_once("://")?.1; @@ -113,15 +136,19 @@ fn inet_aton(h: &str) -> Option { /// Whether `url` may be fetched. `from` is where it was found (a document, or the address that /// redirected to it), or `None` when the user gave it. pub fn check(url: &str, from: Option<&str>) -> Result<(), String> { + check_reach(url, from.map_or(Reach::Local, |u| reach(crate::logins::inner(u)))) +} + +/// Unwraps runtime addresses and applies one inherited reach ceiling through every wrapper. +fn check_reach(url: &str, ceiling: Reach) -> Result<(), String> { crate::logins::check(url).map_err(str::to_string)?; if let Some(bound) = rustgis_data::login_address::parse(url).map_err(str::to_string)? { - return check(bound.url, from.map(crate::logins::inner)); + return check_reach(bound.url, ceiling); } - let from = from.map(crate::logins::inner); // An address read through a saved S3 connection is judged by where it is fetched, which its // own endpoint decides (no login is needed to keep it). if let Some(destination) = crate::s3::bound_destination(url)? { - return check(&destination, from); + return check_reach(&destination, ceiling); } // An `s3://` address is judged by the web address it is fetched from. // An address read anonymously is judged as the address it marks. @@ -129,10 +156,10 @@ pub fn check(url: &str, from: Option<&str>) -> Result<(), String> { if rustgis_data::s3_address::is_wrapper(inner) { return Err("not a saved S3 address".into()); } - return if crate::s3::is_s3(inner) { check(&crate::s3::https_url(url)?, from) } else { check(inner, from) }; + return if crate::s3::is_s3(inner) { check_reach(&crate::s3::https_url(url)?, ceiling) } else { check_reach(inner, ceiling) }; } if crate::s3::is_s3(url) { - return check(&crate::s3::https_url(url)?, from); + return check_reach(&crate::s3::https_url(url)?, ceiling); } let lower = url.to_ascii_lowercase(); if !(lower.starts_with("https://") || lower.starts_with("http://")) { @@ -141,11 +168,9 @@ pub fn check(url: &str, from: Option<&str>) -> Result<(), String> { if url.chars().any(|c| c.is_whitespace() || c.is_control()) { return Err(format!("{url}: the address has spaces in it")); } - match (reach(url), from) { - (Reach::Forbidden, _) => Err(format!("{url}: that address is never fetched")), - (Reach::Local, Some(origin)) if reach(origin) == Reach::Public => { - Err(format!("{url}: a document on the web can't point into the local network")) - } + match reach(url) { + Reach::Forbidden => Err(format!("{url}: that address is never fetched")), + Reach::Local if ceiling == Reach::Public => Err(format!("{url}: a document on the web can't point into the local network")), _ => Ok(()), } } diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index d3d89c9250..1ee1e42754 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -52,6 +52,13 @@ pub enum State { /// Called once with a fetch's bytes or error. pub type Done = Box, String>) + Send>; + +#[derive(Debug)] +struct Fetched { + bytes: Vec, + origin: crate::address::Origin, +} +type FetchedDone = Box) + Send>; /// Called once with a download's bytes or what is known of its failure (#509). pub type DoneDetailed = Box, Failure>) + Send>; // The carrier lives with source I/O so lower-layer workers inherit the same context. @@ -99,13 +106,15 @@ pub const DROPPED: &str = "dropped: the view moved on"; pub const NO_RANGES: &str = "the server doesn't answer byte-range requests"; enum Entry { - Ready(Arc, usize), + Ready(Arc, usize, Option), Pending, Failed(String, Instant), } struct Slot { entry: Entry, + /// A restrictive request's refusal must not poison a later explicit local request. + ceiling: crate::address::Reach, used: u64, /// When the map last asked for it ([`get`]): a waiting tile not asked for lately is dropped. asked: Instant, @@ -379,7 +388,7 @@ pub fn in_flight() -> usize { /// to try again). Every failed fetch is recorded in the asking session's diagnostics: use /// [`get_tile`] for a map tile, whose 404 is routine. pub fn get(url: &str, want: Want) -> State { - request(url, want, &[]) + request(url, want, &[], None).0 } /// Like [`get`], for a map tile: a server's 404 (no tile there) or 410 (no longer there) for it @@ -389,22 +398,43 @@ pub fn get(url: &str, want: Want) -> State { /// a tile (a single-image layer, a layout's picture, a style's sprite sheet) goes through /// [`get`], so its 404 is recorded. pub fn get_tile(url: &str, want: Want) -> State { - request(url, want, TILE_OMIT) + get_tile_from(url, want, None) +} + +/// Requests a tile under its supplying document's reach, with routine missing-tile statuses. +pub(crate) fn get_tile_from(url: &str, want: Want, from: Option<&crate::address::Origin>) -> State { + request(url, want, TILE_OMIT, from).0 +} + +/// Requests a document-selected asset without granting it an explicit local allowance. +pub(crate) fn get_from(url: &str, want: Want, from: Option<&crate::address::Origin>) -> State { + request(url, want, &[], from).0 +} + +/// Returns JSON and its effective origin from the same cache entry and lock acquisition. +pub(crate) fn document(url: &str, from: Option<&crate::address::Origin>) -> (State, Option) { + request(url, Want::Json, &[], from) } /// The statuses a map tile's request omits ([`get_tile`]). const TILE_OMIT: &[u16] = &[404, 410]; /// [`get`] with the statuses that are expected answers for this fetch. -fn request(url: &str, want: Want, omit: &'static [u16]) -> State { +fn request(url: &str, want: Want, omit: &'static [u16], from: Option<&crate::address::Origin>) -> (State, Option) { if let Err(e) = crate::logins::check(resource_of(url)) { - return State::Failed(e.to_string()); + return (State::Failed(e.to_string()), None); } // A map tile of a layer in another system: its service's picture as it comes, or made from // its pictures (#769). if let Some(base) = crate::warp::direct(url) { - return request(base, want, omit); + return request(base, want, omit, from); } + if let Some(from) = from + && let Err(e) = from.check(url) + { + return refused_request(url, e); + } + let ceiling = from.map_or(crate::address::Reach::Local, |o| o.reach); let now = Instant::now(); let mut c = lock(); c.tick += 1; @@ -414,16 +444,23 @@ fn request(url: &str, want: Want, omit: &'static [u16]) -> State { slot.used = tick; slot.asked = now; match &slot.entry { - Entry::Ready(p, _) => return State::Ready(p.clone()), - Entry::Pending => return State::Pending, - Entry::Failed(e, at) if at.elapsed() < retry_wait(e, RETRY) => { + Entry::Ready(p, _, origin) => { + if origin.as_ref().is_some_and(|o| o.reach > ceiling) { + let message = format!("{url}: a document on the web can't read a cached local-network answer"); + drop(c); + return refused_request(url, message); + } + return (State::Ready(p.clone()), origin.clone()); + } + Entry::Pending => return (State::Pending, None), + Entry::Failed(e, at) if ceiling <= slot.ceiling && at.elapsed() < retry_wait(e, RETRY) => { // A server that said when to come back (429 or 503 with `Retry-After`) is asked // again then: the map draws again at that time, though nothing else draws it. // One that didn't is asked again by the next draw after a minute, as before. if let Some(wait) = retry_after(e) { notify_at(*at + wait); } - return State::Failed(e.clone()); + return (State::Failed(e.clone()), None); } Entry::Failed(..) => {} } @@ -431,40 +468,62 @@ fn request(url: &str, want: Want, omit: &'static [u16]) -> State { if crate::warp::is_warp(url) { drop(c); // Made once its pictures are here; asked again (as the map draws) until then. - return match crate::warp::make(url) { - crate::warp::Made::Waiting => State::Pending, - crate::warp::Made::Done(payload) => { - let payload = Arc::new(payload); - let n = payload_bytes(&payload); - store(url, Entry::Ready(payload.clone(), n)); - State::Ready(payload) - } - crate::warp::Made::ForNow(payload) => State::Ready(payload), - crate::warp::Made::Failed(e) => { - store(url, Entry::Failed(e.clone(), Instant::now())); - State::Failed(e) - } - }; + return ( + match crate::warp::make(url, from) { + crate::warp::Made::Waiting => State::Pending, + crate::warp::Made::Done(payload) => { + let payload = Arc::new(payload); + let n = payload_bytes(&payload); + store_from( + url, + Entry::Ready(payload.clone(), n, Some(crate::address::Origin { url: url.into(), reach: ceiling })), + ceiling, + ); + State::Ready(payload) + } + crate::warp::Made::ForNow(payload) => State::Ready(payload), + crate::warp::Made::Failed(e) => { + store_from(url, Entry::Failed(e.clone(), Instant::now()), ceiling); + State::Failed(e) + } + }, + None, + ); } let asked_at = crate::mbtiles::epoch(); - c.slots.insert(url.to_string(), Slot { entry: Entry::Pending, used: tick, asked: now }); + c.slots.insert(url.to_string(), Slot { entry: Entry::Pending, ceiling, used: tick, asked: now }); drop(c); - start(url.to_string(), want, omit, asked_at); - State::Pending + start(url.to_string(), want, omit, asked_at, ceiling); + (State::Pending, None) +} + +/// A cache-policy refusal is a failed request too: record it like a transport refusal, +/// after releasing the cache lock, so it cannot become unexplained missing tiles. +fn refused_request(url: &str, message: String) -> (State, Option) { + RequestDiagnostics::new(url, "GET").failed(&Failure::from(message.clone())); + (State::Failed(message), None) } /// What the cache holds for `url`, without fetching it. pub fn peek(url: &str) -> Option> { + peek_from(url, None) +} + +/// A cache-only lookup cannot reuse a local answer for a public document. +pub(crate) fn peek_from(url: &str, from: Option<&crate::address::Origin>) -> Option> { crate::logins::check(resource_of(url)).ok()?; if let Some(base) = crate::warp::direct(url) { - return peek(base); + return peek_from(base, from); } let mut c = lock(); c.tick += 1; let tick = c.tick; let slot = c.slots.get_mut(url)?; match &slot.entry { - Entry::Ready(p, _) => { + Entry::Ready(p, _, origin) => { + if from.is_some_and(|f| f.check(url).is_err() || origin.as_ref().is_some_and(|o| o.reach > f.reach)) { + return None; + } slot.used = tick; Some(p.clone()) } @@ -524,7 +583,7 @@ pub fn forget(url: &str) { .iter() .filter(|(k, s)| !matches!(s.entry, Entry::Pending) && resource_of(k) == url) .map(|(_, s)| match &s.entry { - Entry::Ready(_, n) => *n, + Entry::Ready(_, n, _) => *n, _ => 0, }) .sum(); @@ -594,7 +653,7 @@ pub fn wait_for_news_unless(since: u64, timeout: Duration, stop: impl Fn() -> bo /// A waiting fetch: its decode kind, diagnostics owner, expected statuses, when it was asked for /// ([`land`]) and initiating context. -type Job = (Want, Capture, &'static [u16], u64, Option); +type Job = (Want, Capture, &'static [u16], u64, Option, crate::address::Reach); fn queue() -> &'static Mutex> { static Q: OnceLock>> = OnceLock::new(); @@ -602,7 +661,7 @@ fn queue() -> &'static Mutex> { } /// Asks for a tile: fetched now when its server has room, else it waits its turn. -fn start(url: String, want: Want, omit: &'static [u16], asked_at: u64) { +fn start(url: String, want: Want, omit: &'static [u16], asked_at: u64, ceiling: crate::address::Reach) { let capture = Capture::current(); let still = current_still(); IN_FLIGHT.fetch_add(1, Ordering::Relaxed); @@ -611,7 +670,7 @@ fn start(url: String, want: Want, omit: &'static [u16], asked_at: u64) { let mut q = queue().lock().unwrap_or_else(|e| e.into_inner()); let mut c = lock(); let latest = c.latest.unwrap_or_else(Instant::now); - let now = q.push(url, (want, capture, omit, asked_at, still), |u| c.slots.get(u).map(|s| s.asked), latest, &mut swept); + let now = q.push(url, (want, capture, omit, asked_at, still, ceiling), |u| c.slots.get(u).map(|s| s.asked), latest, &mut swept); // Under the same lock as the choice: a tile asked for again meanwhile isn't forgotten. forget_dropped(&mut c, &swept); now @@ -670,9 +729,9 @@ fn launch(jobs: Vec>) { } let _lower = Lower; while let Some(w) = pop() { - let (want, capture, omit, asked_at, still) = w.job; + let (want, capture, omit, asked_at, still, ceiling) = w.job; let _scope = capture.enter(); - with_optional_still(still, || fetch_tile(w.url, w.host, want, omit, asked_at)); + with_optional_still(still, || fetch_tile(w.url, w.host, want, omit, asked_at, ceiling)); } } // A tile queued after the last `pop` and before the flag went down is started here. @@ -734,10 +793,15 @@ fn land(key: &str, entry: Option, asked_at: u64) { /// Retains the initiating lifetime through deferred decoding. Retired results are forgotten; /// only live successes and resource failures are shared with later callers. -fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16], asked_at: u64) { +fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16], asked_at: u64, ceiling: crate::address::Reach) { let owner = current_still(); + let local = crate::pmtiles::is_tile(&url) || crate::cog::is_tile(&url) || crate::mbtiles::is_tile(&url); let answer = Answer { key: url.clone(), host, entry: None, asked_at }; - let done: Done = Box::new(move |bytes| { + let done: FetchedDone = Box::new(move |fetched| { + let (bytes, origin) = match fetched { + Ok(f) => (Ok(f.bytes), (!local).then_some(f.origin)), + Err(e) => (Err(e), None), + }; // Shared archive reads can answer under another loader's narrower lifetime. Retain // that floor with this tile's owner before a browser decoder suspends completion. let still = with_optional_still(owner, current_still); @@ -752,7 +816,7 @@ fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16], asked Ok(_) if check_still(&still).is_err() => return, Ok(p) => { let size = payload_bytes(&p); - Entry::Ready(Arc::new(p), size) + Entry::Ready(Arc::new(p), size, origin) } // The browser's decoder may answer after the initiating owner was retired. Err(e) if !keep_failure(&e, &still) => return, @@ -769,6 +833,12 @@ fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16], asked let decoder = image_decoder().lock().unwrap_or_else(|e| e.into_inner()).clone(); decode_with(bytes, want, decoder.as_deref(), settle); }); + if !local { + fetch_reached(url, ceiling, done, omit); + return; + } + let origin = crate::address::Origin::named(&url); + let done: Done = Box::new(move |r| done(r.map(|bytes| Fetched { bytes, origin }))); // A tile from a PMTiles archive is read from the archive, by range. if crate::pmtiles::is_tile(&url) { crate::pmtiles::fetch(&url, done); @@ -1253,22 +1323,32 @@ pub fn head(url: &str, done: HeadDone) { /// A status in `omit` is an expected answer: its failure isn't recorded, only noted once per /// server (a tile past a map's edge). pub(crate) fn fetch_bytes(url: String, done: Done, omit: &'static [u16]) { + fetch_reached(url, crate::address::Reach::Local, Box::new(move |r| done(r.map(|f| f.bytes))), omit); +} + +/// Carries destination policy into the transport and returns provenance with the bytes. +fn fetch_reached(url: String, ceiling: crate::address::Reach, done: FetchedDone, omit: &'static [u16]) { if let Err(e) = check_current() { done(Err(e)); return; } let still = current_still(); - let done: Done = Box::new(move |r| with_optional_still(still, || done(r))); + let done: FetchedDone = Box::new(move |r| with_optional_still(still, || done(r))); if let Err(e) = crate::logins::check(resource_of(&url)) { done(Err(e.to_string())); return; } let key = url.clone(); - let done: Done = Box::new(move |r| done(crate::logins::check(resource_of(&key)).map_err(str::to_string).and(r))); + let done: FetchedDone = Box::new(move |r| done(crate::logins::check(resource_of(&key)).map_err(str::to_string).and(r))); let f = fetcher().lock().unwrap_or_else(|e| e.into_inner()).clone(); match f { - Some(f) => f(url, done, omit), - None => native::fetch(url, done, omit), + Some(f) => { + // Browser redirects are refused; the adapter checks the actual normalized URL. + // Custom fetchers must enforce the same destination boundary. + let origin = crate::address::Origin::named(&crate::s3::bound_destination(&url).ok().flatten().unwrap_or_else(|| url.clone())); + f(url, Box::new(move |r| done(r.map(|bytes| Fetched { bytes, origin }))), omit) + } + None => native::fetch_reached(url, ceiling, done, omit), } } @@ -1325,11 +1405,22 @@ fn finished() { } } +/// Stores an explicitly fetched answer; composed answers use their inherited ceiling. +#[cfg(test)] fn store(url: &str, entry: Entry) { + store_from(url, entry, crate::address::Reach::Local); +} + +/// Composed cache entries retain the ceiling their constituent requests used. +fn store_from(url: &str, entry: Entry, ceiling: crate::address::Reach) { if login_gone(url) { return; } - keep(&mut lock(), url, entry); + let mut c = lock(); + keep(&mut c, url, entry); + if let Some(slot) = c.slots.get_mut(url) { + slot.ceiling = ceiling; + } } /// Whether the login `url` was fetched with is gone, in which case nothing of it is kept. @@ -1338,7 +1429,7 @@ fn login_gone(url: &str) -> bool { return false; } let mut c = lock(); - if let Some(Slot { entry: Entry::Ready(_, n), .. }) = c.slots.remove(url) { + if let Some(Slot { entry: Entry::Ready(_, n, _), .. }) = c.slots.remove(url) { c.bytes = c.bytes.saturating_sub(n); } true @@ -1346,12 +1437,13 @@ fn login_gone(url: &str) -> bool { /// [`store`], holding the cache's lock. fn keep(c: &mut Cache, url: &str, entry: Entry) { - let added = if let Entry::Ready(_, n) = &entry { *n } else { 0 }; + let added = if let Entry::Ready(_, n, _) = &entry { *n } else { 0 }; // Dropped while it was fetched (cleared, or evicted): kept anyway, it was asked for. let tick = c.tick; let asked = c.slots.get(url).map_or_else(Instant::now, |s| s.asked); - let old = c.slots.insert(url.to_string(), Slot { entry, used: tick, asked }); - if let Some(Slot { entry: Entry::Ready(_, n), .. }) = old { + let ceiling = c.slots.get(url).map_or(crate::address::Reach::Local, |s| s.ceiling); + let old = c.slots.insert(url.to_string(), Slot { entry, ceiling, used: tick, asked }); + if let Some(Slot { entry: Entry::Ready(_, n, _), .. }) = old { c.bytes = c.bytes.saturating_sub(n); } c.bytes += added; @@ -1366,7 +1458,7 @@ fn evict(c: &mut Cache) { .slots .iter() .filter_map(|(k, s)| match s.entry { - Entry::Ready(_, n) => Some((s.used, k.clone(), n)), + Entry::Ready(_, n, _) => Some((s.used, k.clone(), n)), Entry::Failed(..) => Some((s.used, k.clone(), 0)), Entry::Pending => None, }) @@ -1466,7 +1558,8 @@ mod native { use std::collections::HashMap; use std::sync::{Condvar, Mutex, OnceLock}; - use super::{Capture, DROPPED, Done, DoneDetailed, MAX_BYTES}; + use super::{Capture, DROPPED, Done, DoneDetailed, Fetched, FetchedDone, MAX_BYTES}; + use crate::address::{Origin, Reach}; use crate::failure::{Failure, Reason}; /// Fetches under way at once, over all servers: two servers at their cap. @@ -1482,7 +1575,7 @@ mod native { struct Jobs { /// Waiting, oldest first: the address, its server, callback, initiating owner, and the statuses /// whose failures the caller expects. - waiting: Vec<(String, String, Done, Capture, &'static [u16], Option)>, + waiting: Vec<(String, String, FetchedDone, Capture, &'static [u16], Option, Reach)>, /// Fetches under way, by server. active: HashMap, } @@ -1545,19 +1638,25 @@ mod native { /// Marks a queued job as a local file read. const LOCAL: &str = "local-file:"; + /// Queues an explicitly selected address, keeping the byte-only public callback stable. pub(super) fn fetch(url: String, done: Done, omit: &'static [u16]) { + fetch_reached(url, Reach::Local, Box::new(move |r| done(r.map(|f| f.bytes))), omit); + } + + /// Keeps the destination ceiling with queued work; the web build requires its own fetcher. + pub(super) fn fetch_reached(url: String, ceiling: Reach, done: FetchedDone, omit: &'static [u16]) { let capture = Capture::current(); let q = queue(); let dropped = { let mut jobs = q.jobs.lock().unwrap_or_else(|e| e.into_inner()); let host = host(&url); - jobs.waiting.push((url, host, done, capture, omit, super::current_still())); + jobs.waiting.push((url, host, done, capture, omit, super::current_still(), ceiling)); (jobs.waiting.len() > QUEUE).then(|| jobs.waiting.remove(0)) }; // Every worker may be waiting for a server at its cap, and only some can take this one. q.ready.notify_all(); // Answered once the queue is unlocked: `done` takes the cache's lock and wakes the app. - if let Some((_, _, done, _, _, _)) = dropped { + if let Some((_, _, done, _, _, _, _)) = dropped { done(Err(DROPPED.into())); } } @@ -1571,18 +1670,21 @@ mod native { .http_status_as_error(false) .timeout_global(Some(timeout)) .user_agent(USER_AGENT) + // A proxy could resolve the target itself, past the checked resolver. + .proxy(None) // Redirects are followed by `get`, which checks where each one leads. .max_redirects(0) } + /// Workers share this agent's checked connections and bounded idle pool. pub(super) fn agent() -> ureq::Agent { - config_builder(std::time::Duration::from_secs(30)) + let config = config_builder(std::time::Duration::from_secs(30)) // The workers share the agent: keep a connection open for each fetch a server // may have under way, so the next tile doesn't connect again. .max_idle_connections_per_host(PER_HOST) .max_idle_connections(WORKERS) - .build() - .into() + .build(); + checked_agent(config, ureq::unversioned::resolver::DefaultResolver::default(), false) } /// The agent the workers share, and with it their open connections. @@ -1593,42 +1695,185 @@ mod native { /// An agent for downloads, which may take minutes. pub(super) fn download_agent(timeout: std::time::Duration) -> ureq::Agent { - config_builder(timeout).build().into() + checked_agent(config_builder(timeout).build(), ureq::unversioned::resolver::DefaultResolver::default(), false) } - /// Resolves names as ureq does, keeping only the internet addresses (checked where the - /// connection is made, so a name that resolves elsewhere on another lookup is caught too). - #[derive(Debug, Default)] - struct PublicOnly(ureq::unversioned::resolver::DefaultResolver); + /// Applies the same IP boundary to every native connection, including a new lookup after + /// DNS changes. Document requests keep their inherited public or local ceiling, + /// and forbidden addresses never leave the resolver. Plugin/POST agents narrow + /// this further to internet addresses only. + #[derive(Debug)] + struct CheckedResolver { + inner: R, + public_only: bool, + } - impl ureq::unversioned::resolver::Resolver for PublicOnly { + impl ureq::unversioned::resolver::Resolver for CheckedResolver { + /// Filters the exact socket addresses passed to the connector, without a second lookup. fn resolve( &self, uri: &ureq::http::Uri, config: &ureq::config::Config, timeout: ureq::unversioned::transport::NextTimeout, ) -> Result { - let all = self.0.resolve(uri, config, timeout)?; + let url = uri.to_string(); + crate::address::check(&url, None).map_err(|e| ureq::Error::Io(std::io::Error::new(std::io::ErrorKind::PermissionDenied, e)))?; + let ceiling = if self.public_only { Reach::Public } else { Reach::Local }; + let all = self.inner.resolve(uri, config, timeout)?; + let ceiling = if all.iter().any(|a| crate::address::ip_reach(a.ip()) == Reach::Public) { Reach::Public } else { ceiling }; let mut kept = self.empty(); - for a in all.iter().filter(|a| crate::address::ip_reach(a.ip()) == crate::address::Reach::Public) { + for a in all.iter().filter(|a| { + let reach = crate::address::ip_reach(a.ip()); + reach != crate::address::Reach::Forbidden && ceiling != crate::address::Reach::Forbidden && reach <= ceiling + }) { kept.push(*a); } if kept.is_empty() { let host = uri.host().unwrap_or_default(); - let msg = format!("{host} resolves to this computer or the local network, which isn't fetched from here"); + let msg = + format!("{host} resolves to this computer or the local network, or a forbidden address, which isn't fetched from here"); return Err(ureq::Error::Io(std::io::Error::new(std::io::ErrorKind::PermissionDenied, msg))); } Ok(kept) } } + thread_local! { + static SENT_TO: std::cell::Cell> = const { std::cell::Cell::new(None) }; + } + + /// Reads the reach of this synchronous request, including a pooled connection. This is + /// observed provenance, never an ambient permission; each agent has a fixed IP ceiling. + fn connected(f: impl FnOnce() -> Result) -> Result<(T, Reach), ureq::Error> { + struct Restore(Option); + impl Drop for Restore { + fn drop(&mut self) { + SENT_TO.with(|r| r.set(self.0)); + } + } + let _restore = Restore(SENT_TO.with(|r| r.replace(None))); + f().map(|v| (v, SENT_TO.with(|r| r.get()).unwrap_or(Reach::Public))) + } + + #[derive(Debug, Default)] + struct ReachConnector(C); + + impl>> + ureq::unversioned::transport::Connector<()> for ReachConnector + { + type Out = Box; + fn connect( + &self, + details: &ureq::unversioned::transport::ConnectionDetails, + chained: Option<()>, + ) -> Result, ureq::Error> { + let reach = details.addrs.first().map_or(Reach::Public, |a| crate::address::ip_reach(a.ip())); + Ok(self.0.connect(details, chained)?.map(|inner| Box::new(ReachedTransport { inner, reach }) as Self::Out)) + } + } + + #[derive(Debug)] + struct ReachedTransport { + inner: Box, + reach: Reach, + } + + impl ureq::unversioned::transport::Transport for ReachedTransport { + fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers { + self.inner.buffers() + } + fn transmit_output(&mut self, amount: usize, timeout: ureq::unversioned::transport::NextTimeout) -> Result<(), ureq::Error> { + self.inner.transmit_output(amount, timeout)?; + SENT_TO.with(|r| r.set(Some(self.reach))); + Ok(()) + } + fn await_input(&mut self, timeout: ureq::unversioned::transport::NextTimeout) -> Result { + self.inner.await_input(timeout) + } + fn is_open(&mut self) -> bool { + self.inner.is_open() + } + fn is_tls(&self) -> bool { + self.inner.is_tls() + } + } + + /// Public-document requests cannot reuse an explicitly local connection from another pool. + fn shared_public_agent() -> &'static ureq::Agent { + static A: OnceLock = OnceLock::new(); + A.get_or_init(|| public_agent(std::time::Duration::from_secs(30))) + } + + /// Builds the ordinary and internet-only agents through one connection boundary. The + /// supplied resolver also lets controlled fixtures exercise DNS changes without network DNS. + pub(super) fn checked_agent( + config: ureq::config::Config, + resolver: impl ureq::unversioned::resolver::Resolver, + public_only: bool, + ) -> ureq::Agent { + ureq::Agent::with_parts( + config, + ReachConnector::::default(), + CheckedResolver { inner: resolver, public_only }, + ) + } + + #[cfg(test)] + /// Exercises classified DNS and real HTTP while routing every connection to an owned listener. + pub(super) fn fixture_agent(answer: std::net::SocketAddr, listener: std::net::SocketAddr, public_only: bool) -> ureq::Agent { + use ureq::unversioned::{ + resolver::Resolver, + transport::{ConnectionDetails, Connector, DefaultConnector, Transport}, + }; + #[derive(Debug)] + struct Dns(std::net::SocketAddr); + impl Resolver for Dns { + fn resolve( + &self, + uri: &ureq::http::Uri, + _: &ureq::config::Config, + _: ureq::unversioned::transport::NextTimeout, + ) -> Result { + let mut addresses = self.empty(); + let literal = uri.host().and_then(|h| h.parse::().ok()); + addresses.push(literal.map_or(self.0, |ip| std::net::SocketAddr::new(ip, uri.port_u16().unwrap_or(80)))); + Ok(addresses) + } + } + #[derive(Debug)] + struct Route(std::net::SocketAddr); + impl Connector<()> for Route { + type Out = Box; + fn connect(&self, d: &ConnectionDetails, _: Option<()>) -> Result, ureq::Error> { + let mut addrs = d.addrs.clone(); + for address in addrs.iter_mut() { + *address = self.0; + } + let fixture = ConnectionDetails { + uri: d.uri, + addrs, + config: d.config, + request_level: d.request_level, + resolver: d.resolver, + now: d.now, + timeout: d.timeout, + current_time: d.current_time.clone(), + run_connector: d.run_connector.clone(), + register_shutdown: d.register_shutdown.clone(), + }; + DefaultConnector::default().connect(&fixture, None) + } + } + ureq::Agent::with_parts( + config_builder(std::time::Duration::from_secs(2)).build(), + ReachConnector(Route(listener)), + CheckedResolver { inner: Dns(answer), public_only }, + ) + } + /// A download agent that connects only to internet addresses. pub(super) fn public_agent(timeout: std::time::Duration) -> ureq::Agent { - let config = config_builder(timeout) - // A proxy would resolve the address itself, past `PublicOnly`. - .proxy(None) - .build(); - ureq::Agent::with_parts(config, ureq::unversioned::transport::DefaultConnector::default(), PublicOnly::default()) + checked_agent(config_builder(timeout).build(), ureq::unversioned::resolver::DefaultResolver::default(), true) } /// Asks for an address's headers on a thread of its own. @@ -1664,18 +1909,21 @@ mod native { let answer = (|| -> Result { crate::logins::check(url).map_err(str::to_string)?; let prepared = crate::s3::prepare(crate::logins::inner(url))?; + crate::address::check(&prepared.url, None)?; if prepared.signed { return Ok(super::Head::default()); } let mut at = prepared.url.clone(); let first_origin = crate::logins::origin(&at); let mut same_origin = true; + let mut ceiling = Reach::Local; for redirects in 0..=5 { // The login goes to its own origin only: once a redirect leaves it, never again. let bound = crate::logins::inherit(url, &at); let login = if same_origin { crate::logins::apply(&bound)? } else { None }; carried_login |= login.is_some(); - let mut req = agent.head(login.as_ref().map_or(at.as_str(), |a| a.url.as_str())); + let sending = if ceiling == Reach::Public { shared_public_agent() } else { agent }; + let mut req = sending.head(login.as_ref().map_or(at.as_str(), |a| a.url.as_str())); if let Some((name, value)) = login.as_ref().and_then(|a| a.header.as_ref()) { req = req.header(name.as_str(), value.as_str()); } @@ -1686,12 +1934,14 @@ mod native { } diagnostics.sending(); super::check_current()?; - let resp = req.call().map_err(unanswered)?; + let req = req.config().timeout_global(agent.config().timeouts().global).build(); + let (resp, reached) = connected(|| req.call()).map_err(unanswered)?; + ceiling = ceiling.min(reached); diagnostics.answered(resp.status().as_u16()); let header = |k: &str| resp.headers().get(k).and_then(|v| v.to_str().ok()); if resp.status().is_redirection() { let next = crate::template::join(&at, header("location").ok_or("a redirect with no Location")?); - crate::address::check(&next, Some(&at))?; + Origin { url: at.clone(), reach: ceiling }.check(&next)?; if at.starts_with("https://") && !next.starts_with("https://") { return Err(format!("{next}: a redirect from https to http").into()); } @@ -1811,18 +2061,19 @@ mod native { }); } + /// Dispatches each queued read through the pool matching its inherited IP ceiling. fn work() { let agent = shared_agent(); let q = queue(); loop { - let (url, done, host, capture, omit, still) = { + let (url, done, host, capture, omit, still, ceiling) = { let mut jobs = q.jobs.lock().unwrap_or_else(|e| e.into_inner()); loop { - let hosts: Vec<&str> = jobs.waiting.iter().map(|(_, h, _, _, _, _)| h.as_str()).collect(); + let hosts: Vec<&str> = jobs.waiting.iter().map(|(_, h, _, _, _, _, _)| h.as_str()).collect(); if let Some(i) = pick(&hosts, &jobs.active) { - let (url, host, done, capture, omit, still) = jobs.waiting.remove(i); + let (url, host, done, capture, omit, still, ceiling) = jobs.waiting.remove(i); *jobs.active.entry(host.clone()).or_insert(0) += 1; - break (url, done, host, capture, omit, still); + break (url, done, host, capture, omit, still, ceiling); } jobs = q.ready.wait(jobs).unwrap_or_else(|e| e.into_inner()); } @@ -1830,7 +2081,8 @@ mod native { let _scope = capture.enter(); let result = super::with_optional_still(still.clone(), || { super::check_current()?; - get(agent, &url, omit) + fetch_answer(if ceiling == Reach::Public { shared_public_agent() } else { agent }, &url, ceiling, false, omit) + .map_err(|f| f.message) }); { let mut jobs = q.jobs.lock().unwrap_or_else(|e| e.into_inner()); @@ -1847,11 +2099,11 @@ mod native { } } - /// Fetches `url`, following up to 5 redirects. `url` itself passed `address::check` when - /// it was made (the layer's address, or one its style or TileJSON chose, checked in - /// `source::resolve`); each redirect is checked here. + /// Fetches `url`, following up to 5 redirects. Checks the final network destination here, + /// even when the address was already checked before expansion or signing; every redirect + /// and every resolved connection is checked too. pub(super) fn get(agent: &ureq::Agent, url: &str, omit: &'static [u16]) -> Result, String> { - fetch_answer(agent, url, false, omit).map_err(|f| f.message) + fetch_answer(agent, url, Reach::Local, false, omit).map(|f| f.bytes).map_err(|f| f.message) } /// The User-Agent every request goes with. @@ -1899,7 +2151,7 @@ mod native { /// [`get_detailed`], a status in `omit` not recorded as a failure. pub(super) fn get_detailed_omitting(agent: &ureq::Agent, url: &str, omit: &'static [u16]) -> Result, Failure> { - fetch_answer(agent, url, true, omit).map_err(|f| Failure { + fetch_answer(agent, url, Reach::Local, true, omit).map(|f| f.bytes).map_err(|f| Failure { url: if url.contains(rustgis_data::login_address::PREFIX) { crate::logins::scrub_error(url) } else { url.to_string() }, ..f }) @@ -1917,22 +2169,30 @@ mod native { /// `omit` is an expected answer: its failure isn't recorded, only noted once per server. A /// request carrying a login is recorded by its plain address and scrubbed text only /// ([`RequestDiagnostics::failed`]). - fn fetch_answer(agent: &ureq::Agent, url: &str, detail: bool, omit: &'static [u16]) -> Result, Failure> { + pub(super) fn fetch_answer( + agent: &ureq::Agent, + url: &str, + initial: Reach, + detail: bool, + omit: &'static [u16], + ) -> Result { use std::io::Read; let mut carried_login = false; // Local file failures are not network traffic. let local = crate::mbtiles::is_tile(url) || url.starts_with(LOCAL); let mut diagnostics = super::RequestDiagnostics::new(url, "GET"); diagnostics.omit_statuses(omit); - let answer = (|| -> Result, Failure> { + let answer = (|| -> Result { super::check_current()?; if crate::mbtiles::is_tile(url) { - return crate::mbtiles::read_local(url).map_err(Failure::from); + return crate::mbtiles::read_local(url).map(|bytes| Fetched { bytes, origin: Origin::named(url) }).map_err(Failure::from); } if let Some(local) = url.strip_prefix(LOCAL) { let (path, range) = super::split_range(local); let (a, b) = range.ok_or("a local read without a range")?; - return crate::pmtiles::read_file(path, a, b - a + 1).map_err(Failure::from); + return crate::pmtiles::read_file(path, a, b - a + 1) + .map(|bytes| Fetched { bytes, origin: Origin::named(url) }) + .map_err(Failure::from); } let (url, limit) = super::split_download_limit(url); let max = limit.unwrap_or(MAX_BYTES); @@ -1944,7 +2204,9 @@ mod native { // endpoint is signed. crate::logins::check(url)?; let prepared = crate::s3::prepare(crate::logins::inner(url))?; + Origin { url: prepared.url.clone(), reach: initial }.check(&prepared.url)?; let mut at = prepared.url.clone(); + let mut ceiling = initial; let first_origin = crate::logins::origin(&at); let mut same_origin = true; let mut redirects = 0; @@ -1953,7 +2215,8 @@ mod native { let bound = crate::logins::inherit(url, &at); let login = if !prepared.signed && same_origin { crate::logins::apply(&bound)? } else { None }; carried_login |= login.is_some(); - let mut req = agent.get(login.as_ref().map_or(at.as_str(), |a| a.url.as_str())); + let sending = if ceiling == Reach::Public && initial != Reach::Public { shared_public_agent() } else { agent }; + let mut req = sending.get(login.as_ref().map_or(at.as_str(), |a| a.url.as_str())); if let Some((name, value)) = login.as_ref().and_then(|a| a.header.as_ref()) { req = req.header(name.as_str(), value.as_str()); } @@ -1988,7 +2251,10 @@ mod native { } diagnostics.sending(); super::check_current()?; - let mut resp = req.call().map_err(unanswered)?; + // Narrowing the connection pool must retain the caller's download timeout. + let req = req.config().timeout_global(agent.config().timeouts().global).build(); + let (mut resp, reached) = connected(|| req.call()).map_err(unanswered)?; + ceiling = ceiling.min(reached); let status = resp.status().as_u16(); diagnostics.answered(status); // A data host refusing the request (or the token): say what to do. @@ -2050,7 +2316,7 @@ mod native { return Err(Failure::without_answer("", crate::earthdata::refused_message(url), Reason::Status(401))); } // Where a redirect leads is the server's choice, not the user's. - crate::address::check(&next, Some(&at))?; + Origin { url: at.clone(), reach: ceiling }.check(&next)?; // Never from HTTPS down to plain HTTP. if at.starts_with("https://") && !next.starts_with("https://") { return Err(format!("{next}: a redirect from https to http").into()); @@ -2062,8 +2328,9 @@ mod native { same_origin &= crate::logins::origin(&next) == first_origin; at = next; }; + let origin = Origin { url: if same_origin { crate::logins::inherit(url, &at) } else { at }, reach: ceiling }; if resp.status().as_u16() == 204 { - return Ok(Vec::new()); + return Ok(Fetched { bytes: Vec::new(), origin }); } // A range asked for and the whole file answered (200): fine for a range from the start // that holds the whole file (a small archive's header read), cut to the range; otherwise @@ -2096,7 +2363,7 @@ mod native { if buf.len() > max { return Err(format!("larger than {max} bytes").into()); } - return Ok(buf); + return Ok(Fetched { bytes: buf, origin }); } // ureq limits the wire bytes; `take` also caps the decoded bytes when gzip expands them. resp.body_mut() @@ -2115,7 +2382,7 @@ mod native { } buf.truncate(usize::try_from(b - a + 1).unwrap_or(usize::MAX)); } - Ok(buf) + Ok(Fetched { bytes: buf, origin }) })(); let result = if local { answer } else { diagnostics.record(answer) }; let result = crate::logins::check(super::resource_of(url)).map_err(Failure::from).and(result); @@ -2133,6 +2400,11 @@ mod native { mod native { use super::{Done, DoneDetailed}; + /// Keeps the destination ceiling with queued work; the web build requires its own fetcher. + pub(super) fn fetch_reached(url: String, _ceiling: crate::address::Reach, done: super::FetchedDone, _omit: &'static [u16]) { + done(Err(format!("{url}: this build has no way to fetch tiles"))); + } + pub(super) fn fetch(url: String, done: Done, _omit: &'static [u16]) { done(Err(format!("{url}: this build has no way to fetch tiles"))); } @@ -2189,7 +2461,9 @@ mod tests { let key = crate::mbtiles::template(&archive(0)).replace("{z}", "0").replace("{x}", "0").replace("{y}", "0"); let failed = |why: &str| Some(Entry::Failed(why.into(), Instant::now())); let old = crate::mbtiles::epoch(); - lock().slots.insert(key.clone(), Slot { entry: Entry::Pending, used: 0, asked: Instant::now() }); + lock() + .slots + .insert(key.clone(), Slot { entry: Entry::Pending, ceiling: crate::address::Reach::Local, used: 0, asked: Instant::now() }); for i in 0..=crate::mbtiles::MAX_FORGOTTEN { crate::mbtiles::forget_archive(&archive(i)); } @@ -2213,7 +2487,15 @@ mod tests { let archive = "/maps#cube=1.mbtiles"; let key = crate::mbtiles::template(archive).replace("{z}", "0").replace("{x}", "0").replace("{y}", "0"); let old = crate::mbtiles::epoch(); - lock().slots.insert(key.clone(), Slot { entry: Entry::Failed("old file".into(), Instant::now()), used: 0, asked: Instant::now() }); + lock().slots.insert( + key.clone(), + Slot { + entry: Entry::Failed("old file".into(), Instant::now()), + ceiling: crate::address::Reach::Local, + used: 0, + asked: Instant::now(), + }, + ); forget(archive); assert!(!lock().slots.contains_key(&key)); assert!(crate::mbtiles::forgotten_since(&key, old), "the file is recorded under the path its tiles name"); @@ -2236,7 +2518,9 @@ mod tests { }; let old = crate::mbtiles::epoch(); - lock().slots.insert(key.clone(), Slot { entry: Entry::Pending, used: 0, asked: Instant::now() }); + lock() + .slots + .insert(key.clone(), Slot { entry: Entry::Pending, ceiling: crate::address::Reach::Local, used: 0, asked: Instant::now() }); forget(archive); land(&key, failed("old file"), old); assert_eq!(kept(), None); @@ -2250,7 +2534,9 @@ mod tests { // Or the old one lands while the new one is pending: the new one's answer is what stays. let new = crate::mbtiles::epoch(); - lock().slots.insert(key.clone(), Slot { entry: Entry::Pending, used: 0, asked: Instant::now() }); + lock() + .slots + .insert(key.clone(), Slot { entry: Entry::Pending, ceiling: crate::address::Reach::Local, used: 0, asked: Instant::now() }); land(&key, failed("old file"), old); assert!(matches!(lock().slots.get(&key), Some(Slot { entry: Entry::Pending, .. })), "the new read's slot stays"); land(&key, None, old); @@ -3163,7 +3449,7 @@ mod tests { assert!(requests[0].contains("x-key: a\r\n")); assert!(requests[1].contains("x-key: b\r\n")); assert!(!requests[2].contains("x-key:")); - store(&ua, Entry::Ready(Arc::new(Payload::Json(serde_json::json!({"a": 1}))), 64)); + store(&ua, Entry::Ready(Arc::new(Payload::Json(serde_json::json!({"a": 1}))), 64, None)); assert!(peek(&ua).is_some()); assert!(peek(&ub).is_none() && peek(&raw).is_none()); assert!(logins::deactivate(&a)); @@ -3208,7 +3494,7 @@ mod tests { assert_ne!(url, replacement_url); release_tx.send(()).unwrap(); assert!(worker.join().unwrap().is_err()); - store(&url, Entry::Ready(Arc::new(Payload::Json(serde_json::json!({"old": true}))), 64)); + store(&url, Entry::Ready(Arc::new(Payload::Json(serde_json::json!({"old": true}))), 64, None)); assert!(peek(&url).is_none()); assert!(matches!(get(&url, Want::Json), State::Failed(_))); assert!(peek(&replacement_url).is_none()); @@ -3275,7 +3561,7 @@ mod tests { let mut resolved = None; for _ in 0..200 { for (address, json, len) in &documents { - store(address, Entry::Ready(Arc::new(Payload::Json(json.clone())), *len)); + store(address, Entry::Ready(Arc::new(Payload::Json(json.clone())), *len, None)); } if let source::Resolve::Ready(r) = source::resolve(&spec) { resolved = Some(r); @@ -3430,6 +3716,176 @@ mod tests { assert!(native::get(&agent, &format!("{base}/x"), &[]).unwrap_err().contains("too many")); } + /// A redirected style/TileJSON must use its effective public origin, including from + /// cache. The fixture routes synthetic public IPs only to its own loopback listener. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn redirected_documents_cannot_keep_a_local_allowance() { + let _turn = turn(); + use crate::{ + address::Reach, + source::{Kind, Resolve, Spec}, + }; + for kind in [Kind::Style, Kind::Vector] { + let tiles = "http://{switch:127.0.0.1}/redirected-document/{z}/{x}/{y}"; + let doc = if kind == Kind::Style { + serde_json::json!({"version": 8, "sources": { + "bad": {"type":"vector", "tiles":[tiles]}, + "relative": {"type":"vector", "tiles":["tiles/{z}/{x}/{y}"]} + }, "layers":[{"id":"bad", "type":"fill", "source":"bad", "source-layer":"l"}]}) + } else { + serde_json::json!({"tiles":[tiles]}) + }; + let destination = "http://document.public.example/final/style.json"; + let base = serve(vec![ + format!("HTTP/1.1 302 Found\r\nLocation: {destination}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"), + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{doc}", doc.to_string().len()), + ]); + let listener = base.strip_prefix("http://").unwrap().parse().unwrap(); + let agent = native::fixture_agent("203.0.113.1:80".parse().unwrap(), listener, false); + let asked = format!("{base}/original.json"); + let fetched = native::fetch_answer(&agent, &asked, Reach::Local, false, &[]).unwrap(); + assert_eq!((&*fetched.origin.url, fetched.origin.reach), (destination, Reach::Public)); + let payload = decode(&fetched.bytes, Want::Json).unwrap(); + store(&asked, Entry::Ready(Arc::new(payload), 1024, Some(fetched.origin))); + let spec = Spec { + kind, + url: asked, + subdomains: Vec::new(), + min_zoom: 0, + max_zoom: 14, + tile_size: 256, + tms: false, + attribution: String::new(), + layers: Vec::new(), + style_layers: String::new(), + hidden_layers: Vec::new(), + }; + let Resolve::Ready(resolved) = crate::source::resolve(&spec) else { panic!("the cached document must resolve") }; + if kind == Kind::Style { + let relative = resolved.style.sources.iter().find(|(id, _)| id == "relative").unwrap(); + assert!( + matches!(&relative.1, crate::style::Source::Vector { tiles, .. } if tiles == &["http://document.public.example/final/tiles/{z}/{x}/{y}"]) + ); + } + let sent = Arc::new(AtomicUsize::new(0)); + let observed = sent.clone(); + set_fetcher(move |_, done| { + observed.fetch_add(1, Ordering::SeqCst); + done(Ok(Vec::new())); + }); + let mut picture = Pixmap::new(64, 64).unwrap(); + let drawn = crate::draw::draw(&mut picture, &spec, 1.0, &crate::draw::tests::world_frame(), true); + assert!(drawn.error.as_deref().is_some_and(|e| e.contains("local network")), "{:?}", drawn.error); + assert_eq!(sent.load(Ordering::SeqCst), 0, "the redirected document must send no local tiles"); + clear_fetcher(); + } + // Resolved forbidden IPs never reach even the fixture connector. A regression can + // only contact this test's own listener, not a real metadata or local service. + let target = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + target.set_nonblocking(true).unwrap(); + for ip in ["169.254.169.254:80", "0.0.0.0:80", "[fe80::1]:80", "[::]:80"] { + let agent = native::fixture_agent(ip.parse().unwrap(), target.local_addr().unwrap(), false); + assert!(native::get(&agent, "http://blocked.example/", &[]).unwrap_err().contains("resolves to")); + } + assert_eq!(target.accept().unwrap_err().kind(), std::io::ErrorKind::WouldBlock); + } + + /// A previously explicit local tile cannot silently stand in for a public document's + /// request for the same hostname. The denial is retained in the session diagnostics. + #[test] + fn cached_destinations_report_a_local_answer_refusal() { + let _turn = turn(); + let url = "http://cached-local-destination.example/tile.png"; + let origin = crate::address::Origin { url: url.into(), reach: crate::address::Reach::Local }; + store(url, Entry::Ready(Arc::new(Payload::Empty), 64, Some(origin))); + let from = crate::address::Origin::named("https://public-document.example/style.json"); + let history = rustgis_data::diagnostics::History::default(); + let _scope = history.enter(); + assert!(matches!(get_tile_from(url, Want::Image, Some(&from)), State::Failed(_))); + assert_eq!(network_records(&history, url).len(), 1, "cached destination refusal must be reported"); + } + + /// A public style's WMS composite must not regain a local allowance for its parts or + /// reuse a composite made explicitly from local answers. Cache seeding performs no I/O. + #[test] + fn warped_destinations_keep_the_document_boundary() { + let _turn = turn(); + let url = "http://warped-local-destination.example/wms#rustgis-warp=0/0/0;crs=4326;map=east"; + let (base, (z, x, y), warp) = rustgis_data::ogc::warp::Warp::parse(url).unwrap(); + let extent = crate::template::TileId { z, x, y }.extent(); + let transformer = crate::warp::transformer(warp.epsg).unwrap(); + let plan = warp.plan(base, [extent.min_x, extent.min_y, extent.max_x, extent.max_y], 256, &transformer); + assert!(!plan.parts.is_empty(), "the fixture must require source pictures"); + for part in plan.parts { + let origin = crate::address::Origin { url: part.url.clone(), reach: crate::address::Reach::Local }; + store(&part.url, Entry::Ready(Arc::new(Payload::Image(Pixmap::new(1, 1).unwrap())), 4, Some(origin))); + } + let from = crate::address::Origin::named("https://public-document.example/style.json"); + assert!(matches!(get_tile_from(url, Want::Image, Some(&from)), State::Failed(_)), "composition must inherit public reach"); + assert!(matches!(get_tile(url, Want::Image), State::Ready(_)), "explicit local composition still works"); + assert!(matches!(get_tile_from(url, Want::Image, Some(&from)), State::Failed(_)), "a cached local composite stays local"); + } + + /// Replaces only the fixture's public DNS names; all connections, if a check regresses, + /// can reach only the listener this test just created. + #[cfg(not(target_arch = "wasm32"))] + #[derive(Debug)] + struct LocalAnswer(std::net::SocketAddr); + + #[cfg(not(target_arch = "wasm32"))] + impl ureq::unversioned::resolver::Resolver for LocalAnswer { + fn resolve( + &self, + uri: &ureq::http::Uri, + config: &ureq::config::Config, + timeout: ureq::unversioned::transport::NextTimeout, + ) -> Result { + if uri.host().is_some_and(|host| host.ends_with(".example")) { + let mut answer = self.empty(); + answer.push(self.0); + Ok(answer) + } else { + ureq::unversioned::resolver::DefaultResolver::default().resolve(uri, config, timeout) + } + } + } + + /// GET, HEAD and downloads use the same checked resolver and recheck redirected names. + /// A refusal must occur before connecting, not after a response or timeout. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn native_destinations_filter_dns_before_connecting() { + let _turn = turn(); + let target = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + target.set_nonblocking(true).unwrap(); + let answer = target.local_addr().unwrap(); + let public = format!("http://destinations.example:{}/tile", answer.port()); + for config in [native::agent().config().clone(), native::download_agent(std::time::Duration::from_secs(1)).config().clone()] { + assert!(config.proxy().is_none(), "a proxy must not bypass checked DNS"); + let agent = native::checked_agent(config, LocalAnswer(answer), true); + assert!(native::get(&agent, &public, &[]).unwrap_err().contains("resolves to")); + assert!(native::head_with(&agent, &public).unwrap_err().contains("resolves to")); + } + assert_eq!(target.accept().unwrap_err().kind(), std::io::ErrorKind::WouldBlock, "no forbidden connection was attempted"); + + let split = serve(vec!["HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntile".into()]); + let listener: std::net::SocketAddr = split.strip_prefix("http://").unwrap().parse().unwrap(); + let explicit = format!("http://self-hosted.example:{}/tiles", listener.port()); + let agent = native::checked_agent(native::agent().config().clone(), LocalAnswer(listener), false); + let fetched = native::fetch_answer(&agent, &explicit, crate::address::Reach::Local, false, &[]).unwrap(); + assert_eq!(fetched.bytes, b"tile"); + assert_eq!(fetched.origin.url, explicit, "preserve the hostname used for TLS and relative references"); + assert_eq!(fetched.origin.reach, crate::address::Reach::Local); + + // Explicit local services still work, including a checked same-origin redirect. + let local = serve(vec![ + "HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".into(), + "HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntile".into(), + ]); + assert_eq!(download_blocking_limited(&local, 4).unwrap(), b"tile"); + } + #[cfg(not(target_arch = "wasm32"))] #[test] fn fetches_are_limited_per_server() { diff --git a/crates/tiles/src/draw.rs b/crates/tiles/src/draw.rs index 0e4fc22802..9ff38ceda0 100644 --- a/crates/tiles/src/draw.rs +++ b/crates/tiles/src/draw.rs @@ -52,6 +52,18 @@ pub struct Drawn { pub error: Option, } +/// Retains each distinct refusal alongside any earlier source or sprite warning. +pub(crate) fn add_error(to: &mut Option, error: String) { + match to { + Some(previous) if !previous.contains(&error) => { + previous.push_str("; "); + previous.push_str(&error); + } + None => *to = Some(error), + _ => {} + } +} + /// One tile's worth of drawing: the tile data, where that tile is, and the square it stands in for. #[derive(Clone)] pub(crate) struct Item { @@ -69,7 +81,7 @@ pub(crate) fn left_out(r: &Resolved, frame: &Frame, mesh: bool) -> Option bool| r.style.layers.iter().any(|l| l.visible_at(zoom) && draws(l)); let hi_dpi = frame.ppp >= 1.5; let unread = sprite_notes(shown(StyleLayer::draws_icons), shown(StyleLayer::draws_patterns), hi_dpi, hi_dpi && !mesh, |hi, what| { - sprite::unread_note(&r.sprites, r.style.sprites_listed, &sprite::load(&r.sprites, hi), what) + sprite::unread_note(&r.sprites, r.style.sprites_listed, &sprite::load_from(&r.sprites, hi, r.from.as_ref()), what) }); let all: Vec<&str> = r.left_out.iter().map(String::as_str).chain(unread.iter().map(String::as_str)).collect(); (!all.is_empty()).then(|| all.join("; ")) @@ -117,7 +129,7 @@ pub fn draw(pm: &mut Pixmap, spec: &Spec, opacity: f32, frame: &Frame, labels: b let (w, h) = (pm.width(), pm.height()); // A style drawing vector shapes draws them from tile pictures drawn once (see `picture`): // a map moving across them only puts them in place. - if let Some(pictures) = crate::picture::cover(&r, frame, &frame.extent(w, h), None, &mut out.pending) { + if let Some(pictures) = crate::picture::cover(&r, frame, &frame.extent(w, h), None, &mut out.pending, &mut out.error) { for it in &pictures { draw_raster(pm, it, frame, opacity); } @@ -169,7 +181,7 @@ pub(crate) fn draw_layers( let mut sprite_state: Option>> = None; let mut sprite = |pending: &mut bool| -> Option> { sprite_state - .get_or_insert_with(|| match sprite::load(&r.sprites, frame.ppp >= 1.5) { + .get_or_insert_with(|| match sprite::load_from(&r.sprites, frame.ppp >= 1.5, r.from.as_ref()) { Load::Ready(s) => Some(s), Load::Pending => { *pending = true; @@ -208,7 +220,14 @@ pub(crate) fn draw_layers( // Hillshade reads the heights a pixel past the view's edges: those tiles too. let pad = 2.0 * frame.m_per_px; let reach = Extent::new(view.min_x - pad, view.min_y - pad, view.max_x + pad, view.max_y + pad); - let list = tiles_for(src, if matches!(src, Source::RasterDem { .. }) { &reach } else { view }, frame, zoom, &mut out.pending); + let list = + match tiles_for(src, if matches!(src, Source::RasterDem { .. }) { &reach } else { view }, frame, zoom, &mut out.pending) { + Ok(list) => list, + Err(e) => { + add_error(&mut out.error, e); + Vec::new() + } + }; items.insert(src_id.to_string(), list); } let Some(list) = items.get(src_id) else { continue }; @@ -310,6 +329,7 @@ pub fn place_raster(spec: &Spec, opacity: f32, frame: &Frame, view: &Extent, rea place_as(spec, opacity, frame, view, reach, None, false) } +/// Places raster or prepared vector pictures and retains both shape and label refusals. fn place_as( spec: &Spec, opacity: f32, @@ -340,12 +360,17 @@ fn place_as( return Some(out); } // Without pictures, a style drawing shapes is left to the check below (no raster layers only). - if let Some(pictures) = pictures.then(|| crate::picture::cover(&r, frame, view, Some(reach), &mut out.pending)).flatten() { + if let Some(pictures) = + pictures.then(|| crate::picture::cover(&r, frame, view, Some(reach), &mut out.pending, &mut out.error)).flatten() + { let opacity = opacity.clamp(0.0, 1.0); out.tiles = pictures.into_iter().map(|it| Placement { picture: it.payload, at: it.at, clip: it.clip, opacity }).collect(); if let Some(size) = labels { let mut d = Drawn::default(); draw_layers(None, size, &frame.extent(size.0, size.1), &r, frame, Pass::Labels, &mut d); + if let Some(refused) = d.error { + add_error(&mut out.error, refused); + } out.labels = d.labels; out.pending |= d.pending; } @@ -362,7 +387,14 @@ fn place_as( let Some(src_id) = layer.source.as_deref() else { continue }; if !items.contains_key(src_id) { let Some(src) = r.style.sources.iter().find(|s| s.0 == src_id).map(|s| &s.1) else { continue }; - items.insert(src_id, tiles_in(src, view, Some(reach), frame, zoom, &mut out.pending)); + let list = match tiles_in(src, view, Some(reach), frame, zoom, &mut out.pending) { + Ok(list) => list, + Err(e) => { + add_error(&mut out.error, e); + Vec::new() + } + }; + items.insert(src_id, list); } let o = (layer.raster_opacity(zoom) * opacity).clamp(0.0, 1.0); for it in items.get(src_id).into_iter().flatten() { @@ -428,7 +460,7 @@ pub fn pick(spec: &Spec, x: f64, y: f64, m_per_px: f64, tol_px: f32, limit: usiz if !matches!(src, Source::Vector { .. }) { continue; } - items.insert(src_id.to_string(), tiles_for(src, &view, &frame, zoom, &mut pending)); + items.insert(src_id.to_string(), tiles_for(src, &view, &frame, zoom, &mut pending).unwrap_or_default()); } let Some(list) = items.get(src_id) else { continue }; // Every tile within reach (a feature near a tile's edge may be in its neighbour), the real @@ -522,33 +554,56 @@ fn hits(f: &crate::mvt::Feature, kind: LayerKind, p: [f32; 2], reach: f32) -> bo } /// The tiles of a source in view, with stand-ins for those not here yet. -fn tiles_for(src: &Source, view: &Extent, frame: &Frame, zoom: f32, pending: &mut bool) -> Vec { +fn tiles_for(src: &Source, view: &Extent, frame: &Frame, zoom: f32, pending: &mut bool) -> Result, String> { tiles_in(src, view, None, frame, zoom, pending) } /// The tiles of a source in view and out to `reach` (those in the cache only), with stand-ins -/// for those not here yet. -pub(crate) fn tiles_in(src: &Source, view: &Extent, reach: Option<&Extent>, frame: &Frame, zoom: f32, pending: &mut bool) -> Vec { - let (tiles, min_z, max_z, want, z) = match src { - Source::Raster { tiles, tile_size, min_zoom, max_zoom, .. } | Source::RasterDem { tiles, tile_size, min_zoom, max_zoom, .. } => { +/// for those not here yet. Expanded destinations retain the supplying document's reach; +/// a refusal is reported by the drawing, without touching the shared cache or fetcher. +pub(crate) fn tiles_in( + src: &Source, + view: &Extent, + reach: Option<&Extent>, + frame: &Frame, + zoom: f32, + pending: &mut bool, +) -> Result, String> { + let (tiles, from, min_z, max_z, want, z) = match src { + Source::Raster { tiles, from, tile_size, min_zoom, max_zoom, .. } + | Source::RasterDem { tiles, from, tile_size, min_zoom, max_zoom, .. } => { let z = template::zoom_for(frame.m_per_px * f64::from(frame.ppp), *tile_size, *max_zoom); - (tiles, *min_zoom, *max_zoom, Want::Image, z) + (tiles, from, *min_zoom, *max_zoom, Want::Image, z) } - Source::Vector { tiles, min_zoom, max_zoom, .. } => { - (tiles, *min_zoom, *max_zoom, Want::Vector, (zoom.floor() as u8).min(*max_zoom)) + Source::Vector { tiles, from, min_zoom, max_zoom, .. } => { + (tiles, from, *min_zoom, *max_zoom, Want::Vector, (zoom.floor() as u8).min(*max_zoom)) } - Source::Other => return Vec::new(), + Source::Other => return Ok(Vec::new()), }; if tiles.is_empty() || z < min_z || zoom + 0.5 < f32::from(min_z) { - return Vec::new(); + return Ok(Vec::new()); } // A mosaic over half the world wide (the globe's, a world projection's) takes the zoom whose // tiles all fit, as its far edges are in view; a flat view keeps those nearest its centre. let z = if view.width() > template::HALF_WORLD { template::fitting_zoom(view, z, min_z) } else { z }; - let url = |t: TileId| { + let url = |t: TileId| -> Result, String> { let i = (t.x as usize).wrapping_add(t.y as usize) % tiles.len(); - tiles.get(i).map(|tpl| template::expand(tpl, t, 256, false, &[])) + let Some(tpl) = tiles.get(i) else { return Ok(None) }; + let expanded = template::expand(tpl, t, 256, false, &[]); + // Archive tile keys are made here from the archive the layer explicitly names. They + // aren't network destinations; their HTTP range reads pass the transport checks. + if !crate::pmtiles::is_tile(&expanded) && !crate::mbtiles::is_tile(&expanded) && !crate::cog::is_tile(&expanded) { + if let Some(from) = from { + from.check(&expanded)?; + } else { + crate::address::check(&expanded, None)?; + } + } + Ok(Some(expanded)) }; + // Ancestors and children are cache-only, but an invalid destination must not read a tile + // previously cached by an explicitly local layer either. + let cached_url = |t| url(t).ok().flatten(); let mut exact = Vec::new(); let mut stand_ins = Vec::new(); // A missing tile is stood in for by its nearest ancestor in the cache and, for raster tiles, @@ -557,14 +612,14 @@ pub(crate) fn tiles_in(src: &Source, view: &Extent, reach: Option<&Extent>, fram // at are already here, and they are sharper. Nothing is drawn twice, so a see-through layer // stays even. let stand_in = |out: &mut Vec, p: &Placed| { - let parent = ancestor(p, min_z, &url); + let parent = ancestor(p, min_z, &cached_url, from.as_ref()); if want != Want::Image || p.id.z >= max_z { out.extend(parent); return; } for c in p.id.children() { let cp = Placed { id: c, wx: p.wx * 2 + i64::from(c.x & 1) }; - match url(c).and_then(|u| cache::peek(&u)) { + match cached_url(c).and_then(|u| cache::peek_from(&u, from.as_ref())) { Some(payload) => out.push(Item { payload, at: cp.extent(), z: c.z, clip: cp.extent() }), None => out.extend(parent.as_ref().map(|a| Item { clip: cp.extent(), ..a.clone() })), } @@ -575,12 +630,12 @@ pub(crate) fn tiles_in(src: &Source, view: &Extent, reach: Option<&Extent>, fram placed = template::covering(view, z); } for p in placed { - let Some(u) = url(p.id) else { continue }; + let Some(u) = url(p.id)? else { continue }; // Past the view, only what the cache already has. let state = if p.extent().intersects(view) { - cache::get_tile(&u, want) + cache::get_tile_from(&u, want, from.as_ref()) } else { - match cache::peek(&u) { + match cache::peek_from(&u, from.as_ref()) { Some(payload) => State::Ready(payload), None => { stand_in(&mut stand_ins, &p); @@ -599,14 +654,14 @@ pub(crate) fn tiles_in(src: &Source, view: &Extent, reach: Option<&Extent>, fram } // Stand-ins first, so the real tiles draw over their edges. stand_ins.extend(exact); - stand_ins + Ok(stand_ins) } /// The nearest ancestor of a missing tile already in the cache, clipped to the missing tile. -fn ancestor(p: &Placed, min_z: u8, url: &dyn Fn(TileId) -> Option) -> Option { +fn ancestor(p: &Placed, min_z: u8, url: &dyn Fn(TileId) -> Option, from: Option<&crate::address::Origin>) -> Option { for lv in 1..=p.id.z.saturating_sub(min_z).min(8) { let a = p.id.ancestor(lv)?; - if let Some(payload) = url(a).and_then(|u| cache::peek(&u)) { + if let Some(payload) = url(a).and_then(|u| cache::peek_from(&u, from)) { let ap = Placed { id: a, wx: p.wx.div_euclid(1 << lv) }; return Some(Item { payload, at: ap.extent(), z: a.z, clip: p.extent() }); } diff --git a/crates/tiles/src/picture.rs b/crates/tiles/src/picture.rs index 3e0deb9902..818d7c276d 100644 --- a/crates/tiles/src/picture.rs +++ b/crates/tiles/src/picture.rs @@ -68,6 +68,8 @@ struct Entry { /// Drawn while some of it was on its way, at this cache generation: drawn again once the /// cache has more. waiting: Option, + /// Destination refusals remain visible when the prepared picture is reused. + error: Option, /// For a style with pattern fills, the ids of the sprites it was drawn from. sheets: Option>, } @@ -78,7 +80,7 @@ fn sprites_read(r: &Resolved, zoom: f32, ppp: f32) -> (bool, Option> if !r.style.layers.iter().any(|l| l.visible_at(zoom) && l.draws_patterns()) { return (false, None); } - match sprite::load(&r.sprites, ppp >= 1.5) { + match sprite::load_from(&r.sprites, ppp >= 1.5, r.from.as_ref()) { Load::Pending => (true, None), Load::Ready(s) => (false, Some(s.sheet_ids())), Load::Failed => (false, Some(Vec::new())), @@ -212,7 +214,14 @@ fn draws_shapes(r: &Resolved, zoom: f32) -> bool { /// The pictures of `r`'s shapes over `view`, drawing those missing, and out to `reach` (only /// those already drawn), bottom first; `None` when the style draws no vector shapes here, or the /// view is too wide for pictures (the whole world on the globe). `frame` gives the scale. -pub(crate) fn cover(r: &Arc, frame: &Frame, view: &Extent, reach: Option<&Extent>, pending: &mut bool) -> Option> { +pub(crate) fn cover( + r: &Arc, + frame: &Frame, + view: &Extent, + reach: Option<&Extent>, + pending: &mut bool, + error: &mut Option, +) -> Option> { let ppp = frame.ppp; if !(ppp.is_finite() && ppp > 0.0 && frame.m_per_px.is_finite() && frame.m_per_px > 0.0) || view.width() > template::HALF_WORLD { return None; @@ -261,6 +270,9 @@ pub(crate) fn cover(r: &Arc, frame: &Frame, view: &Extent, reach: Opti e.used = tick; // (A sprite on its way may add to it: an export waits.) *pending |= in_view && (e.waiting.is_some() || sprites_pending); + if let Some(e) = e.error.as_ref().filter(|_| in_view) { + crate::draw::add_error(error, e.clone()); + } found.push((p, Some(e.picture.clone()))); } _ if in_view => { @@ -311,8 +323,12 @@ pub(crate) fn cover(r: &Arc, frame: &Frame, view: &Extent, reach: Opti let tick = s.tick; let mut added = false; for (&i, d) in todo.iter().zip(drawn) { - let (Some(slot), Some((pm, waiting))) = (found.get_mut(i), d) else { continue }; + let (Some(slot), Some(d)) = (found.get_mut(i), d) else { continue }; + let DrawnPicture { image: pm, pending: waiting, error: refused } = d; *pending |= waiting; + if let Some(refused) = &refused { + crate::draw::add_error(error, refused.clone()); + } let bytes = pm.data().len(); let picture = Arc::new(Payload::Image(pm)); let entry = Entry { @@ -321,6 +337,7 @@ pub(crate) fn cover(r: &Arc, frame: &Frame, view: &Extent, reach: Opti bytes, used: tick, waiting: waiting.then_some(generation), + error: refused, sheets: read.clone(), }; s.bytes += bytes; @@ -340,6 +357,9 @@ pub(crate) fn cover(r: &Arc, frame: &Frame, view: &Extent, reach: Opti for &(k, at, z, clip) in covers.get(&i).into_iter().flatten() { if let Some(e) = s.entries.get_mut(&k) { e.used = tick; + if let Some(refused) = &e.error { + crate::draw::add_error(error, refused.clone()); + } stood_in.push(Item { payload: e.picture.clone(), at, z, clip }); } } @@ -426,6 +446,13 @@ fn evict(s: &mut Store) { } } +/// One prepared picture and the pending/refused reads that produced it. +struct DrawnPicture { + image: Pixmap, + pending: bool, + error: Option, +} + /// Draws the pictures of `squares`: each a `side`-pixel pixmap and whether some of it is still /// on its way, `None` for those not drawn. Natively the squares are shared among threads. With /// `until`, drawing stops once it has passed, after a picture (a round of them natively). @@ -437,7 +464,7 @@ fn draw_all( ppp: f32, reach: f64, until: Option, -) -> Vec> { +) -> Vec> { let Some(until) = until else { return draw_some(r, squares, side, m_per_px, ppp, reach) }; let round = if cfg!(target_arch = "wasm32") { 1 } else { std::thread::available_parallelism().map_or(1, |n| n.get()).min(16) }; let mut out = Vec::with_capacity(squares.len()); @@ -452,14 +479,14 @@ fn draw_all( } /// Draws the pictures of all of `squares` (see [`draw_all`]). -fn draw_some(r: &Resolved, squares: &[Placed], side: u32, m_per_px: f64, ppp: f32, reach: f64) -> Vec> { +fn draw_some(r: &Resolved, squares: &[Placed], side: u32, m_per_px: f64, ppp: f32, reach: f64) -> Vec> { let one = |p: &Placed| draw_one(r, p, side, m_per_px, ppp, reach); #[cfg(not(target_arch = "wasm32"))] { let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(squares.len()).min(16); if threads > 1 { let next = std::sync::atomic::AtomicUsize::new(0); - let out: Mutex>> = Mutex::new((0..squares.len()).map(|_| None).collect()); + let out: Mutex>> = Mutex::new((0..squares.len()).map(|_| None).collect()); std::thread::scope(|scope| { for _ in 0..threads { scope.spawn(|| { @@ -481,7 +508,7 @@ fn draw_some(r: &Resolved, squares: &[Placed], side: u32, m_per_px: f64, ppp: f3 } /// Draws the shapes of one square, with those of the tiles around it out to `reach` points. -fn draw_one(r: &Resolved, p: &Placed, side: u32, m_per_px: f64, ppp: f32, reach: f64) -> Option<(Pixmap, bool)> { +fn draw_one(r: &Resolved, p: &Placed, side: u32, m_per_px: f64, ppp: f32, reach: f64) -> Option { let mut pm = Pixmap::new(side, side)?; let e = p.extent(); let frame = Frame { left: e.min_x, top: e.max_y, m_per_px, ppp }; @@ -490,7 +517,7 @@ fn draw_one(r: &Resolved, p: &Placed, side: u32, m_per_px: f64, ppp: f32, reach: let view = Extent::new(e.min_x - pad, e.min_y - pad, e.max_x + pad, e.max_y + pad); let mut d = Drawn::default(); draw_layers(Some(&mut pm), (side, side), &view, r, &frame, Pass::Shapes, &mut d); - Some((pm, d.pending)) + Some(DrawnPicture { image: pm, pending: d.pending, error: d.error }) } #[cfg(test)] @@ -662,33 +689,34 @@ mod tests { drawn(&spec, &f, 1.0); let view = f.extent(300, 200); let mut pending = false; - let held: Vec> = cover(&r, &f, &view, None, &mut pending).unwrap().into_iter().map(|i| i.payload).collect(); + let held: Vec> = cover(&r, &f, &view, None, &mut pending, &mut None).unwrap().into_iter().map(|i| i.payload).collect(); let theirs = |items: &[Item]| items.iter().all(|i| held.iter().any(|h| Arc::ptr_eq(h, &i.payload))); // A little further in, or coarser (as frames drawn mid-motion are): those drawn stand in. // Over the same ground: the view's squares are those drawn, or above or below them. for g in [frame(2.3, 1.0), frame(2.9, 1.0), frame(2.2, 0.6), frame(1.6, 1.0)] { - let (items, lacks) = paced(Pace::Moving, || cover(&r, &g, &view, None, &mut pending).unwrap()); + let (items, lacks) = paced(Pace::Moving, || cover(&r, &g, &view, None, &mut pending, &mut None).unwrap()); assert!(lacks == Some(Unfinished::Later) && !items.is_empty() && theirs(&items), "stood in at {}", g.m_per_px); // Each covering the view's squares, where it is. assert!(items.iter().all(|i| i.at.intersects(&i.clip) && i.clip.width() > 0.0)); } // Still, or over two levels away: drawn. let g = frame(2.3, 1.0); - let still = cover(&r, &g, &view, None, &mut pending).unwrap(); + let still = cover(&r, &g, &view, None, &mut pending, &mut None).unwrap(); assert!(!theirs(&still)); let far = frame(5.5, 1.0); - let (items, lacks) = paced(Pace::Moving, || cover(&r, &far, &view, None, &mut pending).unwrap()); + let (items, lacks) = paced(Pace::Moving, || cover(&r, &far, &view, None, &mut pending, &mut None).unwrap()); // Nothing stands in that far away (a slow machine may leave some for the next frame). assert!(lacks != Some(Unfinished::Later) && !theirs(&items)); // Settling, the exact pictures are drawn while there's time (a round at least). let g = frame(2.35, 1.0); - let (items, lacks) = paced(Pace::Settling, || cover(&r, &g, &view, None, &mut pending).unwrap()); + let (items, lacks) = paced(Pace::Settling, || cover(&r, &g, &view, None, &mut pending, &mut None).unwrap()); // A round at least is new; complete, nothing held stands in. let held_in = |i: &Item| held.iter().any(|h| Arc::ptr_eq(h, &i.payload)); assert!(lacks != Some(Unfinished::Later) && items.iter().any(|i| !held_in(i))); assert!(lacks.is_some() || !items.iter().any(held_in)); // Inside nested calls, the outer one hears of pictures standing in. - let ((_, inner), outer) = paced(Pace::Settling, || paced(Pace::Moving, || cover(&r, &frame(2.25, 1.0), &view, None, &mut pending))); + let ((_, inner), outer) = + paced(Pace::Settling, || paced(Pace::Moving, || cover(&r, &frame(2.25, 1.0), &view, None, &mut pending, &mut None))); assert!(inner == Some(Unfinished::Later) && outer == inner); assert!(PACE.with(|p| p.get()).is_none()); // One deadline for the frame, whoever asks; an inner call keeps within it. @@ -742,16 +770,16 @@ mod tests { let r = resolved(&spec); let mut pending = false; let world = Extent::new(-template::HALF_WORLD * 1.5, -1e6, template::HALF_WORLD * 1.5, 1e6); - assert!(cover(&r, &f, &world, None, &mut pending).is_none()); + assert!(cover(&r, &f, &world, None, &mut pending, &mut None).is_none()); // More squares than a covering holds: drawn straight too, not left blank. let near = frame(12.0, 1.0); let huge = near.extent(20_000, 10_000); assert!(huge.width() < template::HALF_WORLD); - assert!(cover(&r, &near, &huge, None, &mut pending).is_none()); + assert!(cover(&r, &near, &huge, None, &mut pending, &mut None).is_none()); for m_per_px in [0.0, f64::NAN, f64::INFINITY] { - assert!(cover(&r, &Frame { m_per_px, ..f }, &view, None, &mut pending).is_none()); + assert!(cover(&r, &Frame { m_per_px, ..f }, &view, None, &mut pending, &mut None).is_none()); } - assert!(cover(&r, &Frame { ppp: f32::NAN, ..f }, &view, None, &mut pending).is_none()); + assert!(cover(&r, &Frame { ppp: f32::NAN, ..f }, &view, None, &mut pending, &mut None).is_none()); let _ = std::fs::remove_dir_all(dir); } } diff --git a/crates/tiles/src/shapes.rs b/crates/tiles/src/shapes.rs index a105200388..49eaa8669c 100644 --- a/crates/tiles/src/shapes.rs +++ b/crates/tiles/src/shapes.rs @@ -76,16 +76,23 @@ pub fn vector_tiles(spec: &Spec, frame: &Frame, view: &Extent, reach: Option<&Ex let source = drawn_source(&r, level)?; let src = r.style.sources.iter().find(|(id, _)| *id == source).map(|(_, s)| s)?; let mut pending = false; - let tiles = tiles_in(src, view, reach, frame, zoom, &mut pending) - .into_iter() - .map(|it| DataTile { payload: it.payload, at: it.at, z: it.z, clip: it.clip }) - .collect(); + let mut error = crate::draw::left_out(&r, frame, true); + let items = match tiles_in(src, view, reach, frame, zoom, &mut pending) { + Ok(items) => items, + Err(e) => { + crate::draw::add_error(&mut error, e); + Vec::new() + } + }; + let tiles = items.into_iter().map(|it| DataTile { payload: it.payload, at: it.at, z: it.z, clip: it.clip }).collect(); let mut d = Drawn::default(); if let Some(size) = labels { draw_layers(None, size, &frame.extent(size.0, size.1), &r, frame, Pass::Labels, &mut d); } + if let Some(refused) = d.error { + crate::draw::add_error(&mut error, refused); + } let attribution = r.attribution.clone(); - let error = crate::draw::left_out(&r, frame, true); Some(VectorTiles { style: r, source, tiles, level, labels: d.labels, attribution, pending: pending || d.pending, error }) } @@ -226,7 +233,7 @@ fn average(img: &tiny_skia::Pixmap) -> Option<[f32; 4]> { /// the next level, in its order. Its background, when it has one, is the first run, filling the /// square. pub fn shapes(style: &Resolved, source: &str, tile: &Tile, level: u8) -> TileShapes { - shapes_with(style, source, tile, level, &|| sprite::load(&style.sprites, false)) + shapes_with(style, source, tile, level, &|| sprite::load_from(&style.sprites, false, style.from.as_ref())) } /// [`shapes`], with `load` to load the style's sprites. diff --git a/crates/tiles/src/source.rs b/crates/tiles/src/source.rs index ac6f5387c7..98233cc8d2 100644 --- a/crates/tiles/src/source.rs +++ b/crates/tiles/src/source.rs @@ -12,7 +12,7 @@ use std::sync::{Arc, Mutex, OnceLock}; use serde_json::{Value, json}; use web_time::Instant; -use crate::cache::{self, Payload, State, Want}; +use crate::cache::{self, Payload, State}; use crate::style::{Source, Style}; use crate::{address, cog, mbtiles, pmtiles, template}; @@ -75,6 +75,8 @@ impl Spec { /// A style whose sources all have tile addresses. pub struct Resolved { pub style: Style, + /// The effective style document, kept through redirects and cache hits. + pub from: Option, /// The style's sprites as (id, address without `.json`/`.png`), those RustGIS can /// fetch. pub sprites: Vec<(String, String)>, @@ -189,14 +191,18 @@ fn settle(anew: Resolve, cached: Option>, spec: &Spec, key: String /// Reads `spec`'s style and resolves its sources, keeping the result under `key`. fn resolve_anew(spec: &Spec, key: String) -> Resolve { + let mut base = address::Origin::named(&spec.url); let doc = match spec.kind { Kind::Raster => raster_style(spec), Kind::Vector => match vector_style(spec) { Some(d) => d, None => return Resolve::Failed("the built-in vector style could not be read".into()), }, - Kind::Style => match fetch_json(&spec.url) { - Ok(Some(d)) => d, + Kind::Style => match fetch_json(&spec.url, None) { + Ok(Some((d, reached))) => { + base = reached; + d + } Ok(None) => return Resolve::Pending, Err(e) => return Resolve::Failed(e), }, @@ -263,7 +269,6 @@ fn resolve_anew(spec: &Spec, key: String) -> Resolve { style.layers.retain(|l| l.source_layer.as_ref().is_none_or(|s| !spec.hidden_layers.contains(s))); } // Sources given by a TileJSON address. - let base = spec.url.clone(); let Sources { mut left_out, waiting } = match resolve_sources(&mut style, spec, &base) { Ok(Some(sources)) => sources, Ok(None) => return Resolve::Pending, @@ -284,7 +289,15 @@ fn resolve_anew(spec: &Spec, key: String) -> Resolve { } let (sprites, sprites_left_out) = resolve_sprites(&style, &base); left_out.extend(sprites_left_out); - let r = Arc::new(Resolved { style, attribution, sprites, left_out, made: Mutex::new(Instant::now()), waiting }); + let r = Arc::new(Resolved { + style, + from: (spec.kind == Kind::Style).then_some(base), + attribution, + sprites, + left_out, + made: Mutex::new(Instant::now()), + waiting, + }); if let Err(e) = crate::logins::check(&spec.url) { return Resolve::Failed(e.to_string()); } @@ -300,15 +313,15 @@ fn resolve_anew(spec: &Spec, key: String) -> Resolve { /// A style's sprites with their addresses resolved against `base`, those RustGIS can fetch, and /// what was left out of them, and why, if a layer not hidden draws images and so misses them: the /// sprites are the style's choice, and one that can't be fetched only loses its own icons. -fn resolve_sprites(style: &Style, base: &str) -> (Vec<(String, String)>, Vec) { +fn resolve_sprites(style: &Style, base: &address::Origin) -> (Vec<(String, String)>, Vec) { let mut sprites = Vec::new(); let mut left_out = style.sprites_left_out.clone(); let (mut mapbox, mut refused) = (Vec::new(), Vec::new()); for (id, s) in &style.sprites { - let address = template::join(base, s); + let address = template::join(&base.url, s); if s.starts_with("mapbox://") { mapbox.push(id.as_str()); - } else if address::check(&address, Some(base)).is_err() { + } else if base.check(&address).is_err() { refused.push(id.as_str()); } else { sprites.push((id.clone(), address)); @@ -340,7 +353,7 @@ struct Sources { /// Resolves the sources of `style` in place: what was left out of it, or `None` while some the /// style can't be drawn without are still on their way. -fn resolve_sources(style: &mut Style, spec: &Spec, base: &str) -> Result, String> { +fn resolve_sources(style: &mut Style, spec: &Spec, base: &address::Origin) -> Result, String> { let mut pending = false; let mut left_out = Vec::new(); let mut waiting = false; @@ -381,18 +394,22 @@ fn resolve_sources(style: &mut Style, spec: &Spec, base: &str) -> Result Result { - let (tiles, url) = match src { - Source::Vector { tiles, url, .. } | Source::Raster { tiles, url, .. } | Source::RasterDem { tiles, url, .. } => (tiles, url), +fn resolve_source(src: &mut Source, spec: &Spec, base: &address::Origin) -> Result { + let (tiles, url, from) = match src { + Source::Vector { tiles, url, from, .. } | Source::Raster { tiles, url, from, .. } | Source::RasterDem { tiles, url, from, .. } => { + (tiles, url, from) + } Source::Other => return Ok(false), }; + // Kept only in the resolved style, never accepted from a document or saved in the project. + *from = (spec.kind == Kind::Style).then(|| base.clone()); if tiles.is_empty() && let Some(u) = url.clone() { if u.starts_with("mapbox://") { return Err(format!("{u}: mapbox:// addresses need Mapbox's own library")); } - let u = template::join(base, &u); + let u = template::join(&base.url, &u); // An ArcGIS vector tile service answers its TileJSON-like description as JSON only // when asked (`?f=json`); its styles point at the service itself. let path = u.split(['?', '#']).next().unwrap_or(&u).trim_end_matches('/').to_string(); @@ -413,15 +430,21 @@ fn resolve_source(src: &mut Source, spec: &Spec, base: &str) -> Result { + if spec.kind == Kind::Style { + base.check(&u)?; + } else { + address::check(&u, None)?; + } + match fetch_json(&u, (spec.kind == Kind::Style).then_some(base))? { + Some((tj, reached)) => { src.apply_tilejson(&tj); // Tile addresses relative to the TileJSON document, which chose them. - if let Source::Vector { tiles, .. } | Source::Raster { tiles, .. } | Source::RasterDem { tiles, .. } = src { + if let Source::Vector { tiles, from, .. } | Source::Raster { tiles, from, .. } | Source::RasterDem { tiles, from, .. } = src + { + *from = Some(reached.clone()); for t in tiles.iter_mut() { - *t = template::join(&u, t); - address::check(t, Some(&u))?; + *t = template::join(&reached.url, t); + reached.check(t)?; } } } @@ -436,10 +459,14 @@ fn resolve_source(src: &mut Source, spec: &Spec, base: &str) -> Result Option> { } /// The document at `url`: `Ok(None)` while it is on its way. -fn fetch_json(url: &str) -> Result, String> { - match cache::get(url, Want::Json) { - State::Ready(p) => match &*p { - Payload::Json(v) => Ok(Some(v.clone())), +fn fetch_json(url: &str, from: Option<&address::Origin>) -> Result, String> { + match cache::document(url, from) { + (State::Ready(p), origin) => match &*p { + Payload::Json(v) => Ok(Some((v.clone(), origin.unwrap_or_else(|| address::Origin::named(url))))), _ => Err(format!("{}: not a JSON document", crate::logins::inner(url))), }, - State::Pending => Ok(None), - State::Failed(e) => Err(format!("{}: {e}", crate::logins::inner(url))), + (State::Pending, _) => Ok(None), + (State::Failed(e), _) => Err(format!("{}: {e}", crate::logins::inner(url))), } } @@ -763,6 +790,162 @@ mod tests { assert!(matches!(resolve(&spec(Kind::Raster, "http://localhost:8080/{z}/{x}/{y}.png")), Resolve::Ready(_))); } + /// Resolving then drawing exercises the actual source/template/cache chain for raster, + /// vector and elevation tiles. The fetcher is a spy: no fixture address is contacted. + #[test] + fn style_tile_destinations_are_checked_after_expansion() { + let _turn = cache::turn(); + let sent = Arc::new(Mutex::new(Vec::new())); + let observed = sent.clone(); + cache::set_fetcher(move |url, done| { + observed.lock().unwrap().push(url); + done(Ok(Vec::new())); + }); + let frame = crate::draw::tests::world_frame(); + for kind in ["raster", "vector", "raster-dem"] { + for (case, base, host, refusal) in [ + ("forbidden", "https://destinations.example/style.json", "0.0.0.0", Some("never fetched")), + ("local", "https://destinations.example/style.json", "127.0.0.1", Some("local network")), + ("public", "https://destinations.example/style.json", "tiles.example", None), + ("self-hosted", "http://localhost/style.json", "127.0.0.1", None), + ] { + let url = format!("http://{{switch:{host}}}/destination-regression/{kind}/{case}/{{z}}/{{x}}/{{y}}"); + let layer_kind = match kind { + "raster-dem" => "hillshade", + "vector" => "fill", + _ => kind, + }; + let doc = json!({"version": 8, "sources": { + // A document cannot supply the runtime provenance field itself. + "tiles": {"type": kind, "tiles": [url], "from": "http://localhost/"} + }, "layers": [{"id": "tiles", "type": layer_kind, "source": "tiles", "source-layer": "l"}]}); + let mut style = Style::parse(&doc).unwrap(); + let spec = spec(Kind::Style, &format!("{base}?{kind}-{case}")); + let sources = resolve_sources(&mut style, &spec, &address::Origin::named(base)).unwrap().unwrap(); + let r = Resolved { + style, + sprites: Vec::new(), + from: None, + attribution: Vec::new(), + left_out: sources.left_out, + made: Mutex::new(Instant::now()), + waiting: false, + }; + resolved().lock().unwrap().insert(spec.key(), Arc::new(r)); + let before = sent.lock().unwrap().len(); + let mut pm = tiny_skia::Pixmap::new(64, 64).unwrap(); + let drawn = crate::draw::draw(&mut pm, &spec, 1.0, &frame, true); + cache::wait(std::time::Duration::from_secs(2)); + if let Some(why) = refusal { + assert_eq!(sent.lock().unwrap().len(), before, "{kind}/{case} reached the fetcher"); + assert!(drawn.error.as_deref().is_some_and(|e| e.contains(why)), "{kind}/{case}: {:?}", drawn.error); + } else { + assert!(sent.lock().unwrap().len() > before, "{kind}/{case} must still fetch"); + assert!(drawn.error.is_none(), "{kind}/{case}: {:?}", drawn.error); + } + if kind == "vector" { + // A closer view uses prepared pictures. Its refusal must survive a cache + // hit, and the GPU's data-tile path must report it too. + let near = crate::draw::Frame { left: 0.0, top: 64000.0, m_per_px: 1000.0, ppp: 1.0 }; + for _ in 0..2 { + let d = crate::draw::draw(&mut pm, &spec, 1.0, &near, false); + assert_eq!(d.error.is_some(), refusal.is_some(), "prepared picture: {case}"); + } + let view = near.extent(64, 64); + let gpu = crate::shapes::vector_tiles(&spec, &near, &view, None, None).unwrap(); + assert_eq!(gpu.error.is_some(), refusal.is_some(), "GPU data tiles: {case}"); + let moved = crate::draw::Frame { m_per_px: 1100.0, ..near }; + let (d, _) = + crate::picture::paced(crate::picture::Pace::Moving, || crate::draw::draw(&mut pm, &spec, 1.0, &moved, false)); + assert_eq!(d.error.is_some(), refusal.is_some(), "moving stand-in: {case}"); + } + } + } + } + + /// Label-only sources are checked too, even when the GPU or prepared pictures draw + /// a different source's shapes. Neither renderer may discard a label refusal. + #[test] + fn label_only_destinations_report_refusals_in_both_renderers() { + let _turn = cache::turn(); + cache::set_fetcher(|_, done| done(Ok(Vec::new()))); + let frame = crate::draw::Frame { left: 0.0, top: 64000.0, m_per_px: 1000.0, ppp: 1.0 }; + let view = frame.extent(64, 64); + for (host, refused) in [("0.0.0.0", true), ("labels.example", false)] { + let base = format!("https://label-destinations.example/{host}/style.json"); + let doc = json!({"version":8,"sources":{ + "shapes":{"type":"vector","tiles":["https://shapes.example/{z}/{x}/{y}"]}, + "labels":{"type":"vector","tiles":[format!("http://{{switch:{host}}}/{{z}}/{{x}}/{{y}}")]}}, + "layers":[{"id":"shapes","type":"fill","source":"shapes","source-layer":"l"}, + {"id":"labels","type":"symbol","source":"labels","source-layer":"l","layout":{"text-field":"name"}}]}); + let mut style = Style::parse(&doc).unwrap(); + let spec = spec(Kind::Style, &base); + resolve_sources(&mut style, &spec, &address::Origin::named(&base)).unwrap().unwrap(); + resolved().lock().unwrap().insert( + spec.key(), + Arc::new(Resolved { + style, + from: None, + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }), + ); + let gpu = crate::shapes::vector_tiles(&spec, &frame, &view, None, Some((64, 64))).unwrap(); + assert_eq!(gpu.error.is_some(), refused, "label-only GPU: {host}"); + let placed = crate::draw::place(&spec, 1.0, &frame, &view, &view, Some((64, 64))).unwrap(); + assert_eq!(placed.error.is_some(), refused, "label-only placement: {host}"); + } + cache::wait(std::time::Duration::from_secs(2)); + } + + /// A local style can use a public TileJSON, but that document cannot choose local tiles. + /// The metadata is read through the actual cache; only the resulting tile send is refused. + #[test] + fn tilejson_destinations_keep_the_supplying_documents_reach() { + let _turn = cache::turn(); + let sent = Arc::new(Mutex::new(Vec::new())); + let observed = sent.clone(); + let metadata = "https://tilejson-destinations.example/tiles.json"; + cache::set_fetcher(move |url, done| { + observed.lock().unwrap().push(url.clone()); + done(if url == metadata { + Ok(json!({"tiles": ["http://{switch:127.0.0.1}/tilejson-destinations/{z}/{x}/{y}"]}).to_string().into_bytes()) + } else { + Ok(Vec::new()) + }); + }); + let base = "http://localhost/tilejson-destinations/style.json"; + let s = spec(Kind::Style, base); + let doc = json!({"version": 8, "sources": {"tiles": {"type": "raster", "url": metadata}}, + "layers": [{"id": "tiles", "type": "raster", "source": "tiles"}]}); + let mut ready = None; + for _ in 0..3 { + let mut style = Style::parse(&doc).unwrap(); + if let Some(sources) = resolve_sources(&mut style, &s, &address::Origin::named(base)).unwrap() { + ready = Some(Resolved { + style, + sprites: Vec::new(), + from: None, + attribution: Vec::new(), + left_out: sources.left_out, + made: Mutex::new(Instant::now()), + waiting: false, + }); + break; + } + cache::wait(std::time::Duration::from_secs(2)); + } + resolved().lock().unwrap().insert(s.key(), Arc::new(ready.expect("metadata must resolve"))); + let mut pm = tiny_skia::Pixmap::new(64, 64).unwrap(); + let drawn = crate::draw::draw(&mut pm, &s, 1.0, &crate::draw::tests::world_frame(), false); + cache::wait(std::time::Duration::from_secs(2)); + assert!(drawn.error.as_deref().is_some_and(|e| e.contains("local network")), "{:?}", drawn.error); + assert_eq!(*sent.lock().unwrap(), [metadata], "only the metadata, not its chosen local tile, was fetched"); + } + #[test] fn an_unreadable_elevation_source_is_left_out_and_said() { let doc = serde_json::json!({"version": 8, "sources": { @@ -771,22 +954,22 @@ mod tests { "layers": [{"id": "shade", "type": "hillshade", "source": "dem"}, {"id": "sat", "type": "raster", "source": "sat"}]}); let mut style = Style::parse(&doc).unwrap(); let base = "https://t.example.com/style.json"; - let left_out = resolve_sources(&mut style, &spec(Kind::Style, base), base).unwrap().unwrap().left_out; + let left_out = resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)).unwrap().unwrap().left_out; assert_eq!(left_out.len(), 1); assert!(left_out[0].starts_with("elevation source dem left out"), "{left_out:?}"); assert!(matches!(style.sources.iter().find(|s| s.0 == "dem").map(|s| &s.1), Some(Source::Other))); // A source the style can't draw without still fails it. let doc = serde_json::json!({"version": 8, "sources": {"sat": {"type": "raster", "url": "http://169.254.169.254/sat.json"}}, "layers": []}); let mut style = Style::parse(&doc).unwrap(); - assert!(resolve_sources(&mut style, &spec(Kind::Style, base), base).is_err()); + assert!(resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)).is_err()); // Clean sources leave nothing out. let doc = serde_json::json!({"version": 8, "sources": {"sat": {"type": "raster", "tiles": ["https://t.example.com/{z}/{x}/{y}.png"]}}, "layers": []}); let mut style = Style::parse(&doc).unwrap(); - assert_eq!(resolve_sources(&mut style, &spec(Kind::Style, base), base), Ok(Some(Sources::default()))); + assert_eq!(resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)), Ok(Some(Sources::default()))); // Only a source a layer draws from is missed; the others are dropped without a word. let left_out_of = |doc: serde_json::Value| { let mut style = Style::parse(&doc).unwrap(); - resolve_sources(&mut style, &spec(Kind::Style, base), base).unwrap().unwrap().left_out + resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)).unwrap().unwrap().left_out }; let unused = serde_json::json!({"version": 8, "sources": {"dem": {"type": "raster-dem", "url": "http://169.254.169.254/dem.json"}}, "layers": []}); assert!(left_out_of(unused).is_empty()); @@ -806,7 +989,7 @@ mod tests { "layers": [{"id": "shade", "type": "hillshade", "source": "dem"}]}); let mut style = Style::parse(&doc).unwrap(); let base = "https://t.example.com/style.json"; - assert_eq!(resolve_sources(&mut style, &spec(Kind::Style, base), base), Ok(Some(Sources::default()))); + assert_eq!(resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)), Ok(Some(Sources::default()))); assert!( matches!(style.source("dem"), Some(Source::RasterDem { tiles, .. }) if tiles == &["https://t.example.com/{z}/{x}/{y}.png"]) ); @@ -822,14 +1005,16 @@ mod tests { "layers": [{"id": "shade", "type": "hillshade", "source": "dem"}, {"id": "sat", "type": "raster", "source": "sat"}]}); let mut style = Style::parse(&doc).unwrap(); let base = "https://t.example.com/style.json"; - let sources = resolve_sources(&mut style, &spec(Kind::Style, base), base).unwrap().expect("drawn without it meanwhile"); + let sources = resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)) + .unwrap() + .expect("drawn without it meanwhile"); assert_eq!(sources, Sources { left_out: Vec::new(), waiting: true }, "nothing lost, only waited for"); assert!(matches!(style.source("dem"), Some(Source::Other))); assert!(matches!(style.source("sat"), Some(Source::Raster { .. }))); // One no layer draws from isn't waited for. let unused = serde_json::json!({"version": 8, "sources": {"dem": {"type": "raster-dem", "url": "https://unused-on-its-way.example.com/dem.json"}}, "layers": []}); let mut style = Style::parse(&unused).unwrap(); - assert_eq!(resolve_sources(&mut style, &spec(Kind::Style, base), base), Ok(Some(Sources::default()))); + assert_eq!(resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)), Ok(Some(Sources::default()))); assert!(matches!(style.source("dem"), Some(Source::Other))); // Drawn meanwhile, pending (so it is drawn again, and an export waits), without a notice; // resolved again on the next drawing. @@ -837,6 +1022,7 @@ mod tests { let r = Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: Vec::new(), made: Mutex::new(Instant::now()), @@ -864,12 +1050,13 @@ mod tests { "paint": {"color-relief-color": ["interpolate", ["linear"], ["elevation"], 0, "#00f", 10, "#00f"]}}]}); let mut style = Style::parse(&doc).unwrap(); let base = "https://t.example.com/style.json"; - let sources = resolve_sources(&mut style, &spec(Kind::Style, base), base).unwrap().unwrap(); + let sources = resolve_sources(&mut style, &spec(Kind::Style, base), &address::Origin::named(base)).unwrap().unwrap(); assert_eq!(sources.left_out.len(), 1); assert!(matches!(style.source("dem"), Some(Source::Other))); let r = Resolved { style, sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: sources.left_out, made: Mutex::new(Instant::now()), @@ -893,6 +1080,7 @@ mod tests { let r = Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out, made: Mutex::new(Instant::now()), @@ -911,6 +1099,7 @@ mod tests { let stale = Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: vec!["x".into()], made: Mutex::new(long_ago), @@ -920,6 +1109,7 @@ mod tests { let complete = Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: Vec::new(), made: Mutex::new(long_ago), @@ -931,6 +1121,7 @@ mod tests { let expired = Arc::new(Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: vec!["x".into()], made: Mutex::new(long_ago), @@ -963,6 +1154,7 @@ mod tests { Arc::new(Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: left_out.iter().map(|s| s.to_string()).collect(), made: Mutex::new(at), @@ -980,6 +1172,7 @@ mod tests { let other = Arc::new(Resolved { style: Style::parse(&serde_json::json!({"version": 8, "sources": {}, "layers": [{"id": "bg", "type": "background"}]})).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: vec!["dem: gone".into()], made: Mutex::new(Instant::now()), @@ -990,6 +1183,7 @@ mod tests { let arrived = Arc::new(Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: Vec::new(), left_out: vec!["dem: gone".into()], made: Mutex::new(Instant::now()), @@ -1000,6 +1194,7 @@ mod tests { let recredited = Arc::new(Resolved { style: Style::parse(&doc).unwrap(), sprites: Vec::new(), + from: None, attribution: vec!["© Example".into()], left_out: vec!["dem: gone".into()], made: Mutex::new(Instant::now()), @@ -1017,7 +1212,7 @@ mod tests { {"id": "box", "url": "mapbox://sprites/x"}, {"id": "meta", "url": "http://169.254.169.254/s"}, {"url": "no-id"}]}); - let (sprites, said) = resolve_sprites(&Style::parse(&doc).unwrap(), base); + let (sprites, said) = resolve_sprites(&Style::parse(&doc).unwrap(), &address::Origin::named(base)); assert_eq!(sprites, [("good".to_owned(), "https://t.example.com/sprites/a".to_owned())]); assert_eq!(said.len(), 3, "{said:?}"); assert!(said[0].starts_with("1 of 4 sprites left out"), "{said:?}"); @@ -1027,7 +1222,7 @@ mod tests { let two = |layers: &serde_json::Value| { let doc = serde_json::json!({"version": 8, "sources": {}, "layers": layers, "sprite": [ {"id": "a", "url": "mapbox://sprites/a"}, {"id": "b", "url": "mapbox://sprites/b"}]}); - resolve_sprites(&Style::parse(&doc).unwrap(), base).1 + resolve_sprites(&Style::parse(&doc).unwrap(), &address::Origin::named(base)).1 }; // Said only of a style that misses them: not with no layer drawing images, or one hidden. assert!(two(&serde_json::json!([])).is_empty()); @@ -1038,7 +1233,7 @@ mod tests { assert!(said[0].starts_with("2 of 2 sprites left out") && said[0].ends_with(": a, b"), "{said:?}"); // Clean: nothing to say. let doc = serde_json::json!({"version": 8, "sources": {}, "layers": [], "sprite": "https://t.example.com/s"}); - assert!(resolve_sprites(&Style::parse(&doc).unwrap(), base).1.is_empty()); + assert!(resolve_sprites(&Style::parse(&doc).unwrap(), &address::Origin::named(base)).1.is_empty()); } /// Every sprite of a style fetched with a login (not only the first): one at the style's own @@ -1055,7 +1250,7 @@ mod tests { {"id": "near", "url": "sprites/a"}, {"id": "also", "url": format!("{origin}/other/b")}, {"id": "far", "url": "https://elsewhere.example.com/c"}]}); - let (sprites, said) = resolve_sprites(&Style::parse(&doc).unwrap(), &base); + let (sprites, said) = resolve_sprites(&Style::parse(&doc).unwrap(), &address::Origin::named(&base)); assert_eq!(sprites.len(), 3, "{said:?}"); let plain = [format!("{origin}/sprites/a"), format!("{origin}/other/b"), "https://elsewhere.example.com/c".to_string()]; for ((id, address), plain) in sprites.iter().zip(&plain) { diff --git a/crates/tiles/src/sprite.rs b/crates/tiles/src/sprite.rs index 6d2605411d..d9b0c47192 100644 --- a/crates/tiles/src/sprite.rs +++ b/crates/tiles/src/sprite.rs @@ -266,7 +266,12 @@ enum SheetLoad { /// and a sprite has it, else at 1×. Ready once none is on its way, with those that could be /// read; one that can't only loses its images. pub fn load(sprites: &[(String, String)], hi_dpi: bool) -> Load { - load_with(sprites, |base| load_sheet(base, hi_dpi)) + load_from(sprites, hi_dpi, None) +} + +/// Loads style-chosen sheets with the same reach as their supplying document. +pub(crate) fn load_from(sprites: &[(String, String)], hi_dpi: bool, from: Option<&crate::address::Origin>) -> Load { + load_with(sprites, |base| load_sheet(base, hi_dpi, from)) } fn load_with(sprites: &[(String, String)], sheet: impl Fn(&str) -> SheetLoad) -> Load { @@ -301,27 +306,29 @@ fn keep(all: &mut HashMap>, key: String, sheet: Arc, b all.insert(key, sheet); } -fn load_sheet(base: &str, hi_dpi: bool) -> SheetLoad { +/// Tries high density then normal density without changing the document's authority. +fn load_sheet(base: &str, hi_dpi: bool, from: Option<&crate::address::Origin>) -> SheetLoad { // A login revoked or not covering this address: before any cached sheet is served. if crate::logins::check(base).is_err() { return SheetLoad::Failed; } if hi_dpi { - match load_one(base, "@2x") { + match load_one(base, "@2x", from) { SheetLoad::Failed => {} other => return other, } } - load_one(base, "") + load_one(base, "", from) } -fn load_one(base: &str, density: &str) -> SheetLoad { - let key = format!("{base}|{density}"); +/// Caches decoded sheets separately by reach; both index and picture use that ceiling. +fn load_one(base: &str, density: &str, from: Option<&crate::address::Origin>) -> SheetLoad { + let key = format!("{:?}|{base}|{density}", from.map(|o| o.reach)); if let Some(s) = sprites().lock().unwrap_or_else(|p| p.into_inner()).get(&key) { return SheetLoad::Ready(s.clone()); } - let index = cache::get(&with_suffix(base, &format!("{density}.json")), Want::Json); - let sheet = cache::get(&with_suffix(base, &format!("{density}.png")), Want::Image); + let index = cache::get_from(&with_suffix(base, &format!("{density}.json")), Want::Json, from); + let sheet = cache::get_from(&with_suffix(base, &format!("{density}.png")), Want::Image, from); match (index, sheet) { (State::Ready(i), State::Ready(p)) => match (&*i, &*p) { (Payload::Json(index), Payload::Image(sheet)) => { diff --git a/crates/tiles/src/style.rs b/crates/tiles/src/style.rs index e8aa84ca35..91e8d3a4bf 100644 --- a/crates/tiles/src/style.rs +++ b/crates/tiles/src/style.rs @@ -1213,6 +1213,8 @@ impl StyleView { pub enum Source { Vector { tiles: Vec, + /// Runtime provenance of the tile templates; `None` for a user-selected template. + from: Option, url: Option, min_zoom: u8, max_zoom: u8, @@ -1220,6 +1222,8 @@ pub enum Source { }, Raster { tiles: Vec, + /// Runtime provenance of the tile templates; `None` for a user-selected template. + from: Option, url: Option, tile_size: u32, min_zoom: u8, @@ -1229,6 +1233,8 @@ pub enum Source { /// Elevation tiles, for `hillshade` and `color-relief` layers. RasterDem { tiles: Vec, + /// Runtime provenance of the tile templates; `None` for a user-selected template. + from: Option, url: Option, tile_size: u32, min_zoom: u8, @@ -1310,14 +1316,14 @@ impl Source { let (tiles, url, attribution) = (strings(get("tiles")), str_of(get("url")), str_of(get("attribution")).unwrap_or_default()); let (min_zoom, max_zoom) = (zoom_u8(get("minzoom"), 0), zoom_u8(get("maxzoom"), 22)); match get("type").and_then(Json::as_str) { - Some("vector") => Source::Vector { tiles, url, min_zoom, max_zoom, attribution }, + Some("vector") => Source::Vector { tiles, from: None, url, min_zoom, max_zoom, attribution }, Some("raster") => { let tile_size = get("tileSize").and_then(Json::as_u64).map_or(512, |t| t.clamp(1, 4096) as u32); - Source::Raster { tiles, url, tile_size, min_zoom, max_zoom, attribution } + Source::Raster { tiles, from: None, url, tile_size, min_zoom, max_zoom, attribution } } Some("raster-dem") => { let tile_size = get("tileSize").and_then(Json::as_u64).map_or(512, |t| t.clamp(1, 4096) as u32); - Source::RasterDem { tiles, url, tile_size, min_zoom, max_zoom, attribution, encoding: DemEncoding::parse(j) } + Source::RasterDem { tiles, from: None, url, tile_size, min_zoom, max_zoom, attribution, encoding: DemEncoding::parse(j) } } _ => Source::Other, } @@ -2310,6 +2316,7 @@ mod tests { st.source("r"), Some(&Source::Raster { tiles: vec!["https://r/{z}/{x}/{y}.png".into()], + from: None, url: None, tile_size: 256, min_zoom: 0, @@ -2324,6 +2331,7 @@ mod tests { v, Source::Vector { tiles: vec!["https://x/{z}/{x}/{y}.pbf".into()], + from: None, url: Some("https://x/tiles.json".into()), min_zoom: 0, max_zoom: 14, diff --git a/crates/tiles/src/warp.rs b/crates/tiles/src/warp.rs index fa31acbbb0..9955f70384 100644 --- a/crates/tiles/src/warp.rs +++ b/crates/tiles/src/warp.rs @@ -81,8 +81,11 @@ pub(crate) enum Made { /// [`missing`]) is left clear, and one that failed otherwise too, the tile made again once it is /// asked for again; with none there at all, the tile is empty when they were all missing, else /// it fails. -pub(crate) fn make(url: &str) -> Made { - if let Some((at, tile)) = made_for_now().lock().unwrap_or_else(|e| e.into_inner()).get(url) +pub(crate) fn make(url: &str, from: Option<&crate::address::Origin>) -> Made { + // Partial composites are shared only with callers holding the same reach. The plan is + // geometry alone and can be shared; its pictures always pass the inherited boundary. + let key = format!("{:?}|{url}", from.map_or(crate::address::Reach::Local, |o| o.reach)); + if let Some((at, tile)) = made_for_now().lock().unwrap_or_else(|e| e.into_inner()).get(&key) && at.elapsed() < FOR_NOW { return Made::ForNow(tile.clone()); @@ -106,7 +109,8 @@ pub(crate) fn make(url: &str) -> Made { let (mut waiting, mut error) = (false, None); for p in &plan.parts { // A matrix tile past the service's coverage is routine; a WMS picture (any area asked) isn't. - let state = if p.tile.is_some() { cache::get_tile(&p.url, Want::Image) } else { cache::get(&p.url, Want::Image) }; + let state = + if p.tile.is_some() { cache::get_tile_from(&p.url, Want::Image, from) } else { cache::get_from(&p.url, Want::Image, from) }; match state { State::Ready(payload) => pictures.push(Some(payload)), State::Pending => waiting = true, @@ -150,7 +154,7 @@ pub(crate) fn make(url: &str) -> Made { let tile = compose(&plan, &pictures, SIZE).map_or(Payload::Empty, Payload::Image); let mut kept = made_for_now().lock().unwrap_or_else(|e| e.into_inner()); if !for_now { - kept.remove(url); + kept.remove(&key); return Made::Done(tile); } // Those past their while first, then all if it is still full. @@ -159,7 +163,7 @@ pub(crate) fn make(url: &str) -> Made { kept.clear(); } let tile = Arc::new(tile); - kept.insert(url.to_string(), (web_time::Instant::now(), tile.clone())); + kept.insert(key, (web_time::Instant::now(), tile.clone())); Made::ForNow(tile) } diff --git a/docs/user-guide/maps.md b/docs/user-guide/maps.md index eace026222..49289a973f 100644 --- a/docs/user-guide/maps.md +++ b/docs/user-guide/maps.md @@ -225,6 +225,18 @@ Ocean, Atlas, Blank), which need no network, and web tile layers over them: HTTPS tiles use [platform certificate trust](data.md#https-certificates) on desktop and the command line, including private CAs trusted by the operating system. The web app uses the browser. +Tile addresses are checked after filling their placeholders, as well as when a style or +TileJSON document supplies them. A document on the internet cannot direct tiles into the local +network. Explicitly selected self-hosted servers remain supported, including ordinary +hostnames resolved by local DNS. Native fetchers filter the resolved IPs before connecting, +using the supplying document's actual connection reach. Link-local and unspecified addresses +are always refused. Document redirects also change the base for relative references. +Native redirects are checked at each hop. In the web app, use the final service address: +redirect destinations are hidden from the app and are refused instead of followed unchecked. +The browser controls DNS resolution and local-network access; the app cannot inspect its +resolved IPs. Native tile and document requests connect directly, without an environment +HTTP proxy, so a proxy cannot resolve destinations beyond these checks. + The gallery's **Web Basemaps** need no API key: the OpenFreeMap vector maps (Liberty, Bright, Positron, Dark, Fiord) and USGS Imagery. Choosing one replaces the other web basemaps; tick **Stack on the current basemaps** above them to add it over the others instead (clicking one From f6896403b7ef8f990ef7ea732e05b94b3774f22d Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 7 Nov 2026 14:32:39 -0600 Subject: [PATCH 02/10] fix(tiles): retain final origins across fetch and rendering --- apps/rustgis-web/src/web.rs | 26 +- crates/render/src/tile_mesh.rs | 9 +- crates/tiles/src/address.rs | 40 ++- crates/tiles/src/cache.rs | 442 +++++++++++++++++++++++++++++---- crates/tiles/src/draw.rs | 80 +++--- crates/tiles/src/shapes.rs | 19 +- crates/tiles/src/source.rs | 107 ++++++++ crates/tiles/src/template.rs | 9 + crates/tiles/src/warp.rs | 21 +- docs/user-guide/data.md | 10 + docs/user-guide/maps.md | 11 +- 11 files changed, 667 insertions(+), 107 deletions(-) diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index ffa648a257..29b7ea4503 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -597,6 +597,20 @@ async fn fetch_bytes( still: Option, omit: &'static [u16], ) -> Result, String> { + fetch_reached(url, rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::address::Reach::Local, capture, still, omit) + .await + .map(|fetched| fetched.bytes) +} + +/// GET carries its inherited reach into final URL preparation and returns the normalized +/// credential-free destination with its bytes. Fetch never follows an opaque redirect. +async fn fetch_reached( + url: &str, + ceiling: rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::address::Reach, + capture: Capture, + still: Option, + omit: &'static [u16], +) -> Result { use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{cache::RequestDiagnostics, failure::Failure}; let mut diagnostics = RequestDiagnostics::with_capture(url, "GET", capture); diagnostics.omit_statuses(omit); @@ -618,6 +632,8 @@ async fn fetch_bytes( let prepared = rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::s3::prepare(logins::inner(url))?; let url = prepared.url.as_str(); check_destination(url)?; + let destination = web_sys::Url::new(url).map_err(|_| "not a valid request address")?.href(); + address::check_reach(&destination, ceiling)?; // An ArcGIS sign-in's token, for the portal and the servers it covers, renewed first when it // is about to expire. It goes as the request's `token` parameter, not a header: ArcGIS's // query endpoints answer the header's CORS preflight with an error. @@ -635,7 +651,7 @@ async fn fetch_bytes( // URL parsing in a browser can normalize a numeric or escaped hostname. Judge // that actual destination against the already checked, prepared address. let normalized = web_sys::Url::new(request).map_err(|_| "not a valid request address")?; - address::check(&normalized.href(), Some(url))?; + address::check_reach(&normalized.href(), ceiling)?; let authenticated = prepared.signed || arcgis.is_some() || login.is_some(); let err = |e: wasm_bindgen::JsValue| { if authenticated { "the browser couldn't process the authenticated request".to_string() } else { format!("{e:?}") } @@ -675,7 +691,7 @@ async fn fetch_bytes( init.set_referrer_policy(web_sys::ReferrerPolicy::NoReferrer); init.set_credentials(web_sys::RequestCredentials::Omit); } - Ok((window, request.to_string(), init, abort, range, tail, max, prepared.url, bound.to_string(), authenticated)) + Ok((window, normalized.href(), init, abort, range, tail, max, destination, bound.to_string(), authenticated)) }, |(window, request, init, abort, range, tail, max, url, bound, authenticated)| { (window.fetch_with_str_and_init(&request, &init), abort, range, tail, max, url, bound, authenticated) @@ -776,7 +792,7 @@ async fn fetch_bytes( if whole_tail.is_some_and(|n| bytes.len() as u64 > n) { return Err(format!("{url}: {}", cache::NO_RANGES)); } - Ok(bytes) + Ok(cache::Fetched { bytes, origin: address::Origin::named(&url) }) } .await; cache::check_still(&still)?; @@ -1048,9 +1064,9 @@ fn clipboard() -> Option { /// the app must not read what the user saved on the site. fn services(inbox: Inbox, font_inbox: Inbox, notes: Notes, ctx: egui::Context, embedded: bool) -> Services { // Map tiles come through the browser's fetch (the app repaints when they arrive). - rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::cache::set_fetcher_omitting(|url, done, omit| { + rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::cache::set_fetcher_with_origin(|url, ceiling, done, omit| { wasm_bindgen_futures::spawn_local(crate::request::handoff( - move |capture, still| async move { fetch_bytes(&url, capture, still, omit).await }, + move |capture, still| async move { fetch_reached(&url, ceiling, capture, still, omit).await }, done, )); }); diff --git a/crates/render/src/tile_mesh.rs b/crates/render/src/tile_mesh.rs index 28c3fa1627..36de91397a 100644 --- a/crates/render/src/tile_mesh.rs +++ b/crates/render/src/tile_mesh.rs @@ -393,7 +393,7 @@ pub fn meshes( // Whether the style's sprite is still on its way, asked before the store is locked (a // lookup in the sprites the tile cache keeps). let (sprite_pending, sprites_read) = if rustgis_tiles::shapes::draws_patterns(style, level) { - match rustgis_tiles::sprite::load(&style.sprites, false) { + match style.load_sprites(false) { rustgis_tiles::sprite::Load::Pending => (true, Vec::new()), rustgis_tiles::sprite::Load::Ready(s) => (false, s.sheet_ids()), rustgis_tiles::sprite::Load::Failed => (false, Vec::new()), @@ -704,6 +704,13 @@ mod tests { assert!(meshes(&resolved(serde_json::json!({"fill-pattern": "dots"})), "s", &[], 3, None).1); // Without pattern fills, a mesh doesn't wait for sprites. assert!(!meshes(&resolved(serde_json::json!({"fill-color": "#f00"})), "s", &[], 3, None).1); + // A public style's private sprite must be refused synchronously, not queued by + // this preliminary GPU lookup. `serve` answers without opening any sockets. + let mut private = resolved(serde_json::json!({"fill-pattern":"dots"})); + let style = Arc::get_mut(&mut private).unwrap(); + style.from = Some(rustgis_tiles::address::Origin::named("https://mesh-style.example/style.json")); + style.sprites = vec![("default".into(), "http://127.0.0.1/mesh-private-only".into())]; + assert!(!meshes(&private, "s", &[], 3, None).1, "a refused sprite must never become a pending fetch"); } #[test] diff --git a/crates/tiles/src/address.rs b/crates/tiles/src/address.rs index 1ed8c7eaaa..e38a73bf96 100644 --- a/crates/tiles/src/address.rs +++ b/crates/tiles/src/address.rs @@ -35,11 +35,23 @@ pub struct Origin { } impl Origin { - /// Origin for transports without redirects or DNS visibility, such as the browser. - pub(crate) fn named(url: &str) -> Self { + /// Origin for a trusted transport's normalized final destination when DNS is opaque, + /// such as the browser. This does not perform normalization or grant credentials. + pub fn named(url: &str) -> Self { Self { url: url.to_owned(), reach: reach(crate::logins::inner(url)) } } + /// Provenance for a trusted native transport's actual peer, never its DNS candidates. + /// Forbidden peers and non-web destinations are rejected before their bytes are accepted. + pub fn connected(url: &str, peer: IpAddr) -> Result { + let observed = ip_reach(peer); + check_reach(url, observed)?; + if observed == Reach::Forbidden { + return Err("that peer address is never fetched".into()); + } + Ok(Self { url: url.into(), reach: observed }) + } + /// Checks a descendant without replacing the document's observed reach with host spelling. pub(crate) fn check(&self, url: &str) -> Result<(), String> { check_reach(url, self.reach) @@ -139,8 +151,28 @@ pub fn check(url: &str, from: Option<&str>) -> Result<(), String> { check_reach(url, from.map_or(Reach::Local, |u| reach(crate::logins::inner(u)))) } -/// Unwraps runtime addresses and applies one inherited reach ceiling through every wrapper. -fn check_reach(url: &str, ceiling: Reach) -> Result<(), String> { +/// Keeps a selected account only on its own effective server. Explicit anonymous reads stay +/// anonymous for descendants; neither wrapper can grant a wider network reach. +pub(crate) fn inherit(from: &str, to: &str) -> Result { + let (plain, anonymous) = crate::s3::split_anonymous(from); + if anonymous { + return Ok(crate::s3::anonymous(to)); + } + if let Some(bound) = rustgis_data::s3_address::parse(plain).map_err(str::to_string)? { + let old = crate::s3::bound_destination(plain)?.ok_or("not a saved S3 address")?; + let candidate = rustgis_data::s3_address::bind(bound.binding, bound.endpoint, to).map_err(str::to_string)?; + let new = crate::s3::bound_destination(&candidate)?.ok_or("not a saved S3 address")?; + if crate::logins::origin(&old).is_some_and(|origin| Some(origin) == crate::logins::origin(&new)) { + return Ok(candidate); + } + return Ok(to.to_string()); + } + Ok(crate::logins::inherit(from, to)) +} + +/// Checks a transport's normalized destination under its inherited runtime ceiling, unwrapping +/// account markers without widening it. Call before sending; Forbidden is always refused. +pub fn check_reach(url: &str, ceiling: Reach) -> Result<(), String> { crate::logins::check(url).map_err(str::to_string)?; if let Some(bound) = rustgis_data::login_address::parse(url).map_err(str::to_string)? { return check_reach(bound.url, ceiling); diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 1ee1e42754..8e3a26c54b 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -53,12 +53,14 @@ pub enum State { /// Called once with a fetch's bytes or error. pub type Done = Box, String>) + Send>; +/// Bytes and the actual effective origin observed by a replacement transport. #[derive(Debug)] -struct Fetched { - bytes: Vec, - origin: crate::address::Origin, +pub struct Fetched { + pub bytes: Vec, + pub origin: crate::address::Origin, } -type FetchedDone = Box) + Send>; +/// A policy-aware fetch completes once with bytes and their final provenance, or an error. +pub type FetchedDone = Box) + Send>; /// Called once with a download's bytes or what is known of its failure (#509). pub type DoneDetailed = Box, Failure>) + Send>; // The carrier lives with source I/O so lower-layer workers inherit the same context. @@ -86,7 +88,7 @@ pub(crate) fn caller_callback( }) } -type Fetcher = Box; +type Fetcher = Box; /// Hands back a decoded picture: [`RAW`] pixels, or the encoded bytes it was given when it /// couldn't decode them (they are decoded here then). pub type Decoded = Box, Vec>) + Send>; @@ -158,6 +160,9 @@ fn waker() -> &'static Mutex>> { /// Replaces how addresses are fetched (the browser's `fetch`). `done` must be called once. /// Decode download markers with [`split_download_limit`] and enforce their budget while reading. +/// The legacy byte-only hook is a trusted embedding override responsible for its own +/// destination policy. New transports should use set_fetcher_with_origin to receive the +/// inherited ceiling and report actual provenance; this hook cannot report redirects or peers. pub fn set_fetcher(f: impl Fn(String, Done) + Send + Sync + 'static) { set_fetcher_omitting(move |url, done, _| f(url, done)); } @@ -166,7 +171,35 @@ pub fn set_fetcher(f: impl Fn(String, Done) + Send + Sync + 'static) { /// map's edge): a transport keeping diagnostics passes them to /// [`RequestDiagnostics::omit_statuses`], so it records what the native transport records. pub fn set_fetcher_omitting(f: impl Fn(String, Done, &'static [u16]) + Send + Sync + 'static) { - *fetcher().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(Box::new(f))); + *fetcher().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(Box::new(move |url, _, done, omit| { + let origin = crate::address::Origin::named(&crate::s3::bound_destination(&url).ok().flatten().unwrap_or_else(|| url.clone())); + f(url, Box::new(move |r| done(r.map(|bytes| Fetched { bytes, origin }))), omit); + }))); +} + +/// Installs a trusted transport that checks normalized URLs, redirects and (when available) +/// actual peers against the supplied ceiling BEFORE sending. Completion reports final +/// provenance atomically with the bytes. Browser transports use Origin::named on a normalized +/// destination; native transports use Origin::connected with the socket's actual peer. +pub fn set_fetcher_with_origin(f: impl Fn(String, crate::address::Reach, FetchedDone, &'static [u16]) + Send + Sync + 'static) { + *fetcher().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(Box::new(move |url, ceiling, done, omit| { + let original = url.clone(); + f( + url, + ceiling, + Box::new(move |result| { + done(result.and_then(|mut fetched| { + crate::address::check_reach(&fetched.origin.url, ceiling)?; + if fetched.origin.reach > ceiling { + return Err("the fetched answer exceeds its request's network reach".into()); + } + fetched.origin.url = crate::address::inherit(&original, &fetched.origin.url)?; + Ok(fetched) + })); + }), + omit, + ); + }))); } /// Undoes [`set_fetcher`]: addresses are fetched natively again (a test done with its own). @@ -413,16 +446,33 @@ pub(crate) fn get_from(url: &str, want: Want, from: Option<&crate::address::Orig /// Returns JSON and its effective origin from the same cache entry and lock acquisition. pub(crate) fn document(url: &str, from: Option<&crate::address::Origin>) -> (State, Option) { - request(url, Want::Json, &[], from) + let (state, origin, _) = request(url, Want::Json, &[], from); + (state, origin) +} + +/// A drawable tile and any omissions in a usable partial composite, from the same lookup. +pub(crate) fn tile_noted(url: &str, want: Want, from: Option<&crate::address::Origin>) -> (State, Option) { + let (state, _, note) = request(url, want, TILE_OMIT, from); + (state, note) +} + +/// Missing tiles and a view's unfetched dropped work are ordinary gaps, not drawing errors. +pub(crate) fn routine_tile_failure(error: &str) -> bool { + error == DROPPED || http_status(error).is_some_and(|status| TILE_OMIT.contains(&status)) } /// The statuses a map tile's request omits ([`get_tile`]). const TILE_OMIT: &[u16] = &[404, 410]; /// [`get`] with the statuses that are expected answers for this fetch. -fn request(url: &str, want: Want, omit: &'static [u16], from: Option<&crate::address::Origin>) -> (State, Option) { +fn request( + url: &str, + want: Want, + omit: &'static [u16], + from: Option<&crate::address::Origin>, +) -> (State, Option, Option) { if let Err(e) = crate::logins::check(resource_of(url)) { - return (State::Failed(e.to_string()), None); + return (State::Failed(e.to_string()), None, None); } // A map tile of a layer in another system: its service's picture as it comes, or made from // its pictures (#769). @@ -450,9 +500,9 @@ fn request(url: &str, want: Want, omit: &'static [u16], from: Option<&crate::add drop(c); return refused_request(url, message); } - return (State::Ready(p.clone()), origin.clone()); + return (State::Ready(p.clone()), origin.clone(), None); } - Entry::Pending => return (State::Pending, None), + Entry::Pending => return (State::Pending, None, None), Entry::Failed(e, at) if ceiling <= slot.ceiling && at.elapsed() < retry_wait(e, RETRY) => { // A server that said when to come back (429 or 503 with `Retry-After`) is asked // again then: the map draws again at that time, though nothing else draws it. @@ -460,7 +510,7 @@ fn request(url: &str, want: Want, omit: &'static [u16], from: Option<&crate::add if let Some(wait) = retry_after(e) { notify_at(*at + wait); } - return (State::Failed(e.clone()), None); + return (State::Failed(e.clone()), None, None); } Entry::Failed(..) => {} } @@ -468,6 +518,7 @@ fn request(url: &str, want: Want, omit: &'static [u16], from: Option<&crate::add if crate::warp::is_warp(url) { drop(c); // Made once its pictures are here; asked again (as the map draws) until then. + let mut note = None; return ( match crate::warp::make(url, from) { crate::warp::Made::Waiting => State::Pending, @@ -481,27 +532,31 @@ fn request(url: &str, want: Want, omit: &'static [u16], from: Option<&crate::add ); State::Ready(payload) } - crate::warp::Made::ForNow(payload) => State::Ready(payload), + crate::warp::Made::ForNow(payload, omitted) => { + note = Some(omitted); + State::Ready(payload) + } crate::warp::Made::Failed(e) => { store_from(url, Entry::Failed(e.clone(), Instant::now()), ceiling); State::Failed(e) } }, None, + note, ); } let asked_at = crate::mbtiles::epoch(); c.slots.insert(url.to_string(), Slot { entry: Entry::Pending, ceiling, used: tick, asked: now }); drop(c); start(url.to_string(), want, omit, asked_at, ceiling); - (State::Pending, None) + (State::Pending, None, None) } /// A cache-policy refusal is a failed request too: record it like a transport refusal, /// after releasing the cache lock, so it cannot become unexplained missing tiles. -fn refused_request(url: &str, message: String) -> (State, Option) { +fn refused_request(url: &str, message: String) -> (State, Option, Option) { RequestDiagnostics::new(url, "GET").failed(&Failure::from(message.clone())); - (State::Failed(message), None) + (State::Failed(message), None, None) } /// What the cache holds for `url`, without fetching it. @@ -1133,7 +1188,12 @@ fn download_detailed_within(url: &str, max: usize, timeout: std::time::Duration, }); let f = fetcher().lock().unwrap_or_else(|e| e.into_inner()).clone(); match f { - Some(f) => f(marked.clone(), Box::new(move |r| done(r.map_err(|e| Failure::new(&marked, e)))), omit), + Some(f) => f( + marked.clone(), + crate::address::Reach::Local, + Box::new(move |r| done(r.map(|fetched| fetched.bytes).map_err(|e| Failure::new(&marked, e)))), + omit, + ), None => native::download_within(marked, timeout, omit, done), } } @@ -1342,12 +1402,7 @@ fn fetch_reached(url: String, ceiling: crate::address::Reach, done: FetchedDone, let done: FetchedDone = Box::new(move |r| done(crate::logins::check(resource_of(&key)).map_err(str::to_string).and(r))); let f = fetcher().lock().unwrap_or_else(|e| e.into_inner()).clone(); match f { - Some(f) => { - // Browser redirects are refused; the adapter checks the actual normalized URL. - // Custom fetchers must enforce the same destination boundary. - let origin = crate::address::Origin::named(&crate::s3::bound_destination(&url).ok().flatten().unwrap_or_else(|| url.clone())); - f(url, Box::new(move |r| done(r.map(|bytes| Fetched { bytes, origin }))), omit) - } + Some(f) => f(url, ceiling, done, omit), None => native::fetch_reached(url, ceiling, done, omit), } } @@ -1720,7 +1775,6 @@ mod native { crate::address::check(&url, None).map_err(|e| ureq::Error::Io(std::io::Error::new(std::io::ErrorKind::PermissionDenied, e)))?; let ceiling = if self.public_only { Reach::Public } else { Reach::Local }; let all = self.inner.resolve(uri, config, timeout)?; - let ceiling = if all.iter().any(|a| crate::address::ip_reach(a.ip()) == Reach::Public) { Reach::Public } else { ceiling }; let mut kept = self.empty(); for a in all.iter().filter(|a| { let reach = crate::address::ip_reach(a.ip()); @@ -1738,6 +1792,49 @@ mod native { } } + /// Resolver filtering retains explicit local fallback while removing forbidden peers. + /// Public-document requests never receive local candidates. This fixture opens no sockets. + #[cfg(test)] + #[test] + fn mixed_dns_destinations_keep_local_only_when_allowed() { + use ureq::unversioned::resolver::{ResolvedSocketAddrs, Resolver}; + #[derive(Debug)] + struct Answer(bool); + impl Resolver for Answer { + fn resolve( + &self, + _: &ureq::http::Uri, + _: &ureq::config::Config, + _: ureq::unversioned::transport::NextTimeout, + ) -> Result { + let mut all = self.empty(); + all.push("127.0.0.1:80".parse().unwrap()); + all.push("169.254.169.254:80".parse().unwrap()); + if self.0 { + all.push("203.0.113.1:80".parse().unwrap()); + } + Ok(all) + } + } + let uri = "http://mixed-dns.example/".parse().unwrap(); + let config = config_builder(std::time::Duration::from_secs(1)).build(); + for mixed in [true, false] { + for public_only in [true, false] { + let result = CheckedResolver { inner: Answer(mixed), public_only }.resolve(&uri, &config, Default::default()); + if !mixed && public_only { + assert!(result.unwrap_err().to_string().contains("resolves to")); + } else { + let kept = result.unwrap(); + assert_eq!(kept.len(), if mixed && !public_only { 2 } else { 1 }); + assert!(kept.iter().all(|a| crate::address::ip_reach(a.ip()) != Reach::Forbidden)); + if public_only { + assert!(kept.iter().all(|a| crate::address::ip_reach(a.ip()) == Reach::Public)); + } + } + } + } + } + thread_local! { static SENT_TO: std::cell::Cell> = const { std::cell::Cell::new(None) }; } @@ -1767,8 +1864,61 @@ mod native { details: &ureq::unversioned::transport::ConnectionDetails, chained: Option<()>, ) -> Result, ureq::Error> { - let reach = details.addrs.first().map_or(Reach::Public, |a| crate::address::ip_reach(a.ip())); - Ok(self.0.connect(details, chained)?.map(|inner| Box::new(ReachedTransport { inner, reach }) as Self::Out)) + use ureq::unversioned::transport::time::Duration; + let first = details.addrs.first().map_or(Reach::Public, |a| crate::address::ip_reach(a.ip())); + let mixed = details.addrs.iter().any(|a| crate::address::ip_reach(a.ip()) != first); + if !mixed { + return Ok(self.0.connect(details, chained)?.map(|inner| Box::new(ReachedTransport { inner, reach: first }) as Self::Out)); + } + // The transport API does not expose its peer. Try at most two homogeneous + // groups under one deadline, so success identifies the actual reach without + // losing an explicitly selected server's fallback or changing its TLS name. + let other = if first == Reach::Public { Reach::Local } else { Reach::Public }; + let began = web_time::Instant::now(); + let mut last = None; + for (index, reach) in [first, other].into_iter().enumerate() { + let mut addrs = details.resolver.empty(); + for address in details.addrs.iter().filter(|a| crate::address::ip_reach(a.ip()) == reach) { + addrs.push(*address); + } + let mut attempt = connection_at(details, addrs); + attempt.now = (details.current_time)(); + if !details.timeout.after.is_not_happening() { + let left = details.timeout.after.saturating_sub(began.elapsed()); + if left.is_zero() { + return Err(ureq::Error::Timeout(details.timeout.reason)); + } + attempt.timeout.after = Duration::Exact(left / (2 - index) as u32); + } + match self.0.connect(&attempt, chained) { + Ok(Some(inner)) => return Ok(Some(Box::new(ReachedTransport { inner, reach }) as Self::Out)), + Ok(None) => {} + Err(e) => last = Some(e), + } + } + match last { + Some(error) => Err(error), + None => Ok(None), + } + } + } + + /// Reuses URI, TLS, cancellation and resolver settings while narrowing socket candidates. + fn connection_at<'a>( + d: &ureq::unversioned::transport::ConnectionDetails<'a>, + addrs: ureq::unversioned::resolver::ResolvedSocketAddrs, + ) -> ureq::unversioned::transport::ConnectionDetails<'a> { + ureq::unversioned::transport::ConnectionDetails { + uri: d.uri, + addrs, + config: d.config, + request_level: d.request_level, + resolver: d.resolver, + now: d.now, + timeout: d.timeout, + current_time: d.current_time.clone(), + run_connector: d.run_connector.clone(), + register_shutdown: d.register_shutdown.clone(), } } @@ -1821,12 +1971,24 @@ mod native { #[cfg(test)] /// Exercises classified DNS and real HTTP while routing every connection to an owned listener. pub(super) fn fixture_agent(answer: std::net::SocketAddr, listener: std::net::SocketAddr, public_only: bool) -> ureq::Agent { + fixture_answers(vec![answer], listener, public_only, false) + } + + /// Mixed-DNS fixtures fail the synthetic public connection without opening a socket; + /// all other fixture connections go only to the listener owned by the test. + #[cfg(test)] + fn fixture_answers( + answers: Vec, + listener: std::net::SocketAddr, + public_only: bool, + fail_public: bool, + ) -> ureq::Agent { use ureq::unversioned::{ resolver::Resolver, transport::{ConnectionDetails, Connector, DefaultConnector, Transport}, }; #[derive(Debug)] - struct Dns(std::net::SocketAddr); + struct Dns(Vec); impl Resolver for Dns { fn resolve( &self, @@ -1836,41 +1998,55 @@ mod native { ) -> Result { let mut addresses = self.empty(); let literal = uri.host().and_then(|h| h.parse::().ok()); - addresses.push(literal.map_or(self.0, |ip| std::net::SocketAddr::new(ip, uri.port_u16().unwrap_or(80)))); + if let Some(ip) = literal { + addresses.push(std::net::SocketAddr::new(ip, uri.port_u16().unwrap_or(80))); + } else { + for answer in &self.0 { + addresses.push(*answer); + } + } Ok(addresses) } } #[derive(Debug)] - struct Route(std::net::SocketAddr); + struct Route(std::net::SocketAddr, bool); impl Connector<()> for Route { type Out = Box; fn connect(&self, d: &ConnectionDetails, _: Option<()>) -> Result, ureq::Error> { + if self.1 && d.addrs.first().is_some_and(|a| crate::address::ip_reach(a.ip()) == Reach::Public) { + return Err(std::io::Error::from(std::io::ErrorKind::ConnectionRefused).into()); + } let mut addrs = d.addrs.clone(); for address in addrs.iter_mut() { *address = self.0; } - let fixture = ConnectionDetails { - uri: d.uri, - addrs, - config: d.config, - request_level: d.request_level, - resolver: d.resolver, - now: d.now, - timeout: d.timeout, - current_time: d.current_time.clone(), - run_connector: d.run_connector.clone(), - register_shutdown: d.register_shutdown.clone(), - }; + let fixture = connection_at(d, addrs); DefaultConnector::default().connect(&fixture, None) } } ureq::Agent::with_parts( config_builder(std::time::Duration::from_secs(2)).build(), - ReachConnector(Route(listener)), - CheckedResolver { inner: Dns(answer), public_only }, + ReachConnector(Route(listener, fail_public)), + CheckedResolver { inner: Dns(answers), public_only }, ) } + /// Actual requests exercise resolver filtering, connector fallback and retained provenance. + #[cfg(test)] + #[test] + fn mixed_dns_destinations_preserve_explicit_local_fallback() { + let _turn = super::turn(); + let base = super::tests::serve(vec!["HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntile".into()]); + let listener = base.strip_prefix("http://").unwrap().parse().unwrap(); + let answers = vec!["203.0.113.1:80".parse().unwrap(), "127.0.0.1:80".parse().unwrap()]; + let public = fixture_answers(answers.clone(), listener, true, true); + assert!(fetch_answer(&public, "http://mixed-fixture.example/style.json", Reach::Public, false, &[]).is_err()); + let explicit = fixture_answers(answers, listener, false, true); + let got = fetch_answer(&explicit, "http://mixed-fixture.example/style.json", Reach::Local, false, &[]).unwrap(); + assert_eq!(got.bytes, b"tile"); + assert_eq!(got.origin.reach, Reach::Local); + } + /// A download agent that connects only to internet addresses. pub(super) fn public_agent(timeout: std::time::Duration) -> ureq::Agent { checked_agent(config_builder(timeout).build(), ureq::unversioned::resolver::DefaultResolver::default(), true) @@ -1934,7 +2110,11 @@ mod native { } diagnostics.sending(); super::check_current()?; - let req = req.config().timeout_global(agent.config().timeouts().global).build(); + let req = if sending.config().timeouts().global == agent.config().timeouts().global { + req + } else { + req.config().timeout_global(agent.config().timeouts().global).build() + }; let (resp, reached) = connected(|| req.call()).map_err(unanswered)?; ceiling = ceiling.min(reached); diagnostics.answered(resp.status().as_u16()); @@ -2252,7 +2432,11 @@ mod native { diagnostics.sending(); super::check_current()?; // Narrowing the connection pool must retain the caller's download timeout. - let req = req.config().timeout_global(agent.config().timeouts().global).build(); + let req = if sending.config().timeouts().global == agent.config().timeouts().global { + req + } else { + req.config().timeout_global(agent.config().timeouts().global).build() + }; let (mut resp, reached) = connected(|| req.call()).map_err(unanswered)?; ceiling = ceiling.min(reached); let status = resp.status().as_u16(); @@ -2328,7 +2512,8 @@ mod native { same_origin &= crate::logins::origin(&next) == first_origin; at = next; }; - let origin = Origin { url: if same_origin { crate::logins::inherit(url, &at) } else { at }, reach: ceiling }; + let retained = if same_origin || crate::s3::split_anonymous(url).1 { crate::address::inherit(url, &at)? } else { at }; + let origin = Origin { url: retained, reach: ceiling }; if resp.status().as_u16() == 204 { return Ok(Fetched { bytes: Vec::new(), origin }); } @@ -3180,7 +3365,7 @@ mod tests { /// A server on this computer answering each request with the next of `answers`. #[cfg(not(target_arch = "wasm32"))] - fn serve(answers: Vec) -> String { + pub(super) fn serve(answers: Vec) -> String { serve_bytes(answers.into_iter().map(String::into_bytes).collect()) } @@ -3791,6 +3976,169 @@ mod tests { assert_eq!(target.accept().unwrap_err().kind(), std::io::ErrorKind::WouldBlock); } + /// Draws the actual raster selection path against seeded cache entries, without I/O. + fn draw_cached_raster(url: &str, from: crate::address::Origin) -> (Pixmap, crate::Drawn) { + let mut style = crate::style::Style::parse(&serde_json::json!({"version":8, + "sources":{"s":{"type":"raster","tiles":[url],"maxzoom":0}}, + "layers":[{"id":"r","type":"raster","source":"s"}]})) + .unwrap(); + if let Some((_, crate::style::Source::Raster { from: origin, .. })) = style.sources.first_mut() { + *origin = Some(from.clone()); + } + let resolved = crate::source::Resolved { + style, + from: Some(from), + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }; + let frame = crate::draw::tests::world_frame(); + let mut image = Pixmap::new(64, 64).unwrap(); + let mut drawn = crate::Drawn::default(); + crate::draw::draw_layers(Some(&mut image), (64, 64), &frame.extent(64, 64), &resolved, &frame, crate::draw::Pass::All, &mut drawn); + (image, drawn) + } + + /// Browser-normalized metadata reaches the document consumer atomically with its bytes. + /// Descendants carry Public through the real tile queue to the replacement transport, + /// which refuses a normalized local address before its simulated send. + #[test] + fn replacement_transport_destinations_keep_normalized_origins_and_ceiling() { + let _turn = turn(); + let sent = Arc::new(Mutex::new(Vec::new())); + let observed = sent.clone(); + set_fetcher_with_origin(move |url, ceiling, done, _| { + let normalized = if url == "https://public.example\\@localhost/style.json" { + "https://public.example/@localhost/style.json" + } else if url == "http://%31%32%37.0.0.1/tile.png" { + "http://127.0.0.1/tile.png" + } else { + &url + }; + let check = crate::address::check_reach(normalized, ceiling); + done(check.map(|_| { + observed.lock().unwrap().push((normalized.to_string(), ceiling)); + Fetched { bytes: b"{}".to_vec(), origin: crate::address::Origin::named(normalized) } + })); + }); + let root = "https://public.example\\@localhost/style.json"; + document(root, None); + wait(std::time::Duration::from_secs(2)); + let (state, origin) = document(root, None); + assert!(matches!(state, State::Ready(_))); + let origin = origin.unwrap(); + assert_eq!(origin.url, "https://public.example/@localhost/style.json"); + assert_eq!(origin.reach, crate::address::Reach::Public); + let child = "http://%31%32%37.0.0.1/tile.png"; + get_tile_from(child, Want::Image, Some(&origin)); + wait(std::time::Duration::from_secs(2)); + assert!(matches!(get_tile_from(child, Want::Image, Some(&origin)), State::Failed(_))); + assert_eq!(sent.lock().unwrap().len(), 1, "normalized local child never reaches send"); + assert!(origin.check("http://127.0.0.1/tile.png").is_err()); + } + + /// The native response's effective origin keeps the selected saved S3 account for + /// relative documents, without transferring that binding to another server. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn native_destination_origins_keep_saved_s3_selection() { + let _turn = turn(); + let base = serve(vec!["HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}".into()]); + let binding = "0123456789abcdef"; + let name = "destination-origin-binding"; + crate::s3::connect_saved( + crate::s3::Connection { + name: name.into(), + endpoint: Some(base.clone()), + region: "us-east-1".into(), + buckets: vec!["bucket".into()], + credentials: None, + }, + binding, + "destination-account", + ) + .unwrap(); + let bound = rustgis_data::s3_address::bind(binding, &base, "s3://bucket/styles/style.json").unwrap(); + let got = native::fetch_answer(&native::agent(), &bound, crate::address::Reach::Local, false, &[]).unwrap(); + let child = crate::template::join(&got.origin.url, "tiles.json"); + let selected = rustgis_data::s3_address::parse(&child).unwrap(); + assert!(selected.is_some(), "a relative document must retain the selected S3 binding"); + assert_eq!(selected.unwrap().binding, binding); + assert_eq!(crate::s3::prepare(&child).unwrap().url, format!("{base}/bucket/styles/tiles.json")); + let external = crate::template::join(&got.origin.url, "https://other-origin.example/tiles.json"); + assert!(rustgis_data::s3_address::parse(&external).unwrap().is_none()); + crate::s3::disconnect(name); + } + + /// A cached DNS refusal reaches the drawing, while ordinary missing tiles stay quiet. + #[test] + fn cached_destination_failures_reach_raster_drawing() { + let _turn = turn(); + let from = crate::address::Origin::named("https://public-document.example/style.json"); + for (case, reach) in + [("local", crate::address::Reach::Local), ("public", crate::address::Reach::Public), ("missing", crate::address::Reach::Public)] + { + let url = format!("http://cached-draw.example/{case}"); + let entry = if case == "missing" { + Entry::Failed("http status: 404".into(), Instant::now()) + } else { + let mut image = Pixmap::new(1, 1).unwrap(); + image.fill(tiny_skia::Color::from_rgba8(255, 0, 0, 255)); + Entry::Ready(Arc::new(Payload::Image(image)), 4, Some(crate::address::Origin { url: url.clone(), reach })) + }; + store(&url, entry); + let (image, drawn) = draw_cached_raster(&url, from.clone()); + assert_eq!(drawn.error.is_some(), case == "local", "{case}: {:?}", drawn.error); + assert_eq!(image.pixel(32, 32).unwrap().alpha() > 0, case == "public"); + } + } + + /// Partial WMTS composites retain both their useful pixels and the refused-component + /// warning, including the short-lived composite cache hit. + #[test] + fn partial_warp_destinations_report_refused_components() { + use rustgis_data::ogc::warp::{Grid, Matrix, Warp}; + let _turn = turn(); + let warp = Warp { + epsg: 4326, + grid: Grid::Tiles(vec![Matrix { + id: "m".into(), + left: -180.0, + top: 90.0, + cell: 180.0 / 256.0, + tile_w: 256, + tile_h: 256, + cols: 2, + rows: 1, + }]), + area: None, + systems: Vec::new(), + }; + let template = warp.address("http://partial-warp.example/{nm}/{ncol}/{nrow}"); + let url = crate::template::expand(&template, crate::template::TileId { z: 0, x: 0, y: 0 }, 256, false, &[]); + let (base, _, spec) = Warp::parse(&url).unwrap(); + let half = crate::template::HALF_WORLD; + let plan = spec.plan(base, [-half, -half, half, half], 256, &crate::warp::transformer(4326).unwrap()); + assert_eq!(plan.parts.len(), 2); + for (i, part) in plan.parts.iter().enumerate() { + let mut image = Pixmap::new(1, 1).unwrap(); + image.fill(tiny_skia::Color::from_rgba8(255, 0, 0, 255)); + let reach = if i == 0 { crate::address::Reach::Public } else { crate::address::Reach::Local }; + store( + &part.url, + Entry::Ready(Arc::new(Payload::Image(image)), 4, Some(crate::address::Origin { url: part.url.clone(), reach })), + ); + } + let from = crate::address::Origin::named("https://public-document.example/style.json"); + for _ in 0..2 { + let (image, drawn) = draw_cached_raster(&url, from.clone()); + assert!(image.pixels().iter().any(|p| p.alpha() > 0), "valid component still draws"); + assert!(drawn.error.as_deref().is_some_and(|e| e.contains("cached local-network")), "{:?}", drawn.error); + } + } + /// A previously explicit local tile cannot silently stand in for a public document's /// request for the same hostname. The denial is retained in the session diagnostics. #[test] diff --git a/crates/tiles/src/draw.rs b/crates/tiles/src/draw.rs index 9ff38ceda0..3e3bfc2371 100644 --- a/crates/tiles/src/draw.rs +++ b/crates/tiles/src/draw.rs @@ -81,7 +81,7 @@ pub(crate) fn left_out(r: &Resolved, frame: &Frame, mesh: bool) -> Option bool| r.style.layers.iter().any(|l| l.visible_at(zoom) && draws(l)); let hi_dpi = frame.ppp >= 1.5; let unread = sprite_notes(shown(StyleLayer::draws_icons), shown(StyleLayer::draws_patterns), hi_dpi, hi_dpi && !mesh, |hi, what| { - sprite::unread_note(&r.sprites, r.style.sprites_listed, &sprite::load_from(&r.sprites, hi, r.from.as_ref()), what) + sprite::unread_note(&r.sprites, r.style.sprites_listed, &r.load_sprites(hi), what) }); let all: Vec<&str> = r.left_out.iter().map(String::as_str).chain(unread.iter().map(String::as_str)).collect(); (!all.is_empty()).then(|| all.join("; ")) @@ -181,7 +181,7 @@ pub(crate) fn draw_layers( let mut sprite_state: Option>> = None; let mut sprite = |pending: &mut bool| -> Option> { sprite_state - .get_or_insert_with(|| match sprite::load_from(&r.sprites, frame.ppp >= 1.5, r.from.as_ref()) { + .get_or_insert_with(|| match r.load_sprites(frame.ppp >= 1.5) { Load::Ready(s) => Some(s), Load::Pending => { *pending = true; @@ -220,14 +220,14 @@ pub(crate) fn draw_layers( // Hillshade reads the heights a pixel past the view's edges: those tiles too. let pad = 2.0 * frame.m_per_px; let reach = Extent::new(view.min_x - pad, view.min_y - pad, view.max_x + pad, view.max_y + pad); - let list = - match tiles_for(src, if matches!(src, Source::RasterDem { .. }) { &reach } else { view }, frame, zoom, &mut out.pending) { - Ok(list) => list, - Err(e) => { - add_error(&mut out.error, e); - Vec::new() - } - }; + let list = tiles_for( + src, + if matches!(src, Source::RasterDem { .. }) { &reach } else { view }, + frame, + zoom, + &mut out.pending, + &mut out.error, + ); items.insert(src_id.to_string(), list); } let Some(list) = items.get(src_id) else { continue }; @@ -387,13 +387,7 @@ fn place_as( let Some(src_id) = layer.source.as_deref() else { continue }; if !items.contains_key(src_id) { let Some(src) = r.style.sources.iter().find(|s| s.0 == src_id).map(|s| &s.1) else { continue }; - let list = match tiles_in(src, view, Some(reach), frame, zoom, &mut out.pending) { - Ok(list) => list, - Err(e) => { - add_error(&mut out.error, e); - Vec::new() - } - }; + let list = tiles_in(src, view, Some(reach), frame, zoom, &mut out.pending, &mut out.error); items.insert(src_id, list); } let o = (layer.raster_opacity(zoom) * opacity).clamp(0.0, 1.0); @@ -460,7 +454,7 @@ pub fn pick(spec: &Spec, x: f64, y: f64, m_per_px: f64, tol_px: f32, limit: usiz if !matches!(src, Source::Vector { .. }) { continue; } - items.insert(src_id.to_string(), tiles_for(src, &view, &frame, zoom, &mut pending).unwrap_or_default()); + items.insert(src_id.to_string(), tiles_for(src, &view, &frame, zoom, &mut pending, &mut None)); } let Some(list) = items.get(src_id) else { continue }; // Every tile within reach (a feature near a tile's edge may be in its neighbour), the real @@ -554,13 +548,14 @@ fn hits(f: &crate::mvt::Feature, kind: LayerKind, p: [f32; 2], reach: f32) -> bo } /// The tiles of a source in view, with stand-ins for those not here yet. -fn tiles_for(src: &Source, view: &Extent, frame: &Frame, zoom: f32, pending: &mut bool) -> Result, String> { - tiles_in(src, view, None, frame, zoom, pending) +fn tiles_for(src: &Source, view: &Extent, frame: &Frame, zoom: f32, pending: &mut bool, error: &mut Option) -> Vec { + tiles_in(src, view, None, frame, zoom, pending, error) } /// The tiles of a source in view and out to `reach` (those in the cache only), with stand-ins /// for those not here yet. Expanded destinations retain the supplying document's reach; -/// a refusal is reported by the drawing, without touching the shared cache or fetcher. +/// refused tiles are counted in `error` without touching the shared cache or fetcher. Valid +/// tiles still draw, even when another shard of the same source is refused. pub(crate) fn tiles_in( src: &Source, view: &Extent, @@ -568,7 +563,8 @@ pub(crate) fn tiles_in( frame: &Frame, zoom: f32, pending: &mut bool, -) -> Result, String> { + error: &mut Option, +) -> Vec { let (tiles, from, min_z, max_z, want, z) = match src { Source::Raster { tiles, from, tile_size, min_zoom, max_zoom, .. } | Source::RasterDem { tiles, from, tile_size, min_zoom, max_zoom, .. } => { @@ -578,10 +574,10 @@ pub(crate) fn tiles_in( Source::Vector { tiles, from, min_zoom, max_zoom, .. } => { (tiles, from, *min_zoom, *max_zoom, Want::Vector, (zoom.floor() as u8).min(*max_zoom)) } - Source::Other => return Ok(Vec::new()), + Source::Other => return Vec::new(), }; if tiles.is_empty() || z < min_z || zoom + 0.5 < f32::from(min_z) { - return Ok(Vec::new()); + return Vec::new(); } // A mosaic over half the world wide (the globe's, a world projection's) takes the zoom whose // tiles all fit, as its far edges are in view; a flat view keeps those nearest its centre. @@ -606,6 +602,10 @@ pub(crate) fn tiles_in( let cached_url = |t| url(t).ok().flatten(); let mut exact = Vec::new(); let mut stand_ins = Vec::new(); + let mut omitted = 0; + let mut first_error = None; + let mut partial = 0; + let mut first_partial = None; // A missing tile is stood in for by its nearest ancestor in the cache and, for raster tiles, // by its children in the cache, the ancestor only in the quarters they leave: zooming in, the // tiles of the zoom being passed through are often missing while those of the zoom it ends @@ -630,10 +630,23 @@ pub(crate) fn tiles_in( placed = template::covering(view, z); } for p in placed { - let Some(u) = url(p.id)? else { continue }; + let u = match url(p.id) { + Ok(Some(u)) => u, + Ok(None) => continue, + Err(e) => { + omitted += 1; + first_error.get_or_insert(e); + continue; + } + }; // Past the view, only what the cache already has. let state = if p.extent().intersects(view) { - cache::get_tile_from(&u, want, from.as_ref()) + let (state, note) = cache::tile_noted(&u, want, from.as_ref()); + if let Some(note) = note { + partial += 1; + first_partial.get_or_insert(note); + } + state } else { match cache::peek_from(&u, from.as_ref()) { Some(payload) => State::Ready(payload), @@ -645,7 +658,12 @@ pub(crate) fn tiles_in( }; match state { State::Ready(payload) => exact.push(Item { payload, at: p.extent(), z: p.id.z, clip: p.extent() }), - State::Failed(_) => {} + State::Failed(e) => { + if !cache::routine_tile_failure(&e) { + omitted += 1; + first_error.get_or_insert(e); + } + } State::Pending => { *pending = true; stand_in(&mut stand_ins, &p); @@ -654,7 +672,13 @@ pub(crate) fn tiles_in( } // Stand-ins first, so the real tiles draw over their edges. stand_ins.extend(exact); - Ok(stand_ins) + if let Some(reason) = first_error { + add_error(error, format!("{omitted} tiles left out: {reason}")); + } + if let Some(reason) = first_partial { + add_error(error, format!("{partial} tiles are partial (first: {reason})")); + } + stand_ins } /// The nearest ancestor of a missing tile already in the cache, clipped to the missing tile. diff --git a/crates/tiles/src/shapes.rs b/crates/tiles/src/shapes.rs index 49eaa8669c..d906f08096 100644 --- a/crates/tiles/src/shapes.rs +++ b/crates/tiles/src/shapes.rs @@ -18,7 +18,7 @@ use crate::cache::Payload; use crate::draw::{Drawn, Frame, Label, Pass, draw_layers, tiles_in}; use crate::mvt::{GeomKind, Tile}; use crate::source::{Resolve, Resolved, Spec, resolve}; -use crate::sprite::{self, Load, Sprite}; +use crate::sprite::{Load, Sprite}; use crate::style::{FeatureCtx, LayerKind, LineCap, LineJoin, PATTERNS, Rgba, Source, StyleLayer}; use crate::template; @@ -77,13 +77,14 @@ pub fn vector_tiles(spec: &Spec, frame: &Frame, view: &Extent, reach: Option<&Ex let src = r.style.sources.iter().find(|(id, _)| *id == source).map(|(_, s)| s)?; let mut pending = false; let mut error = crate::draw::left_out(&r, frame, true); - let items = match tiles_in(src, view, reach, frame, zoom, &mut pending) { - Ok(items) => items, - Err(e) => { - crate::draw::add_error(&mut error, e); - Vec::new() - } - }; + let items = tiles_in(src, view, reach, frame, zoom, &mut pending, &mut error); + // Mesh backgrounds cover only returned data squares. When selection leaves holes, + // ordinary drawing paints the whole background and the remaining geometry together. + if error.is_some() + && r.style.layers.iter().any(|l| l.kind == LayerKind::Background && shown_in(l, f32::from(level), f32::from(level) + 1.0)) + { + return None; + } let tiles = items.into_iter().map(|it| DataTile { payload: it.payload, at: it.at, z: it.z, clip: it.clip }).collect(); let mut d = Drawn::default(); if let Some(size) = labels { @@ -233,7 +234,7 @@ fn average(img: &tiny_skia::Pixmap) -> Option<[f32; 4]> { /// the next level, in its order. Its background, when it has one, is the first run, filling the /// square. pub fn shapes(style: &Resolved, source: &str, tile: &Tile, level: u8) -> TileShapes { - shapes_with(style, source, tile, level, &|| sprite::load_from(&style.sprites, false, style.from.as_ref())) + shapes_with(style, source, tile, level, &|| style.load_sprites(false)) } /// [`shapes`], with `load` to load the style's sprites. diff --git a/crates/tiles/src/source.rs b/crates/tiles/src/source.rs index 98233cc8d2..5af8e38100 100644 --- a/crates/tiles/src/source.rs +++ b/crates/tiles/src/source.rs @@ -93,6 +93,11 @@ pub struct Resolved { } impl Resolved { + /// Loads every rendering path's sprite sheets under this style's actual origin. + pub fn load_sprites(&self, hi_dpi: bool) -> crate::sprite::Load { + crate::sprite::load_from(&self.sprites, hi_dpi, self.from.as_ref()) + } + /// Whether it may still be used as it is. fn current(&self) -> bool { !self.waiting && (self.left_out.is_empty() || self.made.lock().unwrap_or_else(|e| e.into_inner()).elapsed() < cache::RETRY) @@ -108,6 +113,7 @@ fn kept(old: Arc, new: Arc) -> Arc { || old.style != new.style || old.sprites != new.sprites || old.attribution != new.attribution + || old.from != new.from { return new; } @@ -863,6 +869,103 @@ mod tests { } } + /// A refused shard leaves its squares empty while the same source's valid squares draw. + /// Both a template list and a host switch are exercised without contacting a server. + #[test] + fn refused_tile_shards_leave_the_other_tiles_drawn() { + let _turn = cache::turn(); + let sent = Arc::new(Mutex::new(Vec::new())); + let observed = sent.clone(); + let mut tile = tiny_skia::Pixmap::new(1, 1).unwrap(); + tile.fill(tiny_skia::Color::from_rgba8(255, 0, 0, 255)); + let bytes = tile.encode_png().unwrap(); + cache::set_fetcher(move |url, done| { + observed.lock().unwrap().push(url); + done(Ok(bytes.clone())); + }); + let public = "http://partial-tiles.example/{z}/{x}/{y}"; + let from = address::Origin::named("https://partial-style.example/style.json"); + for x in 0..2 { + for y in 0..2 { + cache::get_tile_from(&format!("http://partial-tiles.example/1/{x}/{y}"), cache::Want::Image, Some(&from)); + } + } + cache::wait(std::time::Duration::from_secs(2)); + for tiles in [ + vec!["http://{switch:127.0.0.1}/{z}/{x}/{y}", public], + vec!["http://{switch:127.0.0.1,partial-tiles.example}/{z}/{x}/{y}"], + vec![public], + ] { + let clean = tiles == [public]; + let doc = json!({"version":8,"sources":{"s":{"type":"raster","tileSize":32,"maxzoom":1,"tiles":tiles}}, + "layers":[{"id":"r","type":"raster","source":"s"}]}); + let mut style = Style::parse(&doc).unwrap(); + let spec = spec(Kind::Style, &format!("{}?{}", from.url, tiles.len())); + let sources = resolve_sources(&mut style, &spec, &from).unwrap().unwrap(); + let r = Resolved { + style, + from: Some(from.clone()), + sprites: Vec::new(), + attribution: Vec::new(), + left_out: sources.left_out, + made: Mutex::new(Instant::now()), + waiting: false, + }; + let frame = crate::draw::tests::world_frame(); + let mut pm = tiny_skia::Pixmap::new(64, 64).unwrap(); + let mut drawn = crate::draw::Drawn::default(); + crate::draw::draw_layers(Some(&mut pm), (64, 64), &frame.extent(64, 64), &r, &frame, crate::draw::Pass::All, &mut drawn); + assert_eq!(pm.pixel(48, 16).unwrap().red(), 255, "a refused shard must not hide a valid tile"); + assert_eq!(pm.pixel(16, 16).unwrap().alpha(), if clean { 255 } else { 0 }); + assert_eq!(drawn.error.is_none(), clean, "{:?}", drawn.error); + if let Some(error) = drawn.error { + assert_eq!(error.matches("local network").count(), 1); + } + assert!(!drawn.pending); + } + assert!(sent.lock().unwrap().iter().all(|url| url.starts_with("http://partial-tiles.example/"))); + } + + /// A refused vector shard must not remove the style background over that square. + /// Falling back to the ordinary drawing path covers both all-refused and mixed shards. + #[test] + fn refused_vector_destinations_keep_the_style_background() { + let _turn = cache::turn(); + cache::set_fetcher(|_, done| done(Ok(Vec::new()))); + let frame = crate::draw::Frame { left: -100.0, top: 100.0, m_per_px: 1.0, ppp: 1.0 }; + for tiles in [ + vec!["http://{switch:127.0.0.1}/{z}/{x}/{y}"], + vec!["http://{switch:127.0.0.1}/{z}/{x}/{y}", "http://background-public.example/{z}/{x}/{y}"], + ] { + let doc = json!({"version":8,"sources":{"s":{"type":"vector","tiles":tiles}},"layers":[ + {"id":"bg","type":"background","paint":{"background-color":"#f00"}}, + {"id":"fill","type":"fill","source":"s","source-layer":"l"}]}); + let mut style = Style::parse(&doc).unwrap(); + let spec = spec(Kind::Style, &format!("https://background-style.example/{}", tiles.len())); + let from = address::Origin::named(&spec.url); + let sources = resolve_sources(&mut style, &spec, &from).unwrap().unwrap(); + let ready = Arc::new(Resolved { + style, + from: Some(from), + sprites: Vec::new(), + attribution: Vec::new(), + left_out: sources.left_out, + made: Mutex::new(Instant::now()), + waiting: false, + }); + resolved().lock().unwrap().insert(spec.key(), ready); + let view = frame.extent(200, 200); + assert!( + crate::shapes::vector_tiles(&spec, &frame, &view, None, None).is_none(), + "background needs the complete fallback drawing" + ); + let mut image = tiny_skia::Pixmap::new(200, 200).unwrap(); + let drawn = crate::draw::draw(&mut image, &spec, 1.0, &frame, false); + assert_eq!(image.pixel(16, 16).unwrap().red(), 255); + assert!(drawn.error.is_some()); + } + } + /// Label-only sources are checked too, even when the GPU or prepared pictures draw /// a different source's shapes. Neither renderer may discard a label refusal. #[test] @@ -1166,6 +1269,10 @@ mod tests { assert!(!old.current()); let r = kept(old.clone(), made(&["dem: gone"], Instant::now())); assert!(Arc::ptr_eq(&r, &old) && r.current()); + // Equal content from another origin must not retain the old root's authority. + let mut moved = made(&["dem: gone"], Instant::now()); + Arc::get_mut(&mut moved).unwrap().from = Some(address::Origin::named("https://refreshed-style.example/style.json")); + assert!(Arc::ptr_eq(&kept(old.clone(), moved.clone()), &moved)); // Something else (or nothing) left out, or another style: the new one. let new = made(&[], Instant::now()); assert!(Arc::ptr_eq(&kept(old.clone(), new.clone()), &new)); diff --git a/crates/tiles/src/template.rs b/crates/tiles/src/template.rs index 279915067a..b1514bf934 100644 --- a/crates/tiles/src/template.rs +++ b/crates/tiles/src/template.rs @@ -201,6 +201,15 @@ pub fn expand(template: &str, t: TileId, tile_size: u32, tms: bool, subdomains: /// Resolves `href` against the document at `base` (for addresses inside a style or TileJSON). pub fn join(base: &str, href: &str) -> String { let href = crate::logins::inner(href); + match rustgis_data::s3_address::parse(base) { + Ok(Some(bound)) => return crate::address::inherit(base, &join(bound.url, href)).unwrap_or_else(|_| base.to_string()), + Err(_) => return base.to_string(), + Ok(None) => {} + } + let (plain, anonymous) = crate::s3::split_anonymous(base); + if anonymous { + return crate::s3::anonymous(&join(plain, href)); + } match rustgis_data::login_address::parse(base) { Ok(Some(bound)) => return crate::logins::inherit(base, &join(bound.url, href)), Err(_) => return base.to_string(), diff --git a/crates/tiles/src/warp.rs b/crates/tiles/src/warp.rs index 9955f70384..b55fb10e9f 100644 --- a/crates/tiles/src/warp.rs +++ b/crates/tiles/src/warp.rs @@ -50,8 +50,8 @@ pub(crate) fn forget() { } /// The map tiles made without a picture that failed: when each was made, and it. -fn made_for_now() -> &'static Mutex)>> { - static KEPT: OnceLock)>>> = OnceLock::new(); +fn made_for_now() -> &'static Mutex, String)>> { + static KEPT: OnceLock, String)>>> = OnceLock::new(); KEPT.get_or_init(Mutex::default) } @@ -72,7 +72,7 @@ pub(crate) enum Made { Done(Payload), /// Made without a picture that failed (the service may answer when asked again): shown, not /// kept in the cache, and made again once the picture is asked for again ([`FOR_NOW`]). - ForNow(Arc), + ForNow(Arc, String), Failed(String), } @@ -85,10 +85,10 @@ pub(crate) fn make(url: &str, from: Option<&crate::address::Origin>) -> Made { // Partial composites are shared only with callers holding the same reach. The plan is // geometry alone and can be shared; its pictures always pass the inherited boundary. let key = format!("{:?}|{url}", from.map_or(crate::address::Reach::Local, |o| o.reach)); - if let Some((at, tile)) = made_for_now().lock().unwrap_or_else(|e| e.into_inner()).get(&key) + if let Some((at, tile, note)) = made_for_now().lock().unwrap_or_else(|e| e.into_inner()).get(&key) && at.elapsed() < FOR_NOW { - return Made::ForNow(tile.clone()); + return Made::ForNow(tile.clone(), note.clone()); } let known = plans().lock().unwrap_or_else(|e| e.into_inner()).get(url).cloned(); let plan = match known { @@ -107,6 +107,7 @@ pub(crate) fn make(url: &str, from: Option<&crate::address::Origin>) -> Made { } let mut pictures = Vec::with_capacity(plan.parts.len()); let (mut waiting, mut error) = (false, None); + let mut failed = 0; for p in &plan.parts { // A matrix tile past the service's coverage is routine; a WMS picture (any area asked) isn't. let state = @@ -121,7 +122,8 @@ pub(crate) fn make(url: &str, from: Option<&crate::address::Origin>) -> Made { // for any raster tile, the map's layer list doesn't name it: a layer drawn in its // own system does, see `native_tiles`.) if p.tile.is_none() || !missing(&e) { - error = Some(e); + failed += 1; + error.get_or_insert(e); } pictures.push(None); } @@ -158,13 +160,14 @@ pub(crate) fn make(url: &str, from: Option<&crate::address::Origin>) -> Made { return Made::Done(tile); } // Those past their while first, then all if it is still full. - kept.retain(|_, (at, _)| at.elapsed() < FOR_NOW); + kept.retain(|_, (at, _, _)| at.elapsed() < FOR_NOW); if kept.len() >= MAX_FOR_NOW { kept.clear(); } let tile = Arc::new(tile); - kept.insert(key, (web_time::Instant::now(), tile.clone())); - Made::ForNow(tile) + let note = format!("{failed} warped tile components left out: {}", error.unwrap_or_default()); + kept.insert(key, (web_time::Instant::now(), tile.clone(), note.clone())); + Made::ForNow(tile, note) } /// Whether a WMTS tile's failure means the service doesn't have it (past its coverage): a 4xx diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index d21dc9a53d..d0ac9bfc97 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -619,6 +619,16 @@ database is kept without its password. The commands `sources.list`, `sources.sav `sources.favourite` and `sources.remove` read and change the lists, for the command line and the [AI Assistant](assistant.md). On the command line they last for the run. +### Network downloads + +On desktop and the command line, web tiles, service documents and file downloads through +**From URL…** connect directly, including `layer.addUrl`. These requests do not use +`HTTP_PROXY`, `HTTPS_PROXY` or `ALL_PROXY`: a proxy could resolve a destination beyond the +app's IP checks. There is no proxy override for these requests. If your network requires a +proxy, download a file with a trusted external tool and open the local copy in RustGIS. +Live tile and service access requires direct connectivity. The web app uses the browser's +network settings; its [tile request limits](maps.md) differ. + ### HTTPS certificates On desktop and the command line, RustGIS's HTTPS clients use platform certificate verification: diff --git a/docs/user-guide/maps.md b/docs/user-guide/maps.md index 49289a973f..2cecf30b98 100644 --- a/docs/user-guide/maps.md +++ b/docs/user-guide/maps.md @@ -228,14 +228,17 @@ command line, including private CAs trusted by the operating system. The web app Tile addresses are checked after filling their placeholders, as well as when a style or TileJSON document supplies them. A document on the internet cannot direct tiles into the local network. Explicitly selected self-hosted servers remain supported, including ordinary -hostnames resolved by local DNS. Native fetchers filter the resolved IPs before connecting, +hostnames resolved by local DNS, with local fallback when the same hostname also returns +public addresses. Native fetchers filter the resolved IPs before connecting, using the supplying document's actual connection reach. Link-local and unspecified addresses are always refused. Document redirects also change the base for relative references. Native redirects are checked at each hop. In the web app, use the final service address: -redirect destinations are hidden from the app and are refused instead of followed unchecked. +tile, style, TileJSON and sprite redirects, including HTTP-to-HTTPS upgrades and CDN hops, +are refused because the browser hides their destinations from the app. The browser controls DNS resolution and local-network access; the app cannot inspect its -resolved IPs. Native tile and document requests connect directly, without an environment -HTTP proxy, so a proxy cannot resolve destinations beyond these checks. +resolved IPs. Native tile and document requests, and file downloads through **From URL…**, +connect directly without an environment HTTP proxy, so a proxy cannot resolve destinations +beyond these checks. See [network downloads](data.md#network-downloads) if your network requires a proxy. The gallery's **Web Basemaps** need no API key: the OpenFreeMap vector maps (Liberty, Bright, Positron, Dark, Fiord) and USGS Imagery. Choosing one replaces the other web basemaps; tick From 155af17834e5aae0a1c3e17727b844422c2eb1de Mon Sep 17 00:00:00 2001 From: mavrick Date: Sat, 7 Nov 2026 15:42:34 -0600 Subject: [PATCH 03/10] fix(tiles): retain transport authority through cache retries --- apps/rustgis-web/src/web.rs | 2 +- crates/tiles/src/address.rs | 19 ++++- crates/tiles/src/cache.rs | 142 +++++++++++++++++++++++++++-------- crates/tiles/src/draw.rs | 6 +- crates/tiles/src/shapes.rs | 8 +- crates/tiles/src/source.rs | 72 ++++++++++++++++++ crates/tiles/src/template.rs | 10 ++- 7 files changed, 219 insertions(+), 40 deletions(-) diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index 29b7ea4503..12d640bc6d 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -792,7 +792,7 @@ async fn fetch_reached( if whole_tail.is_some_and(|n| bytes.len() as u64 > n) { return Err(format!("{url}: {}", cache::NO_RANGES)); } - Ok(cache::Fetched { bytes, origin: address::Origin::named(&url) }) + Ok(cache::Fetched { bytes, origin: address::Origin::named(&url).retain_binding(&bound)? }) } .await; cache::check_still(&still)?; diff --git a/crates/tiles/src/address.rs b/crates/tiles/src/address.rs index e38a73bf96..ae634445f2 100644 --- a/crates/tiles/src/address.rs +++ b/crates/tiles/src/address.rs @@ -45,13 +45,21 @@ impl Origin { /// Forbidden peers and non-web destinations are rejected before their bytes are accepted. pub fn connected(url: &str, peer: IpAddr) -> Result { let observed = ip_reach(peer); - check_reach(url, observed)?; + check_reach(url, Reach::Local)?; if observed == Reach::Forbidden { return Err("that peer address is never fetched".into()); } Ok(Self { url: url.into(), reach: observed }) } + /// Retains a selected binding after the transport verifies it remained authorized through + /// every redirect. Omit this when a hop dropped the login, even on returning to its server. + /// The normalized destination and observed network reach remain unchanged. + pub fn retain_binding(mut self, selected: &str) -> Result { + self.url = inherit(selected, &self.url)?; + Ok(self) + } + /// Checks a descendant without replacing the document's observed reach with host spelling. pub(crate) fn check(&self, url: &str) -> Result<(), String> { check_reach(url, self.reach) @@ -211,6 +219,15 @@ pub fn check_reach(url: &str, ceiling: Reach) -> Result<(), String> { mod tests { use super::*; + #[test] + fn connected_destinations_separate_names_from_observed_peers() { + let origin = Origin::connected("https://selected.internal/style.json", "203.0.113.1".parse().unwrap()).unwrap(); + assert_eq!(origin.reach, Reach::Public); + assert!(origin.check("http://127.0.0.1/tile").is_err()); + assert!(Origin::connected("http://metadata.google.internal/", "203.0.113.1".parse().unwrap()).is_err()); + assert!(Origin::connected("https://selected.internal/", "169.254.169.254".parse().unwrap()).is_err()); + } + #[test] fn judges_addresses() { assert_eq!(reach("https://tile.openstreetmap.org/1/0/0.png"), Reach::Public); diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 8e3a26c54b..7e8bbc4f12 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -180,20 +180,21 @@ pub fn set_fetcher_omitting(f: impl Fn(String, Done, &'static [u16]) + Send + Sy /// Installs a trusted transport that checks normalized URLs, redirects and (when available) /// actual peers against the supplied ceiling BEFORE sending. Completion reports final /// provenance atomically with the bytes. Browser transports use Origin::named on a normalized -/// destination; native transports use Origin::connected with the socket's actual peer. +/// destination; replacement native transports can use Origin::connected with their actual +/// socket peer. The built-in native connector records its peer reach separately. +/// The reported origin also owns login provenance: use Origin::retain_binding only if every +/// hop retained the selected login. Completion never restores a binding the transport dropped. pub fn set_fetcher_with_origin(f: impl Fn(String, crate::address::Reach, FetchedDone, &'static [u16]) + Send + Sync + 'static) { *fetcher().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(Box::new(move |url, ceiling, done, omit| { - let original = url.clone(); f( url, ceiling, Box::new(move |result| { - done(result.and_then(|mut fetched| { + done(result.and_then(|fetched| { crate::address::check_reach(&fetched.origin.url, ceiling)?; if fetched.origin.reach > ceiling { return Err("the fetched answer exceeds its request's network reach".into()); } - fetched.origin.url = crate::address::inherit(&original, &fetched.origin.url)?; Ok(fetched) })); }), @@ -495,15 +496,14 @@ fn request( slot.asked = now; match &slot.entry { Entry::Ready(p, _, origin) => { - if origin.as_ref().is_some_and(|o| o.reach > ceiling) { - let message = format!("{url}: a document on the web can't read a cached local-network answer"); - drop(c); - return refused_request(url, message); + if origin.as_ref().is_none_or(|o| o.reach <= ceiling) { + return (State::Ready(p.clone()), origin.clone(), None); } - return (State::Ready(p.clone()), origin.clone(), None); + // The same name can resolve both locally and publicly. Retry through the + // restrictive pool rather than reusing local bytes or hiding the public answer. } Entry::Pending => return (State::Pending, None, None), - Entry::Failed(e, at) if ceiling <= slot.ceiling && at.elapsed() < retry_wait(e, RETRY) => { + Entry::Failed(e, at) if ceiling == slot.ceiling && at.elapsed() < retry_wait(e, RETRY) => { // A server that said when to come back (429 or 503 with `Retry-After`) is asked // again then: the map draws again at that time, though nothing else draws it. // One that didn't is asked again by the next draw after a minute, as before. @@ -546,7 +546,10 @@ fn request( ); } let asked_at = crate::mbtiles::epoch(); - c.slots.insert(url.to_string(), Slot { entry: Entry::Pending, ceiling, used: tick, asked: now }); + let old = c.slots.insert(url.to_string(), Slot { entry: Entry::Pending, ceiling, used: tick, asked: now }); + if let Some(Slot { entry: Entry::Ready(_, n, _), .. }) = old { + c.bytes = c.bytes.saturating_sub(n); + } drop(c); start(url.to_string(), want, omit, asked_at, ceiling); (State::Pending, None, None) @@ -2077,8 +2080,8 @@ mod native { }); } - /// Asks for `url`'s headers, following up to 5 redirects, each checked as [`get`] checks - /// them. + /// Asks for a caller-selected archive's headers, following up to 5 redirects, each checked + /// as [`get`] checks them. Document-derived reads use the GET queue's inherited ceiling. pub(super) fn head_with(agent: &ureq::Agent, url: &str) -> Result { let mut diagnostics = super::RequestDiagnostics::new(url, "HEAD"); let mut carried_login = false; @@ -2348,7 +2351,8 @@ mod native { /// as tiles a server doesn't have are routine and only their status is read. A status in /// `omit` is an expected answer: its failure isn't recorded, only noted once per server. A /// request carrying a login is recorded by its plain address and scrubbed text only - /// ([`RequestDiagnostics::failed`]). + /// ([`RequestDiagnostics::failed`]). `agent` must already enforce `initial` in its resolver; + /// the worker selects that pool before dispatch. Redirects can only narrow it further. pub(super) fn fetch_answer( agent: &ureq::Agent, url: &str, @@ -4039,6 +4043,72 @@ mod tests { assert!(origin.check("http://127.0.0.1/tile.png").is_err()); } + /// The replacement transport, which sees every redirect, owns the retained binding. + /// Returning anonymously to the original server must not restore a dropped login. + #[test] + fn replacement_destinations_do_not_restore_dropped_logins() { + let _turn = turn(); + let plain = "https://redirect-binding.example/style.json"; + let binding = crate::logins::activate( + "https://redirect-binding.example", + "redirect-fixture", + crate::logins::Method::Header("X-Key".into()), + "fixture", + false, + ) + .unwrap(); + let bound = crate::logins::bind(&binding, plain).unwrap(); + for retained in [false, true] { + let origin = crate::address::Origin::named(plain); + let origin = if retained { origin.retain_binding(&bound).unwrap() } else { origin }; + set_fetcher_with_origin(move |_, _, done, _| { + done(Ok(Fetched { bytes: Vec::new(), origin: origin.clone() })); + }); + let (tx, rx) = std::sync::mpsc::channel(); + fetch_reached(bound.clone(), crate::address::Reach::Local, Box::new(move |r| tx.send(r).unwrap()), &[]); + let fetched = rx.recv_timeout(std::time::Duration::from_secs(2)).unwrap().unwrap(); + let child = crate::template::join(&fetched.origin.url, "tile.json"); + assert_eq!(rustgis_data::login_address::parse(&child).unwrap().is_some(), retained); + } + crate::logins::deactivate(&binding); + } + + /// Local bytes and local failures cannot prevent a public-ceiling fetch of the same name. + #[test] + fn cached_destinations_refetch_under_the_requested_reach() { + use crate::address::{Origin, Reach}; + let _turn = turn(); + let calls = Arc::new(Mutex::new(Vec::new())); + let sent = calls.clone(); + set_fetcher_with_origin(move |_, ceiling, done, _| { + sent.lock().unwrap().push(ceiling); + done(Err("fixture destination refused".into())); + }); + let from = Origin::named("https://public-document.example/style.json"); + for ready in [true, false] { + let url = format!("https://split-cache.example/{ready}"); + let entry = if ready { + Entry::Ready(Arc::new(Payload::Empty), 64, Some(Origin { url: url.clone(), reach: Reach::Local })) + } else { + Entry::Failed("http status: 404".into(), Instant::now()) + }; + store(&url, entry); + assert!(matches!(get_tile_from(&url, Want::Image, Some(&from)), State::Pending), "a restricted fetch must start"); + wait(std::time::Duration::from_secs(2)); + assert!(matches!(get_tile_from(&url, Want::Image, Some(&from)), State::Failed(e) if e == "fixture destination refused")); + } + assert_eq!(*calls.lock().unwrap(), [Reach::Public, Reach::Public]); + let url = "https://split-cache.example/reverse"; + store_from(url, Entry::Failed("restricted refusal".into(), Instant::now()), Reach::Public); + assert!(matches!(get_tile(url, Want::Image), State::Pending), "explicit requests retry restrictive failures too"); + wait(std::time::Duration::from_secs(2)); + assert_eq!(calls.lock().unwrap().last(), Some(&Reach::Local)); + let url = "https://split-cache.example/reusable"; + store(url, Entry::Ready(Arc::new(Payload::Empty), 0, Some(Origin::named(url)))); + assert!(matches!(get_tile_from(url, Want::Image, Some(&from)), State::Ready(_))); + assert_eq!(calls.lock().unwrap().len(), 3, "a permitted ready answer is reused"); + } + /// The native response's effective origin keeps the selected saved S3 account for /// relative documents, without transferring that binding to another server. #[cfg(not(target_arch = "wasm32"))] @@ -4062,6 +4132,8 @@ mod tests { .unwrap(); let bound = rustgis_data::s3_address::bind(binding, &base, "s3://bucket/styles/style.json").unwrap(); let got = native::fetch_answer(&native::agent(), &bound, crate::address::Reach::Local, false, &[]).unwrap(); + let reported = crate::address::Origin::named(&crate::s3::prepare(&bound).unwrap().url).retain_binding(&bound).unwrap(); + assert_eq!(reported, got.origin, "the no-redirect replacement adapter retains the same S3 selection"); let child = crate::template::join(&got.origin.url, "tiles.json"); let selected = rustgis_data::s3_address::parse(&child).unwrap(); assert!(selected.is_some(), "a relative document must retain the selected S3 binding"); @@ -4070,6 +4142,10 @@ mod tests { let external = crate::template::join(&got.origin.url, "https://other-origin.example/tiles.json"); assert!(rustgis_data::s3_address::parse(&external).unwrap().is_none()); crate::s3::disconnect(name); + // Joining is independent of the live login registry. A removed selection keeps the + // child's address and fails preparation, rather than substituting the parent. + assert_eq!(crate::template::join(&got.origin.url, "tiles.json"), child); + assert!(crate::s3::prepare(&child).is_err()); } /// A cached DNS refusal reaches the drawing, while ordinary missing tiles stay quiet. @@ -4077,6 +4153,7 @@ mod tests { fn cached_destination_failures_reach_raster_drawing() { let _turn = turn(); let from = crate::address::Origin::named("https://public-document.example/style.json"); + set_fetcher_with_origin(|_, _, done, _| done(Err("fixture resolves to the local network".into()))); for (case, reach) in [("local", crate::address::Reach::Local), ("public", crate::address::Reach::Public), ("missing", crate::address::Reach::Public)] { @@ -4088,7 +4165,10 @@ mod tests { image.fill(tiny_skia::Color::from_rgba8(255, 0, 0, 255)); Entry::Ready(Arc::new(Payload::Image(image)), 4, Some(crate::address::Origin { url: url.clone(), reach })) }; - store(&url, entry); + let ceiling = if case == "local" { crate::address::Reach::Local } else { crate::address::Reach::Public }; + store_from(&url, entry, ceiling); + draw_cached_raster(&url, from.clone()); + wait(std::time::Duration::from_secs(2)); let (image, drawn) = draw_cached_raster(&url, from.clone()); assert_eq!(drawn.error.is_some(), case == "local", "{case}: {:?}", drawn.error); assert_eq!(image.pixel(32, 32).unwrap().alpha() > 0, case == "public"); @@ -4101,6 +4181,7 @@ mod tests { fn partial_warp_destinations_report_refused_components() { use rustgis_data::ogc::warp::{Grid, Matrix, Warp}; let _turn = turn(); + set_fetcher_with_origin(|_, _, done, _| done(Err("fixture resolves to the local network".into()))); let warp = Warp { epsg: 4326, grid: Grid::Tiles(vec![Matrix { @@ -4132,28 +4213,15 @@ mod tests { ); } let from = crate::address::Origin::named("https://public-document.example/style.json"); + draw_cached_raster(&url, from.clone()); + wait(std::time::Duration::from_secs(2)); for _ in 0..2 { let (image, drawn) = draw_cached_raster(&url, from.clone()); assert!(image.pixels().iter().any(|p| p.alpha() > 0), "valid component still draws"); - assert!(drawn.error.as_deref().is_some_and(|e| e.contains("cached local-network")), "{:?}", drawn.error); + assert!(drawn.error.as_deref().is_some_and(|e| e.contains("local network")), "{:?}", drawn.error); } } - /// A previously explicit local tile cannot silently stand in for a public document's - /// request for the same hostname. The denial is retained in the session diagnostics. - #[test] - fn cached_destinations_report_a_local_answer_refusal() { - let _turn = turn(); - let url = "http://cached-local-destination.example/tile.png"; - let origin = crate::address::Origin { url: url.into(), reach: crate::address::Reach::Local }; - store(url, Entry::Ready(Arc::new(Payload::Empty), 64, Some(origin))); - let from = crate::address::Origin::named("https://public-document.example/style.json"); - let history = rustgis_data::diagnostics::History::default(); - let _scope = history.enter(); - assert!(matches!(get_tile_from(url, Want::Image, Some(&from)), State::Failed(_))); - assert_eq!(network_records(&history, url).len(), 1, "cached destination refusal must be reported"); - } - /// A public style's WMS composite must not regain a local allowance for its parts or /// reuse a composite made explicitly from local answers. Cache seeding performs no I/O. #[test] @@ -4170,9 +4238,17 @@ mod tests { store(&part.url, Entry::Ready(Arc::new(Payload::Image(Pixmap::new(1, 1).unwrap())), 4, Some(origin))); } let from = crate::address::Origin::named("https://public-document.example/style.json"); - assert!(matches!(get_tile_from(url, Want::Image, Some(&from)), State::Failed(_)), "composition must inherit public reach"); + set_fetcher_with_origin(|_, ceiling, done, _| { + assert_eq!(ceiling, crate::address::Reach::Public); + done(Err("fixture resolves to the local network".into())); + }); assert!(matches!(get_tile(url, Want::Image), State::Ready(_)), "explicit local composition still works"); - assert!(matches!(get_tile_from(url, Want::Image, Some(&from)), State::Failed(_)), "a cached local composite stays local"); + assert!(matches!(get_tile_from(url, Want::Image, Some(&from)), State::Pending), "a cached local composite needs restricted parts"); + wait(std::time::Duration::from_secs(2)); + assert!( + matches!(get_tile_from(url, Want::Image, Some(&from)), State::Failed(e) if e.contains("local network")), + "composition must inherit public reach and report its failed restricted fetch" + ); } /// Replaces only the fixture's public DNS names; all connections, if a check regresses, diff --git a/crates/tiles/src/draw.rs b/crates/tiles/src/draw.rs index 3e3bfc2371..5809e8b680 100644 --- a/crates/tiles/src/draw.rs +++ b/crates/tiles/src/draw.rs @@ -634,8 +634,10 @@ pub(crate) fn tiles_in( Ok(Some(u)) => u, Ok(None) => continue, Err(e) => { - omitted += 1; - first_error.get_or_insert(e); + if p.extent().intersects(view) { + omitted += 1; + first_error.get_or_insert(e); + } continue; } }; diff --git a/crates/tiles/src/shapes.rs b/crates/tiles/src/shapes.rs index d906f08096..4e05c5dddf 100644 --- a/crates/tiles/src/shapes.rs +++ b/crates/tiles/src/shapes.rs @@ -77,14 +77,18 @@ pub fn vector_tiles(spec: &Spec, frame: &Frame, view: &Extent, reach: Option<&Ex let src = r.style.sources.iter().find(|(id, _)| *id == source).map(|(_, s)| s)?; let mut pending = false; let mut error = crate::draw::left_out(&r, frame, true); - let items = tiles_in(src, view, reach, frame, zoom, &mut pending, &mut error); + let mut selection_error = None; + let items = tiles_in(src, view, reach, frame, zoom, &mut pending, &mut selection_error); // Mesh backgrounds cover only returned data squares. When selection leaves holes, // ordinary drawing paints the whole background and the remaining geometry together. - if error.is_some() + if selection_error.is_some() && r.style.layers.iter().any(|l| l.kind == LayerKind::Background && shown_in(l, f32::from(level), f32::from(level) + 1.0)) { return None; } + if let Some(note) = selection_error { + crate::draw::add_error(&mut error, note); + } let tiles = items.into_iter().map(|it| DataTile { payload: it.payload, at: it.at, z: it.z, clip: it.clip }).collect(); let mut d = Drawn::default(); if let Some(size) = labels { diff --git a/crates/tiles/src/source.rs b/crates/tiles/src/source.rs index 5af8e38100..a92079e630 100644 --- a/crates/tiles/src/source.rs +++ b/crates/tiles/src/source.rs @@ -926,6 +926,34 @@ mod tests { assert!(sent.lock().unwrap().iter().all(|url| url.starts_with("http://partial-tiles.example/"))); } + /// Refused cache-only shards outside the view do not count as visible holes. + #[test] + fn cache_only_destinations_do_not_report_visible_omissions() { + let _turn = cache::turn(); + cache::set_fetcher(|_, done| done(Ok(Vec::new()))); + let mut style = Style::parse(&json!({"version":8,"sources":{"s":{"type":"vector","tiles":[ + "http://{switch:127.0.0.1}/{z}/{x}/{y}", "https://cache-only.example/{z}/{x}/{y}"]}},"layers":[]})) + .unwrap(); + let (_, src) = style.sources.first_mut().unwrap(); + if let Source::Vector { from, .. } = src { + *from = Some(address::Origin::named("https://cache-only.example/style.json")); + } + let frame = crate::draw::tests::world_frame(); + let reach = crate::template::TileId { z: 0, x: 0, y: 0 }.extent(); + for (x, refused) in [(1, false), (0, true)] { + let mut view = crate::template::TileId { z: 1, x, y: 0 }.extent(); + view.min_x += 1.0; + view.max_x -= 1.0; + view.min_y += 1.0; + view.max_y -= 1.0; + let mut pending = false; + let mut error = None; + crate::draw::tiles_in(src, &view, Some(&reach), &frame, 1.0, &mut pending, &mut error); + assert_eq!(error.is_some(), refused, "{error:?}"); + } + cache::wait(std::time::Duration::from_secs(2)); + } + /// A refused vector shard must not remove the style background over that square. /// Falling back to the ordinary drawing path covers both all-refused and mixed shards. #[test] @@ -966,6 +994,46 @@ mod tests { } } + /// A sprite notice does not remove otherwise usable vector tiles from the mesh path. + #[test] + fn sprite_notices_do_not_disable_vector_meshes() { + let _turn = cache::turn(); + let document = json!({"version":8,"sprite":"unsupported://sheet", + "sources":{"s":{"type":"vector","tiles":["https://notice-tiles.example/{z}/{x}/{y}"],"maxzoom":0}}, + "layers":[{"id":"bg","type":"background"},{"id":"fill","type":"fill","source":"s","source-layer":"l"}, + {"id":"icons","type":"symbol","source":"s","source-layer":"l","layout":{"icon-image":"pin"}}]}); + cache::set_fetcher(move |url, done| { + done(Ok(if url.ends_with("style.json") { document.to_string().into_bytes() } else { Vec::new() })) + }); + let spec = spec(Kind::Style, "https://notice-style.example/style.json"); + resolve(&spec); + cache::wait(std::time::Duration::from_secs(2)); + let Resolve::Ready(ready) = resolve(&spec) else { panic!("the style must be ready") }; + assert!(!ready.left_out.is_empty()); + let frame = crate::draw::tests::world_frame(); + let tiles = crate::shapes::vector_tiles(&spec, &frame, &frame.extent(64, 64), None, None) + .expect("a sprite notice must not force otherwise usable geometry through CPU drawing"); + assert!(tiles.error.is_some(), "the sprite notice is retained"); + } + + /// Resolving a child beyond the saved-address limit must report that child as unusable, + /// rather than asking the cache for the supplying document as though it were a sprite. + #[test] + fn oversized_saved_s3_children_do_not_reuse_the_document() { + let href = "x".repeat(rustgis_data::s3_address::MAX_ADDRESS); + let style = Style::parse(&json!({"version":8,"sources":{},"sprite":href, + "layers":[{"id":"icons","type":"symbol","layout":{"icon-image":"pin"}}]})) + .unwrap(); + let url = + rustgis_data::s3_address::bind("0123456789abcdef", "https://s3-join.example", "https://s3-join.example/bucket/style.json") + .unwrap(); + let base = address::Origin { url, reach: address::Reach::Public }; + let (sprites, notes) = resolve_sprites(&style, &base); + assert!(sprites.is_empty(), "the document is not its own sprite: {sprites:?}"); + assert_eq!(notes.len(), 1); + assert!(notes[0].contains("won't fetch"), "{notes:?}"); + } + /// Label-only sources are checked too, even when the GPU or prepared pictures draw /// a different source's shapes. Neither renderer may discard a label refusal. #[test] @@ -1273,6 +1341,10 @@ mod tests { let mut moved = made(&["dem: gone"], Instant::now()); Arc::get_mut(&mut moved).unwrap().from = Some(address::Origin::named("https://refreshed-style.example/style.json")); assert!(Arc::ptr_eq(&kept(old.clone(), moved.clone()), &moved)); + let mut local = made(&["dem: gone"], Instant::now()); + Arc::get_mut(&mut local).unwrap().from = + Some(address::Origin { url: "https://refreshed-style.example/style.json".into(), reach: address::Reach::Local }); + assert!(Arc::ptr_eq(&kept(local, moved.clone()), &moved), "a public refresh of the same URL drops local authority"); // Something else (or nothing) left out, or another style: the new one. let new = made(&[], Instant::now()); assert!(Arc::ptr_eq(&kept(old.clone(), new.clone()), &new)); diff --git a/crates/tiles/src/template.rs b/crates/tiles/src/template.rs index b1514bf934..03e897433c 100644 --- a/crates/tiles/src/template.rs +++ b/crates/tiles/src/template.rs @@ -202,7 +202,15 @@ pub fn expand(template: &str, t: TileId, tile_size: u32, tms: bool, subdomains: pub fn join(base: &str, href: &str) -> String { let href = crate::logins::inner(href); match rustgis_data::s3_address::parse(base) { - Ok(Some(bound)) => return crate::address::inherit(base, &join(bound.url, href)).unwrap_or_else(|_| base.to_string()), + Ok(Some(bound)) => { + let joined = join(bound.url, href); + return crate::address::inherit(base, &joined).unwrap_or_else(|_| { + // As for bound service logins, keep the actual derived address even when + // it exceeds the parser's limit. Request validation then refuses it; never + // substitute the parent document or drop the selected account. + format!("{}{}:{}:{}:{joined}", rustgis_data::s3_address::PREFIX, bound.binding, bound.endpoint.len(), bound.endpoint) + }); + } Err(_) => return base.to_string(), Ok(None) => {} } From d999767f4c004aa86fa39d27268356bff5f1ac2a Mon Sep 17 00:00:00 2001 From: mavrick Date: Sun, 8 Nov 2026 08:56:10 -0600 Subject: [PATCH 04/10] fix(tiles): preserve account choice and normalized local requests --- apps/rustgis-web/src/request.rs | 33 +++++++++++++++++++ apps/rustgis-web/src/web.rs | 22 ++++++------- crates/tiles/src/cache.rs | 25 +++++++++++++++ crates/tiles/src/source.rs | 56 +++++++++++++++++++++++++++++++++ crates/tiles/src/template.rs | 36 ++++++++++++++++++++- docs/user-guide/maps.md | 11 +++++-- 6 files changed, 167 insertions(+), 16 deletions(-) diff --git a/apps/rustgis-web/src/request.rs b/apps/rustgis-web/src/request.rs index c72f2b93ab..8c094ddb8d 100644 --- a/apps/rustgis-web/src/request.rs +++ b/apps/rustgis-web/src/request.rs @@ -41,6 +41,19 @@ pub(crate) fn fetch_error(url: &str, error: String, authenticated: bool) -> Stri if authenticated && cache::http_status(&error).is_none() { logins::scrub_error(&format!("{url}: {error}")) } else { error } } +/// Checks the browser's normalized destination under the caller's inherited ceiling. +/// Normalization may reveal a local host; its original spelling cannot narrow an explicit +/// local choice or widen a public document's permission. The normalizer performs no I/O. +pub(crate) fn destination( + url: &str, + ceiling: rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::address::Reach, + normalize: impl FnOnce(&str) -> Result, +) -> Result { + let normalized = normalize(url)?; + rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::address::check_reach(&normalized, ceiling)?; + Ok(normalized) +} + #[cfg(test)] mod tests { use super::*; @@ -55,6 +68,26 @@ mod tests { task::{Context, Poll, Waker}, }; + /// The browser parser's output is checked at the same preparation boundary used by GET, + /// HEAD and POST: an explicit local choice survives normalization, but public input does not. + #[test] + fn normalized_destinations_keep_the_request_ceiling() { + use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::address::Reach; + for ceiling in [Reach::Local, Reach::Public] { + for (spelling, normalized, permitted) in [ + ("http://local%68ost:3000/style.json", "http://localhost:3000/style.json", ceiling == Reach::Local), + ("http://%31%36%39.254.169.254/tile", "http://169.254.169.254/tile", false), + ("https://public.example/tile", "https://public.example/tile", true), + ] { + let result = destination(spelling, ceiling, |_| Ok(normalized.into())); + assert_eq!(result.is_ok(), permitted, "{spelling} under {ceiling:?}"); + if permitted { + assert_eq!(result.unwrap(), normalized); + } + } + } + } + /// Status-only authenticated failures retain their machine-readable routine gap/backoff signal. #[test] fn authenticated_fetch_errors_preserve_http_statuses() { diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index e28ecbb26b..c89bf81b73 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -630,9 +630,7 @@ async fn fetch_reached( let bound = url; let prepared = rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::s3::prepare(logins::inner(url))?; let url = prepared.url.as_str(); - check_destination(url)?; - let destination = web_sys::Url::new(url).map_err(|_| "not a valid request address")?.href(); - address::check_reach(&destination, ceiling)?; + let destination = check_destination(url, ceiling)?; // An ArcGIS sign-in's token, for the portal and the servers it covers, renewed first when it // is about to expire. It goes as the request's `token` parameter, not a header: ArcGIS's // query endpoints answer the header's CORS preflight with an error. @@ -813,7 +811,7 @@ async fn post_json( use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{cache::RequestDiagnostics, failure::Failure}; let mut diagnostics = RequestDiagnostics::with_capture(url, "POST", capture); let answer = async { - use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{cache, logins, s3}; + use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{address, cache, logins, s3}; let (promise, authenticated) = crate::request::prepare_and_send( &still, || async { @@ -827,7 +825,7 @@ async fn post_json( let target = logins::inherit(url, &prepared.url); let login = logins::apply(&target).map_err(str::to_string)?; let request = prepared.url.as_str(); - check_destination(request)?; + check_destination(request, address::Reach::Local)?; let err = |e: wasm_bindgen::JsValue| { if login.is_some() { "the browser couldn't process the authenticated request".to_string() } else { format!("{e:?}") } }; @@ -895,7 +893,7 @@ async fn head( use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{cache::RequestDiagnostics, failure::Failure}; let mut diagnostics = RequestDiagnostics::with_capture(url, "HEAD", capture); let answer = async { - use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{cache, logins, s3}; + use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{address, cache, logins, s3}; let sent = crate::request::prepare_and_send( &still, || async { @@ -908,7 +906,7 @@ async fn head( return Ok(None); } let url = prepared.url.as_str(); - check_destination(url)?; + check_destination(url, address::Reach::Local)?; let target = logins::inherit(bound, url); let login = logins::apply(&target).map_err(str::to_string)?; let request = login.as_ref().map_or(url, |applied| applied.url.as_str()); @@ -977,17 +975,17 @@ async fn head( } /// Checks the URL as the browser parses it, before any request or login renewal is sent. -fn check_destination(url: &str) -> Result<(), String> { - use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::address; - let normalized = web_sys::Url::new(url).map_err(|_| "not a valid request address")?; - address::check(&normalized.href(), Some(url)) +fn check_destination(url: &str, ceiling: rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::address::Reach) -> Result { + crate::request::destination(url, ceiling, |url| { + web_sys::Url::new(url).map(|normalized| normalized.href()).map_err(|_| "not a valid request address".into()) + }) } /// A manual redirect is an opaque, zero-status response: Fetch exposes no Location to check. /// Other zero-status responses are unusable too; none is treated as a successful download. fn check_redirect(response: &web_sys::Response) -> Result<(), String> { if response.status() == 0 { - Err("The browser cannot verify this redirect's destination. Use the final service address.".into()) + Err(rustgis_ui_egui::tl!("The browser cannot verify this redirect's destination. Use the final service address.").into()) } else { Ok(()) } diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 4976014519..e9de454cc8 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -4217,6 +4217,7 @@ mod tests { let from = Origin::named("https://public-style.example/style.json"); for case in ["ready", "failures", "retry", "public"] { let url = format!("https://shared-reach.example/{case}"); + let mut retry_times = None; get(&url, Want::Json); wait(std::time::Duration::from_secs(2)); for _ in 0..3 { @@ -4238,11 +4239,35 @@ mod tests { assert_eq!(retry_after(&error), Some(Duration::from_secs(30))); let State::Failed(error) = get(&url, Want::Json) else { panic!("the explicit layer keeps its cooldown") }; assert_eq!(retry_after(&error), Some(Duration::from_secs(120))); + let c = lock(); + let slot = c.slots.get(&url).unwrap(); + let times = [Reach::Local, Reach::Public].map(|reach| match slot.for_reach(reach) { + (Entry::Failed(_, at), stored) if reach == stored => *at, + _ => panic!("each reach keeps its own failed-at time"), + }); + if let Some(previous) = retry_times { + assert_eq!(times, previous, "alternating reads must not postpone either retry"); + } + retry_times = Some(times); } } } let sent = calls.lock().unwrap(); assert_eq!(sent.iter().filter(|(u, _)| u == &url).count(), 2, "one fetch under each reach, not one per frame: {sent:?}"); + drop(sent); + if case == "retry" { + // Move each recorded failure just past its own original wait, without sleeping + // or changing production clocks. The other reach's cooldown remains usable. + for (reach, count) in [(Reach::Public, 3), (Reach::Local, 4)] { + let from = (reach == Reach::Public).then_some(&from); + let State::Failed(error) = get_from(&url, Want::Json, from) else { panic!("the cooldown is still active") }; + let expired = Instant::now() - retry_wait(&error, RETRY) - Duration::from_secs(1); + store_from(&url, Entry::Failed(error, expired), reach); + assert!(matches!(get_from(&url, Want::Json, from), State::Pending), "retry at this reach's deadline"); + wait(std::time::Duration::from_secs(2)); + assert_eq!(calls.lock().unwrap().iter().filter(|(u, _)| u == &url).count(), count); + } + } } } diff --git a/crates/tiles/src/source.rs b/crates/tiles/src/source.rs index b0947be9d8..409771044f 100644 --- a/crates/tiles/src/source.rs +++ b/crates/tiles/src/source.rs @@ -1188,6 +1188,62 @@ mod tests { assert!(notes[0].contains("won't fetch"), "{notes:?}"); } + /// The generated raster style names the user's selected address as both base and tile; + /// resolving it must retain one wrapper rather than nesting the same selection. + #[test] + fn direct_saved_s3_rasters_keep_one_binding() { + let _turn = cache::turn(); + let selected = rustgis_data::s3_address::bind( + "0123456789abcdef", + "https://s3-direct.example", + "https://s3-direct.example/bucket/{z}/{x}/{y}.png", + ) + .unwrap(); + let spec = spec(Kind::Raster, &selected); + let Resolve::Ready(resolved) = resolve(&spec) else { panic!("a direct raster needs no document fetch") }; + let source = &resolved.style.sources.first().unwrap().1; + let Source::Raster { tiles, .. } = source else { panic!("the generated raster source is retained") }; + assert_eq!(tiles, &[selected]); + } + + /// A direct TileJSON selection reaches the cache once with its chosen account, and + /// relative tile addresses inherit that same account from the returned document. + #[test] + fn direct_saved_s3_tilejson_keeps_its_binding() { + let _turn = cache::turn(); + let endpoint = "https://s3-direct-vector.example"; + let binding = crate::s3::new_binding().unwrap(); + let name = "direct TileJSON fixture"; + crate::s3::connect_saved( + crate::s3::Connection { + name: name.into(), + endpoint: Some(endpoint.into()), + region: "us-east-1".into(), + buckets: vec!["bucket".into()], + credentials: None, + }, + &binding, + "direct-vector", + ) + .unwrap(); + let selected = rustgis_data::s3_address::bind(&binding, endpoint, &format!("{endpoint}/bucket/tiles.json")).unwrap(); + let asked = selected.clone(); + cache::set_fetcher_with_origin(move |url, _, done, _| { + assert_eq!(url, asked, "the TileJSON request keeps exactly one selected wrapper"); + let plain = crate::s3::bound_destination(&url).unwrap().unwrap(); + let origin = address::Origin::named(&plain).retain_binding(&url).unwrap(); + done(Ok(cache::Fetched { bytes: br#"{"tiles":["{z}/{x}/{y}.pbf"]}"#.to_vec(), origin })); + }); + let spec = spec(Kind::Vector, &selected); + assert!(matches!(resolve(&spec), Resolve::Pending)); + cache::wait(std::time::Duration::from_secs(2)); + let Resolve::Ready(resolved) = resolve(&spec) else { panic!("the selected TileJSON must load") }; + let Source::Vector { tiles, .. } = &resolved.style.sources.first().unwrap().1 else { panic!("the vector source is retained") }; + let child = rustgis_data::s3_address::parse(&tiles[0]).unwrap().unwrap(); + assert_eq!((child.binding, child.url), (binding.as_str(), &*format!("{endpoint}/bucket/{{z}}/{{x}}/{{y}}.pbf"))); + crate::s3::disconnect(name); + } + /// Label-only sources are checked too, even when the GPU or prepared pictures draw /// a different source's shapes. Neither renderer may discard a label refusal. #[test] diff --git a/crates/tiles/src/template.rs b/crates/tiles/src/template.rs index 85846adf35..56a0d7c1b4 100644 --- a/crates/tiles/src/template.rs +++ b/crates/tiles/src/template.rs @@ -201,7 +201,18 @@ pub fn expand(template: &str, t: TileId, tile_size: u32, tms: bool, subdomains: /// Resolves `href` against the document at `base` (for addresses inside a style or TileJSON). /// An invalid bound base stays refused by request validation rather than resolving anonymously. pub fn join(base: &str, href: &str) -> String { - let href = crate::logins::inner(href); + // Only the supplying document's base carries authority. A link may name a resource, + // but neither a service-login nor an S3 selector in it can choose an account. + let href = match rustgis_data::login_address::parse(href) { + Ok(Some(bound)) => bound.url, + Err(_) => return href.to_string(), + Ok(None) => href, + }; + let href = match rustgis_data::s3_address::parse(href) { + Ok(Some(bound)) => bound.url, + Err(_) => return href.to_string(), + Ok(None) => href, + }; match rustgis_data::s3_address::parse(base) { Ok(Some(bound)) => { let joined = join(bound.url, href); @@ -290,6 +301,29 @@ mod tests { assert_eq!(join("rustgis-login:broken", "/tile"), "rustgis-login:broken"); } + /// A document may name a resource, but cannot choose another saved S3 account for it. + #[test] + fn document_s3_selectors_do_not_choose_an_account() { + let endpoint = "https://s3-links.example"; + let base = rustgis_data::s3_address::bind("0123456789abcdef", endpoint, &format!("{endpoint}/bucket/style.json")).unwrap(); + for url in [format!("{endpoint}/bucket/tile"), "https://other.example/tile".into()] { + let supplied = rustgis_data::s3_address::bind("fedcba9876543210", endpoint, &url).unwrap(); + assert_eq!(join("https://public.example/style.json", &supplied), url, "the supplied selector is removed"); + let joined = join(&base, &supplied); + let selected = rustgis_data::s3_address::parse(&joined).unwrap(); + if url.starts_with(endpoint) { + assert_eq!(selected.unwrap().binding, "0123456789abcdef", "only the document's own account is inherited"); + } else { + assert!(selected.is_none(), "a different server has no inherited account"); + } + } + for malformed in ["rustgis-s3:broken", "rustgis-login:broken"] { + let joined = join(&base, malformed); + assert_eq!(joined, malformed, "malformed selectors remain refused, not relative paths"); + assert!(crate::address::check(&joined, None).is_err()); + } + } + #[test] fn joins_relative_addresses() { let base = "https://x.org/a/b/styles/root.json"; diff --git a/docs/user-guide/maps.md b/docs/user-guide/maps.md index 1e9c3194fa..32897c061d 100644 --- a/docs/user-guide/maps.md +++ b/docs/user-guide/maps.md @@ -280,13 +280,18 @@ HTTPS tiles use [platform certificate trust](data.md#https-certificates) on desk command line, including private CAs trusted by the operating system. The web app uses the browser. Tile addresses are checked after filling their placeholders, before cached bytes or network -requests are used. In the native app, a document on the internet cannot direct tiles into the -local network. Explicitly selected self-hosted servers remain supported, including ordinary +requests are used. In the native app, a document on the internet cannot select a local +destination IP. Explicitly selected self-hosted servers remain supported, including ordinary hostnames resolved by local DNS, with local fallback when the same hostname also returns public addresses. Native fetchers filter the resolved IPs before connecting, -using the supplying document's actual connection reach. Native requests always refuse +using the supplying document's actual connection reach. Native requests refuse link-local and unspecified addresses, including the well-known NAT64 translation of IPv4 addresses. Document redirects also change the base for relative references. +IPv4 decoding covers mapped addresses and the well-known `64:ff9b::/96` prefix. It does not +inspect destinations behind 6to4 tunnels or custom translation gateways, including the +[RFC 8215](https://www.rfc-editor.org/rfc/rfc8215#section-5) `64:ff9b:1::/48` prefix, whose +translation is network-specific. Such routes can hide a private or link-local IPv4 destination +behind a public-looking IPv6 address; restrictions on those routes require network firewall policy. Native redirects are checked at each hop. In the web app, use the final service address: tile, style, TileJSON and sprite redirects, including HTTP-to-HTTPS upgrades and CDN hops, are refused because the browser hides their destinations from the app. The same browser From 3b92d83a639c5cd3cbad9b0314f32aa9e9f376d7 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sun, 8 Nov 2026 10:33:47 -0600 Subject: [PATCH 05/10] fix(tiles): constrain expanded accounts and refresh refusal diagnostics --- crates/data/src/diagnostics.rs | 5 +- crates/render/src/tile_mesh.rs | 11 +++- crates/render/src/tiles.rs | 5 ++ crates/tiles/src/address.rs | 20 ++++++- crates/tiles/src/cache.rs | 76 +++++++++++++++++++++++--- crates/tiles/src/draw.rs | 25 ++++++--- crates/tiles/src/picture.rs | 99 +++++++++++++++++++++++++++++----- crates/tiles/src/source.rs | 75 ++++++++++++++++++++++++++ crates/tiles/src/template.rs | 26 +++++++-- docs/user-guide/data.md | 2 +- docs/user-guide/maps.md | 8 +-- 11 files changed, 316 insertions(+), 36 deletions(-) diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index 584656449d..fd6994deca 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -350,7 +350,10 @@ impl Capture { /// Keeps repeated policy refusals once while their sanitized facts remain in this bounded /// history. Clearing or evicting the record permits it to be reported again. - pub fn network_failure_once(&self, source: &str, message: &str, details: &str, network: Network) { + pub fn network_failure_once(&self, source: &str, message: &str, details: &str, mut network: Network) { + // Rejected tile paths on one server are one policy refusal, not distinct attempts. + let clean = clean_address(&network.url); + network.url = url::Url::parse(&clean).ok().filter(|u| u.origin().is_tuple()).map_or(clean, |u| u.origin().ascii_serialization()); self.keep_network_failure(source, message, details, network, true); } diff --git a/crates/render/src/tile_mesh.rs b/crates/render/src/tile_mesh.rs index 36de91397a..57c28a4a8d 100644 --- a/crates/render/src/tile_mesh.rs +++ b/crates/render/src/tile_mesh.rs @@ -710,7 +710,16 @@ mod tests { let style = Arc::get_mut(&mut private).unwrap(); style.from = Some(rustgis_tiles::address::Origin::named("https://mesh-style.example/style.json")); style.sprites = vec![("default".into(), "http://127.0.0.1/mesh-private-only".into())]; - assert!(!meshes(&private, "s", &[], 3, None).1, "a refused sprite must never become a pending fetch"); + let id = TileId { z: 3, x: 0, y: 0 }; + let tiles = [DataTile { payload: Arc::new(Payload::Vector(tile())), at: id.extent(), z: id.z, clip: id.extent() }]; + let (made, waiting) = meshes(&private, "s", &tiles, 3, None); + assert!(!waiting, "a refused sprite must never become a pending fetch"); + assert!(made.first().and_then(Option::as_ref).is_some_and(|m| !m.waiting)); + assert_eq!( + crate::tiles::tests::PRIVATE_SPRITE_REQUESTS.load(std::sync::atomic::Ordering::Relaxed), + 0, + "neither sprite JSON nor PNG may reach the fetcher" + ); } #[test] diff --git a/crates/render/src/tiles.rs b/crates/render/src/tiles.rs index 2c0487b1a2..2556d0d8af 100644 --- a/crates/render/src/tiles.rs +++ b/crates/render/src/tiles.rs @@ -868,10 +868,15 @@ pub(crate) mod tests { static HOLD: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); static HELD: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + pub(crate) static PRIVATE_SPRITE_REQUESTS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + /// Serves tiles and documents for `https://tiles.test/…` without a network. Under `split/`, /// zoom 11 is red, zoom 13 blue, and zoom 12 never arrives. pub(crate) fn serve() { rustgis_tiles::cache::set_fetcher(|url, done| { + if url.starts_with("http://127.0.0.1/mesh-private-only") { + PRIVATE_SPRITE_REQUESTS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } let color = |r, g, b| { let mut pm = tiny_skia::Pixmap::new(256, 256).unwrap(); pm.fill(tiny_skia::Color::from_rgba8(r, g, b, 255)); diff --git a/crates/tiles/src/address.rs b/crates/tiles/src/address.rs index 5d8c2bc24e..f9159bd20f 100644 --- a/crates/tiles/src/address.rs +++ b/crates/tiles/src/address.rs @@ -62,6 +62,21 @@ impl Origin { /// Checks a descendant without replacing the document's observed reach with host spelling. pub(crate) fn check(&self, url: &str) -> Result<(), String> { + // Expansion can introduce a selector that was hidden from document joining. + // Only the supplying document's own selection may survive that boundary. + let (plain, _) = crate::s3::split_anonymous(url); + if let Some(child) = rustgis_data::login_address::parse(plain).map_err(str::to_string)? { + let parent = rustgis_data::login_address::parse(&self.url).map_err(str::to_string)?; + if parent.is_none_or(|p| p.binding != child.binding || p.origin != child.origin) { + return Err("a document cannot select another service login".into()); + } + } + if let Some(child) = rustgis_data::s3_address::parse(plain).map_err(str::to_string)? { + let parent = rustgis_data::s3_address::parse(&self.url).map_err(str::to_string)?; + if parent.is_none_or(|p| p.binding != child.binding || p.endpoint != child.endpoint) { + return Err("a document cannot select another S3 account".into()); + } + } check_reach(url, self.reach) } } @@ -192,6 +207,9 @@ pub(crate) fn inherit(from: &str, to: &str) -> Result { /// Checks a transport's normalized destination under its inherited runtime ceiling, unwrapping /// account markers without widening it. Call before sending; Forbidden is always refused. pub fn check_reach(url: &str, ceiling: Reach) -> Result<(), String> { + if ceiling == Reach::Forbidden { + return Err("a forbidden document cannot supply request destinations".into()); + } crate::logins::check(url).map_err(str::to_string)?; if let Some(bound) = rustgis_data::login_address::parse(url).map_err(str::to_string)? { return check_reach(bound.url, ceiling); @@ -204,7 +222,7 @@ pub fn check_reach(url: &str, ceiling: Reach) -> Result<(), String> { // An `s3://` address is judged by the web address it is fetched from. // An address read anonymously is judged as the address it marks. if let (inner, true) = crate::s3::split_anonymous(url) { - if rustgis_data::s3_address::is_wrapper(inner) { + if rustgis_data::s3_address::is_wrapper(inner) || crate::s3::split_anonymous(inner).1 { return Err("not a saved S3 address".into()); } return if crate::s3::is_s3(inner) { check_reach(&crate::s3::https_url(url)?, ceiling) } else { check_reach(inner, ceiling) }; diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index e9de454cc8..f2aa94ba7c 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -601,7 +601,9 @@ fn refused_request(url: &str, message: String) -> (State, Option ureq::Agent { checked_agent(config_builder(timeout).build(), ureq::unversioned::resolver::DefaultResolver::default(), true) @@ -2163,6 +2189,11 @@ mod native { /// Asks for a caller-selected archive's headers, following up to 5 redirects, each checked /// as [`get`] checks them. Document-derived reads use the GET queue's inherited ceiling. pub(super) fn head_with(agent: &ureq::Agent, url: &str) -> Result { + head_with_public(agent, shared_public_agent(), url) + } + + /// Uses a separate internet-only pool after a public hop; fixtures supply both pools. + fn head_with_public(agent: &ureq::Agent, public: &ureq::Agent, url: &str) -> Result { let mut diagnostics = super::RequestDiagnostics::new(url, "HEAD"); let mut carried_login = false; let answer = (|| -> Result { @@ -2181,7 +2212,7 @@ mod native { let bound = crate::logins::inherit(url, &at); let login = if same_origin { crate::logins::apply(&bound)? } else { None }; carried_login |= login.is_some(); - let sending = if ceiling == Reach::Public { shared_public_agent() } else { agent }; + let sending = if ceiling == Reach::Public { public } else { agent }; let mut req = sending.head(login.as_ref().map_or(at.as_str(), |a| a.url.as_str())); if let Some((name, value)) = login.as_ref().and_then(|a| a.header.as_ref()) { req = req.header(name.as_str(), value.as_str()); @@ -2439,6 +2470,18 @@ mod native { initial: Reach, detail: bool, omit: &'static [u16], + ) -> Result { + fetch_answer_with_public(agent, shared_public_agent(), url, initial, detail, omit) + } + + /// The request path with both pools explicit, so redirect DNS is tested without live DNS. + fn fetch_answer_with_public( + agent: &ureq::Agent, + public: &ureq::Agent, + url: &str, + initial: Reach, + detail: bool, + omit: &'static [u16], ) -> Result { use std::io::Read; let mut carried_login = false; @@ -2479,7 +2522,7 @@ mod native { let bound = crate::logins::inherit(url, &at); let login = if !prepared.signed && same_origin { crate::logins::apply(&bound)? } else { None }; carried_login |= login.is_some(); - let sending = if ceiling == Reach::Public && initial != Reach::Public { shared_public_agent() } else { agent }; + let sending = if ceiling == Reach::Public && initial != Reach::Public { public } else { agent }; let mut req = sending.get(login.as_ref().map_or(at.as_str(), |a| a.url.as_str())); if let Some((name, value)) = login.as_ref().and_then(|a| a.header.as_ref()) { req = req.header(name.as_str(), value.as_str()); @@ -2699,6 +2742,22 @@ mod native { #[cfg(test)] mod tests { + #[test] + fn policy_refusals_group_paths_without_evicting_other_diagnostics() { + let history = rustgis_data::diagnostics::History::default(); + let _capture = history.enter(); + for i in 0..20 { + let url = format!("http://127.0.0.1/{i}/0/0"); + note_refusal(&url, &format!("{url}: a document on the web can't point into the local network")); + } + assert_eq!(history.records().len(), 1, "expanded paths must share a policy diagnostic"); + note_refusal("http://127.0.0.1/other", "different refusal"); + note_refusal("http://127.0.0.2/0", "different refusal"); + assert_eq!(history.records().len(), 3); + history.clear(); + note_refusal("http://127.0.0.1/0", "different refusal"); + assert_eq!(history.records().len(), 1); + } /// Runs a fixture that needs stable process-wide cache entries without unrelated login /// revocation clearing them. The exact named child must execute its existing assertions. #[cfg(not(target_arch = "wasm32"))] @@ -4096,11 +4155,14 @@ mod tests { fn replacement_transport_destinations_keep_normalized_origins_and_ceiling() { let _turn = turn(); let sent = Arc::new(Mutex::new(Vec::new())); + let attempts = Arc::new(Mutex::new(Vec::new())); + let attempted = attempts.clone(); let observed = sent.clone(); set_fetcher_with_origin(move |url, ceiling, done, _| { + attempted.lock().unwrap().push((url.clone(), ceiling)); let normalized = if url == "https://public.example\\@localhost/style.json" { "https://public.example/@localhost/style.json" - } else if url == "http://%31%32%37.0.0.1/tile.png" { + } else if url == "http://local%68ost/tile.png" { "http://127.0.0.1/tile.png" } else { &url @@ -4119,10 +4181,12 @@ mod tests { let origin = origin.unwrap(); assert_eq!(origin.url, "https://public.example/@localhost/style.json"); assert_eq!(origin.reach, crate::address::Reach::Public); - let child = "http://%31%32%37.0.0.1/tile.png"; + let child = "http://local%68ost/tile.png"; get_tile_from(child, Want::Image, Some(&origin)); wait(std::time::Duration::from_secs(2)); assert!(matches!(get_tile_from(child, Want::Image, Some(&origin)), State::Failed(_))); + assert_eq!(attempts.lock().unwrap().last().unwrap(), &(child.to_string(), crate::address::Reach::Public)); + assert_eq!(attempts.lock().unwrap().len(), 2, "the child reaches the normalization boundary"); assert_eq!(sent.lock().unwrap().len(), 1, "normalized local child never reaches send"); assert!(origin.check("http://127.0.0.1/tile.png").is_err()); } @@ -4460,7 +4524,7 @@ mod tests { /// can reach only the listener this test just created. #[cfg(not(target_arch = "wasm32"))] #[derive(Debug)] - struct LocalAnswer(std::net::SocketAddr); + pub(super) struct LocalAnswer(pub(super) std::net::SocketAddr); #[cfg(not(target_arch = "wasm32"))] impl ureq::unversioned::resolver::Resolver for LocalAnswer { diff --git a/crates/tiles/src/draw.rs b/crates/tiles/src/draw.rs index 1f3afca9e9..3b2ab56fa0 100644 --- a/crates/tiles/src/draw.rs +++ b/crates/tiles/src/draw.rs @@ -54,13 +54,15 @@ pub struct Drawn { /// Retains each distinct refusal alongside any earlier source or sprite warning. pub(crate) fn add_error(to: &mut Option, error: String) { - match to { - Some(previous) if !previous.contains(&error) => { - previous.push_str("; "); - previous.push_str(&error); + for message in error.split("; ") { + match to { + Some(previous) if !previous.split("; ").any(|kept| kept == message) => { + previous.push_str("; "); + previous.push_str(message); + } + None => *to = Some(message.to_string()), + _ => {} } - None => *to = Some(error), - _ => {} } } @@ -1258,6 +1260,17 @@ fn area(r: &[(f32, f32)]) -> f32 { #[cfg(test)] pub(crate) mod tests { + #[test] + fn refusal_counts_are_not_substring_duplicates() { + let mut error = Some("12 tiles left out: refused".to_string()); + super::add_error(&mut error, "2 tiles left out: refused".into()); + assert_eq!(error.as_deref(), Some("12 tiles left out: refused; 2 tiles left out: refused")); + let combined = error.unwrap(); + let mut error = Some("sprite left out".to_string()); + super::add_error(&mut error, combined.clone()); + super::add_error(&mut error, combined); + assert_eq!(error.as_deref(), Some("sprite left out; 12 tiles left out: refused; 2 tiles left out: refused")); + } use super::*; #[test] diff --git a/crates/tiles/src/picture.rs b/crates/tiles/src/picture.rs index 818d7c276d..461879d08f 100644 --- a/crates/tiles/src/picture.rs +++ b/crates/tiles/src/picture.rs @@ -68,8 +68,9 @@ struct Entry { /// Drawn while some of it was on its way, at this cache generation: drawn again once the /// cache has more. waiting: Option, - /// Destination refusals remain visible when the prepared picture is reused. - error: Option, + /// A failed constituent is reconsidered when the tile cache changes, without keeping + /// an export pending while its server is refusing it. + failed: Option, /// For a style with pattern fills, the ids of the sprites it was drawn from. sheets: Option>, } @@ -241,6 +242,14 @@ pub(crate) fn cover( let m_per_px = TileId::span(z) / f64::from(side); let key = |tile| Key { style: Arc::as_ptr(r) as usize, tile, step: step as i32, ppp: ppp.to_bits() }; let generation = cache::generation(); + // Pictures include padding and may stand in at another zoom. Their preparation errors + // do not describe this viewport: gather current diagnostics without repainting shapes. + let mut visible = Drawn::default(); + draw_layers(None, (0, 0), view, r, frame, Pass::Shapes, &mut visible); + *pending |= visible.pending; + if let Some(reason) = visible.error { + crate::draw::add_error(error, reason); + } let (sprites_pending, read) = sprites_read(r, zoom, ppp); // A view of more squares than a covering holds (a huge window) is drawn straight instead; // a reach of more keeps to the view. @@ -265,14 +274,12 @@ pub(crate) fn cover( Some(e) if !in_view || (e.waiting.is_none_or(|g| g == generation) + && e.failed.is_none_or(|g| g == generation) && sprites_serve(e.sheets.as_deref(), sprites_pending, read.as_deref())) => { e.used = tick; // (A sprite on its way may add to it: an export waits.) *pending |= in_view && (e.waiting.is_some() || sprites_pending); - if let Some(e) = e.error.as_ref().filter(|_| in_view) { - crate::draw::add_error(error, e.clone()); - } found.push((p, Some(e.picture.clone()))); } _ if in_view => { @@ -326,9 +333,6 @@ pub(crate) fn cover( let (Some(slot), Some(d)) = (found.get_mut(i), d) else { continue }; let DrawnPicture { image: pm, pending: waiting, error: refused } = d; *pending |= waiting; - if let Some(refused) = &refused { - crate::draw::add_error(error, refused.clone()); - } let bytes = pm.data().len(); let picture = Arc::new(Payload::Image(pm)); let entry = Entry { @@ -337,7 +341,7 @@ pub(crate) fn cover( bytes, used: tick, waiting: waiting.then_some(generation), - error: refused, + failed: refused.is_some().then_some(generation), sheets: read.clone(), }; s.bytes += bytes; @@ -357,9 +361,6 @@ pub(crate) fn cover( for &(k, at, z, clip) in covers.get(&i).into_iter().flatten() { if let Some(e) = s.entries.get_mut(&k) { e.used = tick; - if let Some(refused) = &e.error { - crate::draw::add_error(error, refused.clone()); - } stood_in.push(Item { payload: e.picture.clone(), at, z, clip }); } } @@ -522,6 +523,80 @@ fn draw_one(r: &Resolved, p: &Placed, side: u32, m_per_px: f64, ppp: f32, reach: #[cfg(test)] mod tests { + /// Padded preparation and later cache reuse must report the current viewport only. + #[test] + fn picture_refusals_follow_the_viewport_not_preparation_padding() { + let _turn = cache::turn(); + let bytes = crate::mvt::encode(&Tile::default()); + cache::set_fetcher(move |_, done| done(Ok(bytes.clone()))); + let mut style = crate::style::Style::parse(&serde_json::json!({"version":8, + "sources":{"s":{"type":"vector","tiles":["http://127.0.0.1/{z}/{x}/{y}","https://picture-view.example/{z}/{x}/{y}"]}}, + "layers":[{"id":"line","type":"line","source":"s","source-layer":"l","paint":{"line-width":20}}]})) + .unwrap(); + let from = crate::address::Origin::named("https://picture-view.example/style.json"); + if let Source::Vector { from: selected, .. } = &mut style.sources.first_mut().unwrap().1 { + *selected = Some(from.clone()); + } + let r = Arc::new(Resolved { + style, + from: Some(from), + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }); + let square = TileId { z: 3, x: 6, y: 4 }.extent(); + let f = Frame { left: square.min_x, top: square.max_y, m_per_px: template::HALF_WORLD / (256.0 * 2.2f64.exp2()), ppp: 1.0 }; + let view = Extent::new( + square.min_x + square.width() * 0.3, + square.min_y + square.height() * 0.3, + square.min_x + square.width() * 0.6, + square.min_y + square.height() * 0.6, + ); + for _ in 0..2 { + let mut error = None; + assert!(cover(&r, &f, &view, None, &mut false, &mut error).is_some()); + cache::wait(Duration::from_secs(2)); + assert!(error.is_none(), "only padding crosses a refused shard: {error:?}"); + } + } + + /// A failure retained in a picture must disappear after the same tile recovers. + #[test] + fn picture_failures_refresh_after_recovery() { + let _turn = cache::turn(); + let _alarm = cache::alarm_turn(); + let bytes = crate::mvt::encode(&Tile::default()); + cache::set_fetcher(|_, done| done(Err("http status: 503 (retry after 1 s)".into()))); + let style = crate::style::Style::parse(&serde_json::json!({"version":8, + "sources":{"s":{"type":"vector","tiles":["https://picture-retry.example/tile"],"maxzoom":0}}, + "layers":[{"id":"fill","type":"fill","source":"s","source-layer":"l"}]})) + .unwrap(); + let r = Arc::new(Resolved { + style, + from: None, + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }); + let f = frame(2.2, 1.0); + let view = f.extent(64, 64); + cover(&r, &f, &view, None, &mut false, &mut None); + cache::wait(Duration::from_secs(2)); + let mut error = None; + cover(&r, &f, &view, None, &mut false, &mut error); + assert!(error.is_some()); + cache::set_fetcher(move |_, done| done(Ok(bytes.clone()))); + std::thread::sleep(Duration::from_secs(1)); + cache::get_tile("https://picture-retry.example/tile", cache::Want::Vector); + cache::wait(Duration::from_secs(2)); + let mut error = None; + cover(&r, &f, &view, None, &mut false, &mut error); + assert!(error.is_none(), "recovered tile still has an old picture warning: {error:?}"); + } use super::*; use crate::draw::{draw, place}; use crate::mvt::{Feature, GeomKind, Layer, Tile}; diff --git a/crates/tiles/src/source.rs b/crates/tiles/src/source.rs index 409771044f..42be4cc715 100644 --- a/crates/tiles/src/source.rs +++ b/crates/tiles/src/source.rs @@ -677,6 +677,81 @@ pub fn plain_text(html: &str) -> String { #[cfg(test)] mod tests { + /// An expanded selector must not let a public style choose another saved account. + #[test] + fn expanded_selectors_cannot_choose_a_documents_account() { + let _turn = cache::turn(); + let endpoint = "https://expanded-account.example"; + let binding = crate::s3::new_binding().unwrap(); + let name = "expanded-account-fixture"; + crate::s3::connect_saved( + crate::s3::Connection { + name: name.into(), + endpoint: Some(endpoint.into()), + region: "us-east-1".into(), + buckets: vec!["bucket".into()], + credentials: None, + }, + &binding, + "expanded-account", + ) + .unwrap(); + let selected = rustgis_data::s3_address::bind(&binding, endpoint, &format!("{endpoint}/bucket/tile.pbf")).unwrap(); + assert!( + address::Origin { url: selected.clone(), reach: address::Reach::Public }.check(&selected).is_ok(), + "the document may retain its own selected account" + ); + let login = crate::logins::activate( + endpoint, + "expanded-service-fixture", + crate::logins::Method::Header("X-Fixture".into()), + "fixture", + false, + ) + .unwrap(); + let service = crate::logins::bind(&login, &format!("{endpoint}/tile.pbf")).unwrap(); + assert!(address::Origin { url: service.clone(), reach: address::Reach::Public }.check(&service).is_ok()); + let asked = Arc::new(Mutex::new(Vec::new())); + let seen = asked.clone(); + cache::set_fetcher(move |url, done| { + seen.lock().unwrap().push(url); + done(Ok(crate::mvt::encode(&crate::mvt::Tile::default()))); + }); + for selected in [selected, service] { + let mut style = Style::parse( + &json!({"version":8,"sources":{"s":{"type":"vector","tiles":[format!("{{switch:{selected}}}")]}},"layers":[]}), + ) + .unwrap(); + let base = address::Origin::named("https://expanded-style.example/style.json"); + let spec = spec(Kind::Style, &base.url); + resolve_sources(&mut style, &spec, &base).unwrap(); + let src = &style.sources.first().unwrap().1; + let frame = crate::draw::tests::world_frame(); + let mut error = None; + crate::draw::tiles_in(src, &frame.extent(64, 64), None, &frame, 0.0, &mut false, &mut error); + cache::wait(std::time::Duration::from_secs(2)); + assert!(error.is_some()); + } + crate::logins::deactivate(&login); + crate::s3::disconnect(name); + assert!(asked.lock().unwrap().is_empty(), "the style selected an account after expansion"); + } + + #[test] + fn forbidden_provenance_cannot_queue_sprites() { + let _turn = cache::turn(); + let asked = Arc::new(Mutex::new(Vec::new())); + let seen = asked.clone(); + cache::set_fetcher(move |url, done| { + seen.lock().unwrap().push(url); + done(Err("fixture".into())); + }); + let from = address::Origin::named("http://metadata/style.json"); + let state = crate::sprite::load_from(&[("default".into(), "http://127.0.0.1/forbidden-sprite".into())], false, Some(&from)); + cache::wait(std::time::Duration::from_secs(2)); + assert!(asked.lock().unwrap().is_empty(), "forbidden provenance reached transport"); + assert!(matches!(state, crate::sprite::Load::Failed)); + } use super::*; fn spec(kind: Kind, url: &str) -> Spec { diff --git a/crates/tiles/src/template.rs b/crates/tiles/src/template.rs index 56a0d7c1b4..f4b98d99ba 100644 --- a/crates/tiles/src/template.rs +++ b/crates/tiles/src/template.rs @@ -201,6 +201,16 @@ pub fn expand(template: &str, t: TileId, tile_size: u32, tms: bool, subdomains: /// Resolves `href` against the document at `base` (for addresses inside a style or TileJSON). /// An invalid bound base stays refused by request validation rather than resolving anonymously. pub fn join(base: &str, href: &str) -> String { + let (plain_base, anonymous_base) = crate::s3::split_anonymous(base); + let (plain_href, anonymous_href) = crate::s3::split_anonymous(href); + if anonymous_base || anonymous_href { + if crate::s3::split_anonymous(plain_base).1 || crate::s3::split_anonymous(plain_href).1 { + return if anonymous_base { base } else { href }.to_string(); + } + // An anonymous child narrows authority, including when its parent chose an account. + let base = crate::logins::inner(rustgis_data::s3_address::inner(plain_base)); + return crate::s3::anonymous(&join(base, plain_href)); + } // Only the supplying document's base carries authority. A link may name a resource, // but neither a service-login nor an S3 selector in it can choose an account. let href = match rustgis_data::login_address::parse(href) { @@ -226,10 +236,6 @@ pub fn join(base: &str, href: &str) -> String { Err(_) => return base.to_string(), Ok(None) => {} } - let (plain, anonymous) = crate::s3::split_anonymous(base); - if anonymous { - return crate::s3::anonymous(&join(plain, href)); - } match rustgis_data::login_address::parse(base) { Ok(Some(bound)) => return crate::logins::inherit(base, &join(bound.url, href)), Err(_) => return base.to_string(), @@ -239,7 +245,7 @@ pub fn join(base: &str, href: &str) -> String { return href.to_string(); } let Some((scheme, rest)) = base.split_once("://") else { return href.to_string() }; - let host_end = rest.find('/').unwrap_or(rest.len()); + let host_end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); let host = rest.get(..host_end).unwrap_or(rest); if let Some(path) = href.strip_prefix("//") { return format!("{scheme}://{path}"); @@ -397,5 +403,15 @@ mod tests { assert_eq!(join("https://a.com/styles/x", "/planet"), "https://a.com/planet"); assert_eq!(join("https://a.com/x", "//b.com/y"), "https://b.com/y"); assert_eq!(join("https://a.com/x", "https://c.com/z"), "https://c.com/z"); + assert_eq!(join("https://tiles.example?token=x", "/tile/0"), "https://tiles.example/tile/0"); + assert_eq!(join("https://tiles.example#view", "tile/0"), "https://tiles.example/tile/0"); + } + + #[test] + fn direct_anonymous_sources_keep_one_marker() { + let anonymous = crate::s3::anonymous("https://tiles.example/{z}/{x}/{y}"); + assert_eq!(join(&anonymous, &anonymous), anonymous); + assert_eq!(join("https://tiles.example/style.json", &anonymous), anonymous); + assert_eq!(join(&anonymous, "tile/0"), crate::s3::anonymous("https://tiles.example/{z}/{x}/tile/0")); } } diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 001d84f266..f41ff3bed8 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -634,7 +634,7 @@ On desktop and the command line, web tiles, service documents and file downloads app's IP checks. There is no proxy override for these requests. If your network requires a proxy, download a file with a trusted external tool and open the local copy in RustGIS. Live tile and service access requires direct connectivity. The web app uses the browser's -network settings. Browser GET requests refuse redirects, including HTTP-to-HTTPS upgrades +network settings. Browser GET and HEAD requests refuse redirects, including HTTP-to-HTTPS upgrades and CDN hops, because the app cannot inspect their destinations. This includes **From URL…** file downloads, fonts, service documents, tiles and sprites, and PMTiles/COG archive range reads. Supply the final address; for a redirecting file download, download it with a trusted diff --git a/docs/user-guide/maps.md b/docs/user-guide/maps.md index 32897c061d..04016286cf 100644 --- a/docs/user-guide/maps.md +++ b/docs/user-guide/maps.md @@ -285,8 +285,9 @@ destination IP. Explicitly selected self-hosted servers remain supported, includ hostnames resolved by local DNS, with local fallback when the same hostname also returns public addresses. Native fetchers filter the resolved IPs before connecting, using the supplying document's actual connection reach. Native requests refuse -link-local and unspecified addresses, including the well-known NAT64 translation of IPv4 -addresses. Document redirects also change the base for relative references. +link-local and unspecified addresses, including those represented by IPv4-mapped IPv6 +addresses or the well-known NAT64 prefix. Public IPv4 destinations under that prefix can pass. +Document redirects also change the base for relative references. IPv4 decoding covers mapped addresses and the well-known `64:ff9b::/96` prefix. It does not inspect destinations behind 6to4 tunnels or custom translation gateways, including the [RFC 8215](https://www.rfc-editor.org/rfc/rfc8215#section-5) `64:ff9b:1::/48` prefix, whose @@ -295,7 +296,8 @@ behind a public-looking IPv6 address; restrictions on those routes require netwo Native redirects are checked at each hop. In the web app, use the final service address: tile, style, TileJSON and sprite redirects, including HTTP-to-HTTPS upgrades and CDN hops, are refused because the browser hides their destinations from the app. The same browser -transport refuses redirects for downloads, fonts and archive range reads; see +transport refuses redirects for downloads, fonts, archive range reads and HEAD requests for +file metadata; see [network downloads](data.md#network-downloads). The web app checks the URL's spelling and normalized address, but cannot inspect resolved IPs. A public-looking hostname can reach a private or link-local server if the browser permits it; From a5510b057cc4e941767c0082d2748d030ebebf5d Mon Sep 17 00:00:00 2001 From: mavrick Date: Sun, 8 Nov 2026 11:30:23 -0600 Subject: [PATCH 06/10] fix(tiles): refresh recovered pictures and refuse malformed links --- crates/tiles/src/draw.rs | 22 +++++-- crates/tiles/src/picture.rs | 118 +++++++++++++++++++++++------------ crates/tiles/src/template.rs | 13 +++- 3 files changed, 105 insertions(+), 48 deletions(-) diff --git a/crates/tiles/src/draw.rs b/crates/tiles/src/draw.rs index 3b2ab56fa0..b0ec6ce1dd 100644 --- a/crates/tiles/src/draw.rs +++ b/crates/tiles/src/draw.rs @@ -80,6 +80,8 @@ pub(crate) struct Item { pub(crate) struct TileSelection { pub(crate) items: Vec, pub(crate) mesh_gaps: bool, + /// A failed or partial read may supply more pixels after the tile cache changes. + pub(crate) retry: bool, } /// What of a resolved style was left out, as a drawing's error (the style still draws): what @@ -172,6 +174,7 @@ pub(crate) enum Pass { /// Draws the layers of a resolved style for `pass` into `target` (a `w × h` pixmap at `frame`) /// from the tiles over `view`, collecting labels into `out`. +/// Returns whether a failed or partial tile read could add pixels when the cache changes. pub(crate) fn draw_layers( mut target: Option<&mut Pixmap>, (w, h): (u32, u32), @@ -180,7 +183,8 @@ pub(crate) fn draw_layers( frame: &Frame, pass: Pass, out: &mut Drawn, -) { +) -> bool { + let mut retry = false; let zoom = template::style_zoom(frame.m_per_px * f64::from(frame.ppp)); let mut items: HashMap> = HashMap::new(); let mut boxes: Boxes = HashMap::new(); @@ -229,15 +233,17 @@ pub(crate) fn draw_layers( // Hillshade reads the heights a pixel past the view's edges: those tiles too. let pad = 2.0 * frame.m_per_px; let reach = Extent::new(view.min_x - pad, view.min_y - pad, view.max_x + pad, view.max_y + pad); - let list = tiles_for( + let selected = tiles_in( src, if matches!(src, Source::RasterDem { .. }) { &reach } else { view }, + None, frame, zoom, &mut out.pending, &mut out.error, ); - items.insert(src_id.to_string(), list); + retry |= selected.retry; + items.insert(src_id.to_string(), selected.items); } let Some(list) = items.get(src_id) else { continue }; match layer.kind { @@ -295,6 +301,7 @@ pub(crate) fn draw_layers( } // The cap keeps the labels placed first, whichever layers they came from. crate::symbols::keep_first(&mut out.labels, crate::symbols::MAX_LABELS); + retry } /// One raster tile placed on the map, for a renderer that draws each tile itself (the GPU map @@ -583,10 +590,10 @@ pub(crate) fn tiles_in( Source::Vector { tiles, from, min_zoom, max_zoom, .. } => { (tiles, from, *min_zoom, *max_zoom, Want::Vector, (zoom.floor() as u8).min(*max_zoom)) } - Source::Other => return TileSelection { items: Vec::new(), mesh_gaps: true }, + Source::Other => return TileSelection { items: Vec::new(), mesh_gaps: true, retry: false }, }; if tiles.is_empty() || z < min_z || zoom + 0.5 < f32::from(min_z) { - return TileSelection { items: Vec::new(), mesh_gaps: true }; + return TileSelection { items: Vec::new(), mesh_gaps: true, retry: false }; } // A mosaic over half the world wide (the globe's, a world projection's) takes the zoom whose // tiles all fit, as its far edges are in view; a flat view keeps those nearest its centre. @@ -618,6 +625,7 @@ pub(crate) fn tiles_in( let mut mesh_gaps = ((view.width() / span).ceil() + 1.0) * ((view.height() / span).ceil() + 1.0) > template::MAX_TILES as f64; let mut omitted = 0; let mut first_error = None; + let mut retry = false; let mut partial = 0; let mut first_partial = None; // A missing tile is stood in for by its nearest ancestor in the cache and, for raster tiles, @@ -665,6 +673,7 @@ pub(crate) fn tiles_in( let state = if visible { let (state, note) = cache::tile_noted(&u, want, from.as_ref()); if let Some(note) = note { + retry = true; partial += 1; first_partial.get_or_insert(note); } @@ -684,6 +693,7 @@ pub(crate) fn tiles_in( exact.push(Item { payload, at: area, z: p.id.z, clip: area }); } State::Failed(e) => { + retry |= visible; mesh_gaps |= visible; if !cache::routine_tile_failure(&e) { omitted += 1; @@ -707,7 +717,7 @@ pub(crate) fn tiles_in( if let Some(reason) = first_partial { add_error(error, format!("{partial} tiles are partial (first: {reason})")); } - TileSelection { items: stand_ins, mesh_gaps } + TileSelection { items: stand_ins, mesh_gaps, retry } } /// The nearest ancestor of a missing tile already in the cache, clipped to the missing tile. diff --git a/crates/tiles/src/picture.rs b/crates/tiles/src/picture.rs index 461879d08f..8476af2972 100644 --- a/crates/tiles/src/picture.rs +++ b/crates/tiles/src/picture.rs @@ -68,8 +68,8 @@ struct Entry { /// Drawn while some of it was on its way, at this cache generation: drawn again once the /// cache has more. waiting: Option, - /// A failed constituent is reconsidered when the tile cache changes, without keeping - /// an export pending while its server is refusing it. + /// A failed or partial constituent read is reconsidered when the tile cache changes, + /// without keeping an export pending while its server is refusing it. failed: Option, /// For a style with pattern fills, the ids of the sprites it was drawn from. sheets: Option>, @@ -331,7 +331,7 @@ pub(crate) fn cover( let mut added = false; for (&i, d) in todo.iter().zip(drawn) { let (Some(slot), Some(d)) = (found.get_mut(i), d) else { continue }; - let DrawnPicture { image: pm, pending: waiting, error: refused } = d; + let DrawnPicture { image: pm, pending: waiting, retry } = d; *pending |= waiting; let bytes = pm.data().len(); let picture = Arc::new(Payload::Image(pm)); @@ -341,7 +341,7 @@ pub(crate) fn cover( bytes, used: tick, waiting: waiting.then_some(generation), - failed: refused.is_some().then_some(generation), + failed: retry.then_some(generation), sheets: read.clone(), }; s.bytes += bytes; @@ -447,11 +447,11 @@ fn evict(s: &mut Store) { } } -/// One prepared picture and the pending/refused reads that produced it. +/// One prepared picture and the pending or incomplete reads that produced it. struct DrawnPicture { image: Pixmap, pending: bool, - error: Option, + retry: bool, } /// Draws the pictures of `squares`: each a `side`-pixel pixmap and whether some of it is still @@ -517,8 +517,8 @@ fn draw_one(r: &Resolved, p: &Placed, side: u32, m_per_px: f64, ppp: f32, reach: let pad = reach * f64::from(ppp) * m_per_px; let view = Extent::new(e.min_x - pad, e.min_y - pad, e.max_x + pad, e.max_y + pad); let mut d = Drawn::default(); - draw_layers(Some(&mut pm), (side, side), &view, r, &frame, Pass::Shapes, &mut d); - Some(DrawnPicture { image: pm, pending: d.pending, error: d.error }) + let retry = draw_layers(Some(&mut pm), (side, side), &view, r, &frame, Pass::Shapes, &mut d); + Some(DrawnPicture { image: pm, pending: d.pending, retry }) } #[cfg(test)] @@ -554,48 +554,86 @@ mod tests { square.min_x + square.width() * 0.6, square.min_y + square.height() * 0.6, ); + let mut kept = Vec::new(); for _ in 0..2 { let mut error = None; - assert!(cover(&r, &f, &view, None, &mut false, &mut error).is_some()); + kept = cover(&r, &f, &view, None, &mut false, &mut error).unwrap(); cache::wait(Duration::from_secs(2)); assert!(error.is_none(), "only padding crosses a refused shard: {error:?}"); } + // A deterministic destination refusal does not make these pixels retryable. + cache::get_tile("https://picture-view.example/unrelated", cache::Want::Vector); + cache::wait(Duration::from_secs(2)); + let mut error = None; + let again = cover(&r, &f, &view, None, &mut false, &mut error).unwrap(); + assert!(error.is_none()); + assert_eq!(kept.len(), again.len()); + assert!(kept.iter().zip(&again).all(|(old, new)| Arc::ptr_eq(&old.payload, &new.payload))); } - /// A failure retained in a picture must disappear after the same tile recovers. + /// Routine missing tiles and reported failures both replace their pixels after recovery. #[test] fn picture_failures_refresh_after_recovery() { let _turn = cache::turn(); - let _alarm = cache::alarm_turn(); - let bytes = crate::mvt::encode(&Tile::default()); - cache::set_fetcher(|_, done| done(Err("http status: 503 (retry after 1 s)".into()))); - let style = crate::style::Style::parse(&serde_json::json!({"version":8, - "sources":{"s":{"type":"vector","tiles":["https://picture-retry.example/tile"],"maxzoom":0}}, - "layers":[{"id":"fill","type":"fill","source":"s","source-layer":"l"}]})) - .unwrap(); - let r = Arc::new(Resolved { - style, - from: None, - sprites: Vec::new(), - attribution: Vec::new(), - left_out: Vec::new(), - made: Mutex::new(Instant::now()), - waiting: false, - }); - let f = frame(2.2, 1.0); - let view = f.extent(64, 64); - cover(&r, &f, &view, None, &mut false, &mut None); - cache::wait(Duration::from_secs(2)); - let mut error = None; - cover(&r, &f, &view, None, &mut false, &mut error); - assert!(error.is_some()); - cache::set_fetcher(move |_, done| done(Ok(bytes.clone()))); - std::thread::sleep(Duration::from_secs(1)); - cache::get_tile("https://picture-retry.example/tile", cache::Want::Vector); - cache::wait(Duration::from_secs(2)); - let mut error = None; - cover(&r, &f, &view, None, &mut false, &mut error); - assert!(error.is_none(), "recovered tile still has an old picture warning: {error:?}"); + let tile = Tile { + layers: vec![Layer { + name: "l".into(), + extent: 4096, + features: vec![Feature { + id: None, + kind: GeomKind::Polygon, + props: Vec::new(), + geometry: vec![vec![[0.0, 0.0], [4096.0, 0.0], [4096.0, 4096.0], [0.0, 4096.0], [0.0, 0.0]]], + }], + }], + }; + let mut stale = Vec::new(); + for status in [503, 404, 410] { + cache::set_fetcher(move |_, done| done(Err(format!("http status: {status}")))); + let url = format!("https://picture-retry.example/{status}"); + let style = crate::style::Style::parse(&serde_json::json!({"version":8, + "sources":{"s":{"type":"vector","tiles":[url],"maxzoom":0}}, + "layers":[{"id":"fill","type":"fill","source":"s","source-layer":"l","paint":{"fill-color":"#ff0000"}}]})) + .unwrap(); + let r = Arc::new(Resolved { + style, + from: None, + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }); + let f = frame(2.2, 1.0); + let view = f.extent(64, 64); + cover(&r, &f, &view, None, &mut false, &mut None); + cache::wait(Duration::from_secs(2)); + let mut error = None; + let mut pending = false; + let failed = cover(&r, &f, &view, None, &mut pending, &mut error).unwrap(); + assert!(!pending, "an unavailable tile must not keep export pending"); + assert_eq!(error.is_some(), status == 503, "routine status {status}: {error:?}"); + assert!(!failed.is_empty()); + assert!(failed.iter().all(|it| matches!(it.payload.as_ref(), Payload::Image(pm) if pm.data().iter().all(|b| *b == 0)))); + let bytes = crate::mvt::encode(&tile); + cache::set_fetcher(move |_, done| done(Ok(bytes.clone()))); + cache::forget(&url); + cache::get_tile(&url, cache::Want::Vector); + cache::wait(Duration::from_secs(2)); + assert!(matches!(cache::peek(&url).as_deref(), Some(Payload::Vector(t)) if !t.layers[0].features.is_empty())); + let mut error = None; + let recovered = cover(&r, &f, &view, None, &mut false, &mut error).unwrap(); + assert!(error.is_none(), "recovered tile still has an old picture warning: {error:?}"); + let replaced = failed.len() == recovered.len() + && failed.iter().zip(&recovered).all(|(old, new)| old.at == new.at && !Arc::ptr_eq(&old.payload, &new.payload)); + let painted = recovered + .iter() + .any(|it| matches!(it.payload.as_ref(), Payload::Image(pm) if pm.data().as_chunks::<4>().0.iter().any(|px| px[3] > 0))); + if !replaced || !painted { + stale.push(status); + } + } + assert!(stale.is_empty(), "recovered geometry left old blank pictures cached for HTTP {stale:?}"); } use super::*; use crate::draw::{draw, place}; diff --git a/crates/tiles/src/template.rs b/crates/tiles/src/template.rs index f4b98d99ba..bc0897a9e7 100644 --- a/crates/tiles/src/template.rs +++ b/crates/tiles/src/template.rs @@ -204,8 +204,11 @@ pub fn join(base: &str, href: &str) -> String { let (plain_base, anonymous_base) = crate::s3::split_anonymous(base); let (plain_href, anonymous_href) = crate::s3::split_anonymous(href); if anonymous_base || anonymous_href { - if crate::s3::split_anonymous(plain_base).1 || crate::s3::split_anonymous(plain_href).1 { - return if anonymous_base { base } else { href }.to_string(); + if crate::s3::split_anonymous(plain_base).1 { + return base.to_string(); + } + if crate::s3::split_anonymous(plain_href).1 { + return href.to_string(); } // An anonymous child narrows authority, including when its parent chose an account. let base = crate::logins::inner(rustgis_data::s3_address::inner(plain_base)); @@ -413,5 +416,11 @@ mod tests { assert_eq!(join(&anonymous, &anonymous), anonymous); assert_eq!(join("https://tiles.example/style.json", &anonymous), anonymous); assert_eq!(join(&anonymous, "tile/0"), crate::s3::anonymous("https://tiles.example/{z}/{x}/tile/0")); + let malformed_child = crate::s3::anonymous(&anonymous); + let joined = join(&crate::s3::anonymous("https://tiles.example/style.json"), &malformed_child); + assert!(crate::address::check(&joined, None).is_err(), "a malformed child was replaced with {joined}"); + assert_eq!(joined, malformed_child); + let malformed_base = crate::s3::anonymous(&crate::s3::anonymous("https://tiles.example/style.json")); + assert!(crate::address::check(&join(&malformed_base, "tile/0"), None).is_err()); } } From d3a1cf5ca66f9da109850bf6a85cab5de9f49df3 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sun, 8 Nov 2026 12:36:50 -0600 Subject: [PATCH 07/10] fix(tiles): group wrapped destination refusals by server --- crates/tiles/src/cache.rs | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index f2aa94ba7c..f7a0ad3dcb 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -599,11 +599,21 @@ fn refused_request(url: &str, message: String) -> (State, Option Some(at), + _ if crate::s3::is_s3(plain) => crate::s3::https_url(url).ok(), + _ => None, + }; + let effective = destination.as_deref().unwrap_or(plain); + let mut diagnostics = RequestDiagnostics::new(effective, "GET"); diagnostics.once = true; // Keep the reason independent of expanded tile paths; the history groups by server. - let reason = message.strip_prefix(url).and_then(|s| s.strip_prefix(": ")).unwrap_or(message); - diagnostics.failed(&Failure::new(url, reason)); + let reason = + message.strip_prefix(url).or_else(|| message.strip_prefix(effective)).and_then(|s| s.strip_prefix(": ")).unwrap_or(message); + diagnostics.failed(&Failure::new(effective, reason)); } /// What the cache holds for `url`, without fetching it. @@ -2744,13 +2754,24 @@ mod native { mod tests { #[test] fn policy_refusals_group_paths_without_evicting_other_diagnostics() { + let _turn = turn(); let history = rustgis_data::diagnostics::History::default(); let _capture = history.enter(); for i in 0..20 { let url = format!("http://127.0.0.1/{i}/0/0"); - note_refusal(&url, &format!("{url}: a document on the web can't point into the local network")); + for asked in [ + url.clone(), + crate::s3::anonymous(&url), + rustgis_data::s3_address::bind("0123456789abcdef", "http://127.0.0.1", &url).unwrap(), + rustgis_data::s3_address::bind("0123456789abcdef", "http://127.0.0.1", &format!("s3://fixture-bucket/{i}/0/0")).unwrap(), + ] { + let reason = crate::address::check_reach(&asked, crate::address::Reach::Public).unwrap_err(); + note_refusal(&asked, &reason); + } } assert_eq!(history.records().len(), 1, "expanded paths must share a policy diagnostic"); + assert_eq!(history.records()[0].network.as_ref().unwrap().url, "http://127.0.0.1"); + assert_eq!(history.records()[0].message, "a document on the web can't point into the local network"); note_refusal("http://127.0.0.1/other", "different refusal"); note_refusal("http://127.0.0.2/0", "different refusal"); assert_eq!(history.records().len(), 3); From 041760db3c90e2f44c24410502ce61eb58c39f60 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sun, 8 Nov 2026 18:42:41 -0600 Subject: [PATCH 08/10] fix(tiles): align picture diagnostics and document links --- crates/tiles/src/cache.rs | 26 ++++ crates/tiles/src/draw.rs | 43 ++++-- crates/tiles/src/picture.rs | 251 +++++++++++++++++++++++++++++------ crates/tiles/src/style.rs | 5 + crates/tiles/src/template.rs | 34 +++++ 5 files changed, 304 insertions(+), 55 deletions(-) diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 5f704edc86..9eacf6d727 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -3564,6 +3564,32 @@ mod tests { serve_bytes(answers.into_iter().map(String::into_bytes).collect()) } + /// Both responses on one owned keep-alive socket retain its actual local provenance. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn reused_native_connection_keeps_its_origin() { + use std::io::Write; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + for i in 0..2 { + assert!(read_request(&mut stream).starts_with("GET /"), "both requests must use the accepted socket"); + let connection = if i == 0 { "keep-alive" } else { "close" }; + write!(stream, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: {connection}\r\n\r\n{{}}").unwrap(); + } + }); + let agent = native::agent(); + for path in ["first", "second"] { + let url = format!("http://{address}/{path}"); + let answer = native::fetch_answer(&agent, &url, crate::address::Reach::Local, false, &[]).unwrap(); + assert_eq!(answer.origin.reach, crate::address::Reach::Local); + assert_eq!(answer.origin.url, url); + assert_eq!(answer.bytes, b"{}"); + } + server.join().unwrap(); + } + #[cfg(not(target_arch = "wasm32"))] fn serve_bytes(answers: Vec>) -> String { serve_recorded(answers).0 diff --git a/crates/tiles/src/draw.rs b/crates/tiles/src/draw.rs index 630a16b7c8..2e70556852 100644 --- a/crates/tiles/src/draw.rs +++ b/crates/tiles/src/draw.rs @@ -88,13 +88,23 @@ pub(crate) struct TileSelection { /// resolving it left out, and any sprites that couldn't be read, for the images shown at `frame`. /// A drawing takes its images at the frame's density; a mesh (`mesh`) its pattern fills at 1×. pub(crate) fn left_out(r: &Resolved, frame: &Frame, mesh: bool) -> Option { + left_out_at(r, frame, frame, mesh) +} + +/// Reports icons at their live frame and patterns and relief at their prepared shape frame. +pub(crate) fn left_out_at(r: &Resolved, frame: &Frame, shapes: &Frame, mesh: bool) -> Option { let zoom = template::style_zoom(frame.m_per_px * f64::from(frame.ppp)); - let shown = |draws: fn(&StyleLayer) -> bool| r.style.layers.iter().any(|l| l.visible_at(zoom) && draws(l)); + let shape_zoom = template::style_zoom(shapes.m_per_px * f64::from(shapes.ppp)); + let shown = |zoom, draws: fn(&StyleLayer) -> bool| r.style.layers.iter().any(|l| l.visible_at(zoom) && draws(l)); let hi_dpi = frame.ppp >= 1.5; - let unread = sprite_notes(shown(StyleLayer::draws_icons), shown(StyleLayer::draws_patterns), hi_dpi, hi_dpi && !mesh, |hi, what| { - sprite::unread_note(&r.sprites, r.style.sprites_listed, &r.load_sprites(hi), what) - }); - let relief = relief_note(&r.style.layers, zoom); + let unread = sprite_notes( + shown(zoom, StyleLayer::draws_icons), + shown(shape_zoom, StyleLayer::draws_patterns), + hi_dpi, + shapes.ppp >= 1.5 && !mesh, + |hi, what| sprite::unread_note(&r.sprites, r.style.sprites_listed, &r.load_sprites(hi), what), + ); + let relief = relief_note(&r.style.layers, shape_zoom); let all: Vec<&str> = r.left_out.iter().map(String::as_str).chain(unread.iter().map(String::as_str)).chain(relief.as_deref()).collect(); (!all.is_empty()).then(|| all.join("; ")) } @@ -154,9 +164,9 @@ pub fn draw(pm: &mut Pixmap, spec: &Spec, opacity: f32, frame: &Frame, labels: b } }; out.attribution = r.attribution.clone(); - out.error = left_out(&r, frame, false); out.pending |= r.waiting; if !(frame.m_per_px.is_finite() && frame.m_per_px > 0.0) || opacity <= 0.0 { + out.error = left_out(&r, frame, false); return out; } let (w, h) = (pm.width(), pm.height()); @@ -171,6 +181,7 @@ pub fn draw(pm: &mut Pixmap, spec: &Spec, opacity: f32, frame: &Frame, labels: b } return out; } + out.error = left_out(&r, frame, false); let mut layer_pm = if opacity < 0.999 { Pixmap::new(w, h) } else { None }; let target: &mut Pixmap = match layer_pm.as_mut() { Some(p) => p, @@ -233,7 +244,7 @@ pub(crate) fn draw_layers( Pass::Shapes => layer.kind != LayerKind::Symbol, Pass::Labels => layer.kind == LayerKind::Symbol, }; - if !wanted || !layer.visible_at(zoom) { + if !wanted || !layer.visible_at(zoom) || !drawable(layer) { continue; } if layer.kind == LayerKind::Background { @@ -295,9 +306,6 @@ pub(crate) fn draw_layers( } LayerKind::Fill | LayerKind::FillExtrusion | LayerKind::Line | LayerKind::Circle => { let Some(t) = target.as_deref_mut() else { continue }; - if !drawable(layer) { - continue; - } // A pattern fill without its sprite isn't drawn: its fallback colour is black. let sheet = if pattern(layer).is_some() { match sprite(&mut out.pending) { @@ -401,9 +409,9 @@ fn place_as( } }; out.attribution = r.attribution.clone(); - out.error = left_out(&r, frame, false); out.pending |= r.waiting; if !(frame.m_per_px.is_finite() && frame.m_per_px > 0.0) || opacity <= 0.0 { + out.error = left_out(&r, frame, false); return Some(out); } // Without pictures, a style drawing shapes is left to the check below (no raster layers only). @@ -423,6 +431,7 @@ fn place_as( } return Some(out); } + out.error = left_out(&r, frame, false); let zoom = template::style_zoom(frame.m_per_px * f64::from(frame.ppp)); let shown: Vec<&StyleLayer> = r.style.layers.iter().filter(|l| l.visible_at(zoom) && l.kind != LayerKind::Other).collect(); // Raster layers only; several are composited together before the layer's opacity applies. @@ -915,9 +924,17 @@ fn boxes_of(known: &mut Boxes, it: &Item, layer: &StyleLayer) -> Arc f32 { + let widths: &[&str] = match layer.kind { + LayerKind::Line => &["line-width"], + LayerKind::Circle => &["circle-radius", "circle-stroke-width"], + _ => &[], + }; + if widths.iter().any(|name| layer.paint_uses_features(name)) { + return 512.0; + } let ctx = FeatureCtx::zoom_only(zoom); let reach = match layer.kind { LayerKind::Line => { diff --git a/crates/tiles/src/picture.rs b/crates/tiles/src/picture.rs index d7c25b6027..e08d029cbb 100644 --- a/crates/tiles/src/picture.rs +++ b/crates/tiles/src/picture.rs @@ -228,9 +228,6 @@ pub(crate) fn cover( return None; } let zoom = template::style_zoom(frame.m_per_px * f64::from(ppp)); - if !draws_shapes(r, zoom) { - return None; - } let step = (zoom * STEPS).round(); let level = (step / STEPS).floor(); // Squares of 256 to 512 pixels: those the size of a 256-point tile at one pixel a point, of @@ -240,23 +237,33 @@ pub(crate) fn cover( let z = (level + down) as u8; let side = (512.0 * ppp * (step / STEPS - level - down).exp2()).round().clamp(16.0, MAX_SIDE) as u32; let m_per_px = TileId::span(z) / f64::from(side); + // Tile selection, visibility and retries must describe the pixels actually prepared, + // including rounding across a source zoom boundary and fractional pixel densities. + let picture_frame = Frame { left: frame.left, top: frame.top, m_per_px, ppp }; + let zoom = template::style_zoom(m_per_px * f64::from(ppp)); + if !draws_shapes(r, zoom) { + return None; + } let key = |tile| Key { style: Arc::as_ptr(r) as usize, tile, step: step as i32, ppp: ppp.to_bits() }; let generation = cache::generation(); + // Decline before starting diagnostics or reads if this view needs straight drawing. + let span = TileId::span(z); + if ((view.width() / span).ceil() + 1.0) * ((view.height() / span).ceil() + 1.0) > template::MAX_TILES as f64 { + return None; + } + if let Some(reason) = crate::draw::left_out_at(r, frame, &picture_frame, false) { + crate::draw::add_error(error, reason); + } // Pictures include padding and may stand in at another zoom. Their preparation errors // do not describe this viewport: gather current diagnostics without repainting shapes. let mut visible = Drawn::default(); - draw_layers(None, (0, 0), view, r, frame, Pass::Shapes, &mut visible); + draw_layers(None, (0, 0), view, r, &picture_frame, Pass::Shapes, &mut visible); *pending |= visible.pending; if let Some(reason) = visible.error { crate::draw::add_error(error, reason); } let (sprites_pending, read) = sprites_read(r, zoom, ppp); - // A view of more squares than a covering holds (a huge window) is drawn straight instead; - // a reach of more keeps to the view. - let span = TileId::span(z); - if ((view.width() / span).ceil() + 1.0) * ((view.height() / span).ceil() + 1.0) > template::MAX_TILES as f64 { - return None; - } + // A reach of more squares than a covering holds keeps to the view. let squares = reach .map(|e| template::covering(e, z)) .filter(|s| !s.is_empty() && s.len() < template::MAX_TILES) @@ -523,6 +530,154 @@ fn draw_one(r: &Resolved, p: &Placed, side: u32, m_per_px: f64, ppp: f32, reach: #[cfg(test)] mod tests { + /// Pattern failures follow prepared pixels, while absent patterns leave no sprite warning. + #[test] + fn rounded_picture_patterns_report_sprite_failures() { + let _turn = cache::turn(); + for min in [3, 4] { + let base = format!("https://rounded-pattern.example/{min}"); + let style = serde_json::to_vec(&serde_json::json!({"version":8,"sprite":format!("{base}/sprite"), + "sources":{"s":{"type":"vector","tiles":[format!("{base}/tile")],"maxzoom":0}}, + "layers":[{"id":"pattern","type":"fill","source":"s","source-layer":"l","minzoom":min,"paint":{"fill-pattern":"a"}}]})) + .unwrap(); + let bytes = crate::mvt::encode(&Tile::default()); + cache::set_fetcher(move |url, done| { + if url.ends_with("/style.json") { + done(Ok(style.clone())); + } else if url.contains("/sprite") { + done(Err("HTTP 503".into())); + } else { + done(Ok(bytes.clone())); + } + }); + let spec = Spec { + kind: Kind::Style, + url: format!("{base}/style.json"), + subdomains: Vec::new(), + min_zoom: 0, + max_zoom: 14, + tile_size: 512, + tms: false, + attribution: String::new(), + layers: Vec::new(), + style_layers: String::new(), + hidden_layers: Vec::new(), + }; + let f = frame(2.999, 1.0); + let view = f.extent(64, 64); + resolved(&spec); + let mut pm = Pixmap::new(64, 64).unwrap(); + draw(&mut pm, &spec, 1.0, &f, false); + cache::wait(Duration::from_secs(2)); + let d = draw(&mut pm, &spec, 1.0, &f, false); + assert_eq!(d.error.as_deref().is_some_and(|e| e.contains("pattern")), min == 3, "draw at pattern minzoom {min}: {:?}", d.error); + let placed = place(&spec, 1.0, &f, &view, &view, None).unwrap(); + assert_eq!( + placed.error.as_deref().is_some_and(|e| e.contains("pattern")), + min == 3, + "placement at pattern minzoom {min}: {:?}", + placed.error + ); + } + } + + /// A rounded picture zoom must report and retry the tiles that supplied its pixels. + #[test] + fn picture_failures_at_rounded_zoom_retry_through_cover() { + let _turn = cache::turn(); + let tile = Tile { + layers: vec![Layer { + name: "l".into(), + extent: 4096, + features: vec![Feature { + id: None, + kind: GeomKind::Polygon, + props: Vec::new(), + geometry: vec![vec![[0.0, 0.0], [4096.0, 0.0], [4096.0, 4096.0], [0.0, 4096.0], [0.0, 0.0]]], + }], + }], + }; + for ppp in [1.0, 1.5] { + let root = format!("https://picture-rounded.example/{ppp}"); + let failed_url = format!("{root}/3"); + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let attempts = calls.clone(); + let bytes = crate::mvt::encode(&tile); + cache::set_fetcher(move |url, done| { + if url.ends_with("/3") && attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 { + done(Err("HTTP 503".into())); + } else { + done(Ok(bytes.clone())); + } + }); + let style = crate::style::Style::parse(&serde_json::json!({"version":8, + "sources":{"s":{"type":"vector","tiles":[format!("{root}/{{z}}")]}}, + "layers":[{"id":"fill","type":"fill","source":"s","source-layer":"l"}]})) + .unwrap(); + let r = Arc::new(Resolved { + style, + from: None, + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }); + let f = frame(2.999, ppp); + let view = f.extent(64, 64); + cover(&r, &f, &view, None, &mut false, &mut None); + cache::wait(Duration::from_secs(2)); + let mut error = None; + let failed = cover(&r, &f, &view, None, &mut false, &mut error).unwrap(); + assert!(error.is_some(), "the failed picture input must be reported at {ppp} pixels per point"); + assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1); + assert!(cache::expire_failure(&failed_url)); + cover(&r, &f, &view, None, &mut false, &mut None); + cache::wait(Duration::from_secs(2)); + assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 2, "cover must retry its rounded-zoom input"); + let recovered = cover(&r, &f, &view, None, &mut false, &mut None).unwrap(); + assert!(failed.iter().zip(&recovered).all(|(a, b)| !Arc::ptr_eq(&a.payload, &b.payload))); + assert!( + recovered + .iter() + .any(|it| matches!(it.payload.as_ref(), Payload::Image(pm) if pm.data().as_chunks::<4>().0.iter().any(|p| p[3]>0))) + ); + } + } + + /// A layer the renderer cannot draw must not select or report its separate source. + #[test] + fn undrawn_line_patterns_do_not_report_refused_tiles() { + let _turn = cache::turn(); + let bytes = crate::mvt::encode(&Tile::default()); + cache::set_fetcher(move |url, done| { + assert!(url.starts_with("https://pattern-control.example/")); + done(Ok(bytes.clone())); + }); + let mut style = crate::style::Style::parse(&serde_json::json!({"version":8, + "sources":{"hidden":{"type":"vector","tiles":["http://127.0.0.1/{z}"]},"shown":{"type":"vector","tiles":["https://pattern-control.example/{z}"]}}, + "layers":[{"id":"pattern","type":"line","source":"hidden","source-layer":"l","paint":{"line-pattern":"rail"}}, + {"id":"fill","type":"fill","source":"shown","source-layer":"l"}]})).unwrap(); + for (_, src) in &mut style.sources { + if let Source::Vector { from, .. } = src { + *from = Some(crate::address::Origin::named("https://pattern-control.example/style.json")); + } + } + let r = Resolved { + style, + from: None, + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }; + let f = frame(2.2, 1.0); + let mut drawn = Drawn::default(); + draw_layers(None, (64, 64), &f.extent(64, 64), &r, &f, Pass::Shapes, &mut drawn); + assert!(drawn.error.is_none(), "an unsupported layer cannot omit drawable tiles: {:?}", drawn.error); + } + /// Padded preparation and later cache reuse must report the current viewport only. #[test] fn picture_refusals_follow_the_viewport_not_preparation_padding() { @@ -579,44 +734,56 @@ mod tests { layers: vec![Layer { name: "l".into(), extent: 4096, - features: vec![Feature { id: None, kind: GeomKind::Point, props: Vec::new(), geometry: vec![vec![[4090.0, 1024.0]]] }], + features: vec![Feature { + id: None, + kind: GeomKind::Point, + props: vec![("radius".into(), crate::mvt::Value::Number(40.0))], + geometry: vec![vec![[4090.0, 1024.0]]], + }], }], }); cache::set_fetcher(move |_, done| done(Ok(bytes.clone()))); - let tile = TileId { z: 2, x: 3, y: 2 }.extent(); - let m_per_px = frame(2.2, 1.0).m_per_px; - let f = Frame { left: tile.min_x + 4.0 * m_per_px, top: tile.max_y - tile.height() * 0.25 + 32.0 * m_per_px, m_per_px, ppp: 1.0 }; - let view = f.extent(64, 64); - for restricted in [false, true] { - let mut style = crate::style::Style::parse(&serde_json::json!({"version":8, + for dependent in [false, true] { + let tile = TileId { z: 2, x: 3, y: 2 }.extent(); + let m_per_px = frame(2.2, 1.0).m_per_px; + let f = Frame { + left: tile.min_x + if dependent { 24.0 } else { 4.0 } * m_per_px, + top: tile.max_y - tile.height() * 0.25 + 32.0 * m_per_px, + m_per_px, + ppp: 1.0, + }; + let view = f.extent(64, 64); + for restricted in [false, true] { + let mut style = crate::style::Style::parse(&serde_json::json!({"version":8, "sources":{"s":{"type":"vector","tiles":["http://127.0.0.1/{z}/{x}/{y}","https://picture-neighbour.example/{z}/{x}/{y}"]}}, - "layers":[{"id":"point","type":"circle","source":"s","source-layer":"l","paint":{"circle-radius":20,"circle-color":"#ff0000"}}]})) + "layers":[{"id":"point","type":"circle","source":"s","source-layer":"l","paint":{"circle-radius":if dependent { serde_json::json!(["get","radius"]) } else { serde_json::json!(20) },"circle-color":"#ff0000"}}]})) .unwrap(); - let from = restricted.then(|| crate::address::Origin::named("https://picture-neighbour.example/style.json")); - if let Source::Vector { from: selected, .. } = &mut style.sources.first_mut().unwrap().1 { - *selected = from.clone(); + let from = restricted.then(|| crate::address::Origin::named("https://picture-neighbour.example/style.json")); + if let Source::Vector { from: selected, .. } = &mut style.sources.first_mut().unwrap().1 { + *selected = from.clone(); + } + let r = Arc::new(Resolved { + style, + from, + sprites: Vec::new(), + attribution: Vec::new(), + left_out: Vec::new(), + made: Mutex::new(Instant::now()), + waiting: false, + }); + cover(&r, &f, &view, None, &mut false, &mut None); + cache::wait(Duration::from_secs(2)); + let mut error = None; + let pictures = cover(&r, &f, &view, None, &mut false, &mut error).unwrap(); + let painted = pictures.iter().any(|it| { + let Payload::Image(pm) = it.payload.as_ref() else { return false }; + let x = ((view.min_x + 2.0 * m_per_px - it.at.min_x) / it.at.width() * f64::from(pm.width())).floor(); + let y = ((it.at.max_y - (view.max_y - 32.0 * m_per_px)) / it.at.height() * f64::from(pm.height())).floor(); + x >= 0.0 && y >= 0.0 && pm.pixel(x as u32, y as u32).is_some_and(|px| px.alpha() > 0) + }); + assert_eq!(painted, !restricted, "the permitted neighbour must paint into the actual viewport"); + assert_eq!(error.is_some(), restricted, "visible missing neighbour geometry must be reported: {error:?}"); } - let r = Arc::new(Resolved { - style, - from, - sprites: Vec::new(), - attribution: Vec::new(), - left_out: Vec::new(), - made: Mutex::new(Instant::now()), - waiting: false, - }); - cover(&r, &f, &view, None, &mut false, &mut None); - cache::wait(Duration::from_secs(2)); - let mut error = None; - let pictures = cover(&r, &f, &view, None, &mut false, &mut error).unwrap(); - let painted = pictures.iter().any(|it| { - let Payload::Image(pm) = it.payload.as_ref() else { return false }; - let x = ((view.min_x + 2.0 * m_per_px - it.at.min_x) / it.at.width() * f64::from(pm.width())).floor(); - let y = ((it.at.max_y - (view.max_y - 32.0 * m_per_px)) / it.at.height() * f64::from(pm.height())).floor(); - x >= 0.0 && y >= 0.0 && pm.pixel(x as u32, y as u32).is_some_and(|px| px.alpha() > 0) - }); - assert_eq!(painted, !restricted, "the permitted neighbour must paint into the actual viewport"); - assert_eq!(error.is_some(), restricted, "visible missing neighbour geometry must be reported: {error:?}"); } } diff --git a/crates/tiles/src/style.rs b/crates/tiles/src/style.rs index c306decc2c..f8007739e3 100644 --- a/crates/tiles/src/style.rs +++ b/crates/tiles/src/style.rs @@ -1617,6 +1617,11 @@ impl StyleLayer { self.paint.get(name).is_some_and(Expr::uses_zoom) } + /// Whether the paint property `name` can differ between features at the same zoom. + pub(crate) fn paint_uses_features(&self, name: &str) -> bool { + self.paint.get(name).is_some_and(|expr| !expr.feature_free()) + } + /// Whether the paint property `name` asks the sprite for an image. pub fn paint_uses_image(&self, name: &str) -> bool { self.paint.get(name).is_some_and(Expr::uses_image) diff --git a/crates/tiles/src/template.rs b/crates/tiles/src/template.rs index bc0897a9e7..34d6f2e229 100644 --- a/crates/tiles/src/template.rs +++ b/crates/tiles/src/template.rs @@ -262,6 +262,16 @@ pub fn join(base: &str, href: &str) -> String { return format!("{scheme}://{host}{}{tail}", remove_dots(href_path)); } let path = rest.get(host_end..).unwrap_or("").split(['?', '#']).next().unwrap_or(""); + if href_path.is_empty() { + // RFC 3986 §5.2.2: an empty reference path keeps the document, and keeps its + // query only when the reference supplies none. A base fragment is never inherited. + let query = if href.starts_with('?') { + "" + } else { + rest.split('#').next().and_then(|r| r.find('?').and_then(|i| r.get(i..))).unwrap_or("") + }; + return format!("{scheme}://{host}{path}{query}{tail}"); + } let dir = path.rfind('/').map_or("/", |i| path.get(..=i).unwrap_or("/")); format!("{scheme}://{host}{}{tail}", remove_dots(&format!("{dir}{href_path}"))) } @@ -294,6 +304,30 @@ fn remove_dots(path: &str) -> String { mod tests { use super::*; + /// Empty-path references keep the document path and inherit only an absent query. + #[test] + fn empty_path_links_keep_the_bound_document_path() { + let url = "https://s3.example/dir/style.json?old=1#old"; + for base in [ + url.to_string(), + rustgis_data::s3_address::bind("0123456789abcdef", "https://s3.example", url).unwrap(), + rustgis_data::login_address::bind("0123456789abcdef", "https://s3.example", url).unwrap(), + crate::s3::anonymous(url), + ] { + for (href, expected) in [ + ("?f=json", "https://s3.example/dir/style.json?f=json"), + ("#new", "https://s3.example/dir/style.json?old=1#new"), + ("", "https://s3.example/dir/style.json?old=1"), + ("?", "https://s3.example/dir/style.json?"), + ("#", "https://s3.example/dir/style.json?old=1#"), + ] { + let joined = join(&base, href); + let (plain, _) = crate::s3::split_anonymous(&joined); + assert_eq!(crate::logins::inner(rustgis_data::s3_address::inner(plain)), expected, "reference {href:?}"); + } + } + } + #[test] fn derived_links_keep_only_the_document_binding() { let base = rustgis_data::login_address::bind("0123456789abcdef", "https://a.test", "https://a.test/style.json").unwrap(); From a5be2aa15e288660297f920c8b8aaddeb6893586 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sun, 8 Nov 2026 19:44:23 -0600 Subject: [PATCH 09/10] fix(tiles): bound capped symbol coverage --- crates/tiles/src/draw.rs | 94 ++++++++++++++++++++++++++++++------ crates/tiles/src/picture.rs | 24 +++++---- crates/tiles/src/style.rs | 42 ++++++++++++++-- crates/tiles/src/template.rs | 12 ++++- 4 files changed, 143 insertions(+), 29 deletions(-) diff --git a/crates/tiles/src/draw.rs b/crates/tiles/src/draw.rs index 2e70556852..2576eca554 100644 --- a/crates/tiles/src/draw.rs +++ b/crates/tiles/src/draw.rs @@ -924,27 +924,27 @@ fn boxes_of(known: &mut Boxes, it: &Item, layer: &StyleLayer) -> Arc f32 { - let widths: &[&str] = match layer.kind { - LayerKind::Line => &["line-width"], - LayerKind::Circle => &["circle-radius", "circle-stroke-width"], - _ => &[], - }; - if widths.iter().any(|name| layer.paint_uses_features(name)) { - return 512.0; - } let ctx = FeatureCtx::zoom_only(zoom); let reach = match layer.kind { LayerKind::Line => { - let k = if layer.line_join(&ctx) == crate::style::LineJoin::Miter { 2.0 } else { 0.75 }; - layer.line_width(&ctx) * k + let miter = layer.layout_uses_features("line-join") || layer.line_join(&ctx) == crate::style::LineJoin::Miter; + let Some(width) = layer.paint_width_bound("line-width", zoom, 1.0) else { return 512.0 }; + width * if miter { 2.0 } else { 0.75 } + } + LayerKind::Circle => { + let (Some(radius), Some(stroke)) = + (layer.paint_width_bound("circle-radius", zoom, 5.0), layer.paint_width_bound("circle-stroke-width", zoom, 0.0)) + else { + return 512.0; + }; + radius + stroke } - LayerKind::Circle => layer.circle_radius(&ctx) + layer.circle_stroke_width(&ctx), _ => 0.0, }; - if reach.is_finite() { reach.clamp(OVERHANG, 512.0) } else { OVERHANG } + if reach.is_finite() { reach.clamp(OVERHANG, 512.0) } else { 512.0 } } fn draw_vector(pm: &mut Pixmap, it: &Item, layer: &StyleLayer, frame: &Frame, zoom: f32, sheet: Option<&Sprite>, boxes: &[[f32; 4]]) { @@ -1687,6 +1687,72 @@ pub(crate) mod tests { let _ = std::fs::remove_dir_all(dir); } + #[test] + fn bounded_shape_widths_keep_their_conservative_reach() { + for (radius, expected) in [ + (serde_json::json!(["min", 2, ["get", "radius"]]), 16.0), + (serde_json::json!(["max", 0, ["min", 30, ["get", "radius"]]]), 30.0), + (serde_json::json!(["case", ["has", "radius"], 30, 40]), 40.0), + (serde_json::json!(["step", ["get", "radius"], 20, 10, 30]), 30.0), + (serde_json::json!(["interpolate", ["linear"], ["get", "radius"], 0, 20, 100, 30]), 512.0), + (serde_json::json!(["get", "radius"]), 512.0), + // NaN is ignored by min: it must not masquerade as a finite cap on the other input. + (serde_json::json!(["min", ["sqrt", -1], ["get", "radius"]]), 512.0), + ] { + let layer = StyleLayer::parse(&serde_json::json!({"id":"p", "type":"circle", "paint":{"circle-radius":radius}})).unwrap(); + assert_eq!(overhang(&layer, 4.0), expected); + for value in [ + None, + Some(crate::mvt::Value::Number(-1.0)), + Some(crate::mvt::Value::Number(500.0)), + Some(crate::mvt::Value::String("invalid".into())), + ] { + let props = value.map(|v| vec![("radius".into(), v)]).unwrap_or_default(); + let f = FeatureCtx { props: &props, ..FeatureCtx::zoom_only(4.0) }; + assert!(overhang(&layer, 4.0) >= layer.circle_radius(&f).min(512.0)); + } + } + let line = StyleLayer::parse(&serde_json::json!({"id":"r", "type":"line", "paint":{"line-width":["min", 20, ["get","width"]]}, + "layout":{"line-join":["case", ["has","width"], "miter", "round"]}})) + .unwrap(); + assert_eq!(overhang(&line, 4.0), 40.0, "a feature may choose a mitred corner"); + } + + #[test] + fn large_shape_caps_keep_the_full_margin() { + for (kind, paint) in [ + ("line", serde_json::json!({"line-width":["min",3e38,["get","width"]]})), + ( + "circle", + serde_json::json!({"circle-radius":["min",3e38,["get","radius"]],"circle-stroke-width":["min",3e38,["get","stroke"]]}), + ), + ] { + let layer = StyleLayer::parse(&serde_json::json!({"id":"p","type":kind,"paint":paint})).unwrap(); + assert_eq!(overhang(&layer, 4.0), 512.0, "combining large caps must not shrink the margin"); + } + } + + #[test] + fn interpolated_width_caps_do_not_hide_nonfinite_outputs() { + let stop = serde_json::json!(["min", 2, ["/", -1, ["get", "zero"]]]); + let radius = serde_json::json!(["min", ["interpolate", ["linear"], ["get", "t"], 0, stop, 1, stop], ["get", "radius"]]); + let layer = StyleLayer::parse(&serde_json::json!({"id":"p","type":"circle","paint":{"circle-radius":radius}})).unwrap(); + let props = vec![ + ("zero".into(), crate::mvt::Value::Number(0.0)), + ("t".into(), crate::mvt::Value::Number(0.5)), + ("radius".into(), crate::mvt::Value::Number(100.0)), + ]; + let f = FeatureCtx { props: &props, ..FeatureCtx::zoom_only(4.0) }; + assert_eq!(layer.circle_radius(&f), 100.0, "min ignores the interpolated NaN"); + assert!(overhang(&layer, 4.0) >= layer.circle_radius(&f), "the inferred cap must contain the actual painted radius"); + let radius = serde_json::json!(["interpolate", ["linear"], ["get", "t"], -2e18, -2e18, 0, 150]); + let layer = StyleLayer::parse(&serde_json::json!({"id":"p","type":"circle","paint":{"circle-radius":radius}})).unwrap(); + let props = vec![("t".into(), crate::mvt::Value::Number(-1.0))]; + let f = FeatureCtx { props: &props, ..FeatureCtx::zoom_only(4.0) }; + assert_eq!(layer.circle_radius(&f), 256.0, "finite interpolation may lose precision through cancellation"); + assert!(overhang(&layer, 4.0) >= layer.circle_radius(&f), "finite stop values alone do not prove a conservative cap"); + } + #[test] fn wide_lines_reach_in_from_past_the_pixmap() { // The top-left 64 pixels of a tile drawn 256 across; a road 120 points wide runs 100 diff --git a/crates/tiles/src/picture.rs b/crates/tiles/src/picture.rs index e08d029cbb..fb1c96b6e9 100644 --- a/crates/tiles/src/picture.rs +++ b/crates/tiles/src/picture.rs @@ -743,20 +743,20 @@ mod tests { }], }); cache::set_fetcher(move |_, done| done(Ok(bytes.clone()))); - for dependent in [false, true] { + for (radius, inset, reaches) in [ + (serde_json::json!(20), 4.0, true), + (serde_json::json!(["get", "radius"]), 24.0, true), + (serde_json::json!(["min", 2, ["get", "radius"]]), 300.0, false), + ] { let tile = TileId { z: 2, x: 3, y: 2 }.extent(); let m_per_px = frame(2.2, 1.0).m_per_px; - let f = Frame { - left: tile.min_x + if dependent { 24.0 } else { 4.0 } * m_per_px, - top: tile.max_y - tile.height() * 0.25 + 32.0 * m_per_px, - m_per_px, - ppp: 1.0, - }; + let f = + Frame { left: tile.min_x + inset * m_per_px, top: tile.max_y - tile.height() * 0.25 + 32.0 * m_per_px, m_per_px, ppp: 1.0 }; let view = f.extent(64, 64); for restricted in [false, true] { let mut style = crate::style::Style::parse(&serde_json::json!({"version":8, "sources":{"s":{"type":"vector","tiles":["http://127.0.0.1/{z}/{x}/{y}","https://picture-neighbour.example/{z}/{x}/{y}"]}}, - "layers":[{"id":"point","type":"circle","source":"s","source-layer":"l","paint":{"circle-radius":if dependent { serde_json::json!(["get","radius"]) } else { serde_json::json!(20) },"circle-color":"#ff0000"}}]})) + "layers":[{"id":"point","type":"circle","source":"s","source-layer":"l","paint":{"circle-radius":radius,"circle-color":"#ff0000"}}]})) .unwrap(); let from = restricted.then(|| crate::address::Origin::named("https://picture-neighbour.example/style.json")); if let Source::Vector { from: selected, .. } = &mut style.sources.first_mut().unwrap().1 { @@ -781,8 +781,12 @@ mod tests { let y = ((it.at.max_y - (view.max_y - 32.0 * m_per_px)) / it.at.height() * f64::from(pm.height())).floor(); x >= 0.0 && y >= 0.0 && pm.pixel(x as u32, y as u32).is_some_and(|px| px.alpha() > 0) }); - assert_eq!(painted, !restricted, "the permitted neighbour must paint into the actual viewport"); - assert_eq!(error.is_some(), restricted, "visible missing neighbour geometry must be reported: {error:?}"); + assert_eq!( + painted, + reaches && !restricted, + "the permitted neighbour paints only when its radius reaches the actual viewport" + ); + assert_eq!(error.is_some(), reaches && restricted, "only visible missing neighbour geometry must be reported: {error:?}"); } } } diff --git a/crates/tiles/src/style.rs b/crates/tiles/src/style.rs index f8007739e3..f7bfbf6e2f 100644 --- a/crates/tiles/src/style.rs +++ b/crates/tiles/src/style.rs @@ -544,6 +544,31 @@ fn calls(n: &Node, op: Op) -> bool { } } +/// A finite upper bound for numeric results at this zoom. Unknown arithmetic stays unbounded; +/// a cap or a finite set of possible outputs can still bound a feature-dependent expression. +fn numeric_upper(n: &Node, f: &FeatureCtx) -> Option { + if free(n) { + return eval(n, f, &mut Vec::new()).as_f64().filter(|v| v.is_finite()); + } + let bound = |n| numeric_upper(n, f); + match n { + Node::Call(Op::Min, args) => args.iter().filter_map(bound).reduce(f64::min), + Node::Call(Op::Max | Op::Coalesce | Op::IsNumber, args) => numeric_upper_all(args.iter(), f), + Node::Call(Op::Case, args) => { + numeric_upper_all(args.iter().skip(1).step_by(2).chain(args.last().filter(|_| args.len() % 2 == 1)), f) + } + Node::Match(_, arms, default) => numeric_upper_all(arms.iter().map(|(_, n)| n).chain(std::iter::once(default.as_ref())), f), + Node::Step(_, first, stops) => numeric_upper_all(std::iter::once(first.as_ref()).chain(stops.iter().map(|(_, n)| n)), f), + // Interpolation and other arithmetic may create NaN or lose precision even from + // bounded inputs. Only an enclosing proven cap may narrow their full margin. + _ => None, + } +} + +fn numeric_upper_all<'a>(nodes: impl Iterator, f: &FeatureCtx) -> Option { + nodes.map(|n| numeric_upper(n, f)).try_fold(f64::NEG_INFINITY, |upper, n| Some(upper.max(n?))).filter(|v| v.is_finite()) +} + /// See [`Expr::feature_free`]. fn free(n: &Node) -> bool { match n { @@ -1617,9 +1642,20 @@ impl StyleLayer { self.paint.get(name).is_some_and(Expr::uses_zoom) } - /// Whether the paint property `name` can differ between features at the same zoom. - pub(crate) fn paint_uses_features(&self, name: &str) -> bool { - self.paint.get(name).is_some_and(|expr| !expr.feature_free()) + /// A conservative width at this zoom, including the property's default when evaluation + /// fails. `None` leaves an unbounded expression to the drawing's full margin. + pub(crate) fn paint_width_bound(&self, name: &str, zoom: f32, default: f32) -> Option { + let f = FeatureCtx::zoom_only(zoom); + let Some(expr) = self.paint.get(name) else { return Some(default) }; + if expr.feature_free() { + return Some(self.width(name, &f, default)); + } + numeric_upper(&expr.0, &f).map(|v| (v as f32).max(default)).filter(|v| v.is_finite()) + } + + /// Whether a layout property can differ between features at the same zoom. + pub(crate) fn layout_uses_features(&self, name: &str) -> bool { + self.layout.get(name).is_some_and(|expr| !expr.feature_free()) } /// Whether the paint property `name` asks the sprite for an image. diff --git a/crates/tiles/src/template.rs b/crates/tiles/src/template.rs index 34d6f2e229..8c544362ec 100644 --- a/crates/tiles/src/template.rs +++ b/crates/tiles/src/template.rs @@ -322,8 +322,16 @@ mod tests { ("#", "https://s3.example/dir/style.json?old=1#"), ] { let joined = join(&base, href); - let (plain, _) = crate::s3::split_anonymous(&joined); - assert_eq!(crate::logins::inner(rustgis_data::s3_address::inner(plain)), expected, "reference {href:?}"); + let expected = if let Some(bound) = rustgis_data::s3_address::parse(&base).unwrap() { + rustgis_data::s3_address::bind(bound.binding, bound.endpoint, expected).unwrap() + } else if let Some(bound) = rustgis_data::login_address::parse(&base).unwrap() { + rustgis_data::login_address::bind(bound.binding, bound.origin, expected).unwrap() + } else if crate::s3::split_anonymous(&base).1 { + crate::s3::anonymous(expected) + } else { + expected.to_string() + }; + assert_eq!(joined, expected, "reference {href:?} keeps the document's authority wrapper"); } } } From 85b3fec92eae8b154724dd73ebb1c2ebddb4b9d9 Mon Sep 17 00:00:00 2001 From: mavrick Date: Sun, 8 Nov 2026 20:58:43 -0600 Subject: [PATCH 10/10] fix(tiles): resolve URL text in relative references --- crates/tiles/src/cache.rs | 69 +++++++++++++++++++++++++----------- crates/tiles/src/template.rs | 16 +++++---- 2 files changed, 59 insertions(+), 26 deletions(-) diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 9eacf6d727..cc9fe728b3 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -2193,9 +2193,10 @@ mod native { assert_eq!(target.accept().unwrap_err().kind(), std::io::ErrorKind::WouldBlock); } - /// A download agent that connects only to internet addresses. + /// An internet-only agent with the same idle capacity as the tile workers' ordinary pool. pub(super) fn public_agent(timeout: std::time::Duration) -> ureq::Agent { - checked_agent(config_builder(timeout).build(), ureq::unversioned::resolver::DefaultResolver::default(), true) + let config = config_builder(timeout).max_idle_connections_per_host(PER_HOST).max_idle_connections(WORKERS).build(); + checked_agent(config, ureq::unversioned::resolver::DefaultResolver::default(), true) } /// Asks for an address's headers on a thread of its own. @@ -3564,30 +3565,34 @@ mod tests { serve_bytes(answers.into_iter().map(String::into_bytes).collect()) } - /// Both responses on one owned keep-alive socket retain its actual local provenance. + /// Both responses on one owned keep-alive socket retain its actual local provenance, + /// including when the URL's hostname alone would classify it as public. #[cfg(not(target_arch = "wasm32"))] #[test] fn reused_native_connection_keeps_its_origin() { use std::io::Write; - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let server = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().unwrap(); - for i in 0..2 { - assert!(read_request(&mut stream).starts_with("GET /"), "both requests must use the accepted socket"); - let connection = if i == 0 { "keep-alive" } else { "close" }; - write!(stream, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: {connection}\r\n\r\n{{}}").unwrap(); + for named in [false, true] { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + for i in 0..2 { + assert!(read_request(&mut stream).starts_with("GET /"), "both requests must use the accepted socket"); + let connection = if i == 0 { "keep-alive" } else { "close" }; + write!(stream, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: {connection}\r\n\r\n{{}}").unwrap(); + } + }); + let agent = native::checked_agent(native::agent().config().clone(), LocalAnswer(address), false); + let host = if named { format!("reuse.public.example:{}", address.port()) } else { address.to_string() }; + for path in ["first", "second"] { + let url = format!("http://{host}/{path}"); + let answer = native::fetch_answer(&agent, &url, crate::address::Reach::Local, false, &[]).unwrap(); + assert_eq!(answer.origin.reach, crate::address::Reach::Local); + assert_eq!(answer.origin.url, url); + assert_eq!(answer.bytes, b"{}"); } - }); - let agent = native::agent(); - for path in ["first", "second"] { - let url = format!("http://{address}/{path}"); - let answer = native::fetch_answer(&agent, &url, crate::address::Reach::Local, false, &[]).unwrap(); - assert_eq!(answer.origin.reach, crate::address::Reach::Local); - assert_eq!(answer.origin.url, url); - assert_eq!(answer.bytes, b"{}"); + server.join().unwrap(); } - server.join().unwrap(); } #[cfg(not(target_arch = "wasm32"))] @@ -4122,6 +4127,30 @@ mod tests { assert!(native::get(&agent, &format!("{base}/x"), &[]).unwrap_err().contains("too many")); } + /// URL text inside a redirect query remains data on the same document and server. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn query_only_redirects_keep_the_document_origin() { + let (base, asked) = serve_recorded( + [ + "HTTP/1.1 302 Found\r\nLocation: ?next=https://other.example/tiles\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + "HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\ntile", + ] + .into_iter() + .map(|s| s.as_bytes().to_vec()) + .collect(), + ); + let answer = + native::fetch_answer(&native::agent(), &format!("{base}/style.json?old=1"), crate::address::Reach::Local, false, &[]).unwrap(); + assert_eq!(answer.bytes, b"tile"); + assert_eq!(answer.origin.url, format!("{base}/style.json?next=https://other.example/tiles")); + assert_eq!(answer.origin.reach, crate::address::Reach::Local); + assert!(asked.recv_timeout(std::time::Duration::from_secs(2)).unwrap().starts_with("GET /style.json?old=1 ")); + assert!( + asked.recv_timeout(std::time::Duration::from_secs(2)).unwrap().starts_with("GET /style.json?next=https://other.example/tiles ") + ); + } + /// A redirected style/TileJSON must use its effective public origin, including from /// cache. The fixture routes synthetic public IPs only to its own loopback listener. #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/tiles/src/template.rs b/crates/tiles/src/template.rs index 8c544362ec..e644599714 100644 --- a/crates/tiles/src/template.rs +++ b/crates/tiles/src/template.rs @@ -244,7 +244,12 @@ pub fn join(base: &str, href: &str) -> String { Err(_) => return base.to_string(), Ok(None) => {} } - if href.contains("://") { + // URL-like text in the query or fragment is data, not an absolute reference. + let (href_path, tail) = match href.find(['?', '#']) { + Some(i) => (href.get(..i).unwrap_or(href), href.get(i..).unwrap_or("")), + None => (href, ""), + }; + if href_path.contains("://") { return href.to_string(); } let Some((scheme, rest)) = base.split_once("://") else { return href.to_string() }; @@ -253,11 +258,6 @@ pub fn join(base: &str, href: &str) -> String { if let Some(path) = href.strip_prefix("//") { return format!("{scheme}://{path}"); } - // The path (with `.` and `..` resolved, RFC 3986) and what follows it. - let (href_path, tail) = match href.find(['?', '#']) { - Some(i) => (href.get(..i).unwrap_or(href), href.get(i..).unwrap_or("")), - None => (href, ""), - }; if href.starts_with('/') { return format!("{scheme}://{host}{}{tail}", remove_dots(href_path)); } @@ -316,6 +316,8 @@ mod tests { ] { for (href, expected) in [ ("?f=json", "https://s3.example/dir/style.json?f=json"), + ("?next=https://other.example/tiles", "https://s3.example/dir/style.json?next=https://other.example/tiles"), + ("#https://other.example/tiles", "https://s3.example/dir/style.json?old=1#https://other.example/tiles"), ("#new", "https://s3.example/dir/style.json?old=1#new"), ("", "https://s3.example/dir/style.json?old=1"), ("?", "https://s3.example/dir/style.json?"), @@ -448,6 +450,8 @@ mod tests { assert_eq!(join("https://a.com/styles/x", "/planet"), "https://a.com/planet"); assert_eq!(join("https://a.com/x", "//b.com/y"), "https://b.com/y"); assert_eq!(join("https://a.com/x", "https://c.com/z"), "https://c.com/z"); + assert_eq!(join("https://a.com/x", "tiles.json?next=https://b.com"), "https://a.com/tiles.json?next=https://b.com"); + assert_eq!(join("https://a.com/x", "/tiles.json#https://b.com"), "https://a.com/tiles.json#https://b.com"); assert_eq!(join("https://tiles.example?token=x", "/tile/0"), "https://tiles.example/tile/0"); assert_eq!(join("https://tiles.example#view", "tile/0"), "https://tiles.example/tile/0"); }