From dcc075ddb6527bf826fd2623736489a36608e258 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Fri, 6 Nov 2026 07:28:35 +0100 Subject: [PATCH 1/8] fix: preserve diagnostic reasons after secret-bearing addresses --- apps/rustgis-web/src/web.rs | 30 ++-- crates/assistant/src/chatgpt.rs | 4 +- crates/assistant/src/http.rs | 50 +++++- crates/data/src/diagnostics.rs | 192 ++++++++++++++++++++++ crates/data/src/source/mod.rs | 5 +- crates/data/src/source/remote.rs | 37 ++++- crates/engine/src/cmd/history.rs | 15 +- crates/engine/src/cmd/layer.rs | 30 ++-- crates/engine/src/cmd/plugin_fetch.rs | 24 ++- crates/engine/src/cmd/portals.rs | 2 +- crates/engine/src/cmd/service.rs | 57 ++++--- crates/engine/src/cmd/stac.rs | 27 ++- crates/engine/src/cmd/tiles.rs | 16 +- crates/engine/src/cmd/zarr.rs | 34 ++-- crates/engine/src/lib.rs | 5 +- crates/engine/src/service_logins.rs | 8 +- crates/engine/src/sources.rs | 56 +++++-- crates/engine/src/tests.rs | 99 ++++++++++- crates/engine/src/tests/service_logins.rs | 2 + crates/geolibre/src/install.rs | 6 +- crates/map/src/qgis/tests.rs | 5 +- crates/map/src/source.rs | 23 ++- crates/map/src/tiles.rs | 24 ++- crates/plugins/src/pyenv.rs | 15 +- crates/plugins/src/registry.rs | 23 ++- crates/tiles/src/address.rs | 8 +- crates/tiles/src/arcgis_auth.rs | 8 +- crates/tiles/src/cache.rs | 52 ++++-- crates/tiles/src/cog.rs | 31 ++-- crates/tiles/src/mbtiles.rs | 5 +- crates/tiles/src/pmtiles.rs | 2 +- crates/tiles/src/portal.rs | 10 +- crates/tiles/src/s3.rs | 12 +- crates/tiles/src/source.rs | 10 +- crates/tiles/src/stac.rs | 9 +- crates/ui-egui/src/assistantpane.rs | 5 +- crates/ui-egui/src/failureui.rs | 42 +++-- crates/ui-egui/src/geolibreui.rs | 9 +- crates/ui-egui/src/lib.rs | 36 ++-- crates/ui-egui/src/pluginsui.rs | 5 +- crates/ui-egui/src/serviceui.rs | 15 +- crates/ui-egui/src/stacui.rs | 5 +- crates/ui-egui/src/zarrui.rs | 3 +- crates/ui-egui/tests/smoke.rs | 39 +++++ docs/privacy.md | 8 + 45 files changed, 859 insertions(+), 244 deletions(-) diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index c02f452dcb..6905f317e0 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -5,7 +5,7 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; use std::rc::Rc; -use rustgis_ui_egui::rustgis_engine::data::diagnostics::Capture; +use rustgis_ui_egui::rustgis_engine::{data::diagnostics::Capture, sources::message_with_address}; use rustgis_ui_egui::{Inbox, RustGisApp, Services, Slot}; use crate::parts::Notes; @@ -164,7 +164,7 @@ pub fn start() { // to different maps depending on when the import completes. if let Some(url) = &project { if let Err(e) = app.run("ui.openUrl", serde_json::json!({"url": url})) { - app.session.notify(format!("?project={url}: {e}"), true); + app.session.notify(format!("?project={}", message_with_address(url, &e)), true); } let skipped: Vec<&str> = ["data", "stac", "style", "tiles", "zarr"].into_iter().filter(|k| !params(&q, k, 1).is_empty()).collect(); @@ -199,7 +199,7 @@ pub fn start() { // in a notice. for url in params(links, "data", 16) { if let Err(e) = app.run("ui.addUrl", serde_json::json!({"url": url, "zoom": !linked})) { - app.session.notify(format!("?data={url}: {e}"), true); + app.session.notify(format!("?data={}", message_with_address(&url, &e)), true); } } // `?stac=`: a STAC item (its default asset is added, signed when it is a @@ -210,7 +210,7 @@ pub fn start() { if let Some(url) = stac.first() && let Err(e) = app.run("ui.stacLink", serde_json::json!({"url": url})) { - app.session.notify(format!("?stac={url}: {e}"), true); + app.session.notify(format!("?stac={}", message_with_address(url, &e)), true); } if stac.len() > 1 { app.session.notify(format!("only the first ?stac= link is opened ({} more were left out)", stac.len() - 1), false); @@ -228,7 +228,7 @@ pub fn start() { match app.run("map.addTiles", p) { // The first style added opens at its view, not one that failed. Ok(_) => style_view &= kind.is_none(), - Err(e) => app.session.notify(format!("?{key}={url}: {e}"), true), + Err(e) => app.session.notify(format!("?{key}={}", message_with_address(&url, &e)), true), } } } @@ -237,7 +237,7 @@ pub fn start() { if let Some(url) = params(links, "zarr", 1).into_iter().next() && let Err(e) = app.run("ui.addZarr", serde_json::json!({"address": url})) { - app.session.notify(format!("?zarr={url}: {e}"), true); + app.session.notify(format!("?zarr={}", message_with_address(&url, &e)), true); } // `?registry=` opens the Plugin Manager on another plugin registry for this // visit (a pull request's preview, say); installing still asks as usual. @@ -682,7 +682,7 @@ async fn fetch_bytes(url: &str, capture: Capture, omit: &'static [u16]) -> Resul // holds the whole file (a small archive), checked once it's read; else no ranges. let whole = range.is_some() && resp.status() != 206; if range.is_some_and(|(a, _)| whole && a > 0) { - return Err(format!("{url}: {}", cache::NO_RANGES)); + return Err(message_with_address(url, cache::NO_RANGES)); } // Fonts are at most about 10 MB and plugins 64 MB; refuse anything larger than allowed, by // header when there is one and by the downloaded size otherwise. @@ -691,7 +691,7 @@ async fn fetch_bytes(url: &str, capture: Capture, omit: &'static [u16]) -> Resul { // The last bytes asked for and a longer file answered: no ranges, so it is downloaded whole. if whole_tail.is_some() { - return Err(format!("{url}: {}", cache::NO_RANGES)); + return Err(message_with_address(url, cache::NO_RANGES)); } return Err(format!("{len} bytes is too large")); } @@ -719,7 +719,7 @@ async fn fetch_bytes(url: &str, capture: Capture, omit: &'static [u16]) -> Resul if (bytes.len() + n) as u64 > max { let _ = reader.cancel(); if whole_tail.is_some() { - return Err(format!("{url}: {}", cache::NO_RANGES)); + return Err(message_with_address(url, cache::NO_RANGES)); } return Err(format!("larger than {max} bytes")); } @@ -731,16 +731,16 @@ async fn fetch_bytes(url: &str, capture: Capture, omit: &'static [u16]) -> Resul if let Some((a, b)) = range.filter(|_| whole) && bytes.len() as u64 > b - a + 1 { - return Err(format!("{url}: {}", cache::NO_RANGES)); + return Err(message_with_address(url, cache::NO_RANGES)); } if whole_tail.is_some_and(|n| bytes.len() as u64 > n) { - return Err(format!("{url}: {}", cache::NO_RANGES)); + return Err(message_with_address(url, cache::NO_RANGES)); } Ok(bytes) } .await; logins::check(bound).map_err(str::to_string)?; - result.map_err(|error| if login.is_some() { logins::scrub(&format!("{url}: {error}")) } else { error }) + result.map_err(|error| if login.is_some() { message_with_address(url, &logins::scrub(&error)) } else { error }) } .await; diagnostics.record(answer.map_err(|e| Failure::new(url, e))).map_err(|failure| failure.message) @@ -804,7 +804,7 @@ async fn post_json(url: &str, body: &str, capture: Capture) -> Result, S } .await; logins::check(url).map_err(str::to_string)?; - result.map_err(|error| if login.is_some() { logins::scrub(&format!("{url}: {error}")) } else { error }) + result.map_err(|error| if login.is_some() { message_with_address(url, &logins::scrub(&error)) } else { error }) } .await; diagnostics.record(answer.map_err(|e| Failure::new(url, e))).map_err(|failure| failure.message) @@ -873,7 +873,7 @@ async fn head(url: &str, capture: Capture) -> Result Result { @@ -23,7 +24,9 @@ fn send_with(req: &HttpRequest, timeout: std::time::Duration, mut on_chunk: Opti // The address checked is the one used. let url = req.url.trim(); if !allowed_url(url) { - return Err(format!("{url}: only HTTPS addresses, or plain HTTP on this computer or network, are allowed").into()); + return Err( + diagnostics::message_with_address(url, "only HTTPS addresses, or plain HTTP on this computer or network, are allowed").into() + ); } let agent: ureq::Agent = ureq::Agent::config_builder() // Models can think for minutes on a hard request. @@ -95,7 +98,11 @@ fn transport_error(e: &ureq::Error, url: &str) -> Error { } _ => TransportFailure::Interrupted, }; - Error::transport(kind, format!("couldn't reach {}: {e}", host(url))) + // Establish known-address boundaries before applying the stricter host-only display policy. + // Replacing raw text first could match a prefix of another address's longer credential. + let message = diagnostics::message_with_address(url, &e.to_string()); + let message = message.replace(&diagnostics::address_in_message(url), &host(url)); + Error::transport(kind, format!("couldn't reach {message}")) } #[cfg(test)] @@ -107,6 +114,45 @@ mod tests { time::Duration, }; + #[test] + fn rejected_secret_addresses_keep_the_policy_reason() { + let req = HttpRequest { method: "GET", url: "ftp://models.test/private?token=secret value".into(), headers: vec![], body: None }; + let error = send(&req).unwrap_err().to_string(); + let error = diagnostics::redact(&diagnostics::redact(&error)); + assert!(error.contains("only HTTPS addresses"), "{error}"); + assert!(!error.contains("secret") && !error.contains("value"), "{error}"); + } + + #[test] + fn transport_errors_keep_reasons_without_revealing_request_paths() { + let url = "https://user:password@models.test/private-key?token=secret value"; + let cause = ureq::Error::Io(std::io::Error::other(format!("request {url} failed; retry {url}: connection closed"))); + let error = transport_error(&cause, url); + assert_eq!(error.transport, Some(TransportFailure::Interrupted)); + let error = diagnostics::redact(&diagnostics::redact(&error.to_string())); + assert!(error.contains("couldn't reach https://models.test") && error.contains("connection closed"), "{error}"); + for hidden in ["user", "password", "private-key", "secret", "value"] { + assert!(!error.contains(hidden), "{error}"); + } + } + + #[test] + fn unknown_transport_addresses_keep_conservative_credential_boundaries() { + let url = "https://models.test/private-key?token=abc"; + for reason in [ + format!("request {url}XYZ credential tail: connection closed"), + "request https://other.test/a?token=unknown credential tail: connection closed".into(), + ] { + let cause = ureq::Error::Io(std::io::Error::other(reason)); + let error = transport_error(&cause, url).to_string(); + let error = diagnostics::redact(&diagnostics::redact(&error)); + assert!(error.contains("couldn't reach https://models.test"), "{error}"); + for hidden in ["abc", "XYZ", "unknown", "credential", "tail", "connection closed"] { + assert!(!error.contains(hidden), "{error}"); + } + } + } + #[test] fn native_transport_preserves_connection_timeout_and_interruption_types() { // Check the typed refusal policy independently of OS-specific TCP retry timing. diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index b7577f90df..0fc91e2ab4 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -622,6 +622,101 @@ fn base64_shaped(word: &str) -> bool { credential_shaped(word) || (word.chars().count() >= 8 && word.chars().skip(1).any(char::is_uppercase)) } +/// Sanitizes a known address on its own, before composing it with diagnostic prose. +/// Unlike a raw URL with a secret query, this result can be redacted again without losing +/// the reason that follows it. This does not establish a boundary for unknown URLs in prose. +pub fn address_in_message(address: &str) -> String { + if address.len() > MAX_INPUT_BYTES { + return redact(address); + } + let address = match crate::s3_address::parse(address) { + Ok(Some(bound)) => bound.url, + Err(_) => return "[address omitted]".to_string(), + Ok(None) => address, + }; + let address = match crate::login_address::parse(address) { + Ok(Some(bound)) => bound.url, + Err(_) => return "[address omitted]".to_string(), + Ok(None) => address, + }; + if address.contains(crate::login_address::PREFIX) || address.contains(crate::s3_address::PREFIX) { + return "[address omitted]".to_string(); + } + // A malformed userinfo value can contain a URL-token boundary before its `@`. The generic + // scanner cannot know those later words belong to the address; here the whole input does. + let before_query = address.split(['?', '#']).next().unwrap_or(""); + if before_query.contains('@') + && before_query.chars().any(|c| c.is_whitespace() || hostile_control(c) || matches!(c, '"' | '\'' | '<' | '>' | '`')) + { + return "[address omitted]".to_string(); + } + let url = address.contains("://"); + // Every remaining byte belongs to this known address, even if a value has unescaped + // whitespace and an unfamiliar signing name. None of that suffix is diagnostic prose. + let mut shown = redact(if url { before_query } else { address }); + if url && shown.starts_with("[address omitted]") { + // A malformed authority is omitted whole, including words after its first token. + shown.truncate("[address omitted]".len()); + } + if url && before_query.len() < address.len() { + shown.push_str("[query omitted]"); + } + shown +} + +/// Composes a known address and its failure reason without exposing credentials or swallowing +/// the reason after a secret query. Exact address occurrences (including unescaped whitespace) +/// are sanitized first; unrelated URLs and credential fields still use conservative [`redact`]. +pub fn message_with_address(address: &str, reason: &str) -> String { + if address.is_empty() { + return redact(reason); + } + if reason.len() > MAX_INPUT_BYTES { + return redact(reason); + } + let shown = address_in_message(address); + let mut safe_reason = String::with_capacity(reason.len()); + let mut copied = 0; + // Lower-level transports may name the inner resource of a runtime binding instead. + let inner = crate::login_address::inner(crate::s3_address::inner(address)); + let mut outer_matches = reason.match_indices(address).peekable(); + let mut inner_matches = reason.match_indices(inner).filter(|_| inner != address).peekable(); + loop { + let occurrence = match (outer_matches.peek(), inner_matches.peek()) { + (Some(outer), Some(inner)) if outer.0 <= inner.0 => outer_matches.next(), + (Some(_), Some(_)) | (None, Some(_)) => inner_matches.next(), + (Some(_), None) => outer_matches.next(), + (None, None) => break, + }; + let Some((start, matched)) = occurrence else { break }; + if start < copied { + continue; + } + let end = start + matched.len(); + let before = reason.get(..start).and_then(|s| s.chars().next_back()); + let after = reason.get(end..).unwrap_or(""); + let begins = before.is_none_or(|c| c.is_whitespace() || matches!(c, '"' | '\'' | '<' | '(' | '[' | '{' | ':' | '=' | ',' | ';')); + let ends = after.chars().next().is_none_or(|c| { + c.is_whitespace() + || matches!(c, '"' | '\'' | '>' | ')' | ']' | '}') + || (matches!(c, ':' | ',' | ';' | '.') + && after.get(c.len_utf8()..).is_some_and(|s| s.is_empty() || s.starts_with(char::is_whitespace))) + }); + // A known address must not match the prefix of another address's longer secret. + if begins && ends { + safe_reason.push_str(reason.get(copied..start).unwrap_or("")); + safe_reason.push_str(&shown); + copied = end; + } + } + safe_reason.push_str(reason.get(copied..).unwrap_or("")); + let reason = safe_reason; + let named = reason + .strip_prefix(&shown) + .is_some_and(|tail| tail.is_empty() || tail.starts_with(':') || tail.chars().next().is_some_and(char::is_whitespace)); + if named { redact(&reason) } else { redact(&format!("{shown}: {reason}")) } +} + /// Sanitizes before bounding. Oversized input is omitted *whole*, so a secret can never be /// retained because its identifying key lay beyond a clipped prefix. URL queries/fragments /// are all omitted, including unknown/encoded signing schemes. JSON/XML bodies are not logged. @@ -953,6 +1048,88 @@ mod tests { assert!(!connection.contains("alice") && !connection.contains("hidden"), "{connection}"); } + #[test] + fn known_addresses_preserve_reasons_through_repeated_redaction() { + for address in [ + "https://example.org/data?token=hidden", + "https://example.org/data?sig=hidden#hidden", + "https://example.org/data?%256bey=hidden", + "https://example.org/data#%2574oken=hidden", + "https://alice:hidden@example.org/data?key=hidden", + "https://example.org/data?token=hidden unescaped hidden", + "https://example.org/data#sig=hidden unescaped hidden", + "https://example.org/data?custom=hidden unescaped hidden", + "https://example.org/data#hidden unescaped hidden", + ] { + let clean = message_with_address(address, "HTTP 503 Service Unavailable"); + assert!(clean.contains("HTTP 503 Service Unavailable"), "{address} -> {clean}"); + assert!(!clean.contains("hidden") && !clean.contains("alice"), "{address} -> {clean}"); + assert_eq!(redact(&clean), clean); + assert_eq!(redact(&redact(&clean)), clean); + assert_eq!(message_with_address(address, &format!("{address}: HTTP 503 Service Unavailable")), clean); + assert_eq!(message_with_address(address, &clean), clean); + } + } + + #[test] + fn every_known_address_occurrence_is_sanitized_before_the_reason() { + let address = "https://alice:hidden@example.org/data?%74oken=hidden unescaped hidden"; + let reason = format!("HTTP 404 fetching {address}; retry of {address} failed"); + let clean = message_with_address(address, &reason); + assert!(clean.contains("HTTP 404 fetching") && clean.contains("; retry of") && clean.ends_with(" failed"), "{clean}"); + assert_eq!(clean.matches("https://example.org/data").count(), 3); + assert!(!clean.contains("hidden") && !clean.contains("alice"), "{clean}"); + assert_eq!(redact(&clean), clean); + let history = History::default(); + history.push(Severity::Error, "Network", &clean, &clean); + assert!(history.report(Filter::All).contains("HTTP 404 fetching")); + assert!(!history.report(Filter::All).contains("hidden")); + } + + #[test] + fn known_address_helpers_do_not_relax_unknown_credentials() { + let raw = "https://example.org/data?token=hidden unescaped hidden: HTTP 503"; + assert_eq!(redact(raw), "https://example.org/data[query and the rest omitted]"); + let clean = message_with_address("https://known.example.org/data?key=hidden", raw); + assert!(!clean.contains("hidden") && !clean.contains("HTTP 503"), "{clean}"); + let clean = message_with_address("https://known.example.org/data?key=hidden", "HTTP 503; Password=hidden more hidden"); + assert!(clean.contains("HTTP 503") && !clean.contains("hidden"), "{clean}"); + assert_eq!(message_with_address("", raw), redact(raw)); + assert_eq!(message_with_address("/data/a", &"x".repeat(MAX_INPUT_BYTES + 1)), "[oversized diagnostic omitted]"); + } + + #[test] + fn known_address_prefixes_do_not_expose_extended_credentials() { + for address in ["https://example.org/data?token=abc", "https://example.org/data#sig=abc"] { + for suffix in ["XYZhidden", "&other=hidden", "#hidden", "%20hidden", "/hidden", ":hidden"] { + let reason = format!("{address}{suffix}: HTTP 503"); + let clean = message_with_address(address, &reason); + assert!(!clean.contains("hidden") && !clean.contains("XYZ"), "{reason} -> {clean}"); + assert!(!clean.contains("HTTP 503"), "{reason} -> {clean}"); + } + } + } + + #[test] + fn known_malformed_userinfo_never_leaks_its_remaining_words() { + for address in [ + "https://alice:hidden more-hidden@example.org/data", + "https://alice:hidden\tmore-hidden@example.org/data", + "https://alice:hidden\"more-hidden@example.org/data", + "https://alice:hidden {shown}"); + assert!(!clean.contains("hidden") && !clean.contains("alice"), "{address} -> {clean}"); + assert!(clean.ends_with(": HTTP 401"), "{clean}"); + assert_eq!(redact(&clean), clean); + } + } + #[test] fn credential_words_in_file_paths_preserve_failure_context() { let path = "../token-cache/secret.png"; @@ -965,6 +1142,21 @@ mod tests { assert_eq!(redact(r"Progress 12% complete"), "Progress 12% complete"); } + #[test] + fn known_runtime_bindings_sanitize_outer_and_inner_address_occurrences() { + let inner = "https://example.org/data?token=hidden"; + let login = crate::login_address::bind("0123456789abcdef", "https://example.org", inner).unwrap(); + let s3 = crate::s3_address::bind("0123456789abcdef", "", inner).unwrap(); + for address in [&login, &s3] { + let clean = message_with_address(address, &format!("HTTP 503 for {inner}; retrying {address} failed")); + assert!(clean.contains("HTTP 503") && clean.contains("; retrying") && clean.ends_with(" failed"), "{clean}"); + for private in ["hidden", "0123456789abcdef", crate::login_address::PREFIX, crate::s3_address::PREFIX] { + assert!(!clean.contains(private), "{private} in {clean}"); + } + assert_eq!(redact(&clean), clean); + } + } + #[test] fn prose_naming_credentials_keeps_its_words() { // RustGIS's own notices name credentials without holding one: redacting must not diff --git a/crates/data/src/source/mod.rs b/crates/data/src/source/mod.rs index f8357131fc..3a862973ad 100644 --- a/crates/data/src/source/mod.rs +++ b/crates/data/src/source/mod.rs @@ -424,7 +424,10 @@ fn open_file_now(file: std::sync::Arc) -> Result( // Readers plan within READ_BUDGET; a round asking for far more is a hostile file's offsets. let asked: u64 = wants.iter().map(|w| if let Want::Range(a, b) = w { b - a } else { 0 }).fold(0, u64::saturating_add); if asked > ROUND_BUDGET { - done(Err(DataError::Format(format!("{}: a read asked for {} MB of the file at once", remote.url, asked >> 20)))); + done(Err(DataError::Format(crate::diagnostics::message_with_address( + &remote.url, + &format!("a read asked for {} MB of the file at once", asked >> 20), + )))); return; } if rounds >= MAX_ROUNDS { - done(Err(DataError::Format(format!("{}: still needed more of the file after {MAX_ROUNDS} rounds of reading", remote.url)))); + done(Err(DataError::Format(crate::diagnostics::message_with_address( + &remote.url, + &format!("still needed more of the file after {MAX_ROUNDS} rounds of reading"), + )))); return; } let Some(fetch) = fetcher().lock().unwrap_or_else(|e| e.into_inner()).clone() else { @@ -516,9 +522,7 @@ fn step( }; if let Some((Some(done), failed)) = next { match failed { - Some(e) => { - done(Err(DataError::Format(if e.starts_with(remote2.url()) { e } else { format!("{}: {e}", remote2.url()) }))) - } + Some(e) => done(Err(DataError::Format(crate::diagnostics::message_with_address(remote2.url(), &e)))), None => step(remote2, work2, done, drive, rounds + 1), } } @@ -540,7 +544,7 @@ pub fn drive_blocking(remote: Arc, work: Arc Result<()> + Send + Sync> = Arc::new(move || { + read_remote.bytes(0, 8).ok_or_else(need)?; + Ok(()) + }); + let error = drive_blocking(remote, work).unwrap_err().to_string(); + assert!(error.contains("HTTP 404"), "{error}"); + assert!(!error.contains("hidden"), "{error}"); + assert_eq!(crate::diagnostics::redact(&error), error); + let history = crate::diagnostics::History::default(); + history.push(crate::diagnostics::Severity::Error, "Data", &error, ""); + let report = history.report(crate::diagnostics::Filter::All); + assert!(report.contains("HTTP 404") && !report.contains("hidden"), "{report}"); + } + #[test] fn hostile_offsets_never_overflow_or_explode() { // Ranges ending at the very end of the address space (a hostile file's offsets). diff --git a/crates/engine/src/cmd/history.rs b/crates/engine/src/cmd/history.rs index 3a22bd151f..4331d3f70e 100644 --- a/crates/engine/src/cmd/history.rs +++ b/crates/engine/src/cmd/history.rs @@ -135,12 +135,17 @@ pub(crate) fn redacted(v: &Value) -> Value { } /// `t` without its secrets. A connection is taken whole, though a libpq one (`PG:host=db -/// password=…`) has spaces in it: `redact` takes its password out. Other text with spaces may -/// name addresses among its words; a single word may be one. Any text may still be a connection -/// in a form `redact` doesn't read (libpq's words alone, GDAL's `MYSQL:db,user=…,password=…`, -/// ODBC's `UID=…;PWD=…`): its secret-named values go too. +/// password=…`) has spaces in it. A single-word address keeps its nonsecret request parameters +/// for replay. In prose, an unknown URL's secret value may continue past whitespace: the strict +/// diagnostic scanner must see it before any address splitter removes its identifying key. pub(crate) fn redacted_text(t: &str) -> String { - let r = if connection(t) || !t.contains(char::is_whitespace) { crate::sources::redact(t) } else { crate::sources::redact_text(t) }; + let r = if connection(t) || !t.contains(char::is_whitespace) { + crate::sources::redact(t) + } else if t.contains("://") { + rustgis_data::diagnostics::redact(t) + } else { + t.to_string() + }; // After `redact` too: a prefix it knows may come in a form it doesn't (GDAL's `MYSQL:`). query_pairs_emptied(&private_keys_removed(&r)) } diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 755cadf7b5..7d4bf39a21 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -2903,7 +2903,8 @@ pub fn url_file_name(url: &str, bytes: &[u8]) -> Result { /// Its errors quote addresses without their keys: the assistant's and MCP's tools return them as /// they are. pub(crate) fn add_url(s: &mut Session, p: &Value) -> CmdResult { - add_url_as_given(s, p).map_err(|e| crate::sources::redact_text(&e)) + let url = ps(p, "url").map(str::trim).unwrap_or(""); + add_url_as_given(s, p).map_err(|e| crate::sources::message_with_address(url, &e)) } fn add_url_as_given(s: &mut Session, p: &Value) -> CmdResult { @@ -3028,13 +3029,9 @@ fn add_url_as_given(s: &mut Session, p: &Value) -> CmdResult { } #[cfg(not(target_arch = "wasm32"))] { - // Errors name the address without its keys, and any address they quote (a redirect's) - // without its keys too; those about the address itself name it already. - let shown = crate::sources::redact(url); - let named = |e: String| match e.strip_prefix(url).or_else(|| e.strip_prefix(shown.as_str())) { - Some(rest) => format!("{shown}{}", crate::sources::redact_text(rest)), - None => format!("{shown}: {}", crate::sources::redact_text(&e)), - }; + // Redact the known address before composing any reason: a query value can contain + // unescaped whitespace, so splitting an arbitrary error into URL words is not safe. + let named = |e: String| crate::sources::message_with_address(url, &e); // An address without a file name may be a GeoTIFF too: its first bytes are read to see, // and anything that doesn't answer as one is downloaded whole, as before. let unnamed = !tiff && is_unnamed_url(url); @@ -3121,8 +3118,8 @@ fn is_zip(bytes: &[u8]) -> bool { /// A Shapefile's `.shp` or part that failed, named by its address without its keys or signature /// (the `.shp`'s query, which the parts carry too). fn part_failed(url: &str, e: String) -> String { - let e = e.strip_prefix(url).map(|r| r.trim_start_matches(": ").to_string()).unwrap_or(e); - format!("{}: {}{}", crate::sources::redact(url), crate::sources::redact_text(&e), status_hint(url, &e)) + let why = status_hint(url, &e); + format!("{}{why}", crate::sources::message_with_address(url, &e)) } /// What a Shapefile's 403 may mean: S3 answers a signed request for a file that isn't there as it @@ -3246,10 +3243,10 @@ pub fn download_data(url: &str, max: usize, done: crate::render::rustgis_tiles:: let left = max.saturating_sub(shp.len()); download_parts(name, shp, parts.into_iter(), Vec::new(), left, done); } - // The app names the address it asked for (redacted), as for any download. + // Name the complete known address before the app or session redacts this again. Err(e) => { let why = status_hint(&shp_url, &e); - done(Err(format!("{}{why}", crate::sources::redact_text(&e)))) + done(Err(format!("{}{why}", crate::sources::message_with_address(&shp_url, &e)))) } }), ); @@ -3418,7 +3415,7 @@ pub(crate) fn add_cog(s: &mut Session, p: &Value, url: &str) -> CmdResult { } } else if r.get("dated").is_some() { // Added, but its first date's file couldn't be read: the view stays where it was. - r["warning"] = json!(format!("{url} couldn't be read yet, so the map wasn't moved to it")); + r["warning"] = json!(crate::sources::message_with_address(&url, "couldn't be read yet, so the map wasn't moved to it")); } r["inPlace"] = json!(true); Ok(r) @@ -3877,17 +3874,18 @@ fn add_download(s: &mut Session, p: &Value) -> CmdResult { // A service's features (Add From Service): its answer's axes (against its layer's bounds) and // paging. let bytes = if service { - let label = ps(p, "name").unwrap_or(&url).to_string(); + let label = ps(p, "name").map(str::to_string).unwrap_or_else(|| crate::sources::address_in_message(&url)); super::service::features_answer(s, &label, bytes, bounds) } else { bytes }; // A WCS coverage (Add From Service): a GeoTIFF, or the service's error said as such. let name = if coverage { - super::service::coverage_check(ps(p, "name").unwrap_or(&url), &bytes)?; + let label = ps(p, "name").map(str::to_string).unwrap_or_else(|| crate::sources::address_in_message(&url)); + super::service::coverage_check(&label, &bytes)?; "coverage.tif".to_string() } else { - url_file_name(&url, &bytes).map_err(|e| format!("{url}: {e}"))? + url_file_name(&url, &bytes).map_err(|e| crate::sources::message_with_address(&url, &e))? }; let left_out = parts_left_out(&url, &bytes, p); let r = super::raster::add_named_bytes(s, p, name, bytes, b(p, "zoom"))?; diff --git a/crates/engine/src/cmd/plugin_fetch.rs b/crates/engine/src/cmd/plugin_fetch.rs index 6249680c43..b5cd418d5d 100644 --- a/crates/engine/src/cmd/plugin_fetch.rs +++ b/crates/engine/src/cmd/plugin_fetch.rs @@ -146,14 +146,17 @@ pub fn checked(url: &str) -> Result { } let lower = url.to_ascii_lowercase(); if !(lower.starts_with("https://") || lower.starts_with("http://")) { - let shown: String = url.chars().take(80).collect(); - return Err(format!("{shown}: plugins fetch only http:// and https:// addresses")); + let shown: String = crate::sources::address_in_message(url).chars().take(80).collect(); + return Err(crate::sources::message_with_address(&shown, "plugins fetch only http:// and https:// addresses")); } // A fragment never reaches the server (and RustGIS marks byte ranges with one). let url = url.split('#').next().unwrap_or(url); address::check(url, None)?; if address::reach(url) != address::Reach::Public && !allow_local() { - return Err(format!("{url}: plugins fetch only from the internet, not this computer or the local network")); + return Err(crate::sources::message_with_address( + url, + "plugins fetch only from the internet, not this computer or the local network", + )); } Ok(url.to_string()) } @@ -162,10 +165,10 @@ pub fn checked(url: &str) -> Result { /// or array. pub fn answer(url: &str, bytes: &[u8]) -> Result { if bytes.len() > MAX_BYTES { - return Err(format!("{url}: larger than {MAX_BYTES} bytes")); + return Err(crate::sources::message_with_address(url, &format!("larger than {MAX_BYTES} bytes"))); } let bytes = bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(bytes); - let text = std::str::from_utf8(bytes).map_err(|_| format!("{url}: the answer isn't UTF-8 text"))?; + let text = std::str::from_utf8(bytes).map_err(|_| crate::sources::message_with_address(url, "the answer isn't UTF-8 text"))?; let trimmed = text.trim_start(); let parsed = if trimmed.starts_with('{') || trimmed.starts_with('[') { serde_json::from_str::(text).ok() } else { None }; Ok(match parsed { @@ -196,7 +199,7 @@ fn start(key: &str, url: &str) { &s3::anonymous(&url), MAX_BYTES, Box::new(move |r| { - let answer = r.map_err(|e| format!("{url}: {e}")).and_then(|bytes| answer(&url, &bytes)); + let answer = r.map_err(|e| crate::sources::message_with_address(&url, &e)).and_then(|bytes| answer(&url, &bytes)); cache().keep(&key, answer); }), ); @@ -248,7 +251,10 @@ impl Calls { let started = *self.started.get_or_insert_with(Instant::now); let left = CALL_BUDGET.saturating_sub(started.elapsed()); if left.is_zero() { - return Err(format!("{url}: this plugin call's fetches took longer than {} s in all", CALL_BUDGET.as_secs())); + return Err(crate::sources::message_with_address( + url, + &format!("this plugin call's fetches took longer than {} s in all", CALL_BUDGET.as_secs()), + )); } // Names are checked again where they resolve: a public-looking name for a local address // (or a redirect to one) is refused there. Tests fetch from a server on this computer. @@ -260,7 +266,7 @@ impl Calls { }; // Errors are kept too (for `ERROR_TTL`), so a plugin run again and again on an address // that doesn't answer doesn't wait for it each time. - let answer = bytes.map_err(|e| format!("{url}: {e}")).and_then(|bytes| answer(url, &bytes)); + let answer = bytes.map_err(|e| crate::sources::message_with_address(url, &e)).and_then(|bytes| answer(url, &bytes)); cache().keep(key, answer.clone()); answer } @@ -268,7 +274,7 @@ impl Calls { #[cfg(target_arch = "wasm32")] fn blocking(&mut self, _key: &str, url: &str) -> CmdResult { let _ = self.started; - Err(format!("{url}: the web app can't wait for a fetch")) + Err(crate::sources::message_with_address(url, "the web app can't wait for a fetch")) } } diff --git a/crates/engine/src/cmd/portals.rs b/crates/engine/src/cmd/portals.rs index d2c457dfa6..09e5159408 100644 --- a/crates/engine/src/cmd/portals.rs +++ b/crates/engine/src/cmd/portals.rs @@ -242,7 +242,7 @@ fn first_layer(s: &mut Session, url: &str, kind: &str) -> Result .or_else(|| layers.first()) .and_then(|l| l["name"].as_str()) .map(str::to_string) - .ok_or_else(|| format!("{url}: the service lists no layer RustGIS can add")) + .ok_or_else(|| crate::sources::message_with_address(url, "the service lists no layer RustGIS can add")) } /// A fetch the command waits for (the command line and the assistant; the app fetches off the diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index b9a2bf1e06..c3e88d84f0 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -45,7 +45,7 @@ fn url_of(p: &Value) -> Result<&str, String> { let url = ps(p, "url").map(str::trim).filter(|u| !u.is_empty()).ok_or("url is required")?; let lower = crate::render::rustgis_tiles::logins::inner(url).to_ascii_lowercase(); if !(lower.starts_with("https://") || lower.starts_with("http://")) { - return Err(format!("{url}: a service is given by its http:// or https:// address")); + return Err(crate::sources::message_with_address(url, "a service is given by its http:// or https:// address")); } Ok(url) } @@ -152,7 +152,7 @@ fn capabilities(p: &Value) -> Result { result.next = Some(address); return Ok(result); }; - rustgis_data::ogc::parse(&address, text.as_bytes()) + rustgis_data::ogc::parse(&address, text.as_bytes()).map_err(|e| crate::sources::message_with_address(&address, &e)) } else { fetch_capabilities(original, &address) }; @@ -196,13 +196,13 @@ fn capabilities(p: &Value) -> Result { #[cfg(target_arch = "wasm32")] fn fetch_capabilities(url: &str, _caps: &str) -> Result { - Err(format!("{url}: in the web app, use Home › Add › Add Data › From Service")) + Err(crate::sources::message_with_address(url, "in the web app, use Home › Add › Add Data › From Service")) } #[cfg(not(target_arch = "wasm32"))] fn fetch_capabilities(url: &str, caps: &str) -> Result { - let bytes = crate::render::rustgis_tiles::cache::download_blocking(caps).map_err(|e| format!("{url}: {e}"))?; - rustgis_data::ogc::parse(caps, &bytes).map_err(|e| format!("{url}: {e}")) + let bytes = crate::render::rustgis_tiles::cache::download_blocking(caps).map_err(|e| crate::sources::message_with_address(url, &e))?; + rustgis_data::ogc::parse(caps, &bytes).map_err(|e| crate::sources::message_with_address(url, &e)) } /// Leaves out the layers whose addresses a document on the web may not send RustGIS to: the @@ -279,7 +279,7 @@ pub(crate) fn download_features(s: &mut Session, name: &str, url: &str, max: usi match download_layer(s, name, url, max, false)? { Downloaded::Bytes(bytes) => Ok(bytes), // Not given without protocol buffer pages. - Downloaded::Read(..) => Err(format!("{url}: the pages were read as protocol buffers")), + Downloaded::Read(..) => Err(crate::sources::message_with_address(url, "the pages were read as protocol buffers")), } } @@ -307,7 +307,7 @@ pub(crate) fn download_layer(s: &mut Session, name: &str, url: &str, max: usize, Step::Done(bytes, note) => break (Downloaded::Bytes(bytes), note), Step::Decoded(dataset, report, note) => break (Downloaded::Read(Box::new((*dataset, report))), note), // A first answer that timed out says how to add a large layer instead. - Step::Failed(e) => return Err(format!("{url}: {}", cache::large_file_hint(e))), + Step::Failed(e) => return Err(crate::sources::message_with_address(url, &cache::large_file_hint(e))), Step::Fetch(pages) => pages, }; let describing = pages.describing(); @@ -349,7 +349,7 @@ pub(crate) fn download_layer(s: &mut Session, name: &str, url: &str, max: usize, fn checked_template(service: &str, t: &str) -> Result { let t = t.trim(); if !(t.contains("{z}") && t.contains("{x}") && t.contains("{y}")) || t.len() > 2000 { - return Err(format!("{t}: not a tile template")); + return Err(crate::sources::message_with_address(t, "not a tile template")); } crate::render::rustgis_tiles::address::check(t, Some(service))?; Ok(crate::render::rustgis_tiles::logins::inherit(service, t)) @@ -454,8 +454,8 @@ pub(crate) fn add(s: &mut Session, p: &Value) -> CmdResult { let quoted = format!("'{}'", want.to_lowercase()); let left_out = |n: &&String| n.contains(rustgis_data::ogc::LEFT_OUT_3D) && n.to_lowercase().contains("ed); match found.service.notes.iter().find(left_out) { - Some(why) => format!("{url}: no layer '{want}': {why}"), - None => format!("{url}: no layer '{want}' (service.layers lists them)"), + Some(why) => crate::sources::message_with_address(&url, &format!("no layer '{want}': {why}")), + None => crate::sources::message_with_address(&url, &format!("no layer '{want}' (service.layers lists them)")), } })?; // A time-enabled layer follows the time slider, at each of its steps, unless a time is asked @@ -609,15 +609,22 @@ pub(crate) fn add(s: &mut Session, p: &Value) -> CmdResult { zoom(s); Ok(r) } - Offer::Service { url: open } => { - Err(format!("{url}: '{want}' is a folder or service the directory lists: list its layers (service.layers with url {open})")) - } + Offer::Service { url: open } => Err(crate::sources::message_with_address( + &url, + &format!( + "'{want}' is a folder or service the directory lists: list its layers (service.layers with url {})", + crate::sources::address_in_message(open) + ), + )), Offer::Coverage(c) => { let (get, _) = coverage_request(s, p, c)?; #[cfg(target_arch = "wasm32")] { let _ = get; - Err(format!("{url}: in the web app, Home › Add › Add Data › From Service downloads the coverage")) + Err(crate::sources::message_with_address( + &url, + "in the web app, Home › Add › Add Data › From Service downloads the coverage", + )) } #[cfg(not(target_arch = "wasm32"))] { @@ -634,7 +641,10 @@ pub(crate) fn add(s: &mut Session, p: &Value) -> CmdResult { #[cfg(target_arch = "wasm32")] { let _ = (items, asked); - Err(format!("{url}: in the web app, Home › Add › Add Data › From Service downloads the features")) + Err(crate::sources::message_with_address( + &url, + "in the web app, Home › Add › Add Data › From Service downloads the features", + )) } #[cfg(not(target_arch = "wasm32"))] { @@ -753,8 +763,10 @@ fn coverage(s: &mut Session, p: &Value) -> CmdResult { .filter(|(_, source)| ps(p, "source_url").is_none_or(|s| same_address(s, source))) .map(|(layer, _)| layer) .find(|l| l.name == want || l.name.eq_ignore_ascii_case(want)) - .ok_or_else(|| format!("{url}: no coverage '{want}' (service.layers lists them)"))?; - let Offer::Coverage(c) = &layer.offer else { return Err(format!("{url}: '{want}' isn't a WCS coverage")) }; + .ok_or_else(|| crate::sources::message_with_address(&url, &format!("no coverage '{want}' (service.layers lists them)")))?; + let Offer::Coverage(c) = &layer.offer else { + return Err(crate::sources::message_with_address(&url, &format!("'{want}' isn't a WCS coverage"))); + }; let (get, area) = coverage_request(s, p, c)?; Ok(json!({"url": get, "name": layer.title, "bbox": area, "describe": rustgis_data::ogc::describe_coverage_url(c)})) } @@ -791,7 +803,9 @@ pub(super) fn catalog_search(s: &mut Session, p: &Value) -> CmdResult { let url = url_of(p)?; let request = records_request(s, p)?; let page = match ps(p, "text") { - Some(text) => rustgis_data::ogc::parse_records(&request, text.as_bytes()).map_err(|e| format!("{url}: {e}"))?, + Some(text) => { + rustgis_data::ogc::parse_records(&request, text.as_bytes()).map_err(|e| crate::sources::message_with_address(url, &e))? + } None => fetch_records(url, &request)?, }; let mut dropped = 0usize; @@ -822,13 +836,14 @@ pub(super) fn catalog_search(s: &mut Session, p: &Value) -> CmdResult { #[cfg(not(target_arch = "wasm32"))] fn fetch_records(url: &str, request: &str) -> Result { - let bytes = crate::render::rustgis_tiles::cache::download_blocking(request).map_err(|e| format!("{url}: {e}"))?; - rustgis_data::ogc::parse_records(request, &bytes).map_err(|e| format!("{url}: {e}")) + let bytes = + crate::render::rustgis_tiles::cache::download_blocking(request).map_err(|e| crate::sources::message_with_address(url, &e))?; + rustgis_data::ogc::parse_records(request, &bytes).map_err(|e| crate::sources::message_with_address(url, &e)) } #[cfg(target_arch = "wasm32")] fn fetch_records(url: &str, _: &str) -> Result { - Err(format!("{url}: in the web app, use Home › Add › Add Data › From Service › Search Catalogue")) + Err(crate::sources::message_with_address(url, "in the web app, use Home › Add › Add Data › From Service › Search Catalogue")) } #[cfg(all(test, not(target_arch = "wasm32")))] diff --git a/crates/engine/src/cmd/stac.rs b/crates/engine/src/cmd/stac.rs index 5988e4dba1..02068ba446 100644 --- a/crates/engine/src/cmd/stac.rs +++ b/crates/engine/src/cmd/stac.rs @@ -649,7 +649,7 @@ pub fn stac_cube(s: &mut Session, p: &Value) -> Result { let info = cog::info_blocking(&first)?; let layout = &info.layout; if layout.transform.pixel_height > 0.0 { - return Err(format!("{first} is stored bottom up, which a trend doesn't read yet")); + return Err(crate::sources::message_with_address(&first, "stored bottom up, which a trend doesn't read yet")); } // The level whose cells are nearest the size asked, by ratio (`1000` takes MODIS's 926.6 m // overview, not its 463 m one); the full images by default. @@ -669,9 +669,12 @@ pub fn stac_cube(s: &mut Session, p: &Value) -> Result { // image's size divided by `f` (rounded up), or its cells aren't those blocks. let fits = |full: u32, ov: u32| ov.checked_mul(f as u32) == Some(full); if !(1.0..=64.0).contains(&f) || !fits(full.width, lv.width) || !fits(full.height, lv.height) { - return Err(format!( - "{first}'s overview ({} × {}) isn't its full image ({} × {}) divided by a whole number, so its cells can't be averaged from it: take \"resampling\": \"nearest\"", - lv.width, lv.height, full.width, full.height + return Err(crate::sources::message_with_address( + &first, + &format!( + "its overview ({} × {}) isn't its full image ({} × {}) divided by a whole number, so its cells can't be averaged from it: take \"resampling\": \"nearest\"", + lv.width, lv.height, full.width, full.height + ), )); } (0, f as u32) @@ -819,15 +822,23 @@ impl super::cube::TimeCube for StacCube { // file's header is fetched once however many pieces and dates read it. let info = cog::info_blocking(href)?; if info.layout.crs != self.crs { - return Err(format!("{href} is in another coordinate system than the first file")); + return Err(crate::sources::message_with_address(href, "in another coordinate system than the first file")); } - let lv = info.layout.levels.get(self.level).ok_or_else(|| format!("{href} has fewer overviews than the first file"))?; + let lv = info + .layout + .levels + .get(self.level) + .ok_or_else(|| crate::sources::message_with_address(href, "has fewer overviews than the first file"))?; let lt = info.layout.transform_of(lv); - let (dx, dy) = offset_in(&grid, <).ok_or_else(|| format!("{href} is on another grid of pixels than the first file"))?; + let (dx, dy) = offset_in(&grid, <) + .ok_or_else(|| crate::sources::message_with_address(href, "on another grid of pixels than the first file"))?; // Averaged, a cell is a block of `f` × `f` of the file's pixels: the file's corner // has to fall on a block's, or the blocks would straddle cells. if dx.rem_euclid(i64::from(f)) != 0 || dy.rem_euclid(i64::from(f)) != 0 { - return Err(format!("{href}'s pixels don't fall in whole cells of the area: take \"resampling\": \"nearest\"")); + return Err(crate::sources::message_with_address( + href, + "its pixels don't fall in whole cells of the area: take \"resampling\": \"nearest\"", + )); } // The piece's columns and rows in the file's pixels, within the file. let (fc0, fc1) = (dx.max(0), (dx + wf).min(i64::from(lv.width))); diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index f23cbaed3c..87521050c5 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -217,7 +217,10 @@ pub(crate) fn add_with_metadata_wait(s: &mut Session, p: &Value, metadata_wait: // (`start`, `end`) turns on when it is off. let dated = rustgis_data::time::is_template(url); if dated && !matches!(kind, TileKind::Raster | TileKind::Vector | TileKind::Cog) { - return Err(format!("{url}: dated addresses work for raster, vector and GeoTIFF tile layers, not {}", kind.code())); + return Err(crate::sources::message_with_address( + url, + &format!("dated addresses work for raster, vector and GeoTIFF tile layers, not {}", kind.code()), + )); } let at = if dated { dated_start(s, p)? } else { 0.0 }; let mut t = TileLayer::new( @@ -468,7 +471,7 @@ pub(crate) fn follow_style_view(s: &mut Session, wait: web_time::Duration) -> bo Ok(None) => return false, Err(e) => { s.style_view = None; - s.notify(format!("{url}: {e}"), true); + s.notify(crate::sources::message_with_address(&url, &e), true); return false; } } @@ -481,7 +484,7 @@ pub(crate) fn follow_style_view(s: &mut Session, wait: web_time::Duration) -> bo match super::navigate::go_to_view(s, center, view.zoom, view.bearing, view.pitch) { Ok(()) => true, Err(e) => { - s.notify(format!("{url}: {e}"), true); + s.notify(crate::sources::message_with_address(&url, &e), true); false } } @@ -630,8 +633,11 @@ fn legends(s: &mut Session, connection: Option<&str>, v: &Value) -> Result Result<(HashMap>, usize let len = b.len() as u64; if total.fetch_add(len, std::sync::atomic::Ordering::Relaxed).saturating_add(len) > zarr::MAX_READ { if let Ok(mut f) = failure.lock() { - f.get_or_insert(format!("{root}: more than {} GB to read: take a smaller part", zarr::MAX_READ >> 30)); + f.get_or_insert(crate::sources::message_with_address( + root, + &format!("more than {} GB to read: take a smaller part", zarr::MAX_READ >> 30), + )); } break; } @@ -128,7 +131,7 @@ fn fetch(root: &str, keys: &[String]) -> Result<(HashMap>, usize } Err(e) => { if let Ok(mut f) = failure.lock() { - f.get_or_insert(format!("{root}/{k}: {e}")); + f.get_or_insert(crate::sources::message_with_address(&format!("{root}/{k}"), &e)); } } } @@ -150,10 +153,10 @@ fn store_at(url: &str) -> Result { let (got, forbidden) = fetch(url, &keys)?; // Every metadata file refused: access is denied, not a store without metadata. if got.is_empty() && forbidden > 0 { - return Err(format!("{url}: access denied (HTTP 403)")); + return Err(crate::sources::message_with_address(url, "access denied (HTTP 403)")); } let docs: Vec<(String, Vec)> = got.into_iter().collect(); - zarr::parse_store(&docs).map_err(|e| format!("{url}: {e}")) + zarr::parse_store(&docs).map_err(|e| crate::sources::message_with_address(url, &e.to_string())) } fn info(_s: &mut Session, p: &Value) -> CmdResult { @@ -257,13 +260,17 @@ pub(crate) fn add(s: &mut Session, p: &Value) -> CmdResult { Source::Url(u) => { let store = store_at(&u)?; pick_variable(&store, &mut sel)?; - let plan = zarr::plan(&store, &sel).map_err(|e| format!("{u}: {e}"))?; + let plan = zarr::plan(&store, &sel).map_err(|e| crate::sources::message_with_address(&u, &e.to_string()))?; let (chunks, forbidden) = fetch(&u, &plan.keys)?; if forbidden > 0 { - return Err(format!("{u}: access denied (HTTP 403) to {forbidden} of the slice's chunks")); + return Err(crate::sources::message_with_address( + &u, + &format!("access denied (HTTP 403) to {forbidden} of the slice's chunks"), + )); } let name = ps(p, "name").map(str::to_string).unwrap_or_else(|| layer_name(&u, &sel, &[])); - let mut r = zarr::assemble(&store, &plan, &chunks, &name, crs_parsed).map_err(|e| format!("{u}: {e}"))?; + let mut r = zarr::assemble(&store, &plan, &chunks, &name, crs_parsed) + .map_err(|e| crate::sources::message_with_address(&u, &e.to_string()))?; r.source = Some(u.clone()); // Kept in the project as a GeoTIFF: a web store isn't fetched again when it opens. let tif = raster::geotiff::write(&r, 0).map_err(|e| e.to_string())?; @@ -464,7 +471,9 @@ pub(crate) fn read_step(zt: &ZarrTime, i: usize, key: String) -> Result Result 0 { - return Err(format!("{}: access denied (HTTP 403) to {forbidden} of the slice's chunks", zt.store)); + return Err(crate::sources::message_with_address( + &zt.store, + &format!("access denied (HTTP 403) to {forbidden} of the slice's chunks"), + )); } let crs = crs.map(Crs::parse).transpose()?; zarr::assemble(&st, &plan, &chunks, &sel, crs) @@ -1000,12 +1012,12 @@ fn store_files(root: &str, keys: &[String]) -> Result>, let (got, forbidden) = fetch(root, batch)?; // A file refused (HTTP 403) isn't one never written: said, not read as missing. if forbidden > 0 { - return Err(format!("{root}: access denied (HTTP 403) to {forbidden} of its files")); + return Err(crate::sources::message_with_address(root, &format!("access denied (HTTP 403) to {forbidden} of its files"))); } for (k, b) in got { total = total.saturating_add(b.len() as u64); if total > zarr::MAX_READ { - return Err(format!("{root}: the series is too large to read: take a shorter period")); + return Err(crate::sources::message_with_address(root, "the series is too large to read: take a shorter period")); } out.insert(k, b); } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index e6e9ce830c..9f6e64386e 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1095,7 +1095,10 @@ impl Session { } if !self.url_requests.iter().any(|q| q == p.as_ref()) { if self.url_requests.len() >= MAX_URL_REQUESTS { - return Err(format!("{url}: {MAX_URL_REQUESTS} downloads are already waiting; try again when they have landed")); + return Err(sources::message_with_address( + &url, + &format!("{MAX_URL_REQUESTS} downloads are already waiting; try again when they have landed"), + )); } self.url_requests.push(p.into_owned()); } diff --git a/crates/engine/src/service_logins.rs b/crates/engine/src/service_logins.rs index f539b570cb..6d6729d4bf 100644 --- a/crates/engine/src/service_logins.rs +++ b/crates/engine/src/service_logins.rs @@ -270,8 +270,12 @@ pub(crate) fn bind_logged_in(s: &Session, connection: &str, url: &str) -> Result return Ok(url.to_string()); } let Some(active) = s.web_logins.active.get(&c.id) else { return Ok(url.to_string()) }; - logins::bind(&active.binding, url) - .map_err(|_| format!("{url}: this address can't carry the connection's login (it has a space, a control character or a backslash)")) + logins::bind(&active.binding, url).map_err(|_| { + crate::sources::message_with_address( + url, + "this address can't carry the connection's login (it has a space, a control character or a backslash)", + ) + }) } pub fn known_secret(s: &Session, c: &Connection) -> Result { diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index a008153784..51eaac482e 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -307,23 +307,27 @@ pub fn holds_plain_key(params: &serde_json::Value) -> bool { walk(params, 0) } -/// Text with every web address in it redacted (`sources::redact`): an error's text may quote the -/// address asked for, or one a redirect led to, keys and all. Each address is redacted alone, so -/// the words around it stay; punctuation that ends a sentence isn't taken as part of it. -pub fn redact_text(text: &str) -> String { - text.split_inclusive(char::is_whitespace) - .map(|piece| { - let word = piece.trim_end(); - let Some(at) = word.find("://") else { return piece.to_string() }; - // The address starts at its scheme, after any quote or bracket before it. - let scheme = |c: &char| c.is_ascii_alphanumeric() || "+.-".contains(*c); - let start = word.get(..at).and_then(|b| b.char_indices().rev().find(|(_, c)| !scheme(c))).map_or(0, |(i, c)| i + c.len_utf8()); - let end = word.trim_end_matches([':', ',', ';', '.', ')', ']', '>', '"', '\'', '»', '›', '”', '’']).len().max(at + 3); - let (before, address, after) = - (word.get(..start).unwrap_or(""), word.get(start..end).unwrap_or(""), word.get(end..).unwrap_or("")); - format!("{before}{}{after}{}", redact(address), piece.get(word.len()..).unwrap_or("")) - }) - .collect() +/// A known source address safe to compose into a repeatedly redacted diagnostic, without +/// publishing a web-login runtime binding or the credentials inside a source wrapper. +pub fn address_in_message(address: &str) -> String { + rustgis_data::diagnostics::address_in_message(&redact(address)) +} + +/// Names a known source while keeping its failure reason. Exact address occurrences are +/// sanitized before the conservative free-text redactor sees any unescaped secret whitespace. +pub fn message_with_address(address: &str, reason: &str) -> String { + let message = rustgis_data::diagnostics::message_with_address(address, reason); + let raw_shown = rustgis_data::diagnostics::address_in_message(address); + let shown = address_in_message(address); + let message = if raw_shown.is_empty() || raw_shown == shown { message } else { message.replace(&raw_shown, &shown) }; + // A source label can differ from the diagnostic label (its secret query was removed + // entirely). Reusing this helper must not prepend that same safe address on every hop. + let reason = message.strip_prefix(&shown).and_then(|tail| tail.strip_prefix(": ")); + let message = match reason.filter(|reason| reason.strip_prefix(&shown).is_some_and(|tail| tail.is_empty() || tail.starts_with(": "))) { + Some(reason) => reason, + None => &message, + }; + crate::render::rustgis_tiles::logins::scrub(message) } /// A PMTiles or COG tile template (`pmtiles:{z}/{x}/{y}@archive`, `cog:…@archive`): what comes @@ -692,6 +696,24 @@ MIIE assert!(s.recent[0].is("service", "https://h/wms")); } + #[test] + fn address_messages_stay_private_and_named_once_across_hops() { + for address in [ + "https://alice:hidden@host.example/data?token=hidden hidden-tail", + "https://host.example/data?page=2&token=hidden", + "https://host.example/data?custom=hidden hidden-tail", + ] { + let mut message = message_with_address(address, "HTTP 503 Service Unavailable"); + for _ in 0..3 { + message = message_with_address(address, &message); + let clean = crate::Session::redacted(&message); + assert!(clean.as_str().contains("HTTP 503 Service Unavailable"), "{clean:?}"); + assert_eq!(clean.as_str().matches("host.example/data").count(), 1, "{clean:?}"); + assert!(!clean.as_str().contains("hidden") && !clean.as_str().contains("alice"), "{clean:?}"); + } + } + } + #[test] fn secrets_are_never_kept() { assert_eq!(redact("https://me:pw@host.example/a?token=abc&layer=x&X-Amz-Signature=1#k"), "https://host.example/a?layer=x"); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 44e8624b30..afa2f716a9 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -11882,6 +11882,46 @@ fn url_requests_wait_for_the_front_end() { assert!(s.request_url(&json!({"url": "https://x.org/16"})).is_ok(), "room again once taken"); } +#[test] +fn url_validation_keeps_its_reason_without_unescaped_query_tails() { + let mut s = Session::new(Project::default()); + for suffix in ["?token=hidden hidden-tail", "?%2574oken=hidden hidden-tail", "?custom=hidden hidden-tail", "#hidden hidden-tail"] { + let url = format!("https://validation.example/data.tif{suffix}"); + let mark = s.notice_mark(); + let error = s.run("layer.addUrl", &json!({"url":url, "cellSize":10})).unwrap_err(); + assert!(error.contains("\"cellSize\" goes with \"bbox\""), "{error}"); + assert!(!error.contains("hidden"), "{error}"); + let notice = s.notices_since(mark).last().unwrap().text.clone(); + assert!(notice.contains("\"cellSize\" goes with \"bbox\"") && !notice.contains("hidden"), "{notice}"); + let clean = Session::redacted(&Session::redacted(&error).into_string()); + assert!(clean.as_str().contains("\"cellSize\" goes with \"bbox\"") && !clean.as_str().contains("hidden"), "{clean:?}"); + } + let report = s.diagnostics.report(rustgis_data::diagnostics::Filter::All); + assert!(report.contains("\"cellSize\" goes with \"bbox\"") && !report.contains("hidden"), "{report}"); +} + +#[test] +fn full_download_queue_keeps_its_reason_through_session_notices() { + let mut s = Session::new(Project::default()); + for i in 0..16 { + s.request_url(&json!({"url": format!("https://queue.example/{i}")})).unwrap(); + } + for suffix in ["?token=queue-secret", "#queue-secret", "?token=queue-secret secret-tail", "#queue-secret secret-tail"] { + let url = format!("https://queue.example/refused.geojson{suffix}"); + let mark = s.notice_mark(); + let error = s.guarded("Add Data From URL", true, |s| s.request_url(&json!({"url": url}))).unwrap_err(); + assert!(error.contains("16 downloads are already waiting; try again when they have landed"), "{error}"); + assert!(!error.contains("queue-secret") && !error.contains("secret-tail"), "{error}"); + assert_eq!(Session::redacted(&error).as_str(), error); + assert_eq!(s.notices_since(mark).iter().map(|n| n.text.as_str()).collect::>(), [error.as_str()]); + s.notify(error.clone(), true); + assert_eq!(s.notices_since(mark).last().unwrap().text, error); + assert!(s.diagnostics.records().iter().any(|r| r.message == error)); + } + let report = s.diagnostics.report(rustgis_data::diagnostics::Filter::All); + assert!(!report.contains("queue-secret") && !report.contains("secret-tail"), "{report}"); +} + /// A mutating command that only left a request for the front end (the browser's `layer.addUrl`) /// leaves no undo step and nothing to save; one that changed the project still does, and so /// does one that merely answers "pending" (a plugin's command can't claim it changed nothing). @@ -13906,6 +13946,35 @@ fn notices_since_survives_the_cap() { assert_eq!(s.notices_since(0).len(), 200); } +#[test] +fn service_command_keeps_missing_layer_reason_and_scrubs_every_known_address() { + const WMS: &str = r#" +Test WMS +image/png +Root +roadsRoads"#; + let mut s = Session::new(Project::default()); + for suffix in ["?token=service-secret", "#service-secret", "?token=service-secret secret-tail", "#service-secret secret-tail"] { + let url = format!("https://maps.example.com/wms{suffix}"); + let missing = format!("{url} again {url}"); + let mark = s.notice_mark(); + let error = s.run("service.add", &json!({"url": url, "type": "wms", "layer": missing, "text": WMS})).unwrap_err(); + if suffix.contains(' ') { + // Invalid addresses are refused before capabilities are read. + assert!(error.contains("the address has spaces in it"), "{error}"); + } else { + assert!(error.contains("no layer '") && error.ends_with("(service.layers lists them)"), "{error}"); + assert_eq!(error.matches("maps.example.com/wms").count(), 3, "{error}"); + } + assert!(!error.contains("service-secret") && !error.contains("secret-tail"), "{error}"); + assert_eq!(Session::redacted(&error).as_str(), error); + assert_eq!(s.notices_since(mark).iter().map(|n| n.text.as_str()).collect::>(), [error.as_str()]); + assert!(s.diagnostics.records().iter().any(|r| r.source == "service.add" && r.message == error)); + } + let report = s.diagnostics.report(rustgis_data::diagnostics::Filter::All); + assert!(!report.contains("service-secret") && !report.contains("secret-tail"), "{report}"); +} + #[test] fn diagnostics_capture_notices_without_an_open_pane_or_project_persistence() { use rustgis_data::diagnostics::{Filter, MAX_DETAILS_BYTES, MAX_MESSAGE_BYTES, Severity}; @@ -20535,7 +20604,11 @@ fn a_run_is_copied_from_the_history_as_code() { let text = out["text"].as_str().unwrap(); assert!(text.contains("import rustgis\n\ns = rustgis.Session(") && text.contains("# 1. Buffer ("), "{text}"); assert!(text.contains("UTC)\ns.run(\n \"gp.run\""), "{text}"); - assert!(text.contains("# 3. Buffer: failed when it ran (no layer https://h.example/a)\n# s.run("), "{text}"); + assert_eq!( + text.lines().filter(|line| line.starts_with("s.run(")).count(), + 2, + "failed or unrecognized runs must not become executable Python: {text}" + ); assert!(text.contains("# 4. Mystery: Mystery can't be run again"), "{text}"); assert!(text.contains("# 5. Buffer: Buffer can't be run again") && !text.contains("file.delete"), "{text}"); // Text from the file stays in its comment, whatever line break it holds (a lone `\r` ends a @@ -20613,6 +20686,30 @@ fn secrets_in_a_query_or_a_run_s_error_stay_out_of_copied_code() { ); } +#[test] +fn history_exports_never_keep_unknown_url_credential_continuations() { + let mut s = world(); + let raw = "GET https://history.example/data?token=hidden hidden-tail: HTTP 503"; + s.project.gp_history.push(crate::GpRecord { + tool: "Buffer".into(), + id: "buffer".into(), + params: json!({"input":"Cities", "note":raw, "url":"https://history.example/data?page=2&token=hidden"}), + command: None, + at: 0, + ok: false, + messages: vec![raw.into()], + millis: 0, + outputs: Vec::new(), + }); + let code = s.run("gp.historyCode", &json!({"run":0})).unwrap()["code"].as_str().unwrap().to_string(); + let script = s.run("gp.historyScript", &Value::Null).unwrap()["text"].as_str().unwrap().to_string(); + for exported in [code, script] { + assert!(!exported.contains("hidden"), "{exported}"); + assert!(exported.contains("GET https://history.example/data"), "{exported}"); + assert!(exported.contains("https://history.example/data?page=2"), "{exported}"); + } +} + #[path = "../../../tests/support/ogc_server.rs"] mod ogc_server; diff --git a/crates/engine/src/tests/service_logins.rs b/crates/engine/src/tests/service_logins.rs index 6437dc508c..cbc28b7110 100644 --- a/crates/engine/src/tests/service_logins.rs +++ b/crates/engine/src/tests/service_logins.rs @@ -63,6 +63,8 @@ impl LoginFile { } Err(_) => break, }; + // Accepted sockets can inherit the listener's nonblocking mode. + stream.set_nonblocking(false).unwrap(); stream.set_read_timeout(Some(std::time::Duration::from_secs(5))).unwrap(); let mut request = Vec::new(); let mut chunk = [0u8; 4096]; diff --git a/crates/geolibre/src/install.rs b/crates/geolibre/src/install.rs index bcfeb76a1f..8711c2ffa0 100644 --- a/crates/geolibre/src/install.rs +++ b/crates/geolibre/src/install.rs @@ -44,7 +44,11 @@ fn agent() -> ureq::Agent { } fn get(agent: &ureq::Agent, url: &str) -> Result, String> { - agent.get(url).header("User-Agent", concat!("RustGIS/", env!("CARGO_PKG_VERSION"))).call().map_err(|e| format!("{url}: {e}")) + agent + .get(url) + .header("User-Agent", concat!("RustGIS/", env!("CARGO_PKG_VERSION"))) + .call() + .map_err(|e| rustgis_data::diagnostics::message_with_address(url, &e.to_string())) } /// The latest release's tag and the archive and checksum URLs for `target`. diff --git a/crates/map/src/qgis/tests.rs b/crates/map/src/qgis/tests.rs index 91a27c4147..9ca845e00a 100644 --- a/crates/map/src/qgis/tests.rs +++ b/crates/map/src/qgis/tests.rs @@ -860,6 +860,7 @@ fn raster_renderers_read_as_rustgis_raster_styles() { /// Every generated style and project in `testdata/qgis` reads, each project with its layers. #[test] fn every_corpus_file_reads() { + const PASSWORD: &str = "qgis-corpus-private-password-73194"; let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/qgis"); let files = |dir: &str, extensions: &[&str]| { let mut paths: Vec<_> = std::fs::read_dir(root.join(dir)).unwrap().map(|e| e.unwrap().path()).collect(); @@ -885,12 +886,14 @@ fn every_corpus_file_reads() { } else { String::from_utf8(bytes).unwrap() }; + // The worktree path itself may contain "secret"; use a distinct credential sentinel. + let text = text.replace("password='secret'", &format!("password='{PASSWORD}'")); let dir = path.parent().unwrap().to_string_lossy().to_string(); let mut report = Report::default(); let project = read_project(&text, &dir, &mut report).unwrap_or_else(|e| panic!("{}: {e}", path.display())); assert!(!project.layers.is_empty(), "{}: no layers", path.display()); // The PostGIS and SQL Server projects' made-up password reaches neither the layers nor the notes. - assert!(!format!("{project:?} {report:?}").contains("secret"), "{}: the password came through", path.display()); + assert!(!format!("{project:?} {report:?}").contains(PASSWORD), "{}: the password came through", path.display()); } } diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index 5d1af60e63..e21663d0e4 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -455,7 +455,10 @@ fn open_postgis(pg: &PostgisOrigin) -> rustgis_data::Result rustgis_data::Result> { use rustgis_data::source::{duckdb::Database, mysql}; let Database::Mysql(address) = Database::parse(&my.database) else { - return Err(rustgis_data::DataError::Format(format!("{}: not a MySQL or MariaDB connection", my.database))); + return Err(rustgis_data::DataError::Format(rustgis_data::diagnostics::message_with_address( + &my.database, + "not a MySQL or MariaDB connection", + ))); }; let conn = mysql::connect(&address, false).map_err(database_open_error)?; let table = mysql::find_table(&conn, &my.table)?; @@ -495,7 +501,10 @@ fn open_sqlserver(ms: &SqlServerOrigin) -> rustgis_data::Result rustgis_data::source::open_shared(&name, Arc::new(bytes), table.as_deref()) .map(Arc::from) - .map_err(|e| format!("{url}: {e}")), - Err(rustgis_data::source::remote::Failed::Other(e)) => Err(format!("{url}: {e}")), - Err(e) => Err(format!("{url}: couldn't be downloaded ({e:?})")), + .map_err(|e| rustgis_data::diagnostics::message_with_address(&url, &e.to_string())), + Err(rustgis_data::source::remote::Failed::Other(e)) => Err(rustgis_data::diagnostics::message_with_address(&url, &e)), + Err(e) => Err(rustgis_data::diagnostics::message_with_address(&url, &format!("couldn't be downloaded ({e:?})"))), }; if let Ok(mut s) = land.lock() { *s = Some(r); diff --git a/crates/map/src/tiles.rs b/crates/map/src/tiles.rs index 4932d425db..8514118296 100644 --- a/crates/map/src/tiles.rs +++ b/crates/map/src/tiles.rs @@ -549,13 +549,16 @@ pub fn check_url(url: &str) -> Result<(), String> { } let lower = url.to_ascii_lowercase(); if !(lower.starts_with("https://") || lower.starts_with("http://") || lower.starts_with("s3://")) { - return Err(format!("'{url}': tile addresses start with https://, http:// or s3://")); + return Err(format!( + "'{}': tile addresses start with https://, http:// or s3://", + rustgis_data::diagnostics::address_in_message(url) + )); } if host_of(url).is_none_or(str::is_empty) { - return Err(format!("'{url}' has no host")); + return Err(format!("'{}' has no host", rustgis_data::diagnostics::address_in_message(url))); } if url.chars().any(|c| c.is_whitespace() || c.is_control()) { - return Err(format!("'{url}' has spaces in it")); + return Err(format!("'{}' has spaces in it", rustgis_data::diagnostics::address_in_message(url))); } Ok(()) } @@ -574,6 +577,21 @@ pub fn host_of(url: &str) -> Option<&str> { mod tests { use super::*; + #[test] + fn invalid_secret_tile_addresses_keep_the_validation_reason() { + for (url, reason) in [ + ("ftp://tiles.test/a?token=secret value", "tile addresses start with"), + ("https://?token=secret value", "has no host"), + ("https://tiles.test/a?token=secret value", "has spaces in it"), + ("https://tiles.test/a#token=secret value", "has spaces in it"), + ] { + let error = check_url(url).unwrap_err(); + let error = rustgis_data::diagnostics::redact(&rustgis_data::diagnostics::redact(&error)); + assert!(error.contains(reason), "{error}"); + assert!(!error.contains("secret") && !error.contains("value"), "{error}"); + } + } + #[test] fn runtime_bound_tile_and_legend_addresses_are_web_addresses() { let bound = rustgis_data::login_address::bind("0123456789abcdef", "https://maps.test", "https://maps.test/style.json").unwrap(); diff --git a/crates/plugins/src/pyenv.rs b/crates/plugins/src/pyenv.rs index 490c1aa947..ba5f4e3792 100644 --- a/crates/plugins/src/pyenv.rs +++ b/crates/plugins/src/pyenv.rs @@ -122,7 +122,11 @@ fn agent() -> ureq::Agent { } fn get(agent: &ureq::Agent, url: &str) -> Result, String> { - agent.get(url).header("User-Agent", concat!("RustGIS/", env!("CARGO_PKG_VERSION"))).call().map_err(|e| format!("{url}: {e}")) + agent + .get(url) + .header("User-Agent", concat!("RustGIS/", env!("CARGO_PKG_VERSION"))) + .call() + .map_err(|e| rustgis_data::diagnostics::message_with_address(url, &e.to_string())) } /// uv: the one [`UV_ENV`] names, else the one downloaded before, else a fresh download. @@ -156,9 +160,14 @@ pub fn uv(progress: &mut dyn FnMut(&str)) -> Result { .ok_or_else(|| format!("PyPI has no uv {UV_VERSION} for {tag}"))? .to_string(); let mut bytes = Vec::new(); - get(&agent, &url)?.body_mut().as_reader().take(MAX_WHEEL + 1).read_to_end(&mut bytes).map_err(|e| format!("{url}: {e}"))?; + get(&agent, &url)? + .body_mut() + .as_reader() + .take(MAX_WHEEL + 1) + .read_to_end(&mut bytes) + .map_err(|e| rustgis_data::diagnostics::message_with_address(&url, &e.to_string()))?; if bytes.len() as u64 > MAX_WHEEL { - return Err(format!("{url} is unexpectedly large")); + return Err(rustgis_data::diagnostics::message_with_address(&url, "is unexpectedly large")); } let actual: String = Sha256::digest(&bytes).iter().map(|b| format!("{b:02x}")).collect(); if actual != *sha256 { diff --git a/crates/plugins/src/registry.rs b/crates/plugins/src/registry.rs index 8a6e6dd068..ecb86abc9d 100644 --- a/crates/plugins/src/registry.rs +++ b/crates/plugins/src/registry.rs @@ -226,7 +226,7 @@ pub fn fetch_with_progress( ) -> Result, String> { use std::io::Read; if !allowed_url(url) { - return Err(format!("{url}: only HTTPS addresses are allowed")); + return Err(rustgis_data::diagnostics::message_with_address(url, "only HTTPS addresses are allowed")); } // Redirects stay on HTTPS (a local test registry may use plain HTTP, and only locally). A // server that stops sending ends the download after a minute, not at `timeout`: until a read @@ -239,8 +239,11 @@ pub fn fetch_with_progress( .tls_config(rustgis_data::http::tls_config()) .build() .into(); - let mut resp = - agent.get(url).header("User-Agent", concat!("RustGIS/", env!("CARGO_PKG_VERSION"))).call().map_err(|e| format!("{url}: {e}"))?; + let mut resp = agent + .get(url) + .header("User-Agent", concat!("RustGIS/", env!("CARGO_PKG_VERSION"))) + .call() + .map_err(|e| rustgis_data::diagnostics::message_with_address(url, &e.to_string()))?; let total = resp.headers().get("content-length").and_then(|v| v.to_str().ok()).and_then(|v| v.trim().parse::().ok()); let mut reader = resp.body_mut().with_config().limit(limit as u64 + 1).reader(); let mut buf = Vec::new(); @@ -249,17 +252,17 @@ pub fn fetch_with_progress( let n = match reader.read(&mut piece) { Ok(n) => n, Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, - Err(e) => return Err(format!("{url}: {e}")), + Err(e) => return Err(rustgis_data::diagnostics::message_with_address(url, &e.to_string())), }; if n == 0 { break; } buf.extend_from_slice(piece.get(..n).unwrap_or_default()); if buf.len() > limit { - return Err(format!("{url} is larger than {} MB", limit / (1024 * 1024))); + return Err(rustgis_data::diagnostics::message_with_address(url, &format!("is larger than {} MB", limit / (1024 * 1024)))); } if !progress(buf.len() as u64, total) { - return Err(format!("{url}: cancelled")); + return Err(rustgis_data::diagnostics::message_with_address(url, "cancelled")); } } Ok(buf) @@ -272,6 +275,14 @@ mod fetch_tests { use std::net::TcpListener; use std::time::Duration; + #[test] + fn rejected_secret_addresses_keep_the_policy_reason() { + let error = fetch("ftp://plugins.test/a?token=secret value", 1024).unwrap_err(); + let error = rustgis_data::diagnostics::redact(&rustgis_data::diagnostics::redact(&error)); + assert!(error.contains("only HTTPS addresses are allowed"), "{error}"); + assert!(!error.contains("secret") && !error.contains("value"), "{error}"); + } + /// Serves `body` once on this computer, as an HTTP answer that gives its length. fn serve(body: Vec) -> std::io::Result { let listener = TcpListener::bind("127.0.0.1:0")?; diff --git a/crates/tiles/src/address.rs b/crates/tiles/src/address.rs index 753dc6d1fb..f957689615 100644 --- a/crates/tiles/src/address.rs +++ b/crates/tiles/src/address.rs @@ -136,15 +136,15 @@ pub fn check(url: &str, from: Option<&str>) -> Result<(), String> { } let lower = url.to_ascii_lowercase(); if !(lower.starts_with("https://") || lower.starts_with("http://")) { - return Err(format!("{url}: only http:// and https:// addresses are fetched")); + return Err(rustgis_data::diagnostics::message_with_address(url, "only http:// and https:// addresses are fetched")); } if url.chars().any(|c| c.is_whitespace() || c.is_control()) { - return Err(format!("{url}: the address has spaces in it")); + return Err(rustgis_data::diagnostics::message_with_address(url, "the address has spaces in it")); } match (reach(url), from) { - (Reach::Forbidden, _) => Err(format!("{url}: that address is never fetched")), + (Reach::Forbidden, _) => Err(rustgis_data::diagnostics::message_with_address(url, "that address is never fetched")), (Reach::Local, Some(origin)) if reach(origin) == Reach::Public => { - Err(format!("{url}: a document on the web can't point into the local network")) + Err(rustgis_data::diagnostics::message_with_address(url, "a document on the web can't point into the local network")) } _ => Ok(()), } diff --git a/crates/tiles/src/arcgis_auth.rs b/crates/tiles/src/arcgis_auth.rs index 98c38e3f3f..e43e8f0bea 100644 --- a/crates/tiles/src/arcgis_auth.rs +++ b/crates/tiles/src/arcgis_auth.rs @@ -91,7 +91,8 @@ pub fn portal_base(input: &str) -> Result { if raw.is_empty() { return Ok(ONLINE.to_string()); } - let (origin, path) = split(raw).ok_or_else(|| format!("{raw}: a portal is given by its https:// address"))?; + let (origin, path) = + split(raw).ok_or_else(|| rustgis_data::diagnostics::message_with_address(raw, "a portal is given by its https:// address"))?; if origin == "https://arcgis.com" { return Ok(ONLINE.to_string()); } @@ -426,7 +427,10 @@ pub fn add_token(url: &str, token: &str) -> Result { return Err("an ArcGIS token is one word of letters, digits and - _ . ~ + / =".into()); } let root = server_root(url).ok_or_else(|| { - format!("{}: a token is given with an ArcGIS REST address on https:// (…/rest/services/… or …/sharing/rest)", url.trim()) + rustgis_data::diagnostics::message_with_address( + url.trim(), + "a token is given with an ArcGIS REST address on https:// (…/rest/services/… or …/sharing/rest)", + ) })?; let scopes = Scope::prefix(&root).into_iter().collect(); keep(Session { diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 31eec53255..c55825bb4d 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -511,7 +511,7 @@ fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16]) { Entry::Ready(Arc::new(p), size) } Err(e) => { - log::debug!("{}", crate::logins::scrub(&format!("{}: {e}", answer.key))); + log::debug!("{}", crate::logins::scrub(&rustgis_data::diagnostics::message_with_address(&answer.key, &e))); Entry::Failed(e, Instant::now()) } }); @@ -728,9 +728,10 @@ impl<'a> RequestDiagnostics<'a> { _ => NetworkKind::Other, } }; + let message = failure.message.contains(url).then(|| rustgis_data::diagnostics::message_with_address(url, &failure.message)); self.capture.network_failure( "Network", - &clean(&failure.message), + &clean(message.as_deref().unwrap_or(&failure.message)), &clean(&details), Network { url: url.to_string(), @@ -1406,7 +1407,7 @@ mod native { let next = crate::template::join(&at, header("location").ok_or("a redirect with no Location")?); crate::address::check(&next, Some(&at))?; if at.starts_with("https://") && !next.starts_with("https://") { - return Err(format!("{next}: a redirect from https to http").into()); + return Err(rustgis_data::diagnostics::message_with_address(&next, "a redirect from https to http").into()); } same_origin &= crate::logins::origin(&next) == first_origin; at = next; @@ -1695,7 +1696,7 @@ mod native { let _ = resp.body_mut().with_config().limit(64).read_to_vec(); continue; } - None => return Err(format!("{url}: {}", super::NO_RANGES).into()), + None => return Err(rustgis_data::diagnostics::message_with_address(url, super::NO_RANGES).into()), } } if status >= 400 { @@ -1729,7 +1730,7 @@ mod native { crate::address::check(&next, Some(&at))?; // Never from HTTPS down to plain HTTP. if at.starts_with("https://") && !next.starts_with("https://") { - return Err(format!("{next}: a redirect from https to http").into()); + return Err(rustgis_data::diagnostics::message_with_address(&next, "a redirect from https to http").into()); } redirects += 1; if redirects > 5 { @@ -1748,7 +1749,7 @@ mod native { if let Some((a, _)) = range.filter(|_| whole) && a > 0 { - return Err(format!("{url}: {}", super::NO_RANGES).into()); + return Err(rustgis_data::diagnostics::message_with_address(url, super::NO_RANGES).into()); } let mut buf = Vec::new(); if !resp.status().is_success() { @@ -1767,7 +1768,7 @@ mod native { .read_to_end(&mut buf) .map_err(|e| e.to_string())?; if buf.len() as u64 > n { - return Err(format!("{url}: {}", super::NO_RANGES).into()); + return Err(rustgis_data::diagnostics::message_with_address(url, super::NO_RANGES).into()); } if buf.len() > max { return Err(format!("larger than {max} bytes").into()); @@ -1787,7 +1788,7 @@ mod native { } if let Some((a, b)) = range.filter(|_| whole) { if buf.len() as u64 > b - a + 1 { - return Err(format!("{url}: {}", super::NO_RANGES).into()); + return Err(rustgis_data::diagnostics::message_with_address(url, super::NO_RANGES).into()); } buf.truncate(usize::try_from(b - a + 1).unwrap_or(usize::MAX)); } @@ -1810,20 +1811,20 @@ mod native { use super::{Done, DoneDetailed}; pub(super) fn fetch(url: String, done: Done, _omit: &'static [u16]) { - done(Err(format!("{url}: this build has no way to fetch tiles"))); + done(Err(rustgis_data::diagnostics::message_with_address(&url, "this build has no way to fetch tiles"))); } pub(super) fn download_within(url: String, _timeout: std::time::Duration, _omit: &'static [u16], done: DoneDetailed) { - let message = format!("{url}: this build has no way to fetch tiles"); + let message = rustgis_data::diagnostics::message_with_address(&url, "this build has no way to fetch tiles"); done(Err(crate::failure::Failure::new(&url, message))); } pub(super) fn post(url: String, _body: String, done: Done) { - done(Err(format!("{url}: this build has no way to send requests"))); + done(Err(rustgis_data::diagnostics::message_with_address(&url, "this build has no way to send requests"))); } pub(super) fn head(url: String, done: super::HeadDone) { - done(Err(format!("{url}: this build has no way to send requests"))); + done(Err(rustgis_data::diagnostics::message_with_address(&url, "this build has no way to send requests"))); } pub(super) fn fetch_local(url: String, done: Done) { @@ -1948,6 +1949,29 @@ mod tests { assert!(second.records().is_empty()); } + #[test] + fn network_diagnostics_preserve_known_secret_address_failure_reasons() { + use rustgis_data::diagnostics::{Filter, History}; + for url in [ + "https://diagnostics-secret.test/data?token=hidden", + "https://diagnostics-secret.test/data?%2573ig=hidden unescaped hidden", + "https://alice:hidden@diagnostics-secret.test/data#%256bey=hidden unescaped hidden", + ] { + let history = History::default(); + let _scope = history.enter(); + let mut diagnostics = RequestDiagnostics::new(url, "GET"); + diagnostics.answered(503); + diagnostics.failed(&Failure::new(url, format!("{url}: HTTP 503 Service Unavailable"))); + let records = history.records(); + assert_eq!(records.len(), 1); + assert!(records[0].message.contains("HTTP 503 Service Unavailable"), "{records:?}"); + assert_eq!(rustgis_data::diagnostics::redact(&records[0].message), records[0].message); + let report = history.report(Filter::All); + assert!(report.contains("HTTP 503 Service Unavailable"), "{report}"); + assert!(!report.contains("hidden") && !report.contains("alice"), "{report}"); + } + } + #[cfg(not(target_arch = "wasm32"))] #[test] fn network_diagnostics_redact_downloads_without_retaining_bodies() { @@ -2677,8 +2701,10 @@ mod tests { let f = native::get_detailed(&agent, &format!("{base}/f.parquet{TAIL}4")).unwrap_err(); assert_eq!(f.status(), Some(404), "{}", f.message); let base = serve(vec![no_suffix, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\nConnection: close\r\n\r\n0123456789".into()]); - let f = native::get_detailed(&agent, &format!("{base}/f.parquet{TAIL}4")).unwrap_err(); + let f = native::get_detailed(&agent, &format!("{base}/f.parquet?token=private-range-token{TAIL}4")).unwrap_err(); assert!(f.message.contains(super::NO_RANGES), "{}", f.message); + assert!(!f.message.contains("private-range-token"), "{}", f.message); + assert!(rustgis_data::diagnostics::redact(&f.message).contains(super::NO_RANGES), "{}", f.message); // More last bytes asked for than the file holds: the whole file. let base = serve(vec![ "HTTP/1.1 501 Not Implemented\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".into(), diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index 8e98a73785..7c3ad09604 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -13,6 +13,7 @@ use std::collections::HashMap; use std::sync::{Arc, Mutex, OnceLock}; +use rustgis_data::diagnostics::message_with_address; use rustgis_data::raster::ColorHint; use rustgis_data::raster::cog::{self, Layout, Parse, Sparse}; use rustgis_data::raster::crs::Transformer; @@ -356,7 +357,7 @@ fn load(url: String, done: Box, String>) + Send>) { Box::new(move |h| { let h = h.unwrap_or_default(); if h.content_type.as_deref().is_some_and(not_tiff_type) && !h.accepts_ranges && h.tiff_header_length.is_none() { - return done(Err(format!("{u}: not a GeoTIFF"))); + return done(Err(message_with_address(&u, "not a GeoTIFF"))); } read_header(u, header_length(h.tiff_header_length), done); }), @@ -391,8 +392,8 @@ fn read_header(url: String, len: u64, done: Box, Str len, Box::new(move |r| match r { Ok(b) if cog::is_tiff(&b) => step(u, Sparse::default().with(0, b), 0, done), - Ok(_) => done(Err(format!("{u}: not a GeoTIFF"))), - Err(e) => done(Err(format!("{u}: {e}"))), + Ok(_) => done(Err(message_with_address(&u, "not a GeoTIFF"))), + Err(e) => done(Err(message_with_address(&u, &e))), }), ); } @@ -403,7 +404,7 @@ fn step(url: String, have: Sparse, rounds: usize, done: Box done(Err(format!("{NOT_COG}: {why}"))), Ok(Parse::Need(a, b)) => { let Some(len) = header_read(&have, a, b, rounds) else { - done(Err(format!("{url}: the GeoTIFF's header is too large or too scattered to read in place"))); + done(Err(message_with_address(&url, "the GeoTIFF's header is too large or too scattered to read in place"))); return; }; let u = url.clone(); @@ -417,13 +418,13 @@ fn step(url: String, have: Sparse, rounds: usize, done: Box step(u, have.with_len(a), rounds + 1, done), - Ok(more) if more.is_empty() => done(Err(format!("{u}: the GeoTIFF is cut short"))), + Ok(more) if more.is_empty() => done(Err(message_with_address(&u, "the GeoTIFF is cut short"))), Ok(more) => step(u, have.with(a, more), rounds + 1, done), - Err(e) => done(Err(format!("{u}: {e}"))), + Err(e) => done(Err(message_with_address(&u, &e))), }), ); } - Err(e) => done(Err(format!("{url}: {e}"))), + Err(e) => done(Err(message_with_address(&url, &e.to_string()))), } } @@ -458,12 +459,12 @@ fn stats(url: String, header: Sparse, layout: Layout, done: Box t, Err(e) => { - done(Err(format!("{url}: {e}"))); + done(Err(message_with_address(&url, &e))); return; } }; let Some(full) = layout.levels.first().cloned() else { - done(Err(format!("{url}: no image"))); + done(Err(message_with_address(&url, "no image"))); return; }; // Bounds: the full image's edges, sampled (a projected square isn't square in degrees). @@ -482,7 +483,7 @@ fn stats(url: String, header: Sparse, layout: Layout, done: Box lon.1 || lat.0 > lat.1 { - done(Err(format!("{url}: the GeoTIFF's corners aren't on the globe"))); + done(Err(message_with_address(&url, "the GeoTIFF's corners aren't on the globe"))); return; } let bounds = [lon.0.max(-180.0), lat.0.max(-85.05), lon.1.min(180.0), lat.1.min(85.05)]; @@ -526,7 +527,7 @@ fn stats(url: String, header: Sparse, layout: Layout, done: Box done(Ok(i)), - Err(e) => done(Err(format!("{url}: {e}"))), + Err(e) => done(Err(message_with_address(&url, &e))), }), ); } @@ -872,7 +873,7 @@ fn parse_url(url: &str) -> Option<(u8, u32, u32, String)> { /// Draws a map tile (`cog:z/x/y@address`): empty where the file has nothing. pub(crate) fn fetch(url: &str, done: Done) { let Some((z, x, y, archive)) = parse_url(url) else { - done(Err(format!("{url}: not a COG tile address"))); + done(Err(message_with_address(url, "not a COG tile address"))); return; }; let (file, look) = split(&archive); @@ -1124,7 +1125,7 @@ pub fn probe_blocking(url: &str) -> Result { Ok(Ok(())) => Ok(true), Ok(Err(e)) if read_whole(&e) => Ok(false), Ok(Err(e)) => Err(e), - Err(_) => Err(format!("{url}: no answer in a minute")), + Err(_) => Err(message_with_address(url, "no answer in a minute")), } } @@ -1158,7 +1159,7 @@ pub fn info_blocking(url: &str) -> Result, String> { let url = split(url).0; let (tx, rx) = std::sync::mpsc::channel(); with_info(url, Box::new(move |r| tx.send(r).unwrap_or_default())); - rx.recv_timeout(Duration::from_secs(120)).map_err(|_| format!("{url}: no answer in two minutes"))? + rx.recv_timeout(Duration::from_secs(120)).map_err(|_| message_with_address(url, "no answer in two minutes"))? } /// Band `band`'s values (0-based) in a window of a level: columns `cols.0..cols.1` and rows @@ -1222,7 +1223,7 @@ pub fn read_values_counted_blocking( let (mut out, mut rounded, mut overflowed) = (rustgis_data::raster::filled(w, h, f32::NAN).map_err(|e| e.to_string())?, 0u64, 0u64); let (tx, rx) = std::sync::mpsc::channel(); fetch_bands(info.clone(), url.clone(), level, tiles.clone(), vec![band], Box::new(move |r| tx.send(r).unwrap_or_default())); - let picked = rx.recv_timeout(Duration::from_secs(120)).map_err(|_| format!("{url}: tiles not read in two minutes"))??; + let picked = rx.recv_timeout(Duration::from_secs(120)).map_err(|_| message_with_address(&url, "tiles not read in two minutes"))??; let nodata = info.layout.nodata.map(|v| v as f32); for (&t, p) in tiles.iter().zip(&picked) { let Some((chunk, b)) = p.first() else { continue }; diff --git a/crates/tiles/src/mbtiles.rs b/crates/tiles/src/mbtiles.rs index 9ceb885c98..c1c8f52c7e 100644 --- a/crates/tiles/src/mbtiles.rs +++ b/crates/tiles/src/mbtiles.rs @@ -249,7 +249,7 @@ fn parse_url(url: &str) -> Option<(u8, u32, u32, &str)> { /// is read on the fetch workers, never on the frame. pub(crate) fn fetch(url: &str, done: Done) { let Some((z, x, y, archive)) = parse_url(url) else { - done(Err(format!("{url}: not an MBTiles tile address"))); + done(Err(rustgis_data::diagnostics::message_with_address(url, "not an MBTiles tile address"))); return; }; if let Err(e) = check(archive) { @@ -269,7 +269,8 @@ pub(crate) fn fetch(url: &str, done: Done) { /// Reads a tile from a file (on a fetch worker). #[cfg(not(target_arch = "wasm32"))] pub(crate) fn read_local(url: &str) -> Result, String> { - let (z, x, y, archive) = parse_url(url).ok_or_else(|| format!("{url}: not an MBTiles tile address"))?; + let (z, x, y, archive) = + parse_url(url).ok_or_else(|| rustgis_data::diagnostics::message_with_address(url, "not an MBTiles tile address"))?; check(archive)?; let path = std::path::Path::new(archive.strip_prefix("file://").unwrap_or(archive)); Ok(rustgis_data::mbtiles::tile_path(path, z, x, y)?.unwrap_or_default()) diff --git a/crates/tiles/src/pmtiles.rs b/crates/tiles/src/pmtiles.rs index 90ea8ee01c..d6b4d41826 100644 --- a/crates/tiles/src/pmtiles.rs +++ b/crates/tiles/src/pmtiles.rs @@ -529,7 +529,7 @@ fn finish(archive: &str, r: Result) { /// doesn't have. pub(crate) fn fetch(url: &str, done: Done) { let Some((z, x, y, archive)) = parse_url(url) else { - done(Err(format!("{url}: not a PMTiles tile address"))); + done(Err(rustgis_data::diagnostics::message_with_address(url, "not a PMTiles tile address"))); return; }; let a = archive.clone(); diff --git a/crates/tiles/src/portal.rs b/crates/tiles/src/portal.rs index 92243276f9..caed166653 100644 --- a/crates/tiles/src/portal.rs +++ b/crates/tiles/src/portal.rs @@ -176,7 +176,10 @@ impl Portal { Some(u) => { let host = host_of(u).to_ascii_lowercase(); if !host_ok(&host) { - return Err(format!("'{u}': a Socrata site is its domain, such as data.cityofnewyork.us")); + return Err(rustgis_data::diagnostics::message_with_address( + u, + "a Socrata site is its domain, such as data.cityofnewyork.us", + )); } web(&host)?; // A discovery API itself searches every site; a site keeps the search to it. @@ -194,7 +197,10 @@ impl Portal { Some(u) => { let host = host_of(u).to_ascii_lowercase(); if !host_ok(&host) { - return Err(format!("'{u}': an ArcGIS Hub site is its address, such as hub.arcgis.com")); + return Err(rustgis_data::diagnostics::message_with_address( + u, + "an ArcGIS Hub site is its address, such as hub.arcgis.com", + )); } web(&host)? } diff --git a/crates/tiles/src/s3.rs b/crates/tiles/src/s3.rs index 9c566821db..ac442a5ce3 100644 --- a/crates/tiles/src/s3.rs +++ b/crates/tiles/src/s3.rs @@ -365,13 +365,14 @@ pub fn is_s3(url: &str) -> bool { /// The bucket and key of an `s3://bucket/key` address. pub fn parse(url: &str) -> Result<(String, String), String> { - let rest = url.get(5..).filter(|_| is_s3(url)).ok_or_else(|| format!("{url}: not an s3:// address"))?; + let rest = + url.get(5..).filter(|_| is_s3(url)).ok_or_else(|| rustgis_data::diagnostics::message_with_address(url, "not an s3:// address"))?; let (bucket, key) = rest.split_once('/').unwrap_or((rest, "")); if !valid_bucket(bucket) { - return Err(format!("{url}: '{bucket}' isn't a bucket name")); + return Err(rustgis_data::diagnostics::message_with_address(url, &format!("'{bucket}' isn't a bucket name"))); } if key.len() > 1024 || key.chars().any(char::is_control) { - return Err(format!("{url}: the key isn't one S3 takes")); + return Err(rustgis_data::diagnostics::message_with_address(url, "the key isn't one S3 takes")); } Ok((bucket.to_string(), key.to_string())) } @@ -626,7 +627,7 @@ pub fn prepare(url: &str) -> Result { /// A web address with its path in S3's canonical form: each part decoded, then encoded. fn canonical_url(url: &str) -> Result { - let (scheme, rest) = url.split_once("://").ok_or_else(|| format!("{url}: not a web address"))?; + let (scheme, rest) = url.split_once("://").ok_or_else(|| rustgis_data::diagnostics::message_with_address(url, "not a web address"))?; let rest = rest.split('#').next().unwrap_or(rest); let end = rest.find(['/', '?']).unwrap_or(rest.len()); let (authority, path_query) = rest.split_at(end); @@ -776,7 +777,8 @@ fn sign_get(url: &str, region: &str, cr: &Credentials, date: &str) -> Result Result<(String, String, String), String> { - let rest = url.split_once("://").map(|(_, r)| r).ok_or_else(|| format!("{url}: not a web address"))?; + let rest = + url.split_once("://").map(|(_, r)| r).ok_or_else(|| rustgis_data::diagnostics::message_with_address(url, "not a web address"))?; let rest = rest.split('#').next().unwrap_or(rest); let (authority, path_query) = match rest.find('/') { Some(i) => (rest.get(..i).unwrap_or(""), rest.get(i..).unwrap_or("/")), diff --git a/crates/tiles/src/source.rs b/crates/tiles/src/source.rs index 02e6eba252..ecd03c76d3 100644 --- a/crates/tiles/src/source.rs +++ b/crates/tiles/src/source.rs @@ -216,7 +216,7 @@ fn resolve_anew(spec: &Spec, key: String) -> Resolve { } } Ok(None) => return Resolve::Pending, - Err(e) => return Resolve::Failed(format!("{}: {e}", crate::logins::inner(&spec.url))), + Err(e) => return Resolve::Failed(crate::logins::scrub(&rustgis_data::diagnostics::message_with_address(&spec.url, &e))), }, Kind::Cog => match cog::info(&spec.url) { Ok(Some(info)) => { @@ -257,7 +257,7 @@ fn resolve_anew(spec: &Spec, key: String) -> Resolve { }; let mut style = match Style::parse(&doc) { Ok(s) => s, - Err(e) => return Resolve::Failed(format!("{}: {e}", crate::logins::inner(&spec.url))), + Err(e) => return Resolve::Failed(crate::logins::scrub(&rustgis_data::diagnostics::message_with_address(&spec.url, &e))), }; if !spec.hidden_layers.is_empty() { style.layers.retain(|l| l.source_layer.as_ref().is_none_or(|s| !spec.hidden_layers.contains(s))); @@ -390,7 +390,7 @@ fn resolve_source(src: &mut Source, spec: &Spec, base: &str) -> Result Result, String> { match cache::get(url, Want::Json) { State::Ready(p) => match &*p { Payload::Json(v) => Ok(Some(v.clone())), - _ => Err(format!("{}: not a JSON document", crate::logins::inner(url))), + _ => Err(rustgis_data::diagnostics::message_with_address(url, "not a JSON document")), }, State::Pending => Ok(None), - State::Failed(e) => Err(format!("{}: {e}", crate::logins::inner(url))), + State::Failed(e) => Err(crate::logins::scrub(&rustgis_data::diagnostics::message_with_address(url, &e))), } } diff --git a/crates/tiles/src/stac.rs b/crates/tiles/src/stac.rs index 55ff6b7998..ac59771b29 100644 --- a/crates/tiles/src/stac.rs +++ b/crates/tiles/src/stac.rs @@ -594,7 +594,10 @@ pub fn sas_token_url(href: &str) -> Result { let ok = |s: &str, max: usize| (3..=max).contains(&s.len()) && s.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'); if !ok(account, 24) || !ok(container, 63) { - return Err(format!("{href}: not an Azure Blob Storage address the Planetary Computer signs")); + return Err(rustgis_data::diagnostics::message_with_address( + href, + "not an Azure Blob Storage address the Planetary Computer signs", + )); } Ok(format!("{SAS_API}/{account}/{container}")) } @@ -873,7 +876,9 @@ mod tests { assert!(parse_sas(bad).is_err()); } assert!(sas_token_url("https://evil.org/x.blob.core.windows.net/c/x.tif").is_err()); - assert!(sas_token_url("https://a.blob.core.windows.net/").is_err()); + let error = sas_token_url("https://a.blob.core.windows.net/?token=hidden unescaped hidden").unwrap_err(); + let error = rustgis_data::diagnostics::redact(&error); + assert!(error.contains("not an Azure Blob Storage address") && !error.contains("hidden"), "{error}"); remember_token("k", "sig=1"); assert_eq!(cached_token("k").as_deref(), Some("sig=1")); // A web item can't point into the local network. diff --git a/crates/ui-egui/src/assistantpane.rs b/crates/ui-egui/src/assistantpane.rs index 3f5e04f143..8732c06fd3 100644 --- a/crates/ui-egui/src/assistantpane.rs +++ b/crates/ui-egui/src/assistantpane.rs @@ -414,7 +414,10 @@ fn dispatch(app: &mut RustGisApp, req: HttpRequest, login: Option (String, String) { (tl!("RustGIS couldn't reach the server.").into(), tl!("Check the address and your connection.").into()) } (Reason::Timeout, _) => (tl!("The server took too long to answer.").into(), tl!("Try again later.").into()), - (Reason::Other, _) => (tl!("The request failed.").into(), redact_text(&f.message)), + (Reason::Other, _) => (tl!("The request failed.").into(), message_with_address(&f.url, &f.message)), } } /// What is known of the failure, as (label, value) rows: the labels in English, translated /// where shown. pub fn details(f: &Failure) -> Vec<(&'static str, String)> { - let mut rows = vec![("Address", rustgis_engine::sources::redact(&f.url))]; + let mut rows = vec![("Address", address_in_message(&f.url))]; match f.status() { Some(s) => rows.push(("Status", status_name(s).map_or_else(|| s.to_string(), |n| format!("{s} {n}")))), - None => rows.push(("Error", redact_text(&f.message))), + None => rows.push(("Error", message_with_address(&f.url, &f.message))), } let header = |k: &str| f.header(k).map(str::to_string); for (label, value) in [ @@ -140,26 +140,24 @@ mod tests { let unanswered = |r: Reason| plain(&Failure::without_answer("u", "x", r)).0; assert!(unanswered(Reason::Unreachable).contains("couldn't reach")); assert!(unanswered(Reason::Timeout).contains("too long")); - // Anything else: the error itself, with no status to show. - let other = Failure::new("u", "too many redirects"); - assert_eq!(plain(&other).1, "too many redirects"); - assert!(report(&other).contains("Error: too many redirects")); } - /// An error's text quoting an address shows it without its keys, the words around it kept. + /// Known addresses are sanitized whole before either error surface composes the reason. #[test] - fn addresses_in_an_error_lose_their_keys() { - let said = "https://h.example/a?f=json&token=s3cr3t: the server doesn't answer byte-range requests"; - let f = Failure::new("https://h.example/a?f=json&token=s3cr3t", said); - let shown = plain(&f).1; - assert_eq!(shown, "https://h.example/a?f=json: the server doesn't answer byte-range requests"); - let report = report(&f); - assert!(!report.contains("s3cr3t"), "{report}"); - assert!(report.contains("Error: https://h.example/a?f=json: the server"), "{report}"); - // Quoted, bracketed or ending a sentence; text with no address is left as it is. - assert_eq!(redact_text("(\"http://x.example/?key=k\"), then."), "(\"http://x.example/\"), then."); - assert_eq!(redact_text("«https://x.example/?token=t» gone"), "«https://x.example/» gone"); - assert_eq!(redact_text("io: connection refused"), "io: connection refused"); - assert_eq!(redact_text("://"), "://"); + fn addresses_in_an_error_lose_their_keys_without_losing_the_reason() { + for url in [ + "https://alice:hidden@h.example/a?token=hidden unescaped hidden", + "https://h.example/a?custom=hidden unescaped hidden", + "https://h.example/a#hidden unescaped hidden", + ] { + let f = Failure::new(url, format!("{url}: the server doesn't answer byte-range requests")); + let shown = plain(&f).1; + let error = details(&f).into_iter().find(|(label, _)| *label == "Error").unwrap().1; + for text in [shown, error, report(&f)] { + let clean = rustgis_engine::Session::redacted(&rustgis_engine::Session::redacted(&text).into_string()).into_string(); + assert!(clean.contains("byte-range requests"), "{clean}"); + assert!(!clean.contains("hidden") && !clean.contains("alice"), "{clean}"); + } + } } } diff --git a/crates/ui-egui/src/geolibreui.rs b/crates/ui-egui/src/geolibreui.rs index 001ebe605d..e627c4d113 100644 --- a/crates/ui-egui/src/geolibreui.rs +++ b/crates/ui-egui/src/geolibreui.rs @@ -110,7 +110,7 @@ impl Read { if self.forbidden > 0 { return Some(Err(crate::i18n::fmt( "{url}: access denied (HTTP 403) to {count} of the slice’s chunks", - &[("url", &self.root), ("count", &self.forbidden.to_string())], + &[("url", &rustgis_engine::sources::address_in_message(&self.root)), ("count", &self.forbidden.to_string())], ))); } let name = self.params.get("name").and_then(Value::as_str).unwrap_or("Zarr"); @@ -121,7 +121,10 @@ impl Read { ); } if self.got.is_empty() && self.forbidden > 0 { - return Some(Err(crate::i18n::fmt("{url}: access denied (HTTP 403)", &[("url", &self.root)]))); + return Some(Err(crate::i18n::fmt( + "{url}: access denied (HTTP 403)", + &[("url", &rustgis_engine::sources::address_in_message(&self.root))], + ))); } self.forbidden = 0; let docs = std::mem::take(&mut self.got).into_iter().collect::>(); @@ -158,7 +161,7 @@ pub fn poll(app: &mut RustGisApp, ctx: &egui::Context) { Err(e) => { app.session.download_failed(id); rustgis_engine::cmd::geolibre::discard_pending(&mut app.session, ¶ms); - app.session.notify(crate::i18n::fmt("{url}: {e}", &[("url", &url), ("e", &e)]), true); + app.session.notify(rustgis_engine::sources::message_with_address(&url, &e), true); } } } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 1f35a3fe3d..d892ab2ea4 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -2877,7 +2877,7 @@ impl RustGisApp { }, Box::new(move |r| *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(r)), ); - self.session.notify(i18n::fmt("Opening {url}…", &[("url", url)]), false); + self.session.notify(i18n::fmt("Opening {url}…", &[("url", &rustgis_engine::sources::address_in_message(url))]), false); let id = self.session.download_started(); if let Some((old, ..)) = self.ui.project_download.replace((id, url.to_string(), slot)) { self.session.download_failed(old); @@ -2973,8 +2973,8 @@ impl RustGisApp { ); } // An address read through a saved S3 connection is said as the address it wraps. - let shown = rustgis_engine::data::s3_address::inner(url); - self.session.notify(i18n::fmt("Reading {url}…", &[("url", shown)]), false); + let shown = rustgis_engine::sources::address_in_message(url); + self.session.notify(i18n::fmt("Reading {url}…", &[("url", &shown)]), false); let mut probe = p.clone(); probe["url"] = json!(address); if let Some(o) = probe.as_object_mut() { @@ -3022,8 +3022,8 @@ impl RustGisApp { if let Some(origin) = &origin { q[connectiontree::ORIGIN] = json!(origin); } - let shown = rustgis_engine::data::s3_address::inner(&url); - self.session.notify(i18n::fmt("Downloading {url}…", &[("url", shown)]), false); + let shown = rustgis_engine::sources::address_in_message(&url); + self.session.notify(i18n::fmt("Downloading {url}…", &[("url", &shown)]), false); let id = self.session.download_started(); // A service's features: read page by page from the start, off the frame (an ArcGIS // layer's description first). @@ -4065,14 +4065,17 @@ impl RustGisApp { if !self.session.download_done(id) { if r.is_ok() { self.session.notify( - i18n::fmt("{url}: downloaded after another project was opened, so it wasn't opened", &[("url", &url)]), + i18n::fmt( + "{url}: downloaded after another project was opened, so it wasn't opened", + &[("url", &rustgis_engine::sources::address_in_message(&url))], + ), false, ); } return self.poll_data_downloads(); } if let Err(e) = r.and_then(|bytes| self.open_project_bytes(project_file_name(&url), &bytes)) { - self.session.notify(if e.starts_with(&url) { e } else { format!("{url}: {e}") }, true); + self.session.notify(rustgis_engine::sources::message_with_address(&url, &e), true); } } self.poll_data_downloads(); @@ -4138,7 +4141,7 @@ impl RustGisApp { p["whole"] = json!(true); let _ = connectiontree::run_add_url(self, p, origin.as_deref()); } - Err(e) => self.session.notify(if e.starts_with(&url) { e } else { format!("{url}: {e}") }, true), + Err(e) => self.session.notify(rustgis_engine::sources::message_with_address(&url, &e), true), } } let mut landed = Vec::new(); @@ -4156,12 +4159,7 @@ impl RustGisApp { Err(e) => { self.session.download_failed(id); rustgis_engine::cmd::geolibre::discard_pending(&mut self.session, &p); - // Named without the address's keys, token or signature. - let e = e.strip_prefix(url.as_str()).map(|r| r.trim_start_matches(": ").to_string()).unwrap_or(e); - let why = rustgis_engine::sources::redact_text(&e); - // An error that names its own address (a Shapefile's part) isn't named again. - let named = why.split_once(": ").is_some_and(|(head, _)| head.contains("://") && !head.contains(' ')); - self.session.notify(if named { why } else { format!("{}: {why}", rustgis_engine::sources::redact(&url)) }, true); + self.session.notify(rustgis_engine::sources::message_with_address(&url, &e), true); } } } @@ -4190,7 +4188,13 @@ impl RustGisApp { fn add_download(&mut self, landed: serviceui::Landed, current: bool) { let serviceui::Landed { id, url, params: mut p, origin } = landed; if !current { - self.session.notify(i18n::fmt("{url}: downloaded after its project was closed, so it wasn't added", &[("url", &url)]), false); + self.session.notify( + i18n::fmt( + "{url}: downloaded after its project was closed, so it wasn't added", + &[("url", &rustgis_engine::sources::address_in_message(&url))], + ), + false, + ); return; } p["id"] = json!(id); @@ -4211,7 +4215,7 @@ impl RustGisApp { } Err(e) => { rustgis_engine::cmd::geolibre::discard_pending(&mut self.session, &pending); - self.session.notify(if e.starts_with(&url) { e } else { format!("{url}: {e}") }, true); + self.session.notify(rustgis_engine::sources::message_with_address(&url, &e), true); } } } diff --git a/crates/ui-egui/src/pluginsui.rs b/crates/ui-egui/src/pluginsui.rs index 7ac349f56d..473eb21d24 100644 --- a/crates/ui-egui/src/pluginsui.rs +++ b/crates/ui-egui/src/pluginsui.rs @@ -238,7 +238,10 @@ pub(crate) fn fetch(app: &RustGisApp, url: &str, limit: usize, ctx: Option { app.session.download_failed(landed.id); let url = &landed.url; - app.session.notify(if e.starts_with(url.as_str()) { e } else { format!("{url}: {e}") }, true); + app.session.notify(rustgis_engine::sources::message_with_address(url, &e), true); } Step::Fetch(pages) => { let next = next_request(app, &landed.params, &pages); @@ -477,8 +483,9 @@ fn go_on(app: &mut RustGisApp, landed: Landed, left: usize, step: Step) { /// Says what was left unread of a service layer's features, by its name. fn noted(app: &mut RustGisApp, landed: &Landed, note: Option) { if let Some(note) = note { - let name = landed.params.get("name").and_then(Value::as_str).filter(|n| !n.is_empty()).unwrap_or(&landed.url); - app.session.notify(format!("{name}: {note}"), false); + let name = landed.params.get("name").and_then(Value::as_str).filter(|n| !n.is_empty()); + let shown = name.map(str::to_string).unwrap_or_else(|| rustgis_engine::sources::address_in_message(&landed.url)); + app.session.notify(i18n::fmt("{name}: {note}", &[("name", &shown), ("note", ¬e)]), false); } } diff --git a/crates/ui-egui/src/stacui.rs b/crates/ui-egui/src/stacui.rs index 3bfa7c8a8a..87b5f9d2ce 100644 --- a/crates/ui-egui/src/stacui.rs +++ b/crates/ui-egui/src/stacui.rs @@ -629,7 +629,10 @@ fn landed_link(app: &mut RustGisApp, url: &str, r: Result, String>) { app.ui.stac.info = Some(format!("{} · {}", app.ui.stac.info.clone().unwrap_or_default(), unknown(app))); } } - _ => app.ui.stac.error = Some(i18n::fmt("{url}: not a STAC item or collection", &[("url", url)])), + _ => { + app.ui.stac.error = + Some(i18n::fmt("{url}: not a STAC item or collection", &[("url", &rustgis_engine::sources::address_in_message(url))])) + } } } diff --git a/crates/ui-egui/src/zarrui.rs b/crates/ui-egui/src/zarrui.rs index a6b2851ab7..9e5c4609e7 100644 --- a/crates/ui-egui/src/zarrui.rs +++ b/crates/ui-egui/src/zarrui.rs @@ -127,7 +127,8 @@ pub fn poll(app: &mut RustGisApp, ctx: &egui::Context) { Phase::Meta => { let docs: Vec<(String, Vec)> = std::mem::take(&mut st.got).into_iter().collect(); if docs.is_empty() && st.forbidden > 0 { - st.error = Some(i18n::fmt("{url}: access denied (HTTP 403)", &[("url", &st.root)])); + st.error = + Some(i18n::fmt("{url}: access denied (HTTP 403)", &[("url", &rustgis_engine::sources::address_in_message(&st.root))])); return; } match zarr::parse_store(&docs) { diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index b988db42d7..4b3ed725bd 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -1960,6 +1960,45 @@ fn data_from_a_web_address_lands_later() { assert!(h.state().ui.downloads.is_empty()); } +/// A failed header probe, ordinary download and project download retain their HTTP reason after +/// notices and diagnostic history redact the message again (#1475). +#[test] +fn address_failures_keep_status_without_credentials() { + use ogc_server::{Route, Server}; + let server = Server::new( + ["probe.tif", "remote.fgb", "download.geojson", "project.rgis"] + .into_iter() + .map(|name| Route { status: 503, ..Route::new(&format!("/{name}"), "") }) + .collect(), + ); + let mut h = harness(); + h.step(); + for (command, file, query) in [ + ("ui.addUrl", "probe.tif", "sig=hidden-probe"), + ("ui.addUrl", "remote.fgb", "token=hidden-remote"), + ("ui.addUrl", "download.geojson", "key=hidden-download"), + ("ui.openUrl", "project.rgis", "%74oken=hidden-project"), + ] { + let url = format!("{}/{file}?{query}#hidden-fragment", server.base); + h.state_mut().session.notices.clear(); + h.state_mut().run(command, json!({"url": url})).unwrap(); + for _ in 0..200 { + h.step(); + if h.state().session.notices.iter().any(|n| n.error) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + let notices = &h.state().session.notices; + let failure = notices.iter().find(|n| n.error).unwrap_or_else(|| panic!("{file}: no failure notice")); + assert!(failure.text.contains(file) && failure.text.contains("503"), "{failure:?}"); + assert!(notices.iter().all(|n| !n.text.contains("hidden")), "{notices:?}"); + let report = h.state().session.diagnostics.report(rustgis_engine::data::diagnostics::Filter::All); + assert!(!report.contains("hidden"), "{report}"); + assert!(report.lines().any(|line| line.contains(file) && line.contains("503")), "{report}"); + } +} + /// Add Data From URL reads a cloud-optimised GeoTIFF in place from an address that names no file /// (an API such as Vantor's `…/geotiff?…`): its first bytes are probed off the frame, then the /// layer reads its tiles by range, never downloading it whole. diff --git a/docs/privacy.md b/docs/privacy.md index 8f255a1cc9..70aaff297e 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -21,6 +21,14 @@ After `Basic` (without a colon) at the start of a line, the rest of the text is when the next word looks like base64: it has a digit or punctuation (a sentence-ending mark `.,!?;:)` at its end doesn't count), has 16 characters or more, or has 8 or more with a capital after the first letter. So a very short Basic credential such as `YTpi` is kept. + +Notices and request errors redact a known address before attaching its failure reason, so a +token or signature does not hide the HTTP status or other explanation. The complete query and +fragment are omitted, including values with unescaped spaces. For an unknown URL embedded in +free text, a secret-named query or fragment still omits the rest of that text: those later +words might be part of the credential, not an explanation. Copied history errors use the same +conservative rule. + Request and response bodies are not captured by network diagnostics. Background requests carry their originating session's weak capture token; failures are not broadcast to other embedded or Python sessions. Layer names, file paths and URL paths can From a9529df443a01b7778ecc046e213d7bea748c915 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Sun, 8 Nov 2026 08:19:59 +0100 Subject: [PATCH 2/8] Fix round-two review findings in address diagnostics --- crates/data/src/diagnostics.rs | 83 ++++++++++++++++++++++++++-------- crates/engine/src/sources.rs | 19 +++++++- crates/tiles/src/cache.rs | 2 +- crates/tiles/src/s3.rs | 7 ++- docs/privacy.md | 5 +- 5 files changed, 90 insertions(+), 26 deletions(-) diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index 524641a638..3a05885eb4 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -561,10 +561,11 @@ fn base64_shaped(word: &str) -> bool { credential_shaped(word) || (word.chars().count() >= 8 && word.chars().skip(1).any(char::is_uppercase)) } -/// A slash before `@` can disguise userinfo as an authority and path. +/// A slash (or a backslash, which URL parsers read as one) before `@` can disguise userinfo as +/// an authority and path. fn ambiguous_userinfo(address: &str) -> bool { let Some((_, rest)) = address.split_once("://") else { return false }; - let Some((authority, path)) = rest.split_once('/') else { return false }; + let Some((authority, path)) = rest.split_once(['/', '\\']) else { return false }; let Some((before_at, _)) = path.split_once('@') else { return false }; !authority.contains('@') && !authority.starts_with('[') && (authority.contains(':') || before_at.contains(':')) } @@ -609,10 +610,18 @@ pub fn address_in_message(address: &str) -> String { // A malformed authority is omitted whole, including words after its first token. shown.truncate("[address omitted]".len()); } + // A redacted path secret whose value had unescaped whitespace may continue into the words + // after it: they belong to this known address, not to diagnostic prose. + if url + && before_query.contains(char::is_whitespace) + && let Some(at) = shown.find(REDACTED).map(|at| at + REDACTED.len()) + { + shown.truncate(at); + } // Keep root authorities separate from the omission marker and the message's `: `: // otherwise another redaction can parse the marker or separator as part of the host/port. if let Some((scheme, rest)) = shown.split_once("://") - && (scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https")) + && ["http", "https", "s3"].iter().any(|s| scheme.eq_ignore_ascii_case(s)) && !rest.is_empty() && !rest.chars().any(|c| c == '/' || c.is_whitespace()) { @@ -699,11 +708,23 @@ pub fn redact_parameter(text: &str) -> String { /// fragment whose value may continue past whitespace. Address-only scrubbing cannot know where /// that value ends, so callers use this result instead of retaining the later words. pub fn secret_continuation_cut(text: &str) -> Option { - let strict = redact(text); - strict.contains("[query and the rest omitted]").then_some(strict) + // Detected before the output is bounded: a long prefix must not cut the marker away. + let mut strict = redact_scan(text, true); + if !strict.contains("[query and the rest omitted]") { + return None; + } + cut_with_ellipsis(&mut strict, MAX_DETAILS_BYTES); + Some(strict) } fn redact_impl(text: &str, diagnostic: bool) -> String { + let mut out = redact_scan(text, diagnostic); + cut_with_ellipsis(&mut out, MAX_DETAILS_BYTES); + out +} + +/// [`redact_impl`] before its output is bounded. +fn redact_scan(text: &str, diagnostic: bool) -> String { if text.len() > MAX_INPUT_BYTES { return "[oversized diagnostic omitted]".to_string(); } @@ -730,21 +751,22 @@ fn redact_impl(text: &str, diagnostic: bool) -> String { i += marker.len(); continue; } - if matches!(c, '{' | '[' | '<') && opens_structured(rest) { - if diagnostic { - out.push_str("[structured content omitted]"); + // A credential tag, attributes or not: its value can span words and lines, so omit the + // remainder whatever else the tag looks like. + if c == '<' { + let inner = rest.get(1..).unwrap_or("").trim_start_matches('/'); + let tag = inner.split(|c: char| c.is_whitespace() || matches!(c, '>' | '/')).next().unwrap_or(""); + if tag.chars().next().is_some_and(|c| c.is_alphabetic() || c == '_') + && inner.get(..256).unwrap_or(inner).contains('>') + && secret_name(tag.rsplit(':').next().unwrap_or(tag)) + { + out.push_str(REDACTED); break; } - // A parameter may legitimately be markup. A credential-bearing tag is not - // innocent syntax: its value can span words and lines, so omit the remainder. - if c == '<' { - let tag = - rest.trim_start_matches(['<', '/']).split(|c: char| c.is_whitespace() || matches!(c, '>' | '/')).next().unwrap_or(""); - if secret_name(tag.rsplit(':').next().unwrap_or(tag)) { - out.push_str(REDACTED); - break; - } - } + } + if matches!(c, '{' | '[' | '<') && opens_structured(rest) && diagnostic { + out.push_str("[structured content omitted]"); + break; } if matches!(c, '"' | '\'') { let quote = c; @@ -792,7 +814,7 @@ fn redact_impl(text: &str, diagnostic: bool) -> String { field_before = false; continue; } - if !diagnostic && ambiguous_userinfo(address.split(['?', '#']).next().unwrap_or("")) { + if ambiguous_userinfo(address.split(['?', '#']).next().unwrap_or("")) { out.push_str("[address omitted]"); break; } @@ -942,7 +964,6 @@ fn redact_impl(text: &str, diagnostic: bool) -> String { i += c.len_utf8(); } } - cut_with_ellipsis(&mut out, MAX_DETAILS_BYTES); out } @@ -1069,10 +1090,30 @@ mod tests { "GET https://alice:hidden@example.org?format=json", "GET https://u:123/w@host/x", "GET https://alice/part:123@host/x", + "GET https://u:123\\w@host/x", + r#"hidden hidden-tail https://example.org"#, ] { let clean = redact_parameter(private); assert!(!clean.contains("hidden") && !clean.contains("alice") && !clean.contains("123"), "{private} -> {clean}"); } + for private in [r#"hidden hidden-tail"#, "GET https://u:123\\w@host/x"] { + let clean = redact(private); + assert!(!clean.contains("hidden") && !clean.contains("123"), "{private} -> {clean}"); + } + // An attribute-only tag that names no credential is still plain markup. + let public = r#" https://example.org "#; + assert_eq!(redact_parameter(public), public); + } + + #[test] + fn known_path_secrets_and_long_prefixes_do_not_keep_continuations() { + let address = "https://example.org/token=hidden hidden-tail/data"; + let clean = message_with_address(address, "HTTP 404"); + assert!(clean.contains("HTTP 404") && !clean.contains("hidden"), "{clean}"); + assert_eq!(redact(&clean), clean); + let long = format!("{} https://example.org/a?token=hidden hidden-tail", "word ".repeat(2000)); + let cut = secret_continuation_cut(&long).expect("the cut is seen past the output bound"); + assert!(!cut.contains("hidden") && cut.len() <= MAX_DETAILS_BYTES + '…'.len_utf8(), "{}", cut.len()); } #[test] @@ -1105,6 +1146,7 @@ mod tests { ("https://models.test:8443", "https://models.test:8443/"), ("http://[::1]:8080#token=hidden value", "http://[::1]:8080/"), ("HTTPS://models.test?token=hidden", "HTTPS://models.test/"), + ("s3://bucket?X-Cred=hidden", "s3://bucket/"), ("https://", "https://"), ] { let mut message = message_with_address(address, "HTTP 503"); @@ -1168,6 +1210,7 @@ mod tests { "https://u:123/w@host/x?token=hidden", "https://alice:hidden/path@host/x", "https://alice/part:123@host/x", + "https://u:123\\w@host/x", ] { let shown = address_in_message(address); let clean = message_with_address(address, &format!("{address}: HTTP 401")); diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index 98203e845e..538df9b8bd 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -333,8 +333,12 @@ pub fn message_with_address(address: &str, reason: &str) -> String { reason.strip_prefix(&shown).is_some_and(|tail| tail.is_empty() || tail.starts_with(": ")) || reason.split_once(": ").is_some_and(|(label, _)| { label.contains("://") - && rustgis_data::io::shapefile_part_urls(address) - .is_some_and(|parts| parts.iter().any(|(_, part)| address_in_message(part) == label)) + && rustgis_data::io::shapefile_part_urls(address).is_some_and(|parts| { + // The part's label may come from either layer's composition. + parts.iter().any(|(_, part)| { + address_in_message(part) == label || rustgis_data::diagnostics::address_in_message(part) == label + }) + }) }) }) { Some(reason) => reason, @@ -739,6 +743,17 @@ MIIE } } + #[test] + fn shapefile_parts_with_secret_queries_are_named_once() { + let shp = "https://h.example/roads.shp?token=hidden"; + let part = "https://h.example/roads.dbf?token=hidden"; + // A part failure as the data layer writes it, passed back through the source's helper. + let label = rustgis_data::diagnostics::message_with_address(part, "http status: 404"); + let message = message_with_address(shp, &label); + assert!(message.contains("roads.dbf") && message.contains("404") && !message.contains("hidden"), "{message}"); + assert_eq!(message.matches("://").count(), 1, "{message}"); + } + #[test] fn secrets_are_never_kept() { assert_eq!(redact("https://me:pw@host.example/a?token=abc&layer=x&X-Amz-Signature=1#k"), "https://host.example/a?layer=x"); diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 68920f41e2..101ff66f2f 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -978,7 +978,7 @@ impl<'a> RequestDiagnostics<'a> { _ => NetworkKind::Other, } }; - let message = failure.message.contains(url).then(|| rustgis_data::diagnostics::message_with_address(url, &failure.message)); + let message = failure.message.starts_with(url).then(|| rustgis_data::diagnostics::message_with_address(url, &failure.message)); self.capture.network_failure( "Network", &clean(message.as_deref().unwrap_or(&failure.message)), diff --git a/crates/tiles/src/s3.rs b/crates/tiles/src/s3.rs index 59817d9976..0f24fc956d 100644 --- a/crates/tiles/src/s3.rs +++ b/crates/tiles/src/s3.rs @@ -389,7 +389,8 @@ pub fn parse(url: &str) -> Result<(String, String), String> { url.get(5..).filter(|_| is_s3(url)).ok_or_else(|| rustgis_data::diagnostics::message_with_address(url, "not an s3:// address"))?; let (bucket, key) = rest.split_once('/').unwrap_or((rest, "")); if !valid_bucket(bucket) { - return Err(rustgis_data::diagnostics::message_with_address(url, &format!("'{bucket}' isn't a bucket name"))); + // The bucket text itself is not repeated: without a slash it may hold the query. + return Err(rustgis_data::diagnostics::message_with_address(url, "the bucket name isn't one S3 takes")); } if key.len() > 1024 || key.chars().any(char::is_control) { return Err(rustgis_data::diagnostics::message_with_address(url, "the key isn't one S3 takes")); @@ -1295,6 +1296,10 @@ mod tests { for bad in ["s3://", "s3://A_B/x", "s3://ab/x", "https://x.org/a", "s3://-ab-/x"] { assert!(parse(bad).is_err(), "{bad}"); } + // A query in place of a key isn't repeated in the error, and the bucket stays readable. + let error = parse("s3://bucket?X-Custom-Cred=hidden").unwrap_err(); + assert!(!error.contains("hidden") && error.contains("s3://bucket/") && error.contains("bucket name"), "{error}"); + assert_eq!(rustgis_data::diagnostics::redact(&error), error); assert_eq!(object_url("my-bucket", "a/b c.tif", None), "https://my-bucket.s3.amazonaws.com/a/b%20c.tif"); assert_eq!(object_url("my.bucket", "k", None), "https://s3.amazonaws.com/my.bucket/k"); let aws = Connection { name: "a".into(), endpoint: None, region: "eu-west-1".into(), buckets: vec![], credentials: None }; diff --git a/docs/privacy.md b/docs/privacy.md index a798cd9a00..1b5ade627c 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -32,8 +32,9 @@ a host, port and path. A root address is shown with `/` before an omission marke so repeated redaction keeps its host. Copied run parameters are sanitized as data, without blanket-omitting JSON/XML syntax. -Public request parameters remain available for replay; credential fields and secret URL -values extending past whitespace are still removed. +Typed addresses lose their keys and credential fields are emptied, but opaque text and nested +values a tool doesn't describe are kept for replay and flagged with a notice: they can still +hold credentials, so review copied code before sharing it. Request and response bodies are not captured by network diagnostics. Background requests carry their originating session's weak capture token; failures are not From 6f0fdad7ed5fc328e669e8392616e4c4b3a3fa76 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Sun, 8 Nov 2026 11:09:05 +0100 Subject: [PATCH 3/8] Fix round-three review findings in address diagnostics --- crates/assistant/src/http.rs | 11 +++++--- crates/data/src/diagnostics.rs | 39 +++++++++++++++++++-------- crates/engine/src/cmd/plugin_fetch.rs | 20 +++++++++++++- crates/tiles/src/cache.rs | 2 +- crates/tiles/src/failure.rs | 8 +++--- docs/privacy.md | 8 +++--- 6 files changed, 64 insertions(+), 24 deletions(-) diff --git a/crates/assistant/src/http.rs b/crates/assistant/src/http.rs index 20650844d2..d965380fa5 100644 --- a/crates/assistant/src/http.rs +++ b/crates/assistant/src/http.rs @@ -78,11 +78,16 @@ fn send_with(req: &HttpRequest, timeout: std::time::Duration, mut on_chunk: Opti Ok(HttpResponse { status, body, headers }) } -/// The scheme and authority of an already-sanitized address, without its omission suffix. +/// The scheme and authority of an already-sanitized address, without its omission suffix. A +/// root address keeps the `/` that separates it from a following `: `; a path is dropped whole. fn host(shown: &str) -> &str { let url = shown.strip_suffix("[query omitted]").unwrap_or(shown); let Some((scheme, rest)) = url.split_once("://") else { return url }; - let end = rest.find('/').map_or(url.len(), |at| scheme.len() + 3 + at + 1); + let end = match rest.find('/') { + Some(at) if at + 1 == rest.len() => url.len(), + Some(at) => scheme.len() + 3 + at, + None => url.len(), + }; url.get(..end).unwrap_or(url) } @@ -136,7 +141,7 @@ mod tests { let error = transport_error(&cause, url); assert_eq!(error.transport, Some(TransportFailure::Interrupted)); let error = diagnostics::redact(&diagnostics::redact(&error.to_string())); - assert!(error.contains("couldn't reach https://models.test") && error.contains("connection closed"), "{error}"); + assert!(error.contains("couldn't reach https://models.test: ") && error.contains("connection closed"), "{error}"); for hidden in ["user", "password", "private-key", "secret", "value"] { assert!(!error.contains(hidden), "{error}"); } diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index 3a05885eb4..572062155a 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -637,19 +637,35 @@ pub fn address_in_message(address: &str) -> String { /// the reason after a secret query. Exact address occurrences (including unescaped whitespace) /// are sanitized first; unrelated URLs and credential fields still use conservative [`redact`]. pub fn message_with_address(address: &str, reason: &str) -> String { - if address.is_empty() { - return redact(reason); - } - if reason.len() > MAX_INPUT_BYTES { + if address.is_empty() || reason.len() > MAX_INPUT_BYTES { return redact(reason); } let shown = address_in_message(address); + let reason = address_occurrences_sanitized(address, reason, &shown); + let named = reason + .strip_prefix(&shown) + .is_some_and(|tail| tail.is_empty() || tail.starts_with(':') || tail.chars().next().is_some_and(char::is_whitespace)); + if named { redact(&reason) } else { redact(&format!("{shown}: {reason}")) } +} + +/// [`message_with_address`] without naming the address in front: for text that already says +/// where it failed, anywhere in it (a transport's `HTTP 503 from …`). +pub fn with_address_sanitized(address: &str, text: &str) -> String { + if address.is_empty() || text.len() > MAX_INPUT_BYTES { + return redact(text); + } + redact(&address_occurrences_sanitized(address, text, &address_in_message(address))) +} + +/// `reason` with each exact occurrence of `address` (or its runtime binding's inner resource) +/// replaced by `shown`, before the conservative scanner sees its unescaped secret whitespace. +fn address_occurrences_sanitized(address: &str, reason: &str, shown: &str) -> String { let mut safe_reason = String::with_capacity(reason.len()); let mut copied = 0; // Lower-level transports may name the inner resource of a runtime binding instead. let inner = crate::login_address::inner(crate::s3_address::inner(address)); let mut outer_matches = reason.match_indices(address).peekable(); - let mut inner_matches = reason.match_indices(inner).filter(|_| inner != address).peekable(); + let mut inner_matches = reason.match_indices(inner).filter(|_| inner != address && !inner.is_empty()).peekable(); loop { let occurrence = match (outer_matches.peek(), inner_matches.peek()) { (Some(outer), Some(inner)) if outer.0 <= inner.0 => outer_matches.next(), @@ -674,16 +690,12 @@ pub fn message_with_address(address: &str, reason: &str) -> String { // A known address must not match the prefix of another address's longer secret. if begins && ends { safe_reason.push_str(reason.get(copied..start).unwrap_or("")); - safe_reason.push_str(&shown); + safe_reason.push_str(shown); copied = end; } } safe_reason.push_str(reason.get(copied..).unwrap_or("")); - let reason = safe_reason; - let named = reason - .strip_prefix(&shown) - .is_some_and(|tail| tail.is_empty() || tail.starts_with(':') || tail.chars().next().is_some_and(char::is_whitespace)); - if named { redact(&reason) } else { redact(&format!("{shown}: {reason}")) } + safe_reason } /// Sanitizes before bounding. Oversized input is omitted *whole*, so a secret can never be @@ -1170,6 +1182,11 @@ mod tests { history.push(Severity::Error, "Network", &clean, &clean); assert!(history.report(Filter::All).contains("HTTP 404 fetching")); assert!(!history.report(Filter::All).contains("hidden")); + // Text that already names where it failed keeps its own wording, without a label in front. + let clean = with_address_sanitized(address, &format!("HTTP 503 from {address}; retrying")); + assert!(clean.starts_with("HTTP 503 from https://example.org/data") && clean.ends_with("; retrying"), "{clean}"); + assert!(!clean.contains("hidden") && !clean.contains("alice"), "{clean}"); + assert_eq!(clean.matches("://").count(), 1, "{clean}"); } #[test] diff --git a/crates/engine/src/cmd/plugin_fetch.rs b/crates/engine/src/cmd/plugin_fetch.rs index 0a0193cc9f..99df668ec6 100644 --- a/crates/engine/src/cmd/plugin_fetch.rs +++ b/crates/engine/src/cmd/plugin_fetch.rs @@ -155,7 +155,15 @@ pub fn checked(url: &str) -> Result { } let lower = url.to_ascii_lowercase(); if !(lower.starts_with("https://") || lower.starts_with("http://")) { - let shown: String = crate::sources::address_in_message(url).chars().take(80).collect(); + let full = crate::sources::address_in_message(url); + let mut shown: String = full.chars().take(80).collect(); + if shown.len() < full.len() { + // Never half an omission marker: cut before one the limit would split. + if let Some(at) = shown.rfind('[').filter(|&at| !shown.get(at..).unwrap_or("").contains(']')) { + shown.truncate(at); + } + shown.push('…'); + } return Err(crate::sources::message_with_address(&shown, "plugins fetch only http:// and https:// addresses")); } // A fragment never reaches the server (and RustGIS marks byte ranges with one). @@ -949,6 +957,16 @@ impl Session { mod tests { use super::*; + /// A long refused address is shortened without cutting an omission marker in half. + #[test] + fn a_shortened_refused_address_keeps_whole_markers() { + let url = format!("ftp://h.example/{}?page=1&token=hidden", "a".repeat(52)); + let error = checked(&url).unwrap_err(); + assert!(!error.contains("hidden") && error.contains("plugins fetch only"), "{error}"); + let unclosed = error.rfind('[').is_some_and(|at| !error.get(at..).unwrap_or("").contains(']')); + assert!(!unclosed, "{error}"); + } + /// A failure answers once a run, in the order it came, and a call made again after it is a new /// download: a command trying again gets its next try, not the same failure for ever. What was /// added answers as often as it is asked for (#1319). diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index dc20a7d209..70d745a040 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -983,7 +983,7 @@ impl<'a> RequestDiagnostics<'a> { _ => NetworkKind::Other, } }; - let message = failure.message.starts_with(url).then(|| rustgis_data::diagnostics::message_with_address(url, &failure.message)); + let message = failure.message.contains(url).then(|| rustgis_data::diagnostics::with_address_sanitized(url, &failure.message)); self.capture.network_failure( "Network", &clean(message.as_deref().unwrap_or(&failure.message)), diff --git a/crates/tiles/src/failure.rs b/crates/tiles/src/failure.rs index 3d204778bd..5e09ff3f17 100644 --- a/crates/tiles/src/failure.rs +++ b/crates/tiles/src/failure.rs @@ -69,10 +69,10 @@ impl Failure { pub fn new(url: &str, message: impl Into) -> Self { let message = message.into(); let reason = crate::cache::http_status(&message).filter(|s| *s >= 400).map_or(Reason::Other, Reason::Status); - // A known leading address is sanitized whole first, so its unescaped secret value cannot - // be mistaken for the reason or force the conservative cut that drops the reason. - let message = if !url.is_empty() && message.starts_with(url) { - rustgis_data::diagnostics::message_with_address(url, &message) + // Known address occurrences, wherever they are, are sanitized whole first, so an + // unescaped secret value can't be mistaken for the reason or force the cut that drops it. + let message = if !url.is_empty() && message.contains(url) { + rustgis_data::diagnostics::with_address_sanitized(url, &message) } else { message }; diff --git a/docs/privacy.md b/docs/privacy.md index 1b5ade627c..7f50dc9dcd 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -26,10 +26,10 @@ Notices and request errors redact a known address before attaching its failure r token or signature does not hide the HTTP status or other explanation. The complete query and fragment are omitted, including values with unescaped spaces. For an unknown URL embedded in free text, a secret-named query or fragment still omits the rest of that text: those later -words might be part of the credential, not an explanation. Copied history errors use the same -conservative rule. Ambiguous slash-containing userinfo is omitted rather than guessed to be -a host, port and path. A root address is shown with `/` before an omission marker or reason, -so repeated redaction keeps its host. +words might be part of the credential, not an explanation. The same rule applies to copied +history errors, saved run messages and notebook cell output. Ambiguous slash-containing userinfo +is omitted rather than guessed to be a host, port and path. A root address is shown with `/` +before an omission marker or reason, so repeated redaction keeps its host. Copied run parameters are sanitized as data, without blanket-omitting JSON/XML syntax. Typed addresses lose their keys and credential fields are emptied, but opaque text and nested From 2d8b7ef13db615aa6a94daee36c2c4cb7a2b9040 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Mon, 9 Nov 2026 20:21:53 +0100 Subject: [PATCH 4/8] Say what an ambiguous address omission drops --- crates/data/src/diagnostics.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index 572062155a..68885816f5 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -753,6 +753,7 @@ fn redact_scan(text: &str, diagnostic: bool) -> String { REDACTED, "[structured content omitted]", "[oversized diagnostic omitted]", + "[address and the rest omitted]", "[address omitted]", "[oversized address omitted]", ] @@ -827,7 +828,9 @@ fn redact_scan(text: &str, diagnostic: bool) -> String { continue; } if ambiguous_userinfo(address.split(['?', '#']).next().unwrap_or("")) { - out.push_str("[address omitted]"); + // Its password may run on past whitespace, as a secret query's may: the rest + // goes too, and the marker says so. + out.push_str("[address and the rest omitted]"); break; } let query = address.split_once(['?', '#']).map_or("", |(_, q)| q); @@ -1112,6 +1115,10 @@ mod tests { let clean = redact(private); assert!(!clean.contains("hidden") && !clean.contains("123"), "{private} -> {clean}"); } + // An ambiguous address in prose says the rest of the text went with it. + let clean = redact("upload https://u:123/w@h/x failed: HTTP 500"); + assert_eq!(clean, "upload [address and the rest omitted]"); + assert_eq!(redact(&clean), clean); // An attribute-only tag that names no credential is still plain markup. let public = r#" https://example.org "#; assert_eq!(redact_parameter(public), public); From 6fb5f4ce76bef7ce7facb302bac28ec87399ec1e Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Mon, 9 Nov 2026 23:44:08 +0100 Subject: [PATCH 5/8] Cut saved text on every rest-omitting marker and say when only a fragment went --- crates/data/src/diagnostics.rs | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index ac01e94808..3d6f3699c3 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -747,9 +747,13 @@ pub fn redact_parameter(text: &str) -> String { /// fragment whose value may continue past whitespace. Address-only scrubbing cannot know where /// that value ends, so callers use this result instead of retaining the later words. pub fn secret_continuation_cut(text: &str) -> Option { - // Detected before the output is bounded: a long prefix must not cut the marker away. + // Detected before the output is bounded: a long prefix must not cut the marker away. Text + // too long to scan can't be shown to end its addresses' values, so with an address in it + // the whole text goes, said. let mut strict = redact_scan(text, true); - if !strict.contains("[query and the rest omitted]") { + let cut = ["[query and the rest omitted]", "[address and the rest omitted]"].iter().any(|m| strict.contains(m)) + || (text.len() > MAX_INPUT_BYTES && text.contains("://")); + if !cut { return None; } cut_with_ellipsis(&mut strict, MAX_DETAILS_BYTES); @@ -912,7 +916,9 @@ fn redact_scan(text: &str, diagnostic: bool) -> String { out.push_str("[query and the rest omitted]"); break; } - out.push_str("[query omitted]"); + // Said as what went: a query (always in a diagnostic), or only a fragment. + let query_removed = diagnostic && address.split('#').next().unwrap_or("").contains('?'); + out.push_str(if query_removed { "[query omitted]" } else { "[fragment omitted]" }); if let Some(end) = address.chars().last().filter(|c| matches!(c, ':' | ',' | ';' | ')' | '.')) { out.push(end); } @@ -1122,6 +1128,14 @@ mod tests { assert_eq!(redact_parameter(public), public); assert_eq!(redact(public), "[structured content omitted]"); } + // A replay keeps a public query; only its fragment goes, and the marker says which. + let link = "see https://example.org/a?format=json#frag then"; + let kept = redact_parameter(link); + assert_eq!(kept, "see https://example.org/a?format=json[fragment omitted] then"); + assert_eq!(redact_parameter(&kept), kept); + assert_eq!(redact(link), "see https://example.org/a[query omitted] then"); + assert_eq!(redact_parameter("see https://example.org/a#frag then"), "see https://example.org/a[fragment omitted] then"); + assert_eq!(redact("see https://example.org/a#frag then"), "see https://example.org/a[fragment omitted] then"); for private in [ "hidden hidden-tail https://example.org", "hidden hidden-tail https://example.org", @@ -1160,6 +1174,15 @@ mod tests { let long = format!("{} https://example.org/a?token=hidden hidden-tail", "word ".repeat(2000)); let cut = secret_continuation_cut(&long).expect("the cut is seen past the output bound"); assert!(!cut.contains("hidden") && cut.len() <= MAX_DETAILS_BYTES + '…'.len_utf8(), "{}", cut.len()); + // An ambiguous address cuts as a secret query does. + let ambiguous = "run failed at https://u:123/w@h/x hidden-tail: HTTP 500"; + assert_eq!(secret_continuation_cut(ambiguous).as_deref(), Some("run failed at [address and the rest omitted]")); + // Text too long to scan goes whole when it holds an address; without one it is left + // to the address scrubber, which has nothing to do. + let oversized = format!("{} https://example.org/a?token=hidden hidden-tail", "word ".repeat(4000)); + assert_eq!(secret_continuation_cut(&oversized).as_deref(), Some("[oversized diagnostic omitted]")); + assert_eq!(secret_continuation_cut(&"word ".repeat(4000)), None); + assert_eq!(secret_continuation_cut("failed at https://example.org/a: HTTP 500"), None); } #[test] From c3ce68add793ba6900f474192bdf43ffc08c9670 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Tue, 10 Nov 2026 06:40:55 +0100 Subject: [PATCH 6/8] Close the ? and # userinfo gap, narrow the oversized cut, and label fragment-only addresses --- crates/assistant/src/http.rs | 2 +- crates/data/src/diagnostics.rs | 60 ++++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/crates/assistant/src/http.rs b/crates/assistant/src/http.rs index d965380fa5..51512945f8 100644 --- a/crates/assistant/src/http.rs +++ b/crates/assistant/src/http.rs @@ -81,7 +81,7 @@ fn send_with(req: &HttpRequest, timeout: std::time::Duration, mut on_chunk: Opti /// The scheme and authority of an already-sanitized address, without its omission suffix. A /// root address keeps the `/` that separates it from a following `: `; a path is dropped whole. fn host(shown: &str) -> &str { - let url = shown.strip_suffix("[query omitted]").unwrap_or(shown); + let url = shown.strip_suffix("[query omitted]").or_else(|| shown.strip_suffix("[fragment omitted]")).unwrap_or(shown); let Some((scheme, rest)) = url.split_once("://") else { return url }; let end = match rest.find('/') { Some(at) if at + 1 == rest.len() => url.len(), diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index 3d6f3699c3..8b54c88960 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -589,14 +589,32 @@ fn base64_shaped(word: &str) -> bool { } /// A slash (or a backslash, which URL parsers read as one) before `@` can disguise userinfo as -/// an authority and path. +/// an authority and path; a `?` or `#` before an `@` in the authority disguises it as a query +/// or fragment. Takes the whole address, query included. fn ambiguous_userinfo(address: &str) -> bool { let Some((_, rest)) = address.split_once("://") else { return false }; + // `u:123?w@host`: the password ends at the `@`, not at the `?` a parser stops at. + let raw_authority = rest.split(['/', '\\']).next().unwrap_or(""); + if raw_authority.split_once('@').is_some_and(|(before, _)| before.contains(['?', '#'])) { + return true; + } + let rest = rest.split(['?', '#']).next().unwrap_or(""); let Some((authority, path)) = rest.split_once(['/', '\\']) else { return false }; let Some((before_at, _)) = path.split_once('@') else { return false }; !authority.contains('@') && !authority.starts_with('[') && (authority.contains(':') || before_at.contains(':')) } +/// Whether a word of text too long to scan names an address whose query or userinfo may hold +/// a credential: a linear look at each `://` word, without the full scanner. +fn may_hold_secret(text: &str) -> bool { + text.split_whitespace().filter(|word| word.contains("://")).any(|word| { + ambiguous_userinfo(word) + || word + .split_once(['?', '#']) + .is_some_and(|(_, query)| query.split(['&', ';', '#']).any(|pair| secret_name(pair.split('=').next().unwrap_or("")))) + }) +} + /// Sanitizes a known address on its own, before composing it with diagnostic prose. /// Unlike a raw URL with a secret query, this result can be redacted again without losing /// the reason that follows it. This does not establish a boundary for unknown URLs in prose. @@ -626,7 +644,7 @@ pub fn address_in_message(address: &str) -> String { return "[address omitted]".to_string(); } // Do not guess which interpretation of a slash-containing authority held a password. - if ambiguous_userinfo(before_query) { + if ambiguous_userinfo(address) { return "[address omitted]".to_string(); } let url = address.contains("://"); @@ -654,8 +672,8 @@ pub fn address_in_message(address: &str) -> String { { shown.push('/'); } - if url && before_query.len() < address.len() { - shown.push_str("[query omitted]"); + if url && let Some(cut) = address.get(before_query.len()..).filter(|cut| !cut.is_empty()) { + shown.push_str(if cut.starts_with('#') { "[fragment omitted]" } else { "[query omitted]" }); } shown } @@ -748,12 +766,13 @@ pub fn redact_parameter(text: &str) -> String { /// that value ends, so callers use this result instead of retaining the later words. pub fn secret_continuation_cut(text: &str) -> Option { // Detected before the output is bounded: a long prefix must not cut the marker away. Text - // too long to scan can't be shown to end its addresses' values, so with an address in it - // the whole text goes, said. + // too long to scan goes whole only when one of its addresses may hold a credential, and the + // marker says how much went and why; other long text keeps its words. + if text.len() > MAX_INPUT_BYTES { + return may_hold_secret(text).then(|| format!("[{} bytes of text omitted: too long to check its addresses]", text.len())); + } let mut strict = redact_scan(text, true); - let cut = ["[query and the rest omitted]", "[address and the rest omitted]"].iter().any(|m| strict.contains(m)) - || (text.len() > MAX_INPUT_BYTES && text.contains("://")); - if !cut { + if !["[query and the rest omitted]", "[address and the rest omitted]"].iter().any(|m| strict.contains(m)) { return None; } cut_with_ellipsis(&mut strict, MAX_DETAILS_BYTES); @@ -858,7 +877,7 @@ fn redact_scan(text: &str, diagnostic: bool) -> String { field_before = false; continue; } - if ambiguous_userinfo(address.split(['?', '#']).next().unwrap_or("")) { + if ambiguous_userinfo(address) { // Its password may run on past whitespace, as a secret query's may: the rest // goes too, and the marker says so. out.push_str("[address and the rest omitted]"); @@ -1177,10 +1196,16 @@ mod tests { // An ambiguous address cuts as a secret query does. let ambiguous = "run failed at https://u:123/w@h/x hidden-tail: HTTP 500"; assert_eq!(secret_continuation_cut(ambiguous).as_deref(), Some("run failed at [address and the rest omitted]")); - // Text too long to scan goes whole when it holds an address; without one it is left - // to the address scrubber, which has nothing to do. + // Text too long to scan goes whole, said with its size, only when an address in it may + // hold a credential; a long text with a public link keeps its words. let oversized = format!("{} https://example.org/a?token=hidden hidden-tail", "word ".repeat(4000)); - assert_eq!(secret_continuation_cut(&oversized).as_deref(), Some("[oversized diagnostic omitted]")); + assert_eq!( + secret_continuation_cut(&oversized), + Some(format!("[{} bytes of text omitted: too long to check its addresses]", oversized.len())) + ); + let ambiguous = format!("{} https://u:123?w@host/x tail", "word ".repeat(4000)); + assert!(secret_continuation_cut(&ambiguous).is_some()); + assert_eq!(secret_continuation_cut(&format!("{} see https://example.org/a?page=2 then", "word ".repeat(4000))), None); assert_eq!(secret_continuation_cut(&"word ".repeat(4000)), None); assert_eq!(secret_continuation_cut("failed at https://example.org/a: HTTP 500"), None); } @@ -1284,6 +1309,8 @@ mod tests { "https://u:123/w@host/x?token=hidden", "https://alice:hidden/path@host/x", "https://alice/part:123@host/x", + "https://u:123?w@host/x", + "https://u:123#w@host/x", "https://u:123\\w@host/x", ] { let shown = address_in_message(address); @@ -1304,6 +1331,13 @@ mod tests { assert_eq!(address_in_message(address), address); assert_eq!(message_with_address(address, "HTTP 503"), format!("{address}: HTTP 503")); } + // An `@` in a query after a path is no userinfo: the host and port stay readable. + let address = "https://h:80/x?email=a@b"; + assert_eq!(message_with_address(address, "HTTP 503"), "https://h:80/x[query omitted]: HTTP 503"); + // A fragment alone is said as one, and the label survives redacting again. + let shown = address_in_message("https://h.example/a#frag"); + assert_eq!(shown, "https://h.example/a[fragment omitted]"); + assert_eq!(redact(&shown), shown); let clean = message_with_address("https://alice:123%2Fw@host/x", "HTTP 503"); assert!(clean.contains("HTTP 503") && !clean.contains("alice") && !clean.contains("123"), "{clean}"); } From 6100a0e2c49123ccd1aabb40f498685dad9bbd18 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Tue, 10 Nov 2026 07:21:55 +0100 Subject: [PATCH 7/8] Keep the omission marker through the saved-text cut, and cut scheme-less queries --- crates/data/src/diagnostics.rs | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index 8b54c88960..cd6215043e 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -637,7 +637,10 @@ pub fn address_in_message(address: &str) -> String { } // A malformed userinfo value can contain a URL-token boundary before its `@`. The generic // scanner cannot know those later words belong to the address; here the whole input does. - let before_query = address.split(['?', '#']).next().unwrap_or(""); + // A web address keeps neither query nor fragment. A scheme-less one (often a file path, + // where `#` is part of a name) is cut at its query only. + let url = address.contains("://"); + let before_query = if url { address.split(['?', '#']).next() } else { address.split('?').next() }.unwrap_or(""); if before_query.contains('@') && before_query.chars().any(|c| c.is_whitespace() || hostile_control(c) || matches!(c, '"' | '\'' | '<' | '>' | '`')) { @@ -647,10 +650,9 @@ pub fn address_in_message(address: &str) -> String { if ambiguous_userinfo(address) { return "[address omitted]".to_string(); } - let url = address.contains("://"); // Every remaining byte belongs to this known address, even if a value has unescaped // whitespace and an unfamiliar signing name. None of that suffix is diagnostic prose. - let mut shown = redact(if url { before_query } else { address }); + let mut shown = redact(before_query); if url && shown.starts_with("[address omitted]") { // A malformed authority is omitted whole, including words after its first token. shown.truncate("[address omitted]".len()); @@ -672,7 +674,7 @@ pub fn address_in_message(address: &str) -> String { { shown.push('/'); } - if url && let Some(cut) = address.get(before_query.len()..).filter(|cut| !cut.is_empty()) { + if let Some(cut) = address.get(before_query.len()..).filter(|cut| !cut.is_empty()) { shown.push_str(if cut.starts_with('#') { "[fragment omitted]" } else { "[query omitted]" }); } shown @@ -772,10 +774,12 @@ pub fn secret_continuation_cut(text: &str) -> Option { return may_hold_secret(text).then(|| format!("[{} bytes of text omitted: too long to check its addresses]", text.len())); } let mut strict = redact_scan(text, true); - if !["[query and the rest omitted]", "[address and the rest omitted]"].iter().any(|m| strict.contains(m)) { - return None; - } - cut_with_ellipsis(&mut strict, MAX_DETAILS_BYTES); + // Cut what comes before the marker and keep the marker (and anything after it) whole, so + // the cut text still says what went. + let at = ["[query and the rest omitted]", "[address and the rest omitted]"].iter().filter_map(|m| strict.rfind(m)).max()?; + let tail = strict.split_off(at); + cut_with_ellipsis(&mut strict, MAX_DETAILS_BYTES.saturating_sub(tail.len())); + strict.push_str(&tail); Some(strict) } @@ -1193,6 +1197,9 @@ mod tests { let long = format!("{} https://example.org/a?token=hidden hidden-tail", "word ".repeat(2000)); let cut = secret_continuation_cut(&long).expect("the cut is seen past the output bound"); assert!(!cut.contains("hidden") && cut.len() <= MAX_DETAILS_BYTES + '…'.len_utf8(), "{}", cut.len()); + // The marker sits past the 8 KiB bound in the scan; the cut keeps it whole at the end. + assert!(cut.ends_with("…[query and the rest omitted]"), "{}", cut.get(cut.len().saturating_sub(60)..).unwrap_or("")); + assert_eq!(redact(&cut), cut); // An ambiguous address cuts as a secret query does. let ambiguous = "run failed at https://u:123/w@h/x hidden-tail: HTTP 500"; assert_eq!(secret_continuation_cut(ambiguous).as_deref(), Some("run failed at [address and the rest omitted]")); @@ -1222,6 +1229,9 @@ mod tests { "https://example.org/data#sig=hidden unescaped hidden", "https://example.org/data?custom=hidden unescaped hidden", "https://example.org/data#hidden unescaped hidden", + // A scheme-less address loses its whole query too, unfamiliar keys included. + "data.tif?custom=hidden unescaped hidden", + "data.tif?token=hidden unescaped hidden", ] { let clean = message_with_address(address, "HTTP 503 Service Unavailable"); assert!(clean.contains("HTTP 503 Service Unavailable"), "{address} -> {clean}"); @@ -1231,6 +1241,9 @@ mod tests { assert_eq!(message_with_address(address, &format!("{address}: HTTP 503 Service Unavailable")), clean); assert_eq!(message_with_address(address, &clean), clean); } + // A scheme-less address says its query went; a `#` in a file name is part of the name. + assert_eq!(address_in_message("data.tif?custom=hidden unescaped hidden"), "data.tif[query omitted]"); + assert_eq!(message_with_address("/data/plot#1.gpkg", "no such table"), "/data/plot#1.gpkg: no such table"); } #[test] From dac84ba33da61557abf7df1efa3160d2b7f1356e Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Tue, 10 Nov 2026 08:13:42 +0100 Subject: [PATCH 8/8] Shorten refused addresses without losing their omission marker --- crates/data/src/diagnostics.rs | 6 +++++ crates/engine/src/cmd/plugin_fetch.rs | 20 +++++---------- crates/engine/src/cmd/tiles.rs | 26 +++++++++++++++++--- crates/engine/src/sources.rs | 35 +++++++++++++++++++++++++-- 4 files changed, 68 insertions(+), 19 deletions(-) diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index cd6215043e..ff8ea84831 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -1244,6 +1244,12 @@ mod tests { // A scheme-less address says its query went; a `#` in a file name is part of the name. assert_eq!(address_in_message("data.tif?custom=hidden unescaped hidden"), "data.tif[query omitted]"); assert_eq!(message_with_address("/data/plot#1.gpkg", "no such table"), "/data/plot#1.gpkg: no such table"); + // A secret after a scheme-less `#` is still a field, and goes; a `#` before `?` stays in + // the path while the query is cut. + let shown = address_in_message("host/data#token=hidden"); + assert!(shown.starts_with("host/data#") && !shown.contains("hidden"), "{shown}"); + assert_eq!(redact(&shown), shown); + assert_eq!(address_in_message("/data/plot#1.gpkg?token=hidden"), "/data/plot#1.gpkg[query omitted]"); } #[test] diff --git a/crates/engine/src/cmd/plugin_fetch.rs b/crates/engine/src/cmd/plugin_fetch.rs index b8322e0ff1..7e5466ace3 100644 --- a/crates/engine/src/cmd/plugin_fetch.rs +++ b/crates/engine/src/cmd/plugin_fetch.rs @@ -155,16 +155,7 @@ pub fn checked(url: &str) -> Result { } let lower = url.to_ascii_lowercase(); if !(lower.starts_with("https://") || lower.starts_with("http://")) { - let full = crate::sources::address_in_message(url); - let mut shown: String = full.chars().take(80).collect(); - if shown.len() < full.len() { - // Never half an omission marker: cut before one the limit would split. - if let Some(at) = shown.rfind('[').filter(|&at| !shown.get(at..).unwrap_or("").contains(']')) { - shown.truncate(at); - } - shown.push('…'); - } - return Err(crate::sources::message_with_address(&shown, "plugins fetch only http:// and https:// addresses")); + return Err(crate::sources::shortened_message_with_address(url, 80, "plugins fetch only http:// and https:// addresses")); } // A fragment never reaches the server (and RustGIS marks byte ranges with one). let url = url.split('#').next().unwrap_or(url); @@ -968,14 +959,15 @@ impl Session { mod tests { use super::*; - /// A long refused address is shortened without cutting an omission marker in half. + /// A long refused address is shortened without cutting an omission marker in half: the + /// address before it is cut instead, and its query stays out. #[test] fn a_shortened_refused_address_keeps_whole_markers() { let url = format!("ftp://h.example/{}?page=1&token=hidden", "a".repeat(52)); let error = checked(&url).unwrap_err(); - assert!(!error.contains("hidden") && error.contains("plugins fetch only"), "{error}"); - let unclosed = error.rfind('[').is_some_and(|at| !error.get(at..).unwrap_or("").contains(']')); - assert!(!unclosed, "{error}"); + assert!(!error.contains("hidden") && !error.contains("page=1"), "{error}"); + assert!(error.ends_with("a…[query omitted]: plugins fetch only http:// and https:// addresses"), "{error}"); + assert_eq!(error.split(": plugins").next().map(|a| a.chars().count()), Some(80), "{error}"); } /// A failure answers once a run, in the order it came, and a call made again after it is a new diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index f047f3690b..7622f3da80 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -865,9 +865,9 @@ fn legends(s: &mut Session, connection: Option<&str>, v: &Value) -> Result bool { /// A known source address safe to compose into a repeatedly redacted diagnostic, without /// publishing a web-login runtime binding or the credentials inside a source wrapper. pub fn address_in_message(address: &str) -> String { - rustgis_data::diagnostics::address_in_message(&redact(address)) + let mut shown = rustgis_data::diagnostics::address_in_message(&redact(address)); + // Source redaction can take out a secret query whole; the diagnostic label still says one went. + let raw = rustgis_data::diagnostics::address_in_message(address); + if let Some(marker) = ["[query omitted]", "[fragment omitted]"].into_iter().find(|m| raw.ends_with(m)) + && !shown.ends_with(marker) + && !shown.ends_with("omitted]") + { + shown.push_str(marker); + } + shown +} + +/// An error naming a long known address shortened to `limit` characters, then `reason` (fixed +/// prose, not from the network). A trailing omission marker (`[query omitted]`) is kept whole +/// and the address before it is cut instead, marked `…`; a cut never leaves half a marker. The +/// shortened address is already sanitized, so it isn't parsed again as an address. +pub fn shortened_message_with_address(address: &str, limit: usize, reason: &str) -> String { + let full = address_in_message(address); + if full.chars().count() <= limit { + return message_with_address(address, reason); + } + let (head, marker) = match full.rfind('[') { + Some(at) if full.ends_with(']') => full.split_at(at), + _ => (full.as_str(), ""), + }; + let mut shown: String = head.chars().take(limit.saturating_sub(marker.chars().count() + 1)).collect(); + if let Some(at) = shown.rfind('[').filter(|&at| !shown.get(at..).unwrap_or("").contains(']')) { + shown.truncate(at); + } + shown.push('…'); + shown.push_str(marker); + format!("{shown}: {reason}") } /// Names a known source while keeping its failure reason. Exact address occurrences are @@ -859,7 +890,7 @@ MIIE assert_eq!(crate::Session::redacted(text).as_str(), text); // An error naming the address it refused keeps those words, and its key stays out. let error = message_with_address("https://h.example/a?token=hidden", text); - assert_eq!(error, format!("https://h.example/a: {text}")); + assert_eq!(error, format!("https://h.example/a[query omitted]: {text}")); assert_eq!(crate::Session::redacted(&error).as_str(), error); } // Anything after the scheme is an address: what can hold a credential goes. A hint's