diff --git a/apps/rustgis-web/src/request.rs b/apps/rustgis-web/src/request.rs index 8c094ddb8d..0909c3c631 100644 --- a/apps/rustgis-web/src/request.rs +++ b/apps/rustgis-web/src/request.rs @@ -35,10 +35,11 @@ pub(crate) async fn prepare_and_send> Ok(send(prepared)) } -/// Formats a fetch failure without exposing the authenticated request's credentials. +/// Formats a fetch failure without exposing the authenticated request's credentials. The +/// address is sanitized on its own first, so a secret query doesn't take the reason with it. pub(crate) fn fetch_error(url: &str, error: String, authenticated: bool) -> String { - use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::logins; - if authenticated && cache::http_status(&error).is_none() { logins::scrub_error(&format!("{url}: {error}")) } else { error } + use rustgis_ui_egui::rustgis_engine::{data::diagnostics::message_with_address, render::rustgis_tiles::logins}; + if authenticated && cache::http_status(&error).is_none() { logins::scrub_error(&message_with_address(url, &error)) } else { error } } /// Checks the browser's normalized destination under the caller's inherited ceiling. diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index c89bf81b73..d94e6e46ad 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -5,7 +5,7 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; use std::rc::Rc; -use rustgis_ui_egui::rustgis_engine::data::diagnostics::Capture; +use rustgis_ui_egui::rustgis_engine::{data::diagnostics::Capture, sources::message_with_address}; use rustgis_ui_egui::{Inbox, RustGisApp, Services, Slot}; use crate::parts::Notes; @@ -164,7 +164,7 @@ pub fn start() { // to different maps depending on when the import completes. if let Some(url) = &project { if let Err(e) = app.run("ui.openUrl", serde_json::json!({"url": url})) { - app.session.notify(format!("?project={url}: {e}"), true); + app.session.notify(format!("?project={}", message_with_address(url, &e)), true); } let skipped: Vec<&str> = ["data", "stac", "style", "tiles", "zarr"].into_iter().filter(|k| !params(&q, k, 1).is_empty()).collect(); @@ -199,7 +199,7 @@ pub fn start() { // in a notice. for url in params(links, "data", 16) { if let Err(e) = app.run("ui.addUrl", serde_json::json!({"url": url, "zoom": !linked})) { - app.session.notify(format!("?data={url}: {e}"), true); + app.session.notify(format!("?data={}", message_with_address(&url, &e)), true); } } // `?stac=`: a STAC item (its default asset is added, signed when it is a @@ -210,7 +210,7 @@ pub fn start() { if let Some(url) = stac.first() && let Err(e) = app.run("ui.stacLink", serde_json::json!({"url": url})) { - app.session.notify(format!("?stac={url}: {e}"), true); + app.session.notify(format!("?stac={}", message_with_address(url, &e)), true); } if stac.len() > 1 { app.session.notify(format!("only the first ?stac= link is opened ({} more were left out)", stac.len() - 1), false); @@ -228,7 +228,7 @@ pub fn start() { match app.run("map.addTiles", p) { // The first style added opens at its view, not one that failed. Ok(_) => style_view &= kind.is_none(), - Err(e) => app.session.notify(format!("?{key}={url}: {e}"), true), + Err(e) => app.session.notify(format!("?{key}={}", message_with_address(&url, &e)), true), } } } @@ -237,7 +237,7 @@ pub fn start() { if let Some(url) = params(links, "zarr", 1).into_iter().next() && let Err(e) = app.run("ui.addZarr", serde_json::json!({"address": url})) { - app.session.notify(format!("?zarr={url}: {e}"), true); + app.session.notify(format!("?zarr={}", message_with_address(&url, &e)), true); } // `?registry=` opens the Plugin Manager on another plugin registry for this // visit (a pull request's preview, say); installing still asks as usual. @@ -735,7 +735,7 @@ async fn fetch_reached( // holds the whole file (a small archive), checked once it's read; else no ranges. let whole = range.is_some() && resp.status() != 206; if range.is_some_and(|(a, _)| whole && a > 0) { - return Err(format!("{url}: {}", cache::NO_RANGES)); + return Err(message_with_address(&url, cache::NO_RANGES)); } // Fonts are at most about 10 MB and plugins 64 MB; refuse anything larger than allowed, by // header when there is one and by the downloaded size otherwise. @@ -744,7 +744,7 @@ async fn fetch_reached( { // The last bytes asked for and a longer file answered: no ranges, so it is downloaded whole. if whole_tail.is_some() { - return Err(format!("{url}: {}", cache::NO_RANGES)); + return Err(message_with_address(&url, cache::NO_RANGES)); } return Err(format!("{len} bytes is too large")); } @@ -772,7 +772,7 @@ async fn fetch_reached( if (bytes.len() + n) as u64 > max { let _ = reader.cancel(); if whole_tail.is_some() { - return Err(format!("{url}: {}", cache::NO_RANGES)); + return Err(message_with_address(&url, cache::NO_RANGES)); } return Err(format!("larger than {max} bytes")); } @@ -784,10 +784,10 @@ async fn fetch_reached( if let Some((a, b)) = range.filter(|_| whole) && bytes.len() as u64 > b - a + 1 { - return Err(format!("{url}: {}", cache::NO_RANGES)); + return Err(message_with_address(&url, cache::NO_RANGES)); } if whole_tail.is_some_and(|n| bytes.len() as u64 > n) { - return Err(format!("{url}: {}", cache::NO_RANGES)); + return Err(message_with_address(&url, cache::NO_RANGES)); } Ok(cache::Fetched { bytes, origin: address::Origin::named(&url).retain_binding(&bound)? }) } @@ -1249,7 +1249,7 @@ fn services(inbox: Inbox, font_inbox: Inbox, notes: Notes, ctx: egui::Context, e let capture = Capture::current(); let still = rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::cache::current_still(); wasm_bindgen_futures::spawn_local(async move { - let r = fetch_bytes(&url, capture, still, &[]).await.map_err(|e| format!("{url}: {e}")); + let r = fetch_bytes(&url, capture, still, &[]).await.map_err(|e| message_with_address(&url, &e)); *slot.lock().unwrap_or_else(|e| e.into_inner()) = Some(r); ctx.request_repaint(); }); diff --git a/crates/assistant/src/chatgpt.rs b/crates/assistant/src/chatgpt.rs index 7c10d7d45a..24936c6413 100644 --- a/crates/assistant/src/chatgpt.rs +++ b/crates/assistant/src/chatgpt.rs @@ -1136,10 +1136,10 @@ mod native { return Err(Error { status: Some(r.status), request_id: r.request_id().map(str::to_string), - ..Error::auth(format!("{url}: HTTP {}", r.status)) + ..Error::auth(rustgis_engine::data::diagnostics::message_with_address(url, &format!("HTTP {}", r.status))) }); } - serde_json::from_slice(&r.body).map_err(|_| Error::auth(format!("{url}: not JSON"))) + serde_json::from_slice(&r.body).map_err(|_| Error::auth(rustgis_engine::data::diagnostics::message_with_address(url, "not JSON"))) } /// Verifies a new id token with OpenAI's published keys. diff --git a/crates/assistant/src/http.rs b/crates/assistant/src/http.rs index 16c76002a3..51512945f8 100644 --- a/crates/assistant/src/http.rs +++ b/crates/assistant/src/http.rs @@ -2,6 +2,7 @@ use crate::provider::{HttpRequest, HttpResponse, allowed_url, reply_limit}; use crate::{Error, error::TransportFailure}; +use rustgis_engine::data::diagnostics; /// Sends `req` and reads the response, whatever its status. pub fn send(req: &HttpRequest) -> Result { @@ -23,7 +24,9 @@ fn send_with(req: &HttpRequest, timeout: std::time::Duration, mut on_chunk: Opti // The address checked is the one used. let url = req.url.trim(); if !allowed_url(url) { - return Err(format!("{url}: only HTTPS addresses, or plain HTTP on this computer or network, are allowed").into()); + return Err( + diagnostics::message_with_address(url, "only HTTPS addresses, or plain HTTP on this computer or network, are allowed").into() + ); } let agent: ureq::Agent = ureq::Agent::config_builder() // Models can think for minutes on a hard request. @@ -75,12 +78,17 @@ fn send_with(req: &HttpRequest, timeout: std::time::Duration, mut on_chunk: Opti Ok(HttpResponse { status, body, headers }) } -/// The scheme and host of a URL (errors never repeat a path that might carry a key). -fn host(url: &str) -> String { - let (scheme, rest) = url.split_once("://").unwrap_or(("", url)); - let authority = rest.split(['/', '?', '#']).next().unwrap_or(""); - // Never the user name or password. - format!("{scheme}://{}", authority.rsplit('@').next().unwrap_or("")) +/// The scheme and authority of an already-sanitized address, without its omission suffix. A +/// root address keeps the `/` that separates it from a following `: `; a path is dropped whole. +fn host(shown: &str) -> &str { + let url = shown.strip_suffix("[query omitted]").or_else(|| shown.strip_suffix("[fragment omitted]")).unwrap_or(shown); + let Some((scheme, rest)) = url.split_once("://") else { return url }; + let end = match rest.find('/') { + Some(at) if at + 1 == rest.len() => url.len(), + Some(at) => scheme.len() + 3 + at, + None => url.len(), + }; + url.get(..end).unwrap_or(url) } /// Only connection failures can be replayed. Timeouts may already have consumed a long request. @@ -95,7 +103,17 @@ fn transport_error(e: &ureq::Error, url: &str) -> Error { } _ => TransportFailure::Interrupted, }; - Error::transport(kind, format!("couldn't reach {}: {e}", host(url))) + // Establish known-address boundaries before applying the stricter host-only display policy. + // Replacing raw text first could match a prefix of another address's longer credential. + let message = diagnostics::message_with_address(url, &e.to_string()); + let shown = diagnostics::address_in_message(url); + let message = if shown.starts_with("https://") || shown.starts_with("http://") { + let host = host(&shown); + if host == shown { message } else { message.replace(&shown, host) } + } else { + message + }; + Error::transport(kind, format!("couldn't reach {message}")) } #[cfg(test)] @@ -107,6 +125,68 @@ mod tests { time::Duration, }; + #[test] + fn rejected_secret_addresses_keep_the_policy_reason() { + let req = HttpRequest { method: "GET", url: "ftp://models.test/private?token=secret value".into(), headers: vec![], body: None }; + let error = send(&req).unwrap_err().to_string(); + let error = diagnostics::redact(&diagnostics::redact(&error)); + assert!(error.contains("only HTTPS addresses"), "{error}"); + assert!(!error.contains("secret") && !error.contains("value"), "{error}"); + } + + #[test] + fn transport_errors_keep_reasons_without_revealing_request_paths() { + let url = "https://user:password@models.test/private-key?token=secret value"; + let cause = ureq::Error::Io(std::io::Error::other(format!("request {url} failed; retry {url}: connection closed"))); + let error = transport_error(&cause, url); + assert_eq!(error.transport, Some(TransportFailure::Interrupted)); + let error = diagnostics::redact(&diagnostics::redact(&error.to_string())); + assert!(error.contains("couldn't reach https://models.test: ") && error.contains("connection closed"), "{error}"); + for hidden in ["user", "password", "private-key", "secret", "value"] { + assert!(!error.contains(hidden), "{error}"); + } + } + + #[test] + fn omitted_userinfo_is_not_restored_by_host_projection() { + for url in ["https://alice:hidden/x@models.test/private", "https://alice/x:hidden@models.test/private"] { + let cause = ureq::Error::Io(std::io::Error::other(format!("request {url}: connection closed"))); + let error = transport_error(&cause, url).to_string(); + for secret in ["alice", "hidden", "private"] { + assert!(!error.contains(secret), "{url} -> {error}"); + } + assert!(error.contains("connection closed"), "{error}"); + assert_eq!(diagnostics::redact(&diagnostics::redact(&error)), error); + } + } + + #[test] + fn host_only_addresses_keep_the_host_after_repeated_redaction() { + let url = "https://models.test?token=hidden value"; + let cause = ureq::Error::Io(std::io::Error::other("connection closed")); + let error = transport_error(&cause, url).to_string(); + let error = diagnostics::redact(&diagnostics::redact(&error)); + assert!(error.contains("https://models.test") && error.contains("connection closed"), "{error}"); + assert!(!error.contains("hidden") && !error.contains("value"), "{error}"); + } + + #[test] + fn unknown_transport_addresses_keep_conservative_credential_boundaries() { + let url = "https://models.test/private-key?token=abc"; + for reason in [ + format!("request {url}XYZ credential tail: connection closed"), + "request https://other.test/a?token=unknown credential tail: connection closed".into(), + ] { + let cause = ureq::Error::Io(std::io::Error::other(reason)); + let error = transport_error(&cause, url).to_string(); + let error = diagnostics::redact(&diagnostics::redact(&error)); + assert!(error.contains("couldn't reach https://models.test"), "{error}"); + for hidden in ["abc", "XYZ", "unknown", "credential", "tail", "connection closed"] { + assert!(!error.contains(hidden), "{error}"); + } + } + } + #[test] fn native_transport_preserves_connection_timeout_and_interruption_types() { // Check the typed refusal policy independently of OS-specific TCP retry timing. diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index 8e573532e9..ff8ea84831 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -588,6 +588,163 @@ fn base64_shaped(word: &str) -> bool { credential_shaped(word) || (word.chars().count() >= 8 && word.chars().skip(1).any(char::is_uppercase)) } +/// A slash (or a backslash, which URL parsers read as one) before `@` can disguise userinfo as +/// an authority and path; a `?` or `#` before an `@` in the authority disguises it as a query +/// or fragment. Takes the whole address, query included. +fn ambiguous_userinfo(address: &str) -> bool { + let Some((_, rest)) = address.split_once("://") else { return false }; + // `u:123?w@host`: the password ends at the `@`, not at the `?` a parser stops at. + let raw_authority = rest.split(['/', '\\']).next().unwrap_or(""); + if raw_authority.split_once('@').is_some_and(|(before, _)| before.contains(['?', '#'])) { + return true; + } + let rest = rest.split(['?', '#']).next().unwrap_or(""); + let Some((authority, path)) = rest.split_once(['/', '\\']) else { return false }; + let Some((before_at, _)) = path.split_once('@') else { return false }; + !authority.contains('@') && !authority.starts_with('[') && (authority.contains(':') || before_at.contains(':')) +} + +/// Whether a word of text too long to scan names an address whose query or userinfo may hold +/// a credential: a linear look at each `://` word, without the full scanner. +fn may_hold_secret(text: &str) -> bool { + text.split_whitespace().filter(|word| word.contains("://")).any(|word| { + ambiguous_userinfo(word) + || word + .split_once(['?', '#']) + .is_some_and(|(_, query)| query.split(['&', ';', '#']).any(|pair| secret_name(pair.split('=').next().unwrap_or("")))) + }) +} + +/// Sanitizes a known address on its own, before composing it with diagnostic prose. +/// Unlike a raw URL with a secret query, this result can be redacted again without losing +/// the reason that follows it. This does not establish a boundary for unknown URLs in prose. +pub fn address_in_message(address: &str) -> String { + if address.len() > MAX_INPUT_BYTES { + return redact(address); + } + let address = match crate::s3_address::parse(address) { + Ok(Some(bound)) => bound.url, + Err(_) => return "[address omitted]".to_string(), + Ok(None) => address, + }; + let address = match crate::login_address::parse(address) { + Ok(Some(bound)) => bound.url, + Err(_) => return "[address omitted]".to_string(), + Ok(None) => address, + }; + if address.contains(crate::login_address::PREFIX) || address.contains(crate::s3_address::PREFIX) { + return "[address omitted]".to_string(); + } + // A malformed userinfo value can contain a URL-token boundary before its `@`. The generic + // scanner cannot know those later words belong to the address; here the whole input does. + // A web address keeps neither query nor fragment. A scheme-less one (often a file path, + // where `#` is part of a name) is cut at its query only. + let url = address.contains("://"); + let before_query = if url { address.split(['?', '#']).next() } else { address.split('?').next() }.unwrap_or(""); + if before_query.contains('@') + && before_query.chars().any(|c| c.is_whitespace() || hostile_control(c) || matches!(c, '"' | '\'' | '<' | '>' | '`')) + { + return "[address omitted]".to_string(); + } + // Do not guess which interpretation of a slash-containing authority held a password. + if ambiguous_userinfo(address) { + return "[address omitted]".to_string(); + } + // Every remaining byte belongs to this known address, even if a value has unescaped + // whitespace and an unfamiliar signing name. None of that suffix is diagnostic prose. + let mut shown = redact(before_query); + if url && shown.starts_with("[address omitted]") { + // A malformed authority is omitted whole, including words after its first token. + shown.truncate("[address omitted]".len()); + } + // A redacted path secret whose value had unescaped whitespace may continue into the words + // after it: they belong to this known address, not to diagnostic prose. + if url + && before_query.contains(char::is_whitespace) + && let Some(at) = shown.find(REDACTED).map(|at| at + REDACTED.len()) + { + shown.truncate(at); + } + // Keep root authorities separate from the omission marker and the message's `: `: + // otherwise another redaction can parse the marker or separator as part of the host/port. + if let Some((scheme, rest)) = shown.split_once("://") + && ["http", "https", "s3"].iter().any(|s| scheme.eq_ignore_ascii_case(s)) + && !rest.is_empty() + && !rest.chars().any(|c| c == '/' || c.is_whitespace()) + { + shown.push('/'); + } + if let Some(cut) = address.get(before_query.len()..).filter(|cut| !cut.is_empty()) { + shown.push_str(if cut.starts_with('#') { "[fragment omitted]" } else { "[query omitted]" }); + } + shown +} + +/// Composes a known address and its failure reason without exposing credentials or swallowing +/// the reason after a secret query. Exact address occurrences (including unescaped whitespace) +/// are sanitized first; unrelated URLs and credential fields still use conservative [`redact`]. +pub fn message_with_address(address: &str, reason: &str) -> String { + if address.is_empty() || reason.len() > MAX_INPUT_BYTES { + return redact(reason); + } + let shown = address_in_message(address); + let reason = address_occurrences_sanitized(address, reason, &shown); + let named = reason + .strip_prefix(&shown) + .is_some_and(|tail| tail.is_empty() || tail.starts_with(':') || tail.chars().next().is_some_and(char::is_whitespace)); + if named { redact(&reason) } else { redact(&format!("{shown}: {reason}")) } +} + +/// [`message_with_address`] without naming the address in front: for text that already says +/// where it failed, anywhere in it (a transport's `HTTP 503 from …`). +pub fn with_address_sanitized(address: &str, text: &str) -> String { + if address.is_empty() || text.len() > MAX_INPUT_BYTES { + return redact(text); + } + redact(&address_occurrences_sanitized(address, text, &address_in_message(address))) +} + +/// `reason` with each exact occurrence of `address` (or its runtime binding's inner resource) +/// replaced by `shown`, before the conservative scanner sees its unescaped secret whitespace. +fn address_occurrences_sanitized(address: &str, reason: &str, shown: &str) -> String { + let mut safe_reason = String::with_capacity(reason.len()); + let mut copied = 0; + // Lower-level transports may name the inner resource of a runtime binding instead. + let inner = crate::login_address::inner(crate::s3_address::inner(address)); + let mut outer_matches = reason.match_indices(address).peekable(); + let mut inner_matches = reason.match_indices(inner).filter(|_| inner != address && !inner.is_empty()).peekable(); + loop { + let occurrence = match (outer_matches.peek(), inner_matches.peek()) { + (Some(outer), Some(inner)) if outer.0 <= inner.0 => outer_matches.next(), + (Some(_), Some(_)) | (None, Some(_)) => inner_matches.next(), + (Some(_), None) => outer_matches.next(), + (None, None) => break, + }; + let Some((start, matched)) = occurrence else { break }; + if start < copied { + continue; + } + let end = start + matched.len(); + let before = reason.get(..start).and_then(|s| s.chars().next_back()); + let after = reason.get(end..).unwrap_or(""); + let begins = before.is_none_or(|c| c.is_whitespace() || matches!(c, '"' | '\'' | '<' | '(' | '[' | '{' | ':' | '=' | ',' | ';')); + let ends = after.chars().next().is_none_or(|c| { + c.is_whitespace() + || matches!(c, '"' | '\'' | '>' | ')' | ']' | '}') + || (matches!(c, ':' | ',' | ';' | '.') + && after.get(c.len_utf8()..).is_some_and(|s| s.is_empty() || s.starts_with(char::is_whitespace))) + }); + // A known address must not match the prefix of another address's longer secret. + if begins && ends { + safe_reason.push_str(reason.get(copied..start).unwrap_or("")); + safe_reason.push_str(shown); + copied = end; + } + } + safe_reason.push_str(reason.get(copied..).unwrap_or("")); + safe_reason +} + /// Sanitizes before bounding. Oversized input is omitted *whole*, so a secret can never be /// retained because its identifying key lay beyond a clipped prefix. URL queries/fragments /// are all omitted, including unknown/encoded signing schemes. JSON/XML bodies are not logged. @@ -596,6 +753,44 @@ fn base64_shaped(word: &str) -> bool { /// counts as a field is narrower in the middle of a line: `name: several words` there reads as /// prose and is kept (see `docs/privacy.md`). pub fn redact(text: &str) -> String { + redact_impl(text, true) +} + +/// Sanitizes a replay parameter without treating innocent JSON/XML syntax or public URL +/// queries as diagnostic payloads. Credential fields and secret query continuations still +/// discard their remaining text; the same input and scanning bounds apply. +pub fn redact_parameter(text: &str) -> String { + redact_impl(text, false) +} + +/// The conservative diagnostic redaction when generated text has an unknown secret query or +/// fragment whose value may continue past whitespace. Address-only scrubbing cannot know where +/// that value ends, so callers use this result instead of retaining the later words. +pub fn secret_continuation_cut(text: &str) -> Option { + // Detected before the output is bounded: a long prefix must not cut the marker away. Text + // too long to scan goes whole only when one of its addresses may hold a credential, and the + // marker says how much went and why; other long text keeps its words. + if text.len() > MAX_INPUT_BYTES { + return may_hold_secret(text).then(|| format!("[{} bytes of text omitted: too long to check its addresses]", text.len())); + } + let mut strict = redact_scan(text, true); + // Cut what comes before the marker and keep the marker (and anything after it) whole, so + // the cut text still says what went. + let at = ["[query and the rest omitted]", "[address and the rest omitted]"].iter().filter_map(|m| strict.rfind(m)).max()?; + let tail = strict.split_off(at); + cut_with_ellipsis(&mut strict, MAX_DETAILS_BYTES.saturating_sub(tail.len())); + strict.push_str(&tail); + Some(strict) +} + +fn redact_impl(text: &str, diagnostic: bool) -> String { + let mut out = redact_scan(text, diagnostic); + cut_with_ellipsis(&mut out, MAX_DETAILS_BYTES); + out +} + +/// [`redact_impl`] before its output is bounded. +fn redact_scan(text: &str, diagnostic: bool) -> String { if text.len() > MAX_INPUT_BYTES { return "[oversized diagnostic omitted]".to_string(); } @@ -612,6 +807,7 @@ pub fn redact(text: &str) -> String { REDACTED, "[structured content omitted]", "[oversized diagnostic omitted]", + "[address and the rest omitted]", "[address omitted]", "[oversized address omitted]", ] @@ -622,7 +818,20 @@ pub fn redact(text: &str) -> String { i += marker.len(); continue; } - if matches!(c, '{' | '[' | '<') && opens_structured(rest) { + // A credential tag, attributes or not: its value can span words and lines, so omit the + // remainder whatever else the tag looks like. + if c == '<' { + let inner = rest.get(1..).unwrap_or("").trim_start_matches('/'); + let tag = inner.split(|c: char| c.is_whitespace() || matches!(c, '>' | '/')).next().unwrap_or(""); + if tag.chars().next().is_some_and(|c| c.is_alphabetic() || c == '_') + && inner.get(..256).unwrap_or(inner).contains('>') + && secret_name(tag.rsplit(':').next().unwrap_or(tag)) + { + out.push_str(REDACTED); + break; + } + } + if matches!(c, '{' | '[' | '<') && opens_structured(rest) && diagnostic { out.push_str("[structured content omitted]"); break; } @@ -672,6 +881,16 @@ pub fn redact(text: &str) -> String { field_before = false; continue; } + if ambiguous_userinfo(address) { + // Its password may run on past whitespace, as a secret query's may: the rest + // goes too, and the marker says so. + out.push_str("[address and the rest omitted]"); + break; + } + let query = address.split_once(['?', '#']).map_or("", |(_, q)| q); + let names = || query.split(['&', ';', '#']).map(|pair| pair.split('=').next().unwrap_or("")); + let has_name = |name: &str| names().any(|key| key.eq_ignore_ascii_case(name) || key_is(key, name)); + let secret = names().any(secret_name) || (!diagnostic && has_name("code") && has_name("state")); match url::Url::parse(address) { // Nothing to take out: the address stays as written, its punctuation too // ("https://example.org: timeout"), unless it holds control characters. @@ -679,7 +898,7 @@ pub fn redact(text: &str) -> String { if url.has_host() && url.username().is_empty() && url.password().is_none() - && url.query().is_none() + && (url.query().is_none() || (!diagnostic && !secret)) && url.fragment().is_none() && !address.chars().any(hostile_control) => { @@ -700,7 +919,9 @@ pub fn redact(text: &str) -> String { } let _ = url.set_username(""); let _ = url.set_password(None); - url.set_query(None); + if diagnostic || secret { + url.set_query(None); + } url.set_fragment(None); out.push_str(&path_secrets_redacted(url.as_str())); out.extend(kept); @@ -709,18 +930,18 @@ pub fn redact(text: &str) -> String { } i += url_end; field_before = false; - if address.contains(['?', '#']) { + if address.contains(['?', '#']) && (diagnostic || secret || address.contains('#')) { // Its query or fragment is gone, and that is said. When a pair in it named a // secret, an unescaped value may run on past whitespace: the rest of the text // goes too, said. Otherwise the rest keeps going through the field rules, so a // failure's reason after the address isn't lost. - let query = address.split_once(['?', '#']).map_or("", |(_, q)| q); - let secret = query.split(['&', ';', '#']).any(|pair| secret_name(pair.split('=').next().unwrap_or(""))); if secret { out.push_str("[query and the rest omitted]"); break; } - out.push_str("[query omitted]"); + // Said as what went: a query (always in a diagnostic), or only a fragment. + let query_removed = diagnostic && address.split('#').next().unwrap_or("").contains('?'); + out.push_str(if query_removed { "[query omitted]" } else { "[fragment omitted]" }); if let Some(end) = address.chars().last().filter(|c| matches!(c, ':' | ',' | ';' | ')' | '.')) { out.push(end); } @@ -814,7 +1035,6 @@ pub fn redact(text: &str) -> String { i += c.len_utf8(); } } - cut_with_ellipsis(&mut out, MAX_DETAILS_BYTES); out } @@ -925,6 +1145,222 @@ mod tests { assert!(!connection.contains("alice") && !connection.contains("hidden"), "{connection}"); } + #[test] + fn replay_parameters_keep_public_structure_but_not_credentials() { + for public in ["
https://example.org?format=plain
", r#"{"label":"public","link":"https://example.org?format=json"}"#] { + assert_eq!(redact_parameter(public), public); + assert_eq!(redact(public), "[structured content omitted]"); + } + // A replay keeps a public query; only its fragment goes, and the marker says which. + let link = "see https://example.org/a?format=json#frag then"; + let kept = redact_parameter(link); + assert_eq!(kept, "see https://example.org/a?format=json[fragment omitted] then"); + assert_eq!(redact_parameter(&kept), kept); + assert_eq!(redact(link), "see https://example.org/a[query omitted] then"); + assert_eq!(redact_parameter("see https://example.org/a#frag then"), "see https://example.org/a[fragment omitted] then"); + assert_eq!(redact("see https://example.org/a#frag then"), "see https://example.org/a[fragment omitted] then"); + for private in [ + "hidden hidden-tail https://example.org", + "hidden hidden-tail https://example.org", + r#"{"password":"hidden hidden-tail","link":"https://example.org"}"#, + "GET https://example.org?%2574oken=hidden hidden-tail", + "GET https://example.org?code=hidden&state=hidden hidden-tail", + "GET https://example.org?%2563ode=hidden&%2573tate=hidden hidden-tail", + "GET https://alice:hidden@example.org?format=json", + "GET https://u:123/w@host/x", + "GET https://alice/part:123@host/x", + "GET https://u:123\\w@host/x", + r#"hidden hidden-tail https://example.org"#, + ] { + let clean = redact_parameter(private); + assert!(!clean.contains("hidden") && !clean.contains("alice") && !clean.contains("123"), "{private} -> {clean}"); + } + for private in [r#"hidden hidden-tail"#, "GET https://u:123\\w@host/x"] { + let clean = redact(private); + assert!(!clean.contains("hidden") && !clean.contains("123"), "{private} -> {clean}"); + } + // An ambiguous address in prose says the rest of the text went with it. + let clean = redact("upload https://u:123/w@h/x failed: HTTP 500"); + assert_eq!(clean, "upload [address and the rest omitted]"); + assert_eq!(redact(&clean), clean); + // An attribute-only tag that names no credential is still plain markup. + let public = r#" https://example.org "#; + assert_eq!(redact_parameter(public), public); + } + + #[test] + fn known_path_secrets_and_long_prefixes_do_not_keep_continuations() { + let address = "https://example.org/token=hidden hidden-tail/data"; + let clean = message_with_address(address, "HTTP 404"); + assert!(clean.contains("HTTP 404") && !clean.contains("hidden"), "{clean}"); + assert_eq!(redact(&clean), clean); + let long = format!("{} https://example.org/a?token=hidden hidden-tail", "word ".repeat(2000)); + let cut = secret_continuation_cut(&long).expect("the cut is seen past the output bound"); + assert!(!cut.contains("hidden") && cut.len() <= MAX_DETAILS_BYTES + '…'.len_utf8(), "{}", cut.len()); + // The marker sits past the 8 KiB bound in the scan; the cut keeps it whole at the end. + assert!(cut.ends_with("…[query and the rest omitted]"), "{}", cut.get(cut.len().saturating_sub(60)..).unwrap_or("")); + assert_eq!(redact(&cut), cut); + // An ambiguous address cuts as a secret query does. + let ambiguous = "run failed at https://u:123/w@h/x hidden-tail: HTTP 500"; + assert_eq!(secret_continuation_cut(ambiguous).as_deref(), Some("run failed at [address and the rest omitted]")); + // Text too long to scan goes whole, said with its size, only when an address in it may + // hold a credential; a long text with a public link keeps its words. + let oversized = format!("{} https://example.org/a?token=hidden hidden-tail", "word ".repeat(4000)); + assert_eq!( + secret_continuation_cut(&oversized), + Some(format!("[{} bytes of text omitted: too long to check its addresses]", oversized.len())) + ); + let ambiguous = format!("{} https://u:123?w@host/x tail", "word ".repeat(4000)); + assert!(secret_continuation_cut(&ambiguous).is_some()); + assert_eq!(secret_continuation_cut(&format!("{} see https://example.org/a?page=2 then", "word ".repeat(4000))), None); + assert_eq!(secret_continuation_cut(&"word ".repeat(4000)), None); + assert_eq!(secret_continuation_cut("failed at https://example.org/a: HTTP 500"), None); + } + + #[test] + fn known_addresses_preserve_reasons_through_repeated_redaction() { + for address in [ + "https://example.org/data?token=hidden", + "https://example.org/data?sig=hidden#hidden", + "https://example.org/data?%256bey=hidden", + "https://example.org/data#%2574oken=hidden", + "https://alice:hidden@example.org/data?key=hidden", + "https://example.org/data?token=hidden unescaped hidden", + "https://example.org/data#sig=hidden unescaped hidden", + "https://example.org/data?custom=hidden unescaped hidden", + "https://example.org/data#hidden unescaped hidden", + // A scheme-less address loses its whole query too, unfamiliar keys included. + "data.tif?custom=hidden unescaped hidden", + "data.tif?token=hidden unescaped hidden", + ] { + let clean = message_with_address(address, "HTTP 503 Service Unavailable"); + assert!(clean.contains("HTTP 503 Service Unavailable"), "{address} -> {clean}"); + assert!(!clean.contains("hidden") && !clean.contains("alice"), "{address} -> {clean}"); + assert_eq!(redact(&clean), clean); + assert_eq!(redact(&redact(&clean)), clean); + assert_eq!(message_with_address(address, &format!("{address}: HTTP 503 Service Unavailable")), clean); + assert_eq!(message_with_address(address, &clean), clean); + } + // A scheme-less address says its query went; a `#` in a file name is part of the name. + assert_eq!(address_in_message("data.tif?custom=hidden unescaped hidden"), "data.tif[query omitted]"); + assert_eq!(message_with_address("/data/plot#1.gpkg", "no such table"), "/data/plot#1.gpkg: no such table"); + // A secret after a scheme-less `#` is still a field, and goes; a `#` before `?` stays in + // the path while the query is cut. + let shown = address_in_message("host/data#token=hidden"); + assert!(shown.starts_with("host/data#") && !shown.contains("hidden"), "{shown}"); + assert_eq!(redact(&shown), shown); + assert_eq!(address_in_message("/data/plot#1.gpkg?token=hidden"), "/data/plot#1.gpkg[query omitted]"); + } + + #[test] + fn root_addresses_keep_context_through_repeated_redaction() { + for (address, origin) in [ + ("https://models.test?token=hidden value", "https://models.test/"), + ("https://models.test:8443", "https://models.test:8443/"), + ("http://[::1]:8080#token=hidden value", "http://[::1]:8080/"), + ("HTTPS://models.test?token=hidden", "HTTPS://models.test/"), + ("s3://bucket?X-Cred=hidden", "s3://bucket/"), + ("https://", "https://"), + ] { + let mut message = message_with_address(address, "HTTP 503"); + for _ in 0..3 { + assert!(message.contains(origin) && message.contains("HTTP 503") && !message.contains("hidden"), "{address} -> {message}"); + message = redact(&message); + } + } + } + + #[test] + fn every_known_address_occurrence_is_sanitized_before_the_reason() { + let address = "https://alice:hidden@example.org/data?%74oken=hidden unescaped hidden"; + let reason = format!("HTTP 404 fetching {address}; retry of {address} failed"); + let clean = message_with_address(address, &reason); + assert!(clean.contains("HTTP 404 fetching") && clean.contains("; retry of") && clean.ends_with(" failed"), "{clean}"); + assert_eq!(clean.matches("https://example.org/data").count(), 3); + assert!(!clean.contains("hidden") && !clean.contains("alice"), "{clean}"); + assert_eq!(redact(&clean), clean); + let history = History::default(); + history.push(Severity::Error, "Network", &clean, &clean); + assert!(history.report(Filter::All).contains("HTTP 404 fetching")); + assert!(!history.report(Filter::All).contains("hidden")); + // Text that already names where it failed keeps its own wording, without a label in front. + let clean = with_address_sanitized(address, &format!("HTTP 503 from {address}; retrying")); + assert!(clean.starts_with("HTTP 503 from https://example.org/data") && clean.ends_with("; retrying"), "{clean}"); + assert!(!clean.contains("hidden") && !clean.contains("alice"), "{clean}"); + assert_eq!(clean.matches("://").count(), 1, "{clean}"); + } + + #[test] + fn known_address_helpers_do_not_relax_unknown_credentials() { + let raw = "https://example.org/data?token=hidden unescaped hidden: HTTP 503"; + assert_eq!(redact(raw), "https://example.org/data[query and the rest omitted]"); + let clean = message_with_address("https://known.example.org/data?key=hidden", raw); + assert!(!clean.contains("hidden") && !clean.contains("HTTP 503"), "{clean}"); + let clean = message_with_address("https://known.example.org/data?key=hidden", "HTTP 503; Password=hidden more hidden"); + assert!(clean.contains("HTTP 503") && !clean.contains("hidden"), "{clean}"); + assert_eq!(message_with_address("", raw), redact(raw)); + assert_eq!(message_with_address("/data/a", &"x".repeat(MAX_INPUT_BYTES + 1)), "[oversized diagnostic omitted]"); + } + + #[test] + fn known_address_prefixes_do_not_expose_extended_credentials() { + for address in ["https://example.org/data?token=abc", "https://example.org/data#sig=abc"] { + for suffix in ["XYZhidden", "&other=hidden", "#hidden", "%20hidden", "/hidden", ":hidden"] { + let reason = format!("{address}{suffix}: HTTP 503"); + let clean = message_with_address(address, &reason); + assert!(!clean.contains("hidden") && !clean.contains("XYZ"), "{reason} -> {clean}"); + assert!(!clean.contains("HTTP 503"), "{reason} -> {clean}"); + } + } + } + + #[test] + fn known_malformed_userinfo_never_leaks_its_remaining_words() { + for address in [ + "https://alice:hidden more-hidden@example.org/data", + "https://alice:hidden\tmore-hidden@example.org/data", + "https://alice:hidden\"more-hidden@example.org/data", + "https://alice:hidden {shown}"); + assert!(!clean.contains("hidden") && !clean.contains("alice"), "{address} -> {clean}"); + assert!(!shown.contains("123") && !clean.contains("123"), "{address} -> {clean}"); + assert!(clean.ends_with(": HTTP 401"), "{clean}"); + assert_eq!(redact(&clean), clean); + } + } + + #[test] + fn unambiguous_ports_and_path_at_signs_keep_address_context() { + for address in + ["https://host:123/data", "https://host/data@edition", "https://host/data@edition:123", "https://[::1]:8080/data@edition"] + { + assert_eq!(address_in_message(address), address); + assert_eq!(message_with_address(address, "HTTP 503"), format!("{address}: HTTP 503")); + } + // An `@` in a query after a path is no userinfo: the host and port stay readable. + let address = "https://h:80/x?email=a@b"; + assert_eq!(message_with_address(address, "HTTP 503"), "https://h:80/x[query omitted]: HTTP 503"); + // A fragment alone is said as one, and the label survives redacting again. + let shown = address_in_message("https://h.example/a#frag"); + assert_eq!(shown, "https://h.example/a[fragment omitted]"); + assert_eq!(redact(&shown), shown); + let clean = message_with_address("https://alice:123%2Fw@host/x", "HTTP 503"); + assert!(clean.contains("HTTP 503") && !clean.contains("alice") && !clean.contains("123"), "{clean}"); + } + #[test] fn credential_words_in_file_paths_preserve_failure_context() { let path = "../token-cache/secret.png"; @@ -937,6 +1373,21 @@ mod tests { assert_eq!(redact(r"Progress 12% complete"), "Progress 12% complete"); } + #[test] + fn known_runtime_bindings_sanitize_outer_and_inner_address_occurrences() { + let inner = "https://example.org/data?token=hidden"; + let login = crate::login_address::bind("0123456789abcdef", "https://example.org", inner).unwrap(); + let s3 = crate::s3_address::bind("0123456789abcdef", "", inner).unwrap(); + for address in [&login, &s3] { + let clean = message_with_address(address, &format!("HTTP 503 for {inner}; retrying {address} failed")); + assert!(clean.contains("HTTP 503") && clean.contains("; retrying") && clean.ends_with(" failed"), "{clean}"); + for private in ["hidden", "0123456789abcdef", crate::login_address::PREFIX, crate::s3_address::PREFIX] { + assert!(!clean.contains(private), "{private} in {clean}"); + } + assert_eq!(redact(&clean), clean); + } + } + #[test] fn prose_naming_credentials_keeps_its_words() { // RustGIS's own notices name credentials without holding one: redacting must not diff --git a/crates/data/src/source/mod.rs b/crates/data/src/source/mod.rs index a28c1b803b..98dee67552 100644 --- a/crates/data/src/source/mod.rs +++ b/crates/data/src/source/mod.rs @@ -424,7 +424,10 @@ fn open_file_now(file: std::sync::Arc) -> Result( // Readers plan within READ_BUDGET; a round asking for far more is a hostile file's offsets. let asked: u64 = wants.iter().map(|w| if let Want::Range(a, b) = w { b - a } else { 0 }).fold(0, u64::saturating_add); if asked > ROUND_BUDGET { - done(Err(DataError::Format(format!("{}: a read asked for {} MB of the file at once", remote.url, asked >> 20)))); + done(Err(DataError::Format(crate::diagnostics::message_with_address( + &remote.url, + &format!("a read asked for {} MB of the file at once", asked >> 20), + )))); return; } if rounds >= MAX_ROUNDS { - done(Err(DataError::Format(format!("{}: still needed more of the file after {MAX_ROUNDS} rounds of reading", remote.url)))); + done(Err(DataError::Format(crate::diagnostics::message_with_address( + &remote.url, + &format!("still needed more of the file after {MAX_ROUNDS} rounds of reading"), + )))); return; } let Some(fetch) = fetcher().lock().unwrap_or_else(|e| e.into_inner()).clone() else { @@ -582,9 +588,7 @@ fn step_in( }; if let Some((Some(done), failed)) = next { match failed { - Some(e) => { - done(Err(DataError::Format(if e.starts_with(remote2.url()) { e } else { format!("{}: {e}", remote2.url()) }))) - } + Some(e) => done(Err(DataError::Format(crate::diagnostics::message_with_address(remote2.url(), &e)))), None => step(remote2, work2, done, drive, rounds + 1, context), } } @@ -610,7 +614,7 @@ pub fn drive_blocking(remote: Arc, work: Arc Result<()> + Send + Sync> = Arc::new(move || { + read_remote.bytes(0, 8).ok_or_else(need)?; + Ok(()) + }); + let error = drive_blocking(remote, work).unwrap_err().to_string(); + assert!(error.contains("HTTP 404"), "{error}"); + assert!(!error.contains("hidden"), "{error}"); + assert_eq!(crate::diagnostics::redact(&error), error); + let history = crate::diagnostics::History::default(); + history.push(crate::diagnostics::Severity::Error, "Data", &error, ""); + let report = history.report(crate::diagnostics::Filter::All); + assert!(report.contains("HTTP 404") && !report.contains("hidden"), "{report}"); + } + #[test] fn hostile_offsets_never_overflow_or_explode() { // Ranges ending at the very end of the address space (a hostile file's offsets). diff --git a/crates/engine/src/cmd/history.rs b/crates/engine/src/cmd/history.rs index 16f0b2a11e..93734ec0ec 100644 --- a/crates/engine/src/cmd/history.rs +++ b/crates/engine/src/cmd/history.rs @@ -139,7 +139,7 @@ pub(crate) fn redacted(v: &Value) -> Value { use crate::sources::looks_like_a_key; use rustgis_data::source::credentials::secret_key; match v { - Value::String(t) => Value::String(redacted_text(t)), + Value::String(t) => Value::String(redacted_parameter(t)), Value::Array(a) => Value::Array(a.iter().map(redacted).collect()), Value::Object(o) => { let beside_address = |k: &str| o.iter().any(|(sibling, v)| sibling != k && crate::project_log::address_context(sibling, v, 0)); @@ -149,20 +149,41 @@ pub(crate) fn redacted(v: &Value) -> Value { && (!k.eq_ignore_ascii_case("key") || v.as_str().is_some_and(looks_like_a_key) || beside_address(k)) }; Value::Object( - o.iter().map(|(k, v)| (redacted_text(k), if secret(k, v) { Value::String(String::new()) } else { redacted(v) })).collect(), + o.iter() + .map(|(k, v)| (redacted_parameter(k), if secret(k, v) { Value::String(String::new()) } else { redacted(v) })) + .collect(), ) } other => other.clone(), } } +/// Replay parameters are data, not diagnostic payloads: preserve innocent markup and public +/// query parameters, while keeping the same conservative credential boundaries. +pub(crate) fn redacted_parameter(t: &str) -> String { + let structured = t.trim_start().starts_with(['{', '[', '<']); + let text = if connection(t) || (!structured && !t.contains(char::is_whitespace)) { + crate::sources::redact(t) + } else if structured || t.contains("://") { + rustgis_data::diagnostics::redact_parameter(t) + } else { + t.to_string() + }; + query_pairs_emptied(&private_keys_removed(&text)) +} + /// `t` without its secrets. A connection is taken whole, though a libpq one (`PG:host=db -/// password=…`) has spaces in it: `redact` takes its password out. Other text with spaces may -/// name addresses among its words; a single word may be one. Any text may still be a connection -/// in a form `redact` doesn't read (libpq's words alone, GDAL's `MYSQL:db,user=…,password=…`, -/// ODBC's `UID=…;PWD=…`): its secret-named values go too. +/// password=…`) has spaces in it. A single-word address keeps its nonsecret request parameters +/// for replay. In prose, an unknown URL's secret value may continue past whitespace: the strict +/// diagnostic scanner must see it before any address splitter removes its identifying key. pub(crate) fn redacted_text(t: &str) -> String { - let r = if connection(t) || !t.contains(char::is_whitespace) { crate::sources::redact(t) } else { crate::sources::redact_text(t) }; + let r = if connection(t) || !t.contains(char::is_whitespace) { + crate::sources::redact(t) + } else if t.contains("://") { + rustgis_data::diagnostics::redact(t) + } else { + t.to_string() + }; // After `redact` too: a prefix it knows may come in a form it doesn't (GDAL's `MYSQL:`). query_pairs_emptied(&private_keys_removed(&r)) } diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 165bd2da47..2e4f347213 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -3251,7 +3251,8 @@ pub(crate) fn fixed_download_urls(p: &Value) -> Option> { /// they are. pub(crate) fn add_url(s: &mut Session, p: &Value) -> CmdResult { let owner = s.continuation()?; - let out = add_url_as_given(s, p).map_err(|e| crate::sources::redact_text(&e))?; + let url = ps(p, "url").map(str::trim).unwrap_or(""); + let out = add_url_as_given(s, p).map_err(|e| crate::sources::message_with_address(url, &e))?; s.check_continuation(&owner)?; Ok(out) } @@ -3295,10 +3296,16 @@ fn add_url_as_given(s: &mut Session, p: &Value) -> CmdResult { .and_then(Value::as_u64) .is_some_and(|id| u32::try_from(id).ok().and_then(|id| s.map().layer(id)).is_none()); if missing || added.gone(s.map()) { - return Err(format!("{url}: the layer its download added was removed or undone before the command ran again")); + return Err(crate::sources::message_with_address( + url, + "the layer its download added was removed or undone before the command ran again", + )); } if super::plugins::fetch::table_gone(&answer, &s.project) { - return Err(format!("{url}: the table its download added was removed or undone before the command ran again")); + return Err(crate::sources::message_with_address( + url, + "the table its download added was removed or undone before the command ran again", + )); } return added.answer_now(s.map(), &answer); } @@ -3399,13 +3406,9 @@ fn add_url_as_given(s: &mut Session, p: &Value) -> CmdResult { } #[cfg(not(target_arch = "wasm32"))] { - // Errors name the address without its keys, and any address they quote (a redirect's) - // without its keys too; those about the address itself name it already. - let shown = crate::sources::redact(url); - let named = |e: String| match e.strip_prefix(url).or_else(|| e.strip_prefix(shown.as_str())) { - Some(rest) => format!("{shown}{}", crate::sources::redact_text(rest)), - None => format!("{shown}: {}", crate::sources::redact_text(&e)), - }; + // Redact the known address before composing any reason: a query value can contain + // unescaped whitespace, so splitting an arbitrary error into URL words is not safe. + let named = |e: String| crate::sources::message_with_address(url, &e); // An address without a file name may be a GeoTIFF too: its first bytes are read to see, // and anything that doesn't answer as one is downloaded whole, as before. let unnamed = !tiff && is_unnamed_url(url); @@ -3492,8 +3495,8 @@ fn is_zip(bytes: &[u8]) -> bool { /// A Shapefile's `.shp` or part that failed, named by its address without its keys or signature /// (the `.shp`'s query, which the parts carry too). fn part_failed(url: &str, e: String) -> String { - let e = e.strip_prefix(url).map(|r| r.trim_start_matches(": ").to_string()).unwrap_or(e); - format!("{}: {}{}", crate::sources::redact(url), crate::sources::redact_text(&e), status_hint(url, &e)) + let why = status_hint(url, &e); + format!("{}{why}", crate::sources::message_with_address(url, &e)) } /// What a Shapefile's 403 may mean: S3 answers a signed request for a file that isn't there as it @@ -3618,10 +3621,10 @@ pub fn download_data(url: &str, max: usize, done: crate::render::rustgis_tiles:: let left = max.saturating_sub(shp.len()); download_parts(name, shp, parts.into_iter(), Vec::new(), left, done); } - // The app names the address it asked for (redacted), as for any download. + // Name the complete known address before the app or session redacts this again. Err(e) => { let why = status_hint(&shp_url, &e); - done(Err(format!("{}{why}", crate::sources::redact_text(&e)))) + done(Err(format!("{}{why}", crate::sources::message_with_address(&shp_url, &e)))) } }), ); @@ -3829,15 +3832,10 @@ pub(crate) fn add_cog_with(s: &mut Session, p: &Value, url: &str, strict: bool, s.notify(format!("{shown}: {line}"), false); } } else if probed || why.is_some() || r.get("dated").is_some() { - // Added, but its (first date's) file couldn't be read: the view stays where it was, and - // the answer says why when the read said (its keys left out). + // Added, but its (first date's) file couldn't be read: the view stays where it was. r["warning"] = json!(match why { - Some(e) => format!( - "{} couldn't be read ({}), so the map wasn't moved to it", - crate::sources::redact(&url), - crate::sources::redact_text(&e) - ), - None => format!("{} couldn't be read yet, so the map wasn't moved to it", crate::sources::redact(&url)), + Some(e) => crate::sources::message_with_address(&url, &format!("couldn't be read ({e}), so the map wasn't moved to it"),), + None => crate::sources::message_with_address(&url, "couldn't be read yet, so the map wasn't moved to it"), }); } r["inPlace"] = json!(true); @@ -3897,11 +3895,8 @@ fn add_in_place( // Its header couldn't be read (a failure recorded and not tried again yet, or now): the view // stays where it was, and the answer says why (the layer says so too, once its read fails). if let Err(e) = &info { - r["warning"] = json!(format!( - "{} couldn't be read ({}), so the map wasn't moved to it", - crate::sources::redact(file), - crate::sources::redact_text(e) - )); + r["warning"] = + json!(crate::sources::message_with_address(file, &format!("couldn't be read ({e}), so the map wasn't moved to it"),)); } if let Ok(Some(info)) = info { let [w, so, e, n] = info.bounds; @@ -4086,7 +4081,7 @@ pub fn window_asked(s: &Session, p: &Value) -> Result { .and_then(|_| super::raster::as_scan(p)) .and_then(|scan| window_checks(p, url, scan)) .and_then(|()| window_settled(s, p, url)); - checked.map_err(|e| crate::sources::redact_text(&e)) + checked.map_err(|e| crate::sources::message_with_address(url, &e)) } /// `layer.addUrl` with `"bbox"`, for those that wait: read here and added. @@ -4112,7 +4107,7 @@ pub fn read_cog_window(asked: &WindowAsked) -> Result { use crate::render::rustgis_tiles::cog; use rustgis_data::raster::cog::Level; let url = asked.url.as_str(); - let named = |e: String| crate::sources::redact_text(&if e.starts_with(url) { e } else { format!("{url}: {e}") }); + let named = |e: String| crate::sources::message_with_address(url, &e); let [west, south, east, north] = asked.bbox; let cell = asked.cell; // The five minutes count from the header's read. @@ -4532,17 +4527,18 @@ fn add_download(s: &mut Session, p: &Value) -> CmdResult { // A service's features (Add From Service): its answer's axes (against its layer's bounds) and // paging. let bytes = if service { - let label = ps(p, "name").unwrap_or(&url).to_string(); + let label = ps(p, "name").map(str::to_string).unwrap_or_else(|| crate::sources::address_in_message(&url)); super::service::features_answer(s, &label, bytes, bounds) } else { bytes }; // A WCS coverage (Add From Service): a GeoTIFF, or the service's error said as such. let name = if coverage { - super::service::coverage_check(ps(p, "name").unwrap_or(&url), &bytes)?; + let label = ps(p, "name").map(str::to_string).unwrap_or_else(|| crate::sources::address_in_message(&url)); + super::service::coverage_check(&label, &bytes)?; "coverage.tif".to_string() } else { - url_file_name(&url, &bytes).map_err(|e| format!("{url}: {e}"))? + url_file_name(&url, &bytes).map_err(|e| crate::sources::message_with_address(&url, &e))? }; let left_out = parts_left_out(&url, &bytes, p); let r = super::raster::add_named_bytes(s, p, name, bytes, b(p, "zoom"))?; diff --git a/crates/engine/src/cmd/plugin_fetch.rs b/crates/engine/src/cmd/plugin_fetch.rs index 39541c1871..7e5466ace3 100644 --- a/crates/engine/src/cmd/plugin_fetch.rs +++ b/crates/engine/src/cmd/plugin_fetch.rs @@ -155,14 +155,16 @@ pub fn checked(url: &str) -> Result { } let lower = url.to_ascii_lowercase(); if !(lower.starts_with("https://") || lower.starts_with("http://")) { - let shown: String = url.chars().take(80).collect(); - return Err(format!("{shown}: plugins fetch only http:// and https:// addresses")); + return Err(crate::sources::shortened_message_with_address(url, 80, "plugins fetch only http:// and https:// addresses")); } // A fragment never reaches the server (and RustGIS marks byte ranges with one). let url = url.split('#').next().unwrap_or(url); address::check(url, None)?; if address::reach(url) != address::Reach::Public && !allow_local() { - return Err(format!("{url}: plugins fetch only from the internet, not this computer or the local network")); + return Err(crate::sources::message_with_address( + url, + "plugins fetch only from the internet, not this computer or the local network", + )); } Ok(url.to_string()) } @@ -171,10 +173,10 @@ pub fn checked(url: &str) -> Result { /// or array. pub fn answer(url: &str, bytes: &[u8]) -> Result { if bytes.len() > MAX_BYTES { - return Err(format!("{url}: larger than {MAX_BYTES} bytes")); + return Err(crate::sources::message_with_address(url, &format!("larger than {MAX_BYTES} bytes"))); } let bytes = bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(bytes); - let text = std::str::from_utf8(bytes).map_err(|_| format!("{url}: the answer isn't UTF-8 text"))?; + let text = std::str::from_utf8(bytes).map_err(|_| crate::sources::message_with_address(url, "the answer isn't UTF-8 text"))?; let trimmed = text.trim_start(); let parsed = if trimmed.starts_with('{') || trimmed.starts_with('[') { serde_json::from_str::(text).ok() } else { None }; Ok(match parsed { @@ -205,7 +207,7 @@ fn start(key: &str, url: &str) { &s3::anonymous(&url), MAX_BYTES, Box::new(move |r| { - let answer = r.map_err(|e| format!("{url}: {e}")).and_then(|bytes| answer(&url, &bytes)); + let answer = r.map_err(|e| crate::sources::message_with_address(&url, &e)).and_then(|bytes| answer(&url, &bytes)); cache().keep(&key, answer); }), ); @@ -257,7 +259,10 @@ impl Calls { let started = *self.started.get_or_insert_with(Instant::now); let left = CALL_BUDGET.saturating_sub(started.elapsed()); if left.is_zero() { - return Err(format!("{url}: this plugin call's fetches took longer than {} s in all", CALL_BUDGET.as_secs())); + return Err(crate::sources::message_with_address( + url, + &format!("this plugin call's fetches took longer than {} s in all", CALL_BUDGET.as_secs()), + )); } // Names are checked again where they resolve: a public-looking name for a local address // (or a redirect to one) is refused there. Tests fetch from a server on this computer. @@ -269,7 +274,7 @@ impl Calls { }; // Errors are kept too (for `ERROR_TTL`), so a plugin run again and again on an address // that doesn't answer doesn't wait for it each time. - let answer = bytes.map_err(|e| format!("{url}: {e}")).and_then(|bytes| answer(url, &bytes)); + let answer = bytes.map_err(|e| crate::sources::message_with_address(url, &e)).and_then(|bytes| answer(url, &bytes)); cache().keep(key, answer.clone()); answer } @@ -277,7 +282,7 @@ impl Calls { #[cfg(target_arch = "wasm32")] fn blocking(&mut self, _key: &str, url: &str) -> CmdResult { let _ = self.started; - Err(format!("{url}: the web app can't wait for a fetch")) + Err(crate::sources::message_with_address(url, "the web app can't wait for a fetch")) } } @@ -333,7 +338,7 @@ pub(crate) struct UrlWaits { /// added twice. added: VecDeque<(u64, Value)>, /// A service's documents being read off the frame, by request number. - reading: Vec<(u64, Arc, String>>>>)>, + reading: Vec<(u64, String, Arc, String>>>>)>, } /// What adding a landed download added, to tell when its command runs again whether it is still @@ -566,27 +571,28 @@ impl UrlWaits { pub(crate) fn read_document(&mut self, n: u64, asked: &Value) { let slot: Arc, String>>>> = Arc::default(); let land = slot.clone(); + let url = asked["url"].as_str().unwrap_or("").to_string(); crate::render::rustgis_tiles::cache::download_within( - asked["url"].as_str().unwrap_or(""), + &url, MAX_DOCUMENT, DOCUMENT_TIMEOUT, Box::new(move |r| *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(r)), ); - self.reading.push((n, slot)); + self.reading.push((n, url, slot)); } /// Lands the documents read by now. One whose command gave up is dropped, and forgotten. fn documents_in(&mut self) { - for (n, slot) in std::mem::take(&mut self.reading) { + for (n, url, slot) in std::mem::take(&mut self.reading) { let answer = slot.lock().unwrap_or_else(|e| e.into_inner()).take(); let Some(answer) = answer else { - self.reading.push((n, slot)); + self.reading.push((n, url, slot)); continue; }; match answer { Ok(bytes) => self.land(n, Ok(json!({})), Added { document: Some(bytes.into()), ..Added::default() }), // Its command gets this as its answer: without the address's keys. - Err(e) => self.land(n, Err(crate::sources::redact_text(&e)), Added::default()), + Err(e) => self.land(n, Err(crate::sources::message_with_address(&url, &e)), Added::default()), } self.refused(n); } @@ -953,6 +959,17 @@ impl Session { mod tests { use super::*; + /// A long refused address is shortened without cutting an omission marker in half: the + /// address before it is cut instead, and its query stays out. + #[test] + fn a_shortened_refused_address_keeps_whole_markers() { + let url = format!("ftp://h.example/{}?page=1&token=hidden", "a".repeat(52)); + let error = checked(&url).unwrap_err(); + assert!(!error.contains("hidden") && !error.contains("page=1"), "{error}"); + assert!(error.ends_with("a…[query omitted]: plugins fetch only http:// and https:// addresses"), "{error}"); + assert_eq!(error.split(": plugins").next().map(|a| a.chars().count()), Some(80), "{error}"); + } + /// A failure answers once a run, in the order it came, and a call made again after it is a new /// download: a command trying again gets its next try, not the same failure for ever. What was /// added answers as often as it is asked for (#1319). @@ -1054,11 +1071,12 @@ mod tests { waits.wait(2, doc(2)); waits.wait(3, doc(3)); waits.remember_given_up(4, doc(4)); + // Each read carries the address it fetched, which the error names. waits.reading = vec![ - (1, reading(Some(Ok(b"caps".to_vec())))), - (2, reading(Some(Err("https://x.org/2?token=hunter2: 404 Not Found".into())))), - (3, reading(None)), - (4, reading(Some(Ok(b"late".to_vec())))), + (1, "https://x.org/1".into(), reading(Some(Ok(b"caps".to_vec())))), + (2, "https://x.org/2?token=hunter2".into(), reading(Some(Err("https://x.org/2?token=hunter2: 404 Not Found".into())))), + (3, "https://x.org/3".into(), reading(None)), + (4, "https://x.org/4".into(), reading(Some(Ok(b"late".to_vec())))), ]; waits.documents_in(); let landed = |n: u64| waits.landed.iter().find(|(m, ..)| *m == n); diff --git a/crates/engine/src/cmd/portals.rs b/crates/engine/src/cmd/portals.rs index 9541bc670c..b6a003b25c 100644 --- a/crates/engine/src/cmd/portals.rs +++ b/crates/engine/src/cmd/portals.rs @@ -242,7 +242,7 @@ fn first_layer(s: &mut Session, url: &str, kind: &str) -> Result .or_else(|| layers.first()) .and_then(|l| l["name"].as_str()) .map(str::to_string) - .ok_or_else(|| format!("{url}: the service lists no layer RustGIS can add")) + .ok_or_else(|| crate::sources::message_with_address(url, "the service lists no layer RustGIS can add")) } /// A fetch the command waits for (the command line and the assistant; the app fetches off the diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index 6d6cf14596..d6dc3a6e62 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -51,7 +51,7 @@ fn url_of(p: &Value) -> Result<&str, String> { let url = ps(p, "url").map(str::trim).filter(|u| !u.is_empty()).ok_or("url is required")?; let lower = crate::render::rustgis_tiles::logins::inner(url).to_ascii_lowercase(); if !(lower.starts_with("https://") || lower.starts_with("http://")) { - return Err(format!("{url}: a service is given by its http:// or https:// address")); + return Err(crate::sources::message_with_address(url, "a service is given by its http:// or https:// address")); } Ok(url) } @@ -216,13 +216,14 @@ fn read_documents(s: &mut Session, p: &Value, left: bool) -> Result Result Result { - Err(format!("{url}: in the web app, use Home › Add › Add Data › From Service")) + Err(crate::sources::message_with_address(url, "in the web app, use Home › Add › Add Data › From Service")) } #[cfg(not(target_arch = "wasm32"))] fn fetch_capabilities(url: &str, caps: &str) -> Result { - let bytes = crate::render::rustgis_tiles::cache::download_blocking(caps).map_err(|e| format!("{url}: {e}"))?; - rustgis_data::ogc::parse(caps, &bytes).map_err(|e| format!("{url}: {e}")) + let bytes = crate::render::rustgis_tiles::cache::download_blocking(caps).map_err(|e| crate::sources::message_with_address(url, &e))?; + rustgis_data::ogc::parse(caps, &bytes).map_err(|e| crate::sources::message_with_address(url, &e)) } /// Leaves out the layers whose addresses a document on the web may not send RustGIS to: the @@ -360,7 +361,7 @@ pub(crate) fn download_features(s: &mut Session, name: &str, url: &str, max: usi match download_layer(s, name, url, max, false)? { Downloaded::Bytes(bytes) => Ok(bytes), // Not given without protocol buffer pages. - Downloaded::Read(..) => Err(format!("{url}: the pages were read as protocol buffers")), + Downloaded::Read(..) => Err(crate::sources::message_with_address(url, "the pages were read as protocol buffers")), } } @@ -388,7 +389,7 @@ pub(crate) fn download_layer(s: &mut Session, name: &str, url: &str, max: usize, Step::Done(bytes, note) => break (Downloaded::Bytes(bytes), note), Step::Decoded(dataset, report, note) => break (Downloaded::Read(Box::new((*dataset, report))), note), // A first answer that timed out says how to add a large layer instead. - Step::Failed(e) => return Err(format!("{url}: {}", cache::large_file_hint(e))), + Step::Failed(e) => return Err(crate::sources::message_with_address(url, &cache::large_file_hint(e))), Step::Fetch(pages) => pages, }; let describing = pages.describing(); @@ -431,7 +432,7 @@ pub(crate) fn download_layer(s: &mut Session, name: &str, url: &str, max: usize, fn checked_template(service: &str, t: &str) -> Result { let t = t.trim(); if !(t.contains("{z}") && t.contains("{x}") && t.contains("{y}")) || t.len() > 2000 { - return Err(format!("{t}: not a tile template")); + return Err(crate::sources::message_with_address(t, "not a tile template")); } crate::render::rustgis_tiles::address::check(t, Some(service))?; // The resolver already chose the destination's login, including its absence after a @@ -477,14 +478,14 @@ fn advertised_template_with( for _ in 0..2 { let shown = crate::sources::redact(&doc); if let Err(e) = crate::render::rustgis_tiles::address::check(&doc, Some(list)) { - let e = crate::sources::redact_text(&e); + let e = crate::sources::message_with_address(&doc, &e); out.fallback = Some(format!("{shown}: this tile set document was refused, so the standard tile address is used: {e}")); return out; } let body = match read(&doc) { Ok(body) => body, Err(e) => { - let e = crate::sources::redact_text(&e); + let e = crate::sources::message_with_address(&doc, &e); out.fallback = Some(format!("{shown}: its tile sets couldn't be read, so the standard tile address is used: {e}")); return out; } @@ -501,7 +502,7 @@ fn advertised_template_with( Ok(t) => out.template = Some(t), // (Its error names the template, which may carry keys.) Err(e) => { - let e = crate::sources::redact_text(&e); + let e = crate::sources::message_with_address(&t, &e); out.fallback = Some(format!("{shown}: its tile template was refused, so the standard tile address is used: {e}")); } } @@ -536,7 +537,7 @@ fn left_template(s: &mut Session, source: &str, list: &str, vector: bool, withhe for _ in 0..2 { let shown = crate::sources::redact(&doc); if let Err(e) = crate::render::rustgis_tiles::address::check(&doc, Some(list)) { - let e = crate::sources::redact_text(&e); + let e = crate::sources::message_with_address(&doc, &e); s.notify(format!("{shown}: this tile set document was refused, so the standard tile address is used: {e}"), false); return Ok(LeftTemplate::Read(out)); } @@ -560,7 +561,7 @@ fn left_template(s: &mut Session, source: &str, list: &str, vector: bool, withhe Ok(t) => out.template = Some(t), // (Its error names the template, which may carry keys.) Err(e) => { - let e = crate::sources::redact_text(&e); + let e = crate::sources::message_with_address(&t, &e); s.notify(format!("{shown}: its tile template was refused, so the standard tile address is used: {e}"), false); } } @@ -730,8 +731,8 @@ fn not_listed(url: &str, what: &str, want: &str, service: &Service) -> String { (None, None) => "service.layers lists them".to_string(), }; match past_limit { - Some(why) => format!("{url}: no {what} '{want}' ({listing}; {why})"), - None => format!("{url}: no {what} '{want}' ({listing})"), + Some(why) => crate::sources::message_with_address(url, &format!("no {what} '{want}' ({listing}; {why})")), + None => crate::sources::message_with_address(url, &format!("no {what} '{want}' ({listing})")), } } @@ -798,7 +799,7 @@ fn add_discovered(s: &mut Session, p: &Value, url: &str, want: &str, found: &Dis let quoted = format!("'{}'", want.to_lowercase()); let left_out = |n: &&String| n.contains(rustgis_data::ogc::LEFT_OUT_3D) && n.to_lowercase().contains("ed); match found.service.notes.iter().find(left_out) { - Some(why) => format!("{url}: no layer '{want}': {why}"), + Some(why) => crate::sources::message_with_address(url, &format!("no layer '{want}': {why}")), None => not_listed(url, "layer", want, &found.service), } })?; @@ -1002,9 +1003,13 @@ fn add_discovered(s: &mut Session, p: &Value, url: &str, want: &str, found: &Dis zoom(s); Ok(r) } - Offer::Service { url: open } => { - Err(format!("{url}: '{want}' is a folder or service the directory lists: list its layers (service.layers with url {open})")) - } + Offer::Service { url: open } => Err(crate::sources::message_with_address( + url, + &format!( + "'{want}' is a folder or service the directory lists: list its layers (service.layers with url {})", + crate::sources::address_in_message(open) + ), + )), Offer::Coverage(c) => { let anonymous = capability_crossed || crosses_origin(source, &c.endpoint) || s.web_logins.is_anonymous(); let (get, _, _) = match ps(p, "describe") { @@ -1025,7 +1030,10 @@ fn add_discovered(s: &mut Session, p: &Value, url: &str, want: &str, found: &Dis #[cfg(target_arch = "wasm32")] { let _ = get; - Err(format!("{url}: in the web app, Home › Add › Add Data › From Service downloads the coverage")) + Err(crate::sources::message_with_address( + &url, + "in the web app, Home › Add › Add Data › From Service downloads the coverage", + )) } #[cfg(not(target_arch = "wasm32"))] { @@ -1049,7 +1057,10 @@ fn add_discovered(s: &mut Session, p: &Value, url: &str, want: &str, found: &Dis #[cfg(target_arch = "wasm32")] { let _ = (items, asked); - Err(format!("{url}: in the web app, Home › Add › Add Data › From Service downloads the features")) + Err(crate::sources::message_with_address( + &url, + "in the web app, Home › Add › Add Data › From Service downloads the features", + )) } #[cfg(not(target_arch = "wasm32"))] { @@ -1226,7 +1237,9 @@ fn coverage(s: &mut Session, p: &Value) -> CmdResult { .filter(|(_, source)| ps(p, "source_url").is_none_or(|s| same_address(s, &source.address))) .find(|(l, _)| l.name == want || l.name.eq_ignore_ascii_case(want)) .ok_or_else(|| not_listed(&url, "coverage", want, &found.service))?; - let Offer::Coverage(c) = &layer.offer else { return Err(format!("{url}: '{want}' isn't a WCS coverage")) }; + let Offer::Coverage(c) = &layer.offer else { + return Err(crate::sources::message_with_address(&url, &format!("'{want}' isn't a WCS coverage"))); + }; let anonymous = source.crossed_origin || crosses_origin(&source.address, &c.endpoint) || s.web_logins.is_anonymous() @@ -1271,7 +1284,9 @@ pub(super) fn catalog_search(s: &mut Session, p: &Value) -> CmdResult { let url = url_of(p)?; let request = records_request(s, p)?; let page = match ps(p, "text") { - Some(text) => rustgis_data::ogc::parse_records(&request, text.as_bytes()).map_err(|e| format!("{url}: {e}"))?, + Some(text) => { + rustgis_data::ogc::parse_records(&request, text.as_bytes()).map_err(|e| crate::sources::message_with_address(url, &e))? + } None => fetch_records(url, &request)?, }; let mut dropped = 0usize; @@ -1302,13 +1317,14 @@ pub(super) fn catalog_search(s: &mut Session, p: &Value) -> CmdResult { #[cfg(not(target_arch = "wasm32"))] fn fetch_records(url: &str, request: &str) -> Result { - let bytes = crate::render::rustgis_tiles::cache::download_blocking(request).map_err(|e| format!("{url}: {e}"))?; - rustgis_data::ogc::parse_records(request, &bytes).map_err(|e| format!("{url}: {e}")) + let bytes = + crate::render::rustgis_tiles::cache::download_blocking(request).map_err(|e| crate::sources::message_with_address(url, &e))?; + rustgis_data::ogc::parse_records(request, &bytes).map_err(|e| crate::sources::message_with_address(url, &e)) } #[cfg(target_arch = "wasm32")] fn fetch_records(url: &str, _: &str) -> Result { - Err(format!("{url}: in the web app, use Home › Add › Add Data › From Service › Search Catalogue")) + Err(crate::sources::message_with_address(url, "in the web app, use Home › Add › Add Data › From Service › Search Catalogue")) } #[cfg(all(test, not(target_arch = "wasm32")))] diff --git a/crates/engine/src/cmd/stac.rs b/crates/engine/src/cmd/stac.rs index 8481224ece..fe90d01dc4 100644 --- a/crates/engine/src/cmd/stac.rs +++ b/crates/engine/src/cmd/stac.rs @@ -716,7 +716,7 @@ pub fn stac_cube(s: &mut Session, p: &Value) -> Result { let info = cog::info_blocking(&first)?; let layout = &info.layout; if layout.transform.pixel_height > 0.0 { - return Err(format!("{first} is stored bottom up, which a trend doesn't read yet")); + return Err(crate::sources::message_with_address(&first, "stored bottom up, which a trend doesn't read yet")); } // The level whose cells are nearest the size asked, by ratio (`1000` takes MODIS's 926.6 m // overview, not its 463 m one); the full images by default. @@ -736,9 +736,12 @@ pub fn stac_cube(s: &mut Session, p: &Value) -> Result { // image's size divided by `f` (rounded up), or its cells aren't those blocks. let fits = |full: u32, ov: u32| ov.checked_mul(f as u32) == Some(full); if !(1.0..=64.0).contains(&f) || !fits(full.width, lv.width) || !fits(full.height, lv.height) { - return Err(format!( - "{first}'s overview ({} × {}) isn't its full image ({} × {}) divided by a whole number, so its cells can't be averaged from it: take \"resampling\": \"nearest\"", - lv.width, lv.height, full.width, full.height + return Err(crate::sources::message_with_address( + &first, + &format!( + "its overview ({} × {}) isn't its full image ({} × {}) divided by a whole number, so its cells can't be averaged from it: take \"resampling\": \"nearest\"", + lv.width, lv.height, full.width, full.height + ), )); } (0, f as u32) @@ -886,15 +889,23 @@ impl super::cube::TimeCube for StacCube { // file's header is fetched once however many pieces and dates read it. let info = cog::info_blocking(href)?; if info.layout.crs != self.crs { - return Err(format!("{href} is in another coordinate system than the first file")); + return Err(crate::sources::message_with_address(href, "in another coordinate system than the first file")); } - let lv = info.layout.levels.get(self.level).ok_or_else(|| format!("{href} has fewer overviews than the first file"))?; + let lv = info + .layout + .levels + .get(self.level) + .ok_or_else(|| crate::sources::message_with_address(href, "has fewer overviews than the first file"))?; let lt = info.layout.transform_of(lv); - let (dx, dy) = offset_in(&grid, <).ok_or_else(|| format!("{href} is on another grid of pixels than the first file"))?; + let (dx, dy) = offset_in(&grid, <) + .ok_or_else(|| crate::sources::message_with_address(href, "on another grid of pixels than the first file"))?; // Averaged, a cell is a block of `f` × `f` of the file's pixels: the file's corner // has to fall on a block's, or the blocks would straddle cells. if dx.rem_euclid(i64::from(f)) != 0 || dy.rem_euclid(i64::from(f)) != 0 { - return Err(format!("{href}'s pixels don't fall in whole cells of the area: take \"resampling\": \"nearest\"")); + return Err(crate::sources::message_with_address( + href, + "its pixels don't fall in whole cells of the area: take \"resampling\": \"nearest\"", + )); } // The piece's columns and rows in the file's pixels, within the file. let (fc0, fc1) = (dx.max(0), (dx + wf).min(i64::from(lv.width))); diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index cdd44e412f..7622f3da80 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -232,7 +232,10 @@ pub(crate) fn add_with_metadata_wait(s: &mut Session, p: &Value, metadata_wait: // (`start`, `end`) turns on when it is off. let dated = rustgis_data::time::is_template(url); if dated && !matches!(kind, TileKind::Raster | TileKind::Vector | TileKind::Cog) { - return Err(format!("{url}: dated addresses work for raster, vector and GeoTIFF tile layers, not {}", kind.code())); + return Err(crate::sources::message_with_address( + url, + &format!("dated addresses work for raster, vector and GeoTIFF tile layers, not {}", kind.code()), + )); } let at = if dated { dated_start(s, p)? } else { 0.0 }; let mut t = TileLayer::new( @@ -483,7 +486,7 @@ pub(crate) fn follow_style_view(s: &mut Session, wait: web_time::Duration) -> bo Ok(None) => return false, Err(e) => { s.style_view = None; - s.notify(format!("{url}: {e}"), true); + s.notify(crate::sources::message_with_address(&url, &e), true); return false; } } @@ -496,7 +499,7 @@ pub(crate) fn follow_style_view(s: &mut Session, wait: web_time::Duration) -> bo match super::navigate::go_to_view(s, center, view.zoom, view.bearing, view.pitch) { Ok(()) => true, Err(e) => { - s.notify(format!("{url}: {e}"), true); + s.notify(crate::sources::message_with_address(&url, &e), true); false } } @@ -862,8 +865,11 @@ fn legends(s: &mut Session, connection: Option<&str>, v: &Value) -> Result Result<(HashMap>, usize if failure.lock().is_ok_and(|f| f.is_some()) { break; } - // A missing file is an expected answer here (see `is_absent`), not a failure: its 404 - // or 403 isn't recorded one by one (one note per server says the - // omission, and a store refusing everything still says "access denied"), while - // a real one (a 500, a timeout) does, with its address, status and duration. + // A missing file is an expected answer here (see `is_absent`), not a failure: + // one omission note replaces its 404/403 diagnostics; real failures still say why. let got = crate::render::rustgis_tiles::cache::download_blocking_omitting(&format!("{root}/{k}"), &[404, 403]); match got { Ok(b) => { let len = b.len() as u64; if total.fetch_add(len, std::sync::atomic::Ordering::Relaxed).saturating_add(len) > zarr::MAX_READ { if let Ok(mut f) = failure.lock() { - f.get_or_insert(format!( - "{root}: more than {} GB to read: take a smaller part", - zarr::MAX_READ >> 30 + f.get_or_insert(crate::sources::message_with_address( + root, + &format!("more than {} GB to read: take a smaller part", zarr::MAX_READ >> 30), )); } break; @@ -133,7 +131,7 @@ fn fetch(root: &str, keys: &[String]) -> Result<(HashMap>, usize } Err(e) => { if let Ok(mut f) = failure.lock() { - f.get_or_insert(format!("{root}/{k}: {e}")); + f.get_or_insert(crate::sources::message_with_address(&format!("{root}/{k}"), &e)); } } } @@ -208,10 +206,10 @@ fn store_at(url: &str) -> Result { let (got, forbidden) = fetch(url, &keys)?; // Every metadata file refused: access is denied, not a store without metadata. if got.is_empty() && forbidden > 0 { - return Err(format!("{url}: access denied (HTTP 403)")); + return Err(crate::sources::message_with_address(url, "access denied (HTTP 403)")); } let docs: Vec<(String, Vec)> = got.into_iter().collect(); - zarr::parse_store(&docs).map_err(|e| format!("{url}: {e}")) + zarr::parse_store(&docs).map_err(|e| crate::sources::message_with_address(url, &e.to_string())) } fn info(_s: &mut Session, p: &Value) -> CmdResult { @@ -327,13 +325,17 @@ pub(crate) fn add(s: &mut Session, p: &Value) -> CmdResult { Source::Url(u) => { let store = store_at(&u)?; pick_variable(&store, &mut sel)?; - let plan = zarr::plan(&store, &sel).map_err(|e| format!("{u}: {e}"))?; + let plan = zarr::plan(&store, &sel).map_err(|e| crate::sources::message_with_address(&u, &e.to_string()))?; let (chunks, forbidden) = fetch(&u, &plan.keys)?; if forbidden > 0 { - return Err(format!("{u}: access denied (HTTP 403) to {forbidden} of the slice's chunks")); + return Err(crate::sources::message_with_address( + &u, + &format!("access denied (HTTP 403) to {forbidden} of the slice's chunks"), + )); } let name = ps(p, "name").map(str::to_string).unwrap_or_else(|| layer_name(&u, &sel, &[])); - let mut r = zarr::assemble(&store, &plan, &chunks, &name, crs_parsed).map_err(|e| format!("{u}: {e}"))?; + let mut r = zarr::assemble(&store, &plan, &chunks, &name, crs_parsed) + .map_err(|e| crate::sources::message_with_address(&u, &e.to_string()))?; crs_lost(&mut r, crs); r.source = Some(u.clone()); // Kept in the project as a GeoTIFF: a web store isn't fetched again when it opens. @@ -542,7 +544,9 @@ pub(crate) fn read_step(zt: &ZarrTime, i: usize, key: String) -> Result Result 0 { - return Err(format!("{}: access denied (HTTP 403) to {forbidden} of the slice's chunks", zt.store)); + return Err(crate::sources::message_with_address( + &zt.store, + &format!("access denied (HTTP 403) to {forbidden} of the slice's chunks"), + )); } let crs = crs.map(Crs::parse).transpose()?; zarr::assemble(&st, &plan, &chunks, &sel, crs) @@ -1110,7 +1117,7 @@ fn add_following(s: &mut Session, p: &Value) -> CmdResult { zt.repair(); // The first step read before the slider is touched, so a store that can't be read leaves // the map as it was. - let first = read_step(&zt, 0, slice_key(&zt, 0)).map_err(|e| format!("{root}: {e}"))?; + let first = read_step(&zt, 0, slice_key(&zt, 0)).map_err(|e| crate::sources::message_with_address(&root, &e))?; // The slider, on over the variable's dates when it is off. if s.map().time.is_none() { let dates: Vec = zt.dates.iter().map(|d| serde_json::json!(d)).collect(); @@ -1120,7 +1127,7 @@ fn add_following(s: &mut Session, p: &Value) -> CmdResult { let want = zt.step_for(window); // The first slice the layer shows sets its style; with no step in the window, the first step's. let r = match want { - Some(i) if i != 0 => read_step(&zt, i, slice_key(&zt, i)).map_err(|e| format!("{root}: {e}"))?, + Some(i) if i != 0 => read_step(&zt, i, slice_key(&zt, i)).map_err(|e| crate::sources::message_with_address(&root, &e))?, _ => first, }; let mut layer = RasterLayer::from_path((*r).clone(), root.clone()); @@ -1228,12 +1235,12 @@ fn store_files(root: &str, keys: &[String]) -> Result>, let (got, forbidden) = fetch(root, batch)?; // A file refused (HTTP 403) isn't one never written: said, not read as missing. if forbidden > 0 { - return Err(format!("{root}: access denied (HTTP 403) to {forbidden} of its files")); + return Err(crate::sources::message_with_address(root, &format!("access denied (HTTP 403) to {forbidden} of its files"))); } for (k, b) in got { total = total.saturating_add(b.len() as u64); if total > zarr::MAX_READ { - return Err(format!("{root}: the series is too large to read: take a shorter period")); + return Err(crate::sources::message_with_address(root, "the series is too large to read: take a shorter period")); } out.insert(k, b); } diff --git a/crates/engine/src/durable.rs b/crates/engine/src/durable.rs index 7f298e0433..6b333fcdf3 100644 --- a/crates/engine/src/durable.rs +++ b/crates/engine/src/durable.rs @@ -348,6 +348,10 @@ impl<'a> Prepare<'a> { pub(crate) fn generated_text(source: &str) -> String { let public = crate::project::without_recorded_login_bindings(source); let source = public.as_deref().unwrap_or(source); + // Address scrubbing keeps later prose, but an unknown secret value may run past whitespace. + if let Some(strict) = rustgis_data::diagnostics::secret_continuation_cut(source) { + return crate::render::rustgis_tiles::logins::scrub(&strict); + } let (clean, _) = rustgis_data::address::scrub_addresses(source); crate::render::rustgis_tiles::logins::scrub(&clean) } diff --git a/crates/engine/src/policy/downloads.rs b/crates/engine/src/policy/downloads.rs index 3e08df13ed..c529d8591f 100644 --- a/crates/engine/src/policy/downloads.rs +++ b/crates/engine/src/policy/downloads.rs @@ -121,7 +121,11 @@ impl Session { return; } if self.url_waits.holds(n, p) { - self.url_waits.land(n, Err(crate::sources::redact_text(error)), crate::cmd::plugins::fetch::Added::default()); + let error = p + .get("url") + .and_then(Value::as_str) + .map_or_else(|| Self::redacted(error).into_string(), |url| crate::sources::message_with_address(url, error)); + self.url_waits.land(n, Err(error), crate::cmd::plugins::fetch::Added::default()); } if self.url_waits.given_up(n, p) { self.url_waits.refused(n); @@ -145,7 +149,7 @@ impl Session { self.landed_reads.retain(|(k, _)| *k != id); #[cfg(not(target_arch = "wasm32"))] self.landed_windows.retain(|(k, _)| *k != id); - let url = crate::sources::redact_text(p.get("url").and_then(Value::as_str).unwrap_or("download")); + let url = crate::sources::address_in_message(p.get("url").and_then(Value::as_str).unwrap_or("download")); if added { Ok(Some(format!("{url}: it was added already for its plugin command, so it wasn't added again"))) } else { @@ -261,7 +265,11 @@ impl Session { }; if (given.is_some() && waiting != Some(number)) || (waiting.is_none() && !self.url_waits.has(number)) { if self.policy.work.queued.len() >= crate::MAX_URL_REQUESTS { - return Err(format!("{} downloads are already waiting; try again when they have landed", crate::MAX_URL_REQUESTS)); + let reason = format!("{} downloads are already waiting; try again when they have landed", crate::MAX_URL_REQUESTS); + return Err(p + .get("url") + .and_then(Value::as_str) + .map_or_else(|| Self::redacted(&reason).into_string(), |url| crate::sources::message_with_address(url, &reason))); } let mut params = asked.clone(); params.as_object_mut().ok_or("download parameters must be an object")?.insert("request".into(), json!(number)); @@ -481,7 +489,12 @@ impl Session { if let Some(number) = claim.params.get("request").and_then(Value::as_u64) && self.url_waits.holds(number, &claim.params) { - self.url_waits.land(number, Err(crate::sources::redact_text(error)), crate::cmd::plugins::fetch::Added::default()); + let error = claim + .params + .get("url") + .and_then(Value::as_str) + .map_or_else(|| Self::redacted(error).into_string(), |url| crate::sources::message_with_address(url, error)); + self.url_waits.land(number, Err(error), crate::cmd::plugins::fetch::Added::default()); } if claim.import { crate::cmd::geolibre::discard_pending(self, &claim.params); diff --git a/crates/engine/src/project_log.rs b/crates/engine/src/project_log.rs index 3d6f75f7fc..1e689aa009 100644 --- a/crates/engine/src/project_log.rs +++ b/crates/engine/src/project_log.rs @@ -191,7 +191,7 @@ fn review_value(v: &Value, depth: usize) -> Result { } } out.push_str(rest); - Value::String(crate::cmd::history::redacted_text(&sql_secrets_removed(&out))) + Value::String(crate::cmd::history::redacted_parameter(&sql_secrets_removed(&out))) } other => other.clone(), }) diff --git a/crates/engine/src/service_logins.rs b/crates/engine/src/service_logins.rs index 2392328dc1..383e49c565 100644 --- a/crates/engine/src/service_logins.rs +++ b/crates/engine/src/service_logins.rs @@ -282,8 +282,12 @@ pub(crate) fn bind_logged_in(s: &Session, connection: &str, url: &str) -> Result return Ok(url.to_string()); } let Some(active) = s.web_logins.active.get(&c.id) else { return Ok(url.to_string()) }; - logins::bind(&active.binding, url) - .map_err(|_| format!("{url}: this address can't carry the connection's login (it has a space, a control character or a backslash)")) + logins::bind(&active.binding, url).map_err(|_| { + crate::sources::message_with_address( + url, + "this address can't carry the connection's login (it has a space, a control character or a backslash)", + ) + }) } pub fn known_secret(s: &Session, c: &Connection) -> Result { diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index aa349ef1f5..bc3aa7eeff 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -313,29 +313,69 @@ pub fn holds_plain_key(params: &serde_json::Value) -> bool { walk(params, 0) } -/// Text with every web address in it redacted (`sources::redact`): an error's text may quote the -/// address asked for, or one a redirect led to, keys and all. Each address is redacted alone, so -/// the words around it stay; punctuation that ends a sentence isn't taken as part of it. -pub fn redact_text(text: &str) -> String { - text.split_inclusive(char::is_whitespace) - .map(|piece| { - let word = piece.trim_end(); - let Some(at) = word.find("://") else { return piece.to_string() }; - // The address starts at its scheme, after any quote or bracket before it. - let scheme = |c: &char| c.is_ascii_alphanumeric() || "+.-".contains(*c); - let start = word.get(..at).and_then(|b| b.char_indices().rev().find(|(_, c)| !scheme(c))).map_or(0, |(i, c)| i + c.len_utf8()); - let end = word.trim_end_matches([':', ',', ';', '.', ')', ']', '>', '"', '\'', '»', '›', '”', '’']).len().max(at + 3); - let (before, address, after) = - (word.get(..start).unwrap_or(""), word.get(start..end).unwrap_or(""), word.get(end..).unwrap_or("")); - // A scheme alone names no address ("give an http:// or s3:// address"), nor does one - // with an ellipsis for the rest (a hint's `https://…`); the notice redaction keeps - // them too. - if rustgis_data::address::scheme_example(address.trim_end_matches('…')) { - return piece.to_string(); - } - format!("{before}{}{after}{}", redact(address), piece.get(word.len()..).unwrap_or("")) - }) - .collect() +/// A known source address safe to compose into a repeatedly redacted diagnostic, without +/// publishing a web-login runtime binding or the credentials inside a source wrapper. +pub fn address_in_message(address: &str) -> String { + let mut shown = rustgis_data::diagnostics::address_in_message(&redact(address)); + // Source redaction can take out a secret query whole; the diagnostic label still says one went. + let raw = rustgis_data::diagnostics::address_in_message(address); + if let Some(marker) = ["[query omitted]", "[fragment omitted]"].into_iter().find(|m| raw.ends_with(m)) + && !shown.ends_with(marker) + && !shown.ends_with("omitted]") + { + shown.push_str(marker); + } + shown +} + +/// An error naming a long known address shortened to `limit` characters, then `reason` (fixed +/// prose, not from the network). A trailing omission marker (`[query omitted]`) is kept whole +/// and the address before it is cut instead, marked `…`; a cut never leaves half a marker. The +/// shortened address is already sanitized, so it isn't parsed again as an address. +pub fn shortened_message_with_address(address: &str, limit: usize, reason: &str) -> String { + let full = address_in_message(address); + if full.chars().count() <= limit { + return message_with_address(address, reason); + } + let (head, marker) = match full.rfind('[') { + Some(at) if full.ends_with(']') => full.split_at(at), + _ => (full.as_str(), ""), + }; + let mut shown: String = head.chars().take(limit.saturating_sub(marker.chars().count() + 1)).collect(); + if let Some(at) = shown.rfind('[').filter(|&at| !shown.get(at..).unwrap_or("").contains(']')) { + shown.truncate(at); + } + shown.push('…'); + shown.push_str(marker); + format!("{shown}: {reason}") +} + +/// Names a known source while keeping its failure reason. Exact address occurrences are +/// sanitized before the conservative free-text redactor sees any unescaped secret whitespace. +pub fn message_with_address(address: &str, reason: &str) -> String { + let message = rustgis_data::diagnostics::message_with_address(address, reason); + let raw_shown = rustgis_data::diagnostics::address_in_message(address); + let shown = address_in_message(address); + let message = if raw_shown.is_empty() || raw_shown == shown { message } else { message.replace(&raw_shown, &shown) }; + // Reusing this helper must not prepend the same safe source label on every hop. + // Only a verified companion belongs to the same download; unrelated targets keep their origin. + let reason = message.strip_prefix(&shown).and_then(|tail| tail.strip_prefix(": ")); + let message = match reason.filter(|reason| { + reason.strip_prefix(&shown).is_some_and(|tail| tail.is_empty() || tail.starts_with(": ")) + || reason.split_once(": ").is_some_and(|(label, _)| { + label.contains("://") + && rustgis_data::io::shapefile_part_urls(address).is_some_and(|parts| { + // The part's label may come from either layer's composition. + parts.iter().any(|(_, part)| { + address_in_message(part) == label || rustgis_data::diagnostics::address_in_message(part) == label + }) + }) + }) + }) { + Some(reason) => reason, + None => &message, + }; + crate::render::rustgis_tiles::logins::scrub(message) } /// A PMTiles or COG tile template (`pmtiles:{z}/{x}/{y}@archive`, `cog:…@archive`): what comes @@ -701,6 +741,50 @@ MIIE assert!(s.recent[0].is("service", "https://h/wms")); } + #[test] + fn address_messages_stay_private_and_named_once_across_hops() { + for address in [ + "https://alice:hidden@host.example/data?token=hidden hidden-tail", + "https://host.example/data?page=2&token=hidden", + "https://host.example/data?custom=hidden hidden-tail", + ] { + let mut message = message_with_address(address, "HTTP 503 Service Unavailable"); + for _ in 0..3 { + message = message_with_address(address, &message); + let clean = crate::Session::redacted(&message); + assert!(clean.as_str().contains("HTTP 503 Service Unavailable"), "{clean:?}"); + assert_eq!(clean.as_str().matches("host.example/data").count(), 1, "{clean:?}"); + assert!(!clean.as_str().contains("hidden") && !clean.as_str().contains("alice"), "{clean:?}"); + } + } + } + + #[test] + fn unrelated_target_failures_retain_the_requested_source() { + let requested = "https://source.example/roads.shp?token=hidden-source"; + let target = "http://127.0.0.1/elsewhere.dbf?token=hidden-target"; + let cause = crate::render::rustgis_tiles::address::check(target, Some(requested)).unwrap_err(); + let mut message = message_with_address(requested, &cause); + for _ in 0..3 { + assert!(message.contains("source.example/roads.shp") && message.contains("127.0.0.1/elsewhere.dbf"), "{message}"); + assert!(message.contains(&cause) && !message.contains("hidden"), "{message}"); + assert_eq!(message.matches("source.example/roads.shp").count(), 1, "{message}"); + assert_eq!(crate::Session::redacted(&message).as_str(), message); + message = message_with_address(requested, &message); + } + } + + #[test] + fn shapefile_parts_with_secret_queries_are_named_once() { + let shp = "https://h.example/roads.shp?token=hidden"; + let part = "https://h.example/roads.dbf?token=hidden"; + // A part failure as the data layer writes it, passed back through the source's helper. + let label = rustgis_data::diagnostics::message_with_address(part, "http status: 404"); + let message = message_with_address(shp, &label); + assert!(message.contains("roads.dbf") && message.contains("404") && !message.contains("hidden"), "{message}"); + assert_eq!(message.matches("://").count(), 1, "{message}"); + } + #[test] fn secrets_are_never_kept() { assert_eq!(redact("https://me:pw@host.example/a?token=abc&layer=x&X-Amz-Signature=1#k"), "https://host.example/a?layer=x"); @@ -793,31 +877,38 @@ MIIE #[test] fn a_scheme_alone_in_prose_is_no_address() { + // Notices and errors go through the one diagnostic redactor: a supported scheme alone in + // prose names no address and is kept. for text in [ "give an http://, https:// or s3:// address", "needs an http:// or https:// address", "uses unencrypted http://. Tick the confirmation", "only http:// and https:// addresses are fetched", - "for example https://…", "(http://) or \"https://\" or 's3://'", "HTTP:// and Https://", - "file:// and ftp://", ] { - assert_eq!(redact_text(text), text); + assert_eq!(crate::Session::redacted(text).as_str(), text); + // An error naming the address it refused keeps those words, and its key stays out. + let error = message_with_address("https://h.example/a?token=hidden", text); + assert_eq!(error, format!("https://h.example/a[query omitted]: {text}")); + assert_eq!(crate::Session::redacted(&error).as_str(), error); } - // Anything after the scheme is an address, redacted as before. + // Anything after the scheme is an address: what can hold a credential goes. A hint's + // `https://…` is read as its scheme; other schemes are taken for addresses. for (text, kept) in [ + ("for example https://…", "for example https:"), + ("file:// and ftp://", "[address omitted] and [address omitted]"), ("see https://user:pw@host/x, or http://", "see https://host/x, or http://"), ("https://user:pw@", "https:"), ("https://user:pw@…", "https:"), ("https://:pw@ failed", "https: failed"), - ("https://?token=x", "https:"), + ("https://?token=x", "[address omitted][query and the rest omitted]"), ("https://#x", "https:"), - ("https://…?token=x", "https:"), + ("https://…?token=x", "[address omitted][query and the rest omitted]"), ("http:///", "http:"), - ("s3://bucket/a.tif?X-Amz-Signature=s", "s3://bucket/a.tif"), + ("s3://bucket/a.tif?X-Amz-Signature=s", "s3://bucket/a.tif[query and the rest omitted]"), ] { - assert_eq!(redact_text(text), kept, "{text}"); + assert_eq!(crate::Session::redacted(text).as_str(), kept, "{text}"); } } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 92343c33b8..a294014d9a 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -9752,15 +9752,26 @@ B,11,51 assert_eq!((s.map().layers.len(), s.undo.len()), (layers + 1, undo), "added once"); assert!(s.take_url_requests().is_empty() && !s.plugin_retries_waiting()); assert!(s.poll_plugin_retries().is_empty(), "run once"); - // A download that fails: the command runs again, and its `layer.addUrl` answers the error. - let q = json!({"op": "layer.addUrl", "params": {"url": "https://x.org/gone.csv"}}); - s.run_leaving_downloads("plugin:eng-rerun/f", &q).unwrap(); - let asked = s.take_url_requests(); - s.request_failed(asked[0].params(), "https://x.org/gone.csv?token=hunter2: 404 Not Found"); - let ran = s.poll_plugin_retries(); - let e = ran[0].1.as_ref().unwrap_err(); - assert!(e.contains("404") && !e.contains("hunter2"), "{ran:?}"); - assert!(!s.plugin_retries_waiting()); + // A failed download runs the command again. A known address keeps its status, even with + // unescaped query whitespace; an unknown URL's secret tail is not safe to retain. + for (url, reason, keeps_status) in [ + // The reason adds an unknown secret query to the requested address: its value may continue. + ("https://x.org/gone.csv", "https://x.org/gone.csv?token=hunter2: 404 Not Found", false), + ("https://x.org/gone.csv?token=hunter2 hidden-tail", "https://x.org/gone.csv?token=hunter2 hidden-tail: 404 Not Found", true), + ("https://x.org/gone.csv", "https://x.org/gone.csv?token=hunter2 hidden-tail: 404 Not Found", false), + ] { + let q = json!({"op":"layer.addUrl","params":{"url":url}}); + s.run_leaving_downloads("plugin:eng-rerun/f", &q).unwrap(); + let asked = s.take_url_requests(); + s.request_failed(asked[0].params(), reason); + let ran = s.poll_plugin_retries(); + let e = ran[0].1.as_ref().unwrap_err(); + assert!(!e.contains("hunter2") && !e.contains("hidden-tail"), "{ran:?}"); + if keeps_status { + assert!(e.contains("404"), "{ran:?}"); + } + assert!(!s.plugin_retries_waiting()); + } // A native request and a plugin request get distinct engine routing numbers. let next = s.next_request + 1; s.request_url(&json!({"url": "https://x.org/other.csv", "request": next})).unwrap(); @@ -13556,6 +13567,46 @@ fn url_requests_wait_for_the_front_end() { assert!(s.request_url(&json!({"url": "https://x.org/16"})).is_ok(), "room again once abandoned"); } +#[test] +fn url_validation_keeps_its_reason_without_unescaped_query_tails() { + let mut s = Session::new(Project::default()); + for suffix in ["?token=hidden hidden-tail", "?%2574oken=hidden hidden-tail", "?custom=hidden hidden-tail", "#hidden hidden-tail"] { + let url = format!("https://validation.example/data.tif{suffix}"); + let mark = s.notice_mark(); + let error = s.run("layer.addUrl", &json!({"url":url, "cellSize":10})).unwrap_err(); + assert!(error.contains("\"cellSize\" goes with \"bbox\""), "{error}"); + assert!(!error.contains("hidden"), "{error}"); + let notice = s.notices_since(mark).last().unwrap().text.clone(); + assert!(notice.contains("\"cellSize\" goes with \"bbox\"") && !notice.contains("hidden"), "{notice}"); + let clean = Session::redacted(&Session::redacted(&error).into_string()); + assert!(clean.as_str().contains("\"cellSize\" goes with \"bbox\"") && !clean.as_str().contains("hidden"), "{clean:?}"); + } + let report = s.diagnostics.report(rustgis_data::diagnostics::Filter::All); + assert!(report.contains("\"cellSize\" goes with \"bbox\"") && !report.contains("hidden"), "{report}"); +} + +#[test] +fn full_download_queue_keeps_its_reason_through_session_notices() { + let mut s = Session::new(Project::default()); + for i in 0..16 { + s.request_url(&json!({"url": format!("https://queue.example/{i}")})).unwrap(); + } + for suffix in ["?token=queue-secret", "#queue-secret", "?token=queue-secret secret-tail", "#queue-secret secret-tail"] { + let url = format!("https://queue.example/refused.geojson{suffix}"); + let mark = s.notice_mark(); + let error = s.guarded("Add Data From URL", true, |s| s.request_url(&json!({"url": url}))).unwrap_err(); + assert!(error.contains("16 downloads are already waiting; try again when they have landed"), "{error}"); + assert!(!error.contains("queue-secret") && !error.contains("secret-tail"), "{error}"); + assert_eq!(Session::redacted(&error).as_str(), error); + assert_eq!(s.notices_since(mark).iter().map(|n| n.text.as_str()).collect::>(), [error.as_str()]); + s.notify(error.clone(), true); + assert_eq!(s.notices_since(mark).last().unwrap().text, error); + assert!(s.diagnostics.records().iter().any(|r| r.message == error)); + } + let report = s.diagnostics.report(rustgis_data::diagnostics::Filter::All); + assert!(!report.contains("queue-secret") && !report.contains("secret-tail"), "{report}"); +} + /// A mutating command that only left a request for the front end (the browser's `layer.addUrl`) /// leaves no undo step and nothing to save; one that changed the project still does, and so /// does one that merely answers "pending" (a plugin's command can't claim it changed nothing). @@ -16546,6 +16597,35 @@ fn notices_since_survives_the_cap() { assert_eq!(s.notices_since(0).len(), 200); } +#[test] +fn service_command_keeps_missing_layer_reason_and_scrubs_every_known_address() { + const WMS: &str = r#" +Test WMS +image/png +Root +roadsRoads"#; + let mut s = Session::new(Project::default()); + for suffix in ["?token=service-secret", "#service-secret", "?token=service-secret secret-tail", "#service-secret secret-tail"] { + let url = format!("https://maps.example.com/wms{suffix}"); + let missing = format!("{url} again {url}"); + let mark = s.notice_mark(); + let error = s.run("service.add", &json!({"url": url, "type": "wms", "layer": missing, "text": WMS})).unwrap_err(); + if suffix.contains(' ') { + // Invalid addresses are refused before capabilities are read. + assert!(error.contains("the address has spaces in it"), "{error}"); + } else { + assert!(error.contains("no layer '") && error.ends_with("(service.layers lists them)"), "{error}"); + assert_eq!(error.matches("maps.example.com/wms").count(), 3, "{error}"); + } + assert!(!error.contains("service-secret") && !error.contains("secret-tail"), "{error}"); + assert_eq!(Session::redacted(&error).as_str(), error); + assert_eq!(s.notices_since(mark).iter().map(|n| n.text.as_str()).collect::>(), [error.as_str()]); + assert!(s.diagnostics.records().iter().any(|r| r.source == "service.add" && r.message == error)); + } + let report = s.diagnostics.report(rustgis_data::diagnostics::Filter::All); + assert!(!report.contains("service-secret") && !report.contains("secret-tail"), "{report}"); +} + #[test] fn diagnostics_capture_notices_without_an_open_pane_or_project_persistence() { use rustgis_data::diagnostics::{Filter, MAX_DETAILS_BYTES, MAX_MESSAGE_BYTES, Severity}; @@ -20470,6 +20550,51 @@ fn zarr_store_at(tag: &str, days: &[i32]) -> (std::path::PathBuf, String) { (dir, path) } +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn timed_zarr_open_failures_keep_status_without_store_credentials() { + use ogc_server::{Route, Server}; + let _cache = zarr_cache_lock(); + let (dir, path) = zarr_store_over_time("private-open-errors"); + let store = std::path::Path::new(&path); + let mut routes = Vec::new(); + for case in ["first", "selected"] { + let root = format!("/{case}?token=hidden-zarr-open"); + for key in [".zmetadata", "lat/0", "lon/0", "time/0"] { + routes.push(Route::bytes(&format!("{root}/{key}"), std::fs::read(store.join(key)).unwrap())); + } + if case == "selected" { + routes.push(Route::bytes(&format!("{root}/t/0.0.0"), std::fs::read(store.join("t/0.0.0")).unwrap())); + } + let step = if case == "first" { 0 } else { 1 }; + routes.push(Route { status: 503, ..Route::new(&format!("{root}/t/{step}.0.0"), "") }); + } + let server = Server::new(routes); + for case in ["first", "selected"] { + let mut s = Session::new(Project::default()); + if case == "selected" { + s.run("time.set", &json!({"dates":["2001-01-01","2001-02-01","2001-03-01"],"date":"2001-02-01"})).unwrap(); + } + let url = format!("{}/{case}?token=hidden-zarr-open", server.base); + let error = s.run("layer.addZarr", &json!({"url":url,"variable":"t","time":true})).unwrap_err(); + assert!(error.contains("503") && !error.contains("hidden-zarr-open"), "{case}: {error}"); + assert!(s.map().layers.is_empty(), "an unreadable initial slice must not add a layer"); + let mark = s.notice_mark(); + s.notify(error, true); + let notices = s.notices_since(mark); + assert!(notices.iter().any(|n| n.error && n.text.contains("503")), "{case}: {notices:?}"); + assert!(notices.iter().all(|n| !n.text.contains("hidden-zarr-open")), "{case}: {notices:?}"); + } + let requests = server.requests(); + for (case, chunk) in [("/first", "t/0.0.0"), ("/selected", "t/0.0.0"), ("/selected", "t/1.0.0")] { + assert!( + requests.iter().any(|r| r.path == case && r.query.iter().any(|(key, value)| key == "token" && value.ends_with(chunk))), + "the first and selected slices must actually be read: {requests:?}" + ); + } + let _ = std::fs::remove_dir_all(dir); +} + /// A Zarr variable's series at a point (#555): the values along its time dimension at the cell /// holding it, read from the chunks holding that cell, through a layer or the store itself. #[cfg(not(target_arch = "wasm32"))] @@ -25533,7 +25658,11 @@ fn a_run_is_copied_from_the_history_as_code() { let text = out["text"].as_str().unwrap(); assert!(text.contains("import rustgis\n\ns = rustgis.Session(") && text.contains("# 1. Buffer ("), "{text}"); assert!(text.contains("UTC)\ns.run(\n \"gp.run\""), "{text}"); - assert!(text.contains("# 3. Buffer: failed when it ran (no layer https://h.example/a)\n# s.run("), "{text}"); + assert_eq!( + text.lines().filter(|line| line.starts_with("s.run(")).count(), + 2, + "failed or unrecognized runs must not become executable Python: {text}" + ); assert!(text.contains("# 4. Mystery: Mystery can't be run again"), "{text}"); assert!(text.contains("# 5. Buffer: Buffer can't be run again") && !text.contains("file.delete"), "{text}"); // Text from the file stays in its comment, whatever line break it holds (a lone `\r` ends a @@ -25614,6 +25743,78 @@ fn secrets_in_a_query_or_a_run_s_error_stay_out_of_copied_code() { ); } +#[test] +fn history_exports_never_keep_unknown_url_credential_continuations() { + let mut s = world(); + let raw = "GET https://history.example/data?token=hidden hidden-tail: HTTP 503"; + s.project.gp_history.push(crate::GpRecord { + tool: "Buffer".into(), + id: "buffer".into(), + params: json!({ + "input":"Cities", + "note":raw, + "url":"https://history.example/data?page=2&token=hidden", + "markup":"hidden hidden-tail https://history.example/data", + "payload":r#"{"password":"hidden hidden-tail","link":"https://history.example/data"}"#, + "compact":r#"{"password":"hidden"}"#, + "markupWithoutUrl":"hidden", + }), + command: None, + at: 0, + ok: false, + messages: vec![raw.into()], + millis: 0, + outputs: Vec::new(), + }); + // Upstream copied code keeps opaque step text replayable and says so; typed addresses and the + // failed run's diagnostic still lose their credentials. + let code = s.run("gp.historyCode", &json!({"run":0})).unwrap()["code"].as_str().unwrap().to_string(); + assert!(code.contains("https://history.example/data?page=2") && !code.contains("page=2&token"), "{code}"); + assert!(s.notices.iter().any(|n| n.text.contains("opaque parameters"))); + let script = s.run("gp.historyScript", &Value::Null).unwrap()["text"].as_str().unwrap().to_string(); + let failure = script.lines().find(|l| l.contains("failed when it ran")).unwrap_or_default(); + assert!(failure.contains("GET https://history.example/data") && !failure.contains("hidden"), "{script}"); + // The parameter sanitizer used for review/redacted output removes every structured secret. + let clean = serde_json::to_string(&crate::cmd::history::redacted(&s.project.gp_history[0].params)).unwrap(); + assert!(!clean.contains("hidden") && clean.contains("https://history.example/data?page=2"), "{clean}"); +} + +#[test] +fn copied_history_replays_public_structured_parameters_without_changing_results() { + let mut s = Session::new(Project::default()); + s.run( + "layer.addDataText", + &json!({ + "fileName":"Words.geojson", + "text":r#"{"type":"FeatureCollection","features":[ + {"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{"word":"A"}}, + {"type":"Feature","geometry":{"type":"Point","coordinates":[1,1]},"properties":{"word":"B"}} + ]}"#, + }), + ) + .unwrap(); + for separator in ["
https://example.org
", r#"{"label":"public","link":"https://example.org?format=json"}"#] { + s.run( + "gp.run", + &json!({ + "tool":"summaryStatistics", + "params":{"input":"Words","statistics":[{"field":"word","statistic":"CONCATENATE"}],"separator":separator} + }), + ) + .unwrap(); + let run = s.project.gp_history.len() - 1; + let copied = s.run("gp.historyCode", &json!({"run":run,"as":"command"})).unwrap(); + let (command, params) = copied["code"].as_str().unwrap().split_once(' ').unwrap(); + let params: Value = serde_json::from_str(params).unwrap(); + s.run(command, ¶ms).unwrap(); + let replayed = s.project.tables.last().unwrap(); + assert_eq!( + replayed.value(0, replayed.field_index("CONCATENATE_word").unwrap()), + &rustgis_data::Value::Text(format!("A{separator}B")) + ); + } +} + #[path = "../../../tests/support/ogc_server.rs"] mod ogc_server; diff --git a/crates/engine/src/tests/service_logins.rs b/crates/engine/src/tests/service_logins.rs index da9ac523a0..bc2ec07a44 100644 --- a/crates/engine/src/tests/service_logins.rs +++ b/crates/engine/src/tests/service_logins.rs @@ -73,6 +73,7 @@ impl LoginFile { } Err(_) => break, }; + // Accepted sockets can inherit the listener's nonblocking mode. stream.set_nonblocking(false).unwrap(); stream.set_read_timeout(Some(std::time::Duration::from_secs(5))).unwrap(); let mut request = Vec::new(); diff --git a/crates/geolibre/src/install.rs b/crates/geolibre/src/install.rs index bcfeb76a1f..8711c2ffa0 100644 --- a/crates/geolibre/src/install.rs +++ b/crates/geolibre/src/install.rs @@ -44,7 +44,11 @@ fn agent() -> ureq::Agent { } fn get(agent: &ureq::Agent, url: &str) -> Result, String> { - agent.get(url).header("User-Agent", concat!("RustGIS/", env!("CARGO_PKG_VERSION"))).call().map_err(|e| format!("{url}: {e}")) + agent + .get(url) + .header("User-Agent", concat!("RustGIS/", env!("CARGO_PKG_VERSION"))) + .call() + .map_err(|e| rustgis_data::diagnostics::message_with_address(url, &e.to_string())) } /// The latest release's tag and the archive and checksum URLs for `target`. diff --git a/crates/map/src/qgis/tests.rs b/crates/map/src/qgis/tests.rs index 91a27c4147..9ca845e00a 100644 --- a/crates/map/src/qgis/tests.rs +++ b/crates/map/src/qgis/tests.rs @@ -860,6 +860,7 @@ fn raster_renderers_read_as_rustgis_raster_styles() { /// Every generated style and project in `testdata/qgis` reads, each project with its layers. #[test] fn every_corpus_file_reads() { + const PASSWORD: &str = "qgis-corpus-private-password-73194"; let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/qgis"); let files = |dir: &str, extensions: &[&str]| { let mut paths: Vec<_> = std::fs::read_dir(root.join(dir)).unwrap().map(|e| e.unwrap().path()).collect(); @@ -885,12 +886,14 @@ fn every_corpus_file_reads() { } else { String::from_utf8(bytes).unwrap() }; + // The worktree path itself may contain "secret"; use a distinct credential sentinel. + let text = text.replace("password='secret'", &format!("password='{PASSWORD}'")); let dir = path.parent().unwrap().to_string_lossy().to_string(); let mut report = Report::default(); let project = read_project(&text, &dir, &mut report).unwrap_or_else(|e| panic!("{}: {e}", path.display())); assert!(!project.layers.is_empty(), "{}: no layers", path.display()); // The PostGIS and SQL Server projects' made-up password reaches neither the layers nor the notes. - assert!(!format!("{project:?} {report:?}").contains("secret"), "{}: the password came through", path.display()); + assert!(!format!("{project:?} {report:?}").contains(PASSWORD), "{}: the password came through", path.display()); } } diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index e014050d3c..9f111bcf9d 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -522,7 +522,10 @@ fn open_postgis(pg: &PostgisOrigin) -> rustgis_data::Result rustgis_data::Result> { use rustgis_data::source::{duckdb::Database, mysql}; let Database::Mysql(address) = Database::parse(&my.database) else { - return Err(rustgis_data::DataError::Format(format!("{}: not a MySQL or MariaDB connection", my.database))); + return Err(rustgis_data::DataError::Format(rustgis_data::diagnostics::message_with_address( + &my.database, + "not a MySQL or MariaDB connection", + ))); }; let conn = mysql::connect(&address, false).map_err(database_open_error)?; let table = mysql::find_table(&conn, &my.table)?; @@ -562,7 +568,10 @@ fn open_sqlserver(ms: &SqlServerOrigin) -> rustgis_data::Result 0 { - return Err(format!( - "{path}: no readable features found while checking the replacement ({} rows could not be read)", + return Err(named(&format!( + "no readable features found while checking the replacement ({} rows could not be read)", probe.skipped - )); + ))); } if probe.skipped > 0 { conversion_notes.push(format!("{} rows could not be read while checking the replacement", probe.skipped)); @@ -1044,9 +1054,11 @@ impl SourceLayer { let r = match r { Ok(bytes) => rustgis_data::source::open_shared(&name, Arc::new(bytes), table.as_deref()) .map(Arc::from) - .map_err(|e| format!("{url}: {e}")), - Err(rustgis_data::source::remote::Failed::Other(e)) => Err(format!("{url}: {e}")), - Err(e) => Err(format!("{url}: couldn't be downloaded ({e:?})")), + .map_err(|e| rustgis_data::diagnostics::message_with_address(&url, &e.to_string())), + Err(rustgis_data::source::remote::Failed::Other(e)) => { + Err(rustgis_data::diagnostics::message_with_address(&url, &e)) + } + Err(e) => Err(rustgis_data::diagnostics::message_with_address(&url, &format!("couldn't be downloaded ({e:?})"))), }; if let Ok(mut s) = land.lock() { *s = Some(r); diff --git a/crates/map/src/tiles.rs b/crates/map/src/tiles.rs index d70adec904..c461159183 100644 --- a/crates/map/src/tiles.rs +++ b/crates/map/src/tiles.rs @@ -573,13 +573,16 @@ pub fn check_url(url: &str) -> Result<(), String> { } let lower = url.to_ascii_lowercase(); if !(lower.starts_with("https://") || lower.starts_with("http://") || lower.starts_with("s3://")) { - return Err(format!("'{url}': tile addresses start with https://, http:// or s3://")); + return Err(format!( + "'{}': tile addresses start with https://, http:// or s3://", + rustgis_data::diagnostics::address_in_message(url) + )); } if host_of(url).is_none_or(str::is_empty) { - return Err(format!("'{url}' has no host")); + return Err(format!("'{}' has no host", rustgis_data::diagnostics::address_in_message(url))); } if url.chars().any(|c| c.is_whitespace() || c.is_control()) { - return Err(format!("'{url}' has spaces in it")); + return Err(format!("'{}' has spaces in it", rustgis_data::diagnostics::address_in_message(url))); } Ok(()) } @@ -658,6 +661,21 @@ impl TileLayer { mod tests { use super::*; + #[test] + fn invalid_secret_tile_addresses_keep_the_validation_reason() { + for (url, reason) in [ + ("ftp://tiles.test/a?token=secret value", "tile addresses start with"), + ("https://?token=secret value", "has no host"), + ("https://tiles.test/a?token=secret value", "has spaces in it"), + ("https://tiles.test/a#token=secret value", "has spaces in it"), + ] { + let error = check_url(url).unwrap_err(); + let error = rustgis_data::diagnostics::redact(&rustgis_data::diagnostics::redact(&error)); + assert!(error.contains(reason), "{error}"); + assert!(!error.contains("secret") && !error.contains("value"), "{error}"); + } + } + #[test] fn runtime_bound_tile_and_legend_addresses_are_web_addresses() { let bound = rustgis_data::login_address::bind("0123456789abcdef", "https://maps.test", "https://maps.test/style.json").unwrap(); diff --git a/crates/plugins/src/pyenv.rs b/crates/plugins/src/pyenv.rs index 490c1aa947..ba5f4e3792 100644 --- a/crates/plugins/src/pyenv.rs +++ b/crates/plugins/src/pyenv.rs @@ -122,7 +122,11 @@ fn agent() -> ureq::Agent { } fn get(agent: &ureq::Agent, url: &str) -> Result, String> { - agent.get(url).header("User-Agent", concat!("RustGIS/", env!("CARGO_PKG_VERSION"))).call().map_err(|e| format!("{url}: {e}")) + agent + .get(url) + .header("User-Agent", concat!("RustGIS/", env!("CARGO_PKG_VERSION"))) + .call() + .map_err(|e| rustgis_data::diagnostics::message_with_address(url, &e.to_string())) } /// uv: the one [`UV_ENV`] names, else the one downloaded before, else a fresh download. @@ -156,9 +160,14 @@ pub fn uv(progress: &mut dyn FnMut(&str)) -> Result { .ok_or_else(|| format!("PyPI has no uv {UV_VERSION} for {tag}"))? .to_string(); let mut bytes = Vec::new(); - get(&agent, &url)?.body_mut().as_reader().take(MAX_WHEEL + 1).read_to_end(&mut bytes).map_err(|e| format!("{url}: {e}"))?; + get(&agent, &url)? + .body_mut() + .as_reader() + .take(MAX_WHEEL + 1) + .read_to_end(&mut bytes) + .map_err(|e| rustgis_data::diagnostics::message_with_address(&url, &e.to_string()))?; if bytes.len() as u64 > MAX_WHEEL { - return Err(format!("{url} is unexpectedly large")); + return Err(rustgis_data::diagnostics::message_with_address(&url, "is unexpectedly large")); } let actual: String = Sha256::digest(&bytes).iter().map(|b| format!("{b:02x}")).collect(); if actual != *sha256 { diff --git a/crates/plugins/src/registry.rs b/crates/plugins/src/registry.rs index 50f188e778..bac33b68a4 100644 --- a/crates/plugins/src/registry.rs +++ b/crates/plugins/src/registry.rs @@ -251,7 +251,7 @@ pub fn fetch_with_progress( ) -> Result, String> { use std::io::Read; if !allowed_url(url) { - return Err(format!("{url}: only HTTPS addresses are allowed")); + return Err(rustgis_data::diagnostics::message_with_address(url, "only HTTPS addresses are allowed")); } // Redirects stay on HTTPS (a local test registry may use plain HTTP, and only locally). A // server that stops sending ends the download after a minute, not at `timeout`: until a read @@ -264,8 +264,11 @@ pub fn fetch_with_progress( .tls_config(rustgis_data::http::tls_config()) .build() .into(); - let mut resp = - agent.get(url).header("User-Agent", concat!("RustGIS/", env!("CARGO_PKG_VERSION"))).call().map_err(|e| format!("{url}: {e}"))?; + let mut resp = agent + .get(url) + .header("User-Agent", concat!("RustGIS/", env!("CARGO_PKG_VERSION"))) + .call() + .map_err(|e| rustgis_data::diagnostics::message_with_address(url, &e.to_string()))?; let total = resp.headers().get("content-length").and_then(|v| v.to_str().ok()).and_then(|v| v.trim().parse::().ok()); let mut reader = resp.body_mut().with_config().limit(limit as u64 + 1).reader(); let mut buf = Vec::new(); @@ -274,17 +277,17 @@ pub fn fetch_with_progress( let n = match reader.read(&mut piece) { Ok(n) => n, Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, - Err(e) => return Err(format!("{url}: {e}")), + Err(e) => return Err(rustgis_data::diagnostics::message_with_address(url, &e.to_string())), }; if n == 0 { break; } buf.extend_from_slice(piece.get(..n).unwrap_or_default()); if buf.len() > limit { - return Err(format!("{url} is larger than {} MB", limit / (1024 * 1024))); + return Err(rustgis_data::diagnostics::message_with_address(url, &format!("is larger than {} MB", limit / (1024 * 1024)))); } if !progress(buf.len() as u64, total) { - return Err(format!("{url}: cancelled")); + return Err(rustgis_data::diagnostics::message_with_address(url, "cancelled")); } } Ok(buf) @@ -297,6 +300,14 @@ mod fetch_tests { use std::net::TcpListener; use std::time::Duration; + #[test] + fn rejected_secret_addresses_keep_the_policy_reason() { + let error = fetch("ftp://plugins.test/a?token=secret value", 1024).unwrap_err(); + let error = rustgis_data::diagnostics::redact(&rustgis_data::diagnostics::redact(&error)); + assert!(error.contains("only HTTPS addresses are allowed"), "{error}"); + assert!(!error.contains("secret") && !error.contains("value"), "{error}"); + } + /// Serves `body` once on this computer, as an HTTP answer that gives its length. fn serve(body: Vec) -> std::io::Result { let listener = TcpListener::bind("127.0.0.1:0")?; diff --git a/crates/tiles/src/address.rs b/crates/tiles/src/address.rs index f9159bd20f..d01ad08fb7 100644 --- a/crates/tiles/src/address.rs +++ b/crates/tiles/src/address.rs @@ -232,14 +232,16 @@ pub fn check_reach(url: &str, ceiling: Reach) -> Result<(), String> { } let lower = url.to_ascii_lowercase(); if !(lower.starts_with("https://") || lower.starts_with("http://")) { - return Err(format!("{url}: only http:// and https:// addresses are fetched")); + return Err(rustgis_data::diagnostics::message_with_address(url, "only http:// and https:// addresses are fetched")); } if url.chars().any(|c| c.is_whitespace() || c.is_control()) { - return Err(format!("{url}: the address has spaces in it")); + return Err(rustgis_data::diagnostics::message_with_address(url, "the address has spaces in it")); } match reach(url) { - Reach::Forbidden => Err(format!("{url}: that address is never fetched")), - Reach::Local if ceiling == Reach::Public => Err(format!("{url}: a document on the web can't point into the local network")), + Reach::Forbidden => Err(rustgis_data::diagnostics::message_with_address(url, "that address is never fetched")), + Reach::Local if ceiling == Reach::Public => { + Err(rustgis_data::diagnostics::message_with_address(url, "a document on the web can't point into the local network")) + } _ => Ok(()), } } diff --git a/crates/tiles/src/arcgis_auth.rs b/crates/tiles/src/arcgis_auth.rs index f7bc2fb0b6..71508f0e6a 100644 --- a/crates/tiles/src/arcgis_auth.rs +++ b/crates/tiles/src/arcgis_auth.rs @@ -91,7 +91,8 @@ pub fn portal_base(input: &str) -> Result { if raw.is_empty() { return Ok(ONLINE.to_string()); } - let (origin, path) = split(raw).ok_or_else(|| format!("{raw}: a portal is given by its https:// address"))?; + let (origin, path) = + split(raw).ok_or_else(|| rustgis_data::diagnostics::message_with_address(raw, "a portal is given by its https:// address"))?; if origin == "https://arcgis.com" { return Ok(ONLINE.to_string()); } @@ -496,7 +497,10 @@ pub fn add_token(url: &str, token: &str) -> Result { return Err("an ArcGIS token is one word of letters, digits and - _ . ~ + / =".into()); } let root = server_root(url).ok_or_else(|| { - format!("{}: a token is given with an ArcGIS REST address on https:// (…/rest/services/… or …/sharing/rest)", url.trim()) + rustgis_data::diagnostics::message_with_address( + url.trim(), + "a token is given with an ArcGIS REST address on https:// (…/rest/services/… or …/sharing/rest)", + ) })?; let scopes = Scope::prefix(&root).into_iter().collect(); keep(Session { diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 883e48dca5..27d451e514 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -948,7 +948,7 @@ fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16], asked // The browser's decoder may answer after the initiating owner was retired. Err(e) if !keep_failure(&e, &still) => return, Err(e) => { - log::debug!("{}", crate::logins::scrub_error(&format!("{}: {e}", answer.key))); + log::debug!("{}", crate::logins::scrub_error(&rustgis_data::diagnostics::message_with_address(&answer.key, &e))); // Refused until a time its server gave: the map draws again then. if let Some(wait) = retry_after(&e) { notify_at(Instant::now() + wait); @@ -1177,6 +1177,8 @@ impl<'a> RequestDiagnostics<'a> { _ => NetworkKind::Other, } }; + let message = failure.message.contains(url).then(|| rustgis_data::diagnostics::with_address_sanitized(url, &failure.message)); + let message = clean(message.as_deref().unwrap_or(&failure.message)); let network = Network { url: url.to_string(), method: self.method.to_string(), @@ -1185,9 +1187,9 @@ impl<'a> RequestDiagnostics<'a> { kind, }; if self.once { - self.capture.network_failure_once("Network", &clean(&failure.message), &clean(&details), network); + self.capture.network_failure_once("Network", &message, &clean(&details), network); } else { - self.capture.network_failure("Network", &clean(&failure.message), &clean(&details), network); + self.capture.network_failure("Network", &message, &clean(&details), network); } } } @@ -2275,7 +2277,7 @@ mod native { let next = crate::template::join(&at, header("location").ok_or("a redirect with no Location")?); Origin { url: at.clone(), reach: ceiling }.check(&next)?; if crate::address::is_https(&at) && !crate::address::is_https(&next) { - return Err(format!("{next}: a redirect from https to http").into()); + return Err(rustgis_data::diagnostics::message_with_address(&next, "a redirect from https to http").into()); } same_origin &= crate::logins::origin(&next) == first_origin; at = next; @@ -2630,7 +2632,7 @@ mod native { let _ = resp.body_mut().with_config().limit(64).read_to_vec(); continue; } - None => return Err(format!("{url}: {}", super::NO_RANGES).into()), + None => return Err(rustgis_data::diagnostics::message_with_address(url, super::NO_RANGES).into()), } } if status >= 400 { @@ -2668,7 +2670,7 @@ mod native { Origin { url: at.clone(), reach: ceiling }.check(&next)?; // Never from HTTPS down to plain HTTP, however either scheme is written. if crate::address::is_https(&at) && !crate::address::is_https(&next) { - return Err(format!("{next}: a redirect from https to http").into()); + return Err(rustgis_data::diagnostics::message_with_address(&next, "a redirect from https to http").into()); } redirects += 1; if redirects > 5 { @@ -2689,7 +2691,7 @@ mod native { if let Some((a, _)) = range.filter(|_| whole) && a > 0 { - return Err(format!("{url}: {}", super::NO_RANGES).into()); + return Err(rustgis_data::diagnostics::message_with_address(url, super::NO_RANGES).into()); } let mut buf = Vec::new(); if !resp.status().is_success() { @@ -2708,7 +2710,7 @@ mod native { .read_to_end(&mut buf) .map_err(|e| e.to_string())?; if buf.len() as u64 > n { - return Err(format!("{url}: {}", super::NO_RANGES).into()); + return Err(rustgis_data::diagnostics::message_with_address(url, super::NO_RANGES).into()); } if buf.len() > max { return Err(format!("larger than {max} bytes").into()); @@ -2728,7 +2730,7 @@ mod native { } if let Some((a, b)) = range.filter(|_| whole) { if buf.len() as u64 > b - a + 1 { - return Err(format!("{url}: {}", super::NO_RANGES).into()); + return Err(rustgis_data::diagnostics::message_with_address(url, super::NO_RANGES).into()); } buf.truncate(usize::try_from(b - a + 1).unwrap_or(usize::MAX)); } @@ -2752,24 +2754,24 @@ mod native { /// Keeps the destination ceiling with queued work; the web build requires its own fetcher. pub(super) fn fetch_reached(url: String, _ceiling: crate::address::Reach, done: super::FetchedDone, _omit: &'static [u16]) { - done(Err(format!("{url}: this build has no way to fetch tiles"))); + done(Err(rustgis_data::diagnostics::message_with_address(&url, "this build has no way to fetch tiles"))); } pub(super) fn fetch(url: String, done: Done, _omit: &'static [u16]) { - done(Err(format!("{url}: this build has no way to fetch tiles"))); + done(Err(rustgis_data::diagnostics::message_with_address(&url, "this build has no way to fetch tiles"))); } pub(super) fn download_within(url: String, _timeout: std::time::Duration, _omit: &'static [u16], done: DoneDetailed) { - let message = format!("{url}: this build has no way to fetch tiles"); + let message = rustgis_data::diagnostics::message_with_address(&url, "this build has no way to fetch tiles"); done(Err(crate::failure::Failure::new(&url, message))); } pub(super) fn post(url: String, _body: String, done: Done) { - done(Err(format!("{url}: this build has no way to send requests"))); + done(Err(rustgis_data::diagnostics::message_with_address(&url, "this build has no way to send requests"))); } pub(super) fn head(url: String, done: super::HeadDone) { - done(Err(format!("{url}: this build has no way to send requests"))); + done(Err(rustgis_data::diagnostics::message_with_address(&url, "this build has no way to send requests"))); } pub(super) fn fetch_local(url: String, done: Done) { @@ -3082,6 +3084,29 @@ mod tests { assert!(second.records().is_empty()); } + #[test] + fn network_diagnostics_preserve_known_secret_address_failure_reasons() { + use rustgis_data::diagnostics::{Filter, History}; + for url in [ + "https://diagnostics-secret.test/data?token=hidden", + "https://diagnostics-secret.test/data?%2573ig=hidden unescaped hidden", + "https://alice:hidden@diagnostics-secret.test/data#%256bey=hidden unescaped hidden", + ] { + let history = History::default(); + let _scope = history.enter(); + let mut diagnostics = RequestDiagnostics::new(url, "GET"); + diagnostics.answered(503); + diagnostics.failed(&Failure::new(url, format!("{url}: HTTP 503 Service Unavailable"))); + let records = history.records(); + assert_eq!(records.len(), 1); + assert!(records[0].message.contains("HTTP 503 Service Unavailable"), "{records:?}"); + assert_eq!(rustgis_data::diagnostics::redact(&records[0].message), records[0].message); + let report = history.report(Filter::All); + assert!(report.contains("HTTP 503 Service Unavailable"), "{report}"); + assert!(!report.contains("hidden") && !report.contains("alice"), "{report}"); + } + } + #[cfg(not(target_arch = "wasm32"))] #[test] fn network_diagnostics_redact_downloads_without_retaining_bodies() { @@ -4069,8 +4094,10 @@ mod tests { let f = native::get_detailed(&agent, &format!("{base}/f.parquet{TAIL}4")).unwrap_err(); assert_eq!(f.status(), Some(404), "{}", f.message); let base = serve(vec![no_suffix, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\nConnection: close\r\n\r\n0123456789".into()]); - let f = native::get_detailed(&agent, &format!("{base}/f.parquet{TAIL}4")).unwrap_err(); + let f = native::get_detailed(&agent, &format!("{base}/f.parquet?token=private-range-token{TAIL}4")).unwrap_err(); assert!(f.message.contains(super::NO_RANGES), "{}", f.message); + assert!(!f.message.contains("private-range-token"), "{}", f.message); + assert!(rustgis_data::diagnostics::redact(&f.message).contains(super::NO_RANGES), "{}", f.message); // More last bytes asked for than the file holds: the whole file. let base = serve(vec![ "HTTP/1.1 501 Not Implemented\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".into(), diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index 02ca65d8da..362be9c191 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -16,6 +16,7 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::{Arc, Mutex, OnceLock}; +use rustgis_data::diagnostics::message_with_address; use rustgis_data::raster::ColorHint; use rustgis_data::raster::cog::{self, Layout, Level, Parse, Sparse}; use rustgis_data::raster::crs::Transformer; @@ -475,7 +476,7 @@ fn load(url: String, done: Box, String>) + Send>) { Box::new(move |h| { let h = h.unwrap_or_default(); if h.content_type.as_deref().is_some_and(not_tiff_type) && !h.accepts_ranges && h.tiff_header_length.is_none() { - return done(Err(format!("{u}: not a GeoTIFF"))); + return done(Err(message_with_address(&u, "not a GeoTIFF"))); } read_header(u, header_length(h.tiff_header_length), done); }), @@ -510,8 +511,8 @@ fn read_header(url: String, len: u64, done: Box, Str len, Box::new(move |r| match r { Ok(b) if cog::is_tiff(&b) => step(u, Sparse::default().with(0, b), 0, done), - Ok(_) => done(Err(format!("{u}: not a GeoTIFF"))), - Err(e) => done(Err(format!("{u}: {e}"))), + Ok(_) => done(Err(message_with_address(&u, "not a GeoTIFF"))), + Err(e) => done(Err(message_with_address(&u, &e))), }), ); } @@ -522,7 +523,7 @@ fn step(url: String, have: Sparse, rounds: usize, done: Box done(Err(format!("{NOT_COG}: {why}"))), Ok(Parse::Need(a, b)) => { let Some(len) = header_read(&have, a, b, rounds) else { - done(Err(format!("{url}: the GeoTIFF's header is too large or too scattered to read in place"))); + done(Err(message_with_address(&url, "the GeoTIFF's header is too large or too scattered to read in place"))); return; }; let u = url.clone(); @@ -536,13 +537,13 @@ fn step(url: String, have: Sparse, rounds: usize, done: Box step(u, have.with_len(a), rounds + 1, done), - Ok(more) if more.is_empty() => done(Err(format!("{u}: the GeoTIFF is cut short"))), + Ok(more) if more.is_empty() => done(Err(message_with_address(&u, "the GeoTIFF is cut short"))), Ok(more) => step(u, have.with(a, more), rounds + 1, done), - Err(e) => done(Err(format!("{u}: {e}"))), + Err(e) => done(Err(message_with_address(&u, &e))), }), ); } - Err(e) => done(Err(format!("{url}: {e}"))), + Err(e) => done(Err(message_with_address(&url, &e.to_string()))), } } @@ -577,12 +578,12 @@ fn stats(url: String, header: Sparse, layout: Layout, done: Box t, Err(e) => { - done(Err(format!("{url}: {e}"))); + done(Err(message_with_address(&url, &e))); return; } }; let Some(full) = layout.levels.first().cloned() else { - done(Err(format!("{url}: no image"))); + done(Err(message_with_address(&url, "no image"))); return; }; // Bounds: the full image's edges, sampled (a projected square isn't square in degrees). @@ -601,7 +602,7 @@ fn stats(url: String, header: Sparse, layout: Layout, done: Box lon.1 || lat.0 > lat.1 { - done(Err(format!("{url}: the GeoTIFF's corners aren't on the globe"))); + done(Err(message_with_address(&url, "the GeoTIFF's corners aren't on the globe"))); return; } let bounds = [lon.0.max(-180.0), lat.0.max(-85.05), lon.1.min(180.0), lat.1.min(85.05)]; @@ -1117,7 +1118,7 @@ fn parse_url(url: &str) -> Option<(u8, u32, u32, String)> { /// Draws a map tile (`cog:z/x/y@address`): empty where the file has nothing. pub(crate) fn fetch(url: &str, done: Done) { let Some((z, x, y, archive)) = parse_url(url) else { - done(Err(format!("{url}: not a COG tile address"))); + done(Err(message_with_address(url, "not a COG tile address"))); return; }; let (file, look) = split(&archive); @@ -1398,7 +1399,7 @@ pub fn probe_blocking(url: &str) -> Result { Ok(Ok(())) => Ok(true), Ok(Err(e)) if read_whole(&e) => Ok(false), Ok(Err(e)) => Err(e), - Err(_) => Err(format!("{url}: no answer in a minute")), + Err(_) => Err(message_with_address(url, "no answer in a minute")), } } @@ -1432,7 +1433,7 @@ pub fn info_blocking(url: &str) -> Result, String> { let url = split(url).0; let (tx, rx) = std::sync::mpsc::channel(); with_info(url, Box::new(move |r| tx.send(r).unwrap_or_default())); - rx.recv_timeout(Duration::from_secs(120)).map_err(|_| format!("{url}: no answer in two minutes"))? + rx.recv_timeout(Duration::from_secs(120)).map_err(|_| message_with_address(url, "no answer in two minutes"))? } /// Band `band`'s values (0-based) in a window of a level: columns `cols.0..cols.1` and rows @@ -1496,7 +1497,7 @@ pub fn read_values_counted_blocking( let (mut out, mut rounded, mut overflowed) = (rustgis_data::raster::filled(w, h, f32::NAN).map_err(|e| e.to_string())?, 0u64, 0u64); let (tx, rx) = std::sync::mpsc::channel(); fetch_bands(info.clone(), url.clone(), level, tiles.clone(), vec![band], Box::new(move |r| tx.send(r).unwrap_or_default())); - let picked = rx.recv_timeout(Duration::from_secs(120)).map_err(|_| format!("{url}: tiles not read in two minutes"))??; + let picked = rx.recv_timeout(Duration::from_secs(120)).map_err(|_| message_with_address(&url, "tiles not read in two minutes"))??; let nodata = info.layout.nodata.map(|v| v as f32); for (&t, p) in tiles.iter().zip(&picked) { let Some((chunk, b)) = p.first() else { continue }; diff --git a/crates/tiles/src/failure.rs b/crates/tiles/src/failure.rs index 58cbf0b3a5..5e09ff3f17 100644 --- a/crates/tiles/src/failure.rs +++ b/crates/tiles/src/failure.rs @@ -69,6 +69,13 @@ impl Failure { pub fn new(url: &str, message: impl Into) -> Self { let message = message.into(); let reason = crate::cache::http_status(&message).filter(|s| *s >= 400).map_or(Reason::Other, Reason::Status); + // Known address occurrences, wherever they are, are sanitized whole first, so an + // unescaped secret value can't be mistaken for the reason or force the cut that drops it. + let message = if !url.is_empty() && message.contains(url) { + rustgis_data::diagnostics::with_address_sanitized(url, &message) + } else { + message + }; let message = crate::logins::scrub_error(&message); let url = if url.contains(rustgis_data::login_address::PREFIX) { crate::logins::scrub_error(url) } else { url.to_string() }; Failure { url, message, reason, answer: None } diff --git a/crates/tiles/src/logins.rs b/crates/tiles/src/logins.rs index 64ffbe67d8..a40433d31f 100644 --- a/crates/tiles/src/logins.rs +++ b/crates/tiles/src/logins.rs @@ -322,6 +322,9 @@ pub fn scrub(text: &str) -> String { /// Runtime request bindings and successful data values must not use this output projection. pub fn scrub_error(text: &str) -> String { let unbound = scrub_patterns(text, &[]); + if let Some(strict) = rustgis_data::diagnostics::secret_continuation_cut(&unbound) { + return scrub(&strict); + } let (public, _) = rustgis_data::address::scrub_addresses(&unbound); scrub(&public) } diff --git a/crates/tiles/src/mbtiles.rs b/crates/tiles/src/mbtiles.rs index 0d2451c753..401c0534d5 100644 --- a/crates/tiles/src/mbtiles.rs +++ b/crates/tiles/src/mbtiles.rs @@ -342,7 +342,7 @@ fn parse_url(url: &str) -> Option<(u8, u32, u32, &str)> { /// is read on the fetch workers, never on the frame. pub(crate) fn fetch(url: &str, done: Done) { let Some((z, x, y, archive)) = parse_url(url) else { - done(Err(format!("{url}: not an MBTiles tile address"))); + done(Err(rustgis_data::diagnostics::message_with_address(url, "not an MBTiles tile address"))); return; }; if let Err(e) = check(archive) { @@ -362,7 +362,8 @@ pub(crate) fn fetch(url: &str, done: Done) { /// Reads a tile from a file (on a fetch worker). #[cfg(not(target_arch = "wasm32"))] pub(crate) fn read_local(url: &str) -> Result, String> { - let (z, x, y, archive) = parse_url(url).ok_or_else(|| format!("{url}: not an MBTiles tile address"))?; + let (z, x, y, archive) = + parse_url(url).ok_or_else(|| rustgis_data::diagnostics::message_with_address(url, "not an MBTiles tile address"))?; check(archive)?; Ok(rustgis_data::mbtiles::tile_path(file_of(archive), z, x, y)?.unwrap_or_default()) } diff --git a/crates/tiles/src/pmtiles.rs b/crates/tiles/src/pmtiles.rs index 0de9570d29..d4e5c23d0d 100644 --- a/crates/tiles/src/pmtiles.rs +++ b/crates/tiles/src/pmtiles.rs @@ -552,7 +552,7 @@ fn finish(archive: &str, r: Result) { /// doesn't have. pub(crate) fn fetch(url: &str, done: Done) { let Some((z, x, y, archive)) = parse_url(url) else { - done(Err(format!("{url}: not a PMTiles tile address"))); + done(Err(rustgis_data::diagnostics::message_with_address(url, "not a PMTiles tile address"))); return; }; let a = archive.clone(); diff --git a/crates/tiles/src/portal.rs b/crates/tiles/src/portal.rs index 92243276f9..caed166653 100644 --- a/crates/tiles/src/portal.rs +++ b/crates/tiles/src/portal.rs @@ -176,7 +176,10 @@ impl Portal { Some(u) => { let host = host_of(u).to_ascii_lowercase(); if !host_ok(&host) { - return Err(format!("'{u}': a Socrata site is its domain, such as data.cityofnewyork.us")); + return Err(rustgis_data::diagnostics::message_with_address( + u, + "a Socrata site is its domain, such as data.cityofnewyork.us", + )); } web(&host)?; // A discovery API itself searches every site; a site keeps the search to it. @@ -194,7 +197,10 @@ impl Portal { Some(u) => { let host = host_of(u).to_ascii_lowercase(); if !host_ok(&host) { - return Err(format!("'{u}': an ArcGIS Hub site is its address, such as hub.arcgis.com")); + return Err(rustgis_data::diagnostics::message_with_address( + u, + "an ArcGIS Hub site is its address, such as hub.arcgis.com", + )); } web(&host)? } diff --git a/crates/tiles/src/s3.rs b/crates/tiles/src/s3.rs index bcbc1b00cc..0f24fc956d 100644 --- a/crates/tiles/src/s3.rs +++ b/crates/tiles/src/s3.rs @@ -385,13 +385,15 @@ pub fn is_s3(url: &str) -> bool { /// The bucket and key of an `s3://bucket/key` address. pub fn parse(url: &str) -> Result<(String, String), String> { - let rest = url.get(5..).filter(|_| is_s3(url)).ok_or_else(|| format!("{url}: not an s3:// address"))?; + let rest = + url.get(5..).filter(|_| is_s3(url)).ok_or_else(|| rustgis_data::diagnostics::message_with_address(url, "not an s3:// address"))?; let (bucket, key) = rest.split_once('/').unwrap_or((rest, "")); if !valid_bucket(bucket) { - return Err(format!("{url}: '{bucket}' isn't a bucket name")); + // The bucket text itself is not repeated: without a slash it may hold the query. + return Err(rustgis_data::diagnostics::message_with_address(url, "the bucket name isn't one S3 takes")); } if key.len() > 1024 || key.chars().any(char::is_control) { - return Err(format!("{url}: the key isn't one S3 takes")); + return Err(rustgis_data::diagnostics::message_with_address(url, "the key isn't one S3 takes")); } Ok((bucket.to_string(), key.to_string())) } @@ -698,7 +700,7 @@ pub fn prepare(url: &str) -> Result { /// A web address with its path in S3's canonical form: each part decoded, then encoded. fn canonical_url(url: &str) -> Result { - let (scheme, rest) = url.split_once("://").ok_or_else(|| format!("{url}: not a web address"))?; + let (scheme, rest) = url.split_once("://").ok_or_else(|| rustgis_data::diagnostics::message_with_address(url, "not a web address"))?; let rest = rest.split('#').next().unwrap_or(rest); let end = rest.find(['/', '?']).unwrap_or(rest.len()); let (authority, path_query) = rest.split_at(end); @@ -856,7 +858,8 @@ fn sign_get(url: &str, region: &str, cr: &Credentials, date: &str) -> Result Result<(String, String, String), String> { - let rest = url.split_once("://").map(|(_, r)| r).ok_or_else(|| format!("{url}: not a web address"))?; + let rest = + url.split_once("://").map(|(_, r)| r).ok_or_else(|| rustgis_data::diagnostics::message_with_address(url, "not a web address"))?; let rest = rest.split('#').next().unwrap_or(rest); let (authority, path_query) = match rest.find('/') { Some(i) => (rest.get(..i).unwrap_or(""), rest.get(i..).unwrap_or("/")), @@ -1293,6 +1296,10 @@ mod tests { for bad in ["s3://", "s3://A_B/x", "s3://ab/x", "https://x.org/a", "s3://-ab-/x"] { assert!(parse(bad).is_err(), "{bad}"); } + // A query in place of a key isn't repeated in the error, and the bucket stays readable. + let error = parse("s3://bucket?X-Custom-Cred=hidden").unwrap_err(); + assert!(!error.contains("hidden") && error.contains("s3://bucket/") && error.contains("bucket name"), "{error}"); + assert_eq!(rustgis_data::diagnostics::redact(&error), error); assert_eq!(object_url("my-bucket", "a/b c.tif", None), "https://my-bucket.s3.amazonaws.com/a/b%20c.tif"); assert_eq!(object_url("my.bucket", "k", None), "https://s3.amazonaws.com/my.bucket/k"); let aws = Connection { name: "a".into(), endpoint: None, region: "eu-west-1".into(), buckets: vec![], credentials: None }; diff --git a/crates/tiles/src/source.rs b/crates/tiles/src/source.rs index a785679ef8..2153b5f92b 100644 --- a/crates/tiles/src/source.rs +++ b/crates/tiles/src/source.rs @@ -231,7 +231,7 @@ fn resolve_anew(spec: &Spec, key: String) -> Resolve { } } Ok(None) => return Resolve::Pending, - Err(e) => return Resolve::Failed(format!("{}: {e}", crate::logins::inner(&spec.url))), + Err(e) => return Resolve::Failed(crate::logins::scrub(&rustgis_data::diagnostics::message_with_address(&spec.url, &e))), }, Kind::Cog => match cog::info(&spec.url) { Ok(Some(info)) => { @@ -272,7 +272,7 @@ fn resolve_anew(spec: &Spec, key: String) -> Resolve { }; let mut style = match Style::parse(&doc) { Ok(s) => s, - Err(e) => return Resolve::Failed(format!("{}: {e}", crate::logins::inner(&spec.url))), + Err(e) => return Resolve::Failed(crate::logins::scrub(&rustgis_data::diagnostics::message_with_address(&spec.url, &e))), }; if !spec.hidden_layers.is_empty() { style.layers.retain(|l| l.source_layer.as_ref().is_none_or(|s| !spec.hidden_layers.contains(s))); @@ -481,7 +481,7 @@ fn resolve_source(src: &mut Source, spec: &Spec, base: &address::Origin) -> Resu && let Some(u) = url.clone() { if u.starts_with("mapbox://") { - return Err(format!("{u}: mapbox:// addresses need Mapbox's own library")); + return Err(rustgis_data::diagnostics::message_with_address(&u, "mapbox:// addresses need Mapbox's own library")); } let u = template::join(&base.url, &u); // An ArcGIS vector tile service answers its TileJSON-like description as JSON only @@ -560,10 +560,10 @@ fn fetch_json(url: &str, from: Option<&address::Origin>) -> Result match &*p { Payload::Json(v) => Ok(Some((v.clone(), origin.unwrap_or_else(|| address::Origin::named(url))))), - _ => Err(format!("{}: not a JSON document", crate::logins::inner(url))), + _ => Err(rustgis_data::diagnostics::message_with_address(url, "not a JSON document")), }, (State::Pending, _) => Ok(None), - (State::Failed(e), _) => Err(format!("{}: {e}", crate::logins::inner(url))), + (State::Failed(e), _) => Err(crate::logins::scrub(&rustgis_data::diagnostics::message_with_address(url, &e))), } } diff --git a/crates/tiles/src/stac.rs b/crates/tiles/src/stac.rs index b1097841f8..ba0a847a44 100644 --- a/crates/tiles/src/stac.rs +++ b/crates/tiles/src/stac.rs @@ -629,7 +629,10 @@ pub fn sas_token_url(href: &str) -> Result { let ok = |s: &str, max: usize| (3..=max).contains(&s.len()) && s.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'); if !ok(account, 24) || !ok(container, 63) { - return Err(format!("{href}: not an Azure Blob Storage address the Planetary Computer signs")); + return Err(rustgis_data::diagnostics::message_with_address( + href, + "not an Azure Blob Storage address the Planetary Computer signs", + )); } Ok(format!("{SAS_API}/{account}/{container}")) } @@ -911,7 +914,9 @@ mod tests { assert!(parse_sas(bad).is_err()); } assert!(sas_token_url("https://evil.org/x.blob.core.windows.net/c/x.tif").is_err()); - assert!(sas_token_url("https://a.blob.core.windows.net/").is_err()); + let error = sas_token_url("https://a.blob.core.windows.net/?token=hidden unescaped hidden").unwrap_err(); + let error = rustgis_data::diagnostics::redact(&error); + assert!(error.contains("not an Azure Blob Storage address") && !error.contains("hidden"), "{error}"); remember_token("k", "sig=1"); assert_eq!(cached_token("k").as_deref(), Some("sig=1")); // A web item can't point into the local network. diff --git a/crates/ui-egui/src/assistantpane.rs b/crates/ui-egui/src/assistantpane.rs index c0f0f483e7..286d142ee1 100644 --- a/crates/ui-egui/src/assistantpane.rs +++ b/crates/ui-egui/src/assistantpane.rs @@ -414,7 +414,10 @@ fn dispatch(app: &mut RustGisApp, req: HttpRequest, login: Option (String, String) { (tl!("RustGIS couldn't reach the server.").into(), tl!("Check the address and your connection.").into()) } (Reason::Timeout, _) => (tl!("The server took too long to answer.").into(), tl!("Try again later.").into()), - (Reason::Other, _) => (tl!("The request failed.").into(), redact_text(&f.message)), + (Reason::Other, _) => (tl!("The request failed.").into(), message_with_address(&f.url, &f.message)), } } /// What is known of the failure, as (label, value) rows: the labels in English, translated /// where shown. pub fn details(f: &Failure) -> Vec<(&'static str, String)> { - let mut rows = vec![("Address", rustgis_engine::sources::redact(&f.url))]; + let mut rows = vec![("Address", address_in_message(&f.url))]; match f.status() { Some(s) => rows.push(("Status", status_name(s).map_or_else(|| s.to_string(), |n| format!("{s} {n}")))), - None => rows.push(("Error", redact_text(&f.message))), + None => rows.push(("Error", message_with_address(&f.url, &f.message))), } let header = |k: &str| f.header(k).map(str::to_string); for (label, value) in [ @@ -140,26 +140,24 @@ mod tests { let unanswered = |r: Reason| plain(&Failure::without_answer("u", "x", r)).0; assert!(unanswered(Reason::Unreachable).contains("couldn't reach")); assert!(unanswered(Reason::Timeout).contains("too long")); - // Anything else: the error itself, with no status to show. - let other = Failure::new("u", "too many redirects"); - assert_eq!(plain(&other).1, "too many redirects"); - assert!(report(&other).contains("Error: too many redirects")); } - /// An error's text quoting an address shows it without its keys, the words around it kept. + /// Known addresses are sanitized whole before either error surface composes the reason. #[test] - fn addresses_in_an_error_lose_their_keys() { - let said = "https://h.example/a?f=json&token=s3cr3t: the server doesn't answer byte-range requests"; - let f = Failure::new("https://h.example/a?f=json&token=s3cr3t", said); - let shown = plain(&f).1; - assert_eq!(shown, "https://h.example/a?f=json: the server doesn't answer byte-range requests"); - let report = report(&f); - assert!(!report.contains("s3cr3t"), "{report}"); - assert!(report.contains("Error: https://h.example/a?f=json: the server"), "{report}"); - // Quoted, bracketed or ending a sentence; text with no address is left as it is. - assert_eq!(redact_text("(\"http://x.example/?key=k\"), then."), "(\"http://x.example/\"), then."); - assert_eq!(redact_text("«https://x.example/?token=t» gone"), "«https://x.example/» gone"); - assert_eq!(redact_text("io: connection refused"), "io: connection refused"); - assert_eq!(redact_text("://"), "://"); + fn addresses_in_an_error_lose_their_keys_without_losing_the_reason() { + for url in [ + "https://alice:hidden@h.example/a?token=hidden unescaped hidden", + "https://h.example/a?custom=hidden unescaped hidden", + "https://h.example/a#hidden unescaped hidden", + ] { + let f = Failure::new(url, format!("{url}: the server doesn't answer byte-range requests")); + let shown = plain(&f).1; + let error = details(&f).into_iter().find(|(label, _)| *label == "Error").unwrap().1; + for text in [shown, error, report(&f)] { + let clean = rustgis_engine::Session::redacted(&rustgis_engine::Session::redacted(&text).into_string()).into_string(); + assert!(clean.contains("byte-range requests"), "{clean}"); + assert!(!clean.contains("hidden") && !clean.contains("alice"), "{clean}"); + } + } } } diff --git a/crates/ui-egui/src/geolibreui.rs b/crates/ui-egui/src/geolibreui.rs index 0b77da5e0b..fe32607c3d 100644 --- a/crates/ui-egui/src/geolibreui.rs +++ b/crates/ui-egui/src/geolibreui.rs @@ -110,7 +110,7 @@ impl Read { if self.forbidden > 0 { return Some(Err(crate::i18n::fmt( "{url}: access denied (HTTP 403) to {count} of the slice’s chunks", - &[("url", &self.root), ("count", &self.forbidden.to_string())], + &[("url", &rustgis_engine::sources::address_in_message(&self.root)), ("count", &self.forbidden.to_string())], ))); } let name = self.params.get("name").and_then(Value::as_str).unwrap_or("Zarr"); @@ -121,7 +121,10 @@ impl Read { ); } if self.got.is_empty() && self.forbidden > 0 { - return Some(Err(crate::i18n::fmt("{url}: access denied (HTTP 403)", &[("url", &self.root)]))); + return Some(Err(crate::i18n::fmt( + "{url}: access denied (HTTP 403)", + &[("url", &rustgis_engine::sources::address_in_message(&self.root))], + ))); } self.forbidden = 0; let docs = std::mem::take(&mut self.got).into_iter().collect::>(); @@ -167,7 +170,7 @@ pub fn poll(app: &mut RustGisApp, ctx: &egui::Context) { Err(e) => { app.session.download_failed(id); rustgis_engine::cmd::geolibre::discard_pending(&mut app.session, ¶ms); - app.session.notify(crate::i18n::fmt("{url}: {e}", &[("url", &url), ("e", &e)]), true); + app.session.notify(rustgis_engine::sources::message_with_address(&url, &e), true); } } } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 59102e35e4..136fb0beea 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -2221,7 +2221,7 @@ impl RustGisApp { fn retire_cog_probes(&mut self) { for (p, _, _) in std::mem::take(&mut self.ui.cog_probes) { let url = p.get("url").and_then(Value::as_str).unwrap_or(""); - let url = serviceui::address_in_message(rustgis_engine::data::s3_address::inner(url)); + let url = rustgis_engine::sources::address_in_message(url); let why = i18n::fmt("{url}: its project was closed while it was read, so it wasn't added", &[("url", &url)]); self.landed_in_dialog(&p, Err(&why)); self.session.notify(why, false); @@ -3165,7 +3165,7 @@ impl RustGisApp { }, Box::new(move |r| *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(r)), ); - self.session.notify(i18n::fmt("Opening {url}…", &[("url", url)]), false); + self.session.notify(i18n::fmt("Opening {url}…", &[("url", &rustgis_engine::sources::address_in_message(url))]), false); let id = self.session.download_started(); if let Some((old, ..)) = self.ui.project_download.replace((id, url.to_string(), slot)) { self.session.download_failed(old); @@ -4261,14 +4261,17 @@ impl RustGisApp { if !self.session.download_done(id) { if r.is_ok() { self.session.notify( - i18n::fmt("{url}: downloaded after another project was opened, so it wasn't opened", &[("url", &url)]), + i18n::fmt( + "{url}: downloaded after another project was opened, so it wasn't opened", + &[("url", &rustgis_engine::sources::address_in_message(&url))], + ), false, ); } return self.poll_data_downloads(); } if let Err(e) = r.and_then(|bytes| self.open_project_bytes(project_file_name(&url), &bytes)) { - self.session.notify(if e.starts_with(&url) { e } else { format!("{url}: {e}") }, true); + self.session.notify(rustgis_engine::sources::message_with_address(&url, &e), true); } } self.poll_data_downloads(); @@ -4405,8 +4408,8 @@ impl RustGisApp { ); } // An address read through a saved S3 connection is said as the address it wraps. - let shown = rustgis_engine::data::s3_address::inner(url); - self.session.notify(i18n::fmt("Reading {url}…", &[("url", shown)]), false); + let shown = rustgis_engine::sources::address_in_message(url); + self.session.notify(i18n::fmt("Reading {url}…", &[("url", &shown)]), false); let mut probe = p.clone(); probe["url"] = json!(address); if let Some(o) = probe.as_object_mut() { @@ -4467,8 +4470,8 @@ impl RustGisApp { if window { return self.start_window_read(p, url, q, claim); } - let shown = rustgis_engine::data::s3_address::inner(&url); - self.session.notify(i18n::fmt("Downloading {url}…", &[("url", shown)]), false); + let shown = rustgis_engine::sources::address_in_message(&url); + self.session.notify(i18n::fmt("Downloading {url}…", &[("url", &shown)]), false); let id = claim.unwrap_or_else(|| self.session.download_started()); // A service's features: read page by page from the start, off the frame (an ArcGIS // layer's description first). @@ -4559,10 +4562,9 @@ impl RustGisApp { } } Err(e) => { - // A waiting command's error names the address, as the notice does. - // Redacted whole, as the notice is: a key with a space in it isn't left half there. - let e = if e.starts_with(&url) { e } else { format!("{url}: {e}") }; - let e = rustgis_engine::Session::redacted(&e).into_string(); + // A waiting command's error names the address, as the notice does: sanitized + // on its own first, so a key in its query doesn't take the reason with it. + let e = rustgis_engine::sources::message_with_address(&url, &e); self.session.request_failed(&p, &e); if let Some(id) = claim { self.session.download_failed(id); @@ -4620,12 +4622,7 @@ impl RustGisApp { self.session.request_failed(p, &e); self.session.download_failed(id); rustgis_engine::cmd::geolibre::discard_pending(&mut self.session, p); - // Named without the address's keys, token or signature. - let e = e.strip_prefix(url).map(|r| r.trim_start_matches(": ").to_string()).unwrap_or(e); - let why = rustgis_engine::sources::redact_text(&e); - // An error that names its own address (a Shapefile's part) isn't named again. - let named = why.split_once(": ").is_some_and(|(head, _)| head.contains("://") && !head.contains(' ')); - let why = if named { why } else { format!("{}: {why}", rustgis_engine::sources::redact(url)) }; + let why = rustgis_engine::sources::message_with_address(url, &e); self.landed_in_dialog(p, Err(&why)); self.session.notify(why, true); } @@ -4639,7 +4636,7 @@ impl RustGisApp { if self.ui.window_reads.len() >= MAX_WINDOW_READS { return Err(i18n::fmt( "{url}: {count} windows are being read already: ask again once one has been added", - &[("url", &rustgis_engine::sources::redact(&url)), ("count", &MAX_WINDOW_READS.to_string())], + &[("url", &rustgis_engine::sources::address_in_message(&url)), ("count", &MAX_WINDOW_READS.to_string())], )); } Ok(asked) @@ -4687,7 +4684,7 @@ impl RustGisApp { // Answered and given back as a refused window is. let e = i18n::fmt( "{url}: its read didn't start: {error}", - &[("url", &rustgis_engine::sources::redact(&url)), ("error", &e.to_string())], + &[("url", &rustgis_engine::sources::address_in_message(&url)), ("error", &e.to_string())], ); self.session.request_failed(p, &e); if let Some(id) = claim { @@ -4696,7 +4693,7 @@ impl RustGisApp { return Err(e); } // Named redacted, and by the address an S3 connection's wraps. - let shown = rustgis_engine::sources::redact(rustgis_engine::data::s3_address::inner(&url)); + let shown = rustgis_engine::sources::address_in_message(&url); self.session.notify(i18n::fmt("Reading {url}…", &[("url", &shown)]), false); let id = claim.unwrap_or_else(|| self.session.download_started()); self.ui.window_reads.push((id, url, q, slot, left)); @@ -4726,15 +4723,13 @@ impl RustGisApp { fn add_download(&mut self, landed: serviceui::Landed, current: bool) { let serviceui::Landed { id, url, params: mut p, origin } = landed; if !current { - let shown = serviceui::address_in_message(&url); + let shown = rustgis_engine::sources::address_in_message(&url); let why = i18n::fmt("{url}: downloaded after its project was closed, so it wasn't added", &[("url", &shown)]); self.session.request_failed(&p, &why); self.landed_in_dialog(&p, Err(&why)); self.session.notify(why, false); return; } - // Named redacted: a notice with a key in it loses what follows it. - let shown = rustgis_engine::sources::redact(&url); p["id"] = json!(id); p["url"] = json!(url); let zoom = p.get("zoom").and_then(rustgis_engine::cmd::bool_value) == Some(true); @@ -4755,11 +4750,9 @@ impl RustGisApp { Err(e) => { self.session.request_failed(&pending, &e); rustgis_engine::cmd::geolibre::discard_pending(&mut self.session, &pending); - // Running it said the error already: again only to name the address it lacks. An - // error may name it redacted. - let named = e.starts_with(&url) || e.starts_with(&shown); - let e = if named { e } else { format!("{shown}: {e}") }; - self.landed_in_dialog(&pending, Err(&e)); + // Running it said the error already; the dialog alone needs the address. + let why = rustgis_engine::sources::message_with_address(&url, &e); + self.landed_in_dialog(&pending, Err(&why)); } } } @@ -4819,7 +4812,7 @@ impl RustGisApp { if opened == self.session.opened { return true; } - let url = serviceui::address_in_message(address); + let url = rustgis_engine::sources::address_in_message(address); let why = i18n::fmt("{url}: its project was closed while it was read, so it wasn't added", &[("url", &url)]); if pending.is_some() && self.dialogs.info.as_deref() == pending { self.dialogs.info = None; @@ -6177,9 +6170,14 @@ mod record_tests { let failed = add("404 Not Found", crate::serviceui::downloading("Roads")); assert!(!failed.session.map().layers.iter().any(|l| l.name == "Roads")); assert_eq!(failed.dialogs.info, None); - // Why, redacted as the notice is: the address, not its token. + // The failure's HTTP status survives redaction in the dialog as well as the notice. let why = failed.dialogs.error.as_deref().unwrap_or(""); - assert!(why.contains("roads.geojson") && !why.contains("hidden-757"), "{why}"); + assert!(why.contains("roads.geojson") && why.contains("404") && !why.contains("hidden-757"), "{why}"); + assert!( + failed.session.notices.iter().any(|notice| notice.error && notice.text.contains("404") && !notice.text.contains("hidden-757")), + "{:?}", + failed.session.notices + ); // The line has moved on to another layer (or another dialog): it stays as it is. let other = add("200 OK", crate::serviceui::downloading("Rivers")); assert_eq!(other.dialogs.info, Some(crate::serviceui::downloading("Rivers"))); diff --git a/crates/ui-egui/src/pluginsui.rs b/crates/ui-egui/src/pluginsui.rs index 98976c248b..8268c47fee 100644 --- a/crates/ui-egui/src/pluginsui.rs +++ b/crates/ui-egui/src/pluginsui.rs @@ -238,7 +238,10 @@ pub(crate) fn fetch(app: &RustGisApp, url: &str, limit: usize, ctx: Option String { - rustgis_engine::Session::redacted(url).as_str().to_string() -} - /// The add-service dialog's line for a layer on its way: what the dialog shows when a layer's /// download starts, and what [`RustGisApp::landed_in_dialog`] looks for when it lands (#1453). pub(crate) fn downloading(name: &str) -> String { @@ -472,8 +466,8 @@ pub fn poll_pages(app: &mut RustGisApp, ctx: &egui::Context) { } for read in std::mem::take(&mut app.ui.service_pages) { if !app.session.download_is_pending(read.landed.id) { - let url = &address_in_message(&read.landed.url); - let why = i18n::fmt("{url}: downloaded after its project was closed, so it wasn't added", &[("url", url)]); + let url = rustgis_engine::sources::address_in_message(&read.landed.url); + let why = i18n::fmt("{url}: downloaded after its project was closed, so it wasn't added", &[("url", &url)]); app.session.request_failed(&read.landed.params, &why); app.landed_in_dialog(&read.landed.params, Err(&why)); app.session.notify(why, false); @@ -534,7 +528,7 @@ pub(crate) fn go_on(app: &mut RustGisApp, landed: Landed, left: usize, step: Ste app.session.request_failed(&landed.params, &e); app.session.download_failed(landed.id); let url = &landed.url; - let e = if e.starts_with(url.as_str()) { e } else { format!("{url}: {e}") }; + let e = rustgis_engine::sources::message_with_address(url, &e); app.landed_in_dialog(&landed.params, Err(&e)); app.session.notify(e, true); } @@ -559,8 +553,9 @@ pub(crate) fn go_on(app: &mut RustGisApp, landed: Landed, left: usize, step: Ste /// Says what was left unread of a service layer's features, by its name. fn noted(app: &mut RustGisApp, landed: &Landed, note: Option) { if let Some(note) = note { - let name = landed.params.get("name").and_then(Value::as_str).filter(|n| !n.is_empty()).unwrap_or(&landed.url); - app.session.notify(format!("{name}: {note}"), false); + let name = landed.params.get("name").and_then(Value::as_str).filter(|n| !n.is_empty()); + let shown = name.map(str::to_string).unwrap_or_else(|| rustgis_engine::sources::address_in_message(&landed.url)); + app.session.notify(i18n::fmt("{name}: {note}", &[("name", &shown), ("note", ¬e)]), false); } } diff --git a/crates/ui-egui/src/stacui.rs b/crates/ui-egui/src/stacui.rs index d2df0d6d01..c718012799 100644 --- a/crates/ui-egui/src/stacui.rs +++ b/crates/ui-egui/src/stacui.rs @@ -629,7 +629,10 @@ fn landed_link(app: &mut RustGisApp, url: &str, r: Result, String>) { app.ui.stac.info = Some(format!("{} · {}", app.ui.stac.info.clone().unwrap_or_default(), unknown(app))); } } - _ => app.ui.stac.error = Some(i18n::fmt("{url}: not a STAC item or collection", &[("url", url)])), + _ => { + app.ui.stac.error = + Some(i18n::fmt("{url}: not a STAC item or collection", &[("url", &rustgis_engine::sources::address_in_message(url))])) + } } } diff --git a/crates/ui-egui/src/zarrui.rs b/crates/ui-egui/src/zarrui.rs index a6b2851ab7..9e5c4609e7 100644 --- a/crates/ui-egui/src/zarrui.rs +++ b/crates/ui-egui/src/zarrui.rs @@ -127,7 +127,8 @@ pub fn poll(app: &mut RustGisApp, ctx: &egui::Context) { Phase::Meta => { let docs: Vec<(String, Vec)> = std::mem::take(&mut st.got).into_iter().collect(); if docs.is_empty() && st.forbidden > 0 { - st.error = Some(i18n::fmt("{url}: access denied (HTTP 403)", &[("url", &st.root)])); + st.error = + Some(i18n::fmt("{url}: access denied (HTTP 403)", &[("url", &rustgis_engine::sources::address_in_message(&st.root))])); return; } match zarr::parse_store(&docs) { diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index 833a5ac569..66a9b6b8a2 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -2011,6 +2011,75 @@ fn data_from_a_web_address_lands_later() { assert!(h.state().ui.downloads.is_empty()); } +/// A failed header probe, ordinary download and project download retain their HTTP reason after +/// notices and diagnostic history redact the message again (#1475). +#[test] +fn address_failures_keep_status_without_credentials() { + use ogc_server::{Route, Server}; + let server = Server::new( + ["probe.tif", "remote.fgb", "download.geojson", "project.rgis"] + .into_iter() + .map(|name| Route { status: 503, ..Route::new(&format!("/{name}"), "") }) + .collect(), + ); + let mut h = harness(); + h.step(); + for (command, file, query) in [ + ("ui.addUrl", "probe.tif", "sig=hidden-probe"), + ("ui.addUrl", "remote.fgb", "token=hidden-remote"), + ("ui.addUrl", "download.geojson", "key=hidden-download"), + ("ui.openUrl", "project.rgis", "%74oken=hidden-project"), + ] { + let url = format!("{}/{file}?{query}#hidden-fragment", server.base); + h.state_mut().session.notices.clear(); + h.state_mut().run(command, json!({"url": url})).unwrap(); + for _ in 0..200 { + h.step(); + if h.state().session.notices.iter().any(|n| n.error) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + let notices = &h.state().session.notices; + let failure = notices.iter().find(|n| n.error).unwrap_or_else(|| panic!("{file}: no failure notice")); + assert!(failure.text.contains(file) && failure.text.contains("503"), "{failure:?}"); + assert!(notices.iter().all(|n| !n.text.contains("hidden")), "{notices:?}"); + let report = h.state().session.diagnostics.report(rustgis_engine::data::diagnostics::Filter::All); + assert!(!report.contains("hidden"), "{report}"); + assert!(report.lines().any(|line| line.contains(file) && line.contains("503")), "{report}"); + } +} + +/// A required Shapefile part is the failed resource, not a second name for its parent. +#[test] +fn shapefile_part_failures_keep_one_address_and_the_http_reason() { + use ogc_server::{Route, Server}; + let server = Server::new(vec![Route::new("/secret.shp", "shape-fixture"), Route::new("/versioned.shp", "shape-fixture")]); + let mut h = harness(); + h.step(); + for (name, query) in [("secret", "token=hidden-part"), ("versioned", "page=2")] { + h.state_mut().session.notices.clear(); + h.state_mut().run("ui.addUrl", json!({"url":format!("{}/{name}.shp?{query}",server.base)})).unwrap(); + for _ in 0..200 { + h.step(); + if h.state().session.notices.iter().any(|notice| notice.error) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + let failure = h.state().session.notices.iter().find(|notice| notice.error).unwrap(); + assert!(failure.text.contains(&format!("/{name}.dbf")) && failure.text.contains("404"), "{failure:?}"); + assert!(!failure.text.contains(".shp") && !failure.text.contains("hidden"), "{failure:?}"); + assert_eq!(failure.text.matches("://").count(), 1, "{failure:?}"); + let redacted = rustgis_engine::Session::redacted(&failure.text); + assert_eq!(redacted.as_str(), failure.text); + } + assert_eq!( + server.requests().iter().map(|request| request.path.as_str()).collect::>(), + ["/secret.shp", "/secret.dbf", "/versioned.shp", "/versioned.dbf"] + ); +} + /// A local server answering with a small cloud-optimised GeoTIFF by range, its HEAD answers /// announcing the TIFF header's length as Vantor's Transform API does: its address, the bytes it /// has sent and the file's length. diff --git a/docs/privacy.md b/docs/privacy.md index 8f255a1cc9..7f50dc9dcd 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -21,6 +21,21 @@ After `Basic` (without a colon) at the start of a line, the rest of the text is when the next word looks like base64: it has a digit or punctuation (a sentence-ending mark `.,!?;:)` at its end doesn't count), has 16 characters or more, or has 8 or more with a capital after the first letter. So a very short Basic credential such as `YTpi` is kept. + +Notices and request errors redact a known address before attaching its failure reason, so a +token or signature does not hide the HTTP status or other explanation. The complete query and +fragment are omitted, including values with unescaped spaces. For an unknown URL embedded in +free text, a secret-named query or fragment still omits the rest of that text: those later +words might be part of the credential, not an explanation. The same rule applies to copied +history errors, saved run messages and notebook cell output. Ambiguous slash-containing userinfo +is omitted rather than guessed to be a host, port and path. A root address is shown with `/` +before an omission marker or reason, so repeated redaction keeps its host. + +Copied run parameters are sanitized as data, without blanket-omitting JSON/XML syntax. +Typed addresses lose their keys and credential fields are emptied, but opaque text and nested +values a tool doesn't describe are kept for replay and flagged with a notice: they can still +hold credentials, so review copied code before sharing it. + Request and response bodies are not captured by network diagnostics. Background requests carry their originating session's weak capture token; failures are not broadcast to other embedded or Python sessions. Layer names, file paths and URL paths can