From 1ec5ec4cef51983c533923d99b2639ddbd09d786 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Tue, 10 Nov 2026 14:38:50 +0100 Subject: [PATCH 01/12] fix(diagnostics): cut open passwords and name removed queries in messages --- crates/data/src/diagnostics.rs | 83 ++++++++++++++++++++++++++- crates/engine/src/cmd/history.rs | 41 +++++++++++++ crates/engine/src/cmd/layer.rs | 4 +- crates/engine/src/cmd/plugin_fetch.rs | 19 ++++-- crates/engine/src/cmd/service.rs | 28 +++++++-- crates/engine/src/tests.rs | 12 ++++ docs/privacy.md | 18 +++++- 7 files changed, 188 insertions(+), 17 deletions(-) diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index ff8ea84831..d6a90ce0d1 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -604,6 +604,30 @@ fn ambiguous_userinfo(address: &str) -> bool { !authority.contains('@') && !authority.starts_with('[') && (authority.contains(':') || before_at.contains(':')) } +/// How many words after a bare `scheme://user:pass` token may still be its password's. +const PASSWORD_WORDS: usize = 4; + +/// Whether an address token that ends at whitespace may be the front of `scheme://user:pass word@host`: +/// nothing but `x:y` after the scheme (no path, query or `@` yet), and a word with an `@` before a +/// slash on the same line. `tail` is the text after the token. When `y` can't be a port (`pass`), +/// the word may be any of the next [`PASSWORD_WORDS`]. When it can (`12`, `8080`), `x:y` may just +/// as well be a host and port, so only the very next word counts: `https://host:8080 failed, +/// contact admin@example.org` keeps its reason. The caller omits the rest. `https://host:8080/x +/// user@example.org` has a path and keeps its words. +fn password_runs_on(address: &str, tail: &str) -> bool { + let Some((_, rest)) = address.split_once("://") else { return false }; + let Some((user, pass)) = rest.split_once(':') else { return false }; + if user.is_empty() || pass.is_empty() || rest.contains(['/', '\\', '?', '#', '@', '[', ']']) { + return false; + } + let digits = pass.trim_end_matches([',', ';', '.', ')', ':']); + let port = !digits.is_empty() && digits.len() <= 5 && digits.bytes().all(|b| b.is_ascii_digit()); + let words = if port { 1 } else { PASSWORD_WORDS }; + tail.split('\n').next().unwrap_or("").split_whitespace().take(words).any(|word| { + word.split_once('@').is_some_and(|(before, _)| !before.is_empty() && !before.contains(['/', '\\']) && !word.contains("://")) + }) +} + /// Whether a word of text too long to scan names an address whose query or userinfo may hold /// a credential: a linear look at each `://` word, without the full scanner. fn may_hold_secret(text: &str) -> bool { @@ -881,7 +905,7 @@ fn redact_scan(text: &str, diagnostic: bool) -> String { field_before = false; continue; } - if ambiguous_userinfo(address) { + if ambiguous_userinfo(address) || password_runs_on(address, rest.get(url_end..).unwrap_or("")) { // Its password may run on past whitespace, as a secret query's may: the rest // goes too, and the marker says so. out.push_str("[address and the rest omitted]"); @@ -1342,6 +1366,63 @@ mod tests { } } + #[test] + fn unknown_address_passwords_with_spaces_are_cut_through_their_at_sign() { + for text in [ + "see https://alice:pass word@host/x failed", + "see https://alice:12 word@host/x failed", + "https://alice:pass wo rd@host/x failed", + "https://alice:pa ss word@host:80/x failed", + "https://alice:12 word@example.org", + // `host:8080` reads as `alice:12`: an email right after it is cut, and the marker says so. + "see https://host:8080 user@example.org", + "see https://alice:pa\tss@host/x failed", + ] { + for clean in [redact(text), redact_parameter(text)] { + assert!(clean.ends_with("[address and the rest omitted]"), "{text} -> {clean}"); + for private in ["alice", "pass", "pa ", "12", "word", "wo ", "rd@", "ss@", "@", "failed"] { + assert!(!clean.contains(private), "{private} in {text} -> {clean}"); + } + assert_eq!(redact(&clean), clean); + } + // A saved copy is cut the same way, and says so. + let saved = secret_continuation_cut(text).unwrap_or_default(); + assert!(saved.ends_with("[address and the rest omitted]") && !saved.contains('@'), "{text} -> {saved}"); + } + // The cut is on one line: the next one is another message. + assert_eq!(redact("https://alice:pass word\nthen user@example.org"), "[address omitted] word\nthen user@example.org"); + } + + #[test] + fn addresses_without_an_open_password_keep_the_words_after_them() { + for text in [ + // A path ends the authority: the colon is a port's, and the email is prose. + "see https://host:8080/x user@example.org failed", + "see https://host:8080/ user@example.org failed", + // No colon in the authority, so no password to run on. + "see https://host user@example.org failed", + "https://host/x and mail user@example.org", + // No `@` within reach of the address. + "see https://host:8080 is down, ask your admin to restart it at ops@example.org", + "see https://host:8080 failed", + // A host and port is ambiguous with a short password: only the next word counts. + "see https://host:8080 failed, contact admin@example.org", + "see https://host:8080\nuser@example.org", + "https://example.org: timeout, mail ops@example.org", + // The local part of a word before `@` is a path, not a password's tail. + "see https://host:8080 /tmp/a@2x.png failed", + ] { + assert_eq!(redact(text), text, "{text}"); + assert_eq!(redact_parameter(text), text, "{text}"); + assert_eq!(secret_continuation_cut(text), None, "{text}"); + } + // A later address is its own: only its user information goes, not the words around it. + assert_eq!(redact("see https://host:8080 https://u@example.org/x failed"), "see https://host:8080 https://example.org/x failed"); + // A port followed by `:` is no valid address (omitted whole), but its reason stays. + let clean = redact("https://host:8080: HTTP 503 (ask ops@example.org)"); + assert_eq!(clean, "[address omitted] HTTP 503 (ask ops@example.org)"); + } + #[test] fn unambiguous_ports_and_path_at_signs_keep_address_context() { for address in diff --git a/crates/engine/src/cmd/history.rs b/crates/engine/src/cmd/history.rs index 93734ec0ec..b9b201e891 100644 --- a/crates/engine/src/cmd/history.rs +++ b/crates/engine/src/cmd/history.rs @@ -309,6 +309,16 @@ fn secret_pairs_emptied(t: &str) -> String { rest = rest.get(next..).unwrap_or(""); continue; }; + // A secret directly after a `?` in a word (`data.tif?token=abc def`) is a query, not a + // libpq pair: a URL query holds no raw space, so its later words may be the value's. The + // rest goes, said, as for an address with a scheme. + if let Some((k, _)) = pair.filter(|_| item.contains('?')) + && let Some(q) = k.find('?') + { + out.push_str(rest.get(..q).unwrap_or("")); + out.push_str("[query and the rest omitted]"); + return out; + } // The name and the `=`, as they are (spaces before the value go: the separator follows). out.push_str(rest.get(..at).unwrap_or("").trim_end()); // Past the value: to its closer when it opens with a quote or a brace, then to the next @@ -445,3 +455,34 @@ fn shell(t: &str) -> String { fn line(t: &str) -> String { t.chars().map(|c| if c.is_control() || c == '\u{2028}' || c == '\u{2029}' { ' ' } else { c }).take(300).collect() } + +#[cfg(test)] +mod tests { + use super::*; + + /// A URL query holds no raw space: a secret in a scheme-less address's query may run on past + /// one, so the rest is cut, said. Spaces in other text and libpq pairs stay as they were. + #[test] + fn a_secret_query_in_a_schemeless_address_cuts_the_rest() { + for text in ["data.tif?token=abc def", "data.tif?token=abc def ghi", "see data.tif?access_token=abc def"] { + for clean in [redacted_parameter(text), redacted_text(text)] { + assert!(clean.ends_with("[query and the rest omitted]"), "{text} -> {clean}"); + for private in ["abc", "def", "ghi", "token"] { + assert!(!clean.contains(private), "{private} in {text} -> {clean}"); + } + } + } + assert_eq!(redacted_parameter("see data.tif?token=abc def"), "see data.tif[query and the rest omitted]"); + for text in [ + "/data/a b.tif?x=1", + "/data/a.tif?sig=abc def&x=1 y", + "a.tif?key=abc def", + "data.tif?v=2 and more words", + "host=h dbname=g user=u", + ] { + assert_eq!(redacted_parameter(text), text, "{text}"); + } + // A libpq pair has no `?`: its value ends at the space, the next word is a pair of its own. + assert_eq!(redacted_parameter("host=h password=abc dbname=g"), "host=h password= dbname=g"); + } +} diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 9703fe0769..d444f5c303 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -3661,7 +3661,7 @@ fn parts_left_out(url: &str, bytes: &[u8], p: &Value) -> Option { if more > 0 { said.push(format!("and {more} more Shapefiles without some of theirs")); } - (!said.is_empty()).then(|| format!("{}: {}", crate::sources::redact(url), said.join("; "))) + (!said.is_empty()).then(|| format!("{}: {}", crate::sources::address_in_message(url), said.join("; "))) } /// A Shapefile by its `.shp` address, waited for (#1238): the `.shp`, then its `.dbf`, `.shx`, @@ -4632,7 +4632,7 @@ fn add_download(s: &mut Session, p: &Value) -> CmdResult { } // No window landed for it: a whole file isn't added in a window's place. if p.get("window").is_some_and(Value::is_object) { - let shown = crate::sources::redact(&url); + let shown = crate::sources::address_in_message(&url); return Err(format!("{shown}: \"window\" is for a window the app read off the frame: use layer.addUrl with \"bbox\"")); } // A download read already (an ArcGIS layer's protocol buffer pages, joined): added as read, diff --git a/crates/engine/src/cmd/plugin_fetch.rs b/crates/engine/src/cmd/plugin_fetch.rs index e1e267a95a..3c8c984ed5 100644 --- a/crates/engine/src/cmd/plugin_fetch.rs +++ b/crates/engine/src/cmd/plugin_fetch.rs @@ -431,9 +431,9 @@ fn same_tile(t: &rustgis_map::tiles::TileLayer, added: &rustgis_map::tiles::Tile fn not_started(cancelled: &[String]) -> String { match cancelled { [] => String::new(), - [one] => format!("; its download of {} wasn't started, and won't be", crate::sources::redact(one)), + [one] => format!("; its download of {} wasn't started, and won't be", crate::sources::address_in_message(one)), many => { - let shown: Vec<_> = many.iter().map(|u| crate::sources::redact(u)).collect(); + let shown: Vec<_> = many.iter().map(|u| crate::sources::address_in_message(u)).collect(); format!("; its {} downloads not yet started won't be: {}", many.len(), shown.join(", ")) } } @@ -546,7 +546,7 @@ impl UrlWaits { let mut addresses = Vec::new(); for (n, asked) in late { self.land(n, Err(format!("no download in {} minutes", REQUEST_WAIT.as_secs() / 60)), Added::default()); - addresses.extend(asked["url"].as_str().map(crate::sources::redact)); + addresses.extend(asked["url"].as_str().map(crate::sources::address_in_message)); self.remember_given_up(n, asked); } addresses @@ -1242,7 +1242,7 @@ mod tests { s.poll_plugin_retries(); let said: Vec<_> = s.notices_since(mark).iter().filter(|n| n.error && n.text.contains("stopped waiting")).collect(); assert_eq!(said.len(), 1, "{said:?}"); - assert!(said[0].text.starts_with("https://x.org/slow.csv") && !said[0].text.contains("hunter2")); + assert!(said[0].text.starts_with("https://x.org/slow.csv[query omitted]: no download") && !said[0].text.contains("hunter2")); // A real late claim is refused before its bytes can add anything. let id = s.download_started_for(&taken[0]).unwrap(); assert!(s.download_landed(id, b"x,y\n1,2\n".to_vec())); @@ -1259,8 +1259,9 @@ mod tests { assert_eq!(said.len(), 1, "{said:?}"); assert!( said[0].text.starts_with("2 downloads") - && said[0].text.contains("https://x.org/a.csv") + && said[0].text.contains("https://x.org/a.csv[query omitted]") && said[0].text.contains("https://x.org/b.csv") + && !said[0].text.contains("b.csv[") ); assert!(!said[0].text.contains("hunter2")); rustgis_plugins::unload("eng-slow-real"); @@ -1365,7 +1366,13 @@ mod tests { for cancelled in [vec!["https://x.org/a?token=hunter2".into()], vec!["https://x.org/a?token=hunter2".into(), "https://x.org/b".into()]] { - assert!(!not_started(&cancelled).contains("hunter2")); + let said = not_started(&cancelled); + assert!(!said.contains("hunter2") && said.contains("https://x.org/a[query omitted]"), "{said}"); + // A second address with nothing taken out is named as it is. + assert_eq!(said.contains("https://x.org/b"), cancelled.len() == 2, "{said}"); + } + for clean in [vec!["https://x.org/a".into()], vec!["https://x.org/a".into(), "https://x.org/b".into()]] { + assert!(!not_started(&clean).contains("omitted]"), "no query went, none said"); } } diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index d6dc3a6e62..e25c281c40 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -476,7 +476,7 @@ fn advertised_template_with( let mut doc = list.to_string(); // Each fallback is said, as [`left_template`] says it. for _ in 0..2 { - let shown = crate::sources::redact(&doc); + let shown = crate::sources::address_in_message(&doc); if let Err(e) = crate::render::rustgis_tiles::address::check(&doc, Some(list)) { let e = crate::sources::message_with_address(&doc, &e); out.fallback = Some(format!("{shown}: this tile set document was refused, so the standard tile address is used: {e}")); @@ -515,8 +515,10 @@ fn advertised_template_with( } } } - out.fallback = - Some(format!("{}: its tile sets lead on past two documents, so the standard tile address is used", crate::sources::redact(list))); + out.fallback = Some(format!( + "{}: its tile sets lead on past two documents, so the standard tile address is used", + crate::sources::address_in_message(list) + )); out } @@ -535,7 +537,7 @@ fn left_template(s: &mut Session, source: &str, list: &str, vector: bool, withhe let mut out = AdvertisedTemplate { template: None, crossed_origin: crosses_origin(source, list), fallback: None }; let mut doc = list.to_string(); for _ in 0..2 { - let shown = crate::sources::redact(&doc); + let shown = crate::sources::address_in_message(&doc); if let Err(e) = crate::render::rustgis_tiles::address::check(&doc, Some(list)) { let e = crate::sources::message_with_address(&doc, &e); s.notify(format!("{shown}: this tile set document was refused, so the standard tile address is used: {e}"), false); @@ -575,7 +577,10 @@ fn left_template(s: &mut Session, source: &str, list: &str, vector: bool, withhe } } s.notify( - format!("{}: its tile sets lead on past two documents, so the standard tile address is used", crate::sources::redact(list)), + format!( + "{}: its tile sets lead on past two documents, so the standard tile address is used", + crate::sources::address_in_message(list) + ), false, ); Ok(LeftTemplate::Read(out)) @@ -1416,6 +1421,19 @@ mod tests { let next = json!({"tilesets":[{"tileMatrixSetId":"WebMercatorQuad","links":[{"rel":"self","href":"https://tiles-fallback.example.invalid/next"}]}]}); let far = advertised_template_with(list, list, false, &mut withheld, |_| Ok(serde_json::to_vec(&next).unwrap_or_default())); assert!(far.fallback.as_ref().is_some_and(|n| n.contains("past two documents")), "{:?}", far.fallback); + // The address named in the notice says its secret query went; one without a query says nothing. + let keyed = "https://tiles-fallback.example.invalid/tiles?key=hunter2"; + let far = advertised_template_with(keyed, keyed, false, &mut withheld, |_| Ok(serde_json::to_vec(&next).unwrap_or_default())); + let said = far.fallback.unwrap_or_default(); + assert!(said.starts_with("https://tiles-fallback.example.invalid/tiles[query omitted]: its tile sets lead on past"), "{said}"); + assert!(!said.contains("hunter2"), "{said}"); + let bare = "https://tiles-fallback.example.invalid/tiles"; + let far = advertised_template_with(bare, bare, false, &mut withheld, |_| Ok(serde_json::to_vec(&next).unwrap_or_default())); + let said = far.fallback.unwrap_or_default(); + assert!( + said.starts_with("https://tiles-fallback.example.invalid/tiles: its tile sets lead on past") && !said.contains("omitted]"), + "{said}" + ); } #[test] diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 109d7c9969..8617cc38db 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13826,6 +13826,7 @@ fn shapefile_by_its_shp_address() { parts.iter().map(|(e, b)| (format!("/PI/TERESINA/HIDROGRAFIA/PI_2211001_RIOS.{e}?token=1"), b.clone())).collect(); // The same without its .prj and .cpg, and one without its .dbf. files.extend(parts.iter().filter(|(e, _)| !matches!(*e, "prj" | "cpg")).map(|(e, b)| (format!("/plain/rios.{e}"), b.clone()))); + files.extend(parts.iter().filter(|(e, _)| !matches!(*e, "prj" | "cpg")).map(|(e, b)| (format!("/keyed/rios.{e}?token=zz"), b.clone()))); files.extend(parts.iter().filter(|(e, _)| *e != "dbf").map(|(e, b)| (format!("/nodbf/rios.{e}"), b.clone()))); files.extend(parts.iter().filter(|(e, _)| *e != "shx").map(|(e, b)| (format!("/noshx/rios.{e}"), b.clone()))); // One whose signature doesn't cover its .prj, and one whose address answers with its parts zipped. @@ -13870,6 +13871,7 @@ fn shapefile_by_its_shp_address() { assert_eq!(r["features"], 1, "{r}"); let said = &s.notices.last().unwrap().text; assert!(said.contains("no .prj, .cpg found or readable beside it (2 of 3") && said.contains("longitude/latitude"), "{said}"); + assert!(said.contains("/plain/rios.shp: rios.shp: no .prj") && !said.contains("omitted]"), "{said}"); // Their 404s are expected: noted once for the server, not recorded as failed requests. let records = s.diagnostics.records(); let failed = |r: &&rustgis_data::diagnostics::Record| { @@ -13881,6 +13883,10 @@ fn shapefile_by_its_shp_address() { s.run("layer.addUrl", &json!({"url": format!("{base}/plain/rios.shp"), "name": "Com crs", "crs": "EPSG:31983"})).unwrap(); let said = &s.notices.last().unwrap().text; assert!(said.contains("no .prj, .cpg found or readable beside it") && !said.contains("longitude/latitude"), "{said}"); + // A secret query taken out of the address in the notice is said; no query, no marker. + s.run("layer.addUrl", &json!({"url": format!("{base}/keyed/rios.shp?token=zz"), "name": "Keyed", "crs": "EPSG:31983"})).unwrap(); + let said = &s.notices.last().unwrap().text; + assert!(said.contains("/keyed/rios.shp[query omitted]: rios.shp: no .prj") && !said.contains("zz"), "{said}"); // A long name keeps its extension (and is still a .shp's zip). let long = crate::cmd::layer::url_file_name(&format!("https://x.org/{}.shp", "a".repeat(200)), b"PK\x03\x04").unwrap(); assert!(long.ends_with(".zip") && long.chars().count() <= 120, "{long}"); @@ -33633,6 +33639,12 @@ fn a_cog_windows_read_is_apart_from_adding_it() { assert!(s.download_landed(id, b"{}".to_vec())); let e = s.run("layer.addDownload", &json!({"id": id, "url": url, "window": {"bbox": [1, 2, 3, 4]}})).unwrap_err(); assert!(e.contains("\"window\" is for") && !e.contains("hidden-1330"), "{e}"); + assert!(e.contains("scene.tif[query omitted]: \"window\" is for"), "{e}"); + let id = s.download_started(); + assert!(s.download_landed(id, b"{}".to_vec())); + let plain = url.split('?').next().unwrap_or(&url).to_string(); + let e = s.run("layer.addDownload", &json!({"id": id, "url": plain, "window": {"bbox": [1, 2, 3, 4]}})).unwrap_err(); + assert!(e.contains("scene.tif: \"window\" is for") && !e.contains("omitted]"), "no query went, none said: {e}"); // A project opened while it was read: it isn't kept to be added. let read = read_cog_window(&window_asked(&s, &p).unwrap()).unwrap(); let id = s.download_started(); diff --git a/docs/privacy.md b/docs/privacy.md index ad52308259..ce4f1270a1 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -27,9 +27,21 @@ token or signature does not hide the HTTP status or other explanation. The compl fragment are omitted, including values with unescaped spaces. For an unknown URL embedded in free text, a secret-named query or fragment still omits the rest of that text: those later words might be part of the credential, not an explanation. The same rule applies to copied -history errors, saved run messages and notebook cell output. Ambiguous slash-containing userinfo -is omitted rather than guessed to be a host, port and path. A root address is shown with `/` -before an omission marker or reason, so repeated redaction keeps its host. +history errors, saved run messages and notebook cell output, and to a secret directly after the +`?` of an address without a scheme (`data.tif?token=abc def`). A query with no secret name +(`?custom=hidden unescaped hidden`) is omitted only to its first space: the redactor can't tell +a value's later words from prose, so they are kept. Ambiguous slash-containing userinfo +is omitted rather than guessed to be a host, port and path. So is a bare `scheme://name:text` +(no path, query or `@` yet) of an unknown URL when a word holding an `@` follows on its line +(`https://alice:pass word@host/x`): the text may be a password with a space in it, so the rest of +the text is omitted too, saying `[address and the rest omitted]`. When `text` could be a port +(`https://host:8080 user@example.org`, `https://alice:12 word@host`) only the next word counts, +since a port and a short password look alike and prose often follows a host and port; otherwise +any of the next four words. A path (`https://host:8080/x user@example.org`) ends the ambiguity, +and the words are kept. A root address is shown with `/` before an omission marker or reason, so +repeated redaction keeps its host. A message that names a known address says when its query or +fragment was taken out (`[query omitted]`, `[fragment omitted]`); the address kept in a saved +source or a project never carries a marker. Copied run parameters are sanitized as data, without blanket-omitting JSON/XML syntax. Typed addresses lose their keys and credential fields are emptied, but opaque text and nested From e14a39823e200b74b7819bc6bdaea4f548e5db64 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Tue, 10 Nov 2026 14:50:31 +0100 Subject: [PATCH 02/12] fix(diagnostics): review round 1 fixes --- crates/data/src/diagnostics.rs | 14 +++++-- crates/engine/src/cmd/history.rs | 71 +++++++++++++++++++++++--------- docs/privacy.md | 27 ++++++------ 3 files changed, 77 insertions(+), 35 deletions(-) diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index d6a90ce0d1..5c53aff01e 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -613,11 +613,12 @@ const PASSWORD_WORDS: usize = 4; /// the word may be any of the next [`PASSWORD_WORDS`]. When it can (`12`, `8080`), `x:y` may just /// as well be a host and port, so only the very next word counts: `https://host:8080 failed, /// contact admin@example.org` keeps its reason. The caller omits the rest. `https://host:8080/x -/// user@example.org` has a path and keeps its words. +/// user@example.org` has a path and keeps its words. An empty `y` (`https://example.org:`) is no +/// password, and a `[` is an IPv6 host, so neither is cut; an empty `x` (`redis://:pa ss@host`) is. fn password_runs_on(address: &str, tail: &str) -> bool { let Some((_, rest)) = address.split_once("://") else { return false }; - let Some((user, pass)) = rest.split_once(':') else { return false }; - if user.is_empty() || pass.is_empty() || rest.contains(['/', '\\', '?', '#', '@', '[', ']']) { + let Some((_, pass)) = rest.split_once(':') else { return false }; + if pass.is_empty() || rest.contains(['/', '\\', '?', '#', '@', '[']) { return false; } let digits = pass.trim_end_matches([',', ';', '.', ')', ':']); @@ -1374,6 +1375,11 @@ mod tests { "https://alice:pass wo rd@host/x failed", "https://alice:pa ss word@host:80/x failed", "https://alice:12 word@example.org", + // An empty name, a `]` in the text, the fourth word, and a number too long for a port. + "redis://:pa ss@host/x failed", + "https://alice:pa]ss word@host failed", + "https://alice:pass a b c d@host failed", + "https://alice:123456 a b c@host failed", // `host:8080` reads as `alice:12`: an email right after it is cut, and the marker says so. "see https://host:8080 user@example.org", "see https://alice:pa\tss@host/x failed", @@ -1407,6 +1413,8 @@ mod tests { "see https://host:8080 failed", // A host and port is ambiguous with a short password: only the next word counts. "see https://host:8080 failed, contact admin@example.org", + "see https://[::1]:8080 user@example.org failed", + "https://x:12345 a b@example.org failed", "see https://host:8080\nuser@example.org", "https://example.org: timeout, mail ops@example.org", // The local part of a word before `@` is a path, not a password's tail. diff --git a/crates/engine/src/cmd/history.rs b/crates/engine/src/cmd/history.rs index b9b201e891..94e6718d6e 100644 --- a/crates/engine/src/cmd/history.rs +++ b/crates/engine/src/cmd/history.rs @@ -167,7 +167,7 @@ pub(crate) fn redacted_parameter(t: &str) -> String { } else if structured || t.contains("://") { rustgis_data::diagnostics::redact_parameter(t) } else { - t.to_string() + schemeless_query_cut(t) }; query_pairs_emptied(&private_keys_removed(&text)) } @@ -182,12 +182,41 @@ pub(crate) fn redacted_text(t: &str) -> String { } else if t.contains("://") { rustgis_data::diagnostics::redact(t) } else { - t.to_string() + schemeless_query_cut(t) }; // After `redact` too: a prefix it knows may come in a form it doesn't (GDAL's `MYSQL:`). query_pairs_emptied(&private_keys_removed(&r)) } +/// Whether `name=value` is a secret pair. A bare `key`, `sig` or `sas` is as likely a field in an +/// expression (`key=5`): a secret only when its value reads as one. +fn secret_pair(name: &str, value: &str) -> bool { + let k = name.trim(); + !k.is_empty() + && rustgis_data::source::credentials::secret_key(k) + && (!["key", "sig", "sas"].iter().any(|p| k.eq_ignore_ascii_case(p)) + || crate::sources::looks_like_a_key(value.trim_matches(['\'', '"', '{', '}']))) +} + +/// Text with no scheme that has a secret pair in the query of one of its words (`data.tif?token=abc +/// def`, `a.tif?x=1&token=abc def`): a URL query holds no raw space, so the words after it may be +/// the value's. The word's query and the rest of the text go, said, as for an address with a +/// scheme. A libpq-style `password=abc` has no `?`: its value ends at the space. +fn schemeless_query_cut(t: &str) -> String { + let mut start = 0; + for word in t.split_inclusive(char::is_whitespace) { + let w = word.trim_end_matches(char::is_whitespace); + if let Some(q) = w.find('?') { + let query = w.get(q + 1..).unwrap_or(""); + if query.split('&').filter_map(|pair| pair.split_once('=')).any(|(k, v)| secret_pair(k, v)) { + return format!("{}[query and the rest omitted]", t.get(..start + q).unwrap_or("")); + } + } + start += word.len(); + } + t.to_string() +} + /// `t` with its secret pairs emptied (`secret_pairs_emptied`), parted first at each `&` that /// starts another `name=` pair, as a query string's do (`a=1&password=…`, with no `://` for /// `redact` to read it by). An `&` inside a value (`password=ab&cd`), or in a value quoted or @@ -291,16 +320,9 @@ fn secret_pairs_emptied(t: &str) -> String { let next = word.get(..word.find(parts).unwrap_or(word.len())).unwrap_or(""); if word.len() < value.len() && !next.is_empty() && !next.contains('=') { (k, at + value.len() - word.len()) } else { (k, at) } }); - // A bare `key`, `sig` or `sas` is as likely a field in an expression (`key=5`): a secret - // only when its value reads as one. let secret = |&(k, at): &(&str, usize)| { - let k = k.trim(); let value = rest.get(at..).unwrap_or(""); - let v = value.get(..value.find(parts).unwrap_or(value.len())).unwrap_or(""); - !k.is_empty() - && rustgis_data::source::credentials::secret_key(k) - && (!["key", "sig", "sas"].iter().any(|p| k.eq_ignore_ascii_case(p)) - || crate::sources::looks_like_a_key(v.trim_matches(['\'', '"', '{', '}']))) + secret_pair(k, value.get(..value.find(parts).unwrap_or(value.len())).unwrap_or("")) }; let Some((_, at)) = pair.filter(secret) else { // Not a secret: the item and the separator after it, as they are. @@ -309,16 +331,6 @@ fn secret_pairs_emptied(t: &str) -> String { rest = rest.get(next..).unwrap_or(""); continue; }; - // A secret directly after a `?` in a word (`data.tif?token=abc def`) is a query, not a - // libpq pair: a URL query holds no raw space, so its later words may be the value's. The - // rest goes, said, as for an address with a scheme. - if let Some((k, _)) = pair.filter(|_| item.contains('?')) - && let Some(q) = k.find('?') - { - out.push_str(rest.get(..q).unwrap_or("")); - out.push_str("[query and the rest omitted]"); - return out; - } // The name and the `=`, as they are (spaces before the value go: the separator follows). out.push_str(rest.get(..at).unwrap_or("").trim_end()); // Past the value: to its closer when it opens with a quote or a brace, then to the next @@ -473,6 +485,25 @@ mod tests { } } assert_eq!(redacted_parameter("see data.tif?token=abc def"), "see data.tif[query and the rest omitted]"); + // A secret anywhere in the word's query, and the exact names a path word would hide. + for text in [ + "a.tif?x=1&token=abc def", + "a.tif?token=abc&x=1 def", + "data.tif?pwd=hunter2 def", + "data.tif?sig=Ab3dE9fG7hJ2kL5mN8pQ4rS6tU1vW0xY def", + ] { + let clean = redacted_parameter(text); + assert!( + clean.ends_with("[query and the rest omitted]") && !clean.contains("def") && !clean.contains("hunter2"), + "{text} -> {clean}" + ); + assert!(!clean.contains("x=1"), "{text} -> {clean}"); + } + // A credential word in the path is no secret query: the words after it stay. + for text in ["/data/tokens/a.tif?v=2 failed", r"C:\Users\me\secrets\a.tif?x=1 done"] { + let clean = redacted_parameter(text); + assert!(!clean.contains("omitted]") && (clean.ends_with("failed") || clean.ends_with("done")), "{text} -> {clean}"); + } for text in [ "/data/a b.tif?x=1", "/data/a.tif?sig=abc def&x=1 y", diff --git a/docs/privacy.md b/docs/privacy.md index ce4f1270a1..42d617f693 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -27,18 +27,21 @@ token or signature does not hide the HTTP status or other explanation. The compl fragment are omitted, including values with unescaped spaces. For an unknown URL embedded in free text, a secret-named query or fragment still omits the rest of that text: those later words might be part of the credential, not an explanation. The same rule applies to copied -history errors, saved run messages and notebook cell output, and to a secret directly after the -`?` of an address without a scheme (`data.tif?token=abc def`). A query with no secret name -(`?custom=hidden unescaped hidden`) is omitted only to its first space: the redactor can't tell -a value's later words from prose, so they are kept. Ambiguous slash-containing userinfo -is omitted rather than guessed to be a host, port and path. So is a bare `scheme://name:text` -(no path, query or `@` yet) of an unknown URL when a word holding an `@` follows on its line -(`https://alice:pass word@host/x`): the text may be a password with a space in it, so the rest of -the text is omitted too, saying `[address and the rest omitted]`. When `text` could be a port -(`https://host:8080 user@example.org`, `https://alice:12 word@host`) only the next word counts, -since a port and a short password look alike and prose often follows a host and port; otherwise -any of the next four words. A path (`https://host:8080/x user@example.org`) ends the ambiguity, -and the words are kept. A root address is shown with `/` before an omission marker or reason, so +history errors, saved run messages and notebook cell output, and to a secret in the query of a +word of scheme-less text (`data.tif?token=abc def`, `a.tif?x=1&token=abc def`), which goes with +the rest of the text, saying `[query and the rest omitted]`. For an unknown `scheme://` URL, a +query with no secret name (`?custom=hidden unescaped hidden`) is omitted only to its first +space: the redactor can't tell a value's later words from prose, so they are kept. Ambiguous +slash-containing userinfo is omitted rather than guessed to be a host, port and path. So is a +bare `scheme://name:text` (no path, query, `@` or IPv6 `[` in it, and a `text` that isn't empty) +of an unknown URL when a word with an `@` (and something before it, no slash) follows on its +line (`https://alice:pass word@host/x`, `redis://:pa ss@host`): the text may be a password with a +space in it, so the rest of the text is omitted too, saying `[address and the rest omitted]`. +When `text` could be a port (one to five digits: `https://host:8080 user@example.org`, +`https://alice:12 word@host`) only the next word counts, since a port and a short password look +alike and prose often follows a host and port; otherwise any of the next four words. A path +(`https://host:8080/x user@example.org`) ends the ambiguity, and the words are kept. A root +address is shown with `/` before an omission marker or reason, so repeated redaction keeps its host. A message that names a known address says when its query or fragment was taken out (`[query omitted]`, `[fragment omitted]`); the address kept in a saved source or a project never carries a marker. From 5cd0af050244e3dc1fbbec341ef144ee77132552 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Tue, 10 Nov 2026 14:59:27 +0100 Subject: [PATCH 03/12] fix(diagnostics): review round 2 fixes --- crates/data/src/diagnostics.rs | 34 +++++++++++++++++++++++++++----- crates/engine/src/cmd/history.rs | 18 +++++++++++------ docs/privacy.md | 22 +++++++++++---------- 3 files changed, 53 insertions(+), 21 deletions(-) diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index 5c53aff01e..aeeb23d0cd 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -622,18 +622,35 @@ fn password_runs_on(address: &str, tail: &str) -> bool { return false; } let digits = pass.trim_end_matches([',', ';', '.', ')', ':']); - let port = !digits.is_empty() && digits.len() <= 5 && digits.bytes().all(|b| b.is_ascii_digit()); - let words = if port { 1 } else { PASSWORD_WORDS }; - tail.split('\n').next().unwrap_or("").split_whitespace().take(words).any(|word| { - word.split_once('@').is_some_and(|(before, _)| !before.is_empty() && !before.contains(['/', '\\']) && !word.contains("://")) - }) + if digits.is_empty() { + return false; + } + let port = digits.len() <= 5 && digits.bytes().all(|b| b.is_ascii_digit()); + // Words of the token's own line, a few of them, found without scanning the rest of the text. + let mut rest = tail; + for _ in 0..if port { 1 } else { PASSWORD_WORDS } { + rest = rest.trim_start_matches(|c: char| c.is_whitespace() && c != '\n'); + if rest.is_empty() || rest.starts_with('\n') { + return false; + } + let end = rest.find(char::is_whitespace).unwrap_or(rest.len()); + let word = rest.get(..end).unwrap_or(""); + if word.split_once('@').is_some_and(|(before, _)| !before.is_empty() && !before.contains(['/', '\\']) && !word.contains("://")) { + return true; + } + rest = rest.get(end..).unwrap_or(""); + } + false } /// Whether a word of text too long to scan names an address whose query or userinfo may hold /// a credential: a linear look at each `://` word, without the full scanner. fn may_hold_secret(text: &str) -> bool { text.split_whitespace().filter(|word| word.contains("://")).any(|word| { + // Where the word ends in the text, so the words after it can be looked at. + let after = (word.as_ptr() as usize).checked_sub(text.as_ptr() as usize).and_then(|at| text.get(at + word.len()..)).unwrap_or(""); ambiguous_userinfo(word) + || password_runs_on(word, after) || word .split_once(['?', '#']) .is_some_and(|(_, query)| query.split(['&', ';', '#']).any(|pair| secret_name(pair.split('=').next().unwrap_or("")))) @@ -1238,6 +1255,11 @@ mod tests { let ambiguous = format!("{} https://u:123?w@host/x tail", "word ".repeat(4000)); assert!(secret_continuation_cut(&ambiguous).is_some()); assert_eq!(secret_continuation_cut(&format!("{} see https://example.org/a?page=2 then", "word ".repeat(4000))), None); + // A password with a space goes whole in long text too, as in short; a port and prose keep. + let open = format!("{} see https://alice:pass word@host/x failed", "word ".repeat(4000)); + assert!(secret_continuation_cut(&open).is_some_and(|cut| !cut.contains("word@"))); + let port = format!("{} see https://host:8080 failed, contact admin@example.org", "word ".repeat(4000)); + assert_eq!(secret_continuation_cut(&port), None); assert_eq!(secret_continuation_cut(&"word ".repeat(4000)), None); assert_eq!(secret_continuation_cut("failed at https://example.org/a: HTTP 500"), None); } @@ -1426,6 +1448,8 @@ mod tests { } // A later address is its own: only its user information goes, not the words around it. assert_eq!(redact("see https://host:8080 https://u@example.org/x failed"), "see https://host:8080 https://example.org/x failed"); + // Nothing left of the text once its punctuation is trimmed is no password to run on. + assert!(redact("https://example.org:, mail ops@example.org").ends_with("mail ops@example.org")); // A port followed by `:` is no valid address (omitted whole), but its reason stays. let clean = redact("https://host:8080: HTTP 503 (ask ops@example.org)"); assert_eq!(clean, "[address omitted] HTTP 503 (ask ops@example.org)"); diff --git a/crates/engine/src/cmd/history.rs b/crates/engine/src/cmd/history.rs index 94e6718d6e..8828935844 100644 --- a/crates/engine/src/cmd/history.rs +++ b/crates/engine/src/cmd/history.rs @@ -198,17 +198,18 @@ fn secret_pair(name: &str, value: &str) -> bool { || crate::sources::looks_like_a_key(value.trim_matches(['\'', '"', '{', '}']))) } -/// Text with no scheme that has a secret pair in the query of one of its words (`data.tif?token=abc -/// def`, `a.tif?x=1&token=abc def`): a URL query holds no raw space, so the words after it may be -/// the value's. The word's query and the rest of the text go, said, as for an address with a -/// scheme. A libpq-style `password=abc` has no `?`: its value ends at the space. +/// Text with no scheme that has a secret pair in the query or fragment of one of its words +/// (`data.tif?token=abc def`, `a.tif?x=1&token=abc def`, `a.tif?x=1;token=abc def`): a URL query +/// holds no raw space, so the words after it may be the value's. The word's query and the rest of +/// the text go, said, as for an address with a scheme. A libpq-style `password=abc` has no `?` or +/// `#`: its value ends at the space. fn schemeless_query_cut(t: &str) -> String { let mut start = 0; for word in t.split_inclusive(char::is_whitespace) { let w = word.trim_end_matches(char::is_whitespace); - if let Some(q) = w.find('?') { + if let Some(q) = w.find(['?', '#']) { let query = w.get(q + 1..).unwrap_or(""); - if query.split('&').filter_map(|pair| pair.split_once('=')).any(|(k, v)| secret_pair(k, v)) { + if query.split(['&', ';', '#', '?']).filter_map(|pair| pair.split_once('=')).any(|(k, v)| secret_pair(k, v)) { return format!("{}[query and the rest omitted]", t.get(..start + q).unwrap_or("")); } } @@ -491,6 +492,10 @@ mod tests { "a.tif?token=abc&x=1 def", "data.tif?pwd=hunter2 def", "data.tif?sig=Ab3dE9fG7hJ2kL5mN8pQ4rS6tU1vW0xY def", + "a.tif?x=1;token=abc def", + "a.tif?x=1#token=abc def", + "a.tif?x=1?token=abc def", + "data.tif#token=abc def", ] { let clean = redacted_parameter(text); assert!( @@ -509,6 +514,7 @@ mod tests { "/data/a.tif?sig=abc def&x=1 y", "a.tif?key=abc def", "data.tif?v=2 and more words", + "a.tif?x=1;y=2 def", "host=h dbname=g user=u", ] { assert_eq!(redacted_parameter(text), text, "{text}"); diff --git a/docs/privacy.md b/docs/privacy.md index 42d617f693..0c91451847 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -27,19 +27,21 @@ token or signature does not hide the HTTP status or other explanation. The compl fragment are omitted, including values with unescaped spaces. For an unknown URL embedded in free text, a secret-named query or fragment still omits the rest of that text: those later words might be part of the credential, not an explanation. The same rule applies to copied -history errors, saved run messages and notebook cell output, and to a secret in the query of a -word of scheme-less text (`data.tif?token=abc def`, `a.tif?x=1&token=abc def`), which goes with -the rest of the text, saying `[query and the rest omitted]`. For an unknown `scheme://` URL, a -query with no secret name (`?custom=hidden unescaped hidden`) is omitted only to its first -space: the redactor can't tell a value's later words from prose, so they are kept. Ambiguous -slash-containing userinfo is omitted rather than guessed to be a host, port and path. So is a -bare `scheme://name:text` (no path, query, `@` or IPv6 `[` in it, and a `text` that isn't empty) -of an unknown URL when a word with an `@` (and something before it, no slash) follows on its -line (`https://alice:pass word@host/x`, `redis://:pa ss@host`): the text may be a password with a +history errors, saved run messages and notebook cell output, and to a secret in the query or +fragment of a word of scheme-less text (`data.tif?token=abc def`, `a.tif?x=1;token=abc def`), +which goes with the rest of the text, saying `[query and the rest omitted]`. For an unknown +`scheme://` URL, a query with no secret name (`?custom=hidden unescaped hidden`) is omitted only +to its first space: the redactor can't tell a value's later words from prose, so they are kept. +Ambiguous slash-containing userinfo is omitted rather than guessed to be a host, port and path. +So is a bare `scheme://name:text` (no `/`, `\`, `?`, `#`, `@` or IPv6 `[` in it, and a `text` that +isn't empty once trailing punctuation is trimmed) of an unknown URL when a word with an `@` (and +something before it, no slash, not another address) follows on its line +(`https://alice:pass word@host/x`, `redis://:pa ss@host`): the text may be a password with a space in it, so the rest of the text is omitted too, saying `[address and the rest omitted]`. When `text` could be a port (one to five digits: `https://host:8080 user@example.org`, `https://alice:12 word@host`) only the next word counts, since a port and a short password look -alike and prose often follows a host and port; otherwise any of the next four words. A path +alike and prose often follows a host and port; otherwise any of the next four words, so a +password with more spaces than that, or with a `/` in it, is not found this way. A path (`https://host:8080/x user@example.org`) ends the ambiguity, and the words are kept. A root address is shown with `/` before an omission marker or reason, so repeated redaction keeps its host. A message that names a known address says when its query or From 59e4dd8e6e706040584a2559a40c3b6e55822763 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Tue, 10 Nov 2026 15:24:57 +0100 Subject: [PATCH 04/12] fix(diagnostics): review round 3 fixes --- crates/data/src/diagnostics.rs | 12 ++++++++++-- crates/engine/src/cmd/history.rs | 6 ++++-- docs/privacy.md | 5 +++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/crates/data/src/diagnostics.rs b/crates/data/src/diagnostics.rs index aeeb23d0cd..b6c085a1ad 100644 --- a/crates/data/src/diagnostics.rs +++ b/crates/data/src/diagnostics.rs @@ -614,11 +614,13 @@ const PASSWORD_WORDS: usize = 4; /// as well be a host and port, so only the very next word counts: `https://host:8080 failed, /// contact admin@example.org` keeps its reason. The caller omits the rest. `https://host:8080/x /// user@example.org` has a path and keeps its words. An empty `y` (`https://example.org:`) is no -/// password, and a `[` is an IPv6 host, so neither is cut; an empty `x` (`redis://:pa ss@host`) is. +/// password, and a host in brackets (IPv6) is no `x:y`, so neither is cut; an empty `x` +/// (`redis://:pa ss@host`) is. A token that ended at a quote or bracket, not whitespace, is part of +/// structured text and is left to the other rules. fn password_runs_on(address: &str, tail: &str) -> bool { let Some((_, rest)) = address.split_once("://") else { return false }; let Some((_, pass)) = rest.split_once(':') else { return false }; - if pass.is_empty() || rest.contains(['/', '\\', '?', '#', '@', '[']) { + if !tail.starts_with(char::is_whitespace) || pass.is_empty() || rest.starts_with('[') || rest.contains(['/', '\\', '?', '#', '@']) { return false; } let digits = pass.trim_end_matches([',', ';', '.', ')', ':']); @@ -1400,6 +1402,8 @@ mod tests { // An empty name, a `]` in the text, the fourth word, and a number too long for a port. "redis://:pa ss@host/x failed", "https://alice:pa]ss word@host failed", + "https://alice:pa[ss word@host failed", + "https://alice:[pa ss]@host/x failed", "https://alice:pass a b c d@host failed", "https://alice:123456 a b c@host failed", // `host:8080` reads as `alice:12`: an email right after it is cut, and the marker says so. @@ -1436,6 +1440,7 @@ mod tests { // A host and port is ambiguous with a short password: only the next word counts. "see https://host:8080 failed, contact admin@example.org", "see https://[::1]:8080 user@example.org failed", + "see https://[::1]:8080 failed, contact admin@example.org", "https://x:12345 a b@example.org failed", "see https://host:8080\nuser@example.org", "https://example.org: timeout, mail ops@example.org", @@ -1453,6 +1458,9 @@ mod tests { // A port followed by `:` is no valid address (omitted whole), but its reason stays. let clean = redact("https://host:8080: HTTP 503 (ask ops@example.org)"); assert_eq!(clean, "[address omitted] HTTP 503 (ask ops@example.org)"); + // A token that ends at a quote is part of structured text, not a front of userinfo. + let json = r#"{"url":"https://h:8080","b":"a@b"}"#; + assert_eq!(redact_parameter(json), json); } #[test] diff --git a/crates/engine/src/cmd/history.rs b/crates/engine/src/cmd/history.rs index 8828935844..1965947f9d 100644 --- a/crates/engine/src/cmd/history.rs +++ b/crates/engine/src/cmd/history.rs @@ -201,8 +201,9 @@ fn secret_pair(name: &str, value: &str) -> bool { /// Text with no scheme that has a secret pair in the query or fragment of one of its words /// (`data.tif?token=abc def`, `a.tif?x=1&token=abc def`, `a.tif?x=1;token=abc def`): a URL query /// holds no raw space, so the words after it may be the value's. The word's query and the rest of -/// the text go, said, as for an address with a scheme. A libpq-style `password=abc` has no `?` or -/// `#`: its value ends at the space. +/// the text go, said, as for an address with a scheme. Only the names `secret_key` knows count, +/// as everywhere in this file. A libpq-style `password=abc` has no `?` or `#`: its value ends at +/// the space. fn schemeless_query_cut(t: &str) -> String { let mut start = 0; for word in t.split_inclusive(char::is_whitespace) { @@ -521,5 +522,6 @@ mod tests { } // A libpq pair has no `?`: its value ends at the space, the next word is a pair of its own. assert_eq!(redacted_parameter("host=h password=abc dbname=g"), "host=h password= dbname=g"); + assert_eq!(redacted_parameter("password=a b"), "password= b"); } } diff --git a/docs/privacy.md b/docs/privacy.md index 0c91451847..92c65a31a9 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -33,8 +33,9 @@ which goes with the rest of the text, saying `[query and the rest omitted]`. For `scheme://` URL, a query with no secret name (`?custom=hidden unescaped hidden`) is omitted only to its first space: the redactor can't tell a value's later words from prose, so they are kept. Ambiguous slash-containing userinfo is omitted rather than guessed to be a host, port and path. -So is a bare `scheme://name:text` (no `/`, `\`, `?`, `#`, `@` or IPv6 `[` in it, and a `text` that -isn't empty once trailing punctuation is trimmed) of an unknown URL when a word with an `@` (and +So is a bare `scheme://name:text` (no `/`, `\`, `?`, `#` or `@` in it, not an IPv6 host in +brackets, ended by whitespace rather than a quote, and a `text` that isn't empty once trailing +punctuation is trimmed) of an unknown URL when a word with an `@` (and something before it, no slash, not another address) follows on its line (`https://alice:pass word@host/x`, `redis://:pa ss@host`): the text may be a password with a space in it, so the rest of the text is omitted too, saying `[address and the rest omitted]`. From 38078ebc9023c2b7c979a48dcaad3a9f5512beb0 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Tue, 10 Nov 2026 17:24:26 +0100 Subject: [PATCH 05/12] test(diagnostics): pin that a public query in a notice reads as before --- crates/engine/src/cmd/plugin_fetch.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/engine/src/cmd/plugin_fetch.rs b/crates/engine/src/cmd/plugin_fetch.rs index 3c8c984ed5..a65d7aa9db 100644 --- a/crates/engine/src/cmd/plugin_fetch.rs +++ b/crates/engine/src/cmd/plugin_fetch.rs @@ -1374,6 +1374,13 @@ mod tests { for clean in [vec!["https://x.org/a".into()], vec!["https://x.org/a".into(), "https://x.org/b".into()]] { assert!(!not_started(&clean).contains("omitted]"), "no query went, none said"); } + // A notice's own redaction already drops a public query and says so, whatever composed it: + // naming the address with `address_in_message` changes that for secret queries only. + let public = "https://x.org/wms?LAYER=a"; + let old = Session::redacted(&format!("its download of {} wasn't started", crate::sources::redact(public))); + let new = Session::redacted(&format!("its download of {} wasn't started", crate::sources::address_in_message(public))); + assert_eq!(old.as_str(), new.as_str()); + assert!(new.as_str().contains("https://x.org/wms[query omitted]"), "{}", new.as_str()); } /// Sharing is bounded by the original plugin incarnation; duplicate clicks of that owner From 24c42d8bb96fdcf3ea93938e210dd2df1e45e331 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Tue, 10 Nov 2026 17:24:26 +0100 Subject: [PATCH 06/12] docs(privacy): say per surface where scheme-less queries are cut, and the key and @ limits --- docs/privacy.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/privacy.md b/docs/privacy.md index 92c65a31a9..b7876cddda 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -27,9 +27,12 @@ token or signature does not hide the HTTP status or other explanation. The compl fragment are omitted, including values with unescaped spaces. For an unknown URL embedded in free text, a secret-named query or fragment still omits the rest of that text: those later words might be part of the credential, not an explanation. The same rule applies to copied -history errors, saved run messages and notebook cell output, and to a secret in the query or -fragment of a word of scheme-less text (`data.tif?token=abc def`, `a.tif?x=1;token=abc def`), -which goes with the rest of the text, saying `[query and the rest omitted]`. For an unknown +history errors, saved run messages and notebook cell output. Scheme-less text is cut only in +copied history, replay parameters and the project log's saved errors: a secret in the query or +fragment of one of its words (`data.tif?token=abc def`, `a.tif?x=1;token=abc def`) goes with the +rest of the text, saying `[query and the rest omitted]`. A `key`, `sig` or `sas` counts only when +its value reads as a key, so `a.tif?key=abc def` stays. Notices and notebook output cut only +`scheme://` addresses and keep scheme-less text as written. For an unknown `scheme://` URL, a query with no secret name (`?custom=hidden unescaped hidden`) is omitted only to its first space: the redactor can't tell a value's later words from prose, so they are kept. Ambiguous slash-containing userinfo is omitted rather than guessed to be a host, port and path. @@ -42,7 +45,8 @@ space in it, so the rest of the text is omitted too, saying `[address and the re When `text` could be a port (one to five digits: `https://host:8080 user@example.org`, `https://alice:12 word@host`) only the next word counts, since a port and a short password look alike and prose often follows a host and port; otherwise any of the next four words, so a -password with more spaces than that, or with a `/` in it, is not found this way. A path +password with more spaces than that, or with a `/` or an `@` in it (`https://alice:pa@ss word@host` +reads as user information followed by an email), is not found this way. A path (`https://host:8080/x user@example.org`) ends the ambiguity, and the words are kept. A root address is shown with `/` before an omission marker or reason, so repeated redaction keeps its host. A message that names a known address says when its query or From 7bcf17b74a4252fccdeb76c147c4634e4bd8aefd Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Tue, 10 Nov 2026 19:33:29 +0100 Subject: [PATCH 07/12] fix(history): judge a query secret by its own name, so data.tif?pwd=... is emptied --- crates/engine/src/cmd/history.rs | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/engine/src/cmd/history.rs b/crates/engine/src/cmd/history.rs index 1965947f9d..ef36af26d3 100644 --- a/crates/engine/src/cmd/history.rs +++ b/crates/engine/src/cmd/history.rs @@ -324,7 +324,9 @@ fn secret_pairs_emptied(t: &str) -> String { }); let secret = |&(k, at): &(&str, usize)| { let value = rest.get(at..).unwrap_or(""); - secret_pair(k, value.get(..value.find(parts).unwrap_or(value.len())).unwrap_or("")) + // The name after the last `?`, `&` or `#`: a word's path before it is no part of the name. + let name = k.rsplit(['?', '&', '#']).next().unwrap_or(k); + secret_pair(name, value.get(..value.find(parts).unwrap_or(value.len())).unwrap_or("")) }; let Some((_, at)) = pair.filter(secret) else { // Not a secret: the item and the separator after it, as they are. @@ -524,4 +526,32 @@ mod tests { assert_eq!(redacted_parameter("host=h password=abc dbname=g"), "host=h password= dbname=g"); assert_eq!(redacted_parameter("password=a b"), "password= b"); } + + /// A secret in the query of one word is judged by its own name, not by the whole word: an exact + /// name (`pwd`, a key-shaped `sig`) is no less a secret behind a path, and a credential word in + /// a folder name is no secret query. + #[test] + fn a_secret_in_the_query_of_one_word_is_emptied_by_its_own_name() { + for (text, kept) in [ + ("data.tif?pwd=hunter2", "data.tif?pwd="), + ("data.tif?sig=Ab3dE9fG7hJ2kL5mN8pQ4rS6tU1vW0xY", "data.tif?sig="), + ("data.tif?token=abc", "data.tif?token="), + ("a.tif?x=1&pwd=hunter2", "a.tif?x=1&pwd="), + ("data.tif?v=2", "data.tif?v=2"), + ("/data/tokens/a.tif?v=2", "/data/tokens/a.tif?v=2"), + ("data.tif?sig=abc", "data.tif?sig=abc"), + ] { + assert_eq!(redacted_parameter(text), kept, "{text}"); + assert_eq!(redacted_text(text), kept, "{text}"); + } + } + + /// Copy as code and the project log pass every string of a parameter through the same cut. + #[test] + fn text_parameters_of_copied_code_get_the_schemeless_cut() { + let params = serde_json::json!({"text": "data.tif?token=abc def", "n": 1}); + assert_eq!(redacted(¶ms), serde_json::json!({"text": "data.tif[query and the rest omitted]", "n": 1})); + let clean = serde_json::json!({"text": "data.tif?v=2 def", "n": 1}); + assert_eq!(redacted(&clean), clean); + } } From 48dae37e5428d115f2243285dce42919453b7cc9 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Tue, 10 Nov 2026 19:33:29 +0100 Subject: [PATCH 08/12] docs(privacy): name the surfaces of the scheme-less cut and the single-word case --- docs/privacy.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/privacy.md b/docs/privacy.md index b7876cddda..f4f2483e7d 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -28,11 +28,13 @@ fragment are omitted, including values with unescaped spaces. For an unknown URL free text, a secret-named query or fragment still omits the rest of that text: those later words might be part of the credential, not an explanation. The same rule applies to copied history errors, saved run messages and notebook cell output. Scheme-less text is cut only in -copied history, replay parameters and the project log's saved errors: a secret in the query or -fragment of one of its words (`data.tif?token=abc def`, `a.tif?x=1;token=abc def`) goes with the -rest of the text, saying `[query and the rest omitted]`. A `key`, `sig` or `sas` counts only when -its value reads as a key, so `a.tif?key=abc def` stays. Notices and notebook output cut only -`scheme://` addresses and keep scheme-less text as written. For an unknown +copied history errors, the parameters of copied code, and the project log's saved parameters and +errors: a secret in the query or fragment of one of its words (`data.tif?token=abc def`, +`a.tif?x=1;token=abc def`) goes with the rest of the text, saying +`[query and the rest omitted]`. A word alone (`data.tif?pwd=hunter2`) has only the secret's +value emptied, with no marker. A `key`, `sig` or `sas` counts only when its value reads as a key, +so `a.tif?key=abc def` stays. Notices and notebook output cut only `scheme://` addresses and +keep scheme-less text as written. For an unknown `scheme://` URL, a query with no secret name (`?custom=hidden unescaped hidden`) is omitted only to its first space: the redactor can't tell a value's later words from prose, so they are kept. Ambiguous slash-containing userinfo is omitted rather than guessed to be a host, port and path. From ffdec4909b3b173fc9bc66c169606e29888fe04e Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Tue, 10 Nov 2026 20:30:57 +0100 Subject: [PATCH 09/12] fix(history): judge a field name whole unless its word has a query --- crates/engine/src/cmd/history.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/cmd/history.rs b/crates/engine/src/cmd/history.rs index ef36af26d3..aded61f453 100644 --- a/crates/engine/src/cmd/history.rs +++ b/crates/engine/src/cmd/history.rs @@ -324,8 +324,9 @@ fn secret_pairs_emptied(t: &str) -> String { }); let secret = |&(k, at): &(&str, usize)| { let value = rest.get(at..).unwrap_or(""); - // The name after the last `?`, `&` or `#`: a word's path before it is no part of the name. - let name = k.rsplit(['?', '&', '#']).next().unwrap_or(k); + // In a word with a `?`, the name after the last `?`, `&` or `#`: its path before is no part + // of the name. Without one, a field name is judged whole (`password#1`). + let name = if k.contains('?') { k.rsplit(['?', '&', '#']).next().unwrap_or(k) } else { k }; secret_pair(name, value.get(..value.find(parts).unwrap_or(value.len())).unwrap_or("")) }; let Some((_, at)) = pair.filter(secret) else { @@ -540,15 +541,20 @@ mod tests { ("data.tif?v=2", "data.tif?v=2"), ("/data/tokens/a.tif?v=2", "/data/tokens/a.tif?v=2"), ("data.tif?sig=abc", "data.tif?sig=abc"), + // A field name that holds a `#` or an `&` but no `?` is judged whole. + ("password#1=hunter2", "password#1="), + ("token#x=abc", "token#x="), + ("password#1=hunter2 failed", "password#1= failed"), ] { assert_eq!(redacted_parameter(text), kept, "{text}"); assert_eq!(redacted_text(text), kept, "{text}"); } } - /// Copy as code and the project log pass every string of a parameter through the same cut. + /// The project log passes every string of a parameter through the same cut. Copy as code does + /// not: it keeps opaque text and says so (see `durable::output`). #[test] - fn text_parameters_of_copied_code_get_the_schemeless_cut() { + fn text_parameters_of_the_project_log_get_the_schemeless_cut() { let params = serde_json::json!({"text": "data.tif?token=abc def", "n": 1}); assert_eq!(redacted(¶ms), serde_json::json!({"text": "data.tif[query and the rest omitted]", "n": 1})); let clean = serde_json::json!({"text": "data.tif?v=2 def", "n": 1}); From a153f4ef34b25f39c5130772cdb63917559cda98 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Tue, 10 Nov 2026 20:30:57 +0100 Subject: [PATCH 10/12] docs(privacy): Copy as code keeps opaque text; the cut covers copied errors and the log --- docs/privacy.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/privacy.md b/docs/privacy.md index f4f2483e7d..3982c1afb4 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -28,8 +28,9 @@ fragment are omitted, including values with unescaped spaces. For an unknown URL free text, a secret-named query or fragment still omits the rest of that text: those later words might be part of the credential, not an explanation. The same rule applies to copied history errors, saved run messages and notebook cell output. Scheme-less text is cut only in -copied history errors, the parameters of copied code, and the project log's saved parameters and -errors: a secret in the query or fragment of one of its words (`data.tif?token=abc def`, +copied history errors and the project log's saved parameters and errors (Copy as code keeps opaque +text parameters and says so, as below): +a secret in the query or fragment of one of its words (`data.tif?token=abc def`, `a.tif?x=1;token=abc def`) goes with the rest of the text, saying `[query and the rest omitted]`. A word alone (`data.tif?pwd=hunter2`) has only the secret's value emptied, with no marker. A `key`, `sig` or `sas` counts only when its value reads as a key, From 933313767deb5f4f5ab131d3291dc5f3374ed3aa Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Wed, 11 Nov 2026 15:26:15 +0100 Subject: [PATCH 11/12] fix(history): a word without a query is secret by its whole name or its tail --- crates/engine/src/cmd/history.rs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/cmd/history.rs b/crates/engine/src/cmd/history.rs index aded61f453..44326a4a54 100644 --- a/crates/engine/src/cmd/history.rs +++ b/crates/engine/src/cmd/history.rs @@ -325,9 +325,11 @@ fn secret_pairs_emptied(t: &str) -> String { let secret = |&(k, at): &(&str, usize)| { let value = rest.get(at..).unwrap_or(""); // In a word with a `?`, the name after the last `?`, `&` or `#`: its path before is no part - // of the name. Without one, a field name is judged whole (`password#1`). - let name = if k.contains('?') { k.rsplit(['?', '&', '#']).next().unwrap_or(k) } else { k }; - secret_pair(name, value.get(..value.find(parts).unwrap_or(value.len())).unwrap_or("")) + // of the name. Without one, the whole field name (`password#1`) or the part after its last + // `&` or `#` (`data.tif#pwd`) makes it a secret. + let tail = k.rsplit(['?', '&', '#']).next().unwrap_or(k); + let value = value.get(..value.find(parts).unwrap_or(value.len())).unwrap_or(""); + if k.contains('?') { secret_pair(tail, value) } else { secret_pair(k, value) || secret_pair(tail, value) } }; let Some((_, at)) = pair.filter(secret) else { // Not a secret: the item and the separator after it, as they are. @@ -545,6 +547,9 @@ mod tests { ("password#1=hunter2", "password#1="), ("token#x=abc", "token#x="), ("password#1=hunter2 failed", "password#1= failed"), + // A word with no `?` is secret by its whole name or by the part after its last `&` or `#`. + ("data.tif#pwd=hunter2", "data.tif#pwd="), + ("data.tif#v=2", "data.tif#v=2"), ] { assert_eq!(redacted_parameter(text), kept, "{text}"); assert_eq!(redacted_text(text), kept, "{text}"); @@ -560,4 +565,17 @@ mod tests { let clean = serde_json::json!({"text": "data.tif?v=2 def", "n": 1}); assert_eq!(redacted(&clean), clean); } + + /// A scheme-less secret query in text that also holds a `://` address goes through `redact`'s + /// field rule: the secret and the words after it are gone. + #[test] + fn a_schemeless_secret_beside_an_address_is_cut_by_the_field_rule() { + let secret = "see https://example.org/ok and data.tif?token=abc def"; + let clean = "see https://example.org/ok and data.tif?v=2 def"; + for cut in [redacted_text(secret), redacted_parameter(secret)] { + assert_eq!(cut, "see https://example.org/ok and data.tif?[REDACTED]"); + } + assert_eq!(redacted_text(clean), clean); + assert_eq!(redacted_parameter(clean), clean); + } } From 8abef92c8ba6a7cc70778f01783bfacd44bf6ccc Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Wed, 11 Nov 2026 15:26:15 +0100 Subject: [PATCH 12/12] docs(privacy): a fragment secret in a single word is emptied too --- docs/privacy.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/privacy.md b/docs/privacy.md index 3982c1afb4..ae2c290101 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -32,8 +32,8 @@ copied history errors and the project log's saved parameters and errors (Copy as text parameters and says so, as below): a secret in the query or fragment of one of its words (`data.tif?token=abc def`, `a.tif?x=1;token=abc def`) goes with the rest of the text, saying -`[query and the rest omitted]`. A word alone (`data.tif?pwd=hunter2`) has only the secret's -value emptied, with no marker. A `key`, `sig` or `sas` counts only when its value reads as a key, +`[query and the rest omitted]`. A word alone (`data.tif?pwd=hunter2`, `data.tif#pwd=hunter2`) has +only the secret's value emptied, with no marker. A `key`, `sig` or `sas` counts only when its value reads as a key, so `a.tif?key=abc def` stays. Notices and notebook output cut only `scheme://` addresses and keep scheme-less text as written. For an unknown `scheme://` URL, a query with no secret name (`?custom=hidden unescaped hidden`) is omitted only