From 7e3f62dcf6aac0f81e34e6dcf090d5a095cad62d Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 16:53:00 -0300 Subject: [PATCH 1/5] feat(engine, ui): Add From Service's filter searches all of a service's layers (#1577) service.layers takes a filter: words a layer's title or name must all contain, in any case. Every layer read is searched, not only the first 2000 a listing shows; the first 2000 matches are listed, with matched and total. Add From Service, when its listing stops short, runs service.layers again on the documents it keeps, with the filter, once typing pauses for 300 ms: nothing is downloaded, and a layer past position 2000 can be found and added from the dialog. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/ogc.rs | 2 +- crates/engine/src/cmd/service.rs | 51 ++++++++++--- crates/engine/src/tests.rs | 35 +++++++++ crates/ui-egui/src/dialogs.rs | 123 ++++++++++++++++++++++++++++--- crates/ui-egui/src/lib.rs | 19 +++++ crates/ui-egui/tests/smoke.rs | 24 +++++- docs/user-guide/data-formats.md | 5 ++ 7 files changed, 236 insertions(+), 23 deletions(-) diff --git a/crates/data/src/ogc.rs b/crates/data/src/ogc.rs index 31670fb4d2..e7857fcdb1 100644 --- a/crates/data/src/ogc.rs +++ b/crates/data/src/ogc.rs @@ -469,7 +469,7 @@ pub struct Service { /// between answers (#1519). Reading stops at [`MAX_READ_LAYERS`], or sooner when the layers' /// request addresses are long ([`read_limit`], [`MAX_COPIED_ADDRESS_BYTES`]); past that none is /// read. An ArcGIS list is read only as far as it is listed. -const MAX_LISTED_LAYERS: usize = 2_000; +pub const MAX_LISTED_LAYERS: usize = 2_000; /// Most layers or collections read from one document, which is hostile input. const MAX_READ_LAYERS: usize = if cfg!(test) { 3_000 } else { 100_000 }; /// Most bytes of a document's own request addresses (its GetMap's, GetFeature's…) copied into diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index d6dc3a6e62..acdbbbe150 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -31,7 +31,7 @@ const DOCUMENT_ROLES: &[crate::ParamRole] = &[ pub fn commands() -> Vec { let mut commands = vec![ C::new("service.layers", "Service Layers", "Home › Add", layers).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Network)).view().params( - r#"{"url": "a WMS, WMTS or WFS address (GetCapabilities is added), an OGC API collections address or landing page (its map tiles and linked collections), or an ArcGIS FeatureServer, MapServer, ImageServer or VectorTileServer, or an ArcGIS services directory or folder (its folders and services, each listed by its own url)", "type"?: "wms"|"wmts"|"wfs"|"ogcapi"|"arcgis" (guessed from the address), "text"?: "the capabilities document, when already fetched", "documents"?: [{"url": "a discovered document address", "text": "its fetched JSON"}], "connection"?: "the saved web-service connection id whose login the requests use"} → layers with source_url, and next when supplied documents need another fetch"#, + r#"{"url": "a WMS, WMTS or WFS address (GetCapabilities is added), an OGC API collections address or landing page (its map tiles and linked collections), or an ArcGIS FeatureServer, MapServer, ImageServer or VectorTileServer, or an ArcGIS services directory or folder (its folders and services, each listed by its own url)", "type"?: "wms"|"wmts"|"wfs"|"ogcapi"|"arcgis" (guessed from the address), "text"?: "the capabilities document, when already fetched", "documents"?: [{"url": "a discovered document address", "text": "its fetched JSON"}], "connection"?: "the saved web-service connection id whose login the requests use", "filter"?: "words a layer's title or name must all contain, in any case: every layer read is searched, not only those a listing shows, and the first 2000 that match are listed"} → layers with source_url (with filter, total: the layers read, and matched: how many match), and next when supplied documents need another fetch"#, ), C::new("service.add", "Add Service Layer", "Home › Add", add).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Network)).params( r#"{"url": "as service.layers", "type"?, "layer": "the layer's name (service.layers lists them)", "name"?: text, "time"?: "a time-enabled WMS or WMTS layer: by default (or \"slider\", true) it follows the time slider, turned on with the layer's times when it is off; or one of its dates (2024-06-01), or \"default\" (false): its service's default", "text"?: "the capabilities document, when already fetched", "documents"?: "additional fetched documents, as service.layers", "source_url"?: "the listed layer's source_url", "describe"?: "a WCS coverage's DescribeCoverage document, when already fetched", "tiles"?: "an OGC API layer's tile template, when already read from its tileset (the app's)", "bbox"?: [west, south, east, north] | "view" | "full": a WCS coverage's area (the view by default); for a WFS, OGC API or ArcGIS feature layer, only the features that cross it (the whole layer, "full", by default), "connection"?: "the id of the saved web-service connection it comes from (connection.browse gives it): the layer records it, and is read with its login"}"#, @@ -108,6 +108,8 @@ struct Discovery { excluded_layers: bool, /// A document left collections out (past its listing limit, or unreadable items links). incomplete: bool, + /// How many layers were read, before a filter kept those matching it ([`discover_matching`]). + read: usize, } impl Discovery { @@ -153,7 +155,24 @@ fn capabilities(s: &mut Session, p: &Value) -> Result { /// off the frame ([`Session::left_document`]), when a document not in yet is its `next`. Long /// document addresses list fewer layers ([`MAX_LISTED_ADDRESS_BYTES`]), said in a note. fn discover(s: &mut Session, p: &Value, left: bool) -> Result { + discover_matching(s, p, left, &[]) +} + +/// [`discover`], keeping only the layers whose title or name contains each of `words` (lower +/// case), when there are any: every layer read is searched, and the first +/// [`rustgis_data::ogc::MAX_LISTED_LAYERS`] matches are listed (#1577). +fn discover_matching(s: &mut Session, p: &Value, left: bool, words: &[String]) -> Result { let mut found = read_documents(s, p, left)?; + found.read = found.service.layers.len(); + if !words.is_empty() { + let keep: Vec = found.service.layers.iter().map(|l| matches_words(l, words)).collect(); + let mut kept = keep.iter(); + found.service.layers.retain(|_| kept.next().copied().unwrap_or(false)); + let mut kept = keep.iter(); + found.sources.retain(|_| kept.next().copied().unwrap_or(false)); + let max = rustgis_data::ogc::MAX_LISTED_LAYERS; + found.service.listed = (found.service.layers.len() > max).then_some(max); + } let listed = found.service.listed.unwrap_or(usize::MAX).min(found.sources.len()); let mut bytes = 0usize; let cut = found.sources[..listed].iter().position(|source| { @@ -173,6 +192,13 @@ fn discover(s: &mut Session, p: &Value, left: bool) -> Result Ok(found) } +/// Whether a layer's title or name contains each of `words` (lower case): Add From Service's +/// filter, which matched the listed layers in the dialog before #1577. +fn matches_words(layer: &rustgis_data::ogc::ServiceLayer, words: &[String]) -> bool { + let (title, name) = (layer.title.to_lowercase(), layer.name.to_lowercase()); + words.iter().all(|w| title.contains(w.as_str()) || name.contains(w.as_str())) +} + fn read_documents(s: &mut Session, p: &Value, left: bool) -> Result { let original = url_of(p)?; let mut address = @@ -678,12 +704,15 @@ pub(crate) fn layers(s: &mut Session, p: &Value) -> CmdResult { let p = owned.as_ref().unwrap_or(p); let bound = crate::service_logins::request_params(s, p)?; let p = bound.as_ref(); - let mut found = capabilities(s, p)?; + let words: Vec = ps(p, "filter").map(|f| f.split_whitespace().map(str::to_lowercase).collect()).unwrap_or_default(); + let mut found = discover_matching(s, p, false, &words)?; notify_withheld(s, found.service.withheld_addresses); - let of = layer_count(&found.service); - let total = found.service.layers.len(); + let searched = !words.is_empty(); + // The layers read, or with a filter those matching it. + let found_layers = found.service.layers.len(); + let of = if searched { format!("{found_layers} matching layers") } else { layer_count(&found.service) }; let mut cut = None; - if let Some(listed) = found.service.listed.filter(|&listed| listed < total) { + if let Some(listed) = found.service.listed.filter(|&listed| listed < found_layers) { found.service.layers.truncate(listed); let note = format!("only the first {listed} of {of} are listed; service.add takes any of them by name"); found.service.notes.push(note.clone()); @@ -691,11 +720,15 @@ pub(crate) fn layers(s: &mut Session, p: &Value) -> CmdResult { } let mut out = service_json(&found.service); // How many there are, for a window that says it its own way: `listedNote` says the list - // stops (Add From Service: its filter searches only those listed), and `atLeast` that the - // document has more than were read. + // stops (Add From Service searches past it with `filter`), and `atLeast` that the document + // has more than were read. let past_limit = found.service.past_read_limit.is_some(); - if cut.is_some() || past_limit { - out["total"] = json!(total); + if cut.is_some() || past_limit || searched { + out["total"] = json!(found.read); + } + // (With a filter, `total` counts every layer read, and `matched` those matching it.) + if searched { + out["matched"] = json!(found_layers); } if past_limit { out["atLeast"] = json!(true); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 9cda16ccd4..7f9f0b66b5 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -16282,6 +16282,41 @@ fn a_long_document_address_lists_fewer_layers() { assert!(s.run("service.groupPlan", &json!({"url": "https://api.example.org/collections", "text": text})).is_ok()); } +/// `service.layers` with `filter` searches every layer read, not only the first 2000 it lists, and +/// counts the matches (#1577: IBGE's WFS has 10,186 types, in an order that changes). +#[test] +fn a_filter_searches_past_the_listing() { + let types: String = (1..=2500) + .map(|i| { + let title = if (2101..=2103).contains(&i) { format!("Solos do Brasil {i}") } else { format!("Tipo {i}") }; + format!("t:{i}{title}") + }) + .collect(); + let caps = format!(r#"{types}"#); + let url = "https://wfs.example.org/wfs"; + let mut s = Session::new(Project::default()); + let names = |v: &Value| v["layers"].as_array().unwrap().iter().map(|l| l["name"].as_str().unwrap().to_string()).collect::>(); + // Without a filter, as before: the first 2000, and no count of matches. + let all = s.run("service.layers", &json!({"url": url, "text": caps})).unwrap(); + assert_eq!((&all["count"], &all["total"]), (&json!(2000), &json!(2500))); + assert!(all.get("matched").is_none()); + // Words in the title or the name, in any case, all of them: past the listing too. + let found = s.run("service.layers", &json!({"url": url, "text": caps, "filter": " BRASIL solos "})).unwrap(); + assert_eq!(names(&found), ["t:2101", "t:2102", "t:2103"]); + assert_eq!((&found["count"], &found["matched"], &found["total"]), (&json!(3), &json!(3), &json!(2500))); + assert!(found.get("listedNote").is_none() && found["notes"].as_array().unwrap().is_empty(), "{found}"); + assert_eq!(found["layers"][0]["source_url"], all["layers"][0]["source_url"]); + assert_eq!(names(&s.run("service.layers", &json!({"url": url, "text": caps, "filter": "t:2500"})).unwrap()), ["t:2500"]); + let none = s.run("service.layers", &json!({"url": url, "text": caps, "filter": "solos nada"})).unwrap(); + assert_eq!((&none["count"], &none["matched"]), (&json!(0), &json!(0))); + // More matches than a listing holds: the first 2000, said, with how many match. + let many = s.run("service.layers", &json!({"url": url, "text": caps, "filter": "tipo"})).unwrap(); + assert_eq!((&many["count"], &many["matched"], &many["total"]), (&json!(2000), &json!(2497), &json!(2500))); + assert_eq!(many["listedNote"], "only the first 2000 of 2497 matching layers are listed; service.add takes any of them by name"); + // A blank filter is none. + assert!(s.run("service.layers", &json!({"url": url, "text": caps, "filter": " "})).unwrap().get("matched").is_none()); +} + /// A layer past what RustGIS reads of a document is refused for that reason, not as absent. #[test] fn a_layer_past_the_read_limit_is_refused_saying_so() { diff --git a/crates/ui-egui/src/dialogs.rs b/crates/ui-egui/src/dialogs.rs index b68a30a602..3c61e8e488 100644 --- a/crates/ui-egui/src/dialogs.rs +++ b/crates/ui-egui/src/dialogs.rs @@ -852,6 +852,61 @@ fn go_to(app: &mut RustGisApp, url: &str, kind: &str, way: Vec<(String, &'static /// How many of a service's layers Add From Service lists. const SHOWN_LAYERS: usize = 500; +/// How long typing pauses before Add From Service searches all of a listing that stops short: +/// reading the documents again takes a moment for a large one (IBGE's WFS, 12 MB), so not on +/// every key (#1577). +pub(crate) const SEARCH_PAUSE: web_time::Duration = web_time::Duration::from_millis(300); + +/// A filter's words as `service.layers` takes them: lower case, one space apart. +fn filter_key(filter: &str) -> String { + filter.split_whitespace().map(str::to_lowercase).collect::>().join(" ") +} + +/// The search of all of a service's layers with the filter, when its listing stops short of them +/// (`listedNote`) and typing has paused for [`SEARCH_PAUSE`]: `service.layers` again on the +/// documents the dialog keeps, with `filter`, so nothing is downloaded. A listing that holds them +/// all is filtered in the dialog, as before. Gives how long until a search is due, to draw again +/// then. One read of the documents on the UI thread per pause in typing, as for each document +/// that arrives: the parsed layers aren't kept, which would hold up to 100,000 of them. +fn search_all(app: &mut RustGisApp) -> Option { + let conn = app.ui.service.as_mut()?; + let key = filter_key(&app.ui.service_browse.filter); + let short = conn.listing.as_ref().is_some_and(|v| v.get("listedNote").is_some()); + if !short || key.is_empty() || conn.pending.is_some() || conn.search.as_ref().is_some_and(|s| s.filter == key) { + return None; + } + let now = web_time::Instant::now(); + match &conn.typed { + Some((typed, at)) if *typed == key => { + let waited = now.duration_since(*at); + if waited < SEARCH_PAUSE { + return Some(SEARCH_PAUSE - waited); + } + } + _ => { + conn.typed = Some((key, now)); + return Some(SEARCH_PAUSE); + } + } + let mut p = json!({"url": conn.url, "text": conn.text, "documents": conn.documents, "filter": key}); + if conn.kind != "auto" { + p["type"] = json!(conn.kind); + } + if let Some(connection) = &conn.connection { + p["connection"] = json!(connection); + } + // Not `app.run`: an error belongs in the dialog, as the listing's does. + let listing = app.session.run("service.layers", &p); + conn.search = Some(crate::ServiceSearch { filter: key, listing }); + conn.added = None; + None +} + +/// The search's answer for the filter as typed now, when there is one and it holds layers. +fn searched<'a>(conn: &'a crate::ServiceConn, filter: &str) -> Option<&'a Value> { + conn.search.as_ref().filter(|s| s.filter == filter_key(filter)).and_then(|s| s.listing.as_ref().ok()) +} + /// The listed layers matching `filter` (words their title or name contains), with their place in /// the listing. fn matching_rows<'a>(layers: &'a [Value], filter: &str) -> Vec<(usize, &'a Value)> { @@ -892,13 +947,24 @@ fn refresh_added(app: &mut RustGisApp) { if conn.added.as_ref().is_some_and(|a| a.revision == revision && a.map == map && a.filter == *filter) { return; } - let layers: &[Value] = conn.listing.as_ref().and_then(|v| v.get("layers")).and_then(Value::as_array).map_or(&[], Vec::as_slice); - let matching = matching_rows(layers, filter); - let matched = matching.len(); - let shown: Vec = matching.into_iter().take(SHOWN_LAYERS).map(|(i, _)| i).collect(); + // Every layer matching the filter, when all of a listing that stops short were searched: its + // answer holds only matches (the first 2000), and counts them all. + let search = searched(conn, filter); + let was_searched = search.is_some(); + let listing = search.or(conn.listing.as_ref()); + let layers: &[Value] = listing.and_then(|v| v.get("layers")).and_then(Value::as_array).map_or(&[], Vec::as_slice); + let (matched, shown): (usize, Vec) = match search { + Some(v) => { + (v.get("matched").and_then(Value::as_u64).map_or(layers.len(), |m| m as usize), (0..layers.len().min(SHOWN_LAYERS)).collect()) + } + None => { + let matching = matching_rows(layers, filter); + (matching.len(), matching.into_iter().take(SHOWN_LAYERS).map(|(i, _)| i).collect()) + } + }; let removals = rustgis_engine::cmd::service_added::added(app.session.map(), layers, &shown); let rows = shown.into_iter().map(|i| (i, removals.get(i).cloned().flatten())).collect(); - conn.added = Some(crate::ServiceAdded { revision, map, filter: filter.clone(), matched, rows }); + conn.added = Some(crate::ServiceAdded { revision, map, filter: filter.clone(), matched, searched: was_searched, rows }); } /// Add From Service: a WMS, WMTS, WFS, WCS, OGC API Features or ArcGIS REST service by its address @@ -992,6 +1058,9 @@ fn add_service(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { crate::connectionui::picked(&list, app.dialogs.connection.as_deref(), url).is_none().then(|| (url.to_string(), kind)) }); let mut keep_root = false; + if let Some(due) = search_all(app) { + ui.ctx().request_repaint_after(due); + } refresh_added(app); if let Some(conn) = &app.ui.service { ui.separator(); @@ -1038,15 +1107,30 @@ fn add_service(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { ui.label(egui::RichText::new(n).small().weak()); } } + // The filter searches all of them, once typing pauses (`search_all`). if listed_note.is_some() { - ui.label( - egui::RichText::new(crate::i18n::fmt( - "The filter searches the first {n} of them; the service.add command adds any other by its name.", + let key = filter_key(&app.ui.service_browse.filter); + let line = if key.is_empty() { + Some(crate::i18n::fmt( + "The first {n} of them are listed; the filter searches all of them.", &[("n", &layers.len().to_string())], )) - .small() - .weak(), - ); + } else { + match conn.search.as_ref().filter(|s| s.filter == key).map(|s| &s.listing) { + None => { + let total = v.get("total").and_then(Value::as_u64).map_or(layers.len().to_string(), |n| n.to_string()); + Some(crate::i18n::fmt("Searching all {total} layers…", &[("total", &total)])) + } + Some(Err(e)) => { + ui.colored_label(t.red, e); + None + } + Some(Ok(_)) => None, + } + }; + if let Some(line) = line { + ui.label(egui::RichText::new(line).small().weak()); + } } // A long list is filtered by words; a directory's services lead back to it. let browse = &mut app.ui.service_browse; @@ -1111,6 +1195,13 @@ fn add_service(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { } let added = conn.added.as_ref(); let rows: &[(usize, Option)] = added.map_or(&[], |a| a.rows.as_slice()); + // A search's rows are places in its answer, which holds only matches. + let row_layers = match added.filter(|a| a.searched) { + Some(_) => { + searched(conn, &app.ui.service_browse.filter).and_then(|v| v.get("layers")).and_then(Value::as_array).unwrap_or(&none) + } + None => layers, + }; if let Some(a) = added.filter(|a| a.matched > SHOWN_LAYERS) { ui.label( egui::RichText::new(crate::i18n::fmt( @@ -1134,7 +1225,7 @@ fn add_service(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { egui::ScrollArea::vertical().id_salt("add_service_layers_scroll").auto_shrink([false, false]).show(ui, |ui| { let style = ui.style().clone(); for (i, removal) in rows { - let Some(l) = layers.get(*i) else { continue }; + let Some(l) = row_layers.get(*i) else { continue }; ui.horizontal(|ui| { ui.add_space(2.0); let kind = l.get("offer").and_then(|o| o.get("type")).and_then(Value::as_str); @@ -2487,6 +2578,8 @@ mod tests { }], })), added: None, + typed: None, + search: None, error: None, failure: None, }); @@ -2538,6 +2631,8 @@ mod tests { }], })), added: None, + typed: None, + search: None, error: None, failure: None, }); @@ -2599,6 +2694,8 @@ mod tests { }], })), added: None, + typed: None, + search: None, error: None, failure: None, }); @@ -2653,6 +2750,8 @@ mod tests { }], })), added: None, + typed: None, + search: None, error: None, failure: None, }); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index bca74bc326..d20d34fd00 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -427,6 +427,20 @@ pub struct ServiceConn { pub error: Option, /// A capabilities request that failed, with what the server said (#509). pub failure: Option, + /// The filter as last typed, and when: a listing that stops short is searched whole once + /// typing pauses (`dialogs::SEARCH_PAUSE`), not on every key (#1577). + pub typed: Option<(String, web_time::Instant)>, + /// `service.layers` run again with the filter, over every layer read of a listing that stops + /// short. + pub search: Option, +} + +/// A search of all of a service's layers (`service.layers` with `filter`), for Add From Service. +pub struct ServiceSearch { + /// The filter it was run with. + pub filter: String, + /// Its answer (the first matches, with `matched`), or why it failed. + pub listing: Result, } /// Add From Service's rows: the layers of a listing that match the filter, and for each the @@ -441,6 +455,8 @@ pub struct ServiceAdded { pub filter: String, /// How many layers match the filter (the first `dialogs::SHOWN_LAYERS` are shown). pub matched: usize, + /// The rows are places in the search's answer (`ServiceConn::search`), not in the listing. + pub searched: bool, /// Each shown row: its place in the listing, and the command that removes it when it is on /// the map (`None`: the row says Add). pub rows: Vec<(usize, Option)>, @@ -3315,6 +3331,8 @@ impl RustGisApp { added: None, error: None, failure: None, + typed: None, + search: None, }); } "ui.importShapefile" => { @@ -5017,6 +5035,7 @@ impl RustGisApp { } conn.listing = Some(v); conn.added = None; + conn.search = None; } Err(e) => conn.error = Some(e), } diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index 2664f060a8..c113710d30 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -2479,13 +2479,35 @@ Connection: close h.step(); h.step(); h.get_by_label_contains("Local: 2001 layers"); - h.get_by_label("The filter searches the first 2000 of them; the service.add command adds any other by its name."); + h.get_by_label("The first 2000 of them are listed; the filter searches all of them."); h.get_by_label("500 of 2000 are listed."); assert!(h.query_by_label_contains("only the first 2000").is_none(), "the note isn't said twice"); h.state_mut().ui.service_browse.filter = "t:20".into(); h.step(); h.step(); assert!(h.query_by_label_contains("are listed.").is_none(), "its 12 matches are all shown"); + // A layer past the first 2000 is found once typing pauses: all of them are searched (#1577). + h.state_mut().ui.service_browse.filter = "T:2001".into(); + h.step(); + h.step(); + h.get_by_label("Searching all 2001 layers…"); + assert!(h.query_by_label_contains("t:2001").is_none(), "not among the 2000 listed"); + std::thread::sleep(std::time::Duration::from_millis(350)); + h.step(); + h.step(); + assert!(h.query_by_label_contains("Searching all").is_none()); + h.get_by_label_contains("t:2001"); + let conn = h.state().ui.service.as_ref().unwrap(); + let a = conn.added.as_ref().unwrap(); + assert!(a.searched && a.matched == 1 && a.rows.len() == 1); + // Its row adds it from the search's answer, as a listed row would. + let row = &conn.search.as_ref().unwrap().listing.as_ref().unwrap()["layers"][a.rows[0].0]; + assert_eq!(row["name"], "t:2001"); + // Cleared, the listing is filtered here again. + h.state_mut().ui.service_browse.filter.clear(); + h.step(); + h.step(); + h.get_by_label("The first 2000 of them are listed; the filter searches all of them."); } #[test] diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index cc75685e93..711b38def0 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -327,6 +327,11 @@ list each repeating a somewhat shorter one. An ArcGIS service or directory is an A dynamic MapServer offers each of those layers twice, as features and as a map, so its list can be twice as long. A note says how many are left out past either limit. +Add From Service's filter searches all of a service's layers, not only those listed: once +typing pauses, the documents it already has are read again for the layers whose title or name +contains each word (`service.layers` with `"filter"`), and the first 2,000 matches are listed, +with how many match. + A WMS or WMTS layer's tiles are asked for down to the detail its service gives, and stretched only past it: a WMTS layer's over the zooms of its tile matrix set (or those its `TileMatrixSetLimits` name; swisstopo's SWISSIMAGE goes to zoom 20), a WMS layer's between the From 1de3f986a448c4cdac83e3b46e4c660659dce701 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 21:26:20 -0300 Subject: [PATCH 2/5] fix(ui): show a search's own notes, and say what the filter can't find (#1577) - Add From Service shows the search answer's notes beside the listing's, so more than 2000 matches says "only the first 2000 of N matching layers are listed", as does a search cut by its addresses' length. - search_all's doc says why it stays on the frame: service.layers binds a saved connection's login in this session. - The user guide says the filter searches every layer read: layers past the read limits, and an ArcGIS service's past its first 2000, aren't. - Stale comments that said the filter searched only those listed. Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/src/dialogs.rs | 22 +++++++++++++++++++--- crates/ui-egui/tests/smoke.rs | 11 ++++++++++- docs/user-guide/data-formats.md | 10 ++++++---- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/crates/ui-egui/src/dialogs.rs b/crates/ui-egui/src/dialogs.rs index 3c61e8e488..98da820cc0 100644 --- a/crates/ui-egui/src/dialogs.rs +++ b/crates/ui-egui/src/dialogs.rs @@ -867,7 +867,12 @@ fn filter_key(filter: &str) -> String { /// documents the dialog keeps, with `filter`, so nothing is downloaded. A listing that holds them /// all is filtered in the dialog, as before. Gives how long until a search is due, to draw again /// then. One read of the documents on the UI thread per pause in typing, as for each document -/// that arrives: the parsed layers aren't kept, which would hold up to 100,000 of them. +/// that arrives (`poll_service`): the parsed layers aren't kept, which would hold up to 100,000 +/// of them. It stays on the frame because `service.layers` binds a saved connection's login to +/// the addresses its rows carry, in this session: a session of its own off the frame would leave +/// that out, and adding a row would go without it. Nothing is fetched (the documents are +/// kept, and no search starts while one is `pending`); IBGE's WFS, 12 MB and 10,186 layers, +/// takes about 0.3 s in a dev build. fn search_all(app: &mut RustGisApp) -> Option { let conn = app.ui.service.as_mut()?; let key = filter_key(&app.ui.service_browse.filter); @@ -1087,7 +1092,7 @@ fn add_service(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { let layers = v.get("layers").and_then(Value::as_array).unwrap_or(&none); let grouped = kind.starts_with("OGC API"); // A listing that stops short counts them all, and says so here in the dialog's own - // words instead of its note: the filter searches only those listed. + // words instead of its note: the filter searches all of them (`search_all`). let listed_note = v.get("listedNote").and_then(Value::as_str); let count = crate::connectionui::layer_count(v); ui.horizontal(|ui| { @@ -1125,7 +1130,18 @@ fn add_service(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { ui.colored_label(t.red, e); None } - Some(Ok(_)) => None, + // The search's own notes, beside the listing's: more matches than it + // lists (its `listedNote`), or a list cut by its addresses' length. + Some(Ok(found)) => { + let shown: Vec<&str> = + v.get("notes").and_then(Value::as_array).into_iter().flatten().filter_map(Value::as_str).collect(); + for n in found.get("notes").and_then(Value::as_array).into_iter().flatten().filter_map(Value::as_str) { + if !shown.contains(&n) { + ui.label(egui::RichText::new(n).small().weak()); + } + } + None + } } }; if let Some(line) = line { diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index c113710d30..52e1e587b9 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -2435,7 +2435,7 @@ fn a_service_is_listed_when_its_capabilities_land() { } /// A listing that stops short says in Add From Service how many layers there are, that its filter -/// searches only those listed, and how many of them are shown, in one place and not three. +/// searches all of them, and how many of them are shown, in one place and not three. #[test] fn a_cut_listing_says_what_the_filter_searches() { use std::io::{Read, Write}; @@ -2503,6 +2503,15 @@ Connection: close // Its row adds it from the search's answer, as a listed row would. let row = &conn.search.as_ref().unwrap().listing.as_ref().unwrap()["layers"][a.rows[0].0]; assert_eq!(row["name"], "t:2001"); + // More matches than a search lists: its note says so, not only how many rows are shown. + h.state_mut().ui.service_browse.filter = "t:".into(); + h.step(); + h.step(); + std::thread::sleep(std::time::Duration::from_millis(350)); + h.step(); + h.step(); + h.get_by_label("only the first 2000 of 2001 matching layers are listed; service.add takes any of them by name"); + h.get_by_label("500 of 2001 are listed."); // Cleared, the listing is filtered here again. h.state_mut().ui.service_browse.filter.clear(); h.step(); diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 711b38def0..419c1e8f16 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -327,10 +327,12 @@ list each repeating a somewhat shorter one. An ArcGIS service or directory is an A dynamic MapServer offers each of those layers twice, as features and as a map, so its list can be twice as long. A note says how many are left out past either limit. -Add From Service's filter searches all of a service's layers, not only those listed: once -typing pauses, the documents it already has are read again for the layers whose title or name -contains each word (`service.layers` with `"filter"`), and the first 2,000 matches are listed, -with how many match. +When a list stops short of the layers read, Add From Service's filter searches every layer +read, not only those listed: once typing pauses, the documents it already has are read again +for the layers whose title or name contains each word (`service.layers` with `"filter"`), and +the first 2,000 matches are listed, with how many match and a note when more match. Layers past +the read limits above, and an ArcGIS service's past its first 2,000, aren't read, so the filter +doesn't find them either. A WMS or WMTS layer's tiles are asked for down to the detail its service gives, and stretched only past it: a WMTS layer's over the zooms of its tile matrix set (or those its From db34643dffb7a535f18d23a27d429118dd3183ce Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 21:58:10 -0300 Subject: [PATCH 3/5] fix(engine): a search doesn't repeat the listing's notice, and a filter must be text (#1577) - service.layers with a filter reads again what a listing read (Add From Service searches once per pause in typing), so it no longer notifies the withheld addresses again: its notes still say them. Reproduced: one listing and three searches gave the notice four times. - A filter that isn't text (a list, a number) is refused instead of ignored, which answered with the unfiltered listing. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/service.rs | 14 ++++++++++--- crates/engine/src/tests.rs | 35 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index acdbbbe150..b5bb52b420 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -31,7 +31,7 @@ const DOCUMENT_ROLES: &[crate::ParamRole] = &[ pub fn commands() -> Vec { let mut commands = vec![ C::new("service.layers", "Service Layers", "Home › Add", layers).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Network)).view().params( - r#"{"url": "a WMS, WMTS or WFS address (GetCapabilities is added), an OGC API collections address or landing page (its map tiles and linked collections), or an ArcGIS FeatureServer, MapServer, ImageServer or VectorTileServer, or an ArcGIS services directory or folder (its folders and services, each listed by its own url)", "type"?: "wms"|"wmts"|"wfs"|"ogcapi"|"arcgis" (guessed from the address), "text"?: "the capabilities document, when already fetched", "documents"?: [{"url": "a discovered document address", "text": "its fetched JSON"}], "connection"?: "the saved web-service connection id whose login the requests use", "filter"?: "words a layer's title or name must all contain, in any case: every layer read is searched, not only those a listing shows, and the first 2000 that match are listed"} → layers with source_url (with filter, total: the layers read, and matched: how many match), and next when supplied documents need another fetch"#, + r#"{"url": "a WMS, WMTS or WFS address (GetCapabilities is added), an OGC API collections address or landing page (its map tiles and linked collections), or an ArcGIS FeatureServer, MapServer, ImageServer or VectorTileServer, or an ArcGIS services directory or folder (its folders and services, each listed by its own url)", "type"?: "wms"|"wmts"|"wfs"|"ogcapi"|"arcgis" (guessed from the address), "text"?: "the capabilities document, when already fetched", "documents"?: [{"url": "a discovered document address", "text": "its fetched JSON"}], "connection"?: "the saved web-service connection id whose login the requests use", "filter"?: "words a layer's title or name must all contain, in any case: every layer read is searched, not only those a listing shows, and the first 2000 that match are listed; what a listing says in a notice, a search says only in its notes"} → layers with source_url (with filter, total: the layers read, and matched: how many match), and next when supplied documents need another fetch"#, ), C::new("service.add", "Add Service Layer", "Home › Add", add).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Network)).params( r#"{"url": "as service.layers", "type"?, "layer": "the layer's name (service.layers lists them)", "name"?: text, "time"?: "a time-enabled WMS or WMTS layer: by default (or \"slider\", true) it follows the time slider, turned on with the layer's times when it is off; or one of its dates (2024-06-01), or \"default\" (false): its service's default", "text"?: "the capabilities document, when already fetched", "documents"?: "additional fetched documents, as service.layers", "source_url"?: "the listed layer's source_url", "describe"?: "a WCS coverage's DescribeCoverage document, when already fetched", "tiles"?: "an OGC API layer's tile template, when already read from its tileset (the app's)", "bbox"?: [west, south, east, north] | "view" | "full": a WCS coverage's area (the view by default); for a WFS, OGC API or ArcGIS feature layer, only the features that cross it (the whole layer, "full", by default), "connection"?: "the id of the saved web-service connection it comes from (connection.browse gives it): the layer records it, and is read with its login"}"#, @@ -704,9 +704,17 @@ pub(crate) fn layers(s: &mut Session, p: &Value) -> CmdResult { let p = owned.as_ref().unwrap_or(p); let bound = crate::service_logins::request_params(s, p)?; let p = bound.as_ref(); - let words: Vec = ps(p, "filter").map(|f| f.split_whitespace().map(str::to_lowercase).collect()).unwrap_or_default(); + let words: Vec = match p.get("filter") { + None | Some(Value::Null) => Vec::new(), + Some(Value::String(f)) => f.split_whitespace().map(str::to_lowercase).collect(), + Some(_) => return Err("filter must be text: the words a layer's title or name must all contain".into()), + }; let mut found = discover_matching(s, p, false, &words)?; - notify_withheld(s, found.service.withheld_addresses); + // A search reads again what a listing read (Add From Service's, once per pause in typing), + // so the listing's notice isn't repeated; the answer's notes still say it (#1577). + if words.is_empty() { + notify_withheld(s, found.service.withheld_addresses); + } let searched = !words.is_empty(); // The layers read, or with a filter those matching it. let found_layers = found.service.layers.len(); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 7f9f0b66b5..9e0187b0ab 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -16317,6 +16317,41 @@ fn a_filter_searches_past_the_listing() { assert!(s.run("service.layers", &json!({"url": url, "text": caps, "filter": " "})).unwrap().get("matched").is_none()); } +/// A search reads again the documents a listing read, so it doesn't say again what reading them +/// says: Add From Service searches once per pause in typing, and the notices would repeat (#1577). +/// The answer's notes still say it. +#[test] +fn a_search_does_not_repeat_the_listings_notice() { + // A landing page whose collections link doesn't get the address's token: one withheld. + let url = "https://collections.invalid/root?f=json&token=RGSYNTH_search"; + let landing = r#"{"links":[{"rel":"data","type":"application/json","href":"/catalog?f=json"}]}"#; + let collections: Vec = (1..=3) + .map(|i| json!({"id": format!("c{i}"), "title": format!("C {i}"), "links": [{"rel": "items", "type": "application/geo+json", "href": format!("https://collections.invalid/collections/c{i}/items")}]})) + .collect(); + let documents = json!([{"url": "https://collections.invalid/catalog?f=json", "text": json!({"collections": collections}).to_string()}]); + let mut s = Session::new(Project::default()); + let withheld = |s: &Session, mark| s.notices_since(mark).iter().filter(|n| n.text.contains("didn't get its keys")).count(); + let said = |v: &Value| v["notes"].as_array().unwrap().iter().any(|n| n.as_str().unwrap().contains("didn't get its keys")); + let mark = s.notice_mark(); + let p = json!({"url": url, "type": "ogcapi", "text": landing, "documents": documents}); + let listing = s.run("service.layers", &p).unwrap(); + assert_eq!(withheld(&s, mark), 1, "{:?}", s.notices_since(mark)); + assert!(said(&listing), "{listing}"); + for filter in ["c 1", "c 2", "c"] { + let mut p = p.clone(); + p["filter"] = json!(filter); + let found = s.run("service.layers", &p).unwrap(); + assert!(found["matched"].as_u64().unwrap() > 0 && said(&found), "{found}"); + } + assert_eq!(withheld(&s, mark), 1, "{:?}", s.notices_since(mark)); + // A filter that isn't text is refused, not ignored. + for filter in [json!(["c"]), json!(5)] { + let mut p = p.clone(); + p["filter"] = filter; + assert!(s.run("service.layers", &p).unwrap_err().contains("filter must be text")); + } +} + /// A layer past what RustGIS reads of a document is refused for that reason, not as absent. #[test] fn a_layer_past_the_read_limit_is_refused_saying_so() { From 1df419d4daa363b6b6426e7dad90c50afe7a8180 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 22:14:29 -0300 Subject: [PATCH 4/5] fix(engine): look for each filter word once (#1577) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `matches_words` checks every word of the filter with `all`, which stops only at a miss, so a word given many times was looked for again in each layer containing it ("a a a …" × 50,000 over 100,000 layers). The words are now sorted and deduplicated first, with the same answer. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/service.rs | 8 +++++++- crates/engine/src/tests.rs | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index b5bb52b420..77a5d0403a 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -706,7 +706,13 @@ pub(crate) fn layers(s: &mut Session, p: &Value) -> CmdResult { let p = bound.as_ref(); let words: Vec = match p.get("filter") { None | Some(Value::Null) => Vec::new(), - Some(Value::String(f)) => f.split_whitespace().map(str::to_lowercase).collect(), + // Each word once: a repeated one would be looked for again in every layer it's in. + Some(Value::String(f)) => { + let mut words: Vec = f.split_whitespace().map(str::to_lowercase).collect(); + words.sort_unstable(); + words.dedup(); + words + } Some(_) => return Err("filter must be text: the words a layer's title or name must all contain".into()), }; let mut found = discover_matching(s, p, false, &words)?; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 9e0187b0ab..24357c73d1 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -16344,6 +16344,11 @@ fn a_search_does_not_repeat_the_listings_notice() { assert!(found["matched"].as_u64().unwrap() > 0 && said(&found), "{found}"); } assert_eq!(withheld(&s, mark), 1, "{:?}", s.notices_since(mark)); + // A word given many times is looked for once, with the same answer. + let mut many = p.clone(); + many["filter"] = json!(vec!["c"; 50_000].join(" ") + " 2"); + let found = s.run("service.layers", &many).unwrap(); + assert_eq!(found["matched"], 1, "{found}"); // A filter that isn't text is refused, not ignored. for filter in [json!(["c"]), json!(5)] { let mut p = p.clone(); From 3703b9ca287c735c22819c7e819d3095fea03397 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 22:48:15 -0300 Subject: [PATCH 5/5] test(ui): set the filter's typing pause instead of waiting for it (#1577) `a_cut_listing_says_what_the_filter_searches` expected the search to wait after two frames and to have run after a 350 ms sleep, so a slow runner could fail it. It now sets the instant `search_all` measures the pause from: an hour ahead while the search must wait, a second back once it is due. No sleep is left. Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/tests/smoke.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index 52e1e587b9..b00870e392 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -2434,6 +2434,22 @@ fn a_service_is_listed_when_its_capabilities_land() { h.step(); } +/// Whether typing in Add From Service's filter has paused long enough for its search +/// (`SEARCH_PAUSE`), set rather than waited for: a slow frame neither runs it early nor late. +enum Pause { + Waiting, + Over, +} + +fn typed_at(app: &mut RustGisApp, pause: Pause) { + let now = web_time::Instant::now(); + let at = match pause { + Pause::Waiting => now + std::time::Duration::from_secs(3600), + Pause::Over => now.checked_sub(std::time::Duration::from_secs(1)).expect("a clock past its first second"), + }; + app.ui.service.as_mut().unwrap().typed.as_mut().expect("the filter was typed").1 = at; +} + /// A listing that stops short says in Add From Service how many layers there are, that its filter /// searches all of them, and how many of them are shown, in one place and not three. #[test] @@ -2489,10 +2505,11 @@ Connection: close // A layer past the first 2000 is found once typing pauses: all of them are searched (#1577). h.state_mut().ui.service_browse.filter = "T:2001".into(); h.step(); + typed_at(h.state_mut(), Pause::Waiting); h.step(); h.get_by_label("Searching all 2001 layers…"); assert!(h.query_by_label_contains("t:2001").is_none(), "not among the 2000 listed"); - std::thread::sleep(std::time::Duration::from_millis(350)); + typed_at(h.state_mut(), Pause::Over); h.step(); h.step(); assert!(h.query_by_label_contains("Searching all").is_none()); @@ -2506,8 +2523,7 @@ Connection: close // More matches than a search lists: its note says so, not only how many rows are shown. h.state_mut().ui.service_browse.filter = "t:".into(); h.step(); - h.step(); - std::thread::sleep(std::time::Duration::from_millis(350)); + typed_at(h.state_mut(), Pause::Over); h.step(); h.step(); h.get_by_label("only the first 2000 of 2001 matching layers are listed; service.add takes any of them by name");