diff --git a/crates/data/src/ogc.rs b/crates/data/src/ogc.rs index 31670fb4d2..e7857fcdb1 100644 --- a/crates/data/src/ogc.rs +++ b/crates/data/src/ogc.rs @@ -469,7 +469,7 @@ pub struct Service { /// between answers (#1519). Reading stops at [`MAX_READ_LAYERS`], or sooner when the layers' /// request addresses are long ([`read_limit`], [`MAX_COPIED_ADDRESS_BYTES`]); past that none is /// read. An ArcGIS list is read only as far as it is listed. -const MAX_LISTED_LAYERS: usize = 2_000; +pub const MAX_LISTED_LAYERS: usize = 2_000; /// Most layers or collections read from one document, which is hostile input. const MAX_READ_LAYERS: usize = if cfg!(test) { 3_000 } else { 100_000 }; /// Most bytes of a document's own request addresses (its GetMap's, GetFeature's…) copied into diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index d6dc3a6e62..77a5d0403a 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -31,7 +31,7 @@ const DOCUMENT_ROLES: &[crate::ParamRole] = &[ pub fn commands() -> Vec { let mut commands = vec![ C::new("service.layers", "Service Layers", "Home › Add", layers).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Network)).view().params( - r#"{"url": "a WMS, WMTS or WFS address (GetCapabilities is added), an OGC API collections address or landing page (its map tiles and linked collections), or an ArcGIS FeatureServer, MapServer, ImageServer or VectorTileServer, or an ArcGIS services directory or folder (its folders and services, each listed by its own url)", "type"?: "wms"|"wmts"|"wfs"|"ogcapi"|"arcgis" (guessed from the address), "text"?: "the capabilities document, when already fetched", "documents"?: [{"url": "a discovered document address", "text": "its fetched JSON"}], "connection"?: "the saved web-service connection id whose login the requests use"} → layers with source_url, and next when supplied documents need another fetch"#, + r#"{"url": "a WMS, WMTS or WFS address (GetCapabilities is added), an OGC API collections address or landing page (its map tiles and linked collections), or an ArcGIS FeatureServer, MapServer, ImageServer or VectorTileServer, or an ArcGIS services directory or folder (its folders and services, each listed by its own url)", "type"?: "wms"|"wmts"|"wfs"|"ogcapi"|"arcgis" (guessed from the address), "text"?: "the capabilities document, when already fetched", "documents"?: [{"url": "a discovered document address", "text": "its fetched JSON"}], "connection"?: "the saved web-service connection id whose login the requests use", "filter"?: "words a layer's title or name must all contain, in any case: every layer read is searched, not only those a listing shows, and the first 2000 that match are listed; what a listing says in a notice, a search says only in its notes"} → layers with source_url (with filter, total: the layers read, and matched: how many match), and next when supplied documents need another fetch"#, ), C::new("service.add", "Add Service Layer", "Home › Add", add).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Network)).params( r#"{"url": "as service.layers", "type"?, "layer": "the layer's name (service.layers lists them)", "name"?: text, "time"?: "a time-enabled WMS or WMTS layer: by default (or \"slider\", true) it follows the time slider, turned on with the layer's times when it is off; or one of its dates (2024-06-01), or \"default\" (false): its service's default", "text"?: "the capabilities document, when already fetched", "documents"?: "additional fetched documents, as service.layers", "source_url"?: "the listed layer's source_url", "describe"?: "a WCS coverage's DescribeCoverage document, when already fetched", "tiles"?: "an OGC API layer's tile template, when already read from its tileset (the app's)", "bbox"?: [west, south, east, north] | "view" | "full": a WCS coverage's area (the view by default); for a WFS, OGC API or ArcGIS feature layer, only the features that cross it (the whole layer, "full", by default), "connection"?: "the id of the saved web-service connection it comes from (connection.browse gives it): the layer records it, and is read with its login"}"#, @@ -108,6 +108,8 @@ struct Discovery { excluded_layers: bool, /// A document left collections out (past its listing limit, or unreadable items links). incomplete: bool, + /// How many layers were read, before a filter kept those matching it ([`discover_matching`]). + read: usize, } impl Discovery { @@ -153,7 +155,24 @@ fn capabilities(s: &mut Session, p: &Value) -> Result { /// off the frame ([`Session::left_document`]), when a document not in yet is its `next`. Long /// document addresses list fewer layers ([`MAX_LISTED_ADDRESS_BYTES`]), said in a note. fn discover(s: &mut Session, p: &Value, left: bool) -> Result { + discover_matching(s, p, left, &[]) +} + +/// [`discover`], keeping only the layers whose title or name contains each of `words` (lower +/// case), when there are any: every layer read is searched, and the first +/// [`rustgis_data::ogc::MAX_LISTED_LAYERS`] matches are listed (#1577). +fn discover_matching(s: &mut Session, p: &Value, left: bool, words: &[String]) -> Result { let mut found = read_documents(s, p, left)?; + found.read = found.service.layers.len(); + if !words.is_empty() { + let keep: Vec = found.service.layers.iter().map(|l| matches_words(l, words)).collect(); + let mut kept = keep.iter(); + found.service.layers.retain(|_| kept.next().copied().unwrap_or(false)); + let mut kept = keep.iter(); + found.sources.retain(|_| kept.next().copied().unwrap_or(false)); + let max = rustgis_data::ogc::MAX_LISTED_LAYERS; + found.service.listed = (found.service.layers.len() > max).then_some(max); + } let listed = found.service.listed.unwrap_or(usize::MAX).min(found.sources.len()); let mut bytes = 0usize; let cut = found.sources[..listed].iter().position(|source| { @@ -173,6 +192,13 @@ fn discover(s: &mut Session, p: &Value, left: bool) -> Result Ok(found) } +/// Whether a layer's title or name contains each of `words` (lower case): Add From Service's +/// filter, which matched the listed layers in the dialog before #1577. +fn matches_words(layer: &rustgis_data::ogc::ServiceLayer, words: &[String]) -> bool { + let (title, name) = (layer.title.to_lowercase(), layer.name.to_lowercase()); + words.iter().all(|w| title.contains(w.as_str()) || name.contains(w.as_str())) +} + fn read_documents(s: &mut Session, p: &Value, left: bool) -> Result { let original = url_of(p)?; let mut address = @@ -678,12 +704,29 @@ pub(crate) fn layers(s: &mut Session, p: &Value) -> CmdResult { let p = owned.as_ref().unwrap_or(p); let bound = crate::service_logins::request_params(s, p)?; let p = bound.as_ref(); - let mut found = capabilities(s, p)?; - notify_withheld(s, found.service.withheld_addresses); - let of = layer_count(&found.service); - let total = found.service.layers.len(); + let words: Vec = match p.get("filter") { + None | Some(Value::Null) => Vec::new(), + // Each word once: a repeated one would be looked for again in every layer it's in. + Some(Value::String(f)) => { + let mut words: Vec = f.split_whitespace().map(str::to_lowercase).collect(); + words.sort_unstable(); + words.dedup(); + words + } + Some(_) => return Err("filter must be text: the words a layer's title or name must all contain".into()), + }; + let mut found = discover_matching(s, p, false, &words)?; + // A search reads again what a listing read (Add From Service's, once per pause in typing), + // so the listing's notice isn't repeated; the answer's notes still say it (#1577). + if words.is_empty() { + notify_withheld(s, found.service.withheld_addresses); + } + let searched = !words.is_empty(); + // The layers read, or with a filter those matching it. + let found_layers = found.service.layers.len(); + let of = if searched { format!("{found_layers} matching layers") } else { layer_count(&found.service) }; let mut cut = None; - if let Some(listed) = found.service.listed.filter(|&listed| listed < total) { + if let Some(listed) = found.service.listed.filter(|&listed| listed < found_layers) { found.service.layers.truncate(listed); let note = format!("only the first {listed} of {of} are listed; service.add takes any of them by name"); found.service.notes.push(note.clone()); @@ -691,11 +734,15 @@ pub(crate) fn layers(s: &mut Session, p: &Value) -> CmdResult { } let mut out = service_json(&found.service); // How many there are, for a window that says it its own way: `listedNote` says the list - // stops (Add From Service: its filter searches only those listed), and `atLeast` that the - // document has more than were read. + // stops (Add From Service searches past it with `filter`), and `atLeast` that the document + // has more than were read. let past_limit = found.service.past_read_limit.is_some(); - if cut.is_some() || past_limit { - out["total"] = json!(total); + if cut.is_some() || past_limit || searched { + out["total"] = json!(found.read); + } + // (With a filter, `total` counts every layer read, and `matched` those matching it.) + if searched { + out["matched"] = json!(found_layers); } if past_limit { out["atLeast"] = json!(true); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index bc520d580e..ce2a94cc22 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -16747,6 +16747,81 @@ fn a_long_document_address_lists_fewer_layers() { assert!(s.run("service.groupPlan", &json!({"url": "https://api.example.org/collections", "text": text})).is_ok()); } +/// `service.layers` with `filter` searches every layer read, not only the first 2000 it lists, and +/// counts the matches (#1577: IBGE's WFS has 10,186 types, in an order that changes). +#[test] +fn a_filter_searches_past_the_listing() { + let types: String = (1..=2500) + .map(|i| { + let title = if (2101..=2103).contains(&i) { format!("Solos do Brasil {i}") } else { format!("Tipo {i}") }; + format!("t:{i}{title}") + }) + .collect(); + let caps = format!(r#"{types}"#); + let url = "https://wfs.example.org/wfs"; + let mut s = Session::new(Project::default()); + let names = |v: &Value| v["layers"].as_array().unwrap().iter().map(|l| l["name"].as_str().unwrap().to_string()).collect::>(); + // Without a filter, as before: the first 2000, and no count of matches. + let all = s.run("service.layers", &json!({"url": url, "text": caps})).unwrap(); + assert_eq!((&all["count"], &all["total"]), (&json!(2000), &json!(2500))); + assert!(all.get("matched").is_none()); + // Words in the title or the name, in any case, all of them: past the listing too. + let found = s.run("service.layers", &json!({"url": url, "text": caps, "filter": " BRASIL solos "})).unwrap(); + assert_eq!(names(&found), ["t:2101", "t:2102", "t:2103"]); + assert_eq!((&found["count"], &found["matched"], &found["total"]), (&json!(3), &json!(3), &json!(2500))); + assert!(found.get("listedNote").is_none() && found["notes"].as_array().unwrap().is_empty(), "{found}"); + assert_eq!(found["layers"][0]["source_url"], all["layers"][0]["source_url"]); + assert_eq!(names(&s.run("service.layers", &json!({"url": url, "text": caps, "filter": "t:2500"})).unwrap()), ["t:2500"]); + let none = s.run("service.layers", &json!({"url": url, "text": caps, "filter": "solos nada"})).unwrap(); + assert_eq!((&none["count"], &none["matched"]), (&json!(0), &json!(0))); + // More matches than a listing holds: the first 2000, said, with how many match. + let many = s.run("service.layers", &json!({"url": url, "text": caps, "filter": "tipo"})).unwrap(); + assert_eq!((&many["count"], &many["matched"], &many["total"]), (&json!(2000), &json!(2497), &json!(2500))); + assert_eq!(many["listedNote"], "only the first 2000 of 2497 matching layers are listed; service.add takes any of them by name"); + // A blank filter is none. + assert!(s.run("service.layers", &json!({"url": url, "text": caps, "filter": " "})).unwrap().get("matched").is_none()); +} + +/// A search reads again the documents a listing read, so it doesn't say again what reading them +/// says: Add From Service searches once per pause in typing, and the notices would repeat (#1577). +/// The answer's notes still say it. +#[test] +fn a_search_does_not_repeat_the_listings_notice() { + // A landing page whose collections link doesn't get the address's token: one withheld. + let url = "https://collections.invalid/root?f=json&token=RGSYNTH_search"; + let landing = r#"{"links":[{"rel":"data","type":"application/json","href":"/catalog?f=json"}]}"#; + let collections: Vec = (1..=3) + .map(|i| json!({"id": format!("c{i}"), "title": format!("C {i}"), "links": [{"rel": "items", "type": "application/geo+json", "href": format!("https://collections.invalid/collections/c{i}/items")}]})) + .collect(); + let documents = json!([{"url": "https://collections.invalid/catalog?f=json", "text": json!({"collections": collections}).to_string()}]); + let mut s = Session::new(Project::default()); + let withheld = |s: &Session, mark| s.notices_since(mark).iter().filter(|n| n.text.contains("didn't get its keys")).count(); + let said = |v: &Value| v["notes"].as_array().unwrap().iter().any(|n| n.as_str().unwrap().contains("didn't get its keys")); + let mark = s.notice_mark(); + let p = json!({"url": url, "type": "ogcapi", "text": landing, "documents": documents}); + let listing = s.run("service.layers", &p).unwrap(); + assert_eq!(withheld(&s, mark), 1, "{:?}", s.notices_since(mark)); + assert!(said(&listing), "{listing}"); + for filter in ["c 1", "c 2", "c"] { + let mut p = p.clone(); + p["filter"] = json!(filter); + let found = s.run("service.layers", &p).unwrap(); + assert!(found["matched"].as_u64().unwrap() > 0 && said(&found), "{found}"); + } + assert_eq!(withheld(&s, mark), 1, "{:?}", s.notices_since(mark)); + // A word given many times is looked for once, with the same answer. + let mut many = p.clone(); + many["filter"] = json!(vec!["c"; 50_000].join(" ") + " 2"); + let found = s.run("service.layers", &many).unwrap(); + assert_eq!(found["matched"], 1, "{found}"); + // A filter that isn't text is refused, not ignored. + for filter in [json!(["c"]), json!(5)] { + let mut p = p.clone(); + p["filter"] = filter; + assert!(s.run("service.layers", &p).unwrap_err().contains("filter must be text")); + } +} + /// A layer past what RustGIS reads of a document is refused for that reason, not as absent. #[test] fn a_layer_past_the_read_limit_is_refused_saying_so() { diff --git a/crates/ui-egui/src/dialogs.rs b/crates/ui-egui/src/dialogs.rs index b68a30a602..98da820cc0 100644 --- a/crates/ui-egui/src/dialogs.rs +++ b/crates/ui-egui/src/dialogs.rs @@ -852,6 +852,66 @@ fn go_to(app: &mut RustGisApp, url: &str, kind: &str, way: Vec<(String, &'static /// How many of a service's layers Add From Service lists. const SHOWN_LAYERS: usize = 500; +/// How long typing pauses before Add From Service searches all of a listing that stops short: +/// reading the documents again takes a moment for a large one (IBGE's WFS, 12 MB), so not on +/// every key (#1577). +pub(crate) const SEARCH_PAUSE: web_time::Duration = web_time::Duration::from_millis(300); + +/// A filter's words as `service.layers` takes them: lower case, one space apart. +fn filter_key(filter: &str) -> String { + filter.split_whitespace().map(str::to_lowercase).collect::>().join(" ") +} + +/// The search of all of a service's layers with the filter, when its listing stops short of them +/// (`listedNote`) and typing has paused for [`SEARCH_PAUSE`]: `service.layers` again on the +/// documents the dialog keeps, with `filter`, so nothing is downloaded. A listing that holds them +/// all is filtered in the dialog, as before. Gives how long until a search is due, to draw again +/// then. One read of the documents on the UI thread per pause in typing, as for each document +/// that arrives (`poll_service`): the parsed layers aren't kept, which would hold up to 100,000 +/// of them. It stays on the frame because `service.layers` binds a saved connection's login to +/// the addresses its rows carry, in this session: a session of its own off the frame would leave +/// that out, and adding a row would go without it. Nothing is fetched (the documents are +/// kept, and no search starts while one is `pending`); IBGE's WFS, 12 MB and 10,186 layers, +/// takes about 0.3 s in a dev build. +fn search_all(app: &mut RustGisApp) -> Option { + let conn = app.ui.service.as_mut()?; + let key = filter_key(&app.ui.service_browse.filter); + let short = conn.listing.as_ref().is_some_and(|v| v.get("listedNote").is_some()); + if !short || key.is_empty() || conn.pending.is_some() || conn.search.as_ref().is_some_and(|s| s.filter == key) { + return None; + } + let now = web_time::Instant::now(); + match &conn.typed { + Some((typed, at)) if *typed == key => { + let waited = now.duration_since(*at); + if waited < SEARCH_PAUSE { + return Some(SEARCH_PAUSE - waited); + } + } + _ => { + conn.typed = Some((key, now)); + return Some(SEARCH_PAUSE); + } + } + let mut p = json!({"url": conn.url, "text": conn.text, "documents": conn.documents, "filter": key}); + if conn.kind != "auto" { + p["type"] = json!(conn.kind); + } + if let Some(connection) = &conn.connection { + p["connection"] = json!(connection); + } + // Not `app.run`: an error belongs in the dialog, as the listing's does. + let listing = app.session.run("service.layers", &p); + conn.search = Some(crate::ServiceSearch { filter: key, listing }); + conn.added = None; + None +} + +/// The search's answer for the filter as typed now, when there is one and it holds layers. +fn searched<'a>(conn: &'a crate::ServiceConn, filter: &str) -> Option<&'a Value> { + conn.search.as_ref().filter(|s| s.filter == filter_key(filter)).and_then(|s| s.listing.as_ref().ok()) +} + /// The listed layers matching `filter` (words their title or name contains), with their place in /// the listing. fn matching_rows<'a>(layers: &'a [Value], filter: &str) -> Vec<(usize, &'a Value)> { @@ -892,13 +952,24 @@ fn refresh_added(app: &mut RustGisApp) { if conn.added.as_ref().is_some_and(|a| a.revision == revision && a.map == map && a.filter == *filter) { return; } - let layers: &[Value] = conn.listing.as_ref().and_then(|v| v.get("layers")).and_then(Value::as_array).map_or(&[], Vec::as_slice); - let matching = matching_rows(layers, filter); - let matched = matching.len(); - let shown: Vec = matching.into_iter().take(SHOWN_LAYERS).map(|(i, _)| i).collect(); + // Every layer matching the filter, when all of a listing that stops short were searched: its + // answer holds only matches (the first 2000), and counts them all. + let search = searched(conn, filter); + let was_searched = search.is_some(); + let listing = search.or(conn.listing.as_ref()); + let layers: &[Value] = listing.and_then(|v| v.get("layers")).and_then(Value::as_array).map_or(&[], Vec::as_slice); + let (matched, shown): (usize, Vec) = match search { + Some(v) => { + (v.get("matched").and_then(Value::as_u64).map_or(layers.len(), |m| m as usize), (0..layers.len().min(SHOWN_LAYERS)).collect()) + } + None => { + let matching = matching_rows(layers, filter); + (matching.len(), matching.into_iter().take(SHOWN_LAYERS).map(|(i, _)| i).collect()) + } + }; let removals = rustgis_engine::cmd::service_added::added(app.session.map(), layers, &shown); let rows = shown.into_iter().map(|i| (i, removals.get(i).cloned().flatten())).collect(); - conn.added = Some(crate::ServiceAdded { revision, map, filter: filter.clone(), matched, rows }); + conn.added = Some(crate::ServiceAdded { revision, map, filter: filter.clone(), matched, searched: was_searched, rows }); } /// Add From Service: a WMS, WMTS, WFS, WCS, OGC API Features or ArcGIS REST service by its address @@ -992,6 +1063,9 @@ fn add_service(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { crate::connectionui::picked(&list, app.dialogs.connection.as_deref(), url).is_none().then(|| (url.to_string(), kind)) }); let mut keep_root = false; + if let Some(due) = search_all(app) { + ui.ctx().request_repaint_after(due); + } refresh_added(app); if let Some(conn) = &app.ui.service { ui.separator(); @@ -1018,7 +1092,7 @@ fn add_service(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { let layers = v.get("layers").and_then(Value::as_array).unwrap_or(&none); let grouped = kind.starts_with("OGC API"); // A listing that stops short counts them all, and says so here in the dialog's own - // words instead of its note: the filter searches only those listed. + // words instead of its note: the filter searches all of them (`search_all`). let listed_note = v.get("listedNote").and_then(Value::as_str); let count = crate::connectionui::layer_count(v); ui.horizontal(|ui| { @@ -1038,15 +1112,41 @@ fn add_service(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { ui.label(egui::RichText::new(n).small().weak()); } } + // The filter searches all of them, once typing pauses (`search_all`). if listed_note.is_some() { - ui.label( - egui::RichText::new(crate::i18n::fmt( - "The filter searches the first {n} of them; the service.add command adds any other by its name.", + let key = filter_key(&app.ui.service_browse.filter); + let line = if key.is_empty() { + Some(crate::i18n::fmt( + "The first {n} of them are listed; the filter searches all of them.", &[("n", &layers.len().to_string())], )) - .small() - .weak(), - ); + } else { + match conn.search.as_ref().filter(|s| s.filter == key).map(|s| &s.listing) { + None => { + let total = v.get("total").and_then(Value::as_u64).map_or(layers.len().to_string(), |n| n.to_string()); + Some(crate::i18n::fmt("Searching all {total} layers…", &[("total", &total)])) + } + Some(Err(e)) => { + ui.colored_label(t.red, e); + None + } + // The search's own notes, beside the listing's: more matches than it + // lists (its `listedNote`), or a list cut by its addresses' length. + Some(Ok(found)) => { + let shown: Vec<&str> = + v.get("notes").and_then(Value::as_array).into_iter().flatten().filter_map(Value::as_str).collect(); + for n in found.get("notes").and_then(Value::as_array).into_iter().flatten().filter_map(Value::as_str) { + if !shown.contains(&n) { + ui.label(egui::RichText::new(n).small().weak()); + } + } + None + } + } + }; + if let Some(line) = line { + ui.label(egui::RichText::new(line).small().weak()); + } } // A long list is filtered by words; a directory's services lead back to it. let browse = &mut app.ui.service_browse; @@ -1111,6 +1211,13 @@ fn add_service(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { } let added = conn.added.as_ref(); let rows: &[(usize, Option)] = added.map_or(&[], |a| a.rows.as_slice()); + // A search's rows are places in its answer, which holds only matches. + let row_layers = match added.filter(|a| a.searched) { + Some(_) => { + searched(conn, &app.ui.service_browse.filter).and_then(|v| v.get("layers")).and_then(Value::as_array).unwrap_or(&none) + } + None => layers, + }; if let Some(a) = added.filter(|a| a.matched > SHOWN_LAYERS) { ui.label( egui::RichText::new(crate::i18n::fmt( @@ -1134,7 +1241,7 @@ fn add_service(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { egui::ScrollArea::vertical().id_salt("add_service_layers_scroll").auto_shrink([false, false]).show(ui, |ui| { let style = ui.style().clone(); for (i, removal) in rows { - let Some(l) = layers.get(*i) else { continue }; + let Some(l) = row_layers.get(*i) else { continue }; ui.horizontal(|ui| { ui.add_space(2.0); let kind = l.get("offer").and_then(|o| o.get("type")).and_then(Value::as_str); @@ -2487,6 +2594,8 @@ mod tests { }], })), added: None, + typed: None, + search: None, error: None, failure: None, }); @@ -2538,6 +2647,8 @@ mod tests { }], })), added: None, + typed: None, + search: None, error: None, failure: None, }); @@ -2599,6 +2710,8 @@ mod tests { }], })), added: None, + typed: None, + search: None, error: None, failure: None, }); @@ -2653,6 +2766,8 @@ mod tests { }], })), added: None, + typed: None, + search: None, error: None, failure: None, }); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 966cc90d13..135ce0ff83 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -456,6 +456,20 @@ pub struct ServiceConn { pub error: Option, /// A capabilities request that failed, with what the server said (#509). pub failure: Option, + /// The filter as last typed, and when: a listing that stops short is searched whole once + /// typing pauses (`dialogs::SEARCH_PAUSE`), not on every key (#1577). + pub typed: Option<(String, web_time::Instant)>, + /// `service.layers` run again with the filter, over every layer read of a listing that stops + /// short. + pub search: Option, +} + +/// A search of all of a service's layers (`service.layers` with `filter`), for Add From Service. +pub struct ServiceSearch { + /// The filter it was run with. + pub filter: String, + /// Its answer (the first matches, with `matched`), or why it failed. + pub listing: Result, } /// Add From Service's rows: the layers of a listing that match the filter, and for each the @@ -470,6 +484,8 @@ pub struct ServiceAdded { pub filter: String, /// How many layers match the filter (the first `dialogs::SHOWN_LAYERS` are shown). pub matched: usize, + /// The rows are places in the search's answer (`ServiceConn::search`), not in the listing. + pub searched: bool, /// Each shown row: its place in the listing, and the command that removes it when it is on /// the map (`None`: the row says Add). pub rows: Vec<(usize, Option)>, @@ -3366,6 +3382,8 @@ impl RustGisApp { added: None, error: None, failure: None, + typed: None, + search: None, }); } "ui.importShapefile" => { @@ -5141,6 +5159,7 @@ impl RustGisApp { } conn.listing = Some(v); conn.added = None; + conn.search = None; } Err(e) => conn.error = Some(e), } diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index 2664f060a8..b00870e392 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -2434,8 +2434,24 @@ fn a_service_is_listed_when_its_capabilities_land() { h.step(); } +/// Whether typing in Add From Service's filter has paused long enough for its search +/// (`SEARCH_PAUSE`), set rather than waited for: a slow frame neither runs it early nor late. +enum Pause { + Waiting, + Over, +} + +fn typed_at(app: &mut RustGisApp, pause: Pause) { + let now = web_time::Instant::now(); + let at = match pause { + Pause::Waiting => now + std::time::Duration::from_secs(3600), + Pause::Over => now.checked_sub(std::time::Duration::from_secs(1)).expect("a clock past its first second"), + }; + app.ui.service.as_mut().unwrap().typed.as_mut().expect("the filter was typed").1 = at; +} + /// A listing that stops short says in Add From Service how many layers there are, that its filter -/// searches only those listed, and how many of them are shown, in one place and not three. +/// searches all of them, and how many of them are shown, in one place and not three. #[test] fn a_cut_listing_says_what_the_filter_searches() { use std::io::{Read, Write}; @@ -2479,13 +2495,44 @@ Connection: close h.step(); h.step(); h.get_by_label_contains("Local: 2001 layers"); - h.get_by_label("The filter searches the first 2000 of them; the service.add command adds any other by its name."); + h.get_by_label("The first 2000 of them are listed; the filter searches all of them."); h.get_by_label("500 of 2000 are listed."); assert!(h.query_by_label_contains("only the first 2000").is_none(), "the note isn't said twice"); h.state_mut().ui.service_browse.filter = "t:20".into(); h.step(); h.step(); assert!(h.query_by_label_contains("are listed.").is_none(), "its 12 matches are all shown"); + // A layer past the first 2000 is found once typing pauses: all of them are searched (#1577). + h.state_mut().ui.service_browse.filter = "T:2001".into(); + h.step(); + typed_at(h.state_mut(), Pause::Waiting); + h.step(); + h.get_by_label("Searching all 2001 layers…"); + assert!(h.query_by_label_contains("t:2001").is_none(), "not among the 2000 listed"); + typed_at(h.state_mut(), Pause::Over); + h.step(); + h.step(); + assert!(h.query_by_label_contains("Searching all").is_none()); + h.get_by_label_contains("t:2001"); + let conn = h.state().ui.service.as_ref().unwrap(); + let a = conn.added.as_ref().unwrap(); + assert!(a.searched && a.matched == 1 && a.rows.len() == 1); + // Its row adds it from the search's answer, as a listed row would. + let row = &conn.search.as_ref().unwrap().listing.as_ref().unwrap()["layers"][a.rows[0].0]; + assert_eq!(row["name"], "t:2001"); + // More matches than a search lists: its note says so, not only how many rows are shown. + h.state_mut().ui.service_browse.filter = "t:".into(); + h.step(); + typed_at(h.state_mut(), Pause::Over); + h.step(); + h.step(); + h.get_by_label("only the first 2000 of 2001 matching layers are listed; service.add takes any of them by name"); + h.get_by_label("500 of 2001 are listed."); + // Cleared, the listing is filtered here again. + h.state_mut().ui.service_browse.filter.clear(); + h.step(); + h.step(); + h.get_by_label("The first 2000 of them are listed; the filter searches all of them."); } #[test] diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 9e46e1cd31..ce739c6a53 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -327,6 +327,13 @@ list each repeating a somewhat shorter one. An ArcGIS service or directory is an A dynamic MapServer offers each of those layers twice, as features and as a map, so its list can be twice as long. A note says how many are left out past either limit. +When a list stops short of the layers read, Add From Service's filter searches every layer +read, not only those listed: once typing pauses, the documents it already has are read again +for the layers whose title or name contains each word (`service.layers` with `"filter"`), and +the first 2,000 matches are listed, with how many match and a note when more match. Layers past +the read limits above, and an ArcGIS service's past its first 2,000, aren't read, so the filter +doesn't find them either. + A WMS or WMTS layer's tiles are asked for down to the detail its service gives, and stretched only past it: a WMTS layer's over the zooms of its tile matrix set (or those its `TileMatrixSetLimits` name; swisstopo's SWISSIMAGE goes to zoom 20), a WMS layer's between the