Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/data/src/ogc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,7 @@ pub struct Service {
/// between answers (#1519). Reading stops at [`MAX_READ_LAYERS`], or sooner when the layers'
/// request addresses are long ([`read_limit`], [`MAX_COPIED_ADDRESS_BYTES`]); past that none is
/// read. An ArcGIS list is read only as far as it is listed.
const MAX_LISTED_LAYERS: usize = 2_000;
pub const MAX_LISTED_LAYERS: usize = 2_000;
/// Most layers or collections read from one document, which is hostile input.
const MAX_READ_LAYERS: usize = if cfg!(test) { 3_000 } else { 100_000 };
/// Most bytes of a document's own request addresses (its GetMap's, GetFeature's…) copied into
Expand Down
67 changes: 57 additions & 10 deletions crates/engine/src/cmd/service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ const DOCUMENT_ROLES: &[crate::ParamRole] = &[
pub fn commands() -> Vec<C> {
let mut commands = vec![
C::new("service.layers", "Service Layers", "Home › Add", layers).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Network)).view().params(
r#"{"url": "a WMS, WMTS or WFS address (GetCapabilities is added), an OGC API collections address or landing page (its map tiles and linked collections), or an ArcGIS FeatureServer, MapServer, ImageServer or VectorTileServer, or an ArcGIS services directory or folder (its folders and services, each listed by its own url)", "type"?: "wms"|"wmts"|"wfs"|"ogcapi"|"arcgis" (guessed from the address), "text"?: "the capabilities document, when already fetched", "documents"?: [{"url": "a discovered document address", "text": "its fetched JSON"}], "connection"?: "the saved web-service connection id whose login the requests use"} → layers with source_url, and next when supplied documents need another fetch"#,
r#"{"url": "a WMS, WMTS or WFS address (GetCapabilities is added), an OGC API collections address or landing page (its map tiles and linked collections), or an ArcGIS FeatureServer, MapServer, ImageServer or VectorTileServer, or an ArcGIS services directory or folder (its folders and services, each listed by its own url)", "type"?: "wms"|"wmts"|"wfs"|"ogcapi"|"arcgis" (guessed from the address), "text"?: "the capabilities document, when already fetched", "documents"?: [{"url": "a discovered document address", "text": "its fetched JSON"}], "connection"?: "the saved web-service connection id whose login the requests use", "filter"?: "words a layer's title or name must all contain, in any case: every layer read is searched, not only those a listing shows, and the first 2000 that match are listed; what a listing says in a notice, a search says only in its notes"} → layers with source_url (with filter, total: the layers read, and matched: how many match), and next when supplied documents need another fetch"#,
),
C::new("service.add", "Add Service Layer", "Home › Add", add).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Network)).params(
r#"{"url": "as service.layers", "type"?, "layer": "the layer's name (service.layers lists them)", "name"?: text, "time"?: "a time-enabled WMS or WMTS layer: by default (or \"slider\", true) it follows the time slider, turned on with the layer's times when it is off; or one of its dates (2024-06-01), or \"default\" (false): its service's default", "text"?: "the capabilities document, when already fetched", "documents"?: "additional fetched documents, as service.layers", "source_url"?: "the listed layer's source_url", "describe"?: "a WCS coverage's DescribeCoverage document, when already fetched", "tiles"?: "an OGC API layer's tile template, when already read from its tileset (the app's)", "bbox"?: [west, south, east, north] | "view" | "full": a WCS coverage's area (the view by default); for a WFS, OGC API or ArcGIS feature layer, only the features that cross it (the whole layer, "full", by default), "connection"?: "the id of the saved web-service connection it comes from (connection.browse gives it): the layer records it, and is read with its login"}"#,
Expand Down Expand Up @@ -108,6 +108,8 @@ struct Discovery {
excluded_layers: bool,
/// A document left collections out (past its listing limit, or unreadable items links).
incomplete: bool,
/// How many layers were read, before a filter kept those matching it ([`discover_matching`]).
read: usize,
}

impl Discovery {
Expand Down Expand Up @@ -153,7 +155,24 @@ fn capabilities(s: &mut Session, p: &Value) -> Result<Discovery, String> {
/// off the frame ([`Session::left_document`]), when a document not in yet is its `next`. Long
/// document addresses list fewer layers ([`MAX_LISTED_ADDRESS_BYTES`]), said in a note.
fn discover(s: &mut Session, p: &Value, left: bool) -> Result<Discovery, String> {
discover_matching(s, p, left, &[])
}

/// [`discover`], keeping only the layers whose title or name contains each of `words` (lower
/// case), when there are any: every layer read is searched, and the first
/// [`rustgis_data::ogc::MAX_LISTED_LAYERS`] matches are listed (#1577).
fn discover_matching(s: &mut Session, p: &Value, left: bool, words: &[String]) -> Result<Discovery, String> {
let mut found = read_documents(s, p, left)?;
found.read = found.service.layers.len();
if !words.is_empty() {
let keep: Vec<bool> = found.service.layers.iter().map(|l| matches_words(l, words)).collect();
let mut kept = keep.iter();
found.service.layers.retain(|_| kept.next().copied().unwrap_or(false));
let mut kept = keep.iter();
found.sources.retain(|_| kept.next().copied().unwrap_or(false));
let max = rustgis_data::ogc::MAX_LISTED_LAYERS;
found.service.listed = (found.service.layers.len() > max).then_some(max);
}
let listed = found.service.listed.unwrap_or(usize::MAX).min(found.sources.len());
let mut bytes = 0usize;
let cut = found.sources[..listed].iter().position(|source| {
Expand All @@ -173,6 +192,13 @@ fn discover(s: &mut Session, p: &Value, left: bool) -> Result<Discovery, String>
Ok(found)
}

/// Whether a layer's title or name contains each of `words` (lower case): Add From Service's
/// filter, which matched the listed layers in the dialog before #1577.
fn matches_words(layer: &rustgis_data::ogc::ServiceLayer, words: &[String]) -> bool {
let (title, name) = (layer.title.to_lowercase(), layer.name.to_lowercase());
words.iter().all(|w| title.contains(w.as_str()) || name.contains(w.as_str()))
Comment thread
elpidioassinaturas marked this conversation as resolved.
Resolved
}

fn read_documents(s: &mut Session, p: &Value, left: bool) -> Result<Discovery, String> {
let original = url_of(p)?;
let mut address =
Expand Down Expand Up @@ -678,24 +704,45 @@ pub(crate) fn layers(s: &mut Session, p: &Value) -> CmdResult {
let p = owned.as_ref().unwrap_or(p);
let bound = crate::service_logins::request_params(s, p)?;
let p = bound.as_ref();
let mut found = capabilities(s, p)?;
notify_withheld(s, found.service.withheld_addresses);
let of = layer_count(&found.service);
let total = found.service.layers.len();
let words: Vec<String> = match p.get("filter") {
None | Some(Value::Null) => Vec::new(),
// Each word once: a repeated one would be looked for again in every layer it's in.
Some(Value::String(f)) => {
let mut words: Vec<String> = f.split_whitespace().map(str::to_lowercase).collect();
words.sort_unstable();
words.dedup();
words
}
Some(_) => return Err("filter must be text: the words a layer's title or name must all contain".into()),
};
let mut found = discover_matching(s, p, false, &words)?;
// A search reads again what a listing read (Add From Service's, once per pause in typing),
// so the listing's notice isn't repeated; the answer's notes still say it (#1577).
if words.is_empty() {
notify_withheld(s, found.service.withheld_addresses);
}
let searched = !words.is_empty();
// The layers read, or with a filter those matching it.
let found_layers = found.service.layers.len();
let of = if searched { format!("{found_layers} matching layers") } else { layer_count(&found.service) };
let mut cut = None;
if let Some(listed) = found.service.listed.filter(|&listed| listed < total) {
if let Some(listed) = found.service.listed.filter(|&listed| listed < found_layers) {
found.service.layers.truncate(listed);
let note = format!("only the first {listed} of {of} are listed; service.add takes any of them by name");
found.service.notes.push(note.clone());
cut = Some(note);
}
let mut out = service_json(&found.service);
// How many there are, for a window that says it its own way: `listedNote` says the list
// stops (Add From Service: its filter searches only those listed), and `atLeast` that the
// document has more than were read.
// stops (Add From Service searches past it with `filter`), and `atLeast` that the document
// has more than were read.
let past_limit = found.service.past_read_limit.is_some();
if cut.is_some() || past_limit {
out["total"] = json!(total);
if cut.is_some() || past_limit || searched {
out["total"] = json!(found.read);
}
// (With a filter, `total` counts every layer read, and `matched` those matching it.)
if searched {
out["matched"] = json!(found_layers);
}
if past_limit {
out["atLeast"] = json!(true);
Expand Down
75 changes: 75 additions & 0 deletions crates/engine/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16747,6 +16747,81 @@ fn a_long_document_address_lists_fewer_layers() {
assert!(s.run("service.groupPlan", &json!({"url": "https://api.example.org/collections", "text": text})).is_ok());
}

/// `service.layers` with `filter` searches every layer read, not only the first 2000 it lists, and
/// counts the matches (#1577: IBGE's WFS has 10,186 types, in an order that changes).
#[test]
fn a_filter_searches_past_the_listing() {
let types: String = (1..=2500)
.map(|i| {
let title = if (2101..=2103).contains(&i) { format!("Solos do Brasil {i}") } else { format!("Tipo {i}") };
format!("<FeatureType><Name>t:{i}</Name><Title>{title}</Title></FeatureType>")
})
.collect();
let caps = format!(r#"<WFS_Capabilities version="2.0.0"><FeatureTypeList>{types}</FeatureTypeList></WFS_Capabilities>"#);
let url = "https://wfs.example.org/wfs";
let mut s = Session::new(Project::default());
let names = |v: &Value| v["layers"].as_array().unwrap().iter().map(|l| l["name"].as_str().unwrap().to_string()).collect::<Vec<_>>();
// Without a filter, as before: the first 2000, and no count of matches.
let all = s.run("service.layers", &json!({"url": url, "text": caps})).unwrap();
assert_eq!((&all["count"], &all["total"]), (&json!(2000), &json!(2500)));
assert!(all.get("matched").is_none());
// Words in the title or the name, in any case, all of them: past the listing too.
let found = s.run("service.layers", &json!({"url": url, "text": caps, "filter": " BRASIL solos "})).unwrap();
assert_eq!(names(&found), ["t:2101", "t:2102", "t:2103"]);
assert_eq!((&found["count"], &found["matched"], &found["total"]), (&json!(3), &json!(3), &json!(2500)));
assert!(found.get("listedNote").is_none() && found["notes"].as_array().unwrap().is_empty(), "{found}");
assert_eq!(found["layers"][0]["source_url"], all["layers"][0]["source_url"]);
assert_eq!(names(&s.run("service.layers", &json!({"url": url, "text": caps, "filter": "t:2500"})).unwrap()), ["t:2500"]);
let none = s.run("service.layers", &json!({"url": url, "text": caps, "filter": "solos nada"})).unwrap();
assert_eq!((&none["count"], &none["matched"]), (&json!(0), &json!(0)));
// More matches than a listing holds: the first 2000, said, with how many match.
let many = s.run("service.layers", &json!({"url": url, "text": caps, "filter": "tipo"})).unwrap();
assert_eq!((&many["count"], &many["matched"], &many["total"]), (&json!(2000), &json!(2497), &json!(2500)));
assert_eq!(many["listedNote"], "only the first 2000 of 2497 matching layers are listed; service.add takes any of them by name");
// A blank filter is none.
assert!(s.run("service.layers", &json!({"url": url, "text": caps, "filter": " "})).unwrap().get("matched").is_none());
}

/// A search reads again the documents a listing read, so it doesn't say again what reading them
/// says: Add From Service searches once per pause in typing, and the notices would repeat (#1577).
/// The answer's notes still say it.
#[test]
fn a_search_does_not_repeat_the_listings_notice() {
// A landing page whose collections link doesn't get the address's token: one withheld.
let url = "https://collections.invalid/root?f=json&token=RGSYNTH_search";
let landing = r#"{"links":[{"rel":"data","type":"application/json","href":"/catalog?f=json"}]}"#;
let collections: Vec<Value> = (1..=3)
.map(|i| json!({"id": format!("c{i}"), "title": format!("C {i}"), "links": [{"rel": "items", "type": "application/geo+json", "href": format!("https://collections.invalid/collections/c{i}/items")}]}))
.collect();
let documents = json!([{"url": "https://collections.invalid/catalog?f=json", "text": json!({"collections": collections}).to_string()}]);
let mut s = Session::new(Project::default());
let withheld = |s: &Session, mark| s.notices_since(mark).iter().filter(|n| n.text.contains("didn't get its keys")).count();
let said = |v: &Value| v["notes"].as_array().unwrap().iter().any(|n| n.as_str().unwrap().contains("didn't get its keys"));
let mark = s.notice_mark();
let p = json!({"url": url, "type": "ogcapi", "text": landing, "documents": documents});
let listing = s.run("service.layers", &p).unwrap();
assert_eq!(withheld(&s, mark), 1, "{:?}", s.notices_since(mark));
assert!(said(&listing), "{listing}");
for filter in ["c 1", "c 2", "c"] {
let mut p = p.clone();
p["filter"] = json!(filter);
let found = s.run("service.layers", &p).unwrap();
assert!(found["matched"].as_u64().unwrap() > 0 && said(&found), "{found}");
}
assert_eq!(withheld(&s, mark), 1, "{:?}", s.notices_since(mark));
// A word given many times is looked for once, with the same answer.
let mut many = p.clone();
many["filter"] = json!(vec!["c"; 50_000].join(" ") + " 2");
let found = s.run("service.layers", &many).unwrap();
assert_eq!(found["matched"], 1, "{found}");
// A filter that isn't text is refused, not ignored.
for filter in [json!(["c"]), json!(5)] {
let mut p = p.clone();
p["filter"] = filter;
assert!(s.run("service.layers", &p).unwrap_err().contains("filter must be text"));
}
}

/// A layer past what RustGIS reads of a document is refused for that reason, not as absent.
#[test]
fn a_layer_past_the_read_limit_is_refused_saying_so() {
Expand Down
Loading
Loading