Repository navigation
fix(connections): test XYZ templates with a sample tile (#2356) #2399
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,6 +6,7 @@ use std::collections::BTreeMap; | |
| use serde_json::{Value, json}; | ||
|
|
||
| use super::{b, ps}; | ||
| use crate::connections::tile_test::{self, Sample}; | ||
| use crate::connections::{Connection, DATABASE_KINDS, DESKTOP_ONLY, KINDS}; | ||
| use crate::render::rustgis_tiles::{arcgis_token, s3, stac}; | ||
| use crate::s3_keys as keys; | ||
|
|
@@ -65,7 +66,7 @@ pub fn commands() -> Vec<C> { | |
| ), | ||
| C::new("connection.forgetPassword", "Forget Saved Password", "Home › Add › Data Sources", forget_password).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Reserved)).view().params(r#"{"id": "a database connection's id or name"}"#), | ||
| C::new("connection.test", "Test Connection", "Home › Add › Data Sources", test).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Reserved)).view().params( | ||
| r#"{"id"?: "a connection's id or name", "kind"?: "postgres"|"mysql"|"sqlserver"|"service"|"wms"|"wmts"|"wfs"|"wcs"|"ogcapi"|"arcgis"|"csw"|"stac"|"s3", "name"?: text, "group"?: text, "address"?: text, "options"?: "an unsaved connection, as connection.new (when no id)", "auth"?: text, "user"?: text, "keyName"?: "as connection.new (when no id)", "password"?: "for a named database, authentication refusal restores prior session credentials; a later listing failure keeps the new password; never persisted. A saved web service's: as connection.login (logs in first)", "apiKey"?: "a saved web service's, as connection.login (logs in first)", "allowInsecure"?: "as connection.login", "accessKeyId"?: text, "secretAccessKey"?: text, "sessionToken"?: "as connection.login (logs in first)", "probe"?: "true: only the address the test reads, {fetch} (null for databases, which their driver tests)", "answer"?: "non-database kinds only: what that address returned, when the app fetched it (no network request then); databases ignore answer and test through their driver"}"#, | ||
| r#"{"id"?: "a connection's id or name", "kind"?: "postgres"|"mysql"|"sqlserver"|"service"|"wms"|"wmts"|"wfs"|"wcs"|"ogcapi"|"arcgis"|"csw"|"stac"|"s3", "name"?: text, "group"?: text, "address"?: text, "options"?: "an unsaved connection, as connection.new (when no id)", "auth"?: text, "user"?: text, "keyName"?: "as connection.new (when no id)", "password"?: "for a named database, authentication refusal restores prior session credentials; a later listing failure keeps the new password; never persisted. A saved web service's: as connection.login (logs in first)", "apiKey"?: "a saved web service's, as connection.login (logs in first)", "allowInsecure"?: "as connection.login", "accessKeyId"?: text, "secretAccessKey"?: text, "sessionToken"?: "as connection.login (logs in first)", "probe"?: "true: only the address the test reads, {fetch} (null for databases), with tile: {zoom, vector} for an XYZ sample", "answer"?: "fetched service or catalogue text, parsed without a request; not used for XYZ tiles or databases"} (a direct XYZ test returns result: {kind: \"xyz\", tile: [z,x,y], status: \"received\" or \"missing\"}; missing is not a pass)"#, | ||
| ), | ||
| C::new("connection.browse", "Browse Connection", "Home › Add › Data Sources", browse::browse).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Reserved).with(crate::policy::Effect::Files)).view().params( | ||
| r#"{"id": "a connection's id or name", "node"?: "a child's node, as a listing gives it (the connection's top level when absent)", "probe"?: "true: only the address this level reads, {fetch} (null: read without a fetch, or by the database driver); an S3 connection logs in first, as connection.test does", "answer"?: "what that address returned, when the app fetched it", "documents"?: [{"url": "a further document a listing asked for (next)", "text": "its answer"}]} → {connection, node, children: [{key, name, what, detail?, node? (browse it), children? (listed already), add? {command, params}: the command that adds it}], truncated?, matched?, notes?, service? {url, type}, next? (fetch it, then ask again with answer and documents)}. Database roots list schemas; open a schema node to list its tables."#, | ||
|
|
@@ -271,6 +272,9 @@ fn shown_stripped(s: &mut Session, c: &Connection, stripped: bool, given: &str, | |
| out["strippedParameters"] = json!(names); | ||
| } | ||
| } | ||
| if web::is_service(c) && tile_test::is_template(&c.address) && tile_test::has_path_key(&c.address) { | ||
| s.notify_text(NoticeText::format(tile_test::PATH_KEY_WARNING, []), false); | ||
| } | ||
| out | ||
| } | ||
|
|
||
|
|
@@ -1217,19 +1221,8 @@ fn answered(s: &mut Session, c: &Connection, text: &str) -> CmdResult { | |
|
|
||
| fn test(s: &mut Session, p: &Value) -> CmdResult { | ||
| let c = tested(s, p)?; | ||
| let service = web::is_service(&c); | ||
| // XYZ addresses name individual tiles, not a document the service test can read. | ||
| if service { | ||
| let address = c.address.split('#').next().unwrap_or_default(); | ||
| if (address.contains("{z}") || address.contains("{zoom}")) | ||
| && address.contains("{x}") | ||
| && (address.contains("{y}") || address.contains("{-y}")) | ||
| { | ||
| return Err(crate::connections::XYZ_TEST.into()); | ||
| } | ||
| } | ||
| // An unsaved web service reads anonymously, whatever saved connection shares its origin. | ||
| let anonymous = service && c.id.is_empty(); | ||
| let anonymous = web::is_service(&c) && c.id.is_empty(); | ||
| web::anonymously(s, anonymous, |s| test_connection(s, p, c)).unwrap_or_else(|_| Err("connection.test failed unexpectedly".into())) | ||
| } | ||
|
|
||
|
|
@@ -1266,6 +1259,11 @@ fn test_connection(s: &mut Session, p: &Value, c: Connection) -> CmdResult { | |
| if !c.id.is_empty() { | ||
| out["connection"] = json!(c.id); | ||
| } | ||
| if web::is_service(&c) | ||
| && let Some(sample) = Sample::for_address(&c.address) | ||
| { | ||
| return test_xyz(s, p, &c, out, sample); | ||
| } | ||
| if b(p, "probe").unwrap_or(false) { | ||
| out["fetch"] = json!(probe_url(s, &c)?); | ||
| return Ok(out); | ||
|
|
@@ -1310,3 +1308,33 @@ fn test_connection(s: &mut Session, p: &Value, c: Connection) -> CmdResult { | |
| out["result"] = result; | ||
| Ok(out) | ||
| } | ||
|
|
||
| fn test_xyz(s: &mut Session, p: &Value, c: &Connection, mut out: Value, sample: Sample) -> CmdResult { | ||
| use crate::render::rustgis_tiles::logins; | ||
| let url = sample.address(&c.address)?; | ||
| if !c.auth.is_empty() && logins::origin(&url) != crate::connections::login_origin(c) { | ||
| return Err(tile_test::OTHER_ORIGIN.into()); | ||
| } | ||
| let url = web::request(s, (!c.id.is_empty()).then_some(c.id.as_str()), &url)?; | ||
| if b(p, "probe").unwrap_or(false) { | ||
| out["fetch"] = json!(url); | ||
| out["tile"] = json!(sample); | ||
| return Ok(out); | ||
| } | ||
| if p.get("answer").is_some() { | ||
| return Err(tile_test::TEXT_ANSWER.into()); | ||
| } | ||
| #[cfg(target_arch = "wasm32")] | ||
| return Err("An XYZ sample tile must be fetched by the app.".into()); | ||
| #[cfg(not(target_arch = "wasm32"))] | ||
| { | ||
| use crate::render::rustgis_tiles::cache; | ||
| let answer = cache::download_blocking_omitting_within(&url, cache::MAX_BYTES, cache::BLOCKING_TIMEOUT, &[404]); | ||
| let status = match sample.check(answer)? { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Reject vector sample layers that lack a name or version. A Require the sample check to reject a layer that lacks a name or a version. Keep a correctly formed layer with a name and zero features valid. The check can sit in the sample path or in the shared decoder. The decoder's
|
||
| tile_test::Outcome::Received => "received", | ||
| tile_test::Outcome::Missing => "missing", | ||
| }; | ||
| out["result"] = json!({"kind":"xyz","tile":[sample.zoom,0,0],"status":status}); | ||
| Ok(out) | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| //! One bounded tile download for Test Connection, shared by the engine and the app. | ||
|
|
||
| use crate::render::rustgis_tiles::{address, cache, template}; | ||
| use rustgis_map::tiles::TileKind; | ||
| use serde::{Deserialize, Serialize}; | ||
|
|
||
| pub const NOT_TILE: &str = "The server answered, but the response is not a usable tile."; | ||
| pub const UNEXPANDED: &str = "The tile URL has an unsupported placeholder."; | ||
| pub const OTHER_ORIGIN: &str = "The sample tile changes the connection's origin. Use a concrete hostname to test with a saved login."; | ||
| pub const TEXT_ANSWER: &str = "An XYZ test needs a tile response, not a text answer."; | ||
| pub const PATH_KEY_WARNING: &str = "An API key in the URL path is saved as typed, not in the credential store."; | ||
|
|
||
| /// An XYZ template the connection test recognises, not just a URL with a subdomain placeholder. | ||
| pub fn is_template(url: &str) -> bool { | ||
| template::xyz_sample_zoom(url).is_some() | ||
| } | ||
|
|
||
| /// A nonempty `;key=…` path parameter, as used by Koordinates. Query keys have their own handling. | ||
| pub fn has_path_key(url: &str) -> bool { | ||
| let Some((_, at)) = url.split_once("://") else { return false }; | ||
| let Some((_, path)) = at.split(['?', '#']).next().unwrap_or_default().split_once('/') else { return false }; | ||
| path.split('/').any(|segment| { | ||
| segment.split(';').skip(1).any(|param| param.split_once('=').is_some_and(|(k, v)| k.eq_ignore_ascii_case("key") && !v.is_empty())) | ||
| }) | ||
| } | ||
|
|
||
| #[derive(Clone, Copy, Deserialize, Serialize)] | ||
| pub struct Sample { | ||
| pub zoom: u8, | ||
| pub vector: bool, | ||
| } | ||
|
|
||
| impl Sample { | ||
| pub(crate) fn for_address(url: &str) -> Option<Self> { | ||
| Some(Self { zoom: template::xyz_sample_zoom(url)?, vector: TileKind::guess(url) == TileKind::Vector }) | ||
| } | ||
|
|
||
| pub(crate) fn address(self, url: &str) -> Result<String, String> { | ||
| let template = url.split('#').next().unwrap_or_default(); | ||
| let url = template::expand(template, template::TileId { z: self.zoom, x: 0, y: 0 }, 256, false, &[]); | ||
| if url.contains(['{', '}']) { | ||
| return Err(UNEXPANDED.into()); | ||
| } | ||
| address::check(&url, Some(template))?; | ||
| Ok(url) | ||
| } | ||
|
|
||
| /// A successful HTTP response is a pass only if the data actually decodes as a tile. | ||
| /// The transport supplies an empty body for 204; neither that nor a 404 is a pass. | ||
| pub fn check(self, response: Result<Vec<u8>, String>) -> Result<Outcome, String> { | ||
| match response { | ||
| Ok(bytes) if bytes.is_empty() => Ok(Outcome::Missing), | ||
| Err(e) if matches!(cache::http_status(&e), Some(204 | 404)) => Ok(Outcome::Missing), | ||
| Ok(bytes) => { | ||
| let want = if self.vector { cache::Want::Vector } else { cache::Want::Image }; | ||
| cache::decode(&bytes, want).map_err(|_| NOT_TILE.to_string())?; | ||
| Ok(Outcome::Received) | ||
| } | ||
| Err(e) => Err(e), | ||
| } | ||
| } | ||
| } | ||
|
|
||
| #[derive(Clone, Copy)] | ||
| pub enum Outcome { | ||
| Received, | ||
| Missing, | ||
| } |
| Original file line number | Diff line number | Diff line change | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -10,6 +10,9 @@ use serde_json::{Map, Value, json}; | |||||||||
|
|
||||||||||
| use crate::sources::{self, Sources}; | ||||||||||
|
|
||||||||||
| #[path = "connection_tile_test.rs"] | ||||||||||
| pub mod tile_test; | ||||||||||
|
|
||||||||||
| /// The desktop preference saved connections are kept under. | ||||||||||
| pub const PREF: &str = "connections"; | ||||||||||
| /// The version of that preference this RustGIS reads and writes. One from a newer RustGIS isn't | ||||||||||
|
|
@@ -58,7 +61,6 @@ pub const SERVICE_AUTH: &[&str] = &["none", "basic", "header", "query", "arcgis" | |||||||||
| pub const WEB: &str = "web"; | ||||||||||
| pub const S3_CREDENTIALS: &[&str] = &["keys", "profile", "environment", "none"]; | ||||||||||
| pub const DESKTOP_ONLY: &str = "database connections are used by the desktop app"; | ||||||||||
| pub const XYZ_TEST: &str = "This is an XYZ tile URL template, not a service endpoint. Use Add Tile Layer to add it to the map."; | ||||||||||
|
|
||||||||||
| /// Whether a serialized flag is off (so it is left out). | ||||||||||
| fn is_false(value: &bool) -> bool { | ||||||||||
|
|
@@ -509,7 +511,15 @@ pub fn clean(mut c: Connection) -> Result<(Connection, bool), String> { | |||||||||
| let stripped = clean_login(&mut c)? || stripped; | ||||||||||
| c.name = name_text(&c.name); | ||||||||||
| if c.name.is_empty() { | ||||||||||
| c.name = if c.kind == "s3" && c.address.is_empty() { "AWS S3".into() } else { sources::default_name(&c.address) }; | ||||||||||
| c.name = if c.kind == "s3" && c.address.is_empty() { | ||||||||||
| "AWS S3".into() | ||||||||||
| } else if SERVICE_KINDS.contains(&c.kind.as_str()) && tile_test::is_template(&c.address) { | ||||||||||
| login_origin(&c) | ||||||||||
| .and_then(|origin| origin.split_once("://").map(|(_, host)| host.trim_start_matches("{s}.").chars().take(80).collect())) | ||||||||||
|
Comment on lines
+517
to
+518
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Bug (confidence: high): the default name for a subdomain XYZ template is never the host. Suggested fix: strip the subdomain placeholder from the address before reading the origin.
Suggested change
Please also add a case to |
||||||||||
| .unwrap_or_else(|| "XYZ tiles".into()) | ||||||||||
| } else { | ||||||||||
| sources::default_name(&c.address) | ||||||||||
| }; | ||||||||||
| } | ||||||||||
| c.group = name_text(&c.group); | ||||||||||
| if c.kind != "s3" { | ||||||||||
|
|
||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -31876,28 +31876,90 @@ fn connections_s3_rename_cannot_inherit_a_destination_session() { | |
| } | ||
|
|
||
| #[test] | ||
| fn connection_test_rejects_xyz_templates() { | ||
| let mut s = Session::new(Project::default()); | ||
| for (kind, address) in [ | ||
| ("service", "http://mt0.google.com/vt/lyrs=s&hl=en&x={x}&y={y}&z={z}"), | ||
| ("wmts", "https://tiles.example/{z}/{x}/{y}.png"), | ||
| ("wcs", "https://tiles.example/{z}/{x}/{y}.png"), | ||
| ("ogcapi", "https://tiles.example/tile.png?z={z}&x={x}&y={y}"), | ||
| ("service", "https://tiles.example/{zoom}/{x}/{-y}.png"), | ||
| fn connection_test_expands_xyz_templates() { | ||
| let mut s = Session::new(Project::default()); | ||
| for (kind, address, expected, zoom) in [ | ||
| ("service", "https://tiles.example/vt/lyrs=s&x={x}&y={y}&z={z}", "https://tiles.example/vt/lyrs=s&x=0&y=0&z=0", 0), | ||
| ("wmts", "https://tiles.example/{z}/{x}/{y}.png", "https://tiles.example/0/0/0.png", 0), | ||
| ("ogcapi", "https://tiles.example/tile.png?z={z}&x={x}&y={y}", "https://tiles.example/tile.png?z=0&x=0&y=0", 0), | ||
| ("service", "https://{s}.tiles.example/{zoom}/{x}/{-y}.png", "https://a.tiles.example/0/0/0.png", 0), | ||
| ("service", "https://tiles.example/{q}.jpg", "https://tiles.example/0.jpg", 1), | ||
| ("service", "https://tiles.example/tile.jpg?tile={quadkey}", "https://tiles.example/tile.jpg?tile=0", 1), | ||
| ] { | ||
| let error = s.run("connection.test", &json!({"kind":kind,"address":address,"probe":true})).unwrap_err(); | ||
| assert!(error.contains("XYZ tile URL template") && error.contains("Add Tile Layer"), "{kind}: {error}"); | ||
| let result = s.run("connection.test", &json!({"kind":kind,"address":address,"probe":true})).unwrap(); | ||
| assert_eq!(result["fetch"], expected, "{kind}"); | ||
| assert_eq!(result["tile"]["zoom"], zoom); | ||
| } | ||
|
|
||
| let saved = | ||
| s.run("connection.new", &json!({"kind":"service","name":"Tiles","address":"https://tiles.example/{z}/{x}/{y}.png"})).unwrap(); | ||
| let error = s.run("connection.test", &json!({"id":saved["id"],"answer":"{}"})).unwrap_err(); | ||
| assert!(error.contains("Add Tile Layer"), "{error}"); | ||
|
|
||
| let ordinary = json!({"kind":"ogcapi","address":"https://tiles.example/collections#{z}/{x}/{y}","probe":true}); | ||
| assert_eq!(s.run("connection.test", &ordinary).unwrap()["fetch"], "https://tiles.example/collections?f=json"); | ||
| } | ||
|
|
||
| #[test] | ||
| fn connection_xyz_default_name_and_path_key_notice() { | ||
| let mut s = Session::new(Project::default()); | ||
| let mark = s.notice_mark(); | ||
| let url = "https://tiles.example/{z}/{x}/{y}.png"; | ||
| let added = s.run("connection.new", &json!({"kind":"service","address":url})).unwrap(); | ||
| assert_eq!(added["name"], "tiles.example"); | ||
| assert!(s.notices_since(mark).is_empty()); | ||
| let with_key = "https://tiles.example/services;key=fixture/tiles/{z}/{x}/{y}.png"; | ||
| let added = s.run("connection.new", &json!({"kind":"service","address":with_key,"name":"Relief"})).unwrap(); | ||
| assert_eq!(added["name"], "Relief"); | ||
| assert_eq!(added["address"], with_key); | ||
| assert!(s.notices_since(mark).iter().any(|n| n.text.contains("URL path is saved as typed") && !n.text.contains("fixture"))); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Check every save notice for the path key. The
|
||
| } | ||
|
|
||
| #[cfg(not(target_arch = "wasm32"))] | ||
| #[test] | ||
| fn connection_test_fetches_one_xyz_tile_with_the_selected_login() { | ||
| use std::io::{BufRead, BufReader, Write}; | ||
| use std::net::TcpListener; | ||
| use std::time::Duration; | ||
|
|
||
| let png = tiny_skia::Pixmap::new(1, 1).unwrap().encode_png().unwrap(); | ||
| for (status, body, expected) in [ | ||
| ("200 OK", png.as_slice(), Ok("received")), | ||
| ("204 No Content", &[][..], Ok("missing")), | ||
| ("404 Not Found", &[][..], Ok("missing")), | ||
| ("401 Unauthorized", &[][..], Err("http status: 401")), | ||
| ("200 OK", b"<html>Login required</html>", Err("not a usable tile")), | ||
| ] { | ||
| let listener = TcpListener::bind("127.0.0.1:0").unwrap(); | ||
| let port = listener.local_addr().unwrap().port(); | ||
| let body = body.to_vec(); | ||
| let server = std::thread::spawn(move || { | ||
| let (stream, _) = listener.accept().unwrap(); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Bound the fixture’s wait for the first connection. If
|
||
| stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); | ||
| let mut reader = BufReader::new(stream); | ||
| let mut request = String::new(); | ||
| reader.read_line(&mut request).unwrap(); | ||
| let mut line = String::new(); | ||
| while reader.read_line(&mut line).unwrap() != 0 && line != "\r\n" { | ||
| line.clear(); | ||
| } | ||
| let mut stream = reader.into_inner(); | ||
| write!(stream, "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).unwrap(); | ||
| stream.write_all(&body).unwrap(); | ||
| request | ||
| }); | ||
| let mut s = Session::new(Project::default()); | ||
| let url = format!("http://127.0.0.1:{port}/services;key=xyz-path-fixture/{{z}}/{{x}}/{{y}}.png"); | ||
| let c = s.run("connection.new", &json!({"kind":"service","address":url,"auth":"query","keyName":"rg_xyz_key"})).unwrap(); | ||
| let params = json!({"id":c["id"],"apiKey":"sample+key","allowInsecure":true}); | ||
| let result = s.run("connection.test", ¶ms); | ||
| match expected { | ||
| Ok(expected) => assert_eq!(result.unwrap()["result"], json!({"kind":"xyz","tile":[0,0,0],"status":expected})), | ||
| Err(expected) => { | ||
| let error = result.unwrap_err(); | ||
| assert!(error.contains(expected) && !error.contains("sample") && !error.contains("xyz-path-fixture"), "{error}"); | ||
| let history = s.diagnostics.report(rustgis_data::diagnostics::Filter::All); | ||
| assert!(history.contains(expected) && !history.contains("xyz-path-fixture"), "{history}"); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Check diagnostics for the query credential. The returned-error assertion checks for the query credential, but the diagnostics assertion checks only for the path key. A redacted 401 error can pass this test even if diagnostics contain
|
||
| } | ||
| } | ||
| assert_eq!(server.join().unwrap(), "GET /services;key=xyz-path-fixture/0/0/0.png?rg_xyz_key=sample%2Bkey HTTP/1.1\r\n"); | ||
| } | ||
| } | ||
|
|
||
| #[test] | ||
| fn connection_test_probes_and_answers() { | ||
| #[cfg(not(target_arch = "wasm32"))] | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Keep S3 listing XML in the
answercontract.An S3 caller can still pass a fetched bucket listing as
answer. The command parses that XML throughs3::parse_listwithout another download, but the revised help describes only service or catalogue text. List S3 XML here and in the user guide. Keep the XYZ exclusion. (raw.githubusercontent.com)