diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index c5b0b6e043..e040e6e5e0 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -43,7 +43,7 @@ pub fn commands() -> Vec { r#"{"url": "https://…/data.geojson|.csv|.parquet|.fgb|.gpkg|.zip|.shp (its .dbf fetched beside it, and its .shx, .prj and .cpg when there)|.kml|.kmz|.gpx|.tif|.jp2 (JPEG 2000: desktop, through GDAL) (the format from the address, else the contents) | a .pmtiles archive", "name"?: text, "layer"?: "one layer of a zip, KML or GPX", "zoom"?: "bool: true frames everything added; false keeps the view, even on an empty map; omitted, the first content on an empty map is framed (a GeoParquet, FlatGeobuf or cloud-optimised GeoTIFF read in place likewise; a PMTiles or MBTiles archive ignores zoom and is framed on its bounds unless it covers the whole world)", "x"?, "y"?, "wkt"?, "crs"?, "delimiter"?, "paperSpace"?, "stream"?, "as"? (as layer.addData), "whole"?: "bool: download a cloud-optimised GeoTIFF whole (as a raster layer) instead of reading it in place (an address without a file name is read in place when its first bytes are a cloud-optimised GeoTIFF)", "cog"?: "bool: the address is a cloud-optimised GeoTIFF whatever its name says (the app's probe found it so)", "bbox"?: "[west, south, east, north] in degrees | "view": a cloud-optimised GeoTIFF's window as a raster layer (desktop and command line), read by range at full resolution, in the file's coordinate system, with its colour table and class names; at most 64 million values (all bands)", "cellSize"?: "with bbox: the coarsest overview whose cells are at most this size is read (the file's units; metres for longitude/latitude, a degree taken as 111.32 km)", "cube"?: "bool: a data cube's file: a tile layer that follows the time slider (stac.addCube adds them), kept one when the project opens again", "start"?, "end"?, "unit"?, "interval"?, "dates"?: "a GeoTIFF address with date placeholders ({date:YYYY}, {YYYY}{MM}{DD}), one file per date: the dates it spans and steps through (as time.set), turning the time slider on", "connection"?: "the id of the saved web-service connection it comes from (as service.add)", "anonymous"?: bool (internal: a plugin-origin download must never adopt a saved login), "request"?: "number (internal: the download a plugin command left to the app, #1202)"}"#, ), C::new("layer.addDownload", "Add Downloaded Data", "Home › Add", add_download).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Reserved).with(crate::policy::Effect::Files)) - .params(r#"{"id": a download the app finished (Home › Add › Add Data › From URL), "url": its address, "name"?: text, "layer"?: text, "zoom"?: bool, "x"?, "y"?, "wkt"?, "crs"?, "delimiter"?, "paperSpace"?, "stream"?, "as"? (as layer.addData), "service"?: bool (a service's features: their axes (when they came as GeoJSON) and paging are checked, and the address is kept as the layer's source), "bounds"?: [west, south, east, north] (with service: the layer's longitude/latitude bounds from its capabilities, which its axes are checked against; without bounds they are left as they come; ignored without service), "coverage"?: bool (a WCS coverage: a GeoTIFF, else the service's error, and its request kept as the layer's source), "geolibreProject"?: bool (internal: downloaded project JSON), "geolibreLayer"?: {"token", "record"} (internal: reserved imported layer and its saved style), "geolibreZarr"?: bool (internal: an imported Zarr slice as GeoTIFF), "connection"?: "as layer.addUrl", "anonymous"?: bool (internal: preserves a plugin-origin download's anonymous reads), "request"?: number (internal: the download a plugin command left to the app, #1202)}"#), + .params(r#"{"id": a download the app finished (Home › Add › Add Data › From URL), "url": its address, "name"?: text, "layer"?: text, "zoom"?: bool, "x"?, "y"?, "wkt"?, "crs"?, "delimiter"?, "paperSpace"?, "stream"?, "as"? (as layer.addData), "service"?: bool (a service's features: their axes (when they came as GeoJSON) and paging are checked, and the address is kept as the layer's source), "bounds"?: [west, south, east, north] (with service: the layer's longitude/latitude bounds from its capabilities, which its axes are checked against; without bounds they are left as they come; ignored without service), "coverage"?: bool (a WCS coverage: a GeoTIFF, else the service's error, and its request kept as the layer's source), "geolibreProject"?: bool (internal: downloaded project JSON), "geolibreLayer"?: {"token", "record"} (internal: reserved imported layer and its saved style), "geolibreZarr"?: bool (internal: an imported Zarr slice as GeoTIFF), "connection"?: "as layer.addUrl", "anonymous"?: bool (internal: preserves a plugin-origin download's anonymous reads), "request"?: number (internal: the download a plugin command left to the app, #1202), "window"?: {"bbox", "cellSize"?, "cog"?} (internal: what a cloud-optimised GeoTIFF's window the app read off the frame asked for, for a task to read it again, #1330)}"#), C::new("layer.addEncodedPolyline", "Add Encoded Polyline", "Home › Add", add_encoded_polyline).roles(&[]).effects(crate::policy::Effects::NONE).params( r#"{"polyline": "encoded polyline text" | [texts] (one line each), "precision"?: 5 (Google) | 6 (OSRM, Valhalla), "name"?: text}"#, ), @@ -3123,8 +3123,8 @@ pub(crate) fn add_url(s: &mut Session, p: &Value) -> CmdResult { fn add_url_as_given(s: &mut Session, p: &Value) -> CmdResult { let bound = crate::service_logins::request_params(s, p)?; let p = bound.as_ref(); - if ["geolibreProject", "geolibreLayer", "geolibreZarr"].iter().any(|key| p.get(key).is_some()) && !s.import_completion(p) { - return Err("GeoLibre download state is internal; use project.importGeolibre".into()); + if !s.import_completion(p) { + no_geolibre_state(p)?; } let url = ps(p, "url").map(str::trim).filter(|u| !u.is_empty()).ok_or("url is required")?; // A plugin command run again once the download it left to the app was added (#1202): what @@ -3164,10 +3164,7 @@ fn add_url_as_given(s: &mut Session, p: &Value) -> CmdResult { } // An address read through a saved S3 connection is judged by the address it wraps; the whole // address stays what is fetched and kept. - let lower = rustgis_data::s3_address::inner(crate::render::rustgis_tiles::logins::inner(url)).to_ascii_lowercase(); - if !(lower.starts_with("https://") || lower.starts_with("http://") || lower.starts_with("s3://")) { - return Err(format!("{url}: give an http://, https:// or s3:// address (Add Data opens files)")); - } + let lower = web_address(url)?; // A scan (`"as"`) keeps the pixels and none of the coordinates: the file is downloaded whole and // read as one (`add_named_bytes` checks it is an image), never read in place. let scan = super::raster::as_scan(p)?; @@ -3177,26 +3174,7 @@ fn add_url_as_given(s: &mut Session, p: &Value) -> CmdResult { return Err(format!("{url}: \"cellSize\" goes with \"bbox\": give a box, or leave it out")); } if window { - if scan || b(p, "whole") == Some(true) || rustgis_data::time::is_template(url) { - return Err(format!( - "{url}: \"bbox\" reads a window of one cloud-optimised GeoTIFF with its coordinates: leave out \"as\" and \"whole\", and give a single file's address" - )); - } - // An address naming another kind of file is refused before anything is fetched (one - // with no extension, or a GeoTIFF's, is read; `"cog": true` says it is one). - if b(p, "cog") != Some(true) - && let Some(ext) = - address_extension(rustgis_data::s3_address::inner(url)).filter(|e| !matches!(e.as_str(), "tif" | "tiff" | "geotiff")) - { - return Err(format!( - "{url}: \"bbox\" reads a window of a cloud-optimised GeoTIFF, and this address names a .{ext} file: leave out \"bbox\" (or give \"cog\": true for a GeoTIFF by another name)" - )); - } - // What a window doesn't read is refused, not dropped: it keeps the file's coordinate system. - let unread: Vec<&str> = WINDOW_UNREAD.iter().copied().filter(|k| p.get(*k).is_some_and(|v| !v.is_null())).collect(); - if !unread.is_empty() { - return Err(format!("{url}: a window (\"bbox\") doesn't read {}: leave it out", unread.join(", "))); - } + window_checks(p, url, scan)?; // A plugin's command run by the app leaves what would wait to the app, off the frame; a // window waits for its header and tiles, which the app doesn't take over yet. #[cfg(not(target_arch = "wasm32"))] @@ -3852,13 +3830,132 @@ pub(crate) fn address_extension(url: &str) -> Option { } /// `layer.addUrl` keys for other kinds of address, which a window (`"bbox"`) doesn't read. -const WINDOW_UNREAD: [&str; 13] = - ["layer", "x", "y", "wkt", "crs", "delimiter", "paperSpace", "stream", "start", "end", "unit", "interval", "dates"]; +const WINDOW_UNREAD: [&str; 16] = [ + "layer", + "x", + "y", + "wkt", + "crs", + "delimiter", + "paperSpace", + "stream", + "start", + "end", + "unit", + "interval", + "dates", + "service", + "coverage", + "bounds", +]; /// Metres in a degree, to compare a cell size in metres with a longitude/latitude file's pixels. #[cfg(not(target_arch = "wasm32"))] const METRES_PER_DEGREE: f64 = 111_320.0; +/// A web address's lowercase form (the address it wraps when read through a saved connection), or +/// why it isn't one. +fn web_address(url: &str) -> Result { + let lower = rustgis_data::s3_address::inner(crate::render::rustgis_tiles::logins::inner(url)).to_ascii_lowercase(); + if !(lower.starts_with("https://") || lower.starts_with("http://") || lower.starts_with("s3://")) { + return Err(format!("{url}: give an http://, https:// or s3:// address (Add Data opens files)")); + } + Ok(lower) +} + +/// Refuses a GeoLibre project's download state, which only the app's import sets. +fn no_geolibre_state(p: &Value) -> Result<(), String> { + if ["geolibreProject", "geolibreLayer", "geolibreZarr"].iter().any(|key| p.get(key).is_some()) { + return Err("GeoLibre download state is internal; use project.importGeolibre".into()); + } + Ok(()) +} + +/// Refuses what can't be a window of `url`, before anything is fetched. +fn window_checks(p: &Value, url: &str, scan: bool) -> Result<(), String> { + no_geolibre_state(p)?; + if scan || b(p, "whole") == Some(true) || rustgis_data::time::is_template(url) { + return Err(format!( + "{url}: \"bbox\" reads a window of one cloud-optimised GeoTIFF with its coordinates: leave out \"as\" and \"whole\", and give a single file's address" + )); + } + // An address naming another kind of file is refused before anything is fetched (one + // with no extension, or a GeoTIFF's, is read; `"cog": true` says it is one). + if b(p, "cog") != Some(true) + && let Some(ext) = + address_extension(rustgis_data::s3_address::inner(url)).filter(|e| !matches!(e.as_str(), "tif" | "tiff" | "geotiff")) + { + return Err(format!( + "{url}: \"bbox\" reads a window of a cloud-optimised GeoTIFF, and this address names a .{ext} file: leave out \"bbox\" (or give \"cog\": true for a GeoTIFF by another name)" + )); + } + // What a window doesn't read is refused, not dropped: it keeps the file's coordinate system. + // A data cube's file is a tile layer that follows the time slider, and a window is a raster: + // `"cube": false` asks for nothing. + let cube = (b(p, "cube") == Some(true)).then_some("cube"); + let unread: Vec<&str> = WINDOW_UNREAD.iter().copied().filter(|k| p.get(*k).is_some_and(|v| !v.is_null())).chain(cube).collect(); + if !unread.is_empty() { + return Err(format!("{url}: a window (\"bbox\") doesn't read {}: leave it out", unread.join(", "))); + } + Ok(()) +} + +/// A window asked for, with its box settled (`"view"` is the view when it was asked for): what +/// [`read_cog_window`] needs, on any thread. +#[cfg(not(target_arch = "wasm32"))] +pub struct WindowAsked { + url: String, + bbox: [f64; 4], + cell: Option, + name: Option, +} + +/// A window read by [`read_cog_window`], waiting to be added to the project: its raster made and +/// its GeoTIFF written, so that adding it holds no frame for long. +#[cfg(not(target_arch = "wasm32"))] +pub struct CogWindow { + name: String, + layer: rustgis_map::RasterLayer, + /// What to say of it, once the layer has its name. + notes: Vec, + columns: [u32; 2], + rows: [u32; 2], + level: usize, + cell_size: f64, +} + +/// `p`'s box and cell size, read on the frame, which holds the view. +#[cfg(not(target_arch = "wasm32"))] +fn window_settled(s: &Session, p: &Value, url: &str) -> Result { + let named = |e: String| if e.starts_with(url) { e } else { format!("{url}: {e}") }; + let bbox = window_box(s, p).map_err(named)?; + let cell = match p.get("cellSize").filter(|v| !v.is_null()) { + None => None, + Some(v) => Some(v.as_f64().filter(|c| c.is_finite() && *c > 0.0).ok_or_else(|| named("cellSize is a size above 0".into()))?), + }; + let name = ps(p, "name").map(str::trim).filter(|n| !n.is_empty()).map(String::from); + Ok(WindowAsked { url: url.to_string(), bbox, cell, name }) +} + +/// A window the app reads off the frame: `p`, a `layer.addUrl` with `"bbox"` and its address bound +/// to its login, checked as the command checks it. Errors quote the address without its keys. +#[cfg(not(target_arch = "wasm32"))] +pub fn window_asked(s: &Session, p: &Value) -> Result { + let url = ps(p, "url").map(str::trim).filter(|u| !u.is_empty()).ok_or("url is required")?; + let checked = web_address(url) + .and_then(|_| super::raster::as_scan(p)) + .and_then(|scan| window_checks(p, url, scan)) + .and_then(|()| window_settled(s, p, url)); + checked.map_err(|e| crate::sources::redact_text(&e)) +} + +/// `layer.addUrl` with `"bbox"`, for those that wait: read here and added. +#[cfg(not(target_arch = "wasm32"))] +fn add_cog_window(s: &mut Session, p: &Value, url: &str) -> CmdResult { + let read = read_cog_window(&window_settled(s, p, url)?)?; + add_window(s, p, read) +} + /// `layer.addUrl` with `"bbox"` (opengeos/RustGIS#1204): a window of a cloud-optimised GeoTIFF, /// read by range as a raster layer. The box (longitude/latitude, or the view) is turned into the /// file's coordinate system and snapped outwards onto the pixels of the level read: the full image, @@ -3866,16 +3963,18 @@ const METRES_PER_DEGREE: f64 = 111_320.0; /// takes are read, with the reader the data cubes use. The raster keeps the file's coordinate /// system, no-data, colour table and band and class names, and is kept in the project (as a /// GeoTIFF of the window), not read again from the address. +/// +/// This waits for the file, and needs no [`Session`]: the app runs it on a thread and adds what it +/// reads when it lands (`layer.addDownload`, [`Session::download_landed_window`]; #1330). Errors +/// quote the address without its keys. #[cfg(not(target_arch = "wasm32"))] -fn add_cog_window(s: &mut Session, p: &Value, url: &str) -> CmdResult { +pub fn read_cog_window(asked: &WindowAsked) -> Result { use crate::render::rustgis_tiles::cog; use rustgis_data::raster::cog::Level; - let named = |e: String| if e.starts_with(url) { e } else { format!("{url}: {e}") }; - let [west, south, east, north] = window_box(s, p).map_err(named)?; - let cell = match p.get("cellSize").filter(|v| !v.is_null()) { - None => None, - Some(v) => Some(v.as_f64().filter(|c| c.is_finite() && *c > 0.0).ok_or_else(|| named("cellSize is a size above 0".into()))?), - }; + let url = asked.url.as_str(); + let named = |e: String| crate::sources::redact_text(&if e.starts_with(url) { e } else { format!("{url}: {e}") }); + let [west, south, east, north] = asked.bbox; + let cell = asked.cell; // The five minutes count from the header's read. let started = std::time::Instant::now(); let info = cog::info_blocking(url).map_err(named)?; @@ -4049,8 +4148,7 @@ fn add_cog_window(s: &mut Session, p: &Value, url: &str) -> CmdResult { // A colour table is one band's: a window of more bands comes in without it, and says so. let palette_lost = (layout.palette.is_some() && bands.len() != 1) .then(|| format!("its colour table is left out: the window has {} bands, a palette has one", bands.len())); - let stem = cog_name(url); - let name = ps(p, "name").map(str::trim).filter(|n| !n.is_empty()).unwrap_or(&stem).to_string(); + let name = asked.name.clone().unwrap_or_else(|| cog_name(url)); let mut r = rustgis_data::raster::Raster::new(&name, c1 - c0, r1 - r0, bands, transform, layout.crs, layout.nodata, color, layout.data_type) .map_err(|e| named(e.to_string()))?; @@ -4088,39 +4186,43 @@ fn add_cog_window(s: &mut Session, p: &Value, url: &str) -> CmdResult { }; let (width, height, size) = (r.width, r.height, size_of(lv)); let layer = rustgis_map::RasterLayer::from_bytes(r, format!("{name}.tif"), bytes); - let mut out = super::raster::add_raster_after(s, &name, layer, b(p, "zoom"))?; let of = if level == 0 { "the full image".to_string() } else { format!("overview {level} of {}", layout.levels.len() - 1) }; - let added = s.active_layer.and_then(|id| s.map().layer(id)).map(|l| l.name.clone()).unwrap_or(name); - s.notify(format!("{added}: a {width} × {height} window of {of}, cells of {size:.1}{unit}, read by range from its address"), false); - if let Some(plain) = plain { - s.notify(format!("{added}: {plain}"), false); - } + let mut notes = vec![format!("a {width} × {height} window of {of}, cells of {size:.1}{unit}, read by range from its address")]; + notes.extend(plain); if let Some(c) = finer { - s.notify(format!("{added}: cellSize {c} is finer than the file's finest cells ({size:.1}{unit}): the full image is read"), false); + notes.push(format!("cellSize {c} is finer than the file's finest cells ({size:.1}{unit}): the full image is read")); } if left_out > 0 { let kept = band_count - left_out; - s.notify( - format!("{added}: the project keeps {kept} of its {band_count} bands: {left_out} band(s) are left out once it is reopened"), - false, - ); + notes.push(format!("the project keeps {kept} of its {band_count} bands: {left_out} band(s) are left out once it is reopened")); } if !cut.is_empty() { - s.notify(format!("{added}: the box reaches past the file's {} edge(s): the window is cut to the file", cut.join(", ")), false); + notes.push(format!("the box reaches past the file's {} edge(s): the window is cut to the file", cut.join(", "))); } if unrefined > 0 { - s.notify( - format!("{added}: {unrefined} piece(s) of the box's edges not followed to a quarter of a pixel: the window may fall short of a bent edge"), - false, - ); + notes.push(format!( + "{unrefined} piece(s) of the box's edges not followed to a quarter of a pixel: the window may fall short of a bent edge" + )); } if unplaced > 0 { - s.notify( - format!("{added}: {unplaced} of the {points} points tested along the box's edges have no place in the file's coordinate system: the window is placed from the others"), - false, - ); + notes.push(format!( + "{unplaced} of the {points} points tested along the box's edges have no place in the file's coordinate system: the window is placed from the others" + )); + } + Ok(CogWindow { name, layer, notes, columns: [c0, c1], rows: [r0, r1], level, cell_size: size }) +} + +/// Adds a window read by [`read_cog_window`] and says what it read. `p` is the command adding it +/// (`layer.addUrl`, or `layer.addDownload` when the app read it off the frame). +#[cfg(not(target_arch = "wasm32"))] +fn add_window(s: &mut Session, p: &Value, read: CogWindow) -> CmdResult { + let CogWindow { name, layer, notes, columns, rows, level, cell_size } = read; + let mut out = super::raster::add_raster_after(s, &name, layer, b(p, "zoom"))?; + let added = s.active_layer.and_then(|id| s.map().layer(id)).map(|l| l.name.clone()).unwrap_or(name); + for note in notes { + s.notify(format!("{added}: {note}"), false); } - out["window"] = json!({"columns": [c0, c1], "rows": [r0, r1], "level": level, "cellSize": size}); + out["window"] = json!({"columns": columns, "rows": rows, "level": level, "cellSize": cell_size}); // `"zoom": true` zooms to the window whatever the map held, as for any other address. if b(p, "zoom") == Some(true) { out["zoomed"] = json!(zoom_to_added(s, &out)); @@ -4254,6 +4356,18 @@ fn add_download(s: &mut Session, p: &Value) -> CmdResult { if (service || coverage) && ps(p, "url").is_none_or(|url| url.trim().is_empty()) { return Err("url is required for a service or coverage download".into()); } + // A window the app read off the frame (#1330). + #[cfg(not(target_arch = "wasm32"))] + if let Some(i) = s.landed_windows.iter().position(|(d, _)| *d == id) { + let (_, read) = s.landed_windows.remove(i); + let r = add_window(s, p, read)?; + return added_download(s, p, r, false, &url); + } + // No window landed for it: a whole file isn't added in a window's place. + if p.get("window").is_some_and(Value::is_object) { + let shown = crate::sources::redact(&url); + return Err(format!("{shown}: \"window\" is for a window the app read off the frame: use layer.addUrl with \"bbox\"")); + } // A download read already (an ArcGIS layer's protocol buffer pages, joined): added as read, // with no axes to check against bounds. if let Some(i) = s.landed_reads.iter().position(|(d, _)| *d == id) { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 1e1dc255aa..928f704a34 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -607,6 +607,8 @@ struct PutAside { url_requests: Vec, policy_work: policy::downloads::Work, landed_reads: Vec<(u64, LandedRead)>, + #[cfg(not(target_arch = "wasm32"))] + landed_windows: Vec<(u64, cmd::layer::CogWindow)>, plugin_retries: Vec, url_waits: cmd::plugins::fetch::UrlWaits, /// A style of the project still loading, which mustn't move the next one's view. @@ -812,6 +814,10 @@ pub struct Session { /// Downloads that landed already read (an ArcGIS layer's protocol buffer pages, joined), /// waiting for `layer.addDownload` too. pub(crate) landed_reads: Vec<(u64, LandedRead)>, + /// Windows of cloud-optimised GeoTIFFs the app read off the frame, waiting for + /// `layer.addDownload` too. + #[cfg(not(target_arch = "wasm32"))] + pub(crate) landed_windows: Vec<(u64, cmd::layer::CogWindow)>, /// Downloads a command asked for where nothing may wait (the browser): the parameters of /// `layer.addUrl` calls, for the front end to fetch and add (see [`Session::take_url_requests`]). pub(crate) url_requests: Vec, @@ -1142,6 +1148,8 @@ impl Session { downloading: Vec::new(), downloaded: Vec::new(), landed_reads: Vec::new(), + #[cfg(not(target_arch = "wasm32"))] + landed_windows: Vec::new(), url_requests: Vec::new(), file_outputs: Vec::new(), #[cfg(test)] @@ -1232,6 +1240,18 @@ impl Session { landed } + /// Hands over a window of a cloud-optimised GeoTIFF read off the frame + /// ([`cmd::layer::read_cog_window`]), for `layer.addDownload {"id"}` as + /// [`Session::download_landed`] does. + #[cfg(not(target_arch = "wasm32"))] + pub fn download_landed_window(&mut self, id: u64, window: cmd::layer::CogWindow) -> bool { + let landed = self.landed(id); + if landed { + self.landed_windows.push((id, window)); + } + landed + } + /// Takes a download off those under way: `false` when it isn't one of them (another project /// was opened meanwhile). fn landed(&mut self, id: u64) -> bool { @@ -1250,6 +1270,12 @@ impl Session { true } + /// Whether a download is still under way for this project: `false` once another project was + /// opened. + pub fn download_under_way(&self, id: u64) -> bool { + self.downloading.contains(&id) + } + /// Forgets a download that failed. pub fn download_failed(&mut self, id: u64) { if self.download_owner(id).is_err() { @@ -1793,6 +1819,8 @@ impl Session { downloading: take(&mut self.downloading), downloaded: take(&mut self.downloaded), landed_reads: take(&mut self.landed_reads), + #[cfg(not(target_arch = "wasm32"))] + landed_windows: take(&mut self.landed_windows), url_requests: take(&mut self.url_requests), policy_work: self.take_policy_work(), plugin_retries: take(&mut self.plugin_retries), @@ -1846,6 +1874,8 @@ impl Session { downloading, downloaded, landed_reads, + #[cfg(not(target_arch = "wasm32"))] + landed_windows, url_requests, policy_work, plugin_retries, @@ -1874,6 +1904,10 @@ impl Session { (self.downloading, self.downloaded, self.url_requests) = (downloading, downloaded, url_requests); self.restore_policy_work(policy_work); self.landed_reads = landed_reads; + #[cfg(not(target_arch = "wasm32"))] + { + self.landed_windows = landed_windows; + } (self.trend_results, self.raster_results, self.raster_runs) = (trend_results, raster_results, raster_runs); #[cfg(feature = "geolibre")] { diff --git a/crates/engine/src/policy/downloads.rs b/crates/engine/src/policy/downloads.rs index f3259f1db7..f4d6fe0145 100644 --- a/crates/engine/src/policy/downloads.rs +++ b/crates/engine/src/policy/downloads.rs @@ -143,6 +143,8 @@ impl Session { } self.downloaded.retain(|(k, _)| *k != id); self.landed_reads.retain(|(k, _)| *k != id); + #[cfg(not(target_arch = "wasm32"))] + self.landed_windows.retain(|(k, _)| *k != id); let url = crate::sources::redact_text(p.get("url").and_then(Value::as_str).unwrap_or("download")); if added { Ok(Some(format!("{url}: it was added already for its plugin command, so it wasn't added again"))) @@ -357,6 +359,7 @@ impl Session { if !self.download_is_pending(id) && !self.downloaded.iter().any(|(n, _)| *n == id) && !self.landed_reads.iter().any(|(n, _)| *n == id) + && !self.landed_window(id) { return Err("this download belongs to a replaced project or was dropped".into()); } @@ -372,10 +375,22 @@ impl Session { self.downloading.retain(|n| *n != id); self.downloaded.retain(|(n, _)| *n != id); self.landed_reads.retain(|(n, _)| *n != id); + #[cfg(not(target_arch = "wasm32"))] + self.landed_windows.retain(|(n, _)| *n != id); return Err(e); } Ok(()) } + /// Whether a window read off the frame landed for download `id` (#1330). + fn landed_window(&self, id: u64) -> bool { + #[cfg(not(target_arch = "wasm32"))] + return self.landed_windows.iter().any(|(n, _)| *n == id); + #[cfg(target_arch = "wasm32")] + { + let _ = id; + false + } + } /// Owns a principal-free predicate that survives queues and browser awaits. pub fn download_still(&self, id: u64) -> Option { let claim = self.policy.work.claims.get(&id)?; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 47caf136d2..89e68f4b7c 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -29333,6 +29333,77 @@ fn a_windows_bands_are_started_within_its_wait() { assert_eq!(e, "2 of 4 bands read in 5 minutes, so the window is given up: take a smaller box or a larger cellSize"); } +/// A window's read needs no session: on a thread, as the app reads it, then added when it lands +/// (#1330). What can't be a window is refused before the read starts, and an error names the +/// address without its keys. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn a_cog_windows_read_is_apart_from_adding_it() { + use crate::cmd::layer::{read_cog_window, window_asked}; + use rustgis_data::raster::cog::example; + let (base, _) = serve_ranges(example(1024, 256)); + let mut s = world(); + let url = format!("{base}/scene.tif?token=hidden-1330"); + let p = json!({"url": url, "bbox": [10.25, 45.5, 10.5, 45.75], "name": "Window"}); + let asked = window_asked(&s, &p).unwrap(); + let layers = s.map().layers.len(); + let read = std::thread::spawn(move || read_cog_window(&asked)).join().unwrap().unwrap(); + assert_eq!(s.map().layers.len(), layers, "nothing added yet"); + let id = s.download_started(); + assert!(s.download_landed_window(id, read)); + let mark = s.notice_mark(); + let r = s.run("layer.addDownload", &json!({"id": id, "url": url, "name": "Window"})).unwrap(); + assert_eq!(r["window"], json!({"columns": [256, 512], "rows": [256, 512], "level": 0, "cellSize": 111_320.0 / 1024.0})); + assert!(s.landed_windows.is_empty()); + let said: Vec = s.notices_since(mark).iter().map(|n| n.text.clone()).collect(); + assert!(said.iter().any(|n| n.starts_with("Window: a 256 × 256 window of the full image")), "{said:?}"); + let w = s.map().layer_by_name("Window").and_then(|l| l.raster.as_ref()?.data.clone()).unwrap(); + assert_eq!((w.value(0, 0, 0), w.value(0, 10, 5)), (Some(0.0), Some(15.0))); + assert_eq!(s.map().layer(r["layer"].as_u64().unwrap() as u32).map(|l| l.name.as_str()), Some("Window")); + // A download that isn't a window isn't added as one. + let id = s.download_started(); + assert!(s.download_landed(id, b"{}".to_vec())); + let e = s.run("layer.addDownload", &json!({"id": id, "url": url, "window": {"bbox": [1, 2, 3, 4]}})).unwrap_err(); + assert!(e.contains("\"window\" is for") && !e.contains("hidden-1330"), "{e}"); + // A project opened while it was read: it isn't kept to be added. + let read = read_cog_window(&window_asked(&s, &p).unwrap()).unwrap(); + let id = s.download_started(); + s.run("project.new", &json!({})).unwrap(); + assert!(!s.download_landed_window(id, read)); + assert!(s.landed_windows.is_empty()); + // Refused as the command refuses them, without the address's keys. + let bad = |more: Value| { + let mut q = json!({"url": "https://x.org/a.tif?token=hidden-1330", "bbox": [1, 2, 3, 4]}); + q.as_object_mut().unwrap().extend(more.as_object().unwrap().clone()); + window_asked(&s, &q).err().unwrap() + }; + for (more, why) in [ + (json!({"bbox": "here"}), "bbox is"), + (json!({"cellSize": -1}), "cellSize is"), + (json!({"whole": true}), "leave out"), + (json!({"crs": "EPSG:4326"}), "doesn't read crs"), + (json!({"url": "https://x.org/a.geojson?token=hidden-1330"}), "names a .geojson file"), + (json!({"url": "C:/data/a.tif"}), "address (Add Data opens files)"), + (json!({"geolibreProject": true}), "GeoLibre download state"), + (json!({"geolibreLayer": 1}), "GeoLibre download state"), + (json!({"geolibreZarr": false}), "GeoLibre download state"), + (json!({"service": true}), "doesn't read service"), + (json!({"coverage": true}), "doesn't read coverage"), + (json!({"bounds": [1, 2, 3, 4]}), "doesn't read bounds"), + (json!({"cube": true}), "doesn't read cube"), + ] { + let e = bad(more); + assert!(e.contains(why) && !e.contains("hidden-1330"), "{why}: {e}"); + } + // `"cube": false` asks for nothing. + assert!(window_asked(&s, &json!({"url": "https://x.org/a.tif", "bbox": [1, 2, 3, 4], "cube": false})).is_ok()); + // A file that can't be read: said by the read, without the address's keys. + let (gone, _) = serve_ranges(b"not a tiff at all".to_vec()); + let asked = window_asked(&s, &json!({"url": format!("{gone}/a.tif?token=hidden-1330"), "bbox": [1, 2, 3, 4]})).unwrap(); + let e = read_cog_window(&asked).err().unwrap(); + assert!(e.contains("/a.tif") && !e.contains("hidden-1330"), "{e}"); +} + /// `layer.addUrl` with `"bbox"` (#1204): a window of a cloud-optimised GeoTIFF, read by range as a /// raster layer, at full resolution or at the overview a cell size asks for; a palette file's /// colours and class names come with it; the window is kept in a saved project, class names too. diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 7a51eb24f7..7730063bb3 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -179,6 +179,39 @@ const LAYOUT_FAVOURITE_PREF: &str = "layout_favourite_arrangement"; /// A result an asynchronous service fills in when it's done (the app polls it each frame). pub type Slot = std::sync::Arc>>; +/// One window is read off the frame at a time, however many have started: each may hold 256 MB +/// and more. A read keeps its turn until the frame has taken what it read, so a frame that stalls +/// holds one window, not all of them. Those that wait for their command on the frame don't wait +/// for this. +#[cfg(not(target_arch = "wasm32"))] +static WINDOW_TURN: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// How many windows may be under way off the frame, being read or waiting for their turn: each +/// holds a thread. +#[cfg(not(target_arch = "wasm32"))] +const MAX_WINDOW_READS: usize = 16; + +/// See [`UiState::window_reads`]. +#[cfg(not(target_arch = "wasm32"))] +type WindowRead = + (u64, String, Value, Slot>, std::sync::Arc); + +/// A window's turn to be read off the frame, or `None` once `left` says its project was closed: +/// one waiting for its turn then gives up its place at once, without being read. +#[cfg(not(target_arch = "wasm32"))] +fn window_turn(left: &std::sync::atomic::AtomicBool) -> Option> { + loop { + if left.load(std::sync::atomic::Ordering::Relaxed) { + return None; + } + match WINDOW_TURN.try_lock() { + Ok(turn) => return Some(turn), + Err(std::sync::TryLockError::Poisoned(e)) => return Some(e.into_inner()), + Err(std::sync::TryLockError::WouldBlock) => std::thread::sleep(std::time::Duration::from_millis(20)), + } + } +} + /// A shell's overrides of the Settings › Start-up preferences for one launch. #[derive(Clone, Debug, Default)] pub struct Start { @@ -716,6 +749,11 @@ pub struct UiState { /// that can be read in place are added so, others downloaded (the `ui.addUrl` parameters, /// and the answer). pub cog_probes: Vec<(Value, Option, Slot>)>, + /// Windows of cloud-optimised GeoTIFFs (`"bbox"`) being read on threads: the session's + /// download id, the address, the parameters to add it with, where the read lands, and whether + /// its project was closed. + #[cfg(not(target_arch = "wasm32"))] + pub window_reads: Vec, /// Service layers whose features are being read page by page (an ArcGIS layer's description /// first). pub service_pages: Vec, @@ -898,6 +936,8 @@ impl Default for UiState { project_download: None, sql: Default::default(), cog_probes: Vec::new(), + #[cfg(not(target_arch = "wasm32"))] + window_reads: Vec::new(), service_pages: Vec::new(), link_zoomed: false, service: None, @@ -2422,6 +2462,20 @@ impl RustGisApp { } finished = again; (id, &finished) + } else if id == "layer.addDownload" && params.get("window").is_some_and(Value::is_object) { + // A window read on a thread: the step is the command that reads it, which a replay + // runs again from its address. + let mut again = params.clone(); + if let Some(o) = again.as_object_mut() { + let window = o.remove("window"); + o.remove("id"); + o.remove("request"); + if let Some(Value::Object(window)) = window { + o.extend(window); + } + } + finished = again; + ("layer.addUrl", &finished) } else { (id, params) }; @@ -4186,7 +4240,10 @@ impl RustGisApp { }; let p = bound.as_ref(); let url = rustgis_engine::data::login_address::inner(p.get("url").and_then(Value::as_str).unwrap_or(url)); - if url.to_ascii_lowercase().split(['?', '#']).next().is_some_and(|u| u.ends_with(".pmtiles")) { + // A window of a cloud-optimised GeoTIFF is read off the frame (below), with no + // probe: it is refused unless it is one. In the browser the command refuses it. + let window = cfg!(not(target_arch = "wasm32")) && p.get("bbox").is_some_and(|v| !v.is_null()); + if !window && url.to_ascii_lowercase().split(['?', '#']).next().is_some_and(|u| u.ends_with(".pmtiles")) { let before = self.session.sources.generation; let result = if let Some(id) = claim { self.run_with_prepared("layer.addUrl", p.clone(), AppDispatch::InPlace(id)) @@ -4215,7 +4272,7 @@ impl RustGisApp { && !tiff && rustgis_engine::cmd::layer::is_unnamed_url(url) && ["service", "coverage"].iter().all(|k| p.get(*k).and_then(Value::as_bool) != Some(true)); - if (remote || tiff || unnamed) && !scan && p.get("whole").and_then(Value::as_bool) != Some(true) { + if !window && (remote || tiff || unnamed) && !scan && p.get("whole").and_then(Value::as_bool) != Some(true) { let slot: Slot> = Slot::default(); let land = slot.clone(); if remote { @@ -4298,6 +4355,10 @@ impl RustGisApp { if let Some(origin) = &origin { q[connectiontree::ORIGIN] = json!(origin); } + #[cfg(not(target_arch = "wasm32"))] + if window { + return self.start_window_read(p, url, q, claim); + } let shown = rustgis_engine::data::s3_address::inner(&url); self.session.notify(i18n::fmt("Downloading {url}…", &[("url", shown)]), false); let id = claim.unwrap_or_else(|| self.session.download_started()); @@ -4412,23 +4473,125 @@ impl RustGisApp { let origin = connectiontree::take_origin_key(&mut p); match r { Ok(bytes) => self.add_landed(serviceui::Landed { id, url, params: p, origin }, bytes), - Err(e) => { - self.session.request_failed(&p, &e); - self.session.download_failed(id); - rustgis_engine::cmd::geolibre::discard_pending(&mut self.session, &p); - // Named without the address's keys, token or signature. - let e = e.strip_prefix(url.as_str()).map(|r| r.trim_start_matches(": ").to_string()).unwrap_or(e); - let why = rustgis_engine::sources::redact_text(&e); - // An error that names its own address (a Shapefile's part) isn't named again. - let named = why.split_once(": ").is_some_and(|(head, _)| head.contains("://") && !head.contains(' ')); - let why = if named { why } else { format!("{}: {why}", rustgis_engine::sources::redact(&url)) }; - self.landed_in_dialog(&p, Err(&why)); - self.session.notify(why, true); + Err(e) => self.landed_failed(id, &url, &p, e), + } + } + // Windows read on threads, added as a download is. + #[cfg(not(target_arch = "wasm32"))] + { + let mut read = Vec::new(); + let session = &self.session; + self.ui.window_reads.retain(|(id, url, p, slot, left)| match slot.lock().unwrap_or_else(|e| e.into_inner()).take() { + Some(r) => { + read.push((*id, url.clone(), p.clone(), r)); + false + } + None => { + left.store(!session.download_under_way(*id), std::sync::atomic::Ordering::Relaxed); + true + } + }); + for (id, url, mut p, r) in read { + let origin = connectiontree::take_origin_key(&mut p); + match r { + Ok(window) => { + let current = self.session.download_landed_window(id, window); + self.add_download(serviceui::Landed { id, url, params: p, origin }, current); + } + Err(e) => self.landed_failed(id, &url, &p, e), } } } } + /// Says why a download or a window's read failed, and answers the command waiting for it. + fn landed_failed(&mut self, id: u64, url: &str, p: &Value, e: String) { + self.session.request_failed(p, &e); + self.session.download_failed(id); + rustgis_engine::cmd::geolibre::discard_pending(&mut self.session, p); + // Named without the address's keys, token or signature. + let e = e.strip_prefix(url).map(|r| r.trim_start_matches(": ").to_string()).unwrap_or(e); + let why = rustgis_engine::sources::redact_text(&e); + // An error that names its own address (a Shapefile's part) isn't named again. + let named = why.split_once(": ").is_some_and(|(head, _)| head.contains("://") && !head.contains(' ')); + let why = if named { why } else { format!("{}: {why}", rustgis_engine::sources::redact(url)) }; + self.landed_in_dialog(p, Err(&why)); + self.session.notify(why, true); + } + + /// Starts reading a window of a cloud-optimised GeoTIFF (`p`, a `ui.addUrl` with `"bbox"`) on a + /// thread, to add it with `q` when it lands (#1330). The command line and scripts wait for it + /// in `layer.addUrl` instead. What's wrong with what was asked is refused here, on the frame. + #[cfg(not(target_arch = "wasm32"))] + fn start_window_read(&mut self, p: &Value, url: String, mut q: Value, claim: Option) -> Result { + let asked = rustgis_engine::cmd::layer::window_asked(&self.session, p).and_then(|asked| { + if self.ui.window_reads.len() >= MAX_WINDOW_READS { + return Err(i18n::fmt( + "{url}: {count} windows are being read already: ask again once one has been added", + &[("url", &rustgis_engine::sources::redact(&url)), ("count", &MAX_WINDOW_READS.to_string())], + )); + } + Ok(asked) + }); + // A refused window answers the command waiting for it and gives back the download it + // was claimed under. + let asked = asked.inspect_err(|e| { + self.session.request_failed(p, e); + if let Some(id) = claim { + self.session.download_failed(id); + } + })?; + // What a task needs to read it again, as the download itself can't be (`record_step`). + let mut window = json!({}); + for k in ["bbox", "cellSize", "cog"] { + if let Some(v) = p.get(k).filter(|v| !v.is_null()) { + window[k] = v.clone(); + } + } + q["window"] = window; + let slot: Slot> = Slot::default(); + let land = slot.clone(); + let left = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let leaving = left.clone(); + // Its requests stay in the session's diagnostics. + let diagnostics = self.session.diagnostics.capture(); + let spawned = std::thread::Builder::new().name("cog-window".into()).spawn(move || { + let _diagnostics = diagnostics.enter(); + let turn = window_turn(&leaving); + let read = if turn.is_some() { + // On a thread no command's catch reaches: a bug must not leave it running forever. + let read = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| rustgis_engine::cmd::layer::read_cog_window(&asked))); + read.unwrap_or_else(|_| Err(tl!("Reading the window failed unexpectedly").to_string())) + } else { + Err(tl!("given up before its read began: its project was closed, or more than 64 downloads started after it").to_string()) + }; + *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(read); + // The next read waits for the frame to take this one, or for the app to be gone. + while std::sync::Arc::strong_count(&land) > 1 && land.lock().unwrap_or_else(|e| e.into_inner()).is_some() { + std::thread::sleep(std::time::Duration::from_millis(20)); + } + drop(turn); + }); + if let Err(e) = spawned { + // Answered and given back as a refused window is. + let e = i18n::fmt( + "{url}: its read didn't start: {error}", + &[("url", &rustgis_engine::sources::redact(&url)), ("error", &e.to_string())], + ); + self.session.request_failed(p, &e); + if let Some(id) = claim { + self.session.download_failed(id); + } + return Err(e); + } + // Named redacted, and by the address an S3 connection's wraps. + let shown = rustgis_engine::sources::redact(rustgis_engine::data::s3_address::inner(&url)); + self.session.notify(i18n::fmt("Reading {url}…", &[("url", &shown)]), false); + let id = claim.unwrap_or_else(|| self.session.download_started()); + self.ui.window_reads.push((id, url, q, slot, left)); + Ok(Value::Null) + } + /// Adds a download that landed (`layer.addDownload`), named in Recent after the connection it /// came from; one whose project was closed meanwhile is said and dropped. pub(crate) fn add_landed(&mut self, landed: serviceui::Landed, bytes: Vec) { @@ -4795,6 +4958,11 @@ impl RustGisApp { if self.session.plugin_retries_waiting() { ctx.request_repaint_after(std::time::Duration::from_millis(100)); } + // A window's read lands after its last tile, with nothing to wake the frame. + #[cfg(not(target_arch = "wasm32"))] + if !self.ui.window_reads.is_empty() { + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } // Zarr steps the time slider moved to, read off the frame, show once they land. if self.session.follow_time_landed() { self.map.invalidate(); @@ -5164,7 +5332,9 @@ pub(crate) fn isolate_account_test(name: &str) -> bool { /// tests that wait for what it reads (tiles, an archive's header), which that would drop. #[cfg(test)] pub(crate) mod fetching { - use std::sync::{Mutex, MutexGuard}; + use std::io::{Read, Write}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex, MutexGuard}; static TURN: Mutex<()> = Mutex::new(()); @@ -5184,6 +5354,52 @@ pub(crate) mod fetching { Turn { _held: TURN.lock().unwrap_or_else(|e| e.into_inner()) } } + /// A slow server of a file by range ([`slow_ranges`]). + pub(crate) struct SlowRanges { + pub origin: String, + /// While unset, requests are held. + pub open: Arc, + /// How many requests it has held. + pub held: Arc, + } + + /// Serves `file` by range, holding requests while `open` is unset. A frame waiting for one + /// shows as a slow frame, not as a test that hangs: after 20 s it is answered anyway. + pub(crate) fn slow_ranges(file: Vec) -> SlowRanges { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let (open, held) = (Arc::new(AtomicBool::new(true)), Arc::new(AtomicUsize::new(0))); + let (shut, count, file) = (open.clone(), held.clone(), Arc::new(file)); + std::thread::spawn(move || { + for stream in listener.incoming() { + let (Ok(mut stream), shut, count, file) = (stream, shut.clone(), count.clone(), file.clone()) else { continue }; + std::thread::spawn(move || { + let mut request = [0u8; 4096]; + let read = stream.read(&mut request).unwrap_or(0); + if !shut.load(Ordering::Relaxed) { + count.fetch_add(1, Ordering::Relaxed); + let start = std::time::Instant::now(); + while !shut.load(Ordering::Relaxed) && start.elapsed() < std::time::Duration::from_secs(20) { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + } + let request = String::from_utf8_lossy(&request[..read]).to_ascii_lowercase(); + let range = request.lines().find_map(|l| l.strip_prefix("range: bytes=")).and_then(|r| { + let (a, b) = r.trim().split_once('-')?; + Some((a.parse::().ok()?, b.parse::().ok()?.min(file.len().saturating_sub(1)))) + }); + let (status, part) = match range { + Some((a, b)) if a <= b => ("206 Partial Content", &file[a..=b]), + _ => ("200 OK", &file[..]), + }; + let _ = write!(stream, "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", part.len()); + let _ = stream.write_all(part); + }); + } + }); + SlowRanges { origin, open, held } + } + /// Waits at most ten seconds for a fixture's request, then restores blocking reads so /// its existing stream timeout remains effective. Missing sends fail instead of hanging CI. #[cfg(not(target_arch = "wasm32"))] @@ -5538,6 +5754,190 @@ mod record_tests { } } + /// A window read off the frame is a step all the same: the command that reads it, with its box, + /// not the download that can't be added again (#1330). + #[test] + fn a_window_read_off_the_frame_is_recorded_as_the_command_reading_it() { + let mut app = RustGisApp::with_sample(); + app.ws.recording = Some(Vec::new()); + let window = json!({"bbox": "view", "cellSize": 30, "cog": true}); + let p = json!({"id": 1, "url": "https://example.com/data", "name": "W", "request": 7, "window": window}); + app.record_step("layer.addDownload", &p, None); + let steps = app.ws.recording.as_ref().unwrap(); + assert_eq!(steps.len(), 1, "{:?}", app.session.notices); + assert_eq!(steps[0].command, "layer.addUrl"); + assert_eq!(steps[0].params, json!({"url": "https://example.com/data", "name": "W", "bbox": "view", "cellSize": 30, "cog": true})); + } + + /// `ui.addUrl` with `"bbox"`: a slow server doesn't hold the + /// frame, and the layer lands when the read does (#1330). + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn a_window_is_read_off_the_frame_and_added_when_it_lands() { + use std::sync::atomic::Ordering; + let _turn = crate::fetching::turn(); + // With the header in hand only the window's tiles wait: those a read on the frame would hold + // it for, for the server's 20 s. Without it, the header waits first. + for header_read in [true, false] { + let slow = crate::fetching::slow_ranges(rustgis_engine::data::raster::cog::example(1024, 256)); + let mut h = egui_kittest::Harness::builder().build_ui_state( + |ui, app: &mut RustGisApp| { + let ctx = ui.ctx().clone(); + app.logic(&ctx); + }, + RustGisApp::with_sample(), + ); + let url = format!("{}/scene.tif", slow.origin); + if header_read { + rustgis_engine::render::rustgis_tiles::cog::info_blocking(&url).unwrap(); + } + slow.open.store(false, Ordering::Relaxed); + let p = json!({"url": url, "bbox": [10.25, 45.5, 10.5, 45.75], "name": "Window"}); + let asked = std::time::Instant::now(); + h.state_mut().run("ui.addUrl", p).unwrap(); + assert!(asked.elapsed() < std::time::Duration::from_secs(5), "asking for it held the frame"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while slow.held.load(Ordering::Relaxed) == 0 { + assert!(std::time::Instant::now() < deadline, "no request held"); + let frame = std::time::Instant::now(); + h.step(); + assert!(frame.elapsed() < std::time::Duration::from_secs(5), "the read held the frame"); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + let frame = std::time::Instant::now(); + h.step(); + assert!(frame.elapsed() < std::time::Duration::from_secs(5), "the read held the frame"); + assert!(h.state().session.map().layer_by_name("Window").is_none()); + assert_eq!(h.state().ui.window_reads.len(), 1); + slow.open.store(true, Ordering::Relaxed); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while h.state().session.map().layer_by_name("Window").is_none() { + assert!(std::time::Instant::now() < deadline, "timed out waiting: {:?}", h.state().session.notices); + h.step(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + let app = h.state(); + assert!(app.ui.window_reads.is_empty()); + let data = app.session.map().layer_by_name("Window").and_then(|l| l.raster.as_ref()?.data.clone()).unwrap(); + assert_eq!((data.width, data.height), (256, 256)); + assert!(app.session.notices.iter().any(|n| n.text.starts_with("Window: a 256 × 256 window")), "{:?}", app.session.notices); + } + } + + /// Windows under way are bounded, as each holds a thread (#1330). + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn too_many_windows_under_way_are_refused() { + let mut app = RustGisApp::with_sample(); + for _ in 0..crate::MAX_WINDOW_READS { + app.ui.window_reads.push((0, String::new(), serde_json::Value::Null, Default::default(), Default::default())); + } + let e = app.run("ui.addUrl", json!({"url": "https://x.org/a.tif?token=hidden-1330", "bbox": [1, 2, 3, 4]})).unwrap_err(); + assert!(e.contains("x.org/a.tif") && e.contains("16 windows") && !e.contains("hidden-1330"), "{e}"); + assert_eq!(app.ui.window_reads.len(), crate::MAX_WINDOW_READS); + } + + /// A window still waiting for its turn when its project closes gives up its place at once, and + /// isn't read (#1330). + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn a_window_waiting_when_its_project_closes_isnt_read() { + use std::sync::atomic::Ordering; + let _turn = crate::fetching::turn(); + let slow = crate::fetching::slow_ranges(rustgis_engine::data::raster::cog::example(1024, 256)); + slow.open.store(false, Ordering::Relaxed); + let mut app = RustGisApp::with_sample(); + // Another window's read has the turn, and keeps it throughout. + let reading = crate::WINDOW_TURN.lock().unwrap_or_else(|e| e.into_inner()); + let url = format!("{}/scene.tif", slow.origin); + app.run("ui.addUrl", json!({"url": url, "bbox": [10.25, 45.5, 10.5, 45.75], "name": "Left"})).unwrap(); + app.run("project.new", json!({})).unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !app.ui.window_reads.is_empty() { + assert!(std::time::Instant::now() < deadline, "its place wasn't given up"); + app.poll_data_downloads(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + drop(reading); + assert_eq!(slow.held.load(Ordering::Relaxed), 0, "nothing is asked of the server"); + let said = |n: &rustgis_engine::Notice| n.text.contains("scene.tif") && n.text.contains("before its read began"); + assert!(app.session.notices.iter().any(said), "{:?}", app.session.notices); + } + + /// The next window is read only once the frame has taken the last: a frame that stalls holds + /// one window's values, not those of all that were asked for (#1330). + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn the_next_window_waits_for_the_last_to_be_taken() { + let _turn = crate::fetching::turn(); + let slow = crate::fetching::slow_ranges(rustgis_engine::data::raster::cog::example(1024, 256)); + let mut app = RustGisApp::with_sample(); + let url = format!("{}/scene.tif", slow.origin); + for name in ["First", "Second"] { + app.run("ui.addUrl", json!({"url": url, "bbox": [10.25, 45.5, 10.5, 45.75], "name": name})).unwrap(); + } + let landed = + |app: &RustGisApp| app.ui.window_reads.iter().filter(|r| r.3.lock().unwrap_or_else(|e| e.into_inner()).is_some()).count(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while landed(&app) == 0 { + assert!(std::time::Instant::now() < deadline, "no read landed"); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + // No frame takes it, so the other isn't read. + std::thread::sleep(std::time::Duration::from_millis(500)); + assert_eq!(landed(&app), 1); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while !app.ui.window_reads.is_empty() { + assert!(std::time::Instant::now() < deadline, "timed out waiting: {:?}", app.session.notices); + app.poll_data_downloads(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert!(app.session.map().layer_by_name("First").is_some() && app.session.map().layer_by_name("Second").is_some()); + } + + /// A window that can't be read is refused at once, one read that fails says so once, and one + /// whose project was closed meanwhile isn't added (#1330). + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn a_window_refused_failed_or_left_behind_is_said() { + let until = |app: &mut RustGisApp| { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while !app.ui.window_reads.is_empty() { + assert!(std::time::Instant::now() < deadline, "timed out waiting"); + app.poll_data_downloads(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + }; + let _turn = crate::fetching::turn(); + let mut app = RustGisApp::with_sample(); + let e = app.run("ui.addUrl", json!({"url": "https://x.org/a.tif", "bbox": "here"})).unwrap_err(); + assert!(e.contains("bbox is") && app.ui.window_reads.is_empty(), "{e}"); + // No probe first, either: a window is a GeoTIFF's or nothing. + let e = app.run("ui.addUrl", json!({"url": "https://x.org/a.pmtiles", "bbox": [1, 2, 3, 4]})).unwrap_err(); + assert!(e.contains("names a .pmtiles file") && app.ui.cog_probes.is_empty(), "{e}"); + // Not a GeoTIFF: said once, without the address's keys. + let slow = crate::fetching::slow_ranges(b"not a tiff at all".to_vec()); + let url = format!("{}/a.tif?token=hidden-1330", slow.origin); + let mark = app.session.notice_mark(); + app.run("ui.addUrl", json!({"url": url, "bbox": [1, 2, 3, 4], "name": "Bad"})).unwrap(); + until(&mut app); + let errors: Vec = app.session.notices_since(mark).iter().filter(|n| n.error).map(|n| n.text.clone()).collect(); + assert_eq!(errors.len(), 1, "{errors:?}"); + assert!(errors[0].contains("/a.tif") && !errors[0].contains("hidden-1330"), "{errors:?}"); + let reading = app.session.notices_since(mark).iter().find(|n| n.text.starts_with("Reading")).map(|n| n.text.clone()).unwrap(); + assert!(reading.contains("/a.tif") && !reading.contains("hidden-1330"), "{reading}"); + assert!(app.session.map().layer_by_name("Bad").is_none()); + // Its project closed while it was read. + let slow = crate::fetching::slow_ranges(rustgis_engine::data::raster::cog::example(1024, 256)); + let url = format!("{}/scene.tif", slow.origin); + app.run("ui.addUrl", json!({"url": url, "bbox": [10.25, 45.5, 10.5, 45.75], "name": "Left"})).unwrap(); + app.run("project.new", json!({})).unwrap(); + until(&mut app); + assert!(app.session.map().layer_by_name("Left").is_none()); + let said = |n: &rustgis_engine::Notice| n.text.contains("scene.tif") && n.text.contains("its project was closed"); + assert!(app.session.notices.iter().any(said), "{:?}", app.session.notices); + } + #[test] #[cfg(not(target_arch = "wasm32"))] fn an_anonymous_download_never_acquires_a_saved_login() { diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index e78555ba9f..0a3c97f542 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -189,6 +189,13 @@ tile), without a STAC catalogue: may take two minutes more): past that the window is given up, saying how many bands were read. A window reads 256 bands at most. A window near the limit takes several hundred MB of memory while it is read and kept. +- The Add Data From URL dialog has no box to fill in, so a window is asked for by a command. On + the command line and in a script, `layer.addUrl` waits for it and answers what was added. In the + desktop app, `ui.addUrl` with `"bbox"` reads it off the frame: the map stays in use, and the + layer is added when the read lands. These are read one at a time, with 16 under way at most, + and the next starts once the app has taken the last. If another project is opened meanwhile, one + still waiting isn't read, and one being read can't be cancelled yet: it runs to its end and + isn't added. Each is said in a notice. - The file's no-data, colour table (a palette file draws in its colours) and band names come with it, and class names given as `CLASS_` items in GDAL's metadata tag, as the MapBiomas files on Source Cooperative give theirs. These come from the file's own tags: an `.aux.xml` @@ -199,7 +206,9 @@ tile), without a STAC catalogue: keeps only its first band once the project is reopened, with a notice. Files stored bottom up aren't read this way yet. In the web app, `"bbox"` is refused for now: the file is drawn in place without it. An address that names another kind of file (`.geojson`, `.fgb`) is refused - with `"bbox"` before anything is read; `"cog": true` takes a GeoTIFF by another name. + with `"bbox"` before anything is read; `"cog": true` takes a GeoTIFF by another name. A + window is a raster layer that the time slider doesn't step through, so `"cube": true` is + refused with it. A GeoParquet or FlatGeobuf file by its address is read in place too, a view at a time, however large: the layer is streamed like a large local file, and the view's features are fetched in the