From 15d3ff1659cacd34d334b40b00432cb5e309f9f6 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Fri, 6 Nov 2026 11:00:33 -0300 Subject: [PATCH 1/8] feat(raster): read a COG window off the frame in the app (#1330, part 1) `layer.addUrl` with "bbox" holds the frame while it waits for the file. Split it: `read_cog_window` reads and writes the raster with no `Session`, and `add_window` adds it. `ui.addUrl` with "bbox" starts the read on a thread and `poll_data_downloads` adds what lands through `layer.addDownload`. The command line and scripts still wait for it in the command. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/layer.rs | 196 +++++++++++++++------- crates/engine/src/lib.rs | 30 ++++ crates/engine/src/tests.rs | 56 +++++++ crates/ui-egui/src/lib.rs | 282 ++++++++++++++++++++++++++++++-- docs/user-guide/data-formats.md | 5 + 5 files changed, 499 insertions(+), 70 deletions(-) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index f34bd22e91..8282418850 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -43,7 +43,7 @@ pub fn commands() -> Vec { r#"{"url": "https://…/data.geojson|.csv|.parquet|.fgb|.gpkg|.zip|.shp (its .dbf fetched beside it, and its .shx, .prj and .cpg when there)|.kml|.kmz|.gpx|.tif|.jp2 (JPEG 2000: desktop, through GDAL) (the format from the address, else the contents) | a .pmtiles archive", "name"?: text, "layer"?: "one layer of a zip, KML or GPX", "zoom"?: "bool: true frames everything added; false keeps the view, even on an empty map; omitted, the first content on an empty map is framed (a GeoParquet, FlatGeobuf or cloud-optimised GeoTIFF read in place likewise; a PMTiles or MBTiles archive ignores zoom and is framed on its bounds unless it covers the whole world)", "x"?, "y"?, "wkt"?, "crs"?, "delimiter"?, "paperSpace"?, "stream"?, "as"? (as layer.addData), "whole"?: "bool: download a cloud-optimised GeoTIFF whole (as a raster layer) instead of reading it in place (an address without a file name is read in place when its first bytes are a cloud-optimised GeoTIFF)", "cog"?: "bool: the address is a cloud-optimised GeoTIFF whatever its name says (the app's probe found it so)", "bbox"?: "[west, south, east, north] in degrees | "view": a cloud-optimised GeoTIFF's window as a raster layer (desktop and command line), read by range at full resolution, in the file's coordinate system, with its colour table and class names; at most 64 million values (all bands)", "cellSize"?: "with bbox: the coarsest overview whose cells are at most this size is read (the file's units; metres for longitude/latitude, a degree taken as 111.32 km)", "start"?, "end"?, "unit"?, "interval"?, "dates"?: "a GeoTIFF address with date placeholders ({date:YYYY}, {YYYY}{MM}{DD}), one file per date: the dates it spans and steps through (as time.set), turning the time slider on", "connection"?: "the id of the saved web-service connection it comes from (as service.add)", "anonymous"?: bool (internal: a plugin-origin download must never adopt a saved login), "request"?: "number (internal: the download a plugin command left to the app, #1202)"}"#, ), C::new("layer.addDownload", "Add Downloaded Data", "Home › Add", add_download) - .params(r#"{"id": a download the app finished (Home › Add › Add Data › From URL), "url": its address, "name"?: text, "layer"?: text, "zoom"?: bool, "x"?, "y"?, "wkt"?, "crs"?, "delimiter"?, "paperSpace"?, "stream"?, "as"? (as layer.addData), "service"?: bool (a service's features: their axes (when they came as GeoJSON) and paging are checked, and the address is kept as the layer's source), "bounds"?: [west, south, east, north] (with service: the layer's longitude/latitude bounds from its capabilities, which its axes are checked against; without bounds they are left as they come; ignored without service), "coverage"?: bool (a WCS coverage: a GeoTIFF, else the service's error, and its request kept as the layer's source), "geolibreProject"?: bool (internal: downloaded project JSON), "geolibreLayer"?: {"token", "record"} (internal: reserved imported layer and its saved style), "geolibreZarr"?: bool (internal: an imported Zarr slice as GeoTIFF), "connection"?: "as layer.addUrl", "anonymous"?: bool (internal: preserves a plugin-origin download's anonymous reads), "request"?: number (internal: the download a plugin command left to the app, #1202)}"#), + .params(r#"{"id": a download the app finished (Home › Add › Add Data › From URL), "url": its address, "name"?: text, "layer"?: text, "zoom"?: bool, "x"?, "y"?, "wkt"?, "crs"?, "delimiter"?, "paperSpace"?, "stream"?, "as"? (as layer.addData), "service"?: bool (a service's features: their axes (when they came as GeoJSON) and paging are checked, and the address is kept as the layer's source), "bounds"?: [west, south, east, north] (with service: the layer's longitude/latitude bounds from its capabilities, which its axes are checked against; without bounds they are left as they come; ignored without service), "coverage"?: bool (a WCS coverage: a GeoTIFF, else the service's error, and its request kept as the layer's source), "geolibreProject"?: bool (internal: downloaded project JSON), "geolibreLayer"?: {"token", "record"} (internal: reserved imported layer and its saved style), "geolibreZarr"?: bool (internal: an imported Zarr slice as GeoTIFF), "connection"?: "as layer.addUrl", "anonymous"?: bool (internal: preserves a plugin-origin download's anonymous reads), "request"?: number (internal: the download a plugin command left to the app, #1202), "window"?: {"bbox", "cellSize"?, "cog"?} (internal: what a cloud-optimised GeoTIFF's window the app read off the frame asked for, for a task to read it again, #1330)}"#), C::new("layer.addEncodedPolyline", "Add Encoded Polyline", "Home › Add", add_encoded_polyline).params( r#"{"polyline": "encoded polyline text" | [texts] (one line each), "precision"?: 5 (Google) | 6 (OSRM, Valhalla), "name"?: text}"#, ), @@ -3118,10 +3118,7 @@ fn add_url_as_given(s: &mut Session, p: &Value) -> CmdResult { } // An address read through a saved S3 connection is judged by the address it wraps; the whole // address stays what is fetched and kept. - let lower = rustgis_data::s3_address::inner(crate::render::rustgis_tiles::logins::inner(url)).to_ascii_lowercase(); - if !(lower.starts_with("https://") || lower.starts_with("http://") || lower.starts_with("s3://")) { - return Err(format!("{url}: give an http://, https:// or s3:// address (Add Data opens files)")); - } + let lower = web_address(url)?; // A scan (`"as"`) keeps the pixels and none of the coordinates: the file is downloaded whole and // read as one (`add_named_bytes` checks it is an image), never read in place. let scan = super::raster::as_scan(p)?; @@ -3131,26 +3128,7 @@ fn add_url_as_given(s: &mut Session, p: &Value) -> CmdResult { return Err(format!("{url}: \"cellSize\" goes with \"bbox\": give a box, or leave it out")); } if window { - if scan || b(p, "whole") == Some(true) || rustgis_data::time::is_template(url) { - return Err(format!( - "{url}: \"bbox\" reads a window of one cloud-optimised GeoTIFF with its coordinates: leave out \"as\" and \"whole\", and give a single file's address" - )); - } - // An address naming another kind of file is refused before anything is fetched (one - // with no extension, or a GeoTIFF's, is read; `"cog": true` says it is one). - if b(p, "cog") != Some(true) - && let Some(ext) = - address_extension(rustgis_data::s3_address::inner(url)).filter(|e| !matches!(e.as_str(), "tif" | "tiff" | "geotiff")) - { - return Err(format!( - "{url}: \"bbox\" reads a window of a cloud-optimised GeoTIFF, and this address names a .{ext} file: leave out \"bbox\" (or give \"cog\": true for a GeoTIFF by another name)" - )); - } - // What a window doesn't read is refused, not dropped: it keeps the file's coordinate system. - let unread: Vec<&str> = WINDOW_UNREAD.iter().copied().filter(|k| p.get(*k).is_some_and(|v| !v.is_null())).collect(); - if !unread.is_empty() { - return Err(format!("{url}: a window (\"bbox\") doesn't read {}: leave it out", unread.join(", "))); - } + window_checks(p, url, scan)?; // A plugin's command run by the app leaves what would wait to the app, off the frame; a // window waits for its header and tiles, which the app doesn't take over yet. #[cfg(not(target_arch = "wasm32"))] @@ -3694,6 +3672,101 @@ const WINDOW_UNREAD: [&str; 13] = #[cfg(not(target_arch = "wasm32"))] const METRES_PER_DEGREE: f64 = 111_320.0; +/// One window is read at a time, however many the app has started: each may hold 256 MB and more. +#[cfg(not(target_arch = "wasm32"))] +static WINDOW_TURN: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// A web address's lowercase form (the address it wraps when read through a saved connection), or +/// why it isn't one. +fn web_address(url: &str) -> Result { + let lower = rustgis_data::s3_address::inner(crate::render::rustgis_tiles::logins::inner(url)).to_ascii_lowercase(); + if !(lower.starts_with("https://") || lower.starts_with("http://") || lower.starts_with("s3://")) { + return Err(format!("{url}: give an http://, https:// or s3:// address (Add Data opens files)")); + } + Ok(lower) +} + +/// Refuses what can't be a window of `url`, before anything is fetched. +fn window_checks(p: &Value, url: &str, scan: bool) -> Result<(), String> { + if scan || b(p, "whole") == Some(true) || rustgis_data::time::is_template(url) { + return Err(format!( + "{url}: \"bbox\" reads a window of one cloud-optimised GeoTIFF with its coordinates: leave out \"as\" and \"whole\", and give a single file's address" + )); + } + // An address naming another kind of file is refused before anything is fetched (one + // with no extension, or a GeoTIFF's, is read; `"cog": true` says it is one). + if b(p, "cog") != Some(true) + && let Some(ext) = + address_extension(rustgis_data::s3_address::inner(url)).filter(|e| !matches!(e.as_str(), "tif" | "tiff" | "geotiff")) + { + return Err(format!( + "{url}: \"bbox\" reads a window of a cloud-optimised GeoTIFF, and this address names a .{ext} file: leave out \"bbox\" (or give \"cog\": true for a GeoTIFF by another name)" + )); + } + // What a window doesn't read is refused, not dropped: it keeps the file's coordinate system. + let unread: Vec<&str> = WINDOW_UNREAD.iter().copied().filter(|k| p.get(*k).is_some_and(|v| !v.is_null())).collect(); + if !unread.is_empty() { + return Err(format!("{url}: a window (\"bbox\") doesn't read {}: leave it out", unread.join(", "))); + } + Ok(()) +} + +/// A window asked for, with its box settled (`"view"` is the view when it was asked for): what +/// [`read_cog_window`] needs, on any thread. +#[cfg(not(target_arch = "wasm32"))] +pub struct WindowAsked { + url: String, + bbox: [f64; 4], + cell: Option, + name: Option, +} + +/// A window read by [`read_cog_window`], waiting to be added to the project: its raster made and +/// its GeoTIFF written, so that adding it holds no frame for long. +#[cfg(not(target_arch = "wasm32"))] +pub struct CogWindow { + name: String, + layer: rustgis_map::RasterLayer, + /// What to say of it, once the layer has its name. + notes: Vec, + columns: [u32; 2], + rows: [u32; 2], + level: usize, + cell_size: f64, +} + +/// `p`'s box and cell size, read on the frame, which holds the view. +#[cfg(not(target_arch = "wasm32"))] +fn window_settled(s: &Session, p: &Value, url: &str) -> Result { + let named = |e: String| if e.starts_with(url) { e } else { format!("{url}: {e}") }; + let bbox = window_box(s, p).map_err(named)?; + let cell = match p.get("cellSize").filter(|v| !v.is_null()) { + None => None, + Some(v) => Some(v.as_f64().filter(|c| c.is_finite() && *c > 0.0).ok_or_else(|| named("cellSize is a size above 0".into()))?), + }; + let name = ps(p, "name").map(str::trim).filter(|n| !n.is_empty()).map(String::from); + Ok(WindowAsked { url: url.to_string(), bbox, cell, name }) +} + +/// A window the app reads off the frame: `p`, a `layer.addUrl` with `"bbox"` and its address bound +/// to its login, checked as the command checks it. Errors quote the address without its keys. +#[cfg(not(target_arch = "wasm32"))] +pub fn window_asked(s: &Session, p: &Value) -> Result { + let url = ps(p, "url").map(str::trim).filter(|u| !u.is_empty()).ok_or("url is required")?; + let checked = web_address(url) + .and_then(|_| super::raster::as_scan(p)) + .and_then(|scan| window_checks(p, url, scan)) + .and_then(|()| window_settled(s, p, url)); + checked.map_err(|e| crate::sources::redact_text(&e)) +} + +/// `layer.addUrl` with `"bbox"`, for those that wait: read here and added. +#[cfg(not(target_arch = "wasm32"))] +fn add_cog_window(s: &mut Session, p: &Value, url: &str) -> CmdResult { + let read = read_cog_window(&window_settled(s, p, url)?)?; + add_window(s, p, read) +} + /// `layer.addUrl` with `"bbox"` (opengeos/RustGIS#1204): a window of a cloud-optimised GeoTIFF, /// read by range as a raster layer. The box (longitude/latitude, or the view) is turned into the /// file's coordinate system and snapped outwards onto the pixels of the level read: the full image, @@ -3701,16 +3774,19 @@ const METRES_PER_DEGREE: f64 = 111_320.0; /// takes are read, with the reader the data cubes use. The raster keeps the file's coordinate /// system, no-data, colour table and band and class names, and is kept in the project (as a /// GeoTIFF of the window), not read again from the address. +/// +/// This waits for the file, and needs no [`Session`]: the app runs it on a thread and adds what it +/// reads when it lands (`layer.addDownload`, [`Session::download_landed_window`]; #1330). Errors +/// quote the address without its keys. #[cfg(not(target_arch = "wasm32"))] -fn add_cog_window(s: &mut Session, p: &Value, url: &str) -> CmdResult { +pub fn read_cog_window(asked: &WindowAsked) -> Result { use crate::render::rustgis_tiles::cog; use rustgis_data::raster::cog::Level; - let named = |e: String| if e.starts_with(url) { e } else { format!("{url}: {e}") }; - let [west, south, east, north] = window_box(s, p).map_err(named)?; - let cell = match p.get("cellSize").filter(|v| !v.is_null()) { - None => None, - Some(v) => Some(v.as_f64().filter(|c| c.is_finite() && *c > 0.0).ok_or_else(|| named("cellSize is a size above 0".into()))?), - }; + let url = asked.url.as_str(); + let named = |e: String| crate::sources::redact_text(&if e.starts_with(url) { e } else { format!("{url}: {e}") }); + let [west, south, east, north] = asked.bbox; + let cell = asked.cell; + let _turn = WINDOW_TURN.lock().unwrap_or_else(std::sync::PoisonError::into_inner); // The five minutes count from the header's read. let started = std::time::Instant::now(); let info = cog::info_blocking(url).map_err(named)?; @@ -3884,8 +3960,7 @@ fn add_cog_window(s: &mut Session, p: &Value, url: &str) -> CmdResult { // A colour table is one band's: a window of more bands comes in without it, and says so. let palette_lost = (layout.palette.is_some() && bands.len() != 1) .then(|| format!("its colour table is left out: the window has {} bands, a palette has one", bands.len())); - let stem = cog_name(url); - let name = ps(p, "name").map(str::trim).filter(|n| !n.is_empty()).unwrap_or(&stem).to_string(); + let name = asked.name.clone().unwrap_or_else(|| cog_name(url)); let mut r = rustgis_data::raster::Raster::new(&name, c1 - c0, r1 - r0, bands, transform, layout.crs, layout.nodata, color, layout.data_type) .map_err(|e| named(e.to_string()))?; @@ -3923,39 +3998,43 @@ fn add_cog_window(s: &mut Session, p: &Value, url: &str) -> CmdResult { }; let (width, height, size) = (r.width, r.height, size_of(lv)); let layer = rustgis_map::RasterLayer::from_bytes(r, format!("{name}.tif"), bytes); - let mut out = super::raster::add_raster_after(s, &name, layer, b(p, "zoom"))?; let of = if level == 0 { "the full image".to_string() } else { format!("overview {level} of {}", layout.levels.len() - 1) }; - let added = s.active_layer.and_then(|id| s.map().layer(id)).map(|l| l.name.clone()).unwrap_or(name); - s.notify(format!("{added}: a {width} × {height} window of {of}, cells of {size:.1}{unit}, read by range from its address"), false); - if let Some(plain) = plain { - s.notify(format!("{added}: {plain}"), false); - } + let mut notes = vec![format!("a {width} × {height} window of {of}, cells of {size:.1}{unit}, read by range from its address")]; + notes.extend(plain); if let Some(c) = finer { - s.notify(format!("{added}: cellSize {c} is finer than the file's finest cells ({size:.1}{unit}): the full image is read"), false); + notes.push(format!("cellSize {c} is finer than the file's finest cells ({size:.1}{unit}): the full image is read")); } if left_out > 0 { let kept = band_count - left_out; - s.notify( - format!("{added}: the project keeps {kept} of its {band_count} bands: {left_out} band(s) are left out once it is reopened"), - false, - ); + notes.push(format!("the project keeps {kept} of its {band_count} bands: {left_out} band(s) are left out once it is reopened")); } if !cut.is_empty() { - s.notify(format!("{added}: the box reaches past the file's {} edge(s): the window is cut to the file", cut.join(", ")), false); + notes.push(format!("the box reaches past the file's {} edge(s): the window is cut to the file", cut.join(", "))); } if unrefined > 0 { - s.notify( - format!("{added}: {unrefined} piece(s) of the box's edges not followed to a quarter of a pixel: the window may fall short of a bent edge"), - false, - ); + notes.push(format!( + "{unrefined} piece(s) of the box's edges not followed to a quarter of a pixel: the window may fall short of a bent edge" + )); } if unplaced > 0 { - s.notify( - format!("{added}: {unplaced} of the {points} points tested along the box's edges have no place in the file's coordinate system: the window is placed from the others"), - false, - ); + notes.push(format!( + "{unplaced} of the {points} points tested along the box's edges have no place in the file's coordinate system: the window is placed from the others" + )); + } + Ok(CogWindow { name, layer, notes, columns: [c0, c1], rows: [r0, r1], level, cell_size: size }) +} + +/// Adds a window read by [`read_cog_window`] and says what it read. `p` is the command adding it +/// (`layer.addUrl`, or `layer.addDownload` when the app read it off the frame). +#[cfg(not(target_arch = "wasm32"))] +fn add_window(s: &mut Session, p: &Value, read: CogWindow) -> CmdResult { + let CogWindow { name, layer, notes, columns, rows, level, cell_size } = read; + let mut out = super::raster::add_raster_after(s, &name, layer, b(p, "zoom"))?; + let added = s.active_layer.and_then(|id| s.map().layer(id)).map(|l| l.name.clone()).unwrap_or(name); + for note in notes { + s.notify(format!("{added}: {note}"), false); } - out["window"] = json!({"columns": [c0, c1], "rows": [r0, r1], "level": level, "cellSize": size}); + out["window"] = json!({"columns": columns, "rows": rows, "level": level, "cellSize": cell_size}); // `"zoom": true` zooms to the window whatever the map held, as for any other address. if b(p, "zoom") == Some(true) { out["zoomed"] = json!(zoom_to_added(s, &out)); @@ -4089,6 +4168,13 @@ fn add_download(s: &mut Session, p: &Value) -> CmdResult { if (service || coverage) && ps(p, "url").is_none_or(|url| url.trim().is_empty()) { return Err("url is required for a service or coverage download".into()); } + // A window the app read off the frame (#1330). + #[cfg(not(target_arch = "wasm32"))] + if let Some(i) = s.landed_windows.iter().position(|(d, _)| *d == id) { + let (_, read) = s.landed_windows.remove(i); + let r = add_window(s, p, read)?; + return added_download(s, p, r, false, &url); + } // A download read already (an ArcGIS layer's protocol buffer pages, joined): added as read, // with no axes to check against bounds. if let Some(i) = s.landed_reads.iter().position(|(d, _)| *d == id) { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 3e3da22795..c919694fa7 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -516,6 +516,8 @@ struct PutAside { downloading: Vec, downloaded: Vec<(u64, Vec)>, landed_reads: Vec<(u64, LandedRead)>, + #[cfg(not(target_arch = "wasm32"))] + landed_windows: Vec<(u64, cmd::layer::CogWindow)>, url_requests: Vec, plugin_retries: Vec, url_waits: cmd::plugins::fetch::UrlWaits, @@ -709,6 +711,10 @@ pub struct Session { /// Downloads that landed already read (an ArcGIS layer's protocol buffer pages, joined), /// waiting for `layer.addDownload` too. pub(crate) landed_reads: Vec<(u64, LandedRead)>, + /// Windows of cloud-optimised GeoTIFFs the app read off the frame, waiting for + /// `layer.addDownload` too. + #[cfg(not(target_arch = "wasm32"))] + pub(crate) landed_windows: Vec<(u64, cmd::layer::CogWindow)>, /// Downloads a command asked for where nothing may wait (the browser): the parameters of /// `layer.addUrl` calls, for the front end to fetch and add (see [`Session::take_url_requests`]). pub(crate) url_requests: Vec, @@ -1027,6 +1033,8 @@ impl Session { downloading: Vec::new(), downloaded: Vec::new(), landed_reads: Vec::new(), + #[cfg(not(target_arch = "wasm32"))] + landed_windows: Vec::new(), url_requests: Vec::new(), file_outputs: Vec::new(), #[cfg(test)] @@ -1113,6 +1121,18 @@ impl Session { landed } + /// Hands over a window of a cloud-optimised GeoTIFF read off the frame + /// ([`cmd::layer::read_cog_window`]), for `layer.addDownload {"id"}` as + /// [`Session::download_landed`] does. + #[cfg(not(target_arch = "wasm32"))] + pub fn download_landed_window(&mut self, id: u64, window: cmd::layer::CogWindow) -> bool { + let landed = self.landed(id); + if landed { + self.landed_windows.push((id, window)); + } + landed + } + /// Takes a download off those under way: `false` when it isn't one of them (another project /// was opened meanwhile). fn landed(&mut self, id: u64) -> bool { @@ -1330,6 +1350,8 @@ impl Session { if let Some(d) = params.get("id").and_then(Value::as_u64) { self.downloaded.retain(|(k, _)| *k != d); self.landed_reads.retain(|(k, _)| *k != d); + #[cfg(not(target_arch = "wasm32"))] + self.landed_windows.retain(|(k, _)| *k != d); } // Redacted here too: the assistant and MCP get the error as it is. let url = sources::redact_text(params.get("url").and_then(Value::as_str).unwrap_or("download")); @@ -1718,6 +1740,8 @@ impl Session { downloading: take(&mut self.downloading), downloaded: take(&mut self.downloaded), landed_reads: take(&mut self.landed_reads), + #[cfg(not(target_arch = "wasm32"))] + landed_windows: take(&mut self.landed_windows), url_requests: take(&mut self.url_requests), plugin_retries: take(&mut self.plugin_retries), url_waits: take(&mut self.url_waits), @@ -1770,6 +1794,8 @@ impl Session { downloading, downloaded, landed_reads, + #[cfg(not(target_arch = "wasm32"))] + landed_windows, url_requests, plugin_retries, url_waits, @@ -1796,6 +1822,10 @@ impl Session { self.file_outputs = file_outputs; (self.downloading, self.downloaded, self.url_requests) = (downloading, downloaded, url_requests); self.landed_reads = landed_reads; + #[cfg(not(target_arch = "wasm32"))] + { + self.landed_windows = landed_windows; + } (self.trend_results, self.raster_results, self.raster_runs) = (trend_results, raster_results, raster_runs); #[cfg(feature = "geolibre")] { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index fde19769af..e059c18b2f 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -27394,6 +27394,62 @@ fn a_windows_bands_are_started_within_its_wait() { /// `layer.addUrl` with `"bbox"` (#1204): a window of a cloud-optimised GeoTIFF, read by range as a /// raster layer, at full resolution or at the overview a cell size asks for; a palette file's /// colours and class names come with it; the window is kept in a saved project, class names too. +/// A window's read needs no session: on a thread, as the app reads it, then added when it lands +/// (#1330). What can't be a window is refused before the read starts, and an error names the +/// address without its keys. +#[test] +fn a_cog_windows_read_is_apart_from_adding_it() { + use crate::cmd::layer::{read_cog_window, window_asked}; + use rustgis_data::raster::cog::example; + let (base, _) = serve_ranges(example(1024, 256)); + let mut s = world(); + let url = format!("{base}/scene.tif?token=hidden-1330"); + let p = json!({"url": url, "bbox": [10.25, 45.5, 10.5, 45.75], "name": "Window"}); + let asked = window_asked(&s, &p).unwrap(); + let layers = s.map().layers.len(); + let read = std::thread::spawn(move || read_cog_window(&asked)).join().unwrap().unwrap(); + assert_eq!(s.map().layers.len(), layers, "nothing added yet"); + let id = s.download_started(); + assert!(s.download_landed_window(id, read)); + let mark = s.notice_mark(); + let r = s.run("layer.addDownload", &json!({"id": id, "url": url, "name": "Window"})).unwrap(); + assert_eq!(r["window"], json!({"columns": [256, 512], "rows": [256, 512], "level": 0, "cellSize": 111_320.0 / 1024.0})); + assert!(s.landed_windows.is_empty()); + let said: Vec = s.notices_since(mark).iter().map(|n| n.text.clone()).collect(); + assert!(said.iter().any(|n| n.starts_with("Window: a 256 × 256 window of the full image")), "{said:?}"); + let w = s.map().layer_by_name("Window").and_then(|l| l.raster.as_ref()?.data.clone()).unwrap(); + assert_eq!((w.value(0, 0, 0), w.value(0, 10, 5)), (Some(0.0), Some(15.0))); + assert_eq!(s.map().layer(r["layer"].as_u64().unwrap() as u32).map(|l| l.name.as_str()), Some("Window")); + // A project opened while it was read: it isn't kept to be added. + let read = read_cog_window(&window_asked(&s, &p).unwrap()).unwrap(); + let id = s.download_started(); + s.run("project.new", &json!({})).unwrap(); + assert!(!s.download_landed_window(id, read)); + assert!(s.landed_windows.is_empty()); + // Refused as the command refuses them, without the address's keys. + let bad = |more: Value| { + let mut q = json!({"url": "https://x.org/a.tif?token=hidden-1330", "bbox": [1, 2, 3, 4]}); + q.as_object_mut().unwrap().extend(more.as_object().unwrap().clone()); + window_asked(&s, &q).err().unwrap() + }; + for (more, why) in [ + (json!({"bbox": "here"}), "bbox is"), + (json!({"cellSize": -1}), "cellSize is"), + (json!({"whole": true}), "leave out"), + (json!({"crs": "EPSG:4326"}), "doesn't read crs"), + (json!({"url": "https://x.org/a.geojson?token=hidden-1330"}), "names a .geojson file"), + (json!({"url": "C:/data/a.tif"}), "give an http://"), + ] { + let e = bad(more); + assert!(e.contains(why) && !e.contains("hidden-1330"), "{why}: {e}"); + } + // A file that can't be read: said by the read, without the address's keys. + let (gone, _) = serve_ranges(b"not a tiff at all".to_vec()); + let asked = window_asked(&s, &json!({"url": format!("{gone}/a.tif?token=hidden-1330"), "bbox": [1, 2, 3, 4]})).unwrap(); + let e = read_cog_window(&asked).err().unwrap(); + assert!(e.contains("/a.tif") && !e.contains("hidden-1330"), "{e}"); +} + #[test] fn a_cog_window_comes_in_as_a_raster() { use rustgis_data::raster::ColorHint; diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index e822667d95..f7067f3033 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -178,6 +178,10 @@ const LAYOUT_FAVOURITE_PREF: &str = "layout_favourite_arrangement"; /// A result an asynchronous service fills in when it's done (the app polls it each frame). pub type Slot = std::sync::Arc>>; +/// See [`UiState::window_reads`]. +#[cfg(not(target_arch = "wasm32"))] +type WindowRead = (u64, String, Value, Slot>); + /// A shell's overrides of the Settings › Start-up preferences for one launch. #[derive(Clone, Debug, Default)] pub struct Start { @@ -694,6 +698,10 @@ pub struct UiState { /// that can be read in place are added so, others downloaded (the `ui.addUrl` parameters, /// and the answer). pub cog_probes: Vec<(Value, Slot>)>, + /// Windows of cloud-optimised GeoTIFFs (`"bbox"`) being read on threads: the session's + /// download id, the address, the parameters to add it with, and where the read lands. + #[cfg(not(target_arch = "wasm32"))] + pub window_reads: Vec, /// Service layers whose features are being read page by page (an ArcGIS layer's description /// first). pub service_pages: Vec, @@ -876,6 +884,8 @@ impl Default for UiState { project_download: None, sql: Default::default(), cog_probes: Vec::new(), + #[cfg(not(target_arch = "wasm32"))] + window_reads: Vec::new(), service_pages: Vec::new(), link_zoomed: false, service: None, @@ -2275,6 +2285,20 @@ impl RustGisApp { } finished = again; (id, &finished) + } else if id == "layer.addDownload" && params.get("window").is_some_and(Value::is_object) { + // A window read on a thread: the step is the command that reads it, which a replay + // runs again from its address. + let mut again = params.clone(); + if let Some(o) = again.as_object_mut() { + let window = o.remove("window"); + o.remove("id"); + o.remove("request"); + if let Some(Value::Object(window)) = window { + o.extend(window); + } + } + finished = again; + ("layer.addUrl", &finished) } else { (id, params) }; @@ -2957,7 +2981,10 @@ impl RustGisApp { .map_err(|e| connectionui::translated(&e))?; let p = bound.as_ref(); let url = rustgis_engine::data::login_address::inner(p.get("url").and_then(Value::as_str).unwrap_or(url)); - if url.to_ascii_lowercase().split(['?', '#']).next().is_some_and(|u| u.ends_with(".pmtiles")) { + // A window of a cloud-optimised GeoTIFF is read off the frame (below), with no + // probe: it is refused unless it is one. In the browser the command refuses it. + let window = cfg!(not(target_arch = "wasm32")) && p.get("bbox").is_some_and(|v| !v.is_null()); + if !window && url.to_ascii_lowercase().split(['?', '#']).next().is_some_and(|u| u.ends_with(".pmtiles")) { return connectiontree::run_named(self, "layer.addUrl", p.clone(), origin.as_deref()); } let address = p.get("url").and_then(Value::as_str).unwrap_or(url).to_string(); @@ -2975,7 +3002,7 @@ impl RustGisApp { && !tiff && rustgis_engine::cmd::layer::is_unnamed_url(url) && ["service", "coverage"].iter().all(|k| p.get(*k).and_then(Value::as_bool) != Some(true)); - if (remote || tiff || unnamed) && !scan && p.get("whole").and_then(Value::as_bool) != Some(true) { + if !window && (remote || tiff || unnamed) && !scan && p.get("whole").and_then(Value::as_bool) != Some(true) { let slot: Slot> = Slot::default(); let land = slot.clone(); if remote { @@ -3056,6 +3083,10 @@ impl RustGisApp { if let Some(origin) = &origin { q[connectiontree::ORIGIN] = json!(origin); } + #[cfg(not(target_arch = "wasm32"))] + if window { + return self.start_window_read(p, url, q); + } let shown = rustgis_engine::data::s3_address::inner(&url); self.session.notify(i18n::fmt("Downloading {url}…", &[("url", shown)]), false); let id = self.session.download_started(); @@ -4202,23 +4233,86 @@ impl RustGisApp { let origin = connectiontree::take_origin_key(&mut p); match r { Ok(bytes) => self.add_landed(serviceui::Landed { id, url, params: p, origin }, bytes), - Err(e) => { - self.session.download_failed(id); - rustgis_engine::cmd::geolibre::discard_pending(&mut self.session, &p); - self.session.request_failed(&p, &e); - // Named without the address's keys, token or signature. - let e = e.strip_prefix(url.as_str()).map(|r| r.trim_start_matches(": ").to_string()).unwrap_or(e); - let why = rustgis_engine::sources::redact_text(&e); - // An error that names its own address (a Shapefile's part) isn't named again. - let named = why.split_once(": ").is_some_and(|(head, _)| head.contains("://") && !head.contains(' ')); - let why = if named { why } else { format!("{}: {why}", rustgis_engine::sources::redact(&url)) }; - self.landed_in_dialog(&p, Err(&why)); - self.session.notify(why, true); + Err(e) => self.landed_failed(id, &url, &p, e), + } + } + // Windows read on threads, added as a download is. + #[cfg(not(target_arch = "wasm32"))] + { + let mut read = Vec::new(); + self.ui.window_reads.retain(|(id, url, p, slot)| match slot.lock().unwrap_or_else(|e| e.into_inner()).take() { + Some(r) => { + read.push((*id, url.clone(), p.clone(), r)); + false + } + None => true, + }); + for (id, url, mut p, r) in read { + let origin = connectiontree::take_origin_key(&mut p); + match r { + Ok(window) => { + let current = self.session.download_landed_window(id, window); + self.add_download(serviceui::Landed { id, url, params: p, origin }, current); + } + Err(e) => self.landed_failed(id, &url, &p, e), } } } } + /// Says why a download or a window's read failed, and answers the command waiting for it. + fn landed_failed(&mut self, id: u64, url: &str, p: &Value, e: String) { + self.session.download_failed(id); + rustgis_engine::cmd::geolibre::discard_pending(&mut self.session, p); + self.session.request_failed(p, &e); + // Named without the address's keys, token or signature. + let e = e.strip_prefix(url).map(|r| r.trim_start_matches(": ").to_string()).unwrap_or(e); + let why = rustgis_engine::sources::redact_text(&e); + // An error that names its own address (a Shapefile's part) isn't named again. + let named = why.split_once(": ").is_some_and(|(head, _)| head.contains("://") && !head.contains(' ')); + let why = if named { why } else { format!("{}: {why}", rustgis_engine::sources::redact(url)) }; + self.landed_in_dialog(p, Err(&why)); + self.session.notify(why, true); + } + + /// Starts reading a window of a cloud-optimised GeoTIFF (`p`, a `ui.addUrl` with `"bbox"`) on a + /// thread, to add it with `q` when it lands (#1330). The command line and scripts wait for it + /// in `layer.addUrl` instead. What's wrong with what was asked is refused here, on the frame. + #[cfg(not(target_arch = "wasm32"))] + fn start_window_read(&mut self, p: &Value, url: String, mut q: Value) -> Result { + let asked = rustgis_engine::cmd::layer::window_asked(&self.session, p)?; + // What a task needs to read it again, as the download itself can't be (`record_step`). + let mut window = json!({}); + for k in ["bbox", "cellSize", "cog"] { + if let Some(v) = p.get(k).filter(|v| !v.is_null()) { + window[k] = v.clone(); + } + } + q["window"] = window; + let slot: Slot> = Slot::default(); + let land = slot.clone(); + // Its requests stay in the session's diagnostics. + let diagnostics = self.session.diagnostics.capture(); + let spawned = std::thread::Builder::new().name("cog-window".into()).spawn(move || { + let _diagnostics = diagnostics.enter(); + // On a thread no command's catch reaches: a bug must not leave it running forever. + let read = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| rustgis_engine::cmd::layer::read_cog_window(&asked))); + let read = read.unwrap_or_else(|_| Err(tl!("Reading the window failed unexpectedly").to_string())); + *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(read); + }); + spawned.map_err(|e| { + i18n::fmt( + "{url}: its read didn't start: {error}", + &[("url", &rustgis_engine::sources::redact(&url)), ("error", &e.to_string())], + ) + })?; + let shown = rustgis_engine::data::s3_address::inner(&url); + self.session.notify(i18n::fmt("Reading {url}…", &[("url", shown)]), false); + let id = self.session.download_started(); + self.ui.window_reads.push((id, url, q, slot)); + Ok(Value::Null) + } + /// Adds a download that landed (`layer.addDownload`), named in Recent after the connection it /// came from; one whose project was closed meanwhile is said and dropped. pub(crate) fn add_landed(&mut self, landed: serviceui::Landed, bytes: Vec) { @@ -4515,6 +4609,11 @@ impl RustGisApp { if self.session.plugin_retries_waiting() { ctx.request_repaint_after(std::time::Duration::from_millis(100)); } + // A window's read lands after its last tile, with nothing to wake the frame. + #[cfg(not(target_arch = "wasm32"))] + if !self.ui.window_reads.is_empty() { + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } // Zarr steps the time slider moved to, read off the frame, show once they land. if self.session.follow_time_landed() { self.map.invalidate(); @@ -4860,7 +4959,9 @@ pub(crate) fn click_in_one_frame(h: &mut egui_kittest::Harness<'_, RustGisApp>, /// tests that wait for what it reads (tiles, an archive's header), which that would drop. #[cfg(test)] pub(crate) mod fetching { - use std::sync::{Mutex, MutexGuard}; + use std::io::{Read, Write}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex, MutexGuard}; static TURN: Mutex<()> = Mutex::new(()); @@ -4879,6 +4980,52 @@ pub(crate) mod fetching { pub(crate) fn turn() -> Turn { Turn { _held: TURN.lock().unwrap_or_else(|e| e.into_inner()) } } + + /// A slow server of a file by range ([`slow_ranges`]). + pub(crate) struct SlowRanges { + pub origin: String, + /// While unset, requests are held. + pub open: Arc, + /// How many requests it has held. + pub held: Arc, + } + + /// Serves `file` by range, holding requests while `open` is unset. A frame waiting for one + /// shows as a slow frame, not as a test that hangs: after 20 s it is answered anyway. + pub(crate) fn slow_ranges(file: Vec) -> SlowRanges { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let (open, held) = (Arc::new(AtomicBool::new(true)), Arc::new(AtomicUsize::new(0))); + let (shut, count, file) = (open.clone(), held.clone(), Arc::new(file)); + std::thread::spawn(move || { + for stream in listener.incoming() { + let (Ok(mut stream), shut, count, file) = (stream, shut.clone(), count.clone(), file.clone()) else { continue }; + std::thread::spawn(move || { + let mut request = [0u8; 4096]; + let read = stream.read(&mut request).unwrap_or(0); + if !shut.load(Ordering::Relaxed) { + count.fetch_add(1, Ordering::Relaxed); + let start = std::time::Instant::now(); + while !shut.load(Ordering::Relaxed) && start.elapsed() < std::time::Duration::from_secs(20) { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + } + let request = String::from_utf8_lossy(&request[..read]).to_ascii_lowercase(); + let range = request.lines().find_map(|l| l.strip_prefix("range: bytes=")).and_then(|r| { + let (a, b) = r.trim().split_once('-')?; + Some((a.parse::().ok()?, b.parse::().ok()?.min(file.len().saturating_sub(1)))) + }); + let (status, part) = match range { + Some((a, b)) if a <= b => ("206 Partial Content", &file[a..=b]), + _ => ("200 OK", &file[..]), + }; + let _ = write!(stream, "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", part.len()); + let _ = stream.write_all(part); + }); + } + }); + SlowRanges { origin, open, held } + } } #[cfg(test)] @@ -5188,6 +5335,111 @@ mod record_tests { assert_eq!(app.ws.recording.as_ref().map(Vec::len), Some(1)); } + /// A window read off the frame is a step all the same: the command that reads it, with its box, + /// not the download that can't be added again (#1330). + #[test] + fn a_window_read_off_the_frame_is_recorded_as_the_command_reading_it() { + let mut app = RustGisApp::with_sample(); + app.ws.recording = Some(Vec::new()); + let window = json!({"bbox": "view", "cellSize": 30, "cog": true}); + let p = json!({"id": 1, "url": "https://example.com/data", "name": "W", "request": 7, "window": window}); + app.record_step("layer.addDownload", &p, None); + let steps = app.ws.recording.as_ref().unwrap(); + assert_eq!(steps.len(), 1, "{:?}", app.session.notices); + assert_eq!(steps[0].command, "layer.addUrl"); + assert_eq!(steps[0].params, json!({"url": "https://example.com/data", "name": "W", "bbox": "view", "cellSize": 30, "cog": true})); + } + + /// `ui.addUrl` with `"bbox"` (Command Search's `layer.addUrl`): a slow server doesn't hold the + /// frame, and the layer lands when the read does (#1330). + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn a_window_is_read_off_the_frame_and_added_when_it_lands() { + use std::sync::atomic::Ordering; + let _turn = crate::fetching::turn(); + let slow = crate::fetching::slow_ranges(rustgis_engine::data::raster::cog::example(1024, 256)); + let mut h = egui_kittest::Harness::builder().build_ui_state( + |ui, app: &mut RustGisApp| { + let ctx = ui.ctx().clone(); + app.logic(&ctx); + }, + RustGisApp::with_sample(), + ); + let url = format!("{}/scene.tif", slow.origin); + // The header in hand, so only the window's tiles wait: those a read on the frame would + // hold it for, for the server's 20 s. + rustgis_engine::render::rustgis_tiles::cog::info_blocking(&url).unwrap(); + slow.open.store(false, Ordering::Relaxed); + let p = json!({"url": url, "bbox": [10.25, 45.5, 10.5, 45.75], "name": "Window"}); + h.state_mut().run("ui.addUrl", p).unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while slow.held.load(Ordering::Relaxed) == 0 { + assert!(std::time::Instant::now() < deadline, "no request held"); + let frame = std::time::Instant::now(); + h.step(); + assert!(frame.elapsed() < std::time::Duration::from_secs(5), "the read held the frame"); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + let frame = std::time::Instant::now(); + h.step(); + assert!(frame.elapsed() < std::time::Duration::from_secs(5), "the read held the frame"); + assert!(h.state().session.map().layer_by_name("Window").is_none()); + assert_eq!(h.state().ui.window_reads.len(), 1); + slow.open.store(true, Ordering::Relaxed); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while h.state().session.map().layer_by_name("Window").is_none() { + assert!(std::time::Instant::now() < deadline, "timed out waiting: {:?}", h.state().session.notices); + h.step(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + let app = h.state(); + assert!(app.ui.window_reads.is_empty()); + let data = app.session.map().layer_by_name("Window").and_then(|l| l.raster.as_ref()?.data.clone()).unwrap(); + assert_eq!((data.width, data.height), (256, 256)); + assert!(app.session.notices.iter().any(|n| n.text.starts_with("Window: a 256 × 256 window")), "{:?}", app.session.notices); + } + + /// A window that can't be read is refused at once, one read that fails says so once, and one + /// whose project was closed meanwhile isn't added (#1330). + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn a_window_refused_failed_or_left_behind_is_said() { + let until = |app: &mut RustGisApp| { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while !app.ui.window_reads.is_empty() { + assert!(std::time::Instant::now() < deadline, "timed out waiting"); + app.poll_data_downloads(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + }; + let _turn = crate::fetching::turn(); + let mut app = RustGisApp::with_sample(); + let e = app.run("ui.addUrl", json!({"url": "https://x.org/a.tif", "bbox": "here"})).unwrap_err(); + assert!(e.contains("bbox is") && app.ui.window_reads.is_empty(), "{e}"); + // No probe first, either: a window is a GeoTIFF's or nothing. + let e = app.run("ui.addUrl", json!({"url": "https://x.org/a.pmtiles", "bbox": [1, 2, 3, 4]})).unwrap_err(); + assert!(e.contains("names a .pmtiles file") && app.ui.cog_probes.is_empty(), "{e}"); + // Not a GeoTIFF: said once, without the address's keys. + let slow = crate::fetching::slow_ranges(b"not a tiff at all".to_vec()); + let url = format!("{}/a.tif?token=hidden-1330", slow.origin); + let mark = app.session.notice_mark(); + app.run("ui.addUrl", json!({"url": url, "bbox": [1, 2, 3, 4], "name": "Bad"})).unwrap(); + until(&mut app); + let errors: Vec = app.session.notices_since(mark).iter().filter(|n| n.error).map(|n| n.text.clone()).collect(); + assert_eq!(errors.len(), 1, "{errors:?}"); + assert!(errors[0].contains("/a.tif") && !errors[0].contains("hidden-1330"), "{errors:?}"); + assert!(app.session.map().layer_by_name("Bad").is_none()); + // Its project closed while it was read. + let slow = crate::fetching::slow_ranges(rustgis_engine::data::raster::cog::example(1024, 256)); + let url = format!("{}/scene.tif", slow.origin); + app.run("ui.addUrl", json!({"url": url, "bbox": [10.25, 45.5, 10.5, 45.75], "name": "Left"})).unwrap(); + app.run("project.new", json!({})).unwrap(); + until(&mut app); + assert!(app.session.map().layer_by_name("Left").is_none()); + let said = |n: &rustgis_engine::Notice| n.text.contains("scene.tif") && n.text.contains("its project was closed"); + assert!(app.session.notices.iter().any(said), "{:?}", app.session.notices); + } + #[test] #[cfg(not(target_arch = "wasm32"))] fn an_anonymous_download_never_acquires_a_saved_login() { diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 341b837f7f..8411ea9f89 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -148,6 +148,11 @@ tile), without a STAC catalogue: may take two minutes more): past that the window is given up, saying how many bands were read. A window reads 256 bands at most. A window near the limit takes several hundred MB of memory while it is read and kept. +- In the desktop app (Command Search) the window is read off the frame: the map stays in use, + and the layer is added when the read lands. Windows are read one at a time. A read + under way can't be cancelled yet; if another project is opened + meanwhile, it isn't added, with a notice. On the command line and in a script, the command waits + for it and answers what was added. - The file's no-data, colour table (a palette file draws in its colours) and band names come with it, and class names given as `CLASS_` items in GDAL's metadata tag, as the MapBiomas files on Source Cooperative give theirs. These come from the file's own tags: an `.aux.xml` From 938c40ca1040f3e7c6853a61d627a2df11fa089d Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Fri, 6 Nov 2026 15:00:26 -0300 Subject: [PATCH 2/8] fix(raster): a window on the frame waits for no other; windows waiting are bounded (#1330) Review round 1 of the first part. The lock moves to the app's threads, so a `layer.addUrl` with "bbox" run on the frame waits only for its own read. At most 16 windows wait to be read off the frame, and the next is refused before its thread starts. The guide no longer says Command Search reaches a window: it opens a dialog with no box. The UI test also runs with the header held and times `ui.addUrl`, and the #1204 test gets its doc lines back. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/layer.rs | 5 -- crates/engine/src/tests.rs | 7 ++- crates/ui-egui/src/lib.rs | 105 +++++++++++++++++++++----------- docs/user-guide/data-formats.md | 11 ++-- 4 files changed, 80 insertions(+), 48 deletions(-) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 8282418850..d194b8b991 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -3672,10 +3672,6 @@ const WINDOW_UNREAD: [&str; 13] = #[cfg(not(target_arch = "wasm32"))] const METRES_PER_DEGREE: f64 = 111_320.0; -/// One window is read at a time, however many the app has started: each may hold 256 MB and more. -#[cfg(not(target_arch = "wasm32"))] -static WINDOW_TURN: std::sync::Mutex<()> = std::sync::Mutex::new(()); - /// A web address's lowercase form (the address it wraps when read through a saved connection), or /// why it isn't one. fn web_address(url: &str) -> Result { @@ -3786,7 +3782,6 @@ pub fn read_cog_window(asked: &WindowAsked) -> Result { let named = |e: String| crate::sources::redact_text(&if e.starts_with(url) { e } else { format!("{url}: {e}") }); let [west, south, east, north] = asked.bbox; let cell = asked.cell; - let _turn = WINDOW_TURN.lock().unwrap_or_else(std::sync::PoisonError::into_inner); // The five minutes count from the header's read. let started = std::time::Instant::now(); let info = cog::info_blocking(url).map_err(named)?; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index e059c18b2f..d57ed381b2 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -27391,12 +27391,10 @@ fn a_windows_bands_are_started_within_its_wait() { assert_eq!(e, "2 of 4 bands read in 5 minutes, so the window is given up: take a smaller box or a larger cellSize"); } -/// `layer.addUrl` with `"bbox"` (#1204): a window of a cloud-optimised GeoTIFF, read by range as a -/// raster layer, at full resolution or at the overview a cell size asks for; a palette file's -/// colours and class names come with it; the window is kept in a saved project, class names too. /// A window's read needs no session: on a thread, as the app reads it, then added when it lands /// (#1330). What can't be a window is refused before the read starts, and an error names the /// address without its keys. +#[cfg(not(target_arch = "wasm32"))] #[test] fn a_cog_windows_read_is_apart_from_adding_it() { use crate::cmd::layer::{read_cog_window, window_asked}; @@ -27450,6 +27448,9 @@ fn a_cog_windows_read_is_apart_from_adding_it() { assert!(e.contains("/a.tif") && !e.contains("hidden-1330"), "{e}"); } +/// `layer.addUrl` with `"bbox"` (#1204): a window of a cloud-optimised GeoTIFF, read by range as a +/// raster layer, at full resolution or at the overview a cell size asks for; a palette file's +/// colours and class names come with it; the window is kept in a saved project, class names too. #[test] fn a_cog_window_comes_in_as_a_raster() { use rustgis_data::raster::ColorHint; diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index f7067f3033..2b896161ce 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -178,6 +178,15 @@ const LAYOUT_FAVOURITE_PREF: &str = "layout_favourite_arrangement"; /// A result an asynchronous service fills in when it's done (the app polls it each frame). pub type Slot = std::sync::Arc>>; +/// One window is read off the frame at a time, however many have started: each may hold 256 MB +/// and more. Those that wait for their command on the frame don't wait for this. +#[cfg(not(target_arch = "wasm32"))] +static WINDOW_TURN: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// How many windows may be waiting to be read off the frame: each holds a thread. +#[cfg(not(target_arch = "wasm32"))] +const MAX_WINDOW_READS: usize = 16; + /// See [`UiState::window_reads`]. #[cfg(not(target_arch = "wasm32"))] type WindowRead = (u64, String, Value, Slot>); @@ -4281,6 +4290,12 @@ impl RustGisApp { #[cfg(not(target_arch = "wasm32"))] fn start_window_read(&mut self, p: &Value, url: String, mut q: Value) -> Result { let asked = rustgis_engine::cmd::layer::window_asked(&self.session, p)?; + if self.ui.window_reads.len() >= MAX_WINDOW_READS { + return Err(i18n::fmt( + "{url}: {count} windows are waiting to be read: ask again once one has been added", + &[("url", &rustgis_engine::sources::redact(&url)), ("count", &MAX_WINDOW_READS.to_string())], + )); + } // What a task needs to read it again, as the download itself can't be (`record_step`). let mut window = json!({}); for k in ["bbox", "cellSize", "cog"] { @@ -4295,6 +4310,7 @@ impl RustGisApp { let diagnostics = self.session.diagnostics.capture(); let spawned = std::thread::Builder::new().name("cog-window".into()).spawn(move || { let _diagnostics = diagnostics.enter(); + let _turn = WINDOW_TURN.lock().unwrap_or_else(|e| e.into_inner()); // On a thread no command's catch reaches: a bug must not leave it running forever. let read = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| rustgis_engine::cmd::layer::read_cog_window(&asked))); let read = read.unwrap_or_else(|_| Err(tl!("Reading the window failed unexpectedly").to_string())); @@ -5357,46 +5373,65 @@ mod record_tests { fn a_window_is_read_off_the_frame_and_added_when_it_lands() { use std::sync::atomic::Ordering; let _turn = crate::fetching::turn(); - let slow = crate::fetching::slow_ranges(rustgis_engine::data::raster::cog::example(1024, 256)); - let mut h = egui_kittest::Harness::builder().build_ui_state( - |ui, app: &mut RustGisApp| { - let ctx = ui.ctx().clone(); - app.logic(&ctx); - }, - RustGisApp::with_sample(), - ); - let url = format!("{}/scene.tif", slow.origin); - // The header in hand, so only the window's tiles wait: those a read on the frame would - // hold it for, for the server's 20 s. - rustgis_engine::render::rustgis_tiles::cog::info_blocking(&url).unwrap(); - slow.open.store(false, Ordering::Relaxed); - let p = json!({"url": url, "bbox": [10.25, 45.5, 10.5, 45.75], "name": "Window"}); - h.state_mut().run("ui.addUrl", p).unwrap(); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); - while slow.held.load(Ordering::Relaxed) == 0 { - assert!(std::time::Instant::now() < deadline, "no request held"); + // With the header in hand only the window's tiles wait: those a read on the frame would hold + // it for, for the server's 20 s. Without it, the header waits first. + for header_read in [true, false] { + let slow = crate::fetching::slow_ranges(rustgis_engine::data::raster::cog::example(1024, 256)); + let mut h = egui_kittest::Harness::builder().build_ui_state( + |ui, app: &mut RustGisApp| { + let ctx = ui.ctx().clone(); + app.logic(&ctx); + }, + RustGisApp::with_sample(), + ); + let url = format!("{}/scene.tif", slow.origin); + if header_read { + rustgis_engine::render::rustgis_tiles::cog::info_blocking(&url).unwrap(); + } + slow.open.store(false, Ordering::Relaxed); + let p = json!({"url": url, "bbox": [10.25, 45.5, 10.5, 45.75], "name": "Window"}); + let asked = std::time::Instant::now(); + h.state_mut().run("ui.addUrl", p).unwrap(); + assert!(asked.elapsed() < std::time::Duration::from_secs(5), "asking for it held the frame"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while slow.held.load(Ordering::Relaxed) == 0 { + assert!(std::time::Instant::now() < deadline, "no request held"); + let frame = std::time::Instant::now(); + h.step(); + assert!(frame.elapsed() < std::time::Duration::from_secs(5), "the read held the frame"); + std::thread::sleep(std::time::Duration::from_millis(20)); + } let frame = std::time::Instant::now(); h.step(); assert!(frame.elapsed() < std::time::Duration::from_secs(5), "the read held the frame"); - std::thread::sleep(std::time::Duration::from_millis(20)); + assert!(h.state().session.map().layer_by_name("Window").is_none()); + assert_eq!(h.state().ui.window_reads.len(), 1); + slow.open.store(true, Ordering::Relaxed); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while h.state().session.map().layer_by_name("Window").is_none() { + assert!(std::time::Instant::now() < deadline, "timed out waiting: {:?}", h.state().session.notices); + h.step(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + let app = h.state(); + assert!(app.ui.window_reads.is_empty()); + let data = app.session.map().layer_by_name("Window").and_then(|l| l.raster.as_ref()?.data.clone()).unwrap(); + assert_eq!((data.width, data.height), (256, 256)); + assert!(app.session.notices.iter().any(|n| n.text.starts_with("Window: a 256 × 256 window")), "{:?}", app.session.notices); } - let frame = std::time::Instant::now(); - h.step(); - assert!(frame.elapsed() < std::time::Duration::from_secs(5), "the read held the frame"); - assert!(h.state().session.map().layer_by_name("Window").is_none()); - assert_eq!(h.state().ui.window_reads.len(), 1); - slow.open.store(true, Ordering::Relaxed); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); - while h.state().session.map().layer_by_name("Window").is_none() { - assert!(std::time::Instant::now() < deadline, "timed out waiting: {:?}", h.state().session.notices); - h.step(); - std::thread::sleep(std::time::Duration::from_millis(20)); + } + + /// Windows waiting to be read are bounded, as each holds a thread (#1330). + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn too_many_windows_waiting_are_refused() { + let mut app = RustGisApp::with_sample(); + for _ in 0..crate::MAX_WINDOW_READS { + app.ui.window_reads.push((0, String::new(), serde_json::Value::Null, Default::default())); } - let app = h.state(); - assert!(app.ui.window_reads.is_empty()); - let data = app.session.map().layer_by_name("Window").and_then(|l| l.raster.as_ref()?.data.clone()).unwrap(); - assert_eq!((data.width, data.height), (256, 256)); - assert!(app.session.notices.iter().any(|n| n.text.starts_with("Window: a 256 × 256 window")), "{:?}", app.session.notices); + let e = app.run("ui.addUrl", json!({"url": "https://x.org/a.tif?token=hidden-1330", "bbox": [1, 2, 3, 4]})).unwrap_err(); + assert!(e.contains("x.org/a.tif") && e.contains("16 windows") && !e.contains("hidden-1330"), "{e}"); + assert_eq!(app.ui.window_reads.len(), crate::MAX_WINDOW_READS); } /// A window that can't be read is refused at once, one read that fails says so once, and one diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 8411ea9f89..256b0d1995 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -148,11 +148,12 @@ tile), without a STAC catalogue: may take two minutes more): past that the window is given up, saying how many bands were read. A window reads 256 bands at most. A window near the limit takes several hundred MB of memory while it is read and kept. -- In the desktop app (Command Search) the window is read off the frame: the map stays in use, - and the layer is added when the read lands. Windows are read one at a time. A read - under way can't be cancelled yet; if another project is opened - meanwhile, it isn't added, with a notice. On the command line and in a script, the command waits - for it and answers what was added. +- The Add Data From URL dialog has no box to fill in, so a window is asked for by a command. On + the command line and in a script, `layer.addUrl` waits for it and answers what was added. In the + desktop app, `ui.addUrl` with `"bbox"` reads it off the frame: the map stays in use, and the + layer is added when the read lands. These are read one at a time, with 16 waiting at most. A + read under way can't be cancelled yet; if another project is opened meanwhile, it isn't added, + with a notice. - The file's no-data, colour table (a palette file draws in its colours) and band names come with it, and class names given as `CLASS_` items in GDAL's metadata tag, as the MapBiomas files on Source Cooperative give theirs. These come from the file's own tags: an `.aux.xml` From e930c9998b17a10af7aa9e51bbd624605388a416 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Fri, 6 Nov 2026 15:21:55 -0300 Subject: [PATCH 3/8] fix(raster): a window read off the frame refuses GeoLibre's download state too (#1330) The command refuses it, and `window_asked` says it refuses what the command does, so `ui.addUrl` with "bbox" and one of those keys added a window and dropped the import. Both now share one check. The limit of 16 counts the window being read, so it is called windows under way. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/layer.rs | 13 ++++++++++--- crates/engine/src/tests.rs | 3 +++ crates/ui-egui/src/lib.rs | 9 +++++---- docs/user-guide/data-formats.md | 2 +- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index d194b8b991..59fb0fcdc8 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -3077,9 +3077,7 @@ pub(crate) fn add_url(s: &mut Session, p: &Value) -> CmdResult { fn add_url_as_given(s: &mut Session, p: &Value) -> CmdResult { let bound = crate::service_logins::request_params(s, p)?; let p = bound.as_ref(); - if ["geolibreProject", "geolibreLayer", "geolibreZarr"].iter().any(|key| p.get(key).is_some()) { - return Err("GeoLibre download state is internal; use project.importGeolibre".into()); - } + no_geolibre_state(p)?; let url = ps(p, "url").map(str::trim).filter(|u| !u.is_empty()).ok_or("url is required")?; // A plugin command run again once the download it left to the app was added (#1202): what // adding it answered, as often as the run asks for it (the app downloaded and added it once). @@ -3682,8 +3680,17 @@ fn web_address(url: &str) -> Result { Ok(lower) } +/// Refuses a GeoLibre project's download state, which only the app's import sets. +fn no_geolibre_state(p: &Value) -> Result<(), String> { + if ["geolibreProject", "geolibreLayer", "geolibreZarr"].iter().any(|key| p.get(key).is_some()) { + return Err("GeoLibre download state is internal; use project.importGeolibre".into()); + } + Ok(()) +} + /// Refuses what can't be a window of `url`, before anything is fetched. fn window_checks(p: &Value, url: &str, scan: bool) -> Result<(), String> { + no_geolibre_state(p)?; if scan || b(p, "whole") == Some(true) || rustgis_data::time::is_template(url) { return Err(format!( "{url}: \"bbox\" reads a window of one cloud-optimised GeoTIFF with its coordinates: leave out \"as\" and \"whole\", and give a single file's address" diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index d57ed381b2..a0e9f985b7 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -27437,6 +27437,9 @@ fn a_cog_windows_read_is_apart_from_adding_it() { (json!({"crs": "EPSG:4326"}), "doesn't read crs"), (json!({"url": "https://x.org/a.geojson?token=hidden-1330"}), "names a .geojson file"), (json!({"url": "C:/data/a.tif"}), "give an http://"), + (json!({"geolibreProject": true}), "GeoLibre download state"), + (json!({"geolibreLayer": 1}), "GeoLibre download state"), + (json!({"geolibreZarr": false}), "GeoLibre download state"), ] { let e = bad(more); assert!(e.contains(why) && !e.contains("hidden-1330"), "{why}: {e}"); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 2b896161ce..16519b3472 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -183,7 +183,8 @@ pub type Slot = std::sync::Arc>>; #[cfg(not(target_arch = "wasm32"))] static WINDOW_TURN: std::sync::Mutex<()> = std::sync::Mutex::new(()); -/// How many windows may be waiting to be read off the frame: each holds a thread. +/// How many windows may be under way off the frame, being read or waiting for their turn: each +/// holds a thread. #[cfg(not(target_arch = "wasm32"))] const MAX_WINDOW_READS: usize = 16; @@ -4292,7 +4293,7 @@ impl RustGisApp { let asked = rustgis_engine::cmd::layer::window_asked(&self.session, p)?; if self.ui.window_reads.len() >= MAX_WINDOW_READS { return Err(i18n::fmt( - "{url}: {count} windows are waiting to be read: ask again once one has been added", + "{url}: {count} windows are being read already: ask again once one has been added", &[("url", &rustgis_engine::sources::redact(&url)), ("count", &MAX_WINDOW_READS.to_string())], )); } @@ -5421,10 +5422,10 @@ mod record_tests { } } - /// Windows waiting to be read are bounded, as each holds a thread (#1330). + /// Windows under way are bounded, as each holds a thread (#1330). #[test] #[cfg(not(target_arch = "wasm32"))] - fn too_many_windows_waiting_are_refused() { + fn too_many_windows_under_way_are_refused() { let mut app = RustGisApp::with_sample(); for _ in 0..crate::MAX_WINDOW_READS { app.ui.window_reads.push((0, String::new(), serde_json::Value::Null, Default::default())); diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 256b0d1995..4def5f4d7f 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -151,7 +151,7 @@ tile), without a STAC catalogue: - The Add Data From URL dialog has no box to fill in, so a window is asked for by a command. On the command line and in a script, `layer.addUrl` waits for it and answers what was added. In the desktop app, `ui.addUrl` with `"bbox"` reads it off the frame: the map stays in use, and the - layer is added when the read lands. These are read one at a time, with 16 waiting at most. A + layer is added when the read lands. These are read one at a time, with 16 under way at most. A read under way can't be cancelled yet; if another project is opened meanwhile, it isn't added, with a notice. - The file's no-data, colour table (a palette file draws in its colours) and band names come From a28a47fe29c696d2eeeb6e1ee5bcc9b67b33d349 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Fri, 6 Nov 2026 21:33:04 -0300 Subject: [PATCH 4/8] fix(raster): a window waiting when its project closes isn't read; one landed window is held at a time (#1330) A window still waiting for its turn when its project is closed gives up its place at once, with a notice, and asks nothing of the server. A read keeps its turn until the frame has taken what it read, so a frame that stalls holds one window's values, not those of every window asked for. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/lib.rs | 6 ++ crates/ui-egui/src/lib.rs | 117 ++++++++++++++++++++++++++++---- docs/user-guide/data-formats.md | 7 +- 3 files changed, 115 insertions(+), 15 deletions(-) diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index c919694fa7..21dcb28b8d 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1141,6 +1141,12 @@ impl Session { true } + /// Whether a download is still under way for this project: `false` once another project was + /// opened. + pub fn download_under_way(&self, id: u64) -> bool { + self.downloading.contains(&id) + } + /// Forgets a download that failed. pub fn download_failed(&mut self, id: u64) { self.downloading.retain(|d| *d != id); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 16519b3472..e80ff63305 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -179,7 +179,9 @@ const LAYOUT_FAVOURITE_PREF: &str = "layout_favourite_arrangement"; pub type Slot = std::sync::Arc>>; /// One window is read off the frame at a time, however many have started: each may hold 256 MB -/// and more. Those that wait for their command on the frame don't wait for this. +/// and more. A read keeps its turn until the frame has taken what it read, so a frame that stalls +/// holds one window, not all of them. Those that wait for their command on the frame don't wait +/// for this. #[cfg(not(target_arch = "wasm32"))] static WINDOW_TURN: std::sync::Mutex<()> = std::sync::Mutex::new(()); @@ -190,7 +192,24 @@ const MAX_WINDOW_READS: usize = 16; /// See [`UiState::window_reads`]. #[cfg(not(target_arch = "wasm32"))] -type WindowRead = (u64, String, Value, Slot>); +type WindowRead = + (u64, String, Value, Slot>, std::sync::Arc); + +/// A window's turn to be read off the frame, or `None` once `left` says its project was closed: +/// one waiting for its turn then gives up its place at once, without being read. +#[cfg(not(target_arch = "wasm32"))] +fn window_turn(left: &std::sync::atomic::AtomicBool) -> Option> { + loop { + if left.load(std::sync::atomic::Ordering::Relaxed) { + return None; + } + match WINDOW_TURN.try_lock() { + Ok(turn) => return Some(turn), + Err(std::sync::TryLockError::Poisoned(e)) => return Some(e.into_inner()), + Err(std::sync::TryLockError::WouldBlock) => std::thread::sleep(std::time::Duration::from_millis(20)), + } + } +} /// A shell's overrides of the Settings › Start-up preferences for one launch. #[derive(Clone, Debug, Default)] @@ -709,7 +728,8 @@ pub struct UiState { /// and the answer). pub cog_probes: Vec<(Value, Slot>)>, /// Windows of cloud-optimised GeoTIFFs (`"bbox"`) being read on threads: the session's - /// download id, the address, the parameters to add it with, and where the read lands. + /// download id, the address, the parameters to add it with, where the read lands, and whether + /// its project was closed. #[cfg(not(target_arch = "wasm32"))] pub window_reads: Vec, /// Service layers whose features are being read page by page (an ArcGIS layer's description @@ -4250,12 +4270,16 @@ impl RustGisApp { #[cfg(not(target_arch = "wasm32"))] { let mut read = Vec::new(); - self.ui.window_reads.retain(|(id, url, p, slot)| match slot.lock().unwrap_or_else(|e| e.into_inner()).take() { + let session = &self.session; + self.ui.window_reads.retain(|(id, url, p, slot, left)| match slot.lock().unwrap_or_else(|e| e.into_inner()).take() { Some(r) => { read.push((*id, url.clone(), p.clone(), r)); false } - None => true, + None => { + left.store(!session.download_under_way(*id), std::sync::atomic::Ordering::Relaxed); + true + } }); for (id, url, mut p, r) in read { let origin = connectiontree::take_origin_key(&mut p); @@ -4307,15 +4331,26 @@ impl RustGisApp { q["window"] = window; let slot: Slot> = Slot::default(); let land = slot.clone(); + let left = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let leaving = left.clone(); // Its requests stay in the session's diagnostics. let diagnostics = self.session.diagnostics.capture(); let spawned = std::thread::Builder::new().name("cog-window".into()).spawn(move || { let _diagnostics = diagnostics.enter(); - let _turn = WINDOW_TURN.lock().unwrap_or_else(|e| e.into_inner()); - // On a thread no command's catch reaches: a bug must not leave it running forever. - let read = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| rustgis_engine::cmd::layer::read_cog_window(&asked))); - let read = read.unwrap_or_else(|_| Err(tl!("Reading the window failed unexpectedly").to_string())); + let turn = window_turn(&leaving); + let read = if turn.is_some() { + // On a thread no command's catch reaches: a bug must not leave it running forever. + let read = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| rustgis_engine::cmd::layer::read_cog_window(&asked))); + read.unwrap_or_else(|_| Err(tl!("Reading the window failed unexpectedly").to_string())) + } else { + Err(tl!("its project was closed before its read began, so it wasn't read").to_string()) + }; *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(read); + // The next read waits for the frame to take this one, or for the app to be gone. + while std::sync::Arc::strong_count(&land) > 1 && land.lock().unwrap_or_else(|e| e.into_inner()).is_some() { + std::thread::sleep(std::time::Duration::from_millis(20)); + } + drop(turn); }); spawned.map_err(|e| { i18n::fmt( @@ -4326,7 +4361,7 @@ impl RustGisApp { let shown = rustgis_engine::data::s3_address::inner(&url); self.session.notify(i18n::fmt("Reading {url}…", &[("url", shown)]), false); let id = self.session.download_started(); - self.ui.window_reads.push((id, url, q, slot)); + self.ui.window_reads.push((id, url, q, slot, left)); Ok(Value::Null) } @@ -5367,7 +5402,7 @@ mod record_tests { assert_eq!(steps[0].params, json!({"url": "https://example.com/data", "name": "W", "bbox": "view", "cellSize": 30, "cog": true})); } - /// `ui.addUrl` with `"bbox"` (Command Search's `layer.addUrl`): a slow server doesn't hold the + /// `ui.addUrl` with `"bbox"`: a slow server doesn't hold the /// frame, and the layer lands when the read does (#1330). #[test] #[cfg(not(target_arch = "wasm32"))] @@ -5428,13 +5463,71 @@ mod record_tests { fn too_many_windows_under_way_are_refused() { let mut app = RustGisApp::with_sample(); for _ in 0..crate::MAX_WINDOW_READS { - app.ui.window_reads.push((0, String::new(), serde_json::Value::Null, Default::default())); + app.ui.window_reads.push((0, String::new(), serde_json::Value::Null, Default::default(), Default::default())); } let e = app.run("ui.addUrl", json!({"url": "https://x.org/a.tif?token=hidden-1330", "bbox": [1, 2, 3, 4]})).unwrap_err(); assert!(e.contains("x.org/a.tif") && e.contains("16 windows") && !e.contains("hidden-1330"), "{e}"); assert_eq!(app.ui.window_reads.len(), crate::MAX_WINDOW_READS); } + /// A window still waiting for its turn when its project closes gives up its place at once, and + /// isn't read (#1330). + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn a_window_waiting_when_its_project_closes_isnt_read() { + use std::sync::atomic::Ordering; + let _turn = crate::fetching::turn(); + let slow = crate::fetching::slow_ranges(rustgis_engine::data::raster::cog::example(1024, 256)); + slow.open.store(false, Ordering::Relaxed); + let mut app = RustGisApp::with_sample(); + // Another window's read has the turn, and keeps it throughout. + let reading = crate::WINDOW_TURN.lock().unwrap_or_else(|e| e.into_inner()); + let url = format!("{}/scene.tif", slow.origin); + app.run("ui.addUrl", json!({"url": url, "bbox": [10.25, 45.5, 10.5, 45.75], "name": "Left"})).unwrap(); + app.run("project.new", json!({})).unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !app.ui.window_reads.is_empty() { + assert!(std::time::Instant::now() < deadline, "its place wasn't given up"); + app.poll_data_downloads(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + drop(reading); + assert_eq!(slow.held.load(Ordering::Relaxed), 0, "nothing is asked of the server"); + let said = |n: &rustgis_engine::Notice| n.text.contains("scene.tif") && n.text.contains("before its read began"); + assert!(app.session.notices.iter().any(said), "{:?}", app.session.notices); + } + + /// The next window is read only once the frame has taken the last: a frame that stalls holds + /// one window's values, not those of all that were asked for (#1330). + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn the_next_window_waits_for_the_last_to_be_taken() { + let _turn = crate::fetching::turn(); + let slow = crate::fetching::slow_ranges(rustgis_engine::data::raster::cog::example(1024, 256)); + let mut app = RustGisApp::with_sample(); + let url = format!("{}/scene.tif", slow.origin); + for name in ["First", "Second"] { + app.run("ui.addUrl", json!({"url": url, "bbox": [10.25, 45.5, 10.5, 45.75], "name": name})).unwrap(); + } + let landed = + |app: &RustGisApp| app.ui.window_reads.iter().filter(|r| r.3.lock().unwrap_or_else(|e| e.into_inner()).is_some()).count(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while landed(&app) == 0 { + assert!(std::time::Instant::now() < deadline, "no read landed"); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + // No frame takes it, so the other isn't read. + std::thread::sleep(std::time::Duration::from_millis(500)); + assert_eq!(landed(&app), 1); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while !app.ui.window_reads.is_empty() { + assert!(std::time::Instant::now() < deadline, "timed out waiting: {:?}", app.session.notices); + app.poll_data_downloads(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert!(app.session.map().layer_by_name("First").is_some() && app.session.map().layer_by_name("Second").is_some()); + } + /// A window that can't be read is refused at once, one read that fails says so once, and one /// whose project was closed meanwhile isn't added (#1330). #[test] diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 4def5f4d7f..a0a4c0adb4 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -151,9 +151,10 @@ tile), without a STAC catalogue: - The Add Data From URL dialog has no box to fill in, so a window is asked for by a command. On the command line and in a script, `layer.addUrl` waits for it and answers what was added. In the desktop app, `ui.addUrl` with `"bbox"` reads it off the frame: the map stays in use, and the - layer is added when the read lands. These are read one at a time, with 16 under way at most. A - read under way can't be cancelled yet; if another project is opened meanwhile, it isn't added, - with a notice. + layer is added when the read lands. These are read one at a time, with 16 under way at most, + and the next starts once the last has been added. If another project is opened meanwhile, one + still waiting isn't read, and one being read can't be cancelled yet: it runs to its end and + isn't added. Each is said in a notice. - The file's no-data, colour table (a palette file draws in its colours) and band names come with it, and class names given as `CLASS_` items in GDAL's metadata tag, as the MapBiomas files on Source Cooperative give theirs. These come from the file's own tags: an `.aux.xml` From 589f026805895918c58c5d7b9a9a715eb80639a7 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Fri, 6 Nov 2026 21:45:22 -0300 Subject: [PATCH 5/8] fix(raster): say a window given up without blaming the project; name its read redacted (#1330) A window waiting for its turn is also given up once 64 later downloads forget its id, with its project still open, so the notice names both causes. The "Reading" notice names the address redacted, as the errors do. Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/src/lib.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index e80ff63305..fbc477f915 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -4343,7 +4343,7 @@ impl RustGisApp { let read = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| rustgis_engine::cmd::layer::read_cog_window(&asked))); read.unwrap_or_else(|_| Err(tl!("Reading the window failed unexpectedly").to_string())) } else { - Err(tl!("its project was closed before its read began, so it wasn't read").to_string()) + Err(tl!("given up before its read began: its project was closed, or more than 64 downloads started after it").to_string()) }; *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(read); // The next read waits for the frame to take this one, or for the app to be gone. @@ -4358,8 +4358,9 @@ impl RustGisApp { &[("url", &rustgis_engine::sources::redact(&url)), ("error", &e.to_string())], ) })?; - let shown = rustgis_engine::data::s3_address::inner(&url); - self.session.notify(i18n::fmt("Reading {url}…", &[("url", shown)]), false); + // Named redacted, and by the address an S3 connection's wraps. + let shown = rustgis_engine::sources::redact(rustgis_engine::data::s3_address::inner(&url)); + self.session.notify(i18n::fmt("Reading {url}…", &[("url", &shown)]), false); let id = self.session.download_started(); self.ui.window_reads.push((id, url, q, slot, left)); Ok(Value::Null) @@ -5557,6 +5558,8 @@ mod record_tests { let errors: Vec = app.session.notices_since(mark).iter().filter(|n| n.error).map(|n| n.text.clone()).collect(); assert_eq!(errors.len(), 1, "{errors:?}"); assert!(errors[0].contains("/a.tif") && !errors[0].contains("hidden-1330"), "{errors:?}"); + let reading = app.session.notices_since(mark).iter().find(|n| n.text.starts_with("Reading")).map(|n| n.text.clone()).unwrap(); + assert!(reading.contains("/a.tif") && !reading.contains("hidden-1330"), "{reading}"); assert!(app.session.map().layer_by_name("Bad").is_none()); // Its project closed while it was read. let slow = crate::fetching::slow_ranges(rustgis_engine::data::raster::cog::example(1024, 256)); From 0e2a8c2452e60c8ab9b09be29dee964cf924da1f Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Fri, 6 Nov 2026 22:15:14 -0300 Subject: [PATCH 6/8] fix(raster): a window refuses what it doesn't read of a service; no download is added as a window (#1330) A window drops `service`, `coverage` and `bounds` without a word, so they join what it refuses. `layer.addDownload` with `window` and no window landed for its id added the whole file and left a task recording a window, so it is refused. The guide says the next read starts once the app has taken the last, which is what the code does. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/layer.rs | 25 +++++++++++++++++++++++-- crates/engine/src/tests.rs | 8 ++++++++ docs/user-guide/data-formats.md | 2 +- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 59fb0fcdc8..db760b9887 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -3663,8 +3663,24 @@ pub(crate) fn address_extension(url: &str) -> Option { } /// `layer.addUrl` keys for other kinds of address, which a window (`"bbox"`) doesn't read. -const WINDOW_UNREAD: [&str; 13] = - ["layer", "x", "y", "wkt", "crs", "delimiter", "paperSpace", "stream", "start", "end", "unit", "interval", "dates"]; +const WINDOW_UNREAD: [&str; 16] = [ + "layer", + "x", + "y", + "wkt", + "crs", + "delimiter", + "paperSpace", + "stream", + "start", + "end", + "unit", + "interval", + "dates", + "service", + "coverage", + "bounds", +]; /// Metres in a degree, to compare a cell size in metres with a longitude/latitude file's pixels. #[cfg(not(target_arch = "wasm32"))] @@ -4177,6 +4193,11 @@ fn add_download(s: &mut Session, p: &Value) -> CmdResult { let r = add_window(s, p, read)?; return added_download(s, p, r, false, &url); } + // No window landed for it: a whole file isn't added in a window's place. + if p.get("window").is_some_and(Value::is_object) { + let shown = crate::sources::redact(&url); + return Err(format!("{shown}: \"window\" is for a window the app read off the frame: use layer.addUrl with \"bbox\"")); + } // A download read already (an ArcGIS layer's protocol buffer pages, joined): added as read, // with no axes to check against bounds. if let Some(i) = s.landed_reads.iter().position(|(d, _)| *d == id) { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index a0e9f985b7..c186bd319b 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -27418,6 +27418,11 @@ fn a_cog_windows_read_is_apart_from_adding_it() { let w = s.map().layer_by_name("Window").and_then(|l| l.raster.as_ref()?.data.clone()).unwrap(); assert_eq!((w.value(0, 0, 0), w.value(0, 10, 5)), (Some(0.0), Some(15.0))); assert_eq!(s.map().layer(r["layer"].as_u64().unwrap() as u32).map(|l| l.name.as_str()), Some("Window")); + // A download that isn't a window isn't added as one. + let id = s.download_started(); + assert!(s.download_landed(id, b"{}".to_vec())); + let e = s.run("layer.addDownload", &json!({"id": id, "url": url, "window": {"bbox": [1, 2, 3, 4]}})).unwrap_err(); + assert!(e.contains("\"window\" is for") && !e.contains("hidden-1330"), "{e}"); // A project opened while it was read: it isn't kept to be added. let read = read_cog_window(&window_asked(&s, &p).unwrap()).unwrap(); let id = s.download_started(); @@ -27440,6 +27445,9 @@ fn a_cog_windows_read_is_apart_from_adding_it() { (json!({"geolibreProject": true}), "GeoLibre download state"), (json!({"geolibreLayer": 1}), "GeoLibre download state"), (json!({"geolibreZarr": false}), "GeoLibre download state"), + (json!({"service": true}), "doesn't read service"), + (json!({"coverage": true}), "doesn't read coverage"), + (json!({"bounds": [1, 2, 3, 4]}), "doesn't read bounds"), ] { let e = bad(more); assert!(e.contains(why) && !e.contains("hidden-1330"), "{why}: {e}"); diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index a0a4c0adb4..ce65d6ef22 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -152,7 +152,7 @@ tile), without a STAC catalogue: the command line and in a script, `layer.addUrl` waits for it and answers what was added. In the desktop app, `ui.addUrl` with `"bbox"` reads it off the frame: the map stays in use, and the layer is added when the read lands. These are read one at a time, with 16 under way at most, - and the next starts once the last has been added. If another project is opened meanwhile, one + and the next starts once the app has taken the last. If another project is opened meanwhile, one still waiting isn't read, and one being read can't be cancelled yet: it runs to its end and isn't added. Each is said in a notice. - The file's no-data, colour table (a palette file draws in its colours) and band names come From 2664b32ea322c01871e590468a8c16ee1ed3a3cd Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Sat, 7 Nov 2026 08:05:15 -0300 Subject: [PATCH 7/8] test(raster): match the refusal of a local path by the part redaction keeps (#1330) Since #1521, `redact_text` turns the message's "http://," into "http:,", for `layer.addUrl` too, so the test matches "address (Add Data opens files)". Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 9be317b82a..2c60c9d657 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -28674,7 +28674,7 @@ fn a_cog_windows_read_is_apart_from_adding_it() { (json!({"whole": true}), "leave out"), (json!({"crs": "EPSG:4326"}), "doesn't read crs"), (json!({"url": "https://x.org/a.geojson?token=hidden-1330"}), "names a .geojson file"), - (json!({"url": "C:/data/a.tif"}), "give an http://"), + (json!({"url": "C:/data/a.tif"}), "address (Add Data opens files)"), (json!({"geolibreProject": true}), "GeoLibre download state"), (json!({"geolibreLayer": 1}), "GeoLibre download state"), (json!({"geolibreZarr": false}), "GeoLibre download state"), From ba3089e01ccf92cabadf03366a2a48b1036c72fb Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Sat, 7 Nov 2026 12:35:29 -0300 Subject: [PATCH 8/8] fix(raster): a window refuses "cube": true; a read whose thread doesn't start is answered (#1330) A window is a raster layer, so "cube": true with "bbox" was dropped without a word: it is refused, as other unread keys are ("cube": false asks for nothing). A window whose thread doesn't start now answers its request and gives its claim back, as a refused window does. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/layer.rs | 5 ++++- crates/engine/src/tests.rs | 3 +++ crates/ui-egui/src/lib.rs | 14 ++++++++++---- docs/user-guide/data-formats.md | 4 +++- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index f72510fd8d..e040e6e5e0 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -3890,7 +3890,10 @@ fn window_checks(p: &Value, url: &str, scan: bool) -> Result<(), String> { )); } // What a window doesn't read is refused, not dropped: it keeps the file's coordinate system. - let unread: Vec<&str> = WINDOW_UNREAD.iter().copied().filter(|k| p.get(*k).is_some_and(|v| !v.is_null())).collect(); + // A data cube's file is a tile layer that follows the time slider, and a window is a raster: + // `"cube": false` asks for nothing. + let cube = (b(p, "cube") == Some(true)).then_some("cube"); + let unread: Vec<&str> = WINDOW_UNREAD.iter().copied().filter(|k| p.get(*k).is_some_and(|v| !v.is_null())).chain(cube).collect(); if !unread.is_empty() { return Err(format!("{url}: a window (\"bbox\") doesn't read {}: leave it out", unread.join(", "))); } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index be6df0e6ce..89e68f4b7c 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -29390,10 +29390,13 @@ fn a_cog_windows_read_is_apart_from_adding_it() { (json!({"service": true}), "doesn't read service"), (json!({"coverage": true}), "doesn't read coverage"), (json!({"bounds": [1, 2, 3, 4]}), "doesn't read bounds"), + (json!({"cube": true}), "doesn't read cube"), ] { let e = bad(more); assert!(e.contains(why) && !e.contains("hidden-1330"), "{why}: {e}"); } + // `"cube": false` asks for nothing. + assert!(window_asked(&s, &json!({"url": "https://x.org/a.tif", "bbox": [1, 2, 3, 4], "cube": false})).is_ok()); // A file that can't be read: said by the read, without the address's keys. let (gone, _) = serve_ranges(b"not a tiff at all".to_vec()); let asked = window_asked(&s, &json!({"url": format!("{gone}/a.tif?token=hidden-1330"), "bbox": [1, 2, 3, 4]})).unwrap(); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index e47a62d248..7730063bb3 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -4572,12 +4572,18 @@ impl RustGisApp { } drop(turn); }); - spawned.map_err(|e| { - i18n::fmt( + if let Err(e) = spawned { + // Answered and given back as a refused window is. + let e = i18n::fmt( "{url}: its read didn't start: {error}", &[("url", &rustgis_engine::sources::redact(&url)), ("error", &e.to_string())], - ) - })?; + ); + self.session.request_failed(p, &e); + if let Some(id) = claim { + self.session.download_failed(id); + } + return Err(e); + } // Named redacted, and by the address an S3 connection's wraps. let shown = rustgis_engine::sources::redact(rustgis_engine::data::s3_address::inner(&url)); self.session.notify(i18n::fmt("Reading {url}…", &[("url", &shown)]), false); diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 9c53aee9b4..0a3c97f542 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -206,7 +206,9 @@ tile), without a STAC catalogue: keeps only its first band once the project is reopened, with a notice. Files stored bottom up aren't read this way yet. In the web app, `"bbox"` is refused for now: the file is drawn in place without it. An address that names another kind of file (`.geojson`, `.fgb`) is refused - with `"bbox"` before anything is read; `"cog": true` takes a GeoTIFF by another name. + with `"bbox"` before anything is read; `"cog": true` takes a GeoTIFF by another name. A + window is a raster layer that the time slider doesn't step through, so `"cube": true` is + refused with it. A GeoParquet or FlatGeobuf file by its address is read in place too, a view at a time, however large: the layer is streamed like a large local file, and the view's features are fetched in the