From 54224c51a1dd75441e0319d7e4b3cdd51de4e764 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 08:36:23 -0300 Subject: [PATCH 1/4] feat(app): Cancel for downloads and window reads in the status bar Each download under way gets a Cancel in the status bar (one with its address, several as a menu). Cancel drops it as a failed download is dropped and answers the command waiting for it. A window of a cloud-optimised GeoTIFF also stops reading within a moment: its `left` flag reaches read_cog_window_until, and the blocking waits on the header and tiles (info_until, read_values_counted_blocking) look at it every 100 ms, so a slow server no longer holds WINDOW_TURN for every other window read. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/layer.rs | 13 +++- crates/tiles/src/cog.rs | 58 ++++++++++++++-- crates/ui-egui/src/chrome.rs | 50 ++++++++++++++ crates/ui-egui/src/lib.rs | 123 +++++++++++++++++++++++++++++++-- 4 files changed, 234 insertions(+), 10 deletions(-) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 4000b92f1f..631f5325a3 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -4196,6 +4196,14 @@ fn add_cog_window(s: &mut Session, p: &Value, url: &str) -> CmdResult { /// quote the address without its keys. #[cfg(not(target_arch = "wasm32"))] pub fn read_cog_window(asked: &WindowAsked) -> Result { + read_cog_window_until(asked, &std::sync::atomic::AtomicBool::new(false)) +} + +/// [`read_cog_window`], giving up within a moment of `stop` being set (the app's Cancel of the +/// download, or its project closed): in the wait on the header, before each band and in the wait +/// on its tiles. Making the raster and writing its GeoTIFF, once the values are in, don't look. +#[cfg(not(target_arch = "wasm32"))] +pub fn read_cog_window_until(asked: &WindowAsked, stop: &std::sync::atomic::AtomicBool) -> Result { use crate::render::rustgis_tiles::cog; use rustgis_data::raster::cog::Level; let url = asked.url.as_str(); @@ -4204,7 +4212,7 @@ pub fn read_cog_window(asked: &WindowAsked) -> Result { let cell = asked.cell; // The five minutes count from the header's read. let started = std::time::Instant::now(); - let info = cog::info_blocking(url).map_err(named)?; + let info = cog::info_until(url, stop).map_err(named)?; let layout = &info.layout; let (pw, ph) = (layout.transform.pixel_width, layout.transform.pixel_height); if !(pw.is_finite() && ph.is_finite() && pw != 0.0 && ph != 0.0) { @@ -4357,7 +4365,8 @@ pub fn read_cog_window(asked: &WindowAsked) -> Result { let (mut bands, mut rounded, mut overflowed) = (Vec::with_capacity(count), Vec::with_capacity(count), 0u64); for band in 0..count { band_in_time(band, count, started.elapsed(), WINDOW_WAIT).map_err(named)?; - let (values, changed, past) = cog::read_values_counted_blocking(&info, url, level, band, (c0, c1), (r0, r1)).map_err(named)?; + let (values, changed, past) = + cog::read_values_counted_blocking(&info, url, level, band, (c0, c1), (r0, r1), stop).map_err(named)?; bands.push(values); rounded.push(changed); overflowed += past; diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index 588899acc8..40ba040a06 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -1430,10 +1430,52 @@ pub const MAX_VALUES_READ: u64 = 64_000_000; /// A file's header, waiting for it: for an analysis that reads the values (opengeos/RustGIS#579). #[cfg(not(target_arch = "wasm32"))] pub fn info_blocking(url: &str) -> Result, String> { + info_until(url, &std::sync::atomic::AtomicBool::new(false)) +} + +/// [`info_blocking`], giving up within [`STOP_STEP`] of `stop` being set (a download's Cancel), +/// with [`STOPPED`]. +#[cfg(not(target_arch = "wasm32"))] +pub fn info_until(url: &str, stop: &std::sync::atomic::AtomicBool) -> Result, String> { let url = split(url).0; let (tx, rx) = std::sync::mpsc::channel(); with_info(url, Box::new(move |r| tx.send(r).unwrap_or_default())); - rx.recv_timeout(Duration::from_secs(120)).map_err(|_| message_with_address(url, "no answer in two minutes"))? + wait_until(&rx, Duration::from_secs(120), stop).map_err(|stopped| stopped_or(stopped, url, "no answer in two minutes"))? +} + +/// What a read given up for its `stop` says. +#[cfg(not(target_arch = "wasm32"))] +pub const STOPPED: &str = "stopped before it was read"; + +/// How often a blocking read looks at its `stop` while it waits. +#[cfg(not(target_arch = "wasm32"))] +const STOP_STEP: Duration = Duration::from_millis(100); + +/// What `rx` sends within `limit`, waited for in steps of [`STOP_STEP`]: `Err(true)` once `stop` +/// is set, `Err(false)` when nothing came in time (or the sender is gone). +#[cfg(not(target_arch = "wasm32"))] +fn wait_until(rx: &std::sync::mpsc::Receiver, limit: Duration, stop: &std::sync::atomic::AtomicBool) -> Result { + let end = Instant::now() + limit; + loop { + if stop.load(std::sync::atomic::Ordering::Relaxed) { + return Err(true); + } + let left = end.saturating_duration_since(Instant::now()); + if left.is_zero() { + return Err(false); + } + match rx.recv_timeout(left.min(STOP_STEP)) { + Ok(v) => return Ok(v), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {} + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return Err(false), + } + } +} + +/// [`STOPPED`] for a wait given up for its `stop`, else `late`; both name the address. +#[cfg(not(target_arch = "wasm32"))] +fn stopped_or(stopped: bool, url: &str, late: &str) -> String { + message_with_address(url, if stopped { STOPPED } else { late }) } /// Band `band`'s values (0-based) in a window of a level: columns `cols.0..cols.1` and rows @@ -1451,12 +1493,15 @@ pub fn read_values_blocking( cols: (u32, u32), rows: (u32, u32), ) -> Result, String> { - read_values_counted_blocking(info, url, level, band, cols, rows).map(|(values, _, _)| values) + let never = std::sync::atomic::AtomicBool::new(false); + read_values_counted_blocking(info, url, level, band, cols, rows, &never).map(|(values, _, _)| values) } /// [`read_values_blocking`], with how many of the window's values `f32` rounded (32- and 64-bit /// whole numbers and 64-bit floats it doesn't hold exactly; no-data isn't counted), and how many -/// it can't hold at all (past its range: left out, NaN). +/// it can't hold at all (past its range: left out, NaN). It gives up within [`STOP_STEP`] of +/// `stop` being set (a download's Cancel), with [`STOPPED`]; the tiles already asked for are +/// still fetched, for the cache. #[cfg(not(target_arch = "wasm32"))] pub fn read_values_counted_blocking( info: &Arc, @@ -1465,6 +1510,7 @@ pub fn read_values_counted_blocking( band: usize, cols: (u32, u32), rows: (u32, u32), + stop: &std::sync::atomic::AtomicBool, ) -> Result<(Vec, u64, u64), String> { let url = split(url).0.to_string(); let l = info.layout.levels.get(level).ok_or("no such overview")?; @@ -1495,9 +1541,13 @@ pub fn read_values_counted_blocking( (r0..=r1).flat_map(|ty| (c0..=c1).filter_map(move |tx| ty.checked_mul(across).and_then(|t| t.checked_add(tx)))).collect(); // Reserved first: a window the machine can't hold is an error, not a crash (#1291). let (mut out, mut rounded, mut overflowed) = (rustgis_data::raster::filled(w, h, f32::NAN).map_err(|e| e.to_string())?, 0u64, 0u64); + if stop.load(std::sync::atomic::Ordering::Relaxed) { + return Err(message_with_address(&url, STOPPED)); + } let (tx, rx) = std::sync::mpsc::channel(); fetch_bands(info.clone(), url.clone(), level, tiles.clone(), vec![band], Box::new(move |r| tx.send(r).unwrap_or_default())); - let picked = rx.recv_timeout(Duration::from_secs(120)).map_err(|_| message_with_address(&url, "tiles not read in two minutes"))??; + let picked = + wait_until(&rx, Duration::from_secs(120), stop).map_err(|stopped| stopped_or(stopped, &url, "tiles not read in two minutes"))??; let nodata = info.layout.nodata.map(|v| v as f32); for (&t, p) in tiles.iter().zip(&picked) { let Some((chunk, b)) = p.first() else { continue }; diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index 5dd978b9b9..5fb8c75d36 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -355,6 +355,11 @@ pub fn status_bar(app: &mut RustGisApp, ui: &mut Ui) { sep(ui, &t); crate::readjob::status(app, ui, &t); } + // Downloads and windows being read off the frame, each with its Cancel. + if !app.ui.downloads.is_empty() || window_reads_under_way(app) { + sep(ui, &t); + downloads_status(app, ui, &t); + } ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { // The CPU/GPU backend toggle, then the frame rate (live while the map moves, // "— fps" when idle), then the last render's cost. @@ -455,6 +460,51 @@ pub fn status_bar(app: &mut RustGisApp, ui: &mut Ui) { }); } +/// Whether windows are being read off the frame (none in the browser). +fn window_reads_under_way(app: &RustGisApp) -> bool { + #[cfg(not(target_arch = "wasm32"))] + return !app.ui.window_reads.is_empty(); + #[cfg(target_arch = "wasm32")] + { + let _ = app; + false + } +} + +/// The downloads under way in the status bar: one with its address and Cancel, several as a menu +/// of them, each with its Cancel (`RustGisApp::cancel_download`). +fn downloads_status(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens) { + let under_way = app.downloads_under_way(); + let mut cancel = None; + let row = |ui: &mut Ui, (id, shown): &(u64, String), cancel: &mut Option| { + // Cut short beside the other entries, whole on hover. + let short: String = + if shown.chars().count() > 48 { format!("{}…", shown.chars().take(47).collect::()) } else { shown.clone() }; + ui.label(egui::RichText::new(short).size(11.5)).on_hover_text(shown); + if ui.small_button(tl!("Cancel")).on_hover_text(tl!("Stop waiting for this download; nothing is added")).clicked() { + *cancel = Some(*id); + } + }; + ui.spinner(); + match under_way.as_slice() { + [] => {} + [one] => { + ui.label(egui::RichText::new(tl!("Downloading…")).size(11.5).color(t.text_dim)); + row(ui, one, &mut cancel); + } + many => { + ui.menu_button(crate::i18n::fmt("{n} downloads", &[("n", &many.len().to_string())]), |ui| { + for d in many { + ui.horizontal(|ui| row(ui, d, &mut cancel)); + } + }); + } + } + if let Some(id) = cancel { + app.cancel_download(id); + } +} + fn sep(ui: &mut Ui, t: &Tokens) { let (r, _) = ui.allocate_exact_size(vec2(1.0, 14.0), Sense::hover()); ui.painter().rect_filled(r, 0.0, t.border_strong); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index e1294788c3..221e2b95a2 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -203,8 +203,11 @@ const MAX_WINDOW_READS: usize = 16; type WindowRead = (u64, String, Value, Slot>, std::sync::Arc); -/// A window's turn to be read off the frame, or `None` once `left` says its project was closed: -/// one waiting for its turn then gives up its place at once, without being read. +/// A window's turn to be read off the frame, or `None` once `left` says its project was closed or +/// its download was cancelled: one waiting for its turn then gives up its place at once, without +/// being read. One being read gives up its turn within a moment too ([`read_cog_window_until`]). +/// +/// [`read_cog_window_until`]: rustgis_engine::cmd::layer::read_cog_window_until #[cfg(not(target_arch = "wasm32"))] fn window_turn(left: &std::sync::atomic::AtomicBool) -> Option> { loop { @@ -4649,6 +4652,44 @@ impl RustGisApp { } } + /// The downloads and windows' reads under way off the frame, oldest first: each one's id, for + /// [`Self::cancel_download`], and its address as messages name it. + pub(crate) fn downloads_under_way(&self) -> Vec<(u64, String)> { + let shown = |id: u64, url: &str| (id, rustgis_engine::sources::address_in_message(url)); + let downloads = self.ui.downloads.iter().map(|(id, url, ..)| shown(*id, url)); + #[cfg(not(target_arch = "wasm32"))] + let downloads = downloads.chain(self.ui.window_reads.iter().map(|(id, url, ..)| shown(*id, url))); + let mut under_way: Vec<_> = downloads.collect(); + under_way.sort_by_key(|(id, _)| *id); + under_way + } + + /// Cancels a download or a window's read under way (the status bar's Cancel): it is dropped + /// and said, and the command waiting for it answered, as one that failed is. A window gives up + /// its turn, or its place in the queue for one, within a moment. A whole file's transfer + /// isn't stopped: it runs to its end off the frame, and what it brings is thrown away. + pub(crate) fn cancel_download(&mut self, id: u64) { + let mut cancelled = self.ui.downloads.iter().position(|d| d.0 == id).map(|i| { + let (_, url, p, _) = self.ui.downloads.remove(i); + (url, p) + }); + #[cfg(not(target_arch = "wasm32"))] + if let Some(i) = self.ui.window_reads.iter().position(|d| d.0 == id) { + let (_, url, p, _, left) = self.ui.window_reads.remove(i); + left.store(true, std::sync::atomic::Ordering::Relaxed); + cancelled = Some((url, p)); + } + let Some((url, mut p)) = cancelled else { return }; + connectiontree::take_origin_key(&mut p); + let e = tl!("cancelled"); + self.session.request_failed(&p, e); + self.session.download_failed(id); + rustgis_engine::cmd::geolibre::discard_pending(&mut self.session, &p); + let why = rustgis_engine::sources::message_with_address(&url, e); + self.landed_in_dialog(&p, Err(&why)); + self.session.notify(why, false); + } + /// Says why a download or a window's read failed, and answers the command waiting for it. fn landed_failed(&mut self, id: u64, url: &str, p: &Value, e: String) { self.session.request_failed(p, &e); @@ -4700,8 +4741,18 @@ impl RustGisApp { let turn = window_turn(&leaving); let read = if turn.is_some() { // On a thread no command's catch reaches: a bug must not leave it running forever. - let read = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| rustgis_engine::cmd::layer::read_cog_window(&asked))); - read.unwrap_or_else(|_| Err(tl!("Reading the window failed unexpectedly").to_string())) + let read = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + rustgis_engine::cmd::layer::read_cog_window_until(&asked, &leaving) + })); + let read = read.unwrap_or_else(|_| Err(tl!("Reading the window failed unexpectedly").to_string())); + // Stopped by `left`: said as one given up before its turn is. + read.map_err(|e| { + if leaving.load(std::sync::atomic::Ordering::Relaxed) { + tl!("given up during its read: its project was closed, or more than 64 downloads started after it").to_string() + } else { + e + } + }) } else { Err(tl!("given up before its read began: its project was closed, or more than 64 downloads started after it").to_string()) }; @@ -6028,6 +6079,70 @@ mod record_tests { assert!(app.session.notices.iter().any(said), "{:?}", app.session.notices); } + /// Cancel stops a window's read waiting on a slow server's tiles within a moment: the next + /// read may take its turn, the read is said cancelled, and nothing is added when the tiles + /// come in after all. + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn a_cancelled_window_gives_up_its_turn_at_once() { + use std::sync::atomic::Ordering; + let _turn = crate::fetching::turn(); + let slow = crate::fetching::slow_ranges(rustgis_engine::data::raster::cog::example(1024, 256)); + let url = format!("{}/scene.tif", slow.origin); + // The header in hand: only the tiles wait, for the server's 20 s. + rustgis_engine::render::rustgis_tiles::cog::info_blocking(&url).unwrap(); + slow.open.store(false, Ordering::Relaxed); + let mut app = RustGisApp::with_sample(); + app.run("ui.addUrl", json!({"url": url, "bbox": [10.25, 45.5, 10.5, 45.75], "name": "Cancelled"})).unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while slow.held.load(Ordering::Relaxed) == 0 { + assert!(std::time::Instant::now() < deadline, "no request held"); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + let under_way = app.downloads_under_way(); + assert!(under_way.len() == 1 && under_way[0].1.contains("scene.tif"), "{under_way:?}"); + app.cancel_download(under_way[0].0); + assert!(app.ui.window_reads.is_empty() && app.downloads_under_way().is_empty()); + let said = |n: &rustgis_engine::Notice| n.text.contains("scene.tif") && n.text.contains("cancelled") && !n.error; + assert!(app.session.notices.iter().any(said), "{:?}", app.session.notices); + // The server still holds the tiles, but the read's turn is free within a moment. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); + while crate::WINDOW_TURN.try_lock().is_err() { + assert!(std::time::Instant::now() < deadline, "the cancelled read kept its turn"); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + slow.open.store(true, Ordering::Relaxed); + for _ in 0..25 { + app.poll_data_downloads(); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert!(app.session.map().layer_by_name("Cancelled").is_none()); + } + + /// Cancel drops a whole file's download as one that failed: said, and not added when its + /// bytes come in after all. + #[test] + fn a_cancelled_download_isnt_added_when_it_lands() { + use rustgis_engine::render::rustgis_tiles::cache; + use std::sync::{Arc, Mutex}; + let _turn = crate::fetching::turn(); + let held = Arc::new(Mutex::new(Vec::::new())); + let callbacks = held.clone(); + cache::set_fetcher(move |_, done| callbacks.lock().unwrap().push(done)); + let mut app = RustGisApp::with_sample(); + app.run("ui.addUrl", json!({"url": "https://cancel.invalid/places.geojson", "name": "Cancelled download"})).unwrap(); + let under_way = app.downloads_under_way(); + assert!(under_way.len() == 1 && under_way[0].1.contains("places.geojson"), "{under_way:?}"); + app.cancel_download(under_way[0].0); + assert!(app.ui.downloads.is_empty() && app.downloads_under_way().is_empty()); + let said = |n: &rustgis_engine::Notice| n.text.contains("places.geojson") && n.text.contains("cancelled"); + assert!(app.session.notices.iter().any(said), "{:?}", app.session.notices); + held.lock().unwrap().pop().unwrap()(Ok(br#"{"type":"Point","coordinates":[1,2]}"#.to_vec())); + app.poll_data_downloads(); + assert!(app.session.map().layer_by_name("Cancelled download").is_none()); + cache::clear_fetcher(); + } + /// The next window is read only once the frame has taken the last: a frame that stalls holds /// one window's values, not those of all that were asked for (#1330). #[test] From 9c39471831f196d94333e2622fbe0b55b6d08763 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 10:05:39 -0300 Subject: [PATCH 2/4] fix(app): cancel_download builds for the web app (no unused mut) Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/src/lib.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 221e2b95a2..ed16c26675 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -4669,16 +4669,17 @@ impl RustGisApp { /// its turn, or its place in the queue for one, within a moment. A whole file's transfer /// isn't stopped: it runs to its end off the frame, and what it brings is thrown away. pub(crate) fn cancel_download(&mut self, id: u64) { - let mut cancelled = self.ui.downloads.iter().position(|d| d.0 == id).map(|i| { + let cancelled = self.ui.downloads.iter().position(|d| d.0 == id).map(|i| { let (_, url, p, _) = self.ui.downloads.remove(i); (url, p) }); #[cfg(not(target_arch = "wasm32"))] - if let Some(i) = self.ui.window_reads.iter().position(|d| d.0 == id) { + let cancelled = cancelled.or_else(|| { + let i = self.ui.window_reads.iter().position(|d| d.0 == id)?; let (_, url, p, _, left) = self.ui.window_reads.remove(i); left.store(true, std::sync::atomic::Ordering::Relaxed); - cancelled = Some((url, p)); - } + Some((url, p)) + }); let Some((url, mut p)) = cancelled else { return }; connectiontree::take_origin_key(&mut p); let e = tl!("cancelled"); From d5e2ca8408fa69682286380fd80856d8ca927a70 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 11:04:14 -0300 Subject: [PATCH 3/4] feat(app): download progress in the status bar; Cancel stops the transfer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A native body's read goes through `Wanted`: it looks at the request's `Still` before each chunk, so a download no longer wanted stops at the next bytes, and counts what it reads into the download's `Progress` (bytes read, total from Content-Length), carried to its thread as the `Still` is (`with_progress`). - The app runs a whole file's download under a `Still` that holds while `UiState::downloads` keeps its slot, so Cancel stops its transfer, and keeps its `Progress` beside it. The status bar shows a bar with its percentage once the size is known, the megabytes read before, and "Reading…" for a window's read. - A window read stopped by `left` is said as given up only when it was the stop that ended it; a failure of its own keeps its reason. - Public docs no longer link the private STOP_STEP; the cancelled download test puts the process's fetcher back with a drop guard. Co-Authored-By: Claude Opus 5.5 --- crates/tiles/src/cache.rs | 165 ++++++++++++++++++++++++++++---- crates/tiles/src/cog.rs | 8 +- crates/ui-egui/src/chrome.rs | 48 ++++++---- crates/ui-egui/src/lib.rs | 95 ++++++++++++------ crates/ui-egui/src/serviceui.rs | 2 +- crates/ui-egui/src/stacui.rs | 2 +- 6 files changed, 252 insertions(+), 68 deletions(-) diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index a894ffbf95..3a44c89c5a 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -70,6 +70,52 @@ fn check_current() -> Result<(), String> { check_still(¤t_still()) } +/// How far a whole download has come, for whoever started it (the app's status bar): the bytes +/// read so far, and the total once the server said it (`Content-Length`; 0 while unknown). A file +/// of several parts (a Shapefile's) adds each part's total as it starts. The bytes are those +/// decoded, so a compressed answer may pass its total. Counted natively only. +#[derive(Debug, Default)] +pub struct Progress { + /// Bytes read. + pub done: AtomicU64, + /// Bytes expected, or 0 while unknown. + pub total: AtomicU64, +} + +impl Progress { + /// The bytes read so far. + pub fn received(&self) -> u64 { + self.done.load(Ordering::Relaxed) + } + + /// The part read, 0 to 1, once the total is known. + pub fn fraction(&self) -> Option { + let total = self.total.load(Ordering::Relaxed); + (total > 0).then(|| (self.received() as f64 / total as f64).min(1.0) as f32) + } +} + +thread_local! { static PROGRESS: std::cell::RefCell>> = const { std::cell::RefCell::new(None) }; } + +/// Runs `f` with downloads it starts counting into `progress` (taken along to their threads, as +/// the request's [`Still`] is). +pub fn with_progress(progress: Option>, f: impl FnOnce() -> T) -> T { + struct Restore(Option>); + impl Drop for Restore { + fn drop(&mut self) { + PROGRESS.with(|p| *p.borrow_mut() = self.0.take()); + } + } + let _restore = Restore(PROGRESS.with(|p| p.replace(progress))); + f() +} + +/// What downloads started here count into ([`with_progress`]). +#[cfg_attr(target_arch = "wasm32", allow(dead_code))] +fn current_progress() -> Option> { + PROGRESS.with(|p| p.borrow().clone()) +} + /// Only a live request's failure describes the shared resource. A dropped or retired read /// must let the next caller retry immediately under its own authority, without classifying cancellation by error text. pub(crate) fn keep_failure(message: &str, still: &Option) -> bool { @@ -2546,12 +2592,16 @@ mod native { pub(super) fn download_within(url: String, timeout: std::time::Duration, omit: &'static [u16], done: DoneDetailed) { let capture = Capture::current(); let still = super::current_still(); + let progress = super::current_progress(); std::thread::spawn(move || { super::with_optional_still(still, || { - let _scope = capture.enter(); - let r = - super::check_current().map_err(Failure::from).and_then(|_| get_detailed_omitting(&download_agent(timeout), &url, omit)); - done(r); + super::with_progress(progress, || { + let _scope = capture.enter(); + let r = super::check_current() + .map_err(Failure::from) + .and_then(|_| get_detailed_omitting(&download_agent(timeout), &url, omit)); + done(r); + }) }) }); } @@ -2560,12 +2610,16 @@ mod native { pub(super) fn download_public_within(url: String, timeout: std::time::Duration, done: DoneDetailed) { let capture = Capture::current(); let still = super::current_still(); + let progress = super::current_progress(); std::thread::spawn(move || { super::with_optional_still(still, || { - let _scope = capture.enter(); - let r = - super::check_current().map_err(Failure::from).and_then(|_| get_detailed_omitting(&public_agent(timeout), &url, &[])); - done(r); + super::with_progress(progress, || { + let _scope = capture.enter(); + let r = super::check_current() + .map_err(Failure::from) + .and_then(|_| get_detailed_omitting(&public_agent(timeout), &url, &[])); + done(r); + }) }) }); } @@ -2615,6 +2669,42 @@ mod native { fetch_answer(agent, url, Reach::Local, false, omit).map(|f| f.bytes).map_err(|f| f.message) } + /// What a body read given up by [`Wanted`] says. + pub(crate) const CANCELLED_READ: &str = "cancelled while it was read"; + + /// A body's read that gives up, at the next bytes that come, once its request is no longer + /// wanted ([`super::current_still`]: a download's Cancel, or its owner gone), and counts what + /// it reads into the download's [`super::Progress`], if it has one. A server that sends + /// nothing more holds it until the agent's timeout, as before. + struct Wanted(R, Option>); + + impl Wanted { + /// `body`, read for the request on this thread; `total` is what its answer says it holds. + fn new(body: R, total: Option) -> Self { + let progress = super::current_progress(); + if let (Some(p), Some(n)) = (&progress, total) { + p.total.store(p.received().saturating_add(n), std::sync::atomic::Ordering::Relaxed); + } + Self(body, progress) + } + } + + impl std::io::Read for Wanted { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + super::check_current().map_err(|_| std::io::Error::other(CANCELLED_READ))?; + let n = self.0.read(buf)?; + if let Some(p) = &self.1 { + p.done.fetch_add(n as u64, std::sync::atomic::Ordering::Relaxed); + } + Ok(n) + } + } + + /// What an answer says its body holds (`Content-Length`). + fn body_length(resp: &ureq::http::Response) -> Option { + resp.headers().get("content-length")?.to_str().ok()?.trim().parse().ok() + } + /// The User-Agent every request goes with. const USER_AGENT: &str = concat!("RustGIS/", env!("CARGO_PKG_VERSION"), " (+https://rustgis.app)"); @@ -2877,10 +2967,7 @@ mod native { // Read one byte past `n` (no more, and no error at the limit): a longer file is // reported as a server without ranges, so the file is downloaded whole instead. let limit = n.min(max as u64).saturating_add(1); - resp.body_mut() - .with_config() - .limit(limit.saturating_add(1)) - .reader() + Wanted::new(resp.body_mut().with_config().limit(limit.saturating_add(1)).reader(), None) .take(limit) .read_to_end(&mut buf) .map_err(|e| e.to_string())?; @@ -2893,10 +2980,8 @@ mod native { return Ok(Fetched { bytes: buf, origin }); } // ureq limits the wire bytes; `take` also caps the decoded bytes when gzip expands them. - resp.body_mut() - .with_config() - .limit(max as u64 + 2) - .reader() + let total = body_length(&resp); + Wanted::new(resp.body_mut().with_config().limit(max as u64 + 2).reader(), total) .take(max as u64 + 1) .read_to_end(&mut buf) .map_err(|e| e.to_string())?; @@ -4073,6 +4158,54 @@ mod tests { String::from_utf8_lossy(&request).into_owned() } + /// A slow download counts its bytes as they come, against the total its answer says, and one + /// no longer wanted (`with_still`: a download's Cancel) stops at the next bytes that come, + /// rather than reading the rest of its body. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn a_slow_download_counts_its_progress_and_stops_once_no_longer_wanted() { + use std::io::Write; + use std::sync::atomic::{AtomicBool, Ordering}; + let _turn = crate::cache::turn(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { return }; + let _ = read_request(&mut stream); + let _ = write!(stream, "HTTP/1.1 200 OK\r\nContent-Length: 1000000\r\nConnection: close\r\n\r\n"); + // A kilobyte every 20 ms: 20 s for the whole body. + for _ in 0..1000 { + if stream.write_all(&[0u8; 1000]).is_err() { + return; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + }); + let wanted = std::sync::Arc::new(AtomicBool::new(true)); + let still_wanted = wanted.clone(); + let (tx, rx) = std::sync::mpsc::channel(); + let progress = std::sync::Arc::new(super::Progress::default()); + super::with_still(std::sync::Arc::new(move || still_wanted.load(Ordering::Relaxed)), || { + super::with_progress(Some(progress.clone()), || { + super::download_limited(&format!("http://{addr}/big.bin"), 2_000_000, Box::new(move |r| tx.send(r).unwrap_or_default())) + }) + }); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while progress.received() < 5_000 { + assert!(std::time::Instant::now() < deadline, "nothing counted: {progress:?}"); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert_eq!(progress.total.load(Ordering::Relaxed), 1_000_000); + let first = progress.received(); + std::thread::sleep(std::time::Duration::from_millis(200)); + let then = progress.received(); + assert!(then > first && then < 1_000_000, "{first} then {then}"); + assert!(progress.fraction().is_some_and(|f| f > 0.0 && f < 1.0), "{progress:?}"); + wanted.store(false, Ordering::Relaxed); + let r = rx.recv_timeout(std::time::Duration::from_secs(5)).expect("the download kept reading its body"); + assert!(r.as_ref().is_err_and(|e| e.contains(super::native::CANCELLED_READ)), "{r:?}"); + } + #[cfg(not(target_arch = "wasm32"))] fn serve_recorded(answers: Vec>) -> (String, std::sync::mpsc::Receiver) { use std::io::Write; diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index 40ba040a06..229460081d 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -1433,8 +1433,8 @@ pub fn info_blocking(url: &str) -> Result, String> { info_until(url, &std::sync::atomic::AtomicBool::new(false)) } -/// [`info_blocking`], giving up within [`STOP_STEP`] of `stop` being set (a download's Cancel), -/// with [`STOPPED`]. +/// [`info_blocking`], giving up within 100 ms of `stop` being set (a download's Cancel), with +/// [`STOPPED`]. #[cfg(not(target_arch = "wasm32"))] pub fn info_until(url: &str, stop: &std::sync::atomic::AtomicBool) -> Result, String> { let url = split(url).0; @@ -1499,8 +1499,8 @@ pub fn read_values_blocking( /// [`read_values_blocking`], with how many of the window's values `f32` rounded (32- and 64-bit /// whole numbers and 64-bit floats it doesn't hold exactly; no-data isn't counted), and how many -/// it can't hold at all (past its range: left out, NaN). It gives up within [`STOP_STEP`] of -/// `stop` being set (a download's Cancel), with [`STOPPED`]; the tiles already asked for are +/// it can't hold at all (past its range: left out, NaN). It gives up within 100 ms of `stop` +/// being set (a download's Cancel), with [`STOPPED`]; the tiles already asked for are /// still fetched, for the cache. #[cfg(not(target_arch = "wasm32"))] pub fn read_values_counted_blocking( diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index 9f7ca62da4..101a60ba44 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -474,40 +474,56 @@ fn window_reads_under_way(app: &RustGisApp) -> bool { } } -/// The downloads under way in the status bar: one with its address and Cancel, several as a menu -/// of them, each with its Cancel (`RustGisApp::cancel_download`). +/// The downloads under way in the status bar, each with how far it has come and its Cancel +/// (`RustGisApp::cancel_download`): one in the bar itself, several as a menu of them. fn downloads_status(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens) { let under_way = app.downloads_under_way(); let mut cancel = None; - let row = |ui: &mut Ui, (id, shown): &(u64, String), cancel: &mut Option| { - // Cut short beside the other entries, whole on hover. - let short: String = - if shown.chars().count() > 48 { format!("{}…", shown.chars().take(47).collect::()) } else { shown.clone() }; - ui.label(egui::RichText::new(short).size(11.5)).on_hover_text(shown); - if ui.small_button(tl!("Cancel")).on_hover_text(tl!("Stop waiting for this download; nothing is added")).clicked() { - *cancel = Some(*id); - } - }; ui.spinner(); match under_way.as_slice() { [] => {} - [one] => { - ui.label(egui::RichText::new(tl!("Downloading…")).size(11.5).color(t.text_dim)); - row(ui, one, &mut cancel); - } + [one] => download_row(ui, t, one, &mut cancel), many => { ui.menu_button(crate::i18n::fmt("{n} downloads", &[("n", &many.len().to_string())]), |ui| { for d in many { - ui.horizontal(|ui| row(ui, d, &mut cancel)); + ui.horizontal(|ui| download_row(ui, t, d, &mut cancel)); } }); } } + // The bar moves while the bytes come in on their threads. + ui.ctx().request_repaint_after(std::time::Duration::from_millis(200)); if let Some(id) = cancel { app.cancel_download(id); } } +/// One download in the status bar: what it is doing, its address (cut short, whole on hover), its +/// progress and its Cancel. A file's whole download shows a bar once the server said its size and +/// the megabytes read before; a window's read ("Reading…", as its start notice says) isn't +/// counted, nor is a download in the browser. +fn download_row(ui: &mut Ui, t: &Tokens, d: &crate::UnderWay, cancel: &mut Option) { + let received = d.progress.as_ref().map_or(0, |p| p.received()); + let fraction = d.progress.as_ref().and_then(|p| p.fraction()); + let doing = match (&d.progress, fraction) { + (None, _) => tl!("Reading…").to_string(), + (Some(_), None) if received > 0 => { + crate::i18n::fmt("Downloading… {mb} MB", &[("mb", &format!("{:.1}", received as f64 / 1_048_576.0))]) + } + (Some(_), _) => tl!("Downloading…").to_string(), + }; + ui.label(egui::RichText::new(doing).size(11.5).color(t.text_dim)); + let short: String = + if d.shown.chars().count() > 48 { format!("{}…", d.shown.chars().take(47).collect::()) } else { d.shown.clone() }; + ui.label(egui::RichText::new(short).size(11.5)).on_hover_text(&d.shown); + if let Some(f) = fraction { + ui.add(egui::ProgressBar::new(f).show_percentage().desired_width(120.0).desired_height(12.0)); + } + if ui.small_button(tl!("Cancel")).on_hover_text(tl!("Stop this download; nothing is added")).clicked() { + *cancel = Some(d.id); + } +} + fn sep(ui: &mut Ui, t: &Tokens) { let (r, _) = ui.allocate_exact_size(vec2(1.0, 14.0), Sense::hover()); ui.painter().rect_filled(r, 0.0, t.border_strong); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index bca3cd9f17..f3578577f6 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -198,6 +198,17 @@ static WINDOW_TURN: std::sync::Mutex<()> = std::sync::Mutex::new(()); #[cfg(not(target_arch = "wasm32"))] const MAX_WINDOW_READS: usize = 16; +/// A download or a window's read under way, for the status bar ([`RustGisApp::downloads_under_way`]). +#[derive(Debug)] +pub(crate) struct UnderWay { + /// Its download id, for [`RustGisApp::cancel_download`]. + pub id: u64, + /// Its address, as messages name it. + pub shown: String, + /// How far a whole file's download has come; `None` for a window's read, which isn't counted. + pub progress: Option>, +} + /// See [`UiState::window_reads`]. #[cfg(not(target_arch = "wasm32"))] type WindowRead = @@ -749,8 +760,9 @@ pub struct UiState { /// CRS override for the next Shapefile picked by Import Shapefile with Projection. pub pending_shapefile: Option, /// Data downloading from web addresses (Home › Add › Add Data › From URL): the address, the parameters to - /// add it with, and where the download lands. - pub downloads: Vec<(u64, String, Value, Slot, String>>)>, + /// add it with, where the download lands, and how far it has come (the status bar's). + pub downloads: + Vec<(u64, String, Value, Slot, String>>, std::sync::Arc)>, /// A project downloading from its web address (`ui.openUrl`, a `?project=` link): the /// session's download id, the address and where the download lands. A later one replaces it; /// opening another project meanwhile drops it. @@ -4519,17 +4531,26 @@ impl RustGisApp { } let slot: Slot, String>> = Slot::default(); let land = slot.clone(); + // Wanted while `UiState::downloads` holds its slot: Cancel (or the app gone) stops its + // transfer at the next bytes that come (natively; the browser's fetch runs on). + let watch = std::sync::Arc::downgrade(&slot); + let wanted: rustgis_engine::render::rustgis_tiles::cache::Still = std::sync::Arc::new(move || watch.strong_count() > 1); + let progress = std::sync::Arc::::default(); // A Shapefile address brings its companion parts under the same download owner. - rustgis_engine::cmd::layer::download_data( - &url, - if q.get("geolibreProject").and_then(Value::as_bool) == Some(true) { - rustgis_engine::map::geolibre::MAX_TEXT - } else { - rustgis_engine::render::rustgis_tiles::cache::MAX_DOWNLOAD - }, - Box::new(move |r| *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(r)), - ); - self.ui.downloads.push((id, url, q, slot)); + rustgis_engine::render::rustgis_tiles::cache::with_still(wanted, || { + rustgis_engine::render::rustgis_tiles::cache::with_progress(Some(progress.clone()), || { + rustgis_engine::cmd::layer::download_data( + &url, + if q.get("geolibreProject").and_then(Value::as_bool) == Some(true) { + rustgis_engine::map::geolibre::MAX_TEXT + } else { + rustgis_engine::render::rustgis_tiles::cache::MAX_DOWNLOAD + }, + Box::new(move |r| *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(r)), + ) + }) + }); + self.ui.downloads.push((id, url, q, slot, progress)); Ok(Value::Null) }) @@ -4612,7 +4633,7 @@ impl RustGisApp { } } let mut landed = Vec::new(); - self.ui.downloads.retain(|(id, url, p, slot)| match slot.lock().unwrap_or_else(|e| e.into_inner()).take() { + self.ui.downloads.retain(|(id, url, p, slot, _)| match slot.lock().unwrap_or_else(|e| e.into_inner()).take() { Some(r) => { landed.push((*id, url.clone(), p.clone(), r)); false @@ -4654,25 +4675,29 @@ impl RustGisApp { } } - /// The downloads and windows' reads under way off the frame, oldest first: each one's id, for - /// [`Self::cancel_download`], and its address as messages name it. - pub(crate) fn downloads_under_way(&self) -> Vec<(u64, String)> { - let shown = |id: u64, url: &str| (id, rustgis_engine::sources::address_in_message(url)); - let downloads = self.ui.downloads.iter().map(|(id, url, ..)| shown(*id, url)); + /// The downloads and windows' reads under way off the frame, oldest first. + pub(crate) fn downloads_under_way(&self) -> Vec { + let shown = |id: u64, url: &str, progress: Option<&std::sync::Arc<_>>| UnderWay { + id, + shown: rustgis_engine::sources::address_in_message(url), + progress: progress.cloned(), + }; + let downloads = self.ui.downloads.iter().map(|(id, url, _, _, progress)| shown(*id, url, Some(progress))); #[cfg(not(target_arch = "wasm32"))] - let downloads = downloads.chain(self.ui.window_reads.iter().map(|(id, url, ..)| shown(*id, url))); + let downloads = downloads.chain(self.ui.window_reads.iter().map(|(id, url, ..)| shown(*id, url, None))); let mut under_way: Vec<_> = downloads.collect(); - under_way.sort_by_key(|(id, _)| *id); + under_way.sort_by_key(|d| d.id); under_way } /// Cancels a download or a window's read under way (the status bar's Cancel): it is dropped /// and said, and the command waiting for it answered, as one that failed is. A window gives up - /// its turn, or its place in the queue for one, within a moment. A whole file's transfer - /// isn't stopped: it runs to its end off the frame, and what it brings is thrown away. + /// its turn, or its place in the queue for one, within a moment. A whole file's transfer stops + /// at the next bytes that come, as its download is no longer wanted (`add_url`); in the + /// browser it runs to its end, and what it brings is thrown away. pub(crate) fn cancel_download(&mut self, id: u64) { let cancelled = self.ui.downloads.iter().position(|d| d.0 == id).map(|i| { - let (_, url, p, _) = self.ui.downloads.remove(i); + let (_, url, p, _, _) = self.ui.downloads.remove(i); (url, p) }); #[cfg(not(target_arch = "wasm32"))] @@ -4756,9 +4781,11 @@ impl RustGisApp { rustgis_engine::cmd::layer::read_cog_window_until(&asked, &leaving) })); let read = read.unwrap_or_else(|_| Err(tl!("Reading the window failed unexpectedly").to_string())); - // Stopped by `left`: said as one given up before its turn is. + // Stopped by `left`: said as one given up before its turn is. A read that failed + // for its own reason keeps it. read.map_err(|e| { - if leaving.load(std::sync::atomic::Ordering::Relaxed) { + if leaving.load(std::sync::atomic::Ordering::Relaxed) && e.contains(rustgis_engine::render::rustgis_tiles::cog::STOPPED) + { tl!("given up during its read: its project was closed, or more than 64 downloads started after it").to_string() } else { e @@ -6111,8 +6138,8 @@ mod record_tests { std::thread::sleep(std::time::Duration::from_millis(20)); } let under_way = app.downloads_under_way(); - assert!(under_way.len() == 1 && under_way[0].1.contains("scene.tif"), "{under_way:?}"); - app.cancel_download(under_way[0].0); + assert!(under_way.len() == 1 && under_way[0].shown.contains("scene.tif") && under_way[0].progress.is_none(), "{under_way:?}"); + app.cancel_download(under_way[0].id); assert!(app.ui.window_reads.is_empty() && app.downloads_under_way().is_empty()); let said = |n: &rustgis_engine::Notice| n.text.contains("scene.tif") && n.text.contains("cancelled") && !n.error; assert!(app.session.notices.iter().any(said), "{:?}", app.session.notices); @@ -6137,21 +6164,29 @@ mod record_tests { use rustgis_engine::render::rustgis_tiles::cache; use std::sync::{Arc, Mutex}; let _turn = crate::fetching::turn(); + // The fetcher is the process's: put back however the test ends. + struct Fetcher; + impl Drop for Fetcher { + fn drop(&mut self) { + cache::clear_fetcher(); + } + } let held = Arc::new(Mutex::new(Vec::::new())); let callbacks = held.clone(); cache::set_fetcher(move |_, done| callbacks.lock().unwrap().push(done)); + let _fetcher = Fetcher; let mut app = RustGisApp::with_sample(); app.run("ui.addUrl", json!({"url": "https://cancel.invalid/places.geojson", "name": "Cancelled download"})).unwrap(); + assert_eq!(held.lock().unwrap().len(), 1, "one fetch for the file"); let under_way = app.downloads_under_way(); - assert!(under_way.len() == 1 && under_way[0].1.contains("places.geojson"), "{under_way:?}"); - app.cancel_download(under_way[0].0); + assert!(under_way.len() == 1 && under_way[0].shown.contains("places.geojson") && under_way[0].progress.is_some(), "{under_way:?}"); + app.cancel_download(under_way[0].id); assert!(app.ui.downloads.is_empty() && app.downloads_under_way().is_empty()); let said = |n: &rustgis_engine::Notice| n.text.contains("places.geojson") && n.text.contains("cancelled"); assert!(app.session.notices.iter().any(said), "{:?}", app.session.notices); held.lock().unwrap().pop().unwrap()(Ok(br#"{"type":"Point","coordinates":[1,2]}"#.to_vec())); app.poll_data_downloads(); assert!(app.session.map().layer_by_name("Cancelled download").is_none()); - cache::clear_fetcher(); } /// The next window is read only once the frame has taken the last: a frame that stalls holds diff --git a/crates/ui-egui/src/serviceui.rs b/crates/ui-egui/src/serviceui.rs index db88fefb84..ad6934759c 100644 --- a/crates/ui-egui/src/serviceui.rs +++ b/crates/ui-egui/src/serviceui.rs @@ -627,7 +627,7 @@ mod tests { assert_eq!(seen.len(), 2, "{:?}", app.session.notices); assert!(seen[1].contains("REQUEST=GetCoverage")); assert_eq!(logins::apply(&seen[1]).unwrap().is_some(), !anonymous); - let (_, _, params, _) = app.ui.downloads.first().expect("the coverage download was dispatched"); + let (_, _, params, _, _) = app.ui.downloads.first().expect("the coverage download was dispatched"); assert_eq!(params.get("connection").and_then(Value::as_str), (!anonymous).then_some(id.as_str())); assert_eq!(params.get("anonymous").and_then(Value::as_bool).unwrap_or(false), anonymous); } else { diff --git a/crates/ui-egui/src/stacui.rs b/crates/ui-egui/src/stacui.rs index c718012799..97d6e0027c 100644 --- a/crates/ui-egui/src/stacui.rs +++ b/crates/ui-egui/src/stacui.rs @@ -1264,7 +1264,7 @@ mod tests { app.ui.stac.waiting = Some((asked, slot)); app.ui.stac.connection = two["id"].as_str().map(str::to_string); poll(&mut app, &egui::Context::default()); - let (_, _, params, _) = app.ui.downloads.last().unwrap(); + let (_, _, params, _, _) = app.ui.downloads.last().unwrap(); assert_eq!(params["connection"], one["id"], "token completion must not re-read the picker"); assert_ne!(params["connection"], two["id"]); } From 80bea1f2712a8ef074bcdcc3d7cda02905dbe37f Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 11:59:17 -0300 Subject: [PATCH 4/4] fix(app): unknown part sizes, Cancel out of the network history, named downloads - A whole answer that doesn't say its size leaves the download's total unknown, so a Shapefile part sent in chunks after a sized one shows megabytes rather than a bar stuck at 100 %; a tail read counts without touching the total (`Wanted::part`). - `RequestDiagnostics::failed` leaves `CANCELLED_READ` out of the network history, as it does `DROPPED`: a Cancel isn't a failed request. - `UiState::downloads` holds a named `Download` instead of a tuple. Co-Authored-By: Claude Opus 5.5 --- crates/tiles/src/cache.rs | 80 +++++++++++++++++++++++++++---- crates/ui-egui/src/lib.rs | 34 ++++++++----- crates/ui-egui/src/pluginstart.rs | 4 +- crates/ui-egui/src/serviceui.rs | 2 +- crates/ui-egui/src/stacui.rs | 2 +- 5 files changed, 97 insertions(+), 25 deletions(-) diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 3a44c89c5a..c012b8cc8d 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -150,6 +150,10 @@ pub const MAX_BYTES: usize = 16 << 20; /// The error a fetcher gives a request it dropped unfetched (the view moved on): the address is /// forgotten rather than failed, so it is fetched again when it is next wanted. pub const DROPPED: &str = "dropped: the view moved on"; + +/// What a native body's read given up for its request's `Still` says (a download's Cancel): a +/// cancel, not a failure, so it stays out of the network history as [`DROPPED`] does. +pub(crate) const CANCELLED_READ: &str = "cancelled while it was read"; /// The end of a fetcher's error when a range was asked for and the server answered with the /// whole file: a file read in place by range is downloaded whole instead. pub const NO_RANGES: &str = "the server doesn't answer byte-range requests"; @@ -1302,7 +1306,7 @@ impl<'a> RequestDiagnostics<'a> { pub fn failed(&self, failure: &Failure) { use crate::failure::Reason; use rustgis_data::diagnostics::{Network, NetworkKind}; - if failure.message == DROPPED { + if failure.message == DROPPED || failure.message.contains(CANCELLED_READ) { return; } // The address as asked for, without the download and range markers this module adds. @@ -2669,9 +2673,6 @@ mod native { fetch_answer(agent, url, Reach::Local, false, omit).map(|f| f.bytes).map_err(|f| f.message) } - /// What a body read given up by [`Wanted`] says. - pub(crate) const CANCELLED_READ: &str = "cancelled while it was read"; - /// A body's read that gives up, at the next bytes that come, once its request is no longer /// wanted ([`super::current_still`]: a download's Cancel, or its owner gone), and counts what /// it reads into the download's [`super::Progress`], if it has one. A server that sends @@ -2679,19 +2680,27 @@ mod native { struct Wanted(R, Option>); impl Wanted { - /// `body`, read for the request on this thread; `total` is what its answer says it holds. + /// `body`, a whole answer read for the request on this thread; `total` is what it says it + /// holds. One that doesn't say leaves the download's total unknown (a Shapefile's part + /// sent in chunks after one that said its size), rather than the bar full while it reads. fn new(body: R, total: Option) -> Self { let progress = super::current_progress(); - if let (Some(p), Some(n)) = (&progress, total) { - p.total.store(p.received().saturating_add(n), std::sync::atomic::Ordering::Relaxed); + if let Some(p) = &progress { + let known = total.map_or(0, |n| p.received().saturating_add(n)); + p.total.store(known, std::sync::atomic::Ordering::Relaxed); } Self(body, progress) } + + /// `body`, part of an answer (its last bytes), counted without touching the total. + fn part(body: R) -> Self { + Self(body, super::current_progress()) + } } impl std::io::Read for Wanted { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - super::check_current().map_err(|_| std::io::Error::other(CANCELLED_READ))?; + super::check_current().map_err(|_| std::io::Error::other(super::CANCELLED_READ))?; let n = self.0.read(buf)?; if let Some(p) = &self.1 { p.done.fetch_add(n as u64, std::sync::atomic::Ordering::Relaxed); @@ -2967,7 +2976,7 @@ mod native { // Read one byte past `n` (no more, and no error at the limit): a longer file is // reported as a server without ranges, so the file is downloaded whole instead. let limit = n.min(max as u64).saturating_add(1); - Wanted::new(resp.body_mut().with_config().limit(limit.saturating_add(1)).reader(), None) + Wanted::part(resp.body_mut().with_config().limit(limit.saturating_add(1)).reader()) .take(limit) .read_to_end(&mut buf) .map_err(|e| e.to_string())?; @@ -3763,6 +3772,9 @@ mod tests { let diagnostics = RequestDiagnostics::new(url, "GET"); diagnostics.failed(&Failure::new(url, DROPPED)); assert!(network_records(&history, url).is_empty()); + // Nor is a download's Cancel during its read. + diagnostics.failed(&Failure::new(url, CANCELLED_READ)); + assert!(network_records(&history, url).is_empty()); diagnostics.failed(&Failure::new(url, "the browser couldn't reach it")); let records = network_records(&history, url); assert_eq!(records.len(), 1); @@ -4203,7 +4215,55 @@ mod tests { assert!(progress.fraction().is_some_and(|f| f > 0.0 && f < 1.0), "{progress:?}"); wanted.store(false, Ordering::Relaxed); let r = rx.recv_timeout(std::time::Duration::from_secs(5)).expect("the download kept reading its body"); - assert!(r.as_ref().is_err_and(|e| e.contains(super::native::CANCELLED_READ)), "{r:?}"); + assert!(r.as_ref().is_err_and(|e| e.contains(super::CANCELLED_READ)), "{r:?}"); + } + + /// A later part of a download (a Shapefile's) whose answer doesn't say its size leaves the + /// total unknown, so the bar shows megabytes, not 100 % while its bytes still come. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn a_part_of_unknown_size_leaves_the_total_unknown() { + use std::io::Write; + let _turn = crate::cache::turn(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { return }; + let path = read_request(&mut stream).split_whitespace().nth(1).unwrap_or("").to_string(); + if path == "/a.shp" { + let _ = write!(stream, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\nConnection: close\r\n\r\n0123456789"); + continue; + } + // Chunked, so no length: a 1 kB chunk every 20 ms. + let _ = write!(stream, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"); + for _ in 0..200 { + if write!(stream, "3e8\r\n{}\r\n", "x".repeat(1000)).is_err() { + return; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + let _ = write!(stream, "0\r\n\r\n"); + } + }); + let progress = std::sync::Arc::new(super::Progress::default()); + let (tx, rx) = std::sync::mpsc::channel(); + let first = tx.clone(); + super::with_progress(Some(progress.clone()), || { + super::download_limited(&format!("http://{addr}/a.shp"), 1_000_000, Box::new(move |r| first.send(r).unwrap_or_default())) + }); + rx.recv_timeout(std::time::Duration::from_secs(10)).unwrap().unwrap(); + assert_eq!(progress.fraction(), Some(1.0)); + super::with_progress(Some(progress.clone()), || { + super::download_limited(&format!("http://{addr}/a.dbf"), 1_000_000, Box::new(move |r| tx.send(r).unwrap_or_default())) + }); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while progress.received() < 5_000 { + assert!(std::time::Instant::now() < deadline, "nothing counted: {progress:?}"); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert_eq!(progress.fraction(), None, "{progress:?}"); + assert!(rx.recv_timeout(std::time::Duration::from_secs(20)).unwrap().is_ok()); } #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index f3578577f6..27786071fc 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -198,6 +198,20 @@ static WINDOW_TURN: std::sync::Mutex<()> = std::sync::Mutex::new(()); #[cfg(not(target_arch = "wasm32"))] const MAX_WINDOW_READS: usize = 16; +/// A file downloading from its web address, to be added when it lands ([`UiState::downloads`]). +pub struct Download { + /// The session's download id. + pub id: u64, + /// Its address. + pub url: String, + /// The parameters to add it with. + pub params: Value, + /// Where it lands. While the list holds this, the download is wanted: Cancel takes it out. + pub slot: Slot, String>>, + /// How far it has come (the status bar's). + pub progress: std::sync::Arc, +} + /// A download or a window's read under way, for the status bar ([`RustGisApp::downloads_under_way`]). #[derive(Debug)] pub(crate) struct UnderWay { @@ -759,10 +773,8 @@ pub struct UiState { pub pending_csv: Option, /// CRS override for the next Shapefile picked by Import Shapefile with Projection. pub pending_shapefile: Option, - /// Data downloading from web addresses (Home › Add › Add Data › From URL): the address, the parameters to - /// add it with, where the download lands, and how far it has come (the status bar's). - pub downloads: - Vec<(u64, String, Value, Slot, String>>, std::sync::Arc)>, + /// Data downloading from web addresses (Home › Add › Add Data › From URL). + pub downloads: Vec, /// A project downloading from its web address (`ui.openUrl`, a `?project=` link): the /// session's download id, the address and where the download lands. A later one replaces it; /// opening another project meanwhile drops it. @@ -4550,7 +4562,7 @@ impl RustGisApp { ) }) }); - self.ui.downloads.push((id, url, q, slot, progress)); + self.ui.downloads.push(Download { id, url, params: q, slot, progress }); Ok(Value::Null) }) @@ -4633,9 +4645,9 @@ impl RustGisApp { } } let mut landed = Vec::new(); - self.ui.downloads.retain(|(id, url, p, slot, _)| match slot.lock().unwrap_or_else(|e| e.into_inner()).take() { + self.ui.downloads.retain(|d| match d.slot.lock().unwrap_or_else(|e| e.into_inner()).take() { Some(r) => { - landed.push((*id, url.clone(), p.clone(), r)); + landed.push((d.id, d.url.clone(), d.params.clone(), r)); false } None => true, @@ -4682,7 +4694,7 @@ impl RustGisApp { shown: rustgis_engine::sources::address_in_message(url), progress: progress.cloned(), }; - let downloads = self.ui.downloads.iter().map(|(id, url, _, _, progress)| shown(*id, url, Some(progress))); + let downloads = self.ui.downloads.iter().map(|d| shown(d.id, &d.url, Some(&d.progress))); #[cfg(not(target_arch = "wasm32"))] let downloads = downloads.chain(self.ui.window_reads.iter().map(|(id, url, ..)| shown(*id, url, None))); let mut under_way: Vec<_> = downloads.collect(); @@ -4696,9 +4708,9 @@ impl RustGisApp { /// at the next bytes that come, as its download is no longer wanted (`add_url`); in the /// browser it runs to its end, and what it brings is thrown away. pub(crate) fn cancel_download(&mut self, id: u64) { - let cancelled = self.ui.downloads.iter().position(|d| d.0 == id).map(|i| { - let (_, url, p, _, _) = self.ui.downloads.remove(i); - (url, p) + let cancelled = self.ui.downloads.iter().position(|d| d.id == id).map(|i| { + let d = self.ui.downloads.remove(i); + (d.url, d.params) }); #[cfg(not(target_arch = "wasm32"))] let cancelled = cancelled.or_else(|| { diff --git a/crates/ui-egui/src/pluginstart.rs b/crates/ui-egui/src/pluginstart.rs index 25adeb2d7c..46fa5e1e88 100644 --- a/crates/ui-egui/src/pluginstart.rs +++ b/crates/ui-egui/src/pluginstart.rs @@ -232,8 +232,8 @@ mod tests { let (asked, limit) = cache::split_download_limit(&asked); assert_eq!(asked, url); assert_eq!(limit, Some(cache::MAX_DOWNLOAD)); - assert_eq!(app.ui.downloads[0].2["zoom"], zoom, "completion preserves the claim's JSON representation"); - assert_eq!(app.ui.downloads[0].2["request"], receipt, "completion retains the opaque request's exact receipt"); + assert_eq!(app.ui.downloads[0].params["zoom"], zoom, "completion preserves the claim's JSON representation"); + assert_eq!(app.ui.downloads[0].params["request"], receipt, "completion retains the opaque request's exact receipt"); done(Ok(b"east,north\n1,2\n3,4\n".to_vec())); app.poll_data_downloads(); let places = app.session.map().layer_by_name(&name).unwrap(); diff --git a/crates/ui-egui/src/serviceui.rs b/crates/ui-egui/src/serviceui.rs index ad6934759c..f55b91815b 100644 --- a/crates/ui-egui/src/serviceui.rs +++ b/crates/ui-egui/src/serviceui.rs @@ -627,7 +627,7 @@ mod tests { assert_eq!(seen.len(), 2, "{:?}", app.session.notices); assert!(seen[1].contains("REQUEST=GetCoverage")); assert_eq!(logins::apply(&seen[1]).unwrap().is_some(), !anonymous); - let (_, _, params, _, _) = app.ui.downloads.first().expect("the coverage download was dispatched"); + let params = &app.ui.downloads.first().expect("the coverage download was dispatched").params; assert_eq!(params.get("connection").and_then(Value::as_str), (!anonymous).then_some(id.as_str())); assert_eq!(params.get("anonymous").and_then(Value::as_bool).unwrap_or(false), anonymous); } else { diff --git a/crates/ui-egui/src/stacui.rs b/crates/ui-egui/src/stacui.rs index 97d6e0027c..014c820a3f 100644 --- a/crates/ui-egui/src/stacui.rs +++ b/crates/ui-egui/src/stacui.rs @@ -1264,7 +1264,7 @@ mod tests { app.ui.stac.waiting = Some((asked, slot)); app.ui.stac.connection = two["id"].as_str().map(str::to_string); poll(&mut app, &egui::Context::default()); - let (_, _, params, _, _) = app.ui.downloads.last().unwrap(); + let params = &app.ui.downloads.last().unwrap().params; assert_eq!(params["connection"], one["id"], "token completion must not re-read the picker"); assert_ne!(params["connection"], two["id"]); }