From f1ad89db1fa13e8cac5eb3575927e49d31b03c21 Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Thu, 5 Nov 2026 11:49:54 +0100 Subject: [PATCH 1/3] feat(ui): a Zarr layer following the time slider reads its step off the frame (#1291 step 3) In the desktop app, moving the time slider no longer waits for a Zarr step that hasn't been read: the step is read on a thread of its own (at most two at once), the layer showing the step before until it lands, and the app then draws it (`Session::follow_time_landed`, asked each frame). Playing waits for each step to be read before it moves on, so every step shows, as when the read waits. A step that can't be read empties the layer and says why once it lands; a step read before (the last eight, and a web store's two read ahead) shows at once. Scripts and the command line still wait for each step (`Session::zarr_off_frame` is off), and an animation's frames (`time.export`) read each step before they draw. --- crates/engine/src/cmd/time.rs | 5 +- crates/engine/src/cmd/zarr.rs | 172 ++++++++++++++++++++++++++++++++++ crates/engine/src/lib.rs | 25 +++++ crates/engine/src/tests.rs | 46 +++++++++ crates/map/src/source.rs | 9 ++ crates/ui-egui/src/lib.rs | 6 ++ crates/ui-egui/src/timebar.rs | 7 ++ docs/user-guide/data.md | 14 ++- 8 files changed, 277 insertions(+), 7 deletions(-) diff --git a/crates/engine/src/cmd/time.rs b/crates/engine/src/cmd/time.rs index 9912e7e072..6c1bc77f97 100644 --- a/crates/engine/src/cmd/time.rs +++ b/crates/engine/src/cmd/time.rs @@ -316,8 +316,9 @@ fn frames(s: &mut Session, steps: &[f64], w: f64, h: f64, gif: Option) -> R sl.current = *t; } s.map_mut().apply_time(); - // Zarr layers that follow the slider read this step's slice. - super::zarr::follow_time(s); + // Zarr layers that follow the slider read this step's slice, here: each frame draws + // its own step. + super::zarr::follow_time_now(s); // Dated tile layers have this step's address now: natively, map_png (through // rustgis_render::render_complete) fetches the tiles it lacks and waits for them // before it draws, so each frame shows its own date's imagery. diff --git a/crates/engine/src/cmd/zarr.rs b/crates/engine/src/cmd/zarr.rs index d8add19688..63ab7fc580 100644 --- a/crates/engine/src/cmd/zarr.rs +++ b/crates/engine/src/cmd/zarr.rs @@ -392,6 +392,9 @@ fn read_ahead(zt: ZarrTime, i: usize) { } } READ_AHEAD_THREADS.fetch_sub(1, Ordering::SeqCst); + // A step given back unread (a read before it failed, or panicked) is read off the + // frame by the next follow, which this wakes. + landed(); } } let ids: Vec = keys.iter().map(|(_, k)| k.clone()).collect(); @@ -402,6 +405,8 @@ fn read_ahead(zt: ZarrTime, i: usize) { for (n, (j, k)) in keys.iter().enumerate() { let ok = read_step(&zt, *j, k.clone()).is_ok(); release(&keys[n..=n]); + // A layer waiting for this step off the frame takes it now. + landed(); if !ok { break; } @@ -440,6 +445,11 @@ pub(crate) fn read_step(zt: &ZarrTime, i: usize, key: String) -> Result Res Ok(new) } +/// Steps read off the frame (see [`follow_time`]) whose read failed, by key, with why: said by +/// the next [`follow_time`], once. +#[cfg(not(target_arch = "wasm32"))] +fn failed_reads() -> &'static std::sync::Mutex> { + static F: std::sync::OnceLock>> = std::sync::OnceLock::new(); + F.get_or_init(Default::default) +} + +/// Failed reads kept to be said (see [`failed_reads`]). +#[cfg(not(target_arch = "wasm32"))] +const FAILED_KEPT: usize = 16; + +/// The most threads reading steps off the frame at once: a step past them is started by the +/// next follow, which a read landing wakes. +#[cfg(not(target_arch = "wasm32"))] +const MAX_OFF_FRAME: usize = 2; + +#[cfg(not(target_arch = "wasm32"))] +static OFF_FRAME_THREADS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// Bumped each time a step read off the frame (or ahead) lands, so the app follows again. +#[cfg(not(target_arch = "wasm32"))] +static LANDED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// A step read in the background has landed: the app looks again ([`landed_since`]) and draws it. +#[cfg(not(target_arch = "wasm32"))] +fn landed() { + LANDED.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + rustgis_map::source::wake(); +} + +/// Whether a layer of the active map that follows the time slider waits for its step to be read +/// off the frame: playing waits for it, as it waits for the step to be read where scripts run. +pub(crate) fn waiting(s: &Session) -> bool { + let m = s.map(); + let window = m.time.as_ref().map(rustgis_map::time::TimeSlider::window); + m.layers.iter().filter_map(|l| l.raster.as_ref()?.time_zarr.as_ref()).any(|zt| zt.step_for(window) != zt.shown) +} + +/// Whether a step read in the background has landed since `seen`, which it moves on. +pub(crate) fn landed_since(seen: &mut u64) -> bool { + #[cfg(not(target_arch = "wasm32"))] + { + let now = LANDED.load(std::sync::atomic::Ordering::SeqCst); + std::mem::replace(seen, now) != now + } + #[cfg(target_arch = "wasm32")] + { + let _ = seen; + false + } +} + +/// Starts reading step `i` (kept as `key`) on a thread of its own, unless it is being read +/// already (ahead, or for another layer) or [`MAX_OFF_FRAME`] are under way: when one lands, +/// [`follow_time`] shows it or starts the next. A read that fails is kept to be said; no thread +/// to be had reads it here. +#[cfg(not(target_arch = "wasm32"))] +fn read_off_frame(zt: &ZarrTime, i: usize, key: String) -> Result<(), String> { + use std::sync::atomic::Ordering; + if OFF_FRAME_THREADS.fetch_add(1, Ordering::SeqCst) >= MAX_OFF_FRAME { + OFF_FRAME_THREADS.fetch_sub(1, Ordering::SeqCst); + return Ok(()); + } + if !reading_ahead().lock().is_ok_and(|mut r| r.insert(key.clone())) { + OFF_FRAME_THREADS.fetch_sub(1, Ordering::SeqCst); + return Ok(()); + } + // Whatever ends the read, a panic on a hostile store included, its key and its place among + // the threads are given back, and the app looks again. + struct Done(String); + impl Drop for Done { + fn drop(&mut self) { + if let Ok(mut r) = reading_ahead().lock() { + r.remove(&self.0); + } + OFF_FRAME_THREADS.fetch_sub(1, Ordering::SeqCst); + landed(); + } + } + let (zt, done) = (zt.clone(), Done(key.clone())); + let capture = rustgis_data::diagnostics::Capture::current(); + let spawned = std::thread::Builder::new().name("zarr-step".into()).spawn(move || { + let _diagnostic_scope = capture.enter(); + let _done = done; + let read = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| read_step(&zt, i, key.clone()))) + .unwrap_or_else(|_| Err(format!("{}: reading step {i} failed unexpectedly", zt.store))); + if let Err(e) = read + && let Ok(mut f) = failed_reads().lock() + { + f.retain(|(k, _)| *k != key); + f.push((key, e)); + // The last few: one the slider has long left is read again when it comes back. + let excess = f.len().saturating_sub(FAILED_KEPT); + f.drain(..excess); + } + }); + // No thread: its key was given back with the closure; the step is read here instead. + spawned.map(|_| ()).map_err(|e| e.to_string()) +} + /// Layers that follow the time slider read the step it shows, in every map: the last of their /// steps in the slider's window, or none (the layer draws nothing) when no step falls in it. A /// web store's next steps are then read ahead in the background, so playing doesn't wait for /// each. A slice that can't be read leaves its layer empty, and says why. +/// +/// In the app (`Session::zarr_off_frame`, #1291 step 3) a step not read yet is read on a thread +/// of its own, the layer showing the step before until it lands; the app then follows again +/// ([`Session::follow_time_landed`]). Scripts and the command line wait for it here. pub(crate) fn follow_time(s: &mut Session) { + let wait = !s.zarr_off_frame; + follow(s, wait); +} + +/// [`follow_time`], waiting for each step here wherever it runs: for an animation's frames, +/// each of which draws its own step. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn follow_time_now(s: &mut Session) { + follow(s, true); +} + +#[cfg_attr(target_arch = "wasm32", allow(unused_variables))] +fn follow(s: &mut Session, wait: bool) { #[cfg(not(target_arch = "wasm32"))] { let mut errors = Vec::new(); let mut ahead: Vec<(ZarrTime, usize)> = Vec::new(); + let mut said_failed: Vec = Vec::new(); for m in &mut s.project.maps { let window = m.time.as_ref().map(rustgis_map::time::TimeSlider::window); for l in &mut m.layers { @@ -534,6 +663,44 @@ pub(crate) fn follow_time(s: &mut Session) { rl.data = None; zt.shown = None; } + // Off the frame: a step kept is shown now, one that failed says why, and any + // other is read on a thread of its own while the layer shows the step before. + Some(i) if !wait && !is_kept(&slice_key(zt, i)) => { + let key = slice_key(zt, i); + // Said for every layer on that step, then dropped (below). + let failed = failed_reads().lock().ok().and_then(|f| f.iter().find(|(k, _)| *k == key).map(|(_, e)| e.clone())); + if failed.is_some() { + said_failed.push(key.clone()); + } + match failed { + Some(e) => { + rl.data = None; + zt.shown = Some(i); + errors.push(format!("{name}: {e}")); + } + None => { + // A web store's next steps too, as when the step is read here. + if is_web(&zt.store) { + ahead.push((zt.clone(), i)); + } + if let Err(e) = read_off_frame(zt, i, key) { + // No thread to read it on: read here, as a script does. + log::warn!("{name}: reading step {i} here: {e}"); + match read_waiting(zt, i) { + Ok(r) => { + rl.data = Some(r); + zt.shown = Some(i); + } + Err(e) => { + rl.data = None; + zt.shown = Some(i); + errors.push(format!("{name}: {e}")); + } + } + } + } + } + } Some(i) => match read_waiting(zt, i) { Ok(r) => { rl.data = Some(r); @@ -551,6 +718,11 @@ pub(crate) fn follow_time(s: &mut Session) { } } } + if !said_failed.is_empty() + && let Ok(mut f) = failed_reads().lock() + { + f.retain(|(k, _)| !said_failed.contains(k)); + } for e in errors { s.notify(e, true); } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 688bd9fd02..cee4deb899 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -739,6 +739,13 @@ pub struct Session { /// one succeeds) through this. Python leaves it unset: a run there holds the interpreter's /// lock and can't be interrupted, so it keeps the Script pane's bounds. pub long_runs: Option, + /// Zarr layers that follow the time slider read a step not read yet on a thread of their + /// own, showing the step before until it lands (the desktop app; see + /// [`Session::follow_time_landed`]). Off, as for scripts and the command line, a slider move + /// waits for the step. + pub zarr_off_frame: bool, + /// The Zarr steps landed that [`Session::follow_time_landed`] has seen. + zarr_seen: u64, /// Set where a person at a terminal waits on a command (`rustgis-cli run` with stdin and /// stderr a terminal), who can stop it with Ctrl-C: tools and rasters take no work cap /// ([`rustgis_analysis::control::Control::stoppable`]). A program that calls it (an agent, a @@ -989,6 +996,8 @@ impl Session { raster_results: Vec::new(), raster_runs: Vec::new(), long_runs: None, + zarr_off_frame: false, + zarr_seen: 0, stoppable: false, }; for text in label_notices { @@ -1123,6 +1132,22 @@ impl Session { self.project.map_mut() } + /// Whether a Zarr layer on the active map that follows the time slider is waiting for its + /// step to be read off the frame ([`Session::zarr_off_frame`]): playing holds its next step. + pub fn zarr_waiting(&self) -> bool { + self.zarr_off_frame && cmd::zarr::waiting(self) + } + + /// Shows the Zarr steps read off the frame ([`Session::zarr_off_frame`]) once they land: the + /// app asks each frame. True when one landed, so the map draws again. + pub fn follow_time_landed(&mut self) -> bool { + if !cmd::zarr::landed_since(&mut self.zarr_seen) { + return false; + } + cmd::zarr::follow_time(self); + true + } + /// Moves the map to the view of a style added with `"view": true` once the style has loaded, /// or to an archive's bounds once its header is read (the UI asks each frame). True when the /// map moved. diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index d743bfecdf..832c21a07f 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -16948,6 +16948,52 @@ fn a_zarr_variable_follows_the_time_slider() { let _ = std::fs::remove_dir_all(dir); } +/// In the app a time slider move doesn't wait for a Zarr step (#1291 step 3): the step is read on +/// a thread of its own, the layer showing the step before until it lands; one that fails says why +/// once it lands. An animation's frames still read each step before they draw. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn a_zarr_step_is_read_off_the_frame_in_the_app() { + let (dir, path) = zarr_store_over_time("offframe"); + let mut s = Session::new(Project::default()); + s.zarr_off_frame = true; + let id = s.run("layer.addZarr", &json!({"path": path, "variable": "t", "time": true})).unwrap()["layer"].as_u64().unwrap() as u32; + let first = |s: &Session| s.map().layer(id).and_then(|l| l.raster.as_ref()).and_then(|r| r.data.as_ref()).map(|d| d.bands[0].data[0]); + assert_eq!(first(&s), Some(8.0)); + let landed = |s: &mut Session, want: Option| { + let end = std::time::Instant::now() + std::time::Duration::from_secs(30); + while first(s) != want { + assert!(std::time::Instant::now() < end, "the step never landed: {:?}", first(s)); + s.follow_time_landed(); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + }; + // The move returns at once: the layer shows the step before, then step 2 once it lands; + // playing waits for it meanwhile. + s.run("time.goTo", &json!({"step": 2})).unwrap(); + landed(&mut s, Some(208.0)); + assert!(!s.zarr_waiting(), "the step shown is the slider's"); + // A step read before shows at once. + s.run("time.goTo", &json!({"step": 0})).unwrap(); + assert_eq!(first(&s), Some(8.0), "a kept step shows without waiting"); + // A step that can't be read empties the layer and says why, once. + std::fs::write(std::path::Path::new(&path).join("t/1.0.0"), b"short").unwrap(); + let mark = s.notice_mark(); + s.run("time.goTo", &json!({"step": 1})).unwrap(); + landed(&mut s, None); + let errors: Vec<_> = s.notices_since(mark).iter().filter(|n| n.error).map(|n| n.text.clone()).collect(); + assert_eq!(errors.len(), 1, "{errors:?}"); + s.follow_time_landed(); + assert_eq!(s.notices_since(mark).iter().filter(|n| n.error).count(), 1, "said once"); + // The animation's frames draw their own steps: step 2 is read before its frame. + let zip = std::env::temp_dir().join(format!("rustgis-zarr-offframe-{}.zip", std::process::id())); + s.run("time.goTo", &json!({"step": 0})).unwrap(); + let r = s.run("time.export", &json!({"path": zip.display().to_string(), "width": 64, "height": 48, "from": "2001-03-01"})).unwrap(); + assert_eq!(r["frames"], 1, "{r}"); + let _ = std::fs::remove_file(zip); + let _ = std::fs::remove_dir_all(dir); +} + /// A Zarr store in a folder: its variables listed (with time labels), a slice added by command /// and by Add Data, saved by folder and selection and read again when the project opens; a slice /// the app read itself handed over by id. diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index 3882c81319..5d1af60e63 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -1136,6 +1136,15 @@ fn waker() -> &'static Mutex>> { W.get_or_init(|| Mutex::new(None)) } +/// Something read in the background (outside this module: a Zarr step) has landed: the map +/// redraws ([`generation`]) and the UI is woken. +pub fn wake() { + GENERATION.fetch_add(1, Ordering::Relaxed); + if let Some(w) = waker().lock().ok().and_then(|w| w.clone()) { + w(); + } +} + /// Called whenever a background read finishes (the UI asks for a repaint). pub fn set_waker(f: impl Fn() + Send + Sync + 'static) { if let Ok(mut w) = waker().lock() { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 859f8aaeae..5a997a407e 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1130,6 +1130,8 @@ impl RustGisApp { // The installed fonts a label can be drawn in, indexed while the app starts. rustgis_engine::render::families::warm_up(); let mut s = session; + // A time slider move doesn't wait for a Zarr step to be read (#1291 step 3). + s.zarr_off_frame = cfg!(not(target_arch = "wasm32")); if s.active_layer.is_none() { s.active_layer = s.map().layers.first().map(|l| l.id); } @@ -4377,6 +4379,10 @@ impl RustGisApp { if self.session.plugin_retries_waiting() { ctx.request_repaint_after(std::time::Duration::from_millis(100)); } + // Zarr steps the time slider moved to, read off the frame, show once they land. + if self.session.follow_time_landed() { + self.map.invalidate(); + } // A style added with `"view": true` moves the map once it has loaded. if self.session.style_view.is_some() { if self.session.follow_style_view() { diff --git a/crates/ui-egui/src/timebar.rs b/crates/ui-egui/src/timebar.rs index 5382631e60..2c0c898099 100644 --- a/crates/ui-egui/src/timebar.rs +++ b/crates/ui-egui/src/timebar.rs @@ -125,6 +125,13 @@ fn advance(app: &mut RustGisApp, ui: &Ui) { return; }; let elapsed = elapsed + f64::from(ui.input(|i| i.stable_dt).min(0.25)) * 1000.0; + // A Zarr step still being read holds the next one: playing goes at the pace the steps are + // read, as it does where the read waits, and each step shows. + if elapsed >= f64::from(t.speed_ms) && app.session.zarr_waiting() { + app.time.playing = Some(elapsed); + ui.ctx().request_repaint_after(web_time::Duration::from_millis(50)); + return; + } if elapsed >= f64::from(t.speed_ms) { if t.at_end() && !t.looping { app.time.playing = None; diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 3e33ef6197..c21b838168 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -1089,11 +1089,15 @@ rustgis-cli run --cmd 'layer.addZarr={"url":"https://storage.googleapis.com/gcp- layer whose step is the time slider's. The slider turns on over the variable's dates when it is off, and the layer shows the last of its steps in the slider's window, or nothing when none falls in it. Moving or playing the slider reads that step's slice (only its chunks), keeping the last -eight read. A web store's next two steps are read ahead in the background. The stretch is fixed at -the first slice's range, so a colour means the same value at every step, and **Export** writes an -animation of it. An axis longer than the slider's 20,000 steps (hourly ERA5) takes a period with `"start"` and -`"end"`; given neither (as the dialog adds it), the layer takes 20,000 steps from the one picked. The project keeps such a layer by its store (a folder or a web address) -and selection, not as a slice, and reads the slider's step again when it opens. +eight read. A web store's next two steps are read ahead in the background. In the desktop app the +slider doesn't wait for a step not read yet: the layer shows the step before until the new one is +read, and a step that can't be read empties the layer and says why. Playing waits for each step to +be read before it moves on, so every step shows. Scripts, the command line and **Export** wait for +each step. The stretch is fixed at the first slice's range, so a colour means the same value at +every step, and **Export** writes an animation of it. An axis longer than the slider's 20,000 steps +(hourly ERA5) takes a period with `"start"` and `"end"`; given neither (as the dialog adds it), the +layer takes 20,000 steps from the one picked. The project keeps such a layer by its store (a folder +or a web address) and selection, not as a slice, and reads the slider's step again when it opens. ```sh rustgis-cli run --cmd 'layer.addZarr={"path":"air.zarr","variable":"t","time":true}' --cmd 'time.export={"path":"t.gif"}' From 2e2f9c7b590ec34fcdd3f5bd2f12d62d5c1acb44 Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Thu, 5 Nov 2026 12:37:22 +0100 Subject: [PATCH 2/3] fix(zarr): only the user's own slider moves read off the frame; failures said once per cause - Agents, models' and tasks' steps, notebooks and nested commands wait for the step, as their next command may read the layer; `time.export` restores the slider's step the same way. - A kept step is looked up once; a miss goes off the frame, never read on the UI thread. - A failure kept for a step no layer waits for any more is dropped, so coming back reads it again; a read-ahead that failed isn't read ahead again until the step is read. - Layers emptied by one cause are said in one notice with a count. - A landed step redraws the layout page too, and shows for a whole step while playing. - The Zarr tests that read steps take one lock on the shared slice cache. --- crates/engine/src/cmd/time.rs | 3 +- crates/engine/src/cmd/zarr.rs | 88 +++++++++++++++++++++++++++-------- crates/engine/src/lib.rs | 2 +- crates/engine/src/tests.rs | 25 ++++++++++ crates/ui-egui/src/lib.rs | 2 + crates/ui-egui/src/timebar.rs | 9 ++-- docs/user-guide/data.md | 13 +++--- 7 files changed, 111 insertions(+), 31 deletions(-) diff --git a/crates/engine/src/cmd/time.rs b/crates/engine/src/cmd/time.rs index 6c1bc77f97..a0f5acb9a6 100644 --- a/crates/engine/src/cmd/time.rs +++ b/crates/engine/src/cmd/time.rs @@ -282,7 +282,8 @@ fn export(s: &mut Session, p: &Value) -> CmdResult { let made = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| frames(s, &steps, w, h, gif.then_some(speed)))); s.map_mut().time = Some(saved); s.map_mut().apply_time(); - super::zarr::follow_time(s); + // The step it is back at, read here: a command after the export may read the layer. + super::zarr::follow_time_now(s); let bytes = made.map_err(|_| "exporting the time steps failed unexpectedly".to_string())??; rustgis_data::io::write_atomic(std::path::Path::new(&path), &bytes).map_err(|e| format!("{path}: {e}"))?; Ok(json!({"path": path, "frames": steps.len(), "bytes": bytes.len()})) diff --git a/crates/engine/src/cmd/zarr.rs b/crates/engine/src/cmd/zarr.rs index 63ab7fc580..e0dd7f215c 100644 --- a/crates/engine/src/cmd/zarr.rs +++ b/crates/engine/src/cmd/zarr.rs @@ -364,7 +364,7 @@ fn read_ahead(zt: ZarrTime, i: usize) { let keys: Vec<(usize, String)> = (i + 1..zt.dates.len()) .take(2) .map(|j| (j, slice_key(&zt, j))) - .filter(|(_, k)| !is_kept(k) && reading_ahead().lock().is_ok_and(|mut r| r.insert(k.clone()))) + .filter(|(_, k)| !is_kept(k) && !ahead_failed(k) && reading_ahead().lock().is_ok_and(|mut r| r.insert(k.clone()))) .collect(); if keys.is_empty() { return; @@ -404,6 +404,14 @@ fn read_ahead(zt: ZarrTime, i: usize) { let _done = Done(keys.iter().map(|(_, k)| k.clone()).collect()); for (n, (j, k)) in keys.iter().enumerate() { let ok = read_step(&zt, *j, k.clone()).is_ok(); + // Not read ahead again (each follow would): the slider reaching it reads it off the + // frame, and says why it fails. + if !ok && let Ok(mut f) = failed_ahead().lock() { + f.retain(|x| x != k); + f.push(k.clone()); + let excess = f.len().saturating_sub(FAILED_KEPT); + f.drain(..excess); + } release(&keys[n..=n]); // A layer waiting for this step off the frame takes it now. landed(); @@ -418,6 +426,18 @@ fn read_ahead(zt: ZarrTime, i: usize) { } } +/// Steps whose read ahead failed, the last few: not read ahead again until one is read. +#[cfg(not(target_arch = "wasm32"))] +fn failed_ahead() -> &'static std::sync::Mutex> { + static F: std::sync::OnceLock>> = std::sync::OnceLock::new(); + F.get_or_init(Default::default) +} + +#[cfg(not(target_arch = "wasm32"))] +fn ahead_failed(key: &str) -> bool { + failed_ahead().lock().is_ok_and(|f| f.iter().any(|k| k == key)) +} + /// Waits (up to 15 s) for a step being read ahead, so it isn't fetched twice; after that it is /// read here. #[cfg(not(target_arch = "wasm32"))] @@ -447,8 +467,13 @@ pub(crate) fn read_step(zt: &ZarrTime, i: usize, key: String) -> Result Result<(), String> { { f.retain(|(k, _)| *k != key); f.push((key, e)); - // The last few: one the slider has long left is read again when it comes back. + // The last few, while a layer still waits for the step (`follow` drops the rest). let excess = f.len().saturating_sub(FAILED_KEPT); f.drain(..excess); } @@ -628,13 +653,14 @@ fn read_off_frame(zt: &ZarrTime, i: usize, key: String) -> Result<(), String> { /// of its own, the layer showing the step before until it lands; the app then follows again /// ([`Session::follow_time_landed`]). Scripts and the command line wait for it here. pub(crate) fn follow_time(s: &mut Session) { - let wait = !s.zarr_off_frame; + // Only the user's own move, a command of its own: an agent, a model's or a task's step, a + // notebook and a script run from it wait for the step, as their next command may read it. + let wait = !s.zarr_off_frame || s.actor.is_some() || s.depth > 1 || s.task_running || s.notebook_running; follow(s, wait); } /// [`follow_time`], waiting for each step here wherever it runs: for an animation's frames, -/// each of which draws its own step. -#[cfg(not(target_arch = "wasm32"))] +/// each of which draws its own step, and the step the slider comes back to after them. pub(crate) fn follow_time_now(s: &mut Session) { follow(s, true); } @@ -646,6 +672,9 @@ fn follow(s: &mut Session, wait: bool) { let mut errors = Vec::new(); let mut ahead: Vec<(ZarrTime, usize)> = Vec::new(); let mut said_failed: Vec = Vec::new(); + // The steps layers wait for off the frame: a failure kept for any other is dropped, so + // the slider coming back to its step reads it again. + let mut waiting: Vec = Vec::new(); for m in &mut s.project.maps { let window = m.time.as_ref().map(rustgis_map::time::TimeSlider::window); for l in &mut m.layers { @@ -663,10 +692,19 @@ fn follow(s: &mut Session, wait: bool) { rl.data = None; zt.shown = None; } - // Off the frame: a step kept is shown now, one that failed says why, and any - // other is read on a thread of its own while the layer shows the step before. - Some(i) if !wait && !is_kept(&slice_key(zt, i)) => { + // Off the frame: a step kept is shown now (looked up once), one that failed says + // why, and any other is read on a thread of its own while the layer shows the + // step before. + Some(i) if !wait => { let key = slice_key(zt, i); + if let Some(r) = kept_slice(&key) { + rl.data = Some(r); + zt.shown = Some(i); + if is_web(&zt.store) { + ahead.push((zt.clone(), i)); + } + continue; + } // Said for every layer on that step, then dropped (below). let failed = failed_reads().lock().ok().and_then(|f| f.iter().find(|(k, _)| *k == key).map(|(_, e)| e.clone())); if failed.is_some() { @@ -676,9 +714,10 @@ fn follow(s: &mut Session, wait: bool) { Some(e) => { rl.data = None; zt.shown = Some(i); - errors.push(format!("{name}: {e}")); + errors.push((name, e)); } None => { + waiting.push(key.clone()); // A web store's next steps too, as when the step is read here. if is_web(&zt.store) { ahead.push((zt.clone(), i)); @@ -694,7 +733,7 @@ fn follow(s: &mut Session, wait: bool) { Err(e) => { rl.data = None; zt.shown = Some(i); - errors.push(format!("{name}: {e}")); + errors.push((name, e)); } } } @@ -712,19 +751,30 @@ fn follow(s: &mut Session, wait: bool) { Err(e) => { rl.data = None; zt.shown = Some(i); - errors.push(format!("{name}: {e}")); + errors.push((name, e)); } }, } } } - if !said_failed.is_empty() - && let Ok(mut f) = failed_reads().lock() - { - f.retain(|(k, _)| !said_failed.contains(k)); + if !wait && let Ok(mut f) = failed_reads().lock() { + f.retain(|(k, _)| waiting.contains(k) && !said_failed.contains(k)); } - for e in errors { - s.notify(e, true); + // One notice for each cause, naming the layers it emptied (with a count past one). + let mut causes: Vec<(String, Vec)> = Vec::new(); + for (name, e) in errors { + match causes.iter_mut().find(|(c, _)| *c == e) { + Some((_, names)) => names.push(name), + None => causes.push((e, vec![name])), + } + } + for (e, names) in causes { + let text = match names.as_slice() { + [one] => format!("{one}: {e}"), + [first, rest @ ..] => format!("{first} and {} more layers: {e}", rest.len()), + [] => e, + }; + s.notify(text, true); } for (zt, i) in ahead { read_ahead(zt, i); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index cee4deb899..9c11f61865 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -665,7 +665,7 @@ pub struct Session { /// What the project replaced during the command running held beside itself, to give back if /// the command fails ([`PutAside`]); and how deep commands run within commands now. put_aside: Option>, - depth: u32, + pub(crate) depth: u32, /// Only the command's registered id, never its parameters. Nested commands restore this. pub(crate) diagnostic_source: Option<&'static str>, /// How many of the commands running now restore the project if they fail: a replacement is diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 832c21a07f..cec9ff128d 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -15785,6 +15785,14 @@ fn lidar_sample_refuses_exports_and_statistics() { std::fs::remove_dir_all(dir).unwrap(); } +/// The Zarr tests that read steps share the process's slice cache: one at a time, so another's +/// steps don't push out the one a test expects kept. +#[cfg(not(target_arch = "wasm32"))] +fn zarr_cache_lock() -> std::sync::MutexGuard<'static, ()> { + static L: std::sync::Mutex<()> = std::sync::Mutex::new(()); + L.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + /// A Zarr store in a temporary folder: `t` over three monthly steps (days since 2001-01-01: 0, /// 31, 59) on a 3 × 4 grid, step `k`'s values `100·k + i`. #[cfg(not(target_arch = "wasm32"))] @@ -15839,6 +15847,7 @@ fn zarr_store_at(tag: &str, days: &[i32]) -> (std::path::PathBuf, String) { #[cfg(not(target_arch = "wasm32"))] #[test] fn a_zarr_series_at_a_point() { + let _cache = zarr_cache_lock(); let (dir, path) = zarr_store_over_time("series"); let mut s = Session::new(Project::default()); // At 1°E 30°N: the stored row 2 (latitudes 10, 20, 30), column 0, so 100·k + 8 at step k. @@ -15894,6 +15903,7 @@ fn a_zarr_series_at_a_point() { #[cfg(not(target_arch = "wasm32"))] #[test] fn a_zarr_variable_summarised_over_time() { + let _cache = zarr_cache_lock(); let (dir, path) = zarr_store_over_time("reduce"); let mut s = Session::new(Project::default()); // The first pixel (north up) is the stored cell 8: 8, 108 and 208 at the three steps. @@ -16866,6 +16876,7 @@ fn a_stac_collection_summarised_over_time() { #[cfg(not(target_arch = "wasm32"))] #[test] fn a_zarr_variable_follows_the_time_slider() { + let _cache = zarr_cache_lock(); let (dir, path) = zarr_store_over_time("follow"); let mut s = Session::new(Project::default()); let id = s.run("layer.addZarr", &json!({"path": path, "variable": "t", "time": true})).unwrap()["layer"].as_u64().unwrap() as u32; @@ -16954,6 +16965,7 @@ fn a_zarr_variable_follows_the_time_slider() { #[cfg(not(target_arch = "wasm32"))] #[test] fn a_zarr_step_is_read_off_the_frame_in_the_app() { + let _cache = zarr_cache_lock(); let (dir, path) = zarr_store_over_time("offframe"); let mut s = Session::new(Project::default()); s.zarr_off_frame = true; @@ -16985,6 +16997,19 @@ fn a_zarr_step_is_read_off_the_frame_in_the_app() { assert_eq!(errors.len(), 1, "{errors:?}"); s.follow_time_landed(); assert_eq!(s.notices_since(mark).iter().filter(|n| n.error).count(), 1, "said once"); + // Two layers on the failing step: one notice, with a count. + s.run("time.goTo", &json!({"step": 0})).unwrap(); + s.run("layer.addZarr", &json!({"path": path, "variable": "t", "time": true, "name": "again"})).unwrap(); + let mark = s.notice_mark(); + s.run("time.goTo", &json!({"step": 1})).unwrap(); + let end = std::time::Instant::now() + std::time::Duration::from_secs(30); + while s.zarr_waiting() { + assert!(std::time::Instant::now() < end, "the failing step never landed"); + s.follow_time_landed(); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + let errors: Vec<_> = s.notices_since(mark).iter().filter(|n| n.error).map(|n| n.text.clone()).collect(); + assert!(errors.len() == 1 && errors[0].contains("and 1 more layers"), "{errors:?}"); // The animation's frames draw their own steps: step 2 is read before its frame. let zip = std::env::temp_dir().join(format!("rustgis-zarr-offframe-{}.zip", std::process::id())); s.run("time.goTo", &json!({"step": 0})).unwrap(); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 5a997a407e..b4b2cfebd3 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -4382,6 +4382,8 @@ impl RustGisApp { // Zarr steps the time slider moved to, read off the frame, show once they land. if self.session.follow_time_landed() { self.map.invalidate(); + // A layout page showing the map draws the step too. + self.layout.redraw(); } // A style added with `"view": true` moves the map once it has loaded. if self.session.style_view.is_some() { diff --git a/crates/ui-egui/src/timebar.rs b/crates/ui-egui/src/timebar.rs index 2c0c898099..b79f6e6b27 100644 --- a/crates/ui-egui/src/timebar.rs +++ b/crates/ui-egui/src/timebar.rs @@ -125,10 +125,11 @@ fn advance(app: &mut RustGisApp, ui: &Ui) { return; }; let elapsed = elapsed + f64::from(ui.input(|i| i.stable_dt).min(0.25)) * 1000.0; - // A Zarr step still being read holds the next one: playing goes at the pace the steps are - // read, as it does where the read waits, and each step shows. - if elapsed >= f64::from(t.speed_ms) && app.session.zarr_waiting() { - app.time.playing = Some(elapsed); + // A Zarr step still being read holds the next one, and shows for a whole step once it lands: + // playing goes at the pace the steps are read, as it does where the read waits, and each + // step shows. + if app.session.zarr_waiting() { + app.time.playing = Some(0.0); ui.ctx().request_repaint_after(web_time::Duration::from_millis(50)); return; } diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index c21b838168..c5337c7bc1 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -1092,12 +1092,13 @@ in it. Moving or playing the slider reads that step's slice (only its chunks), k eight read. A web store's next two steps are read ahead in the background. In the desktop app the slider doesn't wait for a step not read yet: the layer shows the step before until the new one is read, and a step that can't be read empties the layer and says why. Playing waits for each step to -be read before it moves on, so every step shows. Scripts, the command line and **Export** wait for -each step. The stretch is fixed at the first slice's range, so a colour means the same value at -every step, and **Export** writes an animation of it. An axis longer than the slider's 20,000 steps -(hourly ERA5) takes a period with `"start"` and `"end"`; given neither (as the dialog adds it), the -layer takes 20,000 steps from the one picked. The project keeps such a layer by its store (a folder -or a web address) and selection, not as a slice, and reads the slider's step again when it opens. +be read before it moves on, so every step shows. Scripts, models, notebooks, AI agents, the command +line and **Export** wait for each step. The stretch is fixed at the first slice's range, so a +colour means the same value at every step, and **Export** writes an animation of it. An axis longer +than the slider's 20,000 steps (hourly ERA5) takes a period with `"start"` and `"end"`; given +neither (as the dialog adds it), the layer takes 20,000 steps from the one picked. The project +keeps such a layer by its store (a folder or a web address) and selection, not as a slice, and +reads the slider's step again when it opens. ```sh rustgis-cli run --cmd 'layer.addZarr={"path":"air.zarr","variable":"t","time":true}' --cmd 'time.export={"path":"t.gif"}' From aab6a2e328c241a7d2ac22ed9e7af3a6e5803a09 Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Thu, 5 Nov 2026 13:02:03 +0100 Subject: [PATCH 3/3] fix(zarr): a script's or plugin's slider move waits for its step `script.run` and a plugin's host calls run their commands directly, at the depth of the command running them, so a Script pane line moving the slider read its Zarr step off the frame and the next line read the step before. `Session::steps_running` counts commands run as another's steps (set by `workflows::steps_of`, on every path), and `follow_time` waits under it. The grouped failure notice says "1 more layer". --- crates/engine/src/cmd/plugins.rs | 2 +- crates/engine/src/cmd/workflows.rs | 11 ++++++++++- crates/engine/src/cmd/zarr.rs | 7 ++++--- crates/engine/src/lib.rs | 5 +++++ crates/engine/src/tests.rs | 14 +++++++++++++- 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index cf18380413..1603f632e5 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -566,7 +566,7 @@ impl Bridge<'_> { let anonymous = self.s.web_logins.set_anonymous(true); let mut res = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let before = if mutates { crate::service_logins::before(self.s, id, params)? } else { None }; - let out = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run(self.s, params))) + let out = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| super::workflows::steps_of(self.s, |s| run(s, params)))) .unwrap_or_else(|_| Err(format!("{label} failed unexpectedly"))); // Mark additions even when a command reports an error after changing the project. // Their addresses must remain anonymous at later logins and after saving/reopening. diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index 9876e2dcb8..0dabd83a86 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -636,12 +636,21 @@ pub(crate) fn run_script(s: &mut Session, p: &Value) -> CmdResult { if let Some(e) = s.streamed_edit(&id, mutates, label, s.active_layer) { return Err(format!("line {n}: {e}")); } - let r = run(s, ¶ms).map_err(|e| format!("line {n}: {id}: {e}"))?; + let r = steps_of(s, |s| run(s, ¶ms)).map_err(|e| format!("line {n}: {id}: {e}"))?; out.push(json!({"line": n, "command": id, "result": r})); } Ok(json!({"ran": out})) } +/// `run` as a step of the command running now ([`Session::steps_running`] counts it, on every +/// path, a panic included, which is raised again). +pub(crate) fn steps_of(s: &mut Session, run: impl FnOnce(&mut Session) -> T) -> T { + s.steps_running += 1; + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run(s))); + s.steps_running = s.steps_running.saturating_sub(1); + r.unwrap_or_else(|p| std::panic::resume_unwind(p)) +} + /// The most lines a script is read to: Run reads no further, and the Script pane colours no further. pub const MAX_SCRIPT_LINES: usize = 10_000; diff --git a/crates/engine/src/cmd/zarr.rs b/crates/engine/src/cmd/zarr.rs index e0dd7f215c..d1b7966495 100644 --- a/crates/engine/src/cmd/zarr.rs +++ b/crates/engine/src/cmd/zarr.rs @@ -654,8 +654,9 @@ fn read_off_frame(zt: &ZarrTime, i: usize, key: String) -> Result<(), String> { /// ([`Session::follow_time_landed`]). Scripts and the command line wait for it here. pub(crate) fn follow_time(s: &mut Session) { // Only the user's own move, a command of its own: an agent, a model's or a task's step, a - // notebook and a script run from it wait for the step, as their next command may read it. - let wait = !s.zarr_off_frame || s.actor.is_some() || s.depth > 1 || s.task_running || s.notebook_running; + // notebook, a script's line and a plugin's call wait for the step, as their next command may + // read it. + let wait = !s.zarr_off_frame || s.actor.is_some() || s.depth > 1 || s.task_running || s.notebook_running || s.steps_running > 0; follow(s, wait); } @@ -771,7 +772,7 @@ fn follow(s: &mut Session, wait: bool) { for (e, names) in causes { let text = match names.as_slice() { [one] => format!("{one}: {e}"), - [first, rest @ ..] => format!("{first} and {} more layers: {e}", rest.len()), + [first, rest @ ..] => format!("{first} and {} more layer{}: {e}", rest.len(), if rest.len() == 1 { "" } else { "s" }), [] => e, }; s.notify(text, true); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 9c11f61865..e6e9ce830c 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -641,6 +641,10 @@ pub struct Session { /// A notebook is running: it may not start another, even through a task or model (no /// unbounded recursion). pub(crate) notebook_running: bool, + /// How many commands are running other commands as their steps by calling them directly + /// (a script's lines, a plugin's host calls): what those steps do is the script's, not the + /// user's own move, so a Zarr step they move to is read before the next line runs. + pub(crate) steps_running: u32, /// How many geoprocessing history records came with the opened project (from its file). pub history_from_file: usize, /// The report open in the Report pane, if any. @@ -949,6 +953,7 @@ impl Session { project_starts: 0, trusted_notebooks: Default::default(), notebook_running: false, + steps_running: 0, history_from_file: 0, active_report: None, active_presentation: None, diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index cec9ff128d..1f58f7a386 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -16959,6 +16959,12 @@ fn a_zarr_variable_follows_the_time_slider() { let _ = std::fs::remove_dir_all(dir); } +/// Step `k`'s cells of [`zarr_store_at`]'s variable (`100 k + cell`), as its chunk file holds them. +#[cfg(not(target_arch = "wasm32"))] +fn f32s_of(base: i32) -> Vec { + (0..12).flat_map(|i| ((base + i) as f32).to_le_bytes()).collect() +} + /// In the app a time slider move doesn't wait for a Zarr step (#1291 step 3): the step is read on /// a thread of its own, the layer showing the step before until it lands; one that fails says why /// once it lands. An animation's frames still read each step before they draw. @@ -17009,7 +17015,13 @@ fn a_zarr_step_is_read_off_the_frame_in_the_app() { std::thread::sleep(std::time::Duration::from_millis(5)); } let errors: Vec<_> = s.notices_since(mark).iter().filter(|n| n.error).map(|n| n.text.clone()).collect(); - assert!(errors.len() == 1 && errors[0].contains("and 1 more layers"), "{errors:?}"); + assert!(errors.len() == 1 && errors[0].contains("and 1 more layer:"), "{errors:?}"); + // A script's line waits for its step, so the next line reads it. + s.run("time.goTo", &json!({"step": 0})).unwrap(); + std::fs::write(std::path::Path::new(&path).join("t/1.0.0"), f32s_of(100)).unwrap(); + let ran = s.run("script.run", &json!({"script": "time.goTo {\"step\": 1}"})).unwrap(); + assert!(ran["ran"].as_array().is_some_and(|r| r.len() == 1), "{ran}"); + assert_eq!(first(&s), Some(108.0), "a script's move is read before its next line"); // The animation's frames draw their own steps: step 2 is read before its frame. let zip = std::env::temp_dir().join(format!("rustgis-zarr-offframe-{}.zip", std::process::id())); s.run("time.goTo", &json!({"step": 0})).unwrap();