diff --git a/crates/engine/src/cmd/zarr.rs b/crates/engine/src/cmd/zarr.rs index 5b6b70271..c21cbec34 100644 --- a/crates/engine/src/cmd/zarr.rs +++ b/crates/engine/src/cmd/zarr.rs @@ -1403,6 +1403,7 @@ fn add_following(s: &mut Session, p: &Value) -> CmdResult { /// The store a series reads, and what of it: from a Zarr layer (one that follows the time slider, /// or a slice: its store, variable and steps), or from the parameters. #[cfg(not(target_arch = "wasm32"))] +#[derive(Clone, Debug)] struct SeriesSource { root: String, sel: zarr::Selection, @@ -1454,12 +1455,25 @@ fn series_source(s: &Session, p: &Value) -> Result { /// store's fetched. #[cfg(not(target_arch = "wasm32"))] fn store_files(root: &str, keys: &[String]) -> Result>, String> { + store_files_by(root, keys, &mut |_| Ok(())) +} + +/// The files of a store read in batches of up to [`SERIES_BATCH`] files, as [`store_files`] reads +/// them; `before` is called with each batch's size before it is read, and its error (a Cancel) +/// stops the read there: requests already sent finish, none more is made. +#[cfg(not(target_arch = "wasm32"))] +fn store_files_by( + root: &str, + keys: &[String], + before: &mut dyn FnMut(usize) -> Result<(), String>, +) -> Result>, String> { if is_web(root) { // In batches, within the same total a folder's read takes: a store chunked a map a step // (one chunk per step) can't fill the memory. let mut out = HashMap::new(); let mut total = 0u64; - for batch in keys.chunks(64) { + for batch in keys.chunks(SERIES_BATCH) { + before(batch.len())?; let (got, forbidden) = fetch(root, batch)?; // A file refused (HTTP 403) isn't one never written: said, not read as missing. if forbidden > 0 { @@ -1477,7 +1491,10 @@ fn store_files(root: &str, keys: &[String]) -> Result>, } let mut out = HashMap::new(); let mut total = 0u64; - for k in keys { + for (i, k) in keys.iter().enumerate() { + if i % SERIES_BATCH == 0 { + before((keys.len() - i).min(SERIES_BATCH))?; + } let path = Path::new(root).join(k); // A file that isn't there is a chunk never written (Zarr's fill value); one that is but // can't be taken is said, so a value isn't silently missing. @@ -1497,22 +1514,46 @@ fn store_files(root: &str, keys: &[String]) -> Result>, /// Bytes of a store's files ([`zarr::Part`]s), each kept by its id, as [`store_files`] reads /// whole files: a web store's fetched by range, a folder's read in place. +/// In batches of up to [`SERIES_BATCH`] parts, `before` called with each batch's size before it +/// is read, as [`store_files_by`] reads whole files. #[cfg(not(target_arch = "wasm32"))] -fn store_parts(root: &str, parts: &[zarr::Part]) -> Result>, String> { - if is_web(root) { - let (got, forbidden) = fetch_parts(root, parts)?; - if forbidden > 0 { - return Err(crate::sources::message_with_address(root, &format!("access denied (HTTP 403) to {forbidden} of its files"))); +fn store_parts( + root: &str, + parts: &[zarr::Part], + before: &mut dyn FnMut(usize) -> Result<(), String>, +) -> Result>, String> { + let mut out = HashMap::new(); + let mut total = 0u64; + for batch in parts.chunks(SERIES_BATCH) { + before(batch.len())?; + if is_web(root) { + let (got, forbidden) = fetch_parts(root, batch)?; + if forbidden > 0 { + return Err(crate::sources::message_with_address(root, &format!("access denied (HTTP 403) to {forbidden} of its files"))); + } + // Within one total over all batches, as a folder's parts and `store_files_by` are. + for (k, b) in got { + total = total.saturating_add(b.len() as u64); + if total > zarr::MAX_READ { + return Err(crate::sources::message_with_address(root, "the series is too large to read: take a shorter period")); + } + out.insert(k, b); + } + } else { + zarr::read_parts(Path::new(root), batch, &mut out, &mut total).map_err(|e| format!("{root}: {e}"))?; } - return Ok(got); } - let mut out = HashMap::new(); - zarr::read_parts(Path::new(root), parts, &mut out, &mut 0).map_err(|e| format!("{root}: {e}"))?; Ok(out) } +/// The files or parts a series reads in one go: a batch of requests in flight together, between +/// two looks at Cancel (opengeos/RustGIS#2282). +#[cfg(not(target_arch = "wasm32"))] +const SERIES_BATCH: usize = 64; + /// `zarr.series`: a variable's values along its time dimension at the cell holding a point, -/// reading only the chunks that hold that cell. +/// reading only the chunks that hold that cell. A script or the assistant waits for it here; the +/// desktop app reads it on a thread of its own with [`series_start`] (opengeos/RustGIS#2282). fn series(s: &mut Session, p: &Value) -> CmdResult { #[cfg(target_arch = "wasm32")] { @@ -1521,12 +1562,75 @@ fn series(s: &mut Session, p: &Value) -> CmdResult { } #[cfg(not(target_arch = "wasm32"))] { - let num = |k: &str| super::f(p, k).ok_or_else(|| format!("{k} is required (degrees)")); - let (lon, lat) = (num("lon")?, num("lat")?); - if !lon.is_finite() || !(-90.0..=90.0).contains(&lat) { - return Err("lon is a longitude and lat a latitude from -90 to 90 (degrees)".into()); + // What reading it said is said though the read then fails, as it was before #2282. + let mut notes = Vec::new(); + let series = series_start(s, p)?.read(&mut notes); + for n in notes { + s.warn(n); } - let mut src = series_source(s, p)?; + series + } +} + +/// A point's series, prepared on the session (the point and period checked, the store and its +/// variable chosen) and read anywhere with [`SeriesRun::run`]: the desktop app reads it on a +/// thread of its own, with the steps read and Cancel, so a long series doesn't hold the window +/// (opengeos/RustGIS#2282). +#[cfg(not(target_arch = "wasm32"))] +#[derive(Clone, Debug)] +pub struct SeriesRun { + lon: f64, + lat: f64, + src: SeriesSource, + start: Option, + end: Option, +} + +/// The steps a series has read, for its progress: the reads made of those it plans (the chunks, +/// or each shard's index and then its chunk), as steps of the series. It looks at Cancel before +/// each batch of reads. +#[cfg(not(target_arch = "wasm32"))] +struct SeriesTally { + steps: usize, + reads: usize, + /// The reads of the batches done, and of the one in flight. + done: usize, + flight: usize, + /// The steps counted so far. + shown: usize, +} + +#[cfg(not(target_arch = "wasm32"))] +impl SeriesTally { + /// Counts the batch before as read, then stops the series if it was cancelled; `n` is the size + /// of the batch about to be read. + fn before(&mut self, n: usize) -> Result<(), String> { + self.done += std::mem::replace(&mut self.flight, n); + let steps = self.steps * self.done.min(self.reads) / self.reads.max(1); + rustgis_data::progress::step(steps.saturating_sub(self.shown)).map_err(|_| "cancelled".to_string())?; + self.shown = self.shown.max(steps); + Ok(()) + } +} + +#[cfg(not(target_arch = "wasm32"))] +impl SeriesRun { + /// Reads the series under `control`: its count is the steps read of those asked, and Cancel + /// stops it before its next batch of reads, with no series. Gives the series as `zarr.series` + /// answers it, and what reading it said, which a read that fails said too. + pub fn run(&self, control: &std::sync::Arc) -> (Result, Vec) { + let mut notes = Vec::new(); + let series = rustgis_analysis::run_under("Time Series at a Point", control, || self.read(&mut notes)); + (series, notes) + } + + /// Reads the series on this thread, looking at the run's Cancel (if any) between batches; + /// what reading it says goes in `notes` as it is found, so an error after it doesn't lose it. + fn read(&self, notes: &mut Vec) -> Result { + let (lon, lat) = (self.lon, self.lat); + let mut src = self.src.clone(); + let check = &mut |_: usize| rustgis_data::progress::check().map_err(|_| "cancelled".to_string()); + check(0)?; let store = if is_web(&src.root) { cached_store(&src.root, &src.sel.variable)? } else { @@ -1536,23 +1640,14 @@ fn series(s: &mut Session, p: &Value) -> CmdResult { let var = src.sel.variable.clone(); let mut keys = zarr::coordinate_keys(&store, &var); keys.extend(zarr::map_coordinate_keys(&store, &var)); - let coords = store_files(&src.root, &keys)?; + let coords = store_files_by(&src.root, &keys, check)?; let (dim, dates) = zarr::time_axis(&store, &var, &coords) .ok_or_else(|| format!("{var} has no time dimension (a coordinate with CF units, such as \"days since 2001-01-01\", or of whole years, such as 2017)"))?; // A period is found along the axis in order, as following the slider takes it. if !dates.iter().all(|d| d.is_finite()) || !dates.windows(2).all(|w| w[0] < w[1]) { return Err(format!("{var}'s time axis isn't in increasing order without repeats, so its series can't be read by date")); } - // The period: the one asked for, else the layer's, else the whole axis. - let date = |k: &str| -> Result, String> { - match p.get(k).filter(|v| !v.is_null()) { - None => Ok(None), - Some(v) => rustgis_data::time::parse_input(v) - .map(Some) - .ok_or_else(|| format!("{k} is a date (2015-06-01, a year, or milliseconds since 1970)")), - } - }; - let (start, end) = (date("start")?, date("end")?); + let (start, end) = (self.start, self.end); let (from, count) = if start.is_some() || end.is_some() { let inside = |d: &f64| start.is_none_or(|a| *d >= a) && end.is_none_or(|z| *d <= z); let first = dates.iter().position(inside).ok_or("no step of the variable falls between start and end")?; @@ -1575,7 +1670,7 @@ fn series(s: &mut Session, p: &Value) -> CmdResult { // A typed system is only a fallback. Invalid text wasn't used on degree axes either. let typed = if given.is_none() { src.crs.as_deref().and_then(|text| Crs::parse(text).ok()) } else { None }; if let Some(note) = zarr::lonlat_assumption(given.as_ref().map(|(c, _)| *c), typed) { - s.warn(format!("{var}: {note}")); + notes.push(format!("{var}: {note}")); } } let crs = if lonlat { @@ -1584,10 +1679,15 @@ fn series(s: &mut Session, p: &Value) -> CmdResult { *c } else { let text = src.crs.as_deref().ok_or("the store doesn't say its coordinate system; give \"crs\" (such as EPSG:3035)")?; - super::parse_crs(s, text)? + // Read as `parse_crs` reads it, its note said with the series'. + let crs = Crs::parse(text)?; + if let Some(note) = raster::wkt::typed_crs_note(text, crs) { + notes.push(format!("crs: {note}")); + } + crs }; if let Some((_, Some(note))) = given { - s.warn(format!("{var}: {note}")); + notes.push(format!("{var}: {note}")); } let m = crs .transformer()? @@ -1597,26 +1697,58 @@ fn series(s: &mut Session, p: &Value) -> CmdResult { let (row, col) = zarr::locate(&store, &var, &coords, x, y).map_err(|e| e.to_string())?.ok_or("the point is outside the variable's grid")?; let plan = zarr::plan_series(&store, &src.sel, &dim, from as u64, count as u64, row, col).map_err(|e| e.to_string())?; - let mut chunks = store_files(&src.root, &plan.keys)?; + // The steps read: the chunks, or each shard's index then its chunk. + rustgis_data::progress::begin(count); + let mut tally = SeriesTally { steps: count, reads: plan.keys.len() + 2 * plan.parts.len(), done: 0, flight: 0, shown: 0 }; + let mut before = |n: usize| tally.before(n); + let mut chunks = store_files_by(&src.root, &plan.keys, &mut before)?; // A sharded variable: the shards' indexes, then the chunk holding the cell in each. if !plan.parts.is_empty() { - chunks.extend(store_parts(&src.root, &plan.parts)?); + chunks.extend(store_parts(&src.root, &plan.parts, &mut before)?); let inner = zarr::series_inner_parts(&store, &plan, &chunks) .map_err(|e| crate::sources::message_with_address(&src.root, &e.to_string()))?; - chunks.extend(store_parts(&src.root, &inner)?); + chunks.extend(store_parts(&src.root, &inner, &mut before)?); } + // The last batch counted as read once it is, before decoding; cancelled during it, nothing + // is decoded, so no part of a series is given. + before(0)?; let values = zarr::assemble_series(&store, &plan, &chunks).map_err(|e| e.to_string())?; let picked = &dates[from..from + count]; let labels: Vec = picked.iter().map(|d| rustgis_data::time::format(*d, "YYYY-MM-DD HH:mm")).collect(); let values: Vec = values.iter().map(|v| if v.is_finite() { serde_json::json!(v) } else { Value::Null }).collect(); let units = store.array(&var).and_then(|a| a.attrs.get("units")).and_then(Value::as_str).map(str::to_string); - Ok(serde_json::json!({ + let series = serde_json::json!({ "variable": var, "units": units, "dimension": dim, "lon": lon, "lat": lat, "row": row, "col": col, "dates": picked, "labels": labels, "values": values, - })) + }); + Ok(series) } } +/// Prepares `zarr.series` with `p` on the session, to read on another thread with +/// [`SeriesRun::run`]: the parameters are checked and the store chosen here, so a wrong request +/// is said at once, before anything is read. +#[cfg(not(target_arch = "wasm32"))] +pub fn series_start(s: &Session, p: &Value) -> Result { + let num = |k: &str| super::f(p, k).ok_or_else(|| format!("{k} is required (degrees)")); + let (lon, lat) = (num("lon")?, num("lat")?); + if !lon.is_finite() || !(-90.0..=90.0).contains(&lat) { + return Err("lon is a longitude and lat a latitude from -90 to 90 (degrees)".into()); + } + let src = series_source(s, p)?; + // The period: the one asked for, else the layer's, else the whole axis. + let date = |k: &str| -> Result, String> { + match p.get(k).filter(|v| !v.is_null()) { + None => Ok(None), + Some(v) => rustgis_data::time::parse_input(v) + .map(Some) + .ok_or_else(|| format!("{k} is a date (2015-06-01, a year, or milliseconds since 1970)")), + } + }; + let (start, end) = (date("start")?, date("end")?); + Ok(SeriesRun { lon, lat, src, start, end }) +} + // --- A summary over time (opengeos/RustGIS#556) ---------------------------------------------------- #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 67e0884f1..e9cccfa26 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -50751,6 +50751,87 @@ fn a_zarr_series_of_a_sharded_variable_reads_only_the_cells_chunks() { assert_eq!(from_dir.unwrap()["values"], json!([204.0, 1204.0])); } +/// What reading a Zarr series says (here, the grid taken as WGS 84) is said though the read then +/// fails (the point outside the grid), by the script's `zarr.series` and by the pane's +/// `SeriesRun::run` alike, as it was before the read moved off the window (#2282). +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn a_zarr_series_says_its_notes_though_the_read_fails() { + let (dir, path) = zarr_store_over_time("notes-on-error"); + let outside = json!({"path": path, "variable": "t", "lon": 100.0, "lat": 80.0}); + let mut s = Session::new(Project::default()); + let mark = s.notice_mark(); + let err = s.run("zarr.series", &outside); + let said: Vec = s.notices_since(mark).iter().map(|n| n.text.clone()).collect(); + assert!(err.as_ref().is_err_and(|e| e.contains("outside")), "{err:?}"); + assert!(said.iter().any(|n| n.contains("WGS 84")), "the grid note is said: {said:?}"); + let (read, notes) = crate::cmd::zarr::series_start(&s, &outside).unwrap().run(&rustgis_analysis::control::Control::stoppable()); + let _ = std::fs::remove_dir_all(dir); + assert!(read.as_ref().is_err_and(|e| e.contains("outside")), "{read:?}"); + assert!(notes.iter().any(|n| n.contains("WGS 84")), "the pane gets the note too: {notes:?}"); +} + +/// A point's series read off the window (opengeos/RustGIS#2282): `SeriesRun::run` gives what +/// `zarr.series` answers a script, counts the steps read, and Cancel stops it between batches of +/// requests. A batch already sent finishes, no batch after it is asked, and a cancelled read +/// gives no series. 130 steps a chunk each: batches of 64, 64 and 2. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn a_zarr_series_read_off_the_window_stops_between_batches() { + use rustgis_analysis::control::Control; + let days: Vec = (0..130).collect(); + let (dir, path) = zarr_store_at("series-cancel", &days); + let store = std::path::Path::new(&path); + let mut files: Vec<(String, Vec)> = [".zmetadata", ".zgroup", "lat/0", "lon/0", "time/0"] + .iter() + .map(|k| (format!("/air.zarr/{k}"), std::fs::read(store.join(k)).unwrap())) + .collect(); + files.extend((0..130).map(|t| (format!("/air.zarr/t/{t}.0.0"), std::fs::read(store.join(format!("t/{t}.0.0"))).unwrap()))); + let _ = std::fs::remove_dir_all(&dir); + // A server of its own each run, so no chunk comes from the download cache. + let serve = || { + let (base, log) = serve_ranged_files(files.clone()); + (json!({"url": format!("{base}/air.zarr"), "variable": "t", "lon": 1.0, "lat": 30.0}), log) + }; + let chunks = |log: &std::sync::Arc>>| { + log.lock().unwrap().iter().filter(|(p, _)| p.starts_with("/air.zarr/t/")).count() + }; + let s = Session::new(Project::default()); + // The same answer as the script's, and every step counted as read. + let (p, _log) = serve(); + let mut script = Session::new(Project::default()); + let answer = script.run("zarr.series", &p).unwrap(); + let (p, _log) = serve(); + let control = Control::stoppable(); + let (series, notes) = crate::cmd::zarr::series_start(&s, &p).unwrap().run(&control); + let series = series.unwrap(); + assert_eq!(series["values"], answer["values"]); + assert_eq!(series["dates"], answer["dates"]); + assert_eq!(series["values"][129], json!(12908.0)); + // What reading it said, for the app to say as the command does: the grid taken as WGS 84. + assert!(notes.iter().any(|n| n.contains("WGS 84")), "{notes:?}"); + assert_eq!(control.progress(), Some((130, 130))); + // Its looks at Cancel, counted on a whole run. + let (p, _log) = serve(); + let counting = Control::cancelled_at_look(u64::MAX); + crate::cmd::zarr::series_start(&s, &p).unwrap().run(&counting).0.unwrap(); + let looks = counting.looked(); + assert!(looks >= 4, "a look before each batch: {looks}"); + let mut seen = Vec::new(); + for n in 1..=looks { + let (p, log) = serve(); + let control = Control::cancelled_at_look(n); + let (got, _) = crate::cmd::zarr::series_start(&s, &p).unwrap().run(&control); + assert!(got.as_ref().is_err_and(|e| e.contains("cancelled")), "look {n}: {got:?}"); + let asked = chunks(&log); + // Whole batches only: the one under way when Cancel came finished, none after it. + assert!(asked % 64 == 0 || asked == 130, "look {n}: {asked} chunks asked"); + assert!(n == looks || asked < 130, "look {n} stopped nothing: {asked} chunks asked"); + seen.push(asked); + } + assert!(seen.contains(&64) && seen.contains(&128), "a cancel after each batch: {seen:?}"); +} + /// A web store whose group places its grid by `spatial:transform` and `proj:code`, as the AlphaEarth /// mosaic (#1720), with axes longer than `MAX_COORDS` and no coordinate arrays: a box is placed by /// arithmetic and reads only the chunks it touches. diff --git a/crates/engine/src/tests/cancellation.rs b/crates/engine/src/tests/cancellation.rs index 6121d3917..c298b372b 100644 --- a/crates/engine/src/tests/cancellation.rs +++ b/crates/engine/src/tests/cancellation.rs @@ -170,6 +170,39 @@ pub(super) fn long_commands(boundaries: bool) { assert!(contents(&shapes.project) == before, "edit.previewShapeConversion changed the project after Cancel"); } exercise(&s, "table.statistics", &json!({"layer": "Cities", "field": "pop_max"}), Kind::Read, &dir.0, boundaries); + // A Zarr point series is read on a thread of its own by the Time Series pane (#2282). It isn't + // one of the queues above (it has no read-ahead), so it is run here: a local store, swept at + // each of its checks in the boundary mode. It changes nothing in the project. + { + // A store per mode: both run in one test process, and each removes its store when done. + let (store, path) = zarr_store_over_time(if boundaries { "cancel-series-boundaries" } else { "cancel-series-first" }); + let p = json!({"path": path, "variable": "t", "lon": 1.0, "lat": 30.0}); + let run = crate::cmd::zarr::series_start(&s, &p).unwrap(); + let before = contents(&s.project); + if boundaries { + let count = Control::cancelled_at_look(u64::MAX); + let expected = run.run(&count).0.unwrap(); + assert!(count.looked() > 0 && count.looked() < 256, "zarr.series: {} checks", count.looked()); + for n in 1..=count.looked() + 1 { + let control = Control::cancelled_at_look(n); + let (result, _) = run.run(&control); + if n <= count.looked() { + assert!(result.as_ref().is_err_and(|e| e.contains("cancelled")), "zarr.series, check {n}: {result:?}"); + // A retry at once gives the whole series. + assert_eq!(run.run(&Control::new()).0.unwrap(), expected, "zarr.series, check {n}: retry"); + } else { + assert_eq!(result.unwrap(), expected, "zarr.series, check {n}: past its last check"); + } + } + } else { + let control = Control::cancelled_once_started(); + let (result, _) = run.run(&control); + assert!(result.as_ref().is_err_and(|e| e.contains("cancelled")), "zarr.series ran on after Cancel: {result:?}"); + assert!(control.looked() > 0, "zarr.series never looks at Cancel"); + } + assert!(contents(&s.project) == before, "zarr.series changed the project"); + let _ = std::fs::remove_dir_all(store); + } // Automatic size bounds scan the numeric field on that queue too. s.run("layer.symbology", &json!({"layer": "Cities", "type": "unique", "field": "name"})).unwrap(); exercise(&s, "layer.sizeBy", &json!({"layer": "Cities", "field": "pop_max"}), Kind::Read, &dir.0, boundaries); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 6d99dfdf6..da7eafeca 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -6471,12 +6471,13 @@ impl RustGisApp { fieldmenu::play(self, &ctx); chrome::shortcuts(self, &ctx); panes::follow_layer_filter(self); - // A Zarr summary, a trend, a workflow, an export or a read running on its thread lands + // A Zarr summary or point series, a trend, a workflow, an export or a read running on its thread lands // whatever is in view, the File menu's backstage too, so a project saved there holds it // (all are looked at: `|`, not `||`), and the frames go on while one runs so it's taken // when it ends. #[cfg(not(target_arch = "wasm32"))] if workspacepanes::land_summary(self) + | workspacepanes::land_series(self) | analysispane::land_trend(self) | analysispane::land_job(self) | exportjob::land(self) diff --git a/crates/ui-egui/src/workspacepanes.rs b/crates/ui-egui/src/workspacepanes.rs index 194e967fd..f5e8e52e5 100644 --- a/crates/ui-egui/src/workspacepanes.rs +++ b/crates/ui-egui/src/workspacepanes.rs @@ -35,6 +35,9 @@ pub struct WorkspaceState { /// The Time Series pane's Zarr layer, and the series read at points (`zarr.series`), newest last. pub series_layer: String, pub series: Vec, + /// A point's series being read on its thread (opengeos/RustGIS#2282). + #[cfg(not(target_arch = "wasm32"))] + pub series_reading: Option, /// The Time Series pane's summary over time (#556): the statistic, its threshold, the period /// and months as typed, and the run under way. pub reduce_stat: String, @@ -491,6 +494,117 @@ pub struct Reducing { pub result: std::sync::Arc, Vec), String>>>>, } +/// A point's series read on a thread of its own (opengeos/RustGIS#2282): the steps read and +/// Cancel, and the series with what reading it said once it ends. +#[cfg(not(target_arch = "wasm32"))] +pub struct SeriesReading { + /// The project it was asked in ([`rustgis_engine::Session::opened`]): a series that ends after + /// another opened is dropped. + pub opened: u64, + pub control: std::sync::Arc, + pub result: std::sync::Arc>>, +} + +/// A series read on its thread, as `zarr.series` answers it, and what reading it said (a read +/// that fails said it too). +#[cfg(not(target_arch = "wasm32"))] +pub type SeriesResult = (Result, Vec); + +/// Starts reading the series at `at` under the Time Series pane's layer on a thread of its own: a +/// wrong request is said at once, and the read lands through [`land_series`]. +#[cfg(not(target_arch = "wasm32"))] +fn start_series(app: &mut RustGisApp, at: rustgis_engine::geom::Coord) { + let p = json!({"layer": app.ws.series_layer, "lon": at.x, "lat": at.y}); + let job = match rustgis_engine::cmd::zarr::series_start(&app.session, &p) { + Ok(job) => job, + Err(e) => { + app.ws.message = Some((e, true)); + return; + } + }; + let control = rustgis_engine::analysis::control::Control::stoppable(); + let r = SeriesReading { opened: app.session.opened, control: control.clone(), result: Default::default() }; + let result = r.result.clone(); + // A panic on the thread (a hostile store) lands as an error, so the read always ends. + let diagnostics = app.session.diagnostics.capture(); + let spawned = std::thread::Builder::new().name("zarr-series".into()).spawn(move || { + let _diagnostics = diagnostics.enter(); + let out = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| job.run(&control))) + .unwrap_or_else(|_| (Err(tl!("The time series failed unexpectedly").to_string()), Vec::new())); + if let Ok(mut g) = result.lock() { + *g = Some(out); + } + }); + match spawned { + Ok(_) => { + app.ws.message = None; + app.ws.series_reading = Some(r); + } + Err(e) => { + app.ws.message = Some((crate::i18n::fmt("The time series couldn't start: {error}", &[("error", &e.to_string())]), true)); + } + } +} + +/// A series that has been read, added to the chart (or its error kept for the pane): called every +/// frame, whatever pane is in view. A cancelled read adds nothing, not even a series it finished +/// as it was cancelled, and says so; one still reading when another project opens is cancelled +/// there and then, so it neither fetches for a project that's gone nor holds the new one's pane. +/// `true` while one is still reading. +#[cfg(not(target_arch = "wasm32"))] +pub fn land_series(app: &mut RustGisApp) -> bool { + let Some(r) = &app.ws.series_reading else { return false }; + if r.opened != app.session.opened { + r.control.cancel(); + app.ws.series_reading = None; + app.ws.message = Some((tl!("The time series was stopped: another project was opened").to_string(), false)); + return false; + } + let Some((res, notes)) = r.result.lock().ok().and_then(|mut g| g.take()) else { return true }; + let cancelled = r.control.is_cancelled(); + app.ws.series_reading = None; + // What reading it said, though it then failed; not for a read that was cancelled. + if !cancelled { + for n in notes { + app.session.warn(n); + } + } + match res { + _ if cancelled => { + app.ws.message = Some((tl!("The time series was cancelled: nothing was added to the chart").to_string(), false)); + } + Ok(series) => { + app.ws.series.push(series); + let excess = app.ws.series.len().saturating_sub(MAX_SERIES); + app.ws.series.drain(..excess); + } + Err(e) => app.ws.message = Some((e, true)), + } + false +} + +/// The series being read, in the chart's place: the steps read of those asked, and Cancel. `true` +/// while one is. +#[cfg(not(target_arch = "wasm32"))] +fn series_progress(app: &mut RustGisApp, ui: &mut Ui) -> bool { + if !land_series(app) { + return false; + } + let Some(r) = &app.ws.series_reading else { return false }; + ui.horizontal(|ui| { + let bar = match r.control.progress() { + Some((done, total)) => egui::ProgressBar::new(done as f32 / total.max(1) as f32) + .text(crate::i18n::fmt("{done} of {total} steps read", &[("done", &done.to_string()), ("total", &total.to_string())])), + None => egui::ProgressBar::new(0.0).text(tl!("Reading the time series…")), + }; + ui.add(bar.desired_width(200.0)); + if ui.button(tl!("Cancel")).clicked() { + r.control.cancel(); + } + }); + true +} + /// The area between a profile's line and its base, as one quad (two triangles) per segment: a /// filled path is drawn as a fan from its first point, which only fills a convex shape, and the /// area under a profile rarely is. @@ -677,6 +791,9 @@ pub fn time_series(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens) { .map(|l| l.name.clone()) .collect(); if layers.is_empty() { + // A read started before its layer was removed can still be cancelled. + #[cfg(not(target_arch = "wasm32"))] + series_progress(app, ui); ui.label(egui::RichText::new(tl!("Add a Zarr variable first (Home › Add › Add Data › Zarr…).")).weak()); return; } @@ -692,9 +809,14 @@ pub fn time_series(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens) { }) .response .on_hover_text(tl!("Layer")); + #[cfg(not(target_arch = "wasm32"))] + let reading = app.ws.series_reading.is_some(); + #[cfg(target_arch = "wasm32")] + let reading = false; ui.horizontal_wrapped(|ui| { let active = app.workflow.pick.as_ref().is_some_and(|p| p.purpose == "ts_point"); - if ui.selectable_label(active, tl!("Pick a Point")).clicked() { + // One series read at a time: the next point is picked once it ends (or is cancelled). + if ui.add_enabled(!reading, egui::Button::selectable(active, tl!("Pick a Point"))).clicked() { app.workflow.pick = if active { None } else { Some(crate::analysispane::Pick { purpose: "ts_point", needed: 1, points: Vec::new() }) }; } @@ -717,15 +839,27 @@ pub fn time_series(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens) { let picked = app.workflow.pick.as_ref().filter(|p| p.purpose == "ts_point").and_then(|p| p.points.first().copied()); if let Some(c) = picked { app.workflow.pick = None; - let layer = app.ws.series_layer.clone(); - if let Some(v) = run(app, "zarr.series", json!({"layer": layer, "lon": c.x, "lat": c.y})) { - app.ws.series.push(v); - let excess = app.ws.series.len().saturating_sub(MAX_SERIES); - app.ws.series.drain(..excess); + // Read on a thread of its own in the desktop app, so a long series doesn't hold the window. + #[cfg(not(target_arch = "wasm32"))] + start_series(app, c); + #[cfg(target_arch = "wasm32")] + { + let layer = app.ws.series_layer.clone(); + if let Some(v) = run(app, "zarr.series", json!({"layer": layer, "lon": c.x, "lat": c.y})) { + app.ws.series.push(v); + let excess = app.ws.series.len().saturating_sub(MAX_SERIES); + app.ws.series.drain(..excess); + } } } + // The series being read, in the chart's place until it lands. + #[cfg(not(target_arch = "wasm32"))] + let reading = series_progress(app, ui); #[cfg(not(target_arch = "wasm32"))] summarise(app, ui); + if reading && app.ws.series.is_empty() { + return; + } if app.ws.series.is_empty() { ui.label(egui::RichText::new(tl!("Pick a point on the map to see the variable's values through time there.")).weak().small()); return; @@ -1135,6 +1269,8 @@ mod tests { app, ); h.run_steps(3); + // Read on its thread (#2282): it lands within a few frames. + wait_for_series(&mut h); assert_eq!(h.state().ws.series_layer, "air"); assert_eq!(h.state().ws.series.len(), 1); assert_eq!(h.state().ws.series[0]["values"], json!([8.0, 108.0, 208.0])); @@ -1172,4 +1308,174 @@ mod tests { assert_eq!(summary.and_then(|r| r.subdataset.clone()), None); let _ = std::fs::remove_dir_all(dir); } + + /// A web store of `t` (three steps on a 3 × 4 grid, step k's values 100·k + i) served from + /// this machine, whose chunk requests wait while the gate is shut: a slow store. Gives its + /// address, the gate and how many chunk requests came. + #[cfg(not(target_arch = "wasm32"))] + fn gated_store() + -> (String, std::sync::Arc<(std::sync::Mutex, std::sync::Condvar)>, std::sync::Arc) { + use std::io::{Read, Write}; + let arr = |shape: &str, dtype: &str, chunks: &str| json!({"shape": serde_json::from_str::(shape).unwrap(), "chunks": serde_json::from_str::(chunks).unwrap(), "dtype": dtype, "compressor": null, "fill_value": null, "filters": null, "order": "C", "zarr_format": 2}); + let meta = json!({"metadata": { + ".zgroup": {"zarr_format": 2}, + "t/.zarray": arr("[3,3,4]", ">(); + let mut files: Vec<(String, Vec)> = vec![ + ("/air.zarr/.zmetadata".into(), meta.to_string().into_bytes()), + ("/air.zarr/.zgroup".into(), br#"{"zarr_format":2}"#.to_vec()), + ("/air.zarr/lat/0".into(), f32s(&[10.0, 20.0, 30.0])), + ("/air.zarr/lon/0".into(), f32s(&[1.0, 2.0, 3.0, 4.0])), + ("/air.zarr/time/0".into(), [0i32, 31, 59].iter().flat_map(|x| x.to_le_bytes()).collect()), + ]; + for k in 0..3 { + let v: Vec = (0..12).map(|i| (k * 100 + i) as f32).collect(); + files.push((format!("/air.zarr/t/{k}.0.0"), f32s(&v))); + } + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let gate = std::sync::Arc::new((std::sync::Mutex::new(true), std::sync::Condvar::new())); + let asked = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let (files, g, n) = (std::sync::Arc::new(files), gate.clone(), asked.clone()); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut s) = stream else { continue }; + let (files, gate, asked) = (files.clone(), g.clone(), n.clone()); + std::thread::spawn(move || { + let (mut head, mut buf) = (Vec::new(), [0u8; 4096]); + while !head.windows(4).any(|w| w == b"\r\n\r\n") && head.len() < 16 << 10 { + match s.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(n) => head.extend_from_slice(&buf[..n]), + } + } + let req = String::from_utf8_lossy(&head).to_string(); + let path = req.split_whitespace().nth(1).unwrap_or("/").to_string(); + if path.strip_prefix("/air.zarr/t/").is_some_and(|k| !k.starts_with('.')) { + asked.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let (open, cv) = &*gate; + let mut o = open.lock().unwrap(); + while !*o { + o = cv.wait(o).unwrap(); + } + } + match files.iter().find(|(p, _)| *p == path) { + Some((_, body)) => { + let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + let _ = s.write_all(body); + } + None => { + let _ = write!(s, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + } + } + }); + } + }); + (base, gate, asked) + } + + /// Runs frames until the pane's series read ends (or a minute passes). + #[cfg(not(target_arch = "wasm32"))] + fn wait_for_series(h: &mut egui_kittest::Harness<'_, RustGisApp>) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); + while h.state().ws.series_reading.is_some() && std::time::Instant::now() < deadline { + h.run_steps(1); + std::thread::yield_now(); + } + // The frame it landed in drew the pane before it: more show the chart and its buttons. + h.run_steps(2); + } + + /// The Time Series pane reads a point's series off the frame (opengeos/RustGIS#2282): while a + /// slow store holds its chunks the frames go on, with the progress and Cancel in the chart's + /// place; Cancel adds nothing and says so; and a series read to its end is what `zarr.series` + /// gives a script. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn the_time_series_pane_reads_off_the_frame_and_cancels() { + use egui_kittest::kittest::Queryable; + let (base, gate, asked) = gated_store(); + let set_gate = |open: bool| { + let (o, cv) = &*gate; + *o.lock().unwrap() = open; + cv.notify_all(); + }; + let mut app = RustGisApp::with_sample(); + // The layer's first step is read with the gate open; the series' other steps wait for it. + app.run("layer.addZarr", json!({"url": format!("{base}/air.zarr"), "variable": "t", "time": true, "name": "air"})).unwrap(); + set_gate(false); + let asked_before = asked.load(std::sync::atomic::Ordering::Relaxed); + let pick = |app: &mut RustGisApp| { + app.workflow.pick = Some(crate::analysispane::Pick { + purpose: "ts_point", + needed: 1, + points: vec![rustgis_engine::geom::Coord::new(1.0, 30.0)], + }); + }; + pick(&mut app); + let mut h = egui_kittest::Harness::builder().allow_missing_glyphs().with_size(egui::vec2(420.0, 600.0)).build_ui_state( + |ui, app: &mut RustGisApp| { + let t = crate::theme::Tokens::get(ui.ctx()); + time_series(app, ui, &t); + }, + app, + ); + // Frames go on while the store holds the series' chunks (asked after the layer's own read). + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); + while asked.load(std::sync::atomic::Ordering::Relaxed) <= asked_before && std::time::Instant::now() < deadline { + h.run_steps(1); + std::thread::yield_now(); + } + assert!(asked.load(std::sync::atomic::Ordering::Relaxed) > asked_before, "the series asked the store for its chunks"); + h.run_steps(5); + assert!(h.state().ws.series_reading.is_some(), "the series is still being read"); + assert!(h.state().ws.series.is_empty()); + h.get_by_label("Cancel").click(); + h.run_steps(1); + set_gate(true); + wait_for_series(&mut h); + assert!(h.state().ws.series_reading.is_none(), "the cancelled read ends"); + assert!(h.state().ws.series.is_empty(), "a cancelled series adds nothing"); + let said = h.state().ws.message.clone(); + assert!(said.as_ref().is_some_and(|(m, err)| m.contains("cancelled") && !err), "{said:?}"); + // Read to its end: the script's answer. + pick(h.state_mut()); + h.run_steps(1); + wait_for_series(&mut h); + assert_eq!(h.state().ws.series.len(), 1); + // The script's answer, from the command as it waits for the read. + let script = h.state_mut().session.run("zarr.series", &json!({"layer": "air", "lon": 1.0, "lat": 30.0})).unwrap(); + assert_eq!(script["values"], json!([8.0, 108.0, 208.0])); + assert_eq!(h.state().ws.series[0]["values"], script["values"]); + assert_eq!(h.state().ws.series[0]["dates"], script["dates"]); + // Another project opened while a read waits on the store: the read is cancelled at once, + // its pane free again, though the store still holds its chunks. + h.state_mut().ws.series.clear(); + set_gate(false); + let asked_before = asked.load(std::sync::atomic::Ordering::Relaxed); + pick(h.state_mut()); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); + while asked.load(std::sync::atomic::Ordering::Relaxed) <= asked_before && std::time::Instant::now() < deadline { + h.run_steps(1); + std::thread::yield_now(); + } + let control = h.state().ws.series_reading.as_ref().map(|r| r.control.clone()).expect("the second read runs"); + h.state_mut().session.opened += 1; + h.run_steps(1); + assert!(control.is_cancelled(), "opening another project cancels the read"); + assert!(h.state().ws.series_reading.is_none(), "the new project's pane isn't held"); + let said = h.state().ws.message.clone(); + assert!(said.as_ref().is_some_and(|(m, err)| m.contains("another project") && !err), "{said:?}"); + set_gate(true); + h.run_steps(2); + assert!(h.state().ws.series.is_empty(), "nothing from the old project lands"); + } }