diff --git a/crates/data/src/raster/zarr.rs b/crates/data/src/raster/zarr.rs index c9e1394e0..28f64c757 100644 --- a/crates/data/src/raster/zarr.rs +++ b/crates/data/src/raster/zarr.rs @@ -2281,7 +2281,10 @@ fn grid_axes(store: &Store, a: &Array, chunks: &HashMap>) -> Res let (Some((y0, dy)), Some((x0, dx))) = (axis(&ys), axis(&xs)) else { return Err(bad(format!("{}: its coordinates aren't evenly spaced", a.path))); }; - Ok(((y0, dy, ys.len() as u64), (x0, dx, xs.len() as u64), geographic(xc))) + // Longitude by the coordinate's name or units, or by the geographic system the store gives + // (`assemble` wraps such a grid by it too, whatever its axes are called). + let geo = geographic(xc) || crs_of(store, a).is_some_and(|(c, _)| c.is_geographic()); + Ok(((y0, dy, ys.len() as u64), (x0, dx, xs.len() as u64), geo)) } /// The chunk keys of `variable`'s map coordinates, for [`window_of`]: each is read whole (its @@ -3440,6 +3443,43 @@ mod tests { } } + /// A grid whose axes are named `x`/`y` but whose store gives a geographic system is wrapped by + /// `assemble` (0–360 read as -180–180), so a box or a point is placed with the same ±360° turns; + /// a projected one isn't turned. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn a_geographic_system_turns_boxes_and_points_on_named_x_y_axes() { + let f64s = |v: Vec| v.iter().flat_map(|x| x.to_le_bytes()).collect::>(); + for (code, geo) in [(4326, true), (3857, false)] { + let mut files = window_store(false); + let mut root: Value = serde_json::from_slice(&files[0].1).unwrap(); + root["consolidated_metadata"]["metadata"]["v"]["attributes"] = serde_json::json!({"epsg_code": code}); + files[0].1 = root.to_string().into_bytes(); + // Columns centred on 185°…295°: east of 180, as a 0–360 grid stores them. + files[2].1 = zstd::encode_all(&f64s((0..12).map(|i| 185.0 + 10.0 * i as f64).collect())[..], 3).unwrap(); + let store = parse_store(&files).unwrap(); + let coords: HashMap> = + files.iter().filter(|(k, _)| window_keys(&store, "v").unwrap().contains(k)).cloned().collect(); + let found = locate(&store, "v", &coords, -165.0, 50.0).unwrap(); + let placed = window_of(&store, "v", &coords, [-170.0, 30.0, -150.0, 60.0]); + if !geo { + assert_eq!(found, None, "EPSG:{code}"); + assert!(placed.is_err(), "EPSG:{code}"); + continue; + } + assert_eq!(found, Some((3, 1)), "EPSG:{code}"); + let window = Window { x0: 1, y0: 2, x1: 3, y1: 5 }; + assert_eq!(placed.unwrap(), (window, false, false), "EPSG:{code}"); + // The same box written 0–360 is the same window. + assert_eq!(window_of(&store, "v", &coords, [190.0, 30.0, 210.0, 60.0]).unwrap().0, window); + let selection = Selection { variable: "v".into(), index: vec![("time".into(), 1)], window: Some(window) }; + let plan = plan(&store, &selection).unwrap(); + let raster = assemble(&store, &plan, &read_plan(&files, &plan), "v", None).unwrap(); + assert_eq!(raster.crs, Crs::LonLat); + assert!((raster.transform.origin_x - -170.0).abs() < 1e-9, "{}", raster.transform.origin_x); + } + } + /// A store with longitude/latitude axes is in the geographic system it gives (ED50 with its /// TOWGS84 shift, a datum with no shift, another body's), whether as WKT, a PROJ definition or an /// EPSG code, not in WGS 84 (#2008). What reading it said is kept once: another body by its body, diff --git a/crates/engine/src/cmd/zarr.rs b/crates/engine/src/cmd/zarr.rs index c21cbec34..23dd7650a 100644 --- a/crates/engine/src/cmd/zarr.rs +++ b/crates/engine/src/cmd/zarr.rs @@ -28,7 +28,7 @@ pub fn commands() -> Vec { r#"{"layer"? (a Zarr layer: its store, variable and steps; default the active layer) | "path": folder | "url": "https://…", "variable"?, "index"?, "stat": "mean"|"min"|"max"|"sum"|"std" (the population standard deviation, n)|"count"|"countAbove", "above"?: number (countAbove's threshold), "start"?, "end"?: dates (default: the layer's period, else the whole axis), "months"?: [1–12] (a season across years: [12, 1, 2]), "minValid"?: 0–1 (a cell with a smaller share of values is left empty; default 0), "name"?: text, "crs"?} → a raster layer of the statistic (desktop)"#, ), C::new("layer.addZarr", "Add Zarr Variable", "Home › Add › Add Data", add).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Network).with(crate::policy::Effect::Files)).params( - r#"{"path": folder | "url": "https://…", "variable"?: "its path in the store (default: the first)", "index"?: {"time": 3, "level": 0} (other dimensions' steps, default 0), "cells"?: [first column, first row, end column, end row] (a window of the map's cells as stored, the ends not included, at least 2 × 2: only the chunks it touches are read, and in shards only their chunks it needs, by range) | "bbox"?: [west, south, east, north] (a window by the grid's coordinates: degrees for a longitude/latitude grid; its coordinates are read to place it), "crs"?: "EPSG:… for a store that doesn't say", "name"?: text, "time"?: bool (follow the time slider: the layer shows the step the slider is at, and turns the slider on over the variable's dates when it is off; desktop), "start"?, "end"?: dates (with time: the period to take, for an axis longer than the slider's 20,000 steps), "ran"?: (internal: a slice the app has read), "summary"?: bool (internal: with ran, a summary over time rather than a slice)}"#, + r#"{"path": folder | "url": "https://…", "variable"?: "its path in the store (default: the first)", "index"?: {"time": 3, "level": 0} (other dimensions' steps, default 0), "cells"?: [first column, first row, end column, end row] (a window of the map's cells as stored, the ends not included, at least 2 × 2: only the chunks it touches are read, and in shards only their chunks it needs, by range) | "bbox"?: [west, south, east, north] (a window by the grid's coordinates: degrees for a longitude/latitude grid; its coordinates are read to place it), "crs"?: "EPSG:… for a store that doesn't say", "name"?: text, "time"?: bool (follow the time slider: the layer shows the step the slider is at, and turns the slider on over the variable's dates when it is off; with cells or bbox, each step reads only that window; desktop), "start"?, "end"?: dates (with time: the period to take, for an axis longer than the slider's 20,000 steps), "ran"?: (internal: a slice the app has read), "summary"?: bool (internal: with ran, a summary over time rather than a slice)}"#, ), ] } @@ -463,10 +463,7 @@ fn place_window(root: &str, store: &zarr::Store, sel: &mut zarr::Selection, aske pub(crate) fn add(s: &mut Session, p: &Value) -> CmdResult { let asked = asked_window(p)?; if b(p, "time") == Some(true) { - if asked.is_some() { - return Err("a window (cells or bbox) isn't read with time yet: leave time out to read it".into()); - } - return add_following(s, p); + return add_following(s, p, asked); } let src = source(p)?; #[cfg_attr(target_arch = "wasm32", expect(unused_mut, reason = "windowed reads are desktop only"))] @@ -616,7 +613,13 @@ fn cached_store(url: &str, variable: &str) -> Result Ok(st) } -/// The key a slice is kept by: its store, selection and coordinate system. +/// The window of cells a following layer's steps read. +#[cfg(not(target_arch = "wasm32"))] +fn zt_window(zt: &ZarrTime) -> Option { + zt.window.map(|[x0, y0, x1, y1]| zarr::Window { x0, y0, x1, y1 }) +} + +/// The key a slice is kept by: its store, selection (with its window) and coordinate system. #[cfg(not(target_arch = "wasm32"))] pub(crate) fn slice_key(zt: &ZarrTime, i: usize) -> String { // A folder store rewritten on disk (a model run adding steps) is read again: its metadata's @@ -803,7 +806,8 @@ pub(crate) fn read_slice(zt: &ZarrTime, i: usize) -> Result 0 { return Err(crate::sources::message_with_address( @@ -1239,10 +1243,10 @@ fn follow(s: &mut Session, wait: bool) { /// `layer.addZarr` with `"time": true`: the variable's time axis read from its coordinate, a layer /// that shows the step the slider is at, and the slider turned on over the axis when it is off. /// The stretch is fixed at the first slice's, so a colour means one value at every step. -fn add_following(s: &mut Session, p: &Value) -> CmdResult { +fn add_following(s: &mut Session, p: &Value, asked: Option) -> CmdResult { #[cfg(target_arch = "wasm32")] { - let _ = (s, p); + let _ = (s, p, asked); Err("in the web app, a Zarr variable can't follow the time slider yet".into()) } #[cfg(not(target_arch = "wasm32"))] @@ -1286,6 +1290,8 @@ fn add_following(s: &mut Session, p: &Value) -> CmdResult { } }; pick_variable(&store, &mut sel)?; + // A window: every step reads only its cells (and in shards only their chunks it needs). + let window_notes = place_window(&root, &store, &mut sel, asked)?; let chunks = if chunks.is_empty() && is_web(&root) { fetch(&root, &zarr::coordinate_keys(&store, &sel.variable))?.0 } else { chunks }; let (dim, dates) = zarr::time_axis(&store, &sel.variable, &chunks).ok_or_else(|| { @@ -1294,6 +1300,12 @@ fn add_following(s: &mut Session, p: &Value) -> CmdResult { sel.variable ) })?; + // A window is checked against the grid as a still window is (within it, 2 × 2 cells at + // least), before the slider is touched: the same for every step, so step 0 stands for all. + if sel.window.is_some() { + let first = zarr::Selection { index: sel.index.iter().filter(|(d, _)| *d != dim).cloned().collect(), ..sel.clone() }; + zarr::plan(&store, &first).map_err(|e| crate::sources::message_with_address(&root, &e.to_string()))?; + } // Each date is a step: an axis out of order or with a date twice (stores joined end to end) // can't be followed step by step. if !dates.iter().all(|d| d.is_finite()) || !dates.windows(2).all(|w| w[0] < w[1]) { @@ -1339,6 +1351,7 @@ fn add_following(s: &mut Session, p: &Value) -> CmdResult { offset: first as u64, dates: picked, crs, + window: sel.window.map(|w| [w.x0, w.y0, w.x1, w.y1]), shown: None, read_generation: rustgis_map::raster::next_read_generation(), }; @@ -1390,9 +1403,12 @@ fn add_following(s: &mut Session, p: &Value) -> CmdResult { }); layer.time_zarr = Some(zt); let mut out = super::raster::add_raster(s, &name, layer)?; - if let Some(w) = part { + let said: Vec = window_notes.into_iter().chain(part).collect(); + for w in &said { s.warn(w.clone()); - out["warning"] = serde_json::json!(w); + } + if !said.is_empty() { + out["warning"] = serde_json::json!(said.join("; ")); } Ok(out) } @@ -1423,7 +1439,7 @@ fn series_source(s: &Session, p: &Value) -> Result { if let Some(zt) = &rl.time_zarr { return Ok(SeriesSource { root: zt.store.clone(), - sel: zarr::Selection { variable: zt.variable.clone(), index: zt.index.clone(), window: None }, + sel: zarr::Selection { variable: zt.variable.clone(), index: zt.index.clone(), window: zt_window(zt) }, period: Some((zt.offset, zt.dates.len())), crs: zt.crs.clone(), read_generation: zt.read_generation, @@ -1954,6 +1970,7 @@ pub fn zarr_cube(s: &Session, p: &Value) -> Result { offset: 0, dates, crs: src.crs.clone(), + window: None, shown: None, read_generation: src.read_generation, }; diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index c79c6fb87..c04bd0f73 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -87,6 +87,7 @@ pub const CAPABILITIES: &[&str] = &[ "layout-element-ink", "field-generation", "crs-wkt", + "zarr-window", ]; /// Independent marker sizes alongside classified point colours (`Layer::size_by`). pub(crate) const SIZE_BY_CAPABILITY: &str = "classified-point-size"; diff --git a/crates/engine/src/project_capabilities.rs b/crates/engine/src/project_capabilities.rs index a7f6f4434..ee32278e4 100644 --- a/crates/engine/src/project_capabilities.rs +++ b/crates/engine/src/project_capabilities.rs @@ -226,13 +226,20 @@ fn relief_light(layer: &Layer) -> bool { ) } -/// The renderer capabilities a layer needs, in [`map_capabilities`]'s order. +/// Whether a layer is a window of a Zarr variable following the time slider (#1720): an older +/// reader would drop the window and read the whole map at each step. +fn zarr_window(layer: &Layer) -> bool { + layer.raster.as_ref().and_then(|r| r.time_zarr.as_ref()).is_some_and(|z| z.window.is_some()) +} + +/// The capabilities a layer needs, in [`map_capabilities`]'s order. fn layer_capabilities(layer: &Layer) -> Vec<&'static str> { [ (relief_effects(layer), "shaded-relief-effects"), (relief_light(layer), "shaded-relief-light"), (layer.size_by.is_some(), super::SIZE_BY_CAPABILITY), (layer.has_picture_markers(), "picture-markers"), + (zarr_window(layer), "zarr-window"), ] .into_iter() .filter_map(|(used, name)| used.then_some(name)) @@ -240,8 +247,8 @@ fn layer_capabilities(layer: &Layer) -> Vec<&'static str> { } fn map_capabilities<'a>(maps: impl IntoIterator) -> Vec<&'static str> { - let (mut block, mut water, mut shore, mut relief, mut light, mut legends, mut sized, mut pictures) = - (false, false, false, false, false, false, false, false); + let (mut block, mut water, mut shore, mut relief, mut light, mut legends, mut sized, mut pictures, mut zarr) = + (false, false, false, false, false, false, false, false, false); for map in maps { if let Some(t) = &map.view.terrain { block |= t.block.is_some(); @@ -253,6 +260,7 @@ fn map_capabilities<'a>(maps: impl IntoIterator) -> Vec<&'static light |= map.layers.iter().any(relief_light); sized |= map.layers.iter().any(|layer| layer.size_by.is_some()); pictures |= map.layers.iter().any(Layer::has_picture_markers); + zarr |= map.layers.iter().any(zarr_window); legends |= map.tiles.iter().any(|tile| { tile.legends.iter().any(|legend| rustgis_data::login_address::inner(&legend.url).len() > rustgis_map::tiles::MAX_LEGEND_URL) }); @@ -266,6 +274,7 @@ fn map_capabilities<'a>(maps: impl IntoIterator) -> Vec<&'static (legends, "normalized-legend-urls"), (sized, super::SIZE_BY_CAPABILITY), (pictures, "picture-markers"), + (zarr, "zarr-window"), ] .into_iter() .filter_map(|(used, name)| used.then_some(name)) @@ -711,6 +720,30 @@ mod tests { assert!(Project::from_json(&unknown.to_string()).unwrap_err().contains("future-terrain")); } + /// A Zarr window following the time slider (#1720) is required by name in a project, a map + /// file and a layer file: an older reader would drop it and read the whole map at each step. + #[test] + fn a_zarr_window_is_required_where_it_is_saved() { + let mut p = Project::from_json(include_str!("../../../schema/fixtures/zarr_window.rgis")).unwrap(); + let layer = p.map().layers[0].clone(); + assert_eq!( + layer.raster.as_ref().and_then(|r| r.time_zarr.as_ref()).and_then(|z| z.window), + Some([1526419, 1162633, 1526643, 1162857]) + ); + let saved = serde_json::to_value(&p).unwrap(); + assert_eq!(saved["requires"], json!(["zarr-window"])); + assert_eq!(json_bytes(&map_file_bytes(p.map()).unwrap())["requires"], saved["requires"]); + assert_eq!(json_bytes(&layer_file_bytes(&layer).unwrap())["requires"], saved["requires"]); + // The whole variable following the slider needs nothing new. + if let Some(z) = p.maps[0].layers[0].raster.as_mut().and_then(|r| r.time_zarr.as_mut()) { + z.window = None; + } + assert!(serde_json::to_value(&p).unwrap().get("requires").is_none()); + let mut unknown = saved; + unknown["requires"] = json!(["zarr-window", "future-zarr"]); + assert!(Project::from_json(&unknown.to_string()).unwrap_err().contains("future-zarr")); + } + /// Nondefault relief effects survive project, map and layer files; ordinary relief needs none. #[test] fn relief_headers_cover_each_effect_and_clean_controls() { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index a4484153b..a7eac0cf1 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -51017,10 +51017,9 @@ fn a_zarr_window_reads_only_its_chunks_by_range() { "{:?}", s.notices_since(mark) ); - // Refused: cells and bbox together, a window with time, cells that aren't whole, a window outside. + // Refused: cells and bbox together, cells that aren't whole, a window outside. for bad in [ json!({"url": url, "variable": "v", "cells": [2, 1, 5, 4], "bbox": [2, 4, 4, 6]}), - json!({"url": url, "variable": "v", "cells": [2, 1, 5, 4], "time": true}), json!({"url": url, "variable": "v", "cells": [2, 1, 5.5, 4]}), json!({"url": url, "variable": "v", "cells": [2, 1, 50, 4]}), json!({"url": url, "variable": "v", "bbox": [100, 4, 110, 6]}), @@ -51090,6 +51089,113 @@ fn a_zarr_series_of_a_sharded_variable_reads_only_the_cells_chunks() { assert_eq!(from_dir.unwrap()["values"], json!([204.0, 1204.0])); } +/// A window of a sharded Zarr variable follows the time slider (#1720: `aef-mosaic` is too large +/// to read whole): each step reads its shard's index and then only the window's chunk, by range; +/// the project keeps the window with the store, and a folder gives the same cells. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn a_zarr_window_follows_the_time_slider() { + let _cache = zarr_cache_lock(); + // `v` (time 2, lat 4, lon 6; int16, `t·1000 + row·100 + col`) in shards of 1 × 4 × 6 holding + // chunks of 1 × 2 × 3, the index at the end; latitudes 3.5 down by 1, longitudes 0.5 up by 1. + let coord = |n: u64, attrs: Value| { + json!({"zarr_format": 3, "node_type": "array", "shape": [n], "data_type": "float64", "attributes": attrs, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [n]}}, "codecs": [{"name": "bytes"}], "fill_value": 0}) + }; + let v = json!({"zarr_format": 3, "node_type": "array", "shape": [2, 4, 6], "data_type": "int16", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [1, 4, 6]}}, + "codecs": [{"name": "sharding_indexed", "configuration": {"chunk_shape": [1, 2, 3], "codecs": [{"name": "bytes"}], + "index_codecs": [{"name": "bytes"}, {"name": "crc32c"}], "index_location": "end"}}], + "fill_value": -1, "dimension_names": ["time", "lat", "lon"]}); + let root = json!({"zarr_format": 3, "node_type": "group", "consolidated_metadata": {"kind": "inline", "metadata": { + "v": v, "lat": coord(4, json!({"units": "degrees_north"})), "lon": coord(6, json!({"units": "degrees_east"})), + "time": coord(2, json!({"units": "days since 2000-01-01"}))}}}); + let f64s = |v: Vec| v.iter().flat_map(|x| x.to_le_bytes()).collect::>(); + let mut files = vec![ + ("zarr.json".to_string(), root.to_string().into_bytes()), + ("lat/c/0".to_string(), f64s((0..4).map(|i| 3.5 - i as f64).collect())), + ("lon/c/0".to_string(), f64s((0..6).map(|i| 0.5 + i as f64).collect())), + ("time/c/0".to_string(), f64s(vec![0.0, 366.0])), + ]; + for t in 0..2u64 { + let (mut body, mut index) = (Vec::new(), Vec::new()); + for (iy, ix) in [(0u64, 0u64), (0, 1), (1, 0), (1, 1)] { + index.extend([body.len() as u64, 12]); + for (r, c) in (0..2).flat_map(|r| (0..3).map(move |c| (r, c))) { + body.extend(((t * 1000 + (iy * 2 + r) * 100 + ix * 3 + c) as i16).to_le_bytes()); + } + } + body.extend(index.iter().flat_map(|v| v.to_le_bytes())); + body.extend([0u8; 4]); + files.push((format!("v/c/{t}/0/0"), body)); + } + let dir = std::env::temp_dir().join(format!("rustgis-zarr-window-time-{}", std::process::id())); + for (k, b) in &files { + let path = dir.join("s.zarr").join(k); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, b).unwrap(); + } + let (base, log) = serve_ranged_files(files.iter().map(|(k, b)| (format!("/s.zarr/{k}"), b.clone())).collect()); + let url = format!("{base}/s.zarr"); + let mut s = Session::new(Project::default()); + // 3°E–6°E, 0°–2°N: columns 3–6 and rows 2–4, the shard's last chunk (bytes 36–47). + let id = s.run("layer.addZarr", &json!({"url": url, "variable": "v", "time": true, "bbox": [3.0, 0.0, 6.0, 2.0]})).unwrap()["layer"] + .as_u64() + .unwrap() as u32; + let cells = |s: &Session| { + let d = s.map().layer(id).and_then(|l| l.raster.as_ref()).and_then(|r| r.data.clone()).unwrap(); + (d.width, d.height, d.bands[0].data.clone()) + }; + let step = |t: f32| (2..4).flat_map(|r| (3..6).map(move |c| t * 1000.0 + (r * 100 + c) as f32)).collect::>(); + assert_eq!(cells(&s), (3, 2, step(0.0))); + s.run("time.goTo", &json!({"step": 1})).unwrap(); + assert_eq!(cells(&s), (3, 2, step(1.0))); + let zt = s.map().layer(id).unwrap().raster.as_ref().unwrap().time_zarr.clone().unwrap(); + assert_eq!(zt.window, Some([3, 2, 6, 4])); + // Each step: its shard's index, then the window's chunk; never a shard whole. + let asked: Vec<(String, String)> = log.lock().unwrap().clone(); + for t in 0..2 { + let mut ranges: Vec<&str> = asked.iter().filter(|(p, _)| *p == format!("/s.zarr/v/c/{t}/0/0")).map(|(_, r)| r.as_str()).collect(); + ranges.sort(); + ranges.dedup(); + assert_eq!(ranges, ["-68", "36-47"], "{asked:?}"); + } + // Saved and opened again: the window is kept with the store, required by name, and read + // again at the step. + let text = serde_json::to_string(&s.project).unwrap(); + assert!(text.contains("\"window\":[3,2,6,4]") && !text.contains("\"embedded\""), "{text}"); + assert_eq!(serde_json::from_str::(&text).unwrap()["requires"], json!(["zarr-window"])); + let mut t = Session::new(Project::default()); + t.run("project.openText", &json!({"text": text})).unwrap(); + let again = t.map().layers.iter().find_map(|l| l.raster.as_ref()?.data.clone()).unwrap(); + assert_eq!((again.width, again.height, again.bands[0].data.clone()), (3, 2, step(1.0))); + // A window edited by hand to one cell wide isn't widened to the whole map: the step is + // refused, and said. + let mut u = Session::new(Project::default()); + let mark = u.notice_mark(); + let narrow = text.replace("\"window\":[3,2,6,4]", "\"window\":[3,2,4,4]"); + let opened = u.run("project.openText", &json!({"text": narrow})); + let data = u.map().layers.iter().find_map(|l| l.raster.as_ref()?.data.clone()); + assert!(data.is_none_or(|d| (d.width, d.height) != (6, 4)), "the whole map was read"); + let said: Vec = u.notices_since(mark).iter().map(|n| n.text.clone()).collect(); + assert!(opened.is_err() || said.iter().any(|n| n.contains("2 × 2")), "{opened:?} {said:?}"); + // A folder: the same cells, by cells. + let mut f = Session::new(Project::default()); + let r = f.run("layer.addZarr", &json!({"path": dir.join("s.zarr"), "variable": "v", "time": true, "cells": [3, 2, 6, 4]})); + let got = r.as_ref().ok().and_then(|_| f.map().layers.iter().find_map(|l| l.raster.as_ref()?.data.clone())); + let _ = std::fs::remove_dir_all(&dir); + let got = got.unwrap_or_else(|| panic!("{r:?}")); + assert_eq!((got.width, got.height, got.bands[0].data.clone()), (3, 2, step(0.0))); + // A window outside the grid, or under 2 × 2 cells, is refused before the slider is touched, + // as a still window is, not read as the whole map. + let mut g = Session::new(Project::default()); + for cells in [json!([3, 2, 60, 4]), json!([3, 2, 4, 4]), json!([3, 3, 6, 4]), json!([100, 0, 101, 2])] { + let e = g.run("layer.addZarr", &json!({"url": url, "variable": "v", "time": true, "cells": cells})).unwrap_err(); + assert!(e.contains("2 × 2"), "{cells}: {e}"); + assert!(g.map().time.is_none() && g.map().layers.is_empty(), "{cells}"); + } +} + /// What reading a Zarr series says (here, the grid taken as WGS 84) is said though the read then /// fails (the point outside the grid), by the script's `zarr.series` and by the pane's /// `SeriesRun::run` alike, as it was before the read moved off the window (#2282). diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index 205409ebd..726cfc66d 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -1144,6 +1144,11 @@ pub struct ZarrTime { /// The coordinate system given for a store that doesn't say. #[serde(skip_serializing_if = "Option::is_none")] pub crs: Option, + /// The window of the map's cells each step reads, `[x0, y0, x1, y1]` as `layer.addZarr`'s + /// `cells` (the ends not included); `None` for the whole map. A project with one requires + /// `zarr-window`: an older reader would read the whole map at each step. + #[serde(skip_serializing_if = "Option::is_none")] + pub window: Option<[u64; 4]>, /// The step whose slice the layer holds (an index into `dates`); `None` when none is. #[serde(skip)] pub shown: Option, @@ -1168,6 +1173,7 @@ impl Default for ZarrTime { offset: 0, dates: Vec::new(), crs: None, + window: None, shown: None, read_generation: next_read_generation(), } @@ -1228,10 +1234,12 @@ impl ZarrTime { } } - /// The selection of step `i` as `zarr::Selection::parse` reads it: `t2m[time=12,level=3]`. + /// The selection of step `i` as `zarr::Selection::parse` reads it: `t2m[time=12,level=3]`, + /// with `cells=x0:y0:x1:y1` after them for a window. pub fn selection(&self, i: usize) -> String { let mut parts = vec![format!("{}={}", self.dim, self.offset.saturating_add(i as u64))]; parts.extend(self.index.iter().filter(|(d, _)| *d != self.dim).map(|(d, v)| format!("{d}={v}"))); + parts.extend(self.window.map(|[x0, y0, x1, y1]| format!("cells={x0}:{y0}:{x1}:{y1}"))); format!("{}[{}]", self.variable, parts.join(",")) } @@ -1245,6 +1253,8 @@ impl ZarrTime { self.dates.clear(); } self.index.truncate(32); + // A window is kept as it is: one a read can't take (a project edited by hand) is refused + // at each step, never widened to the whole map. for t in [&mut self.store, &mut self.variable, &mut self.dim] { *t = t.chars().take(4096).collect(); } @@ -2359,8 +2369,18 @@ impl RasterLayer { if let Some(p) = path { out.push((crate::LayerRole::Raster, p.as_str(), true, embedded.is_none() && crate::paths::is_local(p))); } - if let Some(ZarrTime { store, variable: _, index: _, dim: _, offset: _, dates: _, crs: _, shown: _, read_generation: _ }) = - time_zarr + if let Some(ZarrTime { + store, + variable: _, + index: _, + dim: _, + offset: _, + dates: _, + crs: _, + window: _, + shown: _, + read_generation: _, + }) = time_zarr { out.push((crate::LayerRole::ZarrStore, store.as_str(), true, crate::paths::is_local(store))); } @@ -2673,6 +2693,7 @@ mod tests { offset: 3, dates, crs: None, + window: None, shown: Some(1), read_generation: 0, }; @@ -2686,6 +2707,27 @@ mod tests { } } + /// A following layer's window is part of each step's selection, as `zarr::Selection::parse` + /// reads it; repair keeps one a read refuses (a project edited by hand), never dropping it to + /// the whole map. + #[test] + fn a_zarr_window_is_in_each_steps_selection() { + let mut z = ZarrTime { + variable: "t".into(), + index: vec![("level".into(), 2)], + dim: "time".into(), + offset: 3, + dates: vec![1.0, 2.0], + window: Some([4, 1, 9, 3]), + ..Default::default() + }; + z.repair(); + assert_eq!(z.selection(1), "t[time=4,level=2,cells=4:1:9:3]"); + z.window = Some([4, 1, 5, 3]); + z.repair(); + assert_eq!((z.window, z.selection(0)), (Some([4, 1, 5, 3]), "t[time=3,level=2,cells=4:1:5:3]".to_string())); + } + fn dem() -> Raster { let gt = GeoTransform { origin_x: 0.0, origin_y: 4.0, pixel_width: 1.0, pixel_height: -1.0 }; let data: Vec = (0..16).map(|i| (i % 4 + i / 4) as f32).collect(); diff --git a/docs/maintaining.md b/docs/maintaining.md index caf70fd66..6e38bf7c8 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -129,6 +129,11 @@ refuses them instead of opening the layout and saving it again, or running the c without the background. Layout files (`.rgislayout`) carry no format header yet and are not covered (#2277). +Projects, map files and layer files with a window of a Zarr variable that follows the time slider +(`layer.addZarr` with `"time": true` and `"cells"` or `"bbox"`) require `zarr-window`, so an older +reader refuses them instead of reading the whole map at each step. A saved `layer.addZarr` command +with both needs nothing: an older RustGIS refuses it when it runs. + Projects with an ink on a legend, colorbar, table or chart (`color` on those elements) require `layout-element-ink`, so an older reader refuses them instead of drawing those elements in dark grey and saving them again without it. Saved `layout.addElement` and `layout.update` commands diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 6eaa4dd6b..7f13bb35e 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -1637,9 +1637,12 @@ coordinates, read whole, so it takes grids of up to 4,000,000 cells a side; a lo size. A box reaching past the grid is read within it, and says so; one over a longitude grid's edge (0°/360°, or ±180°) is refused: give a box on each side (one round the whole globe is the whole grid); `raster.zarrInfo` with -`"bbox"` gives the cells a box covers (`window.cells`). A window is at least 2 × 2 cells (a box within one cell is widened to two, and says so), and -doesn't follow the time slider yet. A window is kept in the project as a GeoTIFF, as a web -store's slice is, also from a folder. +`"bbox"` gives the cells a box covers (`window.cells`). A window is at least 2 × 2 cells (a box within one cell is widened to two, and says so). A window +is kept in the project as a GeoTIFF, as a web store's slice is, also from a folder; with +`"time": true` it follows the time slider instead, each step reading only the window, and the +project keeps the window's cells with the store. A grid whose axes aren't named or measured as +longitude, but whose store gives a geographic system, takes a box or a point written either way +round the globe too, as it is drawn. **Follow the time slider** (desktop; `layer.addZarr` with `"time": true`) adds the variable as one layer whose step is the time slider's. The slider turns on over the variable's dates when it is diff --git a/schema/fixtures/zarr_window.rgis b/schema/fixtures/zarr_window.rgis new file mode 100644 index 000000000..fc00f4ded --- /dev/null +++ b/schema/fixtures/zarr_window.rgis @@ -0,0 +1 @@ +{"format":2,"requires":["zarr-window"],"name":"Zarr window","maps":[{"name":"Map","layers":[{"id":1,"name":"embeddings[band=0]","visible":true,"dataset":{"name":"embeddings[band=0]","geometry_type":"None","fields":[],"features":[],"source":null},"renderer":{"Single":{"fill":[224,123,57,170],"stroke":[134,74,34,255],"stroke_width":0.7,"size":0.0,"marker":"Circle"}},"labels":{"enabled":false,"field":"","size":10.0,"color":[43,43,43,255],"halo":true,"bold":false,"min_scale":null,"max_scale":null,"closest":null,"farthest":null,"placement":"Auto"},"transparency":0.0,"min_scale":null,"max_scale":null,"definition_query":null,"selection":[],"expanded":true,"selectable":false,"graphics":false,"blend":"Normal","mask":null,"display_filters":[],"aggregation":null,"extrusion":null,"priority":0,"raster":{"path":null,"file_name":null,"time_zarr":{"store":"https://example.invalid/aef-mosaic","variable":"embeddings","index":[["band",0]],"dim":"time","dates":[1483228800000.0,1514764800000.0,1546300800000.0,1577836800000.0,1609459200000.0,1640995200000.0,1672531200000.0,1704067200000.0,1735689600000.0],"window":[1526419,1162633,1526643,1162857]},"style":{"renderer":{"Stretch":{"band":0,"ramp":{"Matplotlib":"gray"},"invert":false}},"stretch":{"Custom":{"min":-30.88235294117647,"max":47.16470588235293}},"dra":false,"resampling":"Nearest","brightness":0.0,"contrast":0.0,"gamma":1.0,"rotation":0.0}},"geometry_rev":0}],"basemap":"Paper","camera":{"center":{"x":-4772268.170300318,"y":-2363876.6629938707},"res":3.7264918915312997,"bearing":0.0,"pitch":0.0},"bookmarks":[],"graticule":true,"projection":"WebMercator","popup_mode":"Topmost","label_display":{"paused":false,"locked":false,"show_unplaced":false},"editing":{"relationships":[],"snapping":{"enabled":false,"tolerance_px":10.0,"vertex":true,"edge":true,"end":true},"topology":[],"settings":{"enabled":true,"show_vertices":true,"elevation":"None","ground_to_grid":null}},"swipe":null,"view":{"effects":{"stars":true,"sun_time":null,"weather":"None","vignette":false,"edges":false,"tilt_shift":false,"grain":false,"brightness":0.0,"contrast":0.0,"saturation":0.0,"sepia":0.0},"wireframe":false,"clip_distance":null,"terrain":null},"animation":[],"time":{"start":1483228800000.0,"end":1735689600000.0,"current":1704067200000.0,"unit":"year","interval":1,"dates":[1483228800000.0,1514764800000.0,1546300800000.0,1577836800000.0,1609459200000.0,1640995200000.0,1672531200000.0,1704067200000.0,1735689600000.0],"before":0,"after":1,"cumulative":false,"speed_ms":800,"loop":true},"next_layer_id":2}],"active_map":0,"tables":[],"next_table_id":1,"gp_history":[],"layouts":[],"folders":[],"styles":[],"ramps":[],"tasks":[],"gp_env":{"extent":null,"cell_size":null},"models":[],"workspaces":[],"link_views":false,"jobs":[],"offline_areas":[],"reports":[],"presentations":[],"notebooks":[],"dl_models":[],"shared":[],"data_stores":[]} diff --git a/schema/project.schema.json b/schema/project.schema.json index 9401166d5..4dacb5f26 100644 --- a/schema/project.schema.json +++ b/schema/project.schema.json @@ -1082,7 +1082,8 @@ "picture-markers", "layout-element-ink", "field-generation", - "crs-wkt" + "crs-wkt", + "zarr-window" ], "type": "string" }, @@ -1162,7 +1163,8 @@ "picture-markers", "layout-element-ink", "field-generation", - "crs-wkt" + "crs-wkt", + "zarr-window" ], "x-rustgis-reads-format": 2, "x-rustgis-writes-format": 2