Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 41 additions & 1 deletion crates/data/src/raster/zarr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2281,7 +2281,10 @@ fn grid_axes(store: &Store, a: &Array, chunks: &HashMap<String, Vec<u8>>) -> Res
let (Some((y0, dy)), Some((x0, dx))) = (axis(&ys), axis(&xs)) else {
return Err(bad(format!("{}: its coordinates aren't evenly spaced", a.path)));
};
Ok(((y0, dy, ys.len() as u64), (x0, dx, xs.len() as u64), geographic(xc)))
// Longitude by the coordinate's name or units, or by the geographic system the store gives
// (`assemble` wraps such a grid by it too, whatever its axes are called).
let geo = geographic(xc) || crs_of(store, a).is_some_and(|(c, _)| c.is_geographic());
Comment thread
elpidioassinaturas marked this conversation as resolved.
Resolved
Ok(((y0, dy, ys.len() as u64), (x0, dx, xs.len() as u64), geo))
}

/// The chunk keys of `variable`'s map coordinates, for [`window_of`]: each is read whole (its
Expand Down Expand Up @@ -3440,6 +3443,43 @@ mod tests {
}
}

/// A grid whose axes are named `x`/`y` but whose store gives a geographic system is wrapped by
/// `assemble` (0–360 read as -180–180), so a box or a point is placed with the same ±360° turns;
/// a projected one isn't turned.
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn a_geographic_system_turns_boxes_and_points_on_named_x_y_axes() {
let f64s = |v: Vec<f64>| v.iter().flat_map(|x| x.to_le_bytes()).collect::<Vec<u8>>();
for (code, geo) in [(4326, true), (3857, false)] {
let mut files = window_store(false);
let mut root: Value = serde_json::from_slice(&files[0].1).unwrap();
root["consolidated_metadata"]["metadata"]["v"]["attributes"] = serde_json::json!({"epsg_code": code});
files[0].1 = root.to_string().into_bytes();
// Columns centred on 185°…295°: east of 180, as a 0–360 grid stores them.
files[2].1 = zstd::encode_all(&f64s((0..12).map(|i| 185.0 + 10.0 * i as f64).collect())[..], 3).unwrap();
let store = parse_store(&files).unwrap();
let coords: HashMap<String, Vec<u8>> =
files.iter().filter(|(k, _)| window_keys(&store, "v").unwrap().contains(k)).cloned().collect();
let found = locate(&store, "v", &coords, -165.0, 50.0).unwrap();
let placed = window_of(&store, "v", &coords, [-170.0, 30.0, -150.0, 60.0]);
if !geo {
assert_eq!(found, None, "EPSG:{code}");
assert!(placed.is_err(), "EPSG:{code}");
continue;
}
assert_eq!(found, Some((3, 1)), "EPSG:{code}");
let window = Window { x0: 1, y0: 2, x1: 3, y1: 5 };
assert_eq!(placed.unwrap(), (window, false, false), "EPSG:{code}");
// The same box written 0–360 is the same window.
assert_eq!(window_of(&store, "v", &coords, [190.0, 30.0, 210.0, 60.0]).unwrap().0, window);
let selection = Selection { variable: "v".into(), index: vec![("time".into(), 1)], window: Some(window) };
let plan = plan(&store, &selection).unwrap();
let raster = assemble(&store, &plan, &read_plan(&files, &plan), "v", None).unwrap();
assert_eq!(raster.crs, Crs::LonLat);
assert!((raster.transform.origin_x - -170.0).abs() < 1e-9, "{}", raster.transform.origin_x);
}
}

/// A store with longitude/latitude axes is in the geographic system it gives (ED50 with its
/// TOWGS84 shift, a datum with no shift, another body's), whether as WKT, a PROJ definition or an
/// EPSG code, not in WGS 84 (#2008). What reading it said is kept once: another body by its body,
Expand Down
41 changes: 29 additions & 12 deletions crates/engine/src/cmd/zarr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ pub fn commands() -> Vec<C> {
r#"{"layer"? (a Zarr layer: its store, variable and steps; default the active layer) | "path": folder | "url": "https://…", "variable"?, "index"?, "stat": "mean"|"min"|"max"|"sum"|"std" (the population standard deviation, n)|"count"|"countAbove", "above"?: number (countAbove's threshold), "start"?, "end"?: dates (default: the layer's period, else the whole axis), "months"?: [1–12] (a season across years: [12, 1, 2]), "minValid"?: 0–1 (a cell with a smaller share of values is left empty; default 0), "name"?: text, "crs"?} → a raster layer of the statistic (desktop)"#,
),
C::new("layer.addZarr", "Add Zarr Variable", "Home › Add › Add Data", add).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Network).with(crate::policy::Effect::Files)).params(
r#"{"path": folder | "url": "https://…", "variable"?: "its path in the store (default: the first)", "index"?: {"time": 3, "level": 0} (other dimensions' steps, default 0), "cells"?: [first column, first row, end column, end row] (a window of the map's cells as stored, the ends not included, at least 2 × 2: only the chunks it touches are read, and in shards only their chunks it needs, by range) | "bbox"?: [west, south, east, north] (a window by the grid's coordinates: degrees for a longitude/latitude grid; its coordinates are read to place it), "crs"?: "EPSG:… for a store that doesn't say", "name"?: text, "time"?: bool (follow the time slider: the layer shows the step the slider is at, and turns the slider on over the variable's dates when it is off; desktop), "start"?, "end"?: dates (with time: the period to take, for an axis longer than the slider's 20,000 steps), "ran"?: (internal: a slice the app has read), "summary"?: bool (internal: with ran, a summary over time rather than a slice)}"#,
r#"{"path": folder | "url": "https://…", "variable"?: "its path in the store (default: the first)", "index"?: {"time": 3, "level": 0} (other dimensions' steps, default 0), "cells"?: [first column, first row, end column, end row] (a window of the map's cells as stored, the ends not included, at least 2 × 2: only the chunks it touches are read, and in shards only their chunks it needs, by range) | "bbox"?: [west, south, east, north] (a window by the grid's coordinates: degrees for a longitude/latitude grid; its coordinates are read to place it), "crs"?: "EPSG:… for a store that doesn't say", "name"?: text, "time"?: bool (follow the time slider: the layer shows the step the slider is at, and turns the slider on over the variable's dates when it is off; with cells or bbox, each step reads only that window; desktop), "start"?, "end"?: dates (with time: the period to take, for an axis longer than the slider's 20,000 steps), "ran"?: (internal: a slice the app has read), "summary"?: bool (internal: with ran, a summary over time rather than a slice)}"#,
),
]
}
Expand Down Expand Up @@ -463,10 +463,7 @@ fn place_window(root: &str, store: &zarr::Store, sel: &mut zarr::Selection, aske
pub(crate) fn add(s: &mut Session, p: &Value) -> CmdResult {
let asked = asked_window(p)?;
if b(p, "time") == Some(true) {
if asked.is_some() {
return Err("a window (cells or bbox) isn't read with time yet: leave time out to read it".into());
}
return add_following(s, p);
return add_following(s, p, asked);
}
let src = source(p)?;
#[cfg_attr(target_arch = "wasm32", expect(unused_mut, reason = "windowed reads are desktop only"))]
Expand Down Expand Up @@ -616,7 +613,13 @@ fn cached_store(url: &str, variable: &str) -> Result<std::sync::Arc<zarr::Store>
Ok(st)
}

/// The key a slice is kept by: its store, selection and coordinate system.
/// The window of cells a following layer's steps read.
#[cfg(not(target_arch = "wasm32"))]
fn zt_window(zt: &ZarrTime) -> Option<zarr::Window> {
zt.window.map(|[x0, y0, x1, y1]| zarr::Window { x0, y0, x1, y1 })
}

/// The key a slice is kept by: its store, selection (with its window) and coordinate system.
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn slice_key(zt: &ZarrTime, i: usize) -> String {
// A folder store rewritten on disk (a model run adding steps) is read again: its metadata's
Expand Down Expand Up @@ -803,7 +806,8 @@ pub(crate) fn read_slice(zt: &ZarrTime, i: usize) -> Result<Option<raster::Raste
let (read, st) = if is_web(&zt.store) {
let st = cached_store(&zt.store, &zt.variable)?;
let plan = zarr::plan(&st, &zarr::Selection::parse(&sel)).map_err(|e| e.to_string())?;
let (chunks, forbidden) = fetch(&zt.store, &plan.keys)?;
// A window's coordinates and, in shards, only its chunks, by range.
let (chunks, forbidden) = fetch_plan(&zt.store, &st, &plan)?;
// A chunk refused (HTTP 403) isn't one never written: it's said, not read as fill.
if forbidden > 0 {
return Err(crate::sources::message_with_address(
Expand Down Expand Up @@ -1239,10 +1243,10 @@ fn follow(s: &mut Session, wait: bool) {
/// `layer.addZarr` with `"time": true`: the variable's time axis read from its coordinate, a layer
/// that shows the step the slider is at, and the slider turned on over the axis when it is off.
/// The stretch is fixed at the first slice's, so a colour means one value at every step.
fn add_following(s: &mut Session, p: &Value) -> CmdResult {
fn add_following(s: &mut Session, p: &Value, asked: Option<Asked>) -> CmdResult {
Comment thread
elpidioassinaturas marked this conversation as resolved.
Resolved
#[cfg(target_arch = "wasm32")]
{
let _ = (s, p);
let _ = (s, p, asked);
Err("in the web app, a Zarr variable can't follow the time slider yet".into())
}
#[cfg(not(target_arch = "wasm32"))]
Expand Down Expand Up @@ -1286,6 +1290,8 @@ fn add_following(s: &mut Session, p: &Value) -> CmdResult {
}
};
pick_variable(&store, &mut sel)?;
// A window: every step reads only its cells (and in shards only their chunks it needs).
let window_notes = place_window(&root, &store, &mut sel, asked)?;
let chunks =
if chunks.is_empty() && is_web(&root) { fetch(&root, &zarr::coordinate_keys(&store, &sel.variable))?.0 } else { chunks };
let (dim, dates) = zarr::time_axis(&store, &sel.variable, &chunks).ok_or_else(|| {
Expand All @@ -1294,6 +1300,12 @@ fn add_following(s: &mut Session, p: &Value) -> CmdResult {
sel.variable
)
})?;
// A window is checked against the grid as a still window is (within it, 2 × 2 cells at
// least), before the slider is touched: the same for every step, so step 0 stands for all.
if sel.window.is_some() {
let first = zarr::Selection { index: sel.index.iter().filter(|(d, _)| *d != dim).cloned().collect(), ..sel.clone() };
zarr::plan(&store, &first).map_err(|e| crate::sources::message_with_address(&root, &e.to_string()))?;
}
// Each date is a step: an axis out of order or with a date twice (stores joined end to end)
// can't be followed step by step.
if !dates.iter().all(|d| d.is_finite()) || !dates.windows(2).all(|w| w[0] < w[1]) {
Expand Down Expand Up @@ -1339,6 +1351,7 @@ fn add_following(s: &mut Session, p: &Value) -> CmdResult {
offset: first as u64,
dates: picked,
crs,
window: sel.window.map(|w| [w.x0, w.y0, w.x1, w.y1]),
Comment thread
elpidioassinaturas marked this conversation as resolved.
Resolved
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
shown: None,
read_generation: rustgis_map::raster::next_read_generation(),
};
Expand Down Expand Up @@ -1390,9 +1403,12 @@ fn add_following(s: &mut Session, p: &Value) -> CmdResult {
});
layer.time_zarr = Some(zt);
let mut out = super::raster::add_raster(s, &name, layer)?;
if let Some(w) = part {
let said: Vec<String> = window_notes.into_iter().chain(part).collect();
for w in &said {
s.warn(w.clone());
out["warning"] = serde_json::json!(w);
}
if !said.is_empty() {
out["warning"] = serde_json::json!(said.join("; "));
}
Ok(out)
}
Expand Down Expand Up @@ -1423,7 +1439,7 @@ fn series_source(s: &Session, p: &Value) -> Result<SeriesSource, String> {
if let Some(zt) = &rl.time_zarr {
return Ok(SeriesSource {
root: zt.store.clone(),
sel: zarr::Selection { variable: zt.variable.clone(), index: zt.index.clone(), window: None },
sel: zarr::Selection { variable: zt.variable.clone(), index: zt.index.clone(), window: zt_window(zt) },
Comment thread
elpidioassinaturas marked this conversation as resolved.
Resolved
period: Some((zt.offset, zt.dates.len())),
crs: zt.crs.clone(),
read_generation: zt.read_generation,
Expand Down Expand Up @@ -1954,6 +1970,7 @@ pub fn zarr_cube(s: &Session, p: &Value) -> Result<ZarrCube, String> {
offset: 0,
dates,
crs: src.crs.clone(),
window: None,
shown: None,
read_generation: src.read_generation,
};
Expand Down
1 change: 1 addition & 0 deletions crates/engine/src/project.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ pub const CAPABILITIES: &[&str] = &[
"layout-element-ink",
"field-generation",
"crs-wkt",
"zarr-window",
];
/// Independent marker sizes alongside classified point colours (`Layer::size_by`).
pub(crate) const SIZE_BY_CAPABILITY: &str = "classified-point-size";
Expand Down
39 changes: 36 additions & 3 deletions crates/engine/src/project_capabilities.rs
Original file line number Diff line number Diff line change
Expand Up @@ -226,22 +226,29 @@ fn relief_light(layer: &Layer) -> bool {
)
}

/// The renderer capabilities a layer needs, in [`map_capabilities`]'s order.
/// Whether a layer is a window of a Zarr variable following the time slider (#1720): an older
/// reader would drop the window and read the whole map at each step.
fn zarr_window(layer: &Layer) -> bool {
layer.raster.as_ref().and_then(|r| r.time_zarr.as_ref()).is_some_and(|z| z.window.is_some())
}

/// The capabilities a layer needs, in [`map_capabilities`]'s order.
fn layer_capabilities(layer: &Layer) -> Vec<&'static str> {
[
(relief_effects(layer), "shaded-relief-effects"),
(relief_light(layer), "shaded-relief-light"),
(layer.size_by.is_some(), super::SIZE_BY_CAPABILITY),
(layer.has_picture_markers(), "picture-markers"),
(zarr_window(layer), "zarr-window"),
]
.into_iter()
.filter_map(|(used, name)| used.then_some(name))
.collect()
}

fn map_capabilities<'a>(maps: impl IntoIterator<Item = &'a Map>) -> Vec<&'static str> {
let (mut block, mut water, mut shore, mut relief, mut light, mut legends, mut sized, mut pictures) =
(false, false, false, false, false, false, false, false);
let (mut block, mut water, mut shore, mut relief, mut light, mut legends, mut sized, mut pictures, mut zarr) =
(false, false, false, false, false, false, false, false, false);
for map in maps {
if let Some(t) = &map.view.terrain {
block |= t.block.is_some();
Expand All @@ -253,6 +260,7 @@ fn map_capabilities<'a>(maps: impl IntoIterator<Item = &'a Map>) -> Vec<&'static
light |= map.layers.iter().any(relief_light);
sized |= map.layers.iter().any(|layer| layer.size_by.is_some());
pictures |= map.layers.iter().any(Layer::has_picture_markers);
zarr |= map.layers.iter().any(zarr_window);
legends |= map.tiles.iter().any(|tile| {
tile.legends.iter().any(|legend| rustgis_data::login_address::inner(&legend.url).len() > rustgis_map::tiles::MAX_LEGEND_URL)
});
Expand All @@ -266,6 +274,7 @@ fn map_capabilities<'a>(maps: impl IntoIterator<Item = &'a Map>) -> Vec<&'static
(legends, "normalized-legend-urls"),
(sized, super::SIZE_BY_CAPABILITY),
(pictures, "picture-markers"),
(zarr, "zarr-window"),
]
.into_iter()
.filter_map(|(used, name)| used.then_some(name))
Expand Down Expand Up @@ -711,6 +720,30 @@ mod tests {
assert!(Project::from_json(&unknown.to_string()).unwrap_err().contains("future-terrain"));
}

/// A Zarr window following the time slider (#1720) is required by name in a project, a map
/// file and a layer file: an older reader would drop it and read the whole map at each step.
#[test]
fn a_zarr_window_is_required_where_it_is_saved() {
let mut p = Project::from_json(include_str!("../../../schema/fixtures/zarr_window.rgis")).unwrap();
let layer = p.map().layers[0].clone();
assert_eq!(
layer.raster.as_ref().and_then(|r| r.time_zarr.as_ref()).and_then(|z| z.window),
Some([1526419, 1162633, 1526643, 1162857])
);
let saved = serde_json::to_value(&p).unwrap();
assert_eq!(saved["requires"], json!(["zarr-window"]));
assert_eq!(json_bytes(&map_file_bytes(p.map()).unwrap())["requires"], saved["requires"]);
assert_eq!(json_bytes(&layer_file_bytes(&layer).unwrap())["requires"], saved["requires"]);
// The whole variable following the slider needs nothing new.
if let Some(z) = p.maps[0].layers[0].raster.as_mut().and_then(|r| r.time_zarr.as_mut()) {
z.window = None;
}
assert!(serde_json::to_value(&p).unwrap().get("requires").is_none());
let mut unknown = saved;
unknown["requires"] = json!(["zarr-window", "future-zarr"]);
assert!(Project::from_json(&unknown.to_string()).unwrap_err().contains("future-zarr"));
}

/// Nondefault relief effects survive project, map and layer files; ordinary relief needs none.
#[test]
fn relief_headers_cover_each_effect_and_clean_controls() {
Expand Down
Loading
Loading