From 751b29556885be46784541a3ad749fc31845e8b2 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 12 Nov 2026 12:51:32 -0700 Subject: [PATCH 1/5] fix(data, map, tiles, engine): see-through cells are left out of colour statistics and Change Detection (#1810) A colour band's statistics counted every valid cell, including the cells its alpha band draws see-through. A Float32 RGBA image whose see-through cells hold -3.4e38 (as an export from before #1759 wrote them) was stretched from -3.4e38 and drew solid white, and see-through zeros left by Project Raster or Clip lowered minima and raised counts. - Raster statistics leave out, in the colour bands of a raster with alpha (RGBA, grey and alpha), the cells the map draws fully see-through: an alpha of no value, or under half a 256th of opaque. Opaque black cells count, and the alpha band counts every cell. BandStats takes each value's place, so the runs on several threads give the same statistics as one. - A style mask's sampled statistics and a cloud-optimised GeoTIFF's stretch (read from its coarsest overview) leave them out too. - Change Detection gives no-data where either raster is see-through, the second's alpha resampled onto the first's grid as its band is. - The GeoTIFF and COG writers choose bytes from the statistics; they now look at the see-through cells too, so a file is written in the type it was before. --- crates/data/src/raster/geotiff.rs | 10 +- crates/data/src/raster/mod.rs | 250 ++++++++++++++++++++++++------ crates/engine/src/cmd/imagery.rs | 15 +- crates/engine/src/tests.rs | 117 ++++++++++++++ crates/map/src/raster.rs | 43 ++++- crates/tiles/src/cog.rs | 86 +++++++++- docs/user-guide/data.md | 9 ++ 7 files changed, 478 insertions(+), 52 deletions(-) diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index bdea0b2dd2..5c53611d39 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -1596,13 +1596,19 @@ fn as_area(keys: &[u16]) -> Vec { } /// Whether band `b` is written as bytes: whole numbers within 0–255 (or no valid pixels at all), -/// and a no-data value that is one too. +/// and a no-data value that is one too. A colour band's statistics leave out the cells its alpha +/// draws see-through (#1810), so those cells' values are looked at too: a file is written in the +/// type it was before, whatever they hold. pub(super) fn fits_bytes(r: &Raster, b: usize) -> bool { let Some(band) = r.bands.get(b) else { return false }; let s = &band.stats; let whole_bytes = s.min >= 0.0 && s.max <= 255.0 && (s.categories.is_some() || r.data_type == "UInt8"); let nodata_ok = r.nodata.is_none_or(|d| (0.0..=255.0).contains(&d) && d.fract() == 0.0); - nodata_ok && (whole_bytes || s.count == 0) + let byte = |v: f32| !r.is_valid(v) || (v.fract() == 0.0 && (0.0..=255.0).contains(&v)); + // An 8-bit band's are bytes already. + let see_through_bytes = r.data_type == "UInt8" + || r.see_through_cells(b).is_none_or(|hidden| band.data.iter().enumerate().all(|(i, &v)| !hidden(i) || byte(v))); + nodata_ok && (whole_bytes || s.count == 0) && see_through_bytes } /// The bands [`write`] writes of `r` given `band`: RGB and its fourth sample (alpha only when diff --git a/crates/data/src/raster/mod.rs b/crates/data/src/raster/mod.rs index c857850364..8c7bfa7658 100644 --- a/crates/data/src/raster/mod.rs +++ b/crates/data/src/raster/mod.rs @@ -164,16 +164,33 @@ pub struct BandStats { impl BandStats { /// Statistics of the finite values in `data`. pub fn from_values(data: &[f32]) -> BandStats { - BandStats::compute(data, f32::is_finite) - } - - fn compute(data: &[f32], valid: impl Fn(f32) -> bool + Sync) -> BandStats { - BandStats::compute_with(data, valid, None) + BandStats::compute_with(data, |_, v| v.is_finite(), None) + } + + /// Statistics of a raster's band `data`: the values `valid` takes, less, for a colour band of + /// a raster with alpha, the cells its alpha band (`alpha`) draws see-through ([`see_through`], + /// an alpha of `opaque` being opaque). `stored` is the band's whole statistics, for a reduced + /// overview ([`BandStats::compute_with`]); GDAL counts see-through cells in all but 8-bit + /// images, so a band with `alpha` doesn't take them. + fn of_band( + data: &[f32], + valid: impl Fn(f32) -> bool + Sync, + alpha: Option<&[f32]>, + opaque: f32, + stored: Option, + ) -> BandStats { + match alpha { + Some(alpha) => { + BandStats::compute_with(data, |i, v| valid(v) && alpha.get(i).is_some_and(|&a| !see_through(a, valid(a), opaque)), None) + } + None => BandStats::compute_with(data, |_, v| valid(v), stored), + } } - /// [`BandStats::compute`], taking the range, mean and spread from `stored` (statistics of the - /// whole band, when `data` is a reduced overview of it). - fn compute_with(data: &[f32], valid: impl Fn(f32) -> bool + Sync, stored: Option) -> BandStats { + /// Statistics of the values of `data` that `valid` takes (given each one's place in `data` and + /// its value), taking the range, mean and spread from `stored` (statistics of the whole band, + /// when `data` is a reduced overview of it). + fn compute_with(data: &[f32], valid: impl Fn(usize, f32) -> bool + Sync, stored: Option) -> BandStats { BandStats::compute_on(data, valid, stored, threads_for(data.len())) } @@ -183,10 +200,14 @@ impl BandStats { /// the same to the bit; what doesn't depend on the order is counted on the others /// ([`on_runs`]), the range while this thread sums. The range keeps the first of equal values /// (−0 or 0) in the band's order either way. - fn compute_on(data: &[f32], valid: impl Fn(f32) -> bool + Sync, stored: Option, threads: usize) -> BandStats { - let ranged = |run: &[f32], sum: bool| { + fn compute_on(data: &[f32], valid: impl Fn(usize, f32) -> bool + Sync, stored: Option, threads: usize) -> BandStats { + // A run of `data` from `start`. + let ranged = |start: usize, run: &[f32], sum: bool| { let (mut n, mut min, mut max, mut whole, mut sums) = (0usize, f64::INFINITY, f64::NEG_INFINITY, true, (0.0f64, 0.0f64)); - for &v in run.iter().filter(|&&v| valid(v)) { + for (j, &v) in run.iter().enumerate() { + if !valid(start + j, v) { + continue; + } let x = v as f64; n += 1; (min, max) = (lower(min, x), higher(max, x)); @@ -198,10 +219,10 @@ impl BandStats { (n, min, max, whole, sums) }; let (n, mut min, mut max, whole, (sum, sum2)) = if threads <= 1 { - ranged(data, true) + ranged(0, data, true) } else { - let sums = || ranged(data, true).4; - let (runs, sums) = on_runs_beside(data, threads, sums, |run| ranged(run, false)); + let sums = || ranged(0, data, true).4; + let (runs, sums) = on_runs_beside(data, threads, sums, |start, run| ranged(start, run, false)); let mut all = (0usize, f64::INFINITY, f64::NEG_INFINITY, true, sums); for (rn, rmin, rmax, rwhole, _) in runs { all = (all.0 + rn, lower(all.1, rmin), higher(all.2, rmax), all.3 && rwhole, all.4); @@ -224,8 +245,8 @@ impl BandStats { let small_ints = whole && min >= 0.0 && max <= 65_535.0; let k = (HISTOGRAM_BINS as f64 - 1.0) / if span > 0.0 { span } else { 1.0 }; let b = Binning { min, k, small_ints, whole }; - let mut runs = on_runs(data, threads, |run| Tally::run(run, &valid, b)).into_iter(); - let mut tally = runs.next().unwrap_or_else(|| Tally::run(&[], &valid, b)); + let mut runs = on_runs(data, threads, |start, run| Tally::run(start, run, &valid, b)).into_iter(); + let mut tally = runs.next().unwrap_or_else(|| Tally::run(0, &[], &valid, b)); for t in runs { tally.histogram.iter_mut().zip(&t.histogram).for_each(|(a, b)| *a += b); tally.bits.iter_mut().zip(&t.bits).for_each(|(a, b)| *a |= b); @@ -309,32 +330,40 @@ fn threads_for(len: usize) -> usize { if cfg!(target_arch = "wasm32") || len < PARALLEL_VALUES { 1 } else { crate::threads::max().min(len / (PARALLEL_VALUES / 4)) } } -/// `f` over `threads` runs of `data` that cover it in order, its results in that order, the -/// calling thread taking the first. A worker's panic is raised again here; a run whose thread -/// can't start is made on this one. -fn on_runs(data: &[f32], threads: usize, f: impl Fn(&[f32]) -> T + Sync) -> Vec { +/// `f` over `threads` runs of `data` that cover it in order, each given with where it starts in +/// `data`, its results in that order, the calling thread taking the first. A worker's panic is +/// raised again here; a run whose thread can't start is made on this one. +fn on_runs(data: &[f32], threads: usize, f: impl Fn(usize, &[f32]) -> T + Sync) -> Vec { on_runs_beside(data, threads, || (), f).0 } /// [`on_runs`], the calling thread running `beside` first, while the other threads start on /// their runs. -fn on_runs_beside(data: &[f32], threads: usize, beside: impl FnOnce() -> U, f: impl Fn(&[f32]) -> T + Sync) -> (Vec, U) { +fn on_runs_beside( + data: &[f32], + threads: usize, + beside: impl FnOnce() -> U, + f: impl Fn(usize, &[f32]) -> T + Sync, +) -> (Vec, U) { if threads <= 1 || data.is_empty() { let beside = beside(); - return (vec![f(data)], beside); + return (vec![f(0, data)], beside); } - let mut runs = data.chunks(data.len().div_ceil(threads)); - let first = runs.next().unwrap_or_default(); + let size = data.len().div_ceil(threads); + let mut runs = data.chunks(size).enumerate().map(|(k, run)| (k * size, run)); + let (start, first) = runs.next().unwrap_or_default(); let f = &f; std::thread::scope(|scope| { - let started: Vec<_> = runs.map(|run| std::thread::Builder::new().spawn_scoped(scope, move || f(run)).map_err(|_| run)).collect(); + let started: Vec<_> = runs + .map(|(start, run)| std::thread::Builder::new().spawn_scoped(scope, move || f(start, run)).map_err(|_| (start, run))) + .collect(); let beside = beside(); - let mut out = vec![f(first)]; + let mut out = vec![f(start, first)]; for run in started { out.push(match run { Ok(h) => h.join().unwrap_or_else(|p| std::panic::resume_unwind(p)), // The OS refused a thread: made here. - Err(run) => f(run), + Err((start, run)) => f(start, run), }); } (out, beside) @@ -342,14 +371,14 @@ fn on_runs_beside(data: &[f32], threads: usize, beside: impl FnOnce( } impl Tally { - /// `data`'s valid values counted on this thread. - fn run(data: &[f32], valid: &impl Fn(f32) -> bool, b: Binning) -> Tally { + /// `data`'s valid values counted on this thread: a run of a band from `start`. + fn run(start: usize, data: &[f32], valid: &impl Fn(usize, f32) -> bool, b: Binning) -> Tally { let mut histogram = vec![0u64; HISTOGRAM_BINS]; let mut bits = if b.small_ints { vec![0u64; 1024] } else { Vec::new() }; let mut distinct = std::collections::BTreeSet::new(); let mut track = b.whole; - for &v in data { - if !valid(v) { + for (j, &v) in data.iter().enumerate() { + if !valid(start + j, v) { continue; } let x = v as f64; @@ -537,6 +566,13 @@ pub fn alpha_max(data_type: &str) -> f32 { } } +/// Whether an alpha value draws its cell fully see-through, as the map draws it: an alpha of no +/// value (`valid` false: NaN, or the raster's no-data), or one under half a 256th of `opaque` (the +/// pixel type's [`alpha_max`]), which is nothing in 8 bits. +pub fn see_through(a: f32, valid: bool, opaque: f32) -> bool { + !valid || ((a / opaque).clamp(0.0, 1.0) * 255.0).round() == 0.0 +} + /// How a raster's bands were meant to be shown, from the file. #[derive(Clone, Debug, Default, PartialEq)] pub enum ColorHint { @@ -550,6 +586,19 @@ pub enum ColorHint { Palette(Vec<[u8; 4]>), } +impl ColorHint { + /// For a raster of `bands` bands shown this way with alpha, its alpha band and the colour + /// bands that band makes see-through: an RGBA image's fourth band over its first three, grey + /// and alpha's second over its first. + pub fn alpha(&self, bands: usize) -> Option<(usize, std::ops::Range)> { + match self { + ColorHint::Rgb { alpha: true } if bands >= 4 => Some((3, 0..3)), + ColorHint::GrayAlpha if bands >= 2 => Some((1, 0..1)), + _ => None, + } + } +} + /// A raster on a grid: georeferenced, unless `georeferenced` is false (a scan, whose transform /// is only a stand-in). #[derive(Clone)] @@ -737,16 +786,21 @@ impl Raster { let transformer = crs.transformer().map_err(DataError::Format)?; let nd = nodata.filter(|v| v.is_finite()).map(|v| v as f32); let valid = move |v: f32| v.is_finite() && nd.is_none_or(|d| v != d); + let data_type = data_type.into(); + let (alpha, opaque) = (color.alpha(bands.len()), alpha_max(&data_type)); + let stats: Vec = bands + .iter() + .enumerate() + .map(|(i, data)| { + let alpha = alpha.as_ref().filter(|(_, colour)| colour.contains(&i)).and_then(|(a, _)| bands.get(*a)); + if measure { BandStats::of_band(data, valid, alpha.map(Vec::as_slice), opaque, None) } else { BandStats::default() } + }) + .collect(); let bands = bands .into_iter() + .zip(stats) .enumerate() - .map(|(i, data)| Band { - name: format!("Band {}", i + 1), - stats: if measure { BandStats::compute(&data, valid) } else { BandStats::default() }, - data, - classes: Vec::new(), - wavelength: None, - }) + .map(|(i, (data, stats))| Band { name: format!("Band {}", i + 1), stats, data, classes: Vec::new(), wavelength: None }) .collect(); let mut r = Raster { name: name.into(), @@ -757,7 +811,7 @@ impl Raster { crs, nodata: nodata.filter(|v| v.is_finite()), color, - data_type: data_type.into(), + data_type, source: None, overview: None, georeferenced: true, @@ -838,6 +892,16 @@ impl Raster { let nd = self.nodata.map(|v| v as f32); let valid = move |v: f32| v.is_finite() && nd.is_none_or(|d| v != d); let overview = self.overview.is_some(); + // Worked out before the bands change, as a colour band's take its alpha band's values. + let (alpha, opaque) = (self.color.alpha(self.bands.len()), alpha_max(&self.data_type)); + let mut measured: Vec> = (0..self.bands.len()) + .map(|i| { + let stored = pam.band(i + 1).and_then(|p| p.stats).filter(|_| overview); + let data = self.bands.get(i).filter(|_| recompute || stored.is_some())?; + let alpha = alpha.as_ref().filter(|(_, colour)| colour.contains(&i)).and_then(|(a, _)| self.bands.get(*a)); + Some(BandStats::of_band(&data.data, valid, alpha.map(|a| a.data.as_slice()), opaque, stored)) + }) + .collect(); for (i, band) in self.bands.iter_mut().enumerate() { let p = pam.band(i + 1); if let Some(d) = p.and_then(|p| p.description.as_ref()) { @@ -849,9 +913,8 @@ impl Raster { if let Some(w) = p.and_then(|p| p.wavelength) { band.wavelength = Some(w); } - let stored = p.and_then(|p| p.stats).filter(|_| overview); - if recompute || stored.is_some() { - band.stats = BandStats::compute_with(&band.data, valid, stored); + if let Some(stats) = measured.get_mut(i).and_then(Option::take) { + band.stats = stats; } } } @@ -1012,6 +1075,17 @@ impl Raster { self.rounded_some = if marks.contains(&true) { marks } else { Vec::new() }; } + /// The cells of `band` drawn see-through, by their place (row by row): `band` is a colour + /// band of a raster with alpha ([`ColorHint::alpha`]) and the cell's alpha draws it so + /// ([`see_through`]). Its statistics leave them out. `None` when the band has no such cells + /// to have: no alpha, or the alpha band or a band of data beside the colours. + pub fn see_through_cells(&self, band: usize) -> Option bool + Sync + '_> { + let (a, colour) = self.color.alpha(self.bands.len())?; + let alpha = &self.bands.get(a).filter(|_| colour.contains(&band))?.data; + let opaque = alpha_max(&self.data_type); + Some(move |i: usize| alpha.get(i).is_none_or(|&v| see_through(v, self.is_valid(v), opaque))) + } + /// Whether a value is a real measurement (not no-data or NaN). pub fn is_valid(&self, v: f32) -> bool { // Compared in f32, as the pixels (and the statistics) are: a no-data value such as @@ -1540,9 +1614,10 @@ pub(crate) mod tests { ("gaps", (0..n).map(|i| [f32::NAN, -0.0, 0.0, -9999.0, 17.5][at(i) % 5]).collect()), ]; let bits = |s: &BandStats| (s.count, s.min.to_bits(), s.max.to_bits(), s.mean.to_bits(), s.std_dev.to_bits()); - let valid = |v: f32| v.is_finite() && v != -9999.0; + let value = |v: f32| v.is_finite() && v != -9999.0; + let valid = |_: usize, v: f32| value(v); for (name, data) in &bands { - let want = one_pass(data, valid); + let want = one_pass(data, value); for threads in [1, 2, 7, 16] { let got = BandStats::compute_on(data, valid, None, threads); assert_eq!(bits(&got), bits(&want), "{name} on {threads}"); @@ -1558,6 +1633,93 @@ pub(crate) mod tests { assert_eq!(BandStats::compute_on(&vec![f32::NAN; n], valid, None, 4), BandStats::default()); } + /// Cells a raster's alpha draws see-through are left out of its colour bands' statistics, + /// whatever their values (#1810): −3.4e38 in a Float32 image, a stray byte in an 8-bit one. An + /// opaque black cell counts, the alpha band counts every cell, and a raster without alpha (or + /// with a fourth band of data) counts every valid cell, as before. + #[test] + fn see_through_cells_are_left_out_of_colour_statistics() { + let gt = GeoTransform { origin_x: 500_000.0, origin_y: 4_800_000.0, pixel_width: 1.0, pixel_height: -1.0 }; + let utm = Crs::from_epsg(32611).unwrap(); + // 4 × 4: the bottom row see-through (holding `see`), the first cell opaque black, the rest 100. + let colour = |see: f32| { + (0..16) + .map(|i| { + if i >= 12 { + see + } else if i == 0 { + 0.0 + } else { + 100.0 + } + }) + .collect::>() + }; + let alpha = |opaque: f32| (0..16).map(|i| if i >= 12 { 0.0 } else { opaque }).collect::>(); + let stats = |b: &Band| (b.stats.count, b.stats.min, b.stats.max); + for (data_type, see, opaque) in [("Float32", -3.4e38_f32, 1.0), ("UInt8", 7.0, 255.0), ("UInt16", 7.0, 65535.0)] { + let bands = vec![colour(see), colour(see), colour(see), alpha(opaque)]; + let r = Raster::new("rgba", 4, 4, bands, gt, utm, None, ColorHint::Rgb { alpha: true }, data_type).unwrap(); + for b in &r.bands[..3] { + assert_eq!(stats(b), (12, 0.0, 100.0), "{data_type}"); + // Eleven cells of 100 and one of 0: the mean GDAL 3.9.3 gives an 8-bit RGBA band. + assert!((b.stats.mean - 1100.0 / 12.0).abs() < 1e-9, "{data_type}: {}", b.stats.mean); + assert_eq!(b.stats.histogram.iter().sum::(), 12, "{data_type}"); + } + assert_eq!(stats(&r.bands[3]), (16, 0.0, f64::from(opaque)), "{data_type}"); + // Read back from its GeoTIFF, the same. + let back = geotiff::read(std::io::Cursor::new(geotiff::write(&r, 0).unwrap()), "back").unwrap(); + assert_eq!(back.color, ColorHint::Rgb { alpha: true }, "{data_type}"); + // Written in the type it was before: a Float32 image whose see-through cells hold + // −3.4e38 isn't taken for bytes, as a GeoTIFF or a cloud-optimised one. + if data_type == "Float32" { + let cog = geotiff::read(std::io::Cursor::new(cogwrite::write(&r, 0).unwrap()), "cog").unwrap(); + assert_eq!((back.data_type.as_str(), cog.data_type.as_str()), ("Float32", "Float32")); + } + assert_eq!(back.bands.iter().map(stats).collect::>(), r.bands.iter().map(stats).collect::>(), "{data_type}"); + } + // As the map draws it: an alpha under half a 256th of opaque is see-through (16 bits: 100 + // of 65535), one above it isn't (200). + let faint: Vec = (0..16) + .map(|i| { + if i >= 12 { + 100.0 + } else if i >= 8 { + 200.0 + } else { + 65535.0 + } + }) + .collect(); + let r = Raster::new( + "faint", + 4, + 4, + vec![colour(7.0), colour(7.0), colour(7.0), faint], + gt, + utm, + None, + ColorHint::Rgb { alpha: true }, + "UInt16", + ) + .unwrap(); + assert_eq!(r.bands[0].stats.count, 12); + // Grey and alpha: the grey band. + let r = Raster::new("grey", 4, 4, vec![colour(7.0), alpha(255.0)], gt, utm, None, ColorHint::GrayAlpha, "UInt8").unwrap(); + assert_eq!((stats(&r.bands[0]), r.bands[1].stats.count), ((12, 0.0, 100.0), 16)); + // No-data 0: an alpha of no value is see-through, as the map draws it, and the black cell + // is no-data, as before. + let bands = vec![colour(-3.4e38), colour(-3.4e38), colour(-3.4e38), alpha(1.0)]; + let r = Raster::new("nodata", 4, 4, bands, gt, utm, Some(0.0), ColorHint::Rgb { alpha: true }, "Float32").unwrap(); + assert_eq!(stats(&r.bands[0]), (11, 100.0, 100.0)); + // Without alpha every valid cell counts: RGB, and RGB with a fourth band of data. + for nb in [3, 4] { + let bands = (0..nb).map(|_| colour(7.0)).collect(); + let r = Raster::new("rgb", 4, 4, bands, gt, utm, None, ColorHint::Rgb { alpha: false }, "UInt8").unwrap(); + assert!(r.bands.iter().all(|b| stats(b) == (16, 0.0, 100.0)), "{nb} bands"); + } + } + /// A band the machine can't hold is an error naming its size, not the end of the program /// (#1291), on 64-bit (the reservation fails) and wasm32 (the size overflows) alike; one it can /// is the size asked for. diff --git a/crates/engine/src/cmd/imagery.rs b/crates/engine/src/cmd/imagery.rs index 3523c9fa52..8762af0d1f 100644 --- a/crates/engine/src/cmd/imagery.rs +++ b/crates/engine/src/cmd/imagery.rs @@ -864,6 +864,19 @@ pub(crate) fn change_work(s: &Session, p: &Value) -> Result Some(rasterops::resample_to(&bfr, alpha, &a)?), + _ => None, + }; + let b_opaque = raster::alpha_max(&bfr.data_type); + let hidden = |i: usize| { + a_hidden.as_ref().is_some_and(|h| h(i)) + || b_alpha.as_ref().is_some_and(|al| al.get(i).is_none_or(|&v| raster::see_through(v, bfr.is_valid(v), b_opaque))) + }; let n = a.width as usize * a.height as usize; let out = rasterops::per_index( n, @@ -872,7 +885,7 @@ pub(crate) fn change_work(s: &Session, p: &Value) -> Result bool, offset: f32) -> rustgis_data::raster::Raster { + use rustgis_data::raster::{ColorHint, Crs, GeoTransform, Raster}; + let (w, n) = (40usize, 1600usize); + let colour = |b: usize| -> Vec { + (0..n).map(|i| if see(i) { -3.4e38 } else if i < w { 0.0 } else { (10 + (i * 7 + b * 60) % 240) as f32 } + offset).collect() + }; + let alpha = (0..n).map(|i| if see(i) { 0.0 } else { 255.0 }).collect(); + let gt = GeoTransform { origin_x: 500_000.0, origin_y: 4_800_040.0, pixel_width: 1.0, pixel_height: -1.0 }; + let utm = Crs::from_epsg(32611).unwrap(); + Raster::new(name, 40, 40, vec![colour(0), colour(1), colour(2), alpha], gt, utm, None, ColorHint::Rgb { alpha: true }, "Float32") + .unwrap() +} + +/// The cells an RGBA raster's alpha draws see-through are left out of its colour bands' +/// statistics (#1810): read from a file, projected (nearest) and clipped to a polygon. Its opaque +/// black cells count, its alpha band counts every cell, and the stretch shared from it (`like`) +/// spans its opaque colours, not −3.4e38. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn see_through_cells_are_left_out_of_an_rgba_rasters_statistics() { + use rustgis_data::raster::geotiff; + let dir = std::env::temp_dir().join(format!("rustgis-see-through-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("ortho.tif"); + std::fs::write(&path, geotiff::write(&see_through_ortho("ortho", |i| i % 40 < 10, 0.0), 0).unwrap()).unwrap(); + let mut s = Session::new(Project::default()); + s.run("layer.addData", &json!({"path": path, "name": "ortho"})).unwrap(); + let tr = rustgis_data::raster::Crs::from_epsg(32611).unwrap().transformer().unwrap(); + let ll = |x: f64, y: f64| { + let c = tr.to_lonlat(rustgis_geom::Coord::new(x, y)).unwrap(); + json!([c.x, c.y]) + }; + // A square over the middle, across the see-through columns. + s.run("layer.template", &json!({"template": "polygons", "name": "Boundary"})).unwrap(); + let ring = [ + ll(500_005.0, 4_800_035.0), + ll(500_035.0, 4_800_035.0), + ll(500_035.0, 4_800_005.0), + ll(500_005.0, 4_800_005.0), + ll(500_005.0, 4_800_035.0), + ]; + s.run("edit.addFeature", &json!({"layer": "Boundary", "geometry": {"type": "Polygon", "coordinates": [ring]}})).unwrap(); + s.run("raster.project", &json!({"layer": "ortho", "crs": 32611, "cellSize": 1.0, "resampling": "nearest", "name": "projected"})) + .unwrap(); + s.run( + "raster.process", + &json!({"layer": "ortho", "operation": "clip", "polygon": "Boundary", "selectedOnly": false, "name": "clipped"}), + ) + .unwrap(); + for name in ["ortho", "projected", "clipped"] { + let r = s.map().layer_by_name(name).unwrap().raster.as_ref().unwrap().data.clone().unwrap(); + // The cells its alpha shows: alpha above half a 256th of opaque (Float32's 1). + let shown = r.bands[3].data.iter().filter(|a| **a >= 0.5 / 255.0).count(); + assert!(shown > 0 && shown < r.bands[3].data.len(), "{name}: {shown}"); + let info = s.run("raster.info", &json!({"layer": name})).unwrap(); + for b in 0..3 { + let band = &info["bands"][b]; + assert_eq!(band["validPixels"], json!(shown), "{name} band {b}: {band}"); + let (min, max) = (band["min"].as_f64().unwrap(), band["max"].as_f64().unwrap()); + assert!((0.0..=249.0).contains(&min) && (10.0..=249.0).contains(&max), "{name} band {b}: {band}"); + } + assert_eq!(info["bands"][3]["validPixels"], json!(r.bands[3].data.len()), "{name}"); + } + // The file's black row counts. + assert_eq!(s.run("raster.info", &json!({"layer": "ortho"})).unwrap()["bands"][0]["min"], json!(0.0)); + // The range now drawn, shared: within the opaque colours. + s.run("raster.stretch", &json!({"layer": "ortho", "like": "ortho"})).unwrap(); + let style = &s.map().layer_by_name("ortho").unwrap().raster.as_ref().unwrap().style; + let rustgis_map::raster::Stretch::Custom { bands: Some(bands), .. } = style.stretch else { panic!("{:?}", style.stretch) }; + assert!(bands.iter().all(|[lo, hi]| (0.0..=249.0).contains(lo) && (0.0..=249.0).contains(hi) && lo < hi), "{bands:?}"); + let _ = std::fs::remove_dir_all(dir); +} + +/// Change Detection leaves out a cell either raster draws see-through (#1810): the first's left +/// ten columns, holding −3.4e38, and the second's bottom ten rows, holding a stray 7, on the same +/// grid and on one moved half a cell (its alpha resampled as its band is). Every cell both show +/// compares, black ones included: 5 apart. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn change_detection_leaves_out_see_through_cells() { + use rustgis_data::raster::Raster; + let before = see_through_ortho("before", |i| i % 40 < 10, 0.0); + let mut after = see_through_ortho("after", |i| i >= 1200, 5.0); + for b in 0..3 { + for v in after.bands[b].data.iter_mut().skip(1200) { + *v = 7.0; + } + } + let shifted = { + let mut gt = after.transform; + gt.origin_x += 0.4; + let bands = after.bands.iter().map(|b| b.data.clone()).collect(); + Raster::new("after shifted", 40, 40, bands, gt, after.crs, None, after.color.clone(), "Float32").unwrap() + }; + let mut s = Session::new(Project::default()); + for r in [before, after, shifted] { + let name = r.name.clone(); + let bytes = rustgis_data::raster::geotiff::write(&r, 0).unwrap(); + crate::cmd::raster::add_raster(&mut s, &name, rustgis_map::RasterLayer::from_bytes(r, "x.tif", bytes)).unwrap(); + } + for after in ["after", "after shifted"] { + let out = format!("{after} change"); + s.run("raster.change", &json!({"before": "before", "after": after, "method": "difference", "name": out})).unwrap(); + let r = s.map().layer_by_name(&out).unwrap().raster.as_ref().unwrap().data.clone().unwrap(); + let kept: Vec = r.bands[0].data.iter().copied().filter(|v| r.is_valid(*v)).collect(); + // Opaque in both: columns 10 to 39 of rows 0 to 29. The shift of 0.4 m east keeps each + // cell's centre in the same column of the second. + assert_eq!(kept.len(), 30 * 30, "{after}"); + assert!(kept.iter().all(|v| *v == 5.0), "{after}: {:?}", kept.iter().find(|v| **v != 5.0)); + } +} + /// A layer read whole from a Shapefile or GeoJSON file remembers the files and its rows as read /// (#1385): `source.syncState` says when it has edits the file lacks, when the file changed /// since, and neither once an edit is undone. The record is saved with the project. diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index 356297d94e..bf8ab25474 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -1480,10 +1480,22 @@ impl RasterMask { } /// Statistics of `band`'s valid pixels that the mask leaves, from a sample of a large band. + /// A colour band's cells its alpha draws see-through are left out, as the band's own + /// statistics leave them out (#1810). pub fn stats(&self, r: &Raster, band: usize) -> BandStats { let Some(b) = r.bands.get(band) else { return BandStats::default() }; let step = b.data.len().div_ceil(MASK_STATS_SAMPLE).max(1); - let kept: Vec = b.data.iter().step_by(step).map(|v| if r.is_valid(*v) && !self.hides(*v) { *v } else { f32::NAN }).collect(); + let hidden = r.see_through_cells(band); + let kept: Vec = b + .data + .iter() + .enumerate() + .step_by(step) + .map(|(i, v)| { + let shown = r.is_valid(*v) && !self.hides(*v) && !hidden.as_ref().is_some_and(|h| h(i)); + if shown { *v } else { f32::NAN } + }) + .collect(); BandStats::from_values(&kept) } @@ -2490,6 +2502,35 @@ mod tests { } /// #348: values a mask lists, or outside (or inside) its range, are no data. + /// A style mask's statistics leave out the cells a colour band's alpha draws see-through, as + /// the band's own leave them out (#1810). + #[test] + fn a_masks_statistics_leave_out_see_through_cells() { + use rustgis_data::raster::{ColorHint, Crs, GeoTransform}; + let gt = GeoTransform { origin_x: 0.0, origin_y: 4.0, pixel_width: 1.0, pixel_height: -1.0 }; + // The bottom row see-through, holding −3.4e38; 200 elsewhere, and a black first cell. + let colour: Vec = (0..16) + .map(|i| { + if i >= 12 { + -3.4e38 + } else if i == 0 { + 0.0 + } else { + 200.0 + } + }) + .collect(); + let alpha: Vec = (0..16).map(|i| if i >= 12 { 0.0 } else { 1.0 }).collect(); + let bands = vec![colour.clone(), colour.clone(), colour, alpha]; + let r = Raster::new("rgba", 4, 4, bands, gt, Crs::LonLat, None, ColorHint::Rgb { alpha: true }, "Float32").unwrap(); + let mask = RasterMask { values: vec![200.0], ..RasterMask::default() }; + // 200 hidden by the mask, the bottom row by the alpha: the black cell alone. + let st = mask.stats(&r, 0); + assert_eq!((st.count, st.min, st.max), (1, 0.0, 0.0)); + // The alpha band itself: every cell the mask leaves. + assert_eq!(mask.stats(&r, 3).count, 16); + } + #[test] fn masks_hide_values_and_their_statistics() { let r = dem(); diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index 229460081d..c09ea17db6 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -784,22 +784,41 @@ fn with_stretch(info: Arc, file: String, look: Look, then: StretchThen) { w(s.clone()); } }; + // The colour bands, and the alpha band where the file has one: a cell it hides is left out, + // as the map leaves it unpainted (#1810), so a value a see-through cell holds (−3.4e38 in a + // Float32 image) doesn't set the stretch. + let read = info.shown_from(colour.clone()); + let alpha = info.alpha.and_then(|a| read.iter().rposition(|b| *b == a)); + let opaque = rustgis_data::raster::alpha_max(info.layout.data_type); fetch_bands( info, file, coarsest, tiles, - colour.clone(), + read, Box::new(move |r| match r { Ok(picked) => { let s: Vec<(f32, f32)> = (0..colour.len()) .map(|j| { + // The alpha band shown as a colour isn't hidden by itself. + let alpha = alpha.filter(|a| *a != j); percentiles( picked .iter() - .filter_map(|t| t.get(j)) - .filter_map(|(chunk, b)| chunk.get(*b)) - .flat_map(|v| v.iter().map(|x| shown_value(*x, look.aef))) + .filter_map(|t| { + let band = t.get(j).and_then(|(chunk, b)| chunk.get(*b))?; + Some((band, alpha.and_then(|a| t.get(a)).and_then(|(chunk, b)| chunk.get(*b)))) + }) + .flat_map(|(band, hides)| { + band.iter() + .enumerate() + .filter(move |(i, _)| { + hides.is_none_or(|al| { + al.get(*i).is_some_and(|&a| !rustgis_data::raster::see_through(a, !a.is_nan(), opaque)) + }) + }) + .map(|(_, x)| shown_value(*x, look.aef)) + }) .collect(), ) }) @@ -1750,6 +1769,65 @@ mod tests { assert!(info.stretch_for(&own).is_none(), "kept, which a look of its bands would take for its own"); } + /// A colour image's stretch leaves out the cells its alpha band hides (#1810): an RGBA + /// Float32 COG whose see-through half holds −3.4e38, as an export from before #1759 wrote it, + /// is stretched over its opaque half, black cells and all. + #[test] + fn a_stretch_leaves_out_what_the_alpha_band_hides() { + use rustgis_data::raster::{ColorHint, Crs, GeoTransform, Raster}; + let _turn = crate::cache::turn(); + let _alarm = crate::cache::alarm_turn(); + let (w, h) = (512usize, 512usize); + let n = w * h; + // The top half opaque, 10 to 209 and black in its first column; the bottom see-through. + let colour: Vec = (0..n) + .map(|i| { + if i >= n / 2 { + -3.4e38 + } else if i % w == 0 { + 0.0 + } else { + (10 + i % 200) as f32 + } + }) + .collect(); + let alpha: Vec = (0..n).map(|i| if i >= n / 2 { 0.0 } else { 1.0 }).collect(); + let gt = GeoTransform { origin_x: 500_000.0, origin_y: 4_800_000.0, pixel_width: 10.0, pixel_height: -10.0 }; + let utm = Crs::from_epsg(32611).unwrap(); + let bands = vec![colour.clone(), colour.clone(), colour, alpha]; + let r = Raster::new("rgba", w as u32, h as u32, bands, gt, utm, None, ColorHint::Rgb { alpha: true }, "Float32").unwrap(); + let file = Arc::new(rustgis_data::raster::cogwrite::write(&r, 0).unwrap()); + let held = Arc::new(Mutex::new(Vec::<(String, Done)>::new())); + let hold = held.clone(); + crate::cache::set_fetcher(move |url, done| hold.lock().unwrap().push((url, done))); + let drain = || { + for _ in 0..100 { + let pending = std::mem::take(&mut *held.lock().unwrap()); + if pending.is_empty() { + return; + } + for (url, done) in pending { + let (a, b) = crate::cache::split_range(&url).1.unwrap(); + done(Ok(file[a as usize..((b + 1) as usize).min(file.len())].to_vec())); + } + } + panic!("bounded reads"); + }; + let url = "https://see-through.invalid/rgba.tif".to_string(); + let (tx, rx) = std::sync::mpsc::channel(); + with_info(&url, Box::new(move |r| tx.send(r).unwrap())); + drain(); + let info = rx.recv().unwrap().unwrap(); + assert_eq!(info.alpha, Some(3)); + let (tx, rx) = std::sync::mpsc::channel(); + with_stretch(info, url.clone(), Look::default(), Box::new(move |r| tx.send(r).unwrap())); + drain(); + let stretch = rx.recv().unwrap().unwrap(); + assert_eq!(stretch.len(), 3); + assert!(stretch.iter().all(|(lo, hi)| (0.0..=209.0).contains(lo) && (0.0..=209.0).contains(hi) && lo < hi), "{stretch:?}"); + forget_file(&url); + } + /// A joint read whose tiles the fetch queue keeps dropping ends, saying so, rather than /// asking without end: within eight drops a tile (64 at least). #[test] diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 7280b4f6f2..8305117305 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -1198,6 +1198,15 @@ With a raster layer selected in Layers, the **Style** and **Data** tabs show its | Style › Enhancement | Brightness, contrast, gamma | | Data | Raster information (size, bands, statistics, coordinate system) and Export Raster (GeoTIFF) | +**See-through cells** of an image with an alpha band (RGBA, or grey and alpha) are left out of +its colour bands' statistics. These are the cells the map draws fully see-through: alpha 0, or +under half a 256th of opaque in a 16-bit image. Leaving them out changes Raster information's +counts and ranges, and the stretches taken from them: Minimum Maximum, Percent Clip, Standard +Deviation, Histogram Equalize, a style mask's, and a cloud-optimised GeoTIFF's own. So a value +those cells hold, such as −3.4e38 in a Float32 image, doesn't set the stretch, and an opaque +black cell still counts. The alpha band itself counts every cell. **Change Detection** gives +no-data where either image is see-through. + **Cast shadows**, on a Shaded Relief layer (`raster.symbology` `"shadow"`, 0 to 1), darkens the slopes the terrain hides from the sun as well as those facing away from it: a valley behind a ridge is in shadow, with a soft edge. Only the direct light is darkened: a slope in full shadow keeps the From 5eadd9160655f1cb0cb168bf8be57190197ed0e4 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 12 Nov 2026 13:41:50 -0700 Subject: [PATCH 2/5] docs: the see-through rule in every pixel type, not only 16-bit (#1810) --- docs/user-guide/data.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 05bae3c356..2be1fe806c 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -1202,13 +1202,14 @@ With a raster layer selected in Layers, the **Style** and **Data** tabs show its | Data | Raster information (size, bands, statistics, coordinate system) and Export Raster (GeoTIFF) | **See-through cells** of an image with an alpha band (RGBA, or grey and alpha) are left out of -its colour bands' statistics. These are the cells the map draws fully see-through: alpha 0, or -under half a 256th of opaque in a 16-bit image. Leaving them out changes Raster information's -counts and ranges, and the stretches taken from them: Minimum Maximum, Percent Clip, Standard -Deviation, Histogram Equalize, a style mask's, and a cloud-optimised GeoTIFF's own. So a value -those cells hold, such as −3.4e38 in a Float32 image, doesn't set the stretch, and an opaque -black cell still counts. The alpha band itself counts every cell. **Change Detection** gives -no-data where either image is see-through. +its colour bands' statistics. These are the cells the map draws fully see-through: an alpha of no +value, or one under half a 256th of opaque. That is alpha 0 in an 8-bit image, 128 or less of +65535 in a 16-bit one, and under about 0.002 of 1 in a Float32 one. Leaving them out changes +Raster information's counts and ranges, and the stretches taken from them: Minimum Maximum, +Percent Clip, Standard Deviation, Histogram Equalize, a style mask's, and a cloud-optimised +GeoTIFF's own. So a value those cells hold, such as −3.4e38 in a Float32 image, doesn't set the +stretch, and an opaque black cell still counts. The alpha band itself counts every cell. +**Change Detection** gives no-data where either image is see-through. **Cast shadows**, on a Shaded Relief layer (`raster.symbology` `"shadow"`, 0 to 1), darkens the slopes the terrain hides from the sun as well as those facing away from it: a valley behind a ridge From e6e518f893785614770aaef3710d3dbac820a2b2 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 12 Nov 2026 14:38:42 -0700 Subject: [PATCH 3/5] test(engine): Change Detection on a second raster moved by whole cells; review points (#1810) - The Change Detection test adds a second raster moved five cells east, see-through in its own right ten columns, so its alpha is read from other source columns than the first's: resampled, the change keeps columns 10 to 34 (1000 cells); read at the first's cells, it would keep 10 to 29 and compare see-through 7s. - fits_bytes scans a colour band's see-through cells only once its statistics fit (and never for an 8-bit band). - The data guide says the alpha band counts every alpha with a value. - The #348 doc comment is back on its test. --- crates/data/src/raster/geotiff.rs | 11 +++++---- crates/engine/src/tests.rs | 39 ++++++++++++++++++++++++++++--- crates/map/src/raster.rs | 2 +- docs/user-guide/data.md | 3 ++- 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index f76eebefd1..41913b4c0a 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -1628,10 +1628,13 @@ pub(super) fn fits_bytes(r: &Raster, b: usize) -> bool { let whole_bytes = s.min >= 0.0 && s.max <= 255.0 && (s.categories.is_some() || r.data_type == "UInt8"); let nodata_ok = r.nodata.is_none_or(|d| (0.0..=255.0).contains(&d) && d.fract() == 0.0); let byte = |v: f32| !r.is_valid(v) || (v.fract() == 0.0 && (0.0..=255.0).contains(&v)); - // An 8-bit band's are bytes already. - let see_through_bytes = r.data_type == "UInt8" - || r.see_through_cells(b).is_none_or(|hidden| band.data.iter().enumerate().all(|(i, &v)| !hidden(i) || byte(v))); - nodata_ok && (whole_bytes || s.count == 0) && see_through_bytes + // Looked at only once the statistics fit, as it reads the whole band; an 8-bit band's are + // bytes already. + let see_through_bytes = || { + r.data_type == "UInt8" + || r.see_through_cells(b).is_none_or(|hidden| band.data.iter().enumerate().all(|(i, &v)| !hidden(i) || byte(v))) + }; + nodata_ok && (whole_bytes || s.count == 0) && see_through_bytes() } /// The bands [`write`] writes of `r` given `band`: RGB and its fourth sample (alpha only when diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index ce3d431559..2adfd16670 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -43355,13 +43355,34 @@ fn see_through_cells_are_left_out_of_an_rgba_rasters_statistics() { /// Change Detection leaves out a cell either raster draws see-through (#1810): the first's left /// ten columns, holding −3.4e38, and the second's bottom ten rows, holding a stray 7, on the same -/// grid and on one moved half a cell (its alpha resampled as its band is). Every cell both show -/// compares, black ones included: 5 apart. +/// grid and on one moved 0.4 m. Every cell both show compares, black ones included: 5 apart. A +/// second moved five cells east, see-through in its own right ten columns, has its alpha +/// resampled as its band is: read at the first's cells instead, it would hide other columns. #[cfg(not(target_arch = "wasm32"))] #[test] fn change_detection_leaves_out_see_through_cells() { use rustgis_data::raster::Raster; let before = see_through_ortho("before", |i| i % 40 < 10, 0.0); + // Five cells east: its column k is the first's column k + 5, with the first's colours plus 5 + // (its first five columns cover the first's see-through ones), see-through from its column 30. + let moved = { + let see = |i: usize| i % 40 >= 30; + let bands = (0..4) + .map(|b| { + (0..1600) + .map(|i| match (b, see(i)) { + (3, true) => 0.0, + (3, false) => 255.0, + (_, true) => 7.0, + (_, false) => before.bands[b].data[i + 5] + 5.0, + }) + .collect() + }) + .collect(); + let mut gt = before.transform; + gt.origin_x += 5.0; + Raster::new("after moved", 40, 40, bands, gt, before.crs, None, before.color.clone(), "Float32").unwrap() + }; let mut after = see_through_ortho("after", |i| i >= 1200, 5.0); for b in 0..3 { for v in after.bands[b].data.iter_mut().skip(1200) { @@ -43375,7 +43396,7 @@ fn change_detection_leaves_out_see_through_cells() { Raster::new("after shifted", 40, 40, bands, gt, after.crs, None, after.color.clone(), "Float32").unwrap() }; let mut s = Session::new(Project::default()); - for r in [before, after, shifted] { + for r in [before, after, shifted, moved] { let name = r.name.clone(); let bytes = rustgis_data::raster::geotiff::write(&r, 0).unwrap(); crate::cmd::raster::add_raster(&mut s, &name, rustgis_map::RasterLayer::from_bytes(r, "x.tif", bytes)).unwrap(); @@ -43390,6 +43411,18 @@ fn change_detection_leaves_out_see_through_cells() { assert_eq!(kept.len(), 30 * 30, "{after}"); assert!(kept.iter().all(|v| *v == 5.0), "{after}: {:?}", kept.iter().find(|v| **v != 5.0)); } + // Moved: the first's columns 0 to 4 are off the second, 5 to 9 see-through in the first, and + // 35 to 39 the second's see-through 30 to 34; 10 to 34 of every row compare, 5 apart. Its alpha + // read at the first's cells would hide 30 to 39 instead (800 cells, and 7s compared). + s.run("raster.change", &json!({"before": "before", "after": "after moved", "method": "difference", "name": "moved change"})).unwrap(); + let r = s.map().layer_by_name("moved change").unwrap().raster.as_ref().unwrap().data.clone().unwrap(); + let kept: Vec = (0..1600).filter(|i| r.is_valid(r.bands[0].data[*i])).collect(); + assert_eq!(kept.len(), 25 * 40); + assert!( + kept.iter().all(|i| (10..35).contains(&(i % 40)) && r.bands[0].data[*i] == 5.0), + "{:?}", + kept.iter().find(|i| r.bands[0].data[**i] != 5.0) + ); } /// A layer read whole from a Shapefile or GeoJSON file remembers the files and its rows as read diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index 827626d7cd..d117ef39c8 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -2509,7 +2509,6 @@ mod tests { assert!(Colorizer::new(&k, &r, None).color(&[5.0], None).is_some()); } - /// #348: values a mask lists, or outside (or inside) its range, are no data. /// A style mask's statistics leave out the cells a colour band's alpha draws see-through, as /// the band's own leave them out (#1810). #[test] @@ -2539,6 +2538,7 @@ mod tests { assert_eq!(mask.stats(&r, 3).count, 16); } + /// #348: values a mask lists, or outside (or inside) its range, are no data. #[test] fn masks_hide_values_and_their_statistics() { let r = dem(); diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 2be1fe806c..2c3b6b9f4c 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -1208,7 +1208,8 @@ value, or one under half a 256th of opaque. That is alpha 0 in an 8-bit image, 1 Raster information's counts and ranges, and the stretches taken from them: Minimum Maximum, Percent Clip, Standard Deviation, Histogram Equalize, a style mask's, and a cloud-optimised GeoTIFF's own. So a value those cells hold, such as −3.4e38 in a Float32 image, doesn't set the -stretch, and an opaque black cell still counts. The alpha band itself counts every cell. +stretch, and an opaque black cell still counts. The alpha band's own statistics count every +alpha with a value, see-through ones included. **Change Detection** gives no-data where either image is see-through. **Cast shadows**, on a Shaded Relief layer (`raster.symbology` `"shadow"`, 0 to 1), darkens the From ef68f882c8af006d6db85a5b101576375649c644 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 12 Nov 2026 17:57:04 -0700 Subject: [PATCH 4/5] test(engine): the see-through rows are set through an iterator, as clippy 1.100's needless_range_loop asks (#1810) --- crates/engine/src/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 2adfd16670..75689a7e6c 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -43384,8 +43384,8 @@ fn change_detection_leaves_out_see_through_cells() { Raster::new("after moved", 40, 40, bands, gt, before.crs, None, before.color.clone(), "Float32").unwrap() }; let mut after = see_through_ortho("after", |i| i >= 1200, 5.0); - for b in 0..3 { - for v in after.bands[b].data.iter_mut().skip(1200) { + for band in after.bands.iter_mut().take(3) { + for v in band.data.iter_mut().skip(1200) { *v = 7.0; } } From d7a6890685bef1b2cdc40804971f19769dd586b6 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 12 Nov 2026 18:44:22 -0700 Subject: [PATCH 5/5] test(data): statistics with a validity by place are the same on any number of threads (#1810) --- crates/data/src/raster/mod.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/data/src/raster/mod.rs b/crates/data/src/raster/mod.rs index c37f6059b1..17fc0d9298 100644 --- a/crates/data/src/raster/mod.rs +++ b/crates/data/src/raster/mod.rs @@ -1626,6 +1626,21 @@ pub(crate) mod tests { assert_eq!(got.categories, want.categories, "{name} on {threads}"); } } + // A validity that depends on the cell's place, as a see-through cell's does (#1810): each + // run is given where it starts, so several threads leave out the same cells as one. A + // cell's place is checked against its band too: every third one is left out, and so is + // the band's last cell, which a run off by one would miss or count twice. + for (name, data) in &bands { + let by_place = |i: usize, v: f32| value(v) && !i.is_multiple_of(3) && i + 1 != data.len(); + let kept: Vec = data.iter().enumerate().map(|(i, &v)| if by_place(i, v) { v } else { f32::NAN }).collect(); + let want = one_pass(&kept, value); + for threads in [1, 2, 7, 16] { + let got = BandStats::compute_on(data, by_place, None, threads); + assert_eq!(bits(&got), bits(&want), "{name} by place on {threads}"); + assert_eq!(got.histogram, want.histogram, "{name} by place on {threads}"); + assert_eq!(got.categories, want.categories, "{name} by place on {threads}"); + } + } assert!(BandStats::compute_on(&bands[1].1, valid, None, 2).categories.is_none(), "257 small whole values"); assert!(BandStats::compute_on(&bands[3].1, valid, None, 2).categories.is_none(), "400 in two runs of 200"); // A band too small for threads, and one with no valid value.