diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index 81fb2bffa8..41913b4c0a 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -1619,13 +1619,22 @@ fn as_area(keys: &[u16]) -> Vec { } /// Whether band `b` is written as bytes: whole numbers within 0–255 (or no valid pixels at all), -/// and a no-data value that is one too. +/// and a no-data value that is one too. A colour band's statistics leave out the cells its alpha +/// draws see-through (#1810), so those cells' values are looked at too: a file is written in the +/// type it was before, whatever they hold. pub(super) fn fits_bytes(r: &Raster, b: usize) -> bool { let Some(band) = r.bands.get(b) else { return false }; let s = &band.stats; let whole_bytes = s.min >= 0.0 && s.max <= 255.0 && (s.categories.is_some() || r.data_type == "UInt8"); let nodata_ok = r.nodata.is_none_or(|d| (0.0..=255.0).contains(&d) && d.fract() == 0.0); - nodata_ok && (whole_bytes || s.count == 0) + let byte = |v: f32| !r.is_valid(v) || (v.fract() == 0.0 && (0.0..=255.0).contains(&v)); + // Looked at only once the statistics fit, as it reads the whole band; an 8-bit band's are + // bytes already. + let see_through_bytes = || { + r.data_type == "UInt8" + || r.see_through_cells(b).is_none_or(|hidden| band.data.iter().enumerate().all(|(i, &v)| !hidden(i) || byte(v))) + }; + nodata_ok && (whole_bytes || s.count == 0) && see_through_bytes() } /// The bands [`write`] writes of `r` given `band`: RGB and its fourth sample (alpha only when diff --git a/crates/data/src/raster/mod.rs b/crates/data/src/raster/mod.rs index e240e3fd5e..17fc0d9298 100644 --- a/crates/data/src/raster/mod.rs +++ b/crates/data/src/raster/mod.rs @@ -165,16 +165,33 @@ pub struct BandStats { impl BandStats { /// Statistics of the finite values in `data`. pub fn from_values(data: &[f32]) -> BandStats { - BandStats::compute(data, f32::is_finite) - } - - fn compute(data: &[f32], valid: impl Fn(f32) -> bool + Sync) -> BandStats { - BandStats::compute_with(data, valid, None) + BandStats::compute_with(data, |_, v| v.is_finite(), None) + } + + /// Statistics of a raster's band `data`: the values `valid` takes, less, for a colour band of + /// a raster with alpha, the cells its alpha band (`alpha`) draws see-through ([`see_through`], + /// an alpha of `opaque` being opaque). `stored` is the band's whole statistics, for a reduced + /// overview ([`BandStats::compute_with`]); GDAL counts see-through cells in all but 8-bit + /// images, so a band with `alpha` doesn't take them. + fn of_band( + data: &[f32], + valid: impl Fn(f32) -> bool + Sync, + alpha: Option<&[f32]>, + opaque: f32, + stored: Option, + ) -> BandStats { + match alpha { + Some(alpha) => { + BandStats::compute_with(data, |i, v| valid(v) && alpha.get(i).is_some_and(|&a| !see_through(a, valid(a), opaque)), None) + } + None => BandStats::compute_with(data, |_, v| valid(v), stored), + } } - /// [`BandStats::compute`], taking the range, mean and spread from `stored` (statistics of the - /// whole band, when `data` is a reduced overview of it). - fn compute_with(data: &[f32], valid: impl Fn(f32) -> bool + Sync, stored: Option) -> BandStats { + /// Statistics of the values of `data` that `valid` takes (given each one's place in `data` and + /// its value), taking the range, mean and spread from `stored` (statistics of the whole band, + /// when `data` is a reduced overview of it). + fn compute_with(data: &[f32], valid: impl Fn(usize, f32) -> bool + Sync, stored: Option) -> BandStats { BandStats::compute_on(data, valid, stored, threads_for(data.len())) } @@ -184,10 +201,14 @@ impl BandStats { /// the same to the bit; what doesn't depend on the order is counted on the others /// ([`on_runs`]), the range while this thread sums. The range keeps the first of equal values /// (−0 or 0) in the band's order either way. - fn compute_on(data: &[f32], valid: impl Fn(f32) -> bool + Sync, stored: Option, threads: usize) -> BandStats { - let ranged = |run: &[f32], sum: bool| { + fn compute_on(data: &[f32], valid: impl Fn(usize, f32) -> bool + Sync, stored: Option, threads: usize) -> BandStats { + // A run of `data` from `start`. + let ranged = |start: usize, run: &[f32], sum: bool| { let (mut n, mut min, mut max, mut whole, mut sums) = (0usize, f64::INFINITY, f64::NEG_INFINITY, true, (0.0f64, 0.0f64)); - for &v in run.iter().filter(|&&v| valid(v)) { + for (j, &v) in run.iter().enumerate() { + if !valid(start + j, v) { + continue; + } let x = v as f64; n += 1; (min, max) = (lower(min, x), higher(max, x)); @@ -199,10 +220,10 @@ impl BandStats { (n, min, max, whole, sums) }; let (n, mut min, mut max, whole, (sum, sum2)) = if threads <= 1 { - ranged(data, true) + ranged(0, data, true) } else { - let sums = || ranged(data, true).4; - let (runs, sums) = on_runs_beside(data, threads, sums, |run| ranged(run, false)); + let sums = || ranged(0, data, true).4; + let (runs, sums) = on_runs_beside(data, threads, sums, |start, run| ranged(start, run, false)); let mut all = (0usize, f64::INFINITY, f64::NEG_INFINITY, true, sums); for (rn, rmin, rmax, rwhole, _) in runs { all = (all.0 + rn, lower(all.1, rmin), higher(all.2, rmax), all.3 && rwhole, all.4); @@ -225,8 +246,8 @@ impl BandStats { let small_ints = whole && min >= 0.0 && max <= 65_535.0; let k = (HISTOGRAM_BINS as f64 - 1.0) / if span > 0.0 { span } else { 1.0 }; let b = Binning { min, k, small_ints, whole }; - let mut runs = on_runs(data, threads, |run| Tally::run(run, &valid, b)).into_iter(); - let mut tally = runs.next().unwrap_or_else(|| Tally::run(&[], &valid, b)); + let mut runs = on_runs(data, threads, |start, run| Tally::run(start, run, &valid, b)).into_iter(); + let mut tally = runs.next().unwrap_or_else(|| Tally::run(0, &[], &valid, b)); for t in runs { tally.histogram.iter_mut().zip(&t.histogram).for_each(|(a, b)| *a += b); tally.bits.iter_mut().zip(&t.bits).for_each(|(a, b)| *a |= b); @@ -310,32 +331,40 @@ fn threads_for(len: usize) -> usize { if cfg!(target_arch = "wasm32") || len < PARALLEL_VALUES { 1 } else { crate::threads::max().min(len / (PARALLEL_VALUES / 4)) } } -/// `f` over `threads` runs of `data` that cover it in order, its results in that order, the -/// calling thread taking the first. A worker's panic is raised again here; a run whose thread -/// can't start is made on this one. -fn on_runs(data: &[f32], threads: usize, f: impl Fn(&[f32]) -> T + Sync) -> Vec { +/// `f` over `threads` runs of `data` that cover it in order, each given with where it starts in +/// `data`, its results in that order, the calling thread taking the first. A worker's panic is +/// raised again here; a run whose thread can't start is made on this one. +fn on_runs(data: &[f32], threads: usize, f: impl Fn(usize, &[f32]) -> T + Sync) -> Vec { on_runs_beside(data, threads, || (), f).0 } /// [`on_runs`], the calling thread running `beside` first, while the other threads start on /// their runs. -fn on_runs_beside(data: &[f32], threads: usize, beside: impl FnOnce() -> U, f: impl Fn(&[f32]) -> T + Sync) -> (Vec, U) { +fn on_runs_beside( + data: &[f32], + threads: usize, + beside: impl FnOnce() -> U, + f: impl Fn(usize, &[f32]) -> T + Sync, +) -> (Vec, U) { if threads <= 1 || data.is_empty() { let beside = beside(); - return (vec![f(data)], beside); + return (vec![f(0, data)], beside); } - let mut runs = data.chunks(data.len().div_ceil(threads)); - let first = runs.next().unwrap_or_default(); + let size = data.len().div_ceil(threads); + let mut runs = data.chunks(size).enumerate().map(|(k, run)| (k * size, run)); + let (start, first) = runs.next().unwrap_or_default(); let f = &f; std::thread::scope(|scope| { - let started: Vec<_> = runs.map(|run| std::thread::Builder::new().spawn_scoped(scope, move || f(run)).map_err(|_| run)).collect(); + let started: Vec<_> = runs + .map(|(start, run)| std::thread::Builder::new().spawn_scoped(scope, move || f(start, run)).map_err(|_| (start, run))) + .collect(); let beside = beside(); - let mut out = vec![f(first)]; + let mut out = vec![f(start, first)]; for run in started { out.push(match run { Ok(h) => h.join().unwrap_or_else(|p| std::panic::resume_unwind(p)), // The OS refused a thread: made here. - Err(run) => f(run), + Err((start, run)) => f(start, run), }); } (out, beside) @@ -343,14 +372,14 @@ fn on_runs_beside(data: &[f32], threads: usize, beside: impl FnOnce( } impl Tally { - /// `data`'s valid values counted on this thread. - fn run(data: &[f32], valid: &impl Fn(f32) -> bool, b: Binning) -> Tally { + /// `data`'s valid values counted on this thread: a run of a band from `start`. + fn run(start: usize, data: &[f32], valid: &impl Fn(usize, f32) -> bool, b: Binning) -> Tally { let mut histogram = vec![0u64; HISTOGRAM_BINS]; let mut bits = if b.small_ints { vec![0u64; 1024] } else { Vec::new() }; let mut distinct = std::collections::BTreeSet::new(); let mut track = b.whole; - for &v in data { - if !valid(v) { + for (j, &v) in data.iter().enumerate() { + if !valid(start + j, v) { continue; } let x = v as f64; @@ -538,6 +567,13 @@ pub fn alpha_max(data_type: &str) -> f32 { } } +/// Whether an alpha value draws its cell fully see-through, as the map draws it: an alpha of no +/// value (`valid` false: NaN, or the raster's no-data), or one under half a 256th of `opaque` (the +/// pixel type's [`alpha_max`]), which is nothing in 8 bits. +pub fn see_through(a: f32, valid: bool, opaque: f32) -> bool { + !valid || ((a / opaque).clamp(0.0, 1.0) * 255.0).round() == 0.0 +} + /// How a raster's bands were meant to be shown, from the file. #[derive(Clone, Debug, Default, PartialEq)] pub enum ColorHint { @@ -551,6 +587,19 @@ pub enum ColorHint { Palette(Vec<[u8; 4]>), } +impl ColorHint { + /// For a raster of `bands` bands shown this way with alpha, its alpha band and the colour + /// bands that band makes see-through: an RGBA image's fourth band over its first three, grey + /// and alpha's second over its first. + pub fn alpha(&self, bands: usize) -> Option<(usize, std::ops::Range)> { + match self { + ColorHint::Rgb { alpha: true } if bands >= 4 => Some((3, 0..3)), + ColorHint::GrayAlpha if bands >= 2 => Some((1, 0..1)), + _ => None, + } + } +} + /// A raster on a grid: georeferenced, unless `georeferenced` is false (a scan, whose transform /// is only a stand-in). #[derive(Clone)] @@ -738,16 +787,21 @@ impl Raster { let transformer = crs.transformer().map_err(DataError::Format)?; let nd = nodata.filter(|v| v.is_finite()).map(|v| v as f32); let valid = move |v: f32| v.is_finite() && nd.is_none_or(|d| v != d); + let data_type = data_type.into(); + let (alpha, opaque) = (color.alpha(bands.len()), alpha_max(&data_type)); + let stats: Vec = bands + .iter() + .enumerate() + .map(|(i, data)| { + let alpha = alpha.as_ref().filter(|(_, colour)| colour.contains(&i)).and_then(|(a, _)| bands.get(*a)); + if measure { BandStats::of_band(data, valid, alpha.map(Vec::as_slice), opaque, None) } else { BandStats::default() } + }) + .collect(); let bands = bands .into_iter() + .zip(stats) .enumerate() - .map(|(i, data)| Band { - name: format!("Band {}", i + 1), - stats: if measure { BandStats::compute(&data, valid) } else { BandStats::default() }, - data, - classes: Vec::new(), - wavelength: None, - }) + .map(|(i, (data, stats))| Band { name: format!("Band {}", i + 1), stats, data, classes: Vec::new(), wavelength: None }) .collect(); let mut r = Raster { name: name.into(), @@ -758,7 +812,7 @@ impl Raster { crs, nodata: nodata.filter(|v| v.is_finite()), color, - data_type: data_type.into(), + data_type, source: None, overview: None, georeferenced: true, @@ -839,6 +893,16 @@ impl Raster { let nd = self.nodata.map(|v| v as f32); let valid = move |v: f32| v.is_finite() && nd.is_none_or(|d| v != d); let overview = self.overview.is_some(); + // Worked out before the bands change, as a colour band's take its alpha band's values. + let (alpha, opaque) = (self.color.alpha(self.bands.len()), alpha_max(&self.data_type)); + let mut measured: Vec> = (0..self.bands.len()) + .map(|i| { + let stored = pam.band(i + 1).and_then(|p| p.stats).filter(|_| overview); + let data = self.bands.get(i).filter(|_| recompute || stored.is_some())?; + let alpha = alpha.as_ref().filter(|(_, colour)| colour.contains(&i)).and_then(|(a, _)| self.bands.get(*a)); + Some(BandStats::of_band(&data.data, valid, alpha.map(|a| a.data.as_slice()), opaque, stored)) + }) + .collect(); for (i, band) in self.bands.iter_mut().enumerate() { let p = pam.band(i + 1); if let Some(d) = p.and_then(|p| p.description.as_ref()) { @@ -850,9 +914,8 @@ impl Raster { if let Some(w) = p.and_then(|p| p.wavelength) { band.wavelength = Some(w); } - let stored = p.and_then(|p| p.stats).filter(|_| overview); - if recompute || stored.is_some() { - band.stats = BandStats::compute_with(&band.data, valid, stored); + if let Some(stats) = measured.get_mut(i).and_then(Option::take) { + band.stats = stats; } } } @@ -1013,6 +1076,17 @@ impl Raster { self.rounded_some = if marks.contains(&true) { marks } else { Vec::new() }; } + /// The cells of `band` drawn see-through, by their place (row by row): `band` is a colour + /// band of a raster with alpha ([`ColorHint::alpha`]) and the cell's alpha draws it so + /// ([`see_through`]). Its statistics leave them out. `None` when the band has no such cells + /// to have: no alpha, or the alpha band or a band of data beside the colours. + pub fn see_through_cells(&self, band: usize) -> Option bool + Sync + '_> { + let (a, colour) = self.color.alpha(self.bands.len())?; + let alpha = &self.bands.get(a).filter(|_| colour.contains(&band))?.data; + let opaque = alpha_max(&self.data_type); + Some(move |i: usize| alpha.get(i).is_none_or(|&v| see_through(v, self.is_valid(v), opaque))) + } + /// Whether a value is a real measurement (not no-data or NaN). pub fn is_valid(&self, v: f32) -> bool { // Compared in f32, as the pixels (and the statistics) are: a no-data value such as @@ -1541,9 +1615,10 @@ pub(crate) mod tests { ("gaps", (0..n).map(|i| [f32::NAN, -0.0, 0.0, -9999.0, 17.5][at(i) % 5]).collect()), ]; let bits = |s: &BandStats| (s.count, s.min.to_bits(), s.max.to_bits(), s.mean.to_bits(), s.std_dev.to_bits()); - let valid = |v: f32| v.is_finite() && v != -9999.0; + let value = |v: f32| v.is_finite() && v != -9999.0; + let valid = |_: usize, v: f32| value(v); for (name, data) in &bands { - let want = one_pass(data, valid); + let want = one_pass(data, value); for threads in [1, 2, 7, 16] { let got = BandStats::compute_on(data, valid, None, threads); assert_eq!(bits(&got), bits(&want), "{name} on {threads}"); @@ -1551,6 +1626,21 @@ pub(crate) mod tests { assert_eq!(got.categories, want.categories, "{name} on {threads}"); } } + // A validity that depends on the cell's place, as a see-through cell's does (#1810): each + // run is given where it starts, so several threads leave out the same cells as one. A + // cell's place is checked against its band too: every third one is left out, and so is + // the band's last cell, which a run off by one would miss or count twice. + for (name, data) in &bands { + let by_place = |i: usize, v: f32| value(v) && !i.is_multiple_of(3) && i + 1 != data.len(); + let kept: Vec = data.iter().enumerate().map(|(i, &v)| if by_place(i, v) { v } else { f32::NAN }).collect(); + let want = one_pass(&kept, value); + for threads in [1, 2, 7, 16] { + let got = BandStats::compute_on(data, by_place, None, threads); + assert_eq!(bits(&got), bits(&want), "{name} by place on {threads}"); + assert_eq!(got.histogram, want.histogram, "{name} by place on {threads}"); + assert_eq!(got.categories, want.categories, "{name} by place on {threads}"); + } + } assert!(BandStats::compute_on(&bands[1].1, valid, None, 2).categories.is_none(), "257 small whole values"); assert!(BandStats::compute_on(&bands[3].1, valid, None, 2).categories.is_none(), "400 in two runs of 200"); // A band too small for threads, and one with no valid value. @@ -1559,6 +1649,93 @@ pub(crate) mod tests { assert_eq!(BandStats::compute_on(&vec![f32::NAN; n], valid, None, 4), BandStats::default()); } + /// Cells a raster's alpha draws see-through are left out of its colour bands' statistics, + /// whatever their values (#1810): −3.4e38 in a Float32 image, a stray byte in an 8-bit one. An + /// opaque black cell counts, the alpha band counts every cell, and a raster without alpha (or + /// with a fourth band of data) counts every valid cell, as before. + #[test] + fn see_through_cells_are_left_out_of_colour_statistics() { + let gt = GeoTransform { origin_x: 500_000.0, origin_y: 4_800_000.0, pixel_width: 1.0, pixel_height: -1.0 }; + let utm = Crs::from_epsg(32611).unwrap(); + // 4 × 4: the bottom row see-through (holding `see`), the first cell opaque black, the rest 100. + let colour = |see: f32| { + (0..16) + .map(|i| { + if i >= 12 { + see + } else if i == 0 { + 0.0 + } else { + 100.0 + } + }) + .collect::>() + }; + let alpha = |opaque: f32| (0..16).map(|i| if i >= 12 { 0.0 } else { opaque }).collect::>(); + let stats = |b: &Band| (b.stats.count, b.stats.min, b.stats.max); + for (data_type, see, opaque) in [("Float32", -3.4e38_f32, 1.0), ("UInt8", 7.0, 255.0), ("UInt16", 7.0, 65535.0)] { + let bands = vec![colour(see), colour(see), colour(see), alpha(opaque)]; + let r = Raster::new("rgba", 4, 4, bands, gt, utm, None, ColorHint::Rgb { alpha: true }, data_type).unwrap(); + for b in &r.bands[..3] { + assert_eq!(stats(b), (12, 0.0, 100.0), "{data_type}"); + // Eleven cells of 100 and one of 0: the mean GDAL 3.9.3 gives an 8-bit RGBA band. + assert!((b.stats.mean - 1100.0 / 12.0).abs() < 1e-9, "{data_type}: {}", b.stats.mean); + assert_eq!(b.stats.histogram.iter().sum::(), 12, "{data_type}"); + } + assert_eq!(stats(&r.bands[3]), (16, 0.0, f64::from(opaque)), "{data_type}"); + // Read back from its GeoTIFF, the same. + let back = geotiff::read(std::io::Cursor::new(geotiff::write(&r, 0).unwrap()), "back").unwrap(); + assert_eq!(back.color, ColorHint::Rgb { alpha: true }, "{data_type}"); + // Written in the type it was before: a Float32 image whose see-through cells hold + // −3.4e38 isn't taken for bytes, as a GeoTIFF or a cloud-optimised one. + if data_type == "Float32" { + let cog = geotiff::read(std::io::Cursor::new(cogwrite::write(&r, 0).unwrap()), "cog").unwrap(); + assert_eq!((back.data_type.as_str(), cog.data_type.as_str()), ("Float32", "Float32")); + } + assert_eq!(back.bands.iter().map(stats).collect::>(), r.bands.iter().map(stats).collect::>(), "{data_type}"); + } + // As the map draws it: an alpha under half a 256th of opaque is see-through (16 bits: 100 + // of 65535), one above it isn't (200). + let faint: Vec = (0..16) + .map(|i| { + if i >= 12 { + 100.0 + } else if i >= 8 { + 200.0 + } else { + 65535.0 + } + }) + .collect(); + let r = Raster::new( + "faint", + 4, + 4, + vec![colour(7.0), colour(7.0), colour(7.0), faint], + gt, + utm, + None, + ColorHint::Rgb { alpha: true }, + "UInt16", + ) + .unwrap(); + assert_eq!(r.bands[0].stats.count, 12); + // Grey and alpha: the grey band. + let r = Raster::new("grey", 4, 4, vec![colour(7.0), alpha(255.0)], gt, utm, None, ColorHint::GrayAlpha, "UInt8").unwrap(); + assert_eq!((stats(&r.bands[0]), r.bands[1].stats.count), ((12, 0.0, 100.0), 16)); + // No-data 0: an alpha of no value is see-through, as the map draws it, and the black cell + // is no-data, as before. + let bands = vec![colour(-3.4e38), colour(-3.4e38), colour(-3.4e38), alpha(1.0)]; + let r = Raster::new("nodata", 4, 4, bands, gt, utm, Some(0.0), ColorHint::Rgb { alpha: true }, "Float32").unwrap(); + assert_eq!(stats(&r.bands[0]), (11, 100.0, 100.0)); + // Without alpha every valid cell counts: RGB, and RGB with a fourth band of data. + for nb in [3, 4] { + let bands = (0..nb).map(|_| colour(7.0)).collect(); + let r = Raster::new("rgb", 4, 4, bands, gt, utm, None, ColorHint::Rgb { alpha: false }, "UInt8").unwrap(); + assert!(r.bands.iter().all(|b| stats(b) == (16, 0.0, 100.0)), "{nb} bands"); + } + } + /// A band the machine can't hold is an error naming its size, not the end of the program /// (#1291), on 64-bit (the reservation fails) and wasm32 (the size overflows) alike; one it can /// is the size asked for. diff --git a/crates/engine/src/cmd/imagery.rs b/crates/engine/src/cmd/imagery.rs index 9626fbcbfe..2257633433 100644 --- a/crates/engine/src/cmd/imagery.rs +++ b/crates/engine/src/cmd/imagery.rs @@ -884,6 +884,19 @@ pub(crate) fn change_work(s: &Session, p: &Value) -> Result Some(rasterops::resample_to(&bfr, alpha, &a)?), + _ => None, + }; + let b_opaque = raster::alpha_max(&bfr.data_type); + let hidden = |i: usize| { + a_hidden.as_ref().is_some_and(|h| h(i)) + || b_alpha.as_ref().is_some_and(|al| al.get(i).is_none_or(|&v| raster::see_through(v, bfr.is_valid(v), b_opaque))) + }; let n = a.width as usize * a.height as usize; let out = rasterops::per_index( n, @@ -892,7 +905,7 @@ pub(crate) fn change_work(s: &Session, p: &Value) -> Result bool, offset: f32) -> rustgis_data::raster::Raster { + use rustgis_data::raster::{ColorHint, Crs, GeoTransform, Raster}; + let (w, n) = (40usize, 1600usize); + let colour = |b: usize| -> Vec { + (0..n).map(|i| if see(i) { -3.4e38 } else if i < w { 0.0 } else { (10 + (i * 7 + b * 60) % 240) as f32 } + offset).collect() + }; + let alpha = (0..n).map(|i| if see(i) { 0.0 } else { 255.0 }).collect(); + let gt = GeoTransform { origin_x: 500_000.0, origin_y: 4_800_040.0, pixel_width: 1.0, pixel_height: -1.0 }; + let utm = Crs::from_epsg(32611).unwrap(); + Raster::new(name, 40, 40, vec![colour(0), colour(1), colour(2), alpha], gt, utm, None, ColorHint::Rgb { alpha: true }, "Float32") + .unwrap() +} + +/// The cells an RGBA raster's alpha draws see-through are left out of its colour bands' +/// statistics (#1810): read from a file, projected (nearest) and clipped to a polygon. Its opaque +/// black cells count, its alpha band counts every cell, and the stretch shared from it (`like`) +/// spans its opaque colours, not −3.4e38. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn see_through_cells_are_left_out_of_an_rgba_rasters_statistics() { + use rustgis_data::raster::geotiff; + let dir = std::env::temp_dir().join(format!("rustgis-see-through-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("ortho.tif"); + std::fs::write(&path, geotiff::write(&see_through_ortho("ortho", |i| i % 40 < 10, 0.0), 0).unwrap()).unwrap(); + let mut s = Session::new(Project::default()); + s.run("layer.addData", &json!({"path": path, "name": "ortho"})).unwrap(); + let tr = rustgis_data::raster::Crs::from_epsg(32611).unwrap().transformer().unwrap(); + let ll = |x: f64, y: f64| { + let c = tr.to_lonlat(rustgis_geom::Coord::new(x, y)).unwrap(); + json!([c.x, c.y]) + }; + // A square over the middle, across the see-through columns. + s.run("layer.template", &json!({"template": "polygons", "name": "Boundary"})).unwrap(); + let ring = [ + ll(500_005.0, 4_800_035.0), + ll(500_035.0, 4_800_035.0), + ll(500_035.0, 4_800_005.0), + ll(500_005.0, 4_800_005.0), + ll(500_005.0, 4_800_035.0), + ]; + s.run("edit.addFeature", &json!({"layer": "Boundary", "geometry": {"type": "Polygon", "coordinates": [ring]}})).unwrap(); + s.run("raster.project", &json!({"layer": "ortho", "crs": 32611, "cellSize": 1.0, "resampling": "nearest", "name": "projected"})) + .unwrap(); + s.run( + "raster.process", + &json!({"layer": "ortho", "operation": "clip", "polygon": "Boundary", "selectedOnly": false, "name": "clipped"}), + ) + .unwrap(); + for name in ["ortho", "projected", "clipped"] { + let r = s.map().layer_by_name(name).unwrap().raster.as_ref().unwrap().data.clone().unwrap(); + // The cells its alpha shows: alpha above half a 256th of opaque (Float32's 1). + let shown = r.bands[3].data.iter().filter(|a| **a >= 0.5 / 255.0).count(); + assert!(shown > 0 && shown < r.bands[3].data.len(), "{name}: {shown}"); + let info = s.run("raster.info", &json!({"layer": name})).unwrap(); + for b in 0..3 { + let band = &info["bands"][b]; + assert_eq!(band["validPixels"], json!(shown), "{name} band {b}: {band}"); + let (min, max) = (band["min"].as_f64().unwrap(), band["max"].as_f64().unwrap()); + assert!((0.0..=249.0).contains(&min) && (10.0..=249.0).contains(&max), "{name} band {b}: {band}"); + } + assert_eq!(info["bands"][3]["validPixels"], json!(r.bands[3].data.len()), "{name}"); + } + // The file's black row counts. + assert_eq!(s.run("raster.info", &json!({"layer": "ortho"})).unwrap()["bands"][0]["min"], json!(0.0)); + // The range now drawn, shared: within the opaque colours. + s.run("raster.stretch", &json!({"layer": "ortho", "like": "ortho"})).unwrap(); + let style = &s.map().layer_by_name("ortho").unwrap().raster.as_ref().unwrap().style; + let rustgis_map::raster::Stretch::Custom { bands: Some(bands), .. } = style.stretch else { panic!("{:?}", style.stretch) }; + assert!(bands.iter().all(|[lo, hi]| (0.0..=249.0).contains(lo) && (0.0..=249.0).contains(hi) && lo < hi), "{bands:?}"); + let _ = std::fs::remove_dir_all(dir); +} + +/// Change Detection leaves out a cell either raster draws see-through (#1810): the first's left +/// ten columns, holding −3.4e38, and the second's bottom ten rows, holding a stray 7, on the same +/// grid and on one moved 0.4 m. Every cell both show compares, black ones included: 5 apart. A +/// second moved five cells east, see-through in its own right ten columns, has its alpha +/// resampled as its band is: read at the first's cells instead, it would hide other columns. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn change_detection_leaves_out_see_through_cells() { + use rustgis_data::raster::Raster; + let before = see_through_ortho("before", |i| i % 40 < 10, 0.0); + // Five cells east: its column k is the first's column k + 5, with the first's colours plus 5 + // (its first five columns cover the first's see-through ones), see-through from its column 30. + let moved = { + let see = |i: usize| i % 40 >= 30; + let bands = (0..4) + .map(|b| { + (0..1600) + .map(|i| match (b, see(i)) { + (3, true) => 0.0, + (3, false) => 255.0, + (_, true) => 7.0, + (_, false) => before.bands[b].data[i + 5] + 5.0, + }) + .collect() + }) + .collect(); + let mut gt = before.transform; + gt.origin_x += 5.0; + Raster::new("after moved", 40, 40, bands, gt, before.crs, None, before.color.clone(), "Float32").unwrap() + }; + let mut after = see_through_ortho("after", |i| i >= 1200, 5.0); + for band in after.bands.iter_mut().take(3) { + for v in band.data.iter_mut().skip(1200) { + *v = 7.0; + } + } + let shifted = { + let mut gt = after.transform; + gt.origin_x += 0.4; + let bands = after.bands.iter().map(|b| b.data.clone()).collect(); + Raster::new("after shifted", 40, 40, bands, gt, after.crs, None, after.color.clone(), "Float32").unwrap() + }; + let mut s = Session::new(Project::default()); + for r in [before, after, shifted, moved] { + let name = r.name.clone(); + let bytes = rustgis_data::raster::geotiff::write(&r, 0).unwrap(); + crate::cmd::raster::add_raster(&mut s, &name, rustgis_map::RasterLayer::from_bytes(r, "x.tif", bytes)).unwrap(); + } + for after in ["after", "after shifted"] { + let out = format!("{after} change"); + s.run("raster.change", &json!({"before": "before", "after": after, "method": "difference", "name": out})).unwrap(); + let r = s.map().layer_by_name(&out).unwrap().raster.as_ref().unwrap().data.clone().unwrap(); + let kept: Vec = r.bands[0].data.iter().copied().filter(|v| r.is_valid(*v)).collect(); + // Opaque in both: columns 10 to 39 of rows 0 to 29. The shift of 0.4 m east keeps each + // cell's centre in the same column of the second. + assert_eq!(kept.len(), 30 * 30, "{after}"); + assert!(kept.iter().all(|v| *v == 5.0), "{after}: {:?}", kept.iter().find(|v| **v != 5.0)); + } + // Moved: the first's columns 0 to 4 are off the second, 5 to 9 see-through in the first, and + // 35 to 39 the second's see-through 30 to 34; 10 to 34 of every row compare, 5 apart. Its alpha + // read at the first's cells would hide 30 to 39 instead (800 cells, and 7s compared). + s.run("raster.change", &json!({"before": "before", "after": "after moved", "method": "difference", "name": "moved change"})).unwrap(); + let r = s.map().layer_by_name("moved change").unwrap().raster.as_ref().unwrap().data.clone().unwrap(); + let kept: Vec = (0..1600).filter(|i| r.is_valid(r.bands[0].data[*i])).collect(); + assert_eq!(kept.len(), 25 * 40); + assert!( + kept.iter().all(|i| (10..35).contains(&(i % 40)) && r.bands[0].data[*i] == 5.0), + "{:?}", + kept.iter().find(|i| r.bands[0].data[**i] != 5.0) + ); +} + /// A layer read whole from a Shapefile or GeoJSON file remembers the files and its rows as read /// (#1385): `source.syncState` says when it has edits the file lacks, when the file changed /// since, and neither once an edit is undone. The record is saved with the project. diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index 68c80590b5..d117ef39c8 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -1488,10 +1488,22 @@ impl RasterMask { } /// Statistics of `band`'s valid pixels that the mask leaves, from a sample of a large band. + /// A colour band's cells its alpha draws see-through are left out, as the band's own + /// statistics leave them out (#1810). pub fn stats(&self, r: &Raster, band: usize) -> BandStats { let Some(b) = r.bands.get(band) else { return BandStats::default() }; let step = b.data.len().div_ceil(MASK_STATS_SAMPLE).max(1); - let kept: Vec = b.data.iter().step_by(step).map(|v| if r.is_valid(*v) && !self.hides(*v) { *v } else { f32::NAN }).collect(); + let hidden = r.see_through_cells(band); + let kept: Vec = b + .data + .iter() + .enumerate() + .step_by(step) + .map(|(i, v)| { + let shown = r.is_valid(*v) && !self.hides(*v) && !hidden.as_ref().is_some_and(|h| h(i)); + if shown { *v } else { f32::NAN } + }) + .collect(); BandStats::from_values(&kept) } @@ -2497,6 +2509,35 @@ mod tests { assert!(Colorizer::new(&k, &r, None).color(&[5.0], None).is_some()); } + /// A style mask's statistics leave out the cells a colour band's alpha draws see-through, as + /// the band's own leave them out (#1810). + #[test] + fn a_masks_statistics_leave_out_see_through_cells() { + use rustgis_data::raster::{ColorHint, Crs, GeoTransform}; + let gt = GeoTransform { origin_x: 0.0, origin_y: 4.0, pixel_width: 1.0, pixel_height: -1.0 }; + // The bottom row see-through, holding −3.4e38; 200 elsewhere, and a black first cell. + let colour: Vec = (0..16) + .map(|i| { + if i >= 12 { + -3.4e38 + } else if i == 0 { + 0.0 + } else { + 200.0 + } + }) + .collect(); + let alpha: Vec = (0..16).map(|i| if i >= 12 { 0.0 } else { 1.0 }).collect(); + let bands = vec![colour.clone(), colour.clone(), colour, alpha]; + let r = Raster::new("rgba", 4, 4, bands, gt, Crs::LonLat, None, ColorHint::Rgb { alpha: true }, "Float32").unwrap(); + let mask = RasterMask { values: vec![200.0], ..RasterMask::default() }; + // 200 hidden by the mask, the bottom row by the alpha: the black cell alone. + let st = mask.stats(&r, 0); + assert_eq!((st.count, st.min, st.max), (1, 0.0, 0.0)); + // The alpha band itself: every cell the mask leaves. + assert_eq!(mask.stats(&r, 3).count, 16); + } + /// #348: values a mask lists, or outside (or inside) its range, are no data. #[test] fn masks_hide_values_and_their_statistics() { diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index 229460081d..c09ea17db6 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -784,22 +784,41 @@ fn with_stretch(info: Arc, file: String, look: Look, then: StretchThen) { w(s.clone()); } }; + // The colour bands, and the alpha band where the file has one: a cell it hides is left out, + // as the map leaves it unpainted (#1810), so a value a see-through cell holds (−3.4e38 in a + // Float32 image) doesn't set the stretch. + let read = info.shown_from(colour.clone()); + let alpha = info.alpha.and_then(|a| read.iter().rposition(|b| *b == a)); + let opaque = rustgis_data::raster::alpha_max(info.layout.data_type); fetch_bands( info, file, coarsest, tiles, - colour.clone(), + read, Box::new(move |r| match r { Ok(picked) => { let s: Vec<(f32, f32)> = (0..colour.len()) .map(|j| { + // The alpha band shown as a colour isn't hidden by itself. + let alpha = alpha.filter(|a| *a != j); percentiles( picked .iter() - .filter_map(|t| t.get(j)) - .filter_map(|(chunk, b)| chunk.get(*b)) - .flat_map(|v| v.iter().map(|x| shown_value(*x, look.aef))) + .filter_map(|t| { + let band = t.get(j).and_then(|(chunk, b)| chunk.get(*b))?; + Some((band, alpha.and_then(|a| t.get(a)).and_then(|(chunk, b)| chunk.get(*b)))) + }) + .flat_map(|(band, hides)| { + band.iter() + .enumerate() + .filter(move |(i, _)| { + hides.is_none_or(|al| { + al.get(*i).is_some_and(|&a| !rustgis_data::raster::see_through(a, !a.is_nan(), opaque)) + }) + }) + .map(|(_, x)| shown_value(*x, look.aef)) + }) .collect(), ) }) @@ -1750,6 +1769,65 @@ mod tests { assert!(info.stretch_for(&own).is_none(), "kept, which a look of its bands would take for its own"); } + /// A colour image's stretch leaves out the cells its alpha band hides (#1810): an RGBA + /// Float32 COG whose see-through half holds −3.4e38, as an export from before #1759 wrote it, + /// is stretched over its opaque half, black cells and all. + #[test] + fn a_stretch_leaves_out_what_the_alpha_band_hides() { + use rustgis_data::raster::{ColorHint, Crs, GeoTransform, Raster}; + let _turn = crate::cache::turn(); + let _alarm = crate::cache::alarm_turn(); + let (w, h) = (512usize, 512usize); + let n = w * h; + // The top half opaque, 10 to 209 and black in its first column; the bottom see-through. + let colour: Vec = (0..n) + .map(|i| { + if i >= n / 2 { + -3.4e38 + } else if i % w == 0 { + 0.0 + } else { + (10 + i % 200) as f32 + } + }) + .collect(); + let alpha: Vec = (0..n).map(|i| if i >= n / 2 { 0.0 } else { 1.0 }).collect(); + let gt = GeoTransform { origin_x: 500_000.0, origin_y: 4_800_000.0, pixel_width: 10.0, pixel_height: -10.0 }; + let utm = Crs::from_epsg(32611).unwrap(); + let bands = vec![colour.clone(), colour.clone(), colour, alpha]; + let r = Raster::new("rgba", w as u32, h as u32, bands, gt, utm, None, ColorHint::Rgb { alpha: true }, "Float32").unwrap(); + let file = Arc::new(rustgis_data::raster::cogwrite::write(&r, 0).unwrap()); + let held = Arc::new(Mutex::new(Vec::<(String, Done)>::new())); + let hold = held.clone(); + crate::cache::set_fetcher(move |url, done| hold.lock().unwrap().push((url, done))); + let drain = || { + for _ in 0..100 { + let pending = std::mem::take(&mut *held.lock().unwrap()); + if pending.is_empty() { + return; + } + for (url, done) in pending { + let (a, b) = crate::cache::split_range(&url).1.unwrap(); + done(Ok(file[a as usize..((b + 1) as usize).min(file.len())].to_vec())); + } + } + panic!("bounded reads"); + }; + let url = "https://see-through.invalid/rgba.tif".to_string(); + let (tx, rx) = std::sync::mpsc::channel(); + with_info(&url, Box::new(move |r| tx.send(r).unwrap())); + drain(); + let info = rx.recv().unwrap().unwrap(); + assert_eq!(info.alpha, Some(3)); + let (tx, rx) = std::sync::mpsc::channel(); + with_stretch(info, url.clone(), Look::default(), Box::new(move |r| tx.send(r).unwrap())); + drain(); + let stretch = rx.recv().unwrap().unwrap(); + assert_eq!(stretch.len(), 3); + assert!(stretch.iter().all(|(lo, hi)| (0.0..=209.0).contains(lo) && (0.0..=209.0).contains(hi) && lo < hi), "{stretch:?}"); + forget_file(&url); + } + /// A joint read whose tiles the fetch queue keeps dropping ends, saying so, rather than /// asking without end: within eight drops a tile (64 at least). #[test] diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 55ecad8978..2c3b6b9f4c 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -1201,6 +1201,17 @@ With a raster layer selected in Layers, the **Style** and **Data** tabs show its | Style › Enhancement | Brightness, contrast, gamma | | Data | Raster information (size, bands, statistics, coordinate system) and Export Raster (GeoTIFF) | +**See-through cells** of an image with an alpha band (RGBA, or grey and alpha) are left out of +its colour bands' statistics. These are the cells the map draws fully see-through: an alpha of no +value, or one under half a 256th of opaque. That is alpha 0 in an 8-bit image, 128 or less of +65535 in a 16-bit one, and under about 0.002 of 1 in a Float32 one. Leaving them out changes +Raster information's counts and ranges, and the stretches taken from them: Minimum Maximum, +Percent Clip, Standard Deviation, Histogram Equalize, a style mask's, and a cloud-optimised +GeoTIFF's own. So a value those cells hold, such as −3.4e38 in a Float32 image, doesn't set the +stretch, and an opaque black cell still counts. The alpha band's own statistics count every +alpha with a value, see-through ones included. +**Change Detection** gives no-data where either image is see-through. + **Cast shadows**, on a Shaded Relief layer (`raster.symbology` `"shadow"`, 0 to 1), darkens the slopes the terrain hides from the sun as well as those facing away from it: a valley behind a ridge is in shadow, with a soft edge. Only the direct light is darkened: a slope in full shadow keeps the