Skip to content
13 changes: 11 additions & 2 deletions crates/data/src/raster/geotiff.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1619,13 +1619,22 @@ fn as_area(keys: &[u16]) -> Vec<u16> {
}

/// Whether band `b` is written as bytes: whole numbers within 0–255 (or no valid pixels at all),
/// and a no-data value that is one too.
/// and a no-data value that is one too. A colour band's statistics leave out the cells its alpha
/// draws see-through (#1810), so those cells' values are looked at too: a file is written in the
/// type it was before, whatever they hold.
pub(super) fn fits_bytes(r: &Raster, b: usize) -> bool {
let Some(band) = r.bands.get(b) else { return false };
let s = &band.stats;
let whole_bytes = s.min >= 0.0 && s.max <= 255.0 && (s.categories.is_some() || r.data_type == "UInt8");
let nodata_ok = r.nodata.is_none_or(|d| (0.0..=255.0).contains(&d) && d.fract() == 0.0);
nodata_ok && (whole_bytes || s.count == 0)
let byte = |v: f32| !r.is_valid(v) || (v.fract() == 0.0 && (0.0..=255.0).contains(&v));
// Looked at only once the statistics fit, as it reads the whole band; an 8-bit band's are
// bytes already.
let see_through_bytes = || {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Performance (medium-low confidence, about 50%): the earlier concern about scanning when whole_bytes is false is fixed by the closure. The remaining cost is for bands that do pass the cheap tests, such as Float32 colour bands whose visible values are whole bytes. Each such band is scanned in full by every fits_bytes call, and a write calls it several times per band (plan's sample_of, the bytes closure at geotiff.rs:1719, the all(...) checks at geotiff.rs:1789 and :1818, and cogwrite.rs:118). On a 100-million-cell RGBA, that is several full passes with a closure per cell.

The verdict depends only on the band, so it could be worked out once: for example, a flag beside the statistics, computed in the same pass as BandStats (where the see-through cells are already visited), so fits_bytes reads it. Not a blocker, since the output is correct.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right: a Float32 colour band whose shown values are whole bytes is scanned once per fits_bytes call, and a write makes a few of them. The output is correct, and the scan only runs for a colour band of a raster with alpha that passes the cheap tests. A flag worked out with BandStats in the one pass would change Band's fields, which every reader and writer builds. I've left it for the release quality pass.

r.data_type == "UInt8"
|| r.see_through_cells(b).is_none_or(|hidden| band.data.iter().enumerate().all(|(i, &v)| !hidden(i) || byte(v)))
};
nodata_ok && (whole_bytes || s.count == 0) && see_through_bytes()
}

/// The bands [`write`] writes of `r` given `band`: RGB and its fourth sample (alpha only when
Expand Down
265 changes: 221 additions & 44 deletions crates/data/src/raster/mod.rs

Large diffs are not rendered by default.

15 changes: 14 additions & 1 deletion crates/engine/src/cmd/imagery.rs
Original file line number Diff line number Diff line change
Expand Up @@ -884,6 +884,19 @@ pub(crate) fn change_work(s: &Session, p: &Value) -> Result<super::workflows::Wo
Ok(Box::new(move || {
let av = &a.bands.get(band).ok_or("no such band")?.data;
let bv = rasterops::resample_to(&bfr, band, &a)?;
// A cell either raster draws see-through has no colour to compare (#1810): the first's by
// its alpha band, the second's by its alpha band resampled onto the first's grid as its
// band is (a cell outside it, or of no alpha value, is see-through).
let a_hidden = a.see_through_cells(band);
let b_alpha = match bfr.color.alpha(bfr.band_count()) {
Some((alpha, colour)) if colour.contains(&band) => Some(rasterops::resample_to(&bfr, alpha, &a)?),

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Performance (medium confidence): when the second raster has an alpha band and the bands are on different grids, resample_to runs its per-cell reprojection once for the colour band and again here for the alpha band, and holds a second n-float buffer. For an RGBA second raster, that roughly doubles the resampling cost of the change. It is correct as written, so this is not blocking: resampling the colour band and its alpha band in one pass over the cells (or skipping the alpha pass when the grids already match, as resample_to's fast path does for the colour band) would avoid the second reprojection.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Only when the grids differ. On one grid, resample_to takes its fast path for the alpha band as it does for the colour band, so the alpha is a copy, not a second reprojection. On two grids it's a second pass and a second buffer of n floats for an RGBA second raster, as you say. The result is correct either way, and the change runs on its own thread with Cancel. Resampling the band and its alpha in one pass would need a two-band resample_to in rasterops, so I've left it for the release quality pass.

_ => None,
};
let b_opaque = raster::alpha_max(&bfr.data_type);
let hidden = |i: usize| {
a_hidden.as_ref().is_some_and(|h| h(i))
|| b_alpha.as_ref().is_some_and(|al| al.get(i).is_none_or(|&v| raster::see_through(v, bfr.is_valid(v), b_opaque)))
};
let n = a.width as usize * a.height as usize;
let out = rasterops::per_index(
n,
Expand All @@ -892,7 +905,7 @@ pub(crate) fn change_work(s: &Session, p: &Value) -> Result<super::workflows::Wo
|| (),
|_, i| {
let (Some(x), Some(y)) = (av.get(i).copied(), bv.get(i).copied()) else { return NODATA };
if !a.is_valid(x) || y == NODATA {
if !a.is_valid(x) || y == NODATA || hidden(i) {
return NODATA;
}
let v = match method.as_str() {
Expand Down
150 changes: 150 additions & 0 deletions crates/engine/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43275,6 +43275,156 @@ fn stac_search_by_shape_says_what_it_leaves_out() {
assert_eq!(said(&s, "have no geometry"), before);
}

/// An RGBA orthophoto whose see-through cells hold −3.4e38 (a Float32 one, as an export from
/// before #1759 wrote it, its alpha 0 to 255 as that file had it), with opaque black cells, on
/// 40 × 40 one-metre cells in UTM zone 11N: see-through in its left ten columns, black along its
/// top row. With its colours, which are 10 to 249 elsewhere.
#[cfg(not(target_arch = "wasm32"))]
fn see_through_ortho(name: &str, see: impl Fn(usize) -> bool, offset: f32) -> rustgis_data::raster::Raster {
use rustgis_data::raster::{ColorHint, Crs, GeoTransform, Raster};
let (w, n) = (40usize, 1600usize);
let colour = |b: usize| -> Vec<f32> {
(0..n).map(|i| if see(i) { -3.4e38 } else if i < w { 0.0 } else { (10 + (i * 7 + b * 60) % 240) as f32 } + offset).collect()
};
let alpha = (0..n).map(|i| if see(i) { 0.0 } else { 255.0 }).collect();
let gt = GeoTransform { origin_x: 500_000.0, origin_y: 4_800_040.0, pixel_width: 1.0, pixel_height: -1.0 };
let utm = Crs::from_epsg(32611).unwrap();
Raster::new(name, 40, 40, vec![colour(0), colour(1), colour(2), alpha], gt, utm, None, ColorHint::Rgb { alpha: true }, "Float32")
.unwrap()
}

/// The cells an RGBA raster's alpha draws see-through are left out of its colour bands'
/// statistics (#1810): read from a file, projected (nearest) and clipped to a polygon. Its opaque
/// black cells count, its alpha band counts every cell, and the stretch shared from it (`like`)
/// spans its opaque colours, not −3.4e38.
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn see_through_cells_are_left_out_of_an_rgba_rasters_statistics() {
use rustgis_data::raster::geotiff;
let dir = std::env::temp_dir().join(format!("rustgis-see-through-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("ortho.tif");
std::fs::write(&path, geotiff::write(&see_through_ortho("ortho", |i| i % 40 < 10, 0.0), 0).unwrap()).unwrap();
let mut s = Session::new(Project::default());
s.run("layer.addData", &json!({"path": path, "name": "ortho"})).unwrap();
let tr = rustgis_data::raster::Crs::from_epsg(32611).unwrap().transformer().unwrap();
let ll = |x: f64, y: f64| {
let c = tr.to_lonlat(rustgis_geom::Coord::new(x, y)).unwrap();
json!([c.x, c.y])
};
// A square over the middle, across the see-through columns.
s.run("layer.template", &json!({"template": "polygons", "name": "Boundary"})).unwrap();
let ring = [
ll(500_005.0, 4_800_035.0),
ll(500_035.0, 4_800_035.0),
ll(500_035.0, 4_800_005.0),
ll(500_005.0, 4_800_005.0),
ll(500_005.0, 4_800_035.0),
];
s.run("edit.addFeature", &json!({"layer": "Boundary", "geometry": {"type": "Polygon", "coordinates": [ring]}})).unwrap();
s.run("raster.project", &json!({"layer": "ortho", "crs": 32611, "cellSize": 1.0, "resampling": "nearest", "name": "projected"}))
.unwrap();
s.run(
"raster.process",
&json!({"layer": "ortho", "operation": "clip", "polygon": "Boundary", "selectedOnly": false, "name": "clipped"}),
)
.unwrap();
for name in ["ortho", "projected", "clipped"] {
let r = s.map().layer_by_name(name).unwrap().raster.as_ref().unwrap().data.clone().unwrap();
// The cells its alpha shows: alpha above half a 256th of opaque (Float32's 1).
let shown = r.bands[3].data.iter().filter(|a| **a >= 0.5 / 255.0).count();
assert!(shown > 0 && shown < r.bands[3].data.len(), "{name}: {shown}");
let info = s.run("raster.info", &json!({"layer": name})).unwrap();
for b in 0..3 {
let band = &info["bands"][b];
assert_eq!(band["validPixels"], json!(shown), "{name} band {b}: {band}");
let (min, max) = (band["min"].as_f64().unwrap(), band["max"].as_f64().unwrap());
assert!((0.0..=249.0).contains(&min) && (10.0..=249.0).contains(&max), "{name} band {b}: {band}");
}
assert_eq!(info["bands"][3]["validPixels"], json!(r.bands[3].data.len()), "{name}");
}
// The file's black row counts.
assert_eq!(s.run("raster.info", &json!({"layer": "ortho"})).unwrap()["bands"][0]["min"], json!(0.0));
// The range now drawn, shared: within the opaque colours.
s.run("raster.stretch", &json!({"layer": "ortho", "like": "ortho"})).unwrap();
let style = &s.map().layer_by_name("ortho").unwrap().raster.as_ref().unwrap().style;
let rustgis_map::raster::Stretch::Custom { bands: Some(bands), .. } = style.stretch else { panic!("{:?}", style.stretch) };
assert!(bands.iter().all(|[lo, hi]| (0.0..=249.0).contains(lo) && (0.0..=249.0).contains(hi) && lo < hi), "{bands:?}");
let _ = std::fs::remove_dir_all(dir);
}

/// Change Detection leaves out a cell either raster draws see-through (#1810): the first's left
/// ten columns, holding −3.4e38, and the second's bottom ten rows, holding a stray 7, on the same
/// grid and on one moved 0.4 m. Every cell both show compares, black ones included: 5 apart. A
/// second moved five cells east, see-through in its own right ten columns, has its alpha
/// resampled as its band is: read at the first's cells instead, it would hide other columns.
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn change_detection_leaves_out_see_through_cells() {
use rustgis_data::raster::Raster;
let before = see_through_ortho("before", |i| i % 40 < 10, 0.0);
// Five cells east: its column k is the first's column k + 5, with the first's colours plus 5
// (its first five columns cover the first's see-through ones), see-through from its column 30.
let moved = {
let see = |i: usize| i % 40 >= 30;
let bands = (0..4)
.map(|b| {
(0..1600)
.map(|i| match (b, see(i)) {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
(3, true) => 0.0,
(3, false) => 255.0,
(_, true) => 7.0,
(_, false) => before.bands[b].data[i + 5] + 5.0,
})
.collect()
})
.collect();
let mut gt = before.transform;
gt.origin_x += 5.0;
Raster::new("after moved", 40, 40, bands, gt, before.crs, None, before.color.clone(), "Float32").unwrap()
};
let mut after = see_through_ortho("after", |i| i >= 1200, 5.0);
for band in after.bands.iter_mut().take(3) {
for v in band.data.iter_mut().skip(1200) {
*v = 7.0;
}
}
let shifted = {
let mut gt = after.transform;
gt.origin_x += 0.4;
let bands = after.bands.iter().map(|b| b.data.clone()).collect();
Raster::new("after shifted", 40, 40, bands, gt, after.crs, None, after.color.clone(), "Float32").unwrap()
};
let mut s = Session::new(Project::default());
for r in [before, after, shifted, moved] {
let name = r.name.clone();
let bytes = rustgis_data::raster::geotiff::write(&r, 0).unwrap();
crate::cmd::raster::add_raster(&mut s, &name, rustgis_map::RasterLayer::from_bytes(r, "x.tif", bytes)).unwrap();
}
for after in ["after", "after shifted"] {
let out = format!("{after} change");
s.run("raster.change", &json!({"before": "before", "after": after, "method": "difference", "name": out})).unwrap();
let r = s.map().layer_by_name(&out).unwrap().raster.as_ref().unwrap().data.clone().unwrap();
let kept: Vec<f32> = r.bands[0].data.iter().copied().filter(|v| r.is_valid(*v)).collect();
// Opaque in both: columns 10 to 39 of rows 0 to 29. The shift of 0.4 m east keeps each
// cell's centre in the same column of the second.
assert_eq!(kept.len(), 30 * 30, "{after}");
assert!(kept.iter().all(|v| *v == 5.0), "{after}: {:?}", kept.iter().find(|v| **v != 5.0));
}
// Moved: the first's columns 0 to 4 are off the second, 5 to 9 see-through in the first, and
// 35 to 39 the second's see-through 30 to 34; 10 to 34 of every row compare, 5 apart. Its alpha
// read at the first's cells would hide 30 to 39 instead (800 cells, and 7s compared).
s.run("raster.change", &json!({"before": "before", "after": "after moved", "method": "difference", "name": "moved change"})).unwrap();
let r = s.map().layer_by_name("moved change").unwrap().raster.as_ref().unwrap().data.clone().unwrap();
let kept: Vec<usize> = (0..1600).filter(|i| r.is_valid(r.bands[0].data[*i])).collect();
assert_eq!(kept.len(), 25 * 40);
assert!(
kept.iter().all(|i| (10..35).contains(&(i % 40)) && r.bands[0].data[*i] == 5.0),
"{:?}",
kept.iter().find(|i| r.bands[0].data[**i] != 5.0)
);
}

/// A layer read whole from a Shapefile or GeoJSON file remembers the files and its rows as read
/// (#1385): `source.syncState` says when it has edits the file lacks, when the file changed
/// since, and neither once an edit is undone. The record is saved with the project.
Expand Down
43 changes: 42 additions & 1 deletion crates/map/src/raster.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1488,10 +1488,22 @@ impl RasterMask {
}

/// Statistics of `band`'s valid pixels that the mask leaves, from a sample of a large band.
/// A colour band's cells its alpha draws see-through are left out, as the band's own
/// statistics leave them out (#1810).
pub fn stats(&self, r: &Raster, band: usize) -> BandStats {
let Some(b) = r.bands.get(band) else { return BandStats::default() };
let step = b.data.len().div_ceil(MASK_STATS_SAMPLE).max(1);
let kept: Vec<f32> = b.data.iter().step_by(step).map(|v| if r.is_valid(*v) && !self.hides(*v) { *v } else { f32::NAN }).collect();
let hidden = r.see_through_cells(band);
let kept: Vec<f32> = b
.data
.iter()
.enumerate()
.step_by(step)
.map(|(i, v)| {
let shown = r.is_valid(*v) && !self.hides(*v) && !hidden.as_ref().is_some_and(|h| h(i));
if shown { *v } else { f32::NAN }
})
.collect();
BandStats::from_values(&kept)
}

Expand Down Expand Up @@ -2497,6 +2509,35 @@ mod tests {
assert!(Colorizer::new(&k, &r, None).color(&[5.0], None).is_some());
}

/// A style mask's statistics leave out the cells a colour band's alpha draws see-through, as
/// the band's own leave them out (#1810).
#[test]
fn a_masks_statistics_leave_out_see_through_cells() {
use rustgis_data::raster::{ColorHint, Crs, GeoTransform};
let gt = GeoTransform { origin_x: 0.0, origin_y: 4.0, pixel_width: 1.0, pixel_height: -1.0 };
// The bottom row see-through, holding −3.4e38; 200 elsewhere, and a black first cell.
let colour: Vec<f32> = (0..16)
.map(|i| {
if i >= 12 {
-3.4e38
} else if i == 0 {
0.0
} else {
200.0
}
})
.collect();
let alpha: Vec<f32> = (0..16).map(|i| if i >= 12 { 0.0 } else { 1.0 }).collect();
let bands = vec![colour.clone(), colour.clone(), colour, alpha];
let r = Raster::new("rgba", 4, 4, bands, gt, Crs::LonLat, None, ColorHint::Rgb { alpha: true }, "Float32").unwrap();
let mask = RasterMask { values: vec![200.0], ..RasterMask::default() };
// 200 hidden by the mask, the bottom row by the alpha: the black cell alone.
let st = mask.stats(&r, 0);
assert_eq!((st.count, st.min, st.max), (1, 0.0, 0.0));
// The alpha band itself: every cell the mask leaves.
assert_eq!(mask.stats(&r, 3).count, 16);
}

/// #348: values a mask lists, or outside (or inside) its range, are no data.
#[test]
fn masks_hide_values_and_their_statistics() {
Expand Down
Loading
Loading