diff --git a/crates/data/src/raster/cog.rs b/crates/data/src/raster/cog.rs index 7498c5e26..1baf8853a 100644 --- a/crates/data/src/raster/cog.rs +++ b/crates/data/src/raster/cog.rs @@ -748,6 +748,9 @@ pub struct Example { /// Without `classes`: band `b`'s wavelength, 400 + 100 b nanometres, in GDAL's metadata /// (its only item there). pub wavelengths: bool, + /// Overviews half the size before them rounded down (500 of 1001), as terrio 0.8 writes + /// them; GDAL rounds up (501). + pub rounded_down: bool, } impl Default for Example { @@ -770,6 +773,7 @@ impl Default for Example { sparse: false, no_overviews: false, wavelengths: false, + rounded_down: false, } } } @@ -783,7 +787,8 @@ pub fn example_with(e: &Example) -> Vec { let per_chunk = nb / planes; let mut sizes = vec![size]; while !e.no_overviews && *sizes.last().unwrap_or(&tile) > tile { - sizes.push(sizes.last().unwrap_or(&tile).div_ceil(2)); + let last = *sizes.last().unwrap_or(&tile); + sizes.push(if e.rounded_down { last / 2 } else { last.div_ceil(2) }); } let value = |x: u32, y: u32, b: u32| -> u8 { if e.int8 { diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index 8954fba8b..413b2d9c9 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -1430,9 +1430,14 @@ pub(super) fn read_limited_with_nodata( } let (dw, dh) = (full_w.div_ceil(k), full_h.div_ceil(k)); let decimated = (k <= 1 << 20).then_some(u64::from(dw) * u64::from(dh)); + // As detailed as every k-th pixel is an overview of the full size over k, rounded either + // way: GDAL rounds up, some writers down (terrio's 10439 × 9842 for 83517 × 78739 over + // 8, #1965), and one a pixel short must not make the whole full image be read instead. + let as_detailed = |pixels: u64| pixels >= u64::from((full_w / k).max(1)) * u64::from((full_h / k).max(1)); match best { - // An overview at least as detailed as the full image read sparsely: only it is read. - Some((index, ow, oh)) if decimated.is_none_or(|d| u64::from(ow) * u64::from(oh) >= d) => { + // An overview as detailed, to the rounding of its size, as the full image read + // sparsely: only it is read. + Some((index, ow, oh)) if decimated.is_none() || as_detailed(u64::from(ow) * u64::from(oh)) => { dec.seek_to_image(index).map_err(err)?; image = index; (w, h) = (ow, oh); @@ -2922,6 +2927,29 @@ mod tests { assert_eq!(back.value(0, 517, 300), r.value(0, 517, 300)); } + /// #1965: an overview a pixel short of every k-th pixel of the full image, as a writer that + /// rounds its overviews' sizes down makes it (terrio's 10439 × 9842 for 83517 × 78739 over 8), + /// is read rather than the whole full image, whose every k-th pixel is only a pixel larger. A + /// 10 GB mosaic took minutes to open so. + #[test] + fn an_overview_rounded_down_is_read_rather_than_the_full_image() { + use super::super::cog::{Example, example_with}; + // 1001 pixels a side in 256-pixel tiles, overviews of 500 and 250. At most 260,000 + // samples: every second pixel is 501 × 501, the 500 overview a pixel short each way. + let file = example_with(&Example { size: 1001, tile: 256, rounded_down: true, ..Example::default() }); + let max = 260_000; + let whole = read_limited(reader(std::io::Cursor::new(&file)).unwrap(), "rounded", max, None).unwrap(); + assert_eq!((whole.width, whole.height, whole.overview), (500, 500, Some((1001, 1001)))); + // Only the overview's tiles are read, none of the full image's (most of the file). + let (r, parts) = read_in_parts(&file, 64 << 10, max); + assert_eq!((r.width, r.height), (500, 500)); + assert!(parts.held() < file.len() as u64 / 3, "{} of {}", parts.held(), file.len()); + // One rounded up, as GDAL writes it, is read as before. + let up = example_with(&Example { size: 1001, tile: 256, ..Example::default() }); + let r = read_limited(reader(std::io::Cursor::new(&up)).unwrap(), "up", max, None).unwrap(); + assert_eq!((r.width, r.height), (501, 501)); + } + #[test] fn a_striped_file_read_in_parts_is_read_to_its_end() { // No overviews and in strips: every strip is needed, and reads stop at the file's end.