From 6755b8d63dda20a8df30a1f068d2dcc21bfb576e Mon Sep 17 00:00:00 2001 From: mavrick Date: Thu, 12 Nov 2026 18:03:34 -0600 Subject: [PATCH 1/4] perf(data): the untiled GeoTIFF export takes a predictor and Deflate's fastest level (#1111) The untiled GeoTIFF writer (geotiff::write and write_all) writes its strips through the COG writer's code instead of the tiff crate's encoder, which can't write the floating-point predictor: floats take Predictor 3, bytes Predictor 2, a single band of a few classes none at Deflate's default level, everything else Deflate level 1 through miniz_oxide. The samples and tags are as before, apart from the resolution tags, which are no longer written. A file that could pass 4 GiB is now BigTIFF instead of an error. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/raster/cogwrite.rs | 305 ++++++++++++++++++++++++----- crates/data/src/raster/geotiff.rs | 159 +++++---------- docs/user-guide/data.md | 20 +- 3 files changed, 318 insertions(+), 166 deletions(-) diff --git a/crates/data/src/raster/cogwrite.rs b/crates/data/src/raster/cogwrite.rs index b1d38fc065..d1e61638d5 100644 --- a/crates/data/src/raster/cogwrite.rs +++ b/crates/data/src/raster/cogwrite.rs @@ -26,8 +26,12 @@ //! //! Classic TIFF is used when the file can't pass 4 GiB even with every tile left uncompressed, //! BigTIFF otherwise. +//! +//! The untiled GeoTIFF ([`super::geotiff::write`]) is written here too, by [`write_strips`]: one +//! image in strips of rows, with the same predictors and Deflate effort, and no overviews. use std::io::{Seek, SeekFrom, Write}; +use std::ops::Range; use std::path::Path; use super::geotiff::{GeoKeys, fits_bytes, geo_keys}; @@ -48,6 +52,11 @@ fn err(e: impl std::fmt::Display) -> DataError { DataError::Format(format!("COG: {e}")) } +/// An error of the untiled writer ([`write_strips`]). +fn strip_err(e: impl std::fmt::Display) -> DataError { + DataError::Format(format!("GeoTIFF: {e}")) +} + /// Which TIFF flavour to write. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Flavour { @@ -67,8 +76,9 @@ pub struct CogInfo { pub bytes: u64, } +/// The type of every sample in a file. #[derive(Clone, Copy, PartialEq, Eq)] -enum Sample { +pub(super) enum Sample { Byte, Short, Float, @@ -82,6 +92,21 @@ impl Sample { Sample::Float => 4, } } + + /// BitsPerSample and SampleFormat (1 unsigned whole numbers, 3 floats). + fn tiff_type(self) -> (u16, u16) { + match self { + Sample::Byte => (8, 1), + Sample::Short => (16, 1), + Sample::Float => (32, 3), + } + } +} + +/// Whether a band holds classes: it names them, or holds a few distinct values. Their codes don't +/// change smoothly, so they take no predictor (see the module's notes). +pub(super) fn few_classes(b: &super::Band) -> bool { + !b.classes.is_empty() || b.stats.categories.as_ref().is_some_and(|c| c.len() <= 64) } /// What goes into the file: which bands, as what. @@ -202,7 +227,7 @@ fn plan(r: &Raster, band: usize) -> Result { }; let max = if sample == Sample::Byte { 255.0 } else { 65_535.0 }; // A few distinct whole numbers are classes; photographs and rasters with a range are not. - let categorical = !b.classes.is_empty() || (sample != Sample::Float && b.stats.categories.as_ref().is_some_and(|c| c.len() <= 64)); + let categorical = !b.classes.is_empty() || (sample != Sample::Float && few_classes(b)); Ok(Plan { bands: vec![band], sample, @@ -330,20 +355,36 @@ fn tile_samples(r: &Raster, plan: &Plan, lv: &Level, tx: u32, ty: u32) -> Vec u16 { - match plan.sample { - _ if plan.categorical => 1, + predictor_of(plan.sample, plan.categorical) +} + +/// The TIFF predictor for samples of type `sample`: none (1) for classes, differences along the row +/// (2) for whole numbers, the floating-point one (3) for floats. +fn predictor_of(sample: Sample, classes: bool) -> u16 { + match sample { + _ if classes => 1, Sample::Byte | Sample::Short => 2, Sample::Float => 3, } } -/// Applies `plan`'s predictor to a tile's samples (as [`tile_samples`] makes them), one tile row at -/// a time; `row` is scratch space for the floating-point predictor's reordering. +/// Applies `plan`'s predictor to a tile's samples (as [`tile_samples`] makes them); `row` is +/// scratch space for the floating-point predictor's reordering. fn predict(raw: &mut [u8], plan: &Plan, row: &mut Vec) { - let spp = plan.bands.len(); - let size = plan.sample.bytes(); - for line in raw.chunks_exact_mut(TILE as usize * spp * size) { - match (predictor(plan), plan.sample) { + let line = TILE as usize * plan.bands.len() * plan.sample.bytes(); + predict_rows(raw, line, plan.bands.len(), plan.sample, predictor(plan), row); +} + +/// Applies `predictor` to rows of `line` bytes each, `spp` samples of type `sample` to a pixel, one +/// row at a time; `row` is scratch space for the floating-point predictor's reordering. +fn predict_rows(raw: &mut [u8], line: usize, spp: usize, sample: Sample, predictor: u16, row: &mut Vec) { + let size = sample.bytes(); + // An image with no columns has no rows to predict (and `chunks_exact_mut` takes no zero). + if line == 0 { + return; + } + for line in raw.chunks_exact_mut(line) { + match (predictor, sample) { (2, Sample::Byte) => differ(line, spp), (2, Sample::Short) => { // Each 16-bit sample less the one a pixel before it, wrapping; last first, so each @@ -396,8 +437,8 @@ const LONG: u16 = 4; const DOUBLE: u16 = 12; const LONG8: u16 = 16; -/// A directory entry; `patch` marks a tile table, filled in after the tiles are written. -struct Entry { +/// A directory entry; `patch` marks a tile (or strip) table, filled in after they are written. +pub(super) struct Entry { tag: u16, ty: u16, count: u64, @@ -405,19 +446,23 @@ struct Entry { patch: Option, } -fn shorts(tag: u16, v: &[u16]) -> Entry { +/// An entry of 16-bit values. +pub(super) fn shorts(tag: u16, v: &[u16]) -> Entry { Entry { tag, ty: SHORT, count: v.len() as u64, data: v.iter().flat_map(|x| x.to_le_bytes()).collect(), patch: None } } -fn long(tag: u16, v: u32) -> Entry { +/// An entry of one 32-bit value. +pub(super) fn long(tag: u16, v: u32) -> Entry { Entry { tag, ty: LONG, count: 1, data: v.to_le_bytes().to_vec(), patch: None } } -fn doubles(tag: u16, v: &[f64]) -> Entry { +/// An entry of 64-bit floats. +pub(super) fn doubles(tag: u16, v: &[f64]) -> Entry { Entry { tag, ty: DOUBLE, count: v.len() as u64, data: v.iter().flat_map(|x| x.to_le_bytes()).collect(), patch: None } } -fn ascii(tag: u16, s: &str) -> Entry { +/// An entry of text, NUL-terminated as TIFF ASCII is. +pub(super) fn ascii(tag: u16, s: &str) -> Entry { let mut data = s.as_bytes().to_vec(); data.push(0); Entry { tag, ty: ASCII, count: data.len() as u64, data, patch: None } @@ -433,11 +478,7 @@ fn table(tag: u16, n: usize, big: bool, counts: bool) -> Entry { /// full image only, as GDAL writes it. fn entries(r: &Raster, plan: &Plan, lv: &Level, first: bool, geo: &GeoKeys, big: bool) -> Vec { let spp = plan.bands.len(); - let (bits, format) = match plan.sample { - Sample::Byte => (8u16, 1u16), - Sample::Short => (16, 1), - Sample::Float => (32, 3), - }; + let (bits, format) = plan.sample.tiff_type(); let mut e = Vec::new(); if !first { e.push(long(254, 1)); @@ -475,21 +516,6 @@ fn entries(r: &Raster, plan: &Plan, lv: &Level, first: bool, geo: &GeoKeys, big: } e.push(shorts(339, &vec![format; spp])); if first { - if let Some(c) = r.copyright.as_deref().map(super::tag_text).filter(|c| !c.is_empty()) { - e.push(ascii(33432, &c)); - } - let gt = r.transform; - e.push(doubles(33550, &[gt.pixel_width.abs(), gt.pixel_height.abs(), 0.0])); - e.push(doubles(33922, &[0.0, 0.0, 0.0, gt.origin_x, gt.origin_y, 0.0])); - e.push(shorts(34735, &geo.keys)); - if let Some(d) = &geo.custom { - if !d.doubles.is_empty() { - e.push(doubles(34736, &d.doubles)); - } - if !d.ascii.is_empty() { - e.push(ascii(34737, &d.ascii)); - } - } let n = r.width as usize * r.height as usize; let nodata = match (plan.sample, r.nodata) { // Alpha says which pixels have no value; a no-data tag would make readers that apply it @@ -502,15 +528,39 @@ fn entries(r: &Raster, plan: &Plan, lv: &Level, first: bool, geo: &GeoKeys, big: plan.bands.iter().any(|b| r.bands.get(*b).is_some_and(|b| b.stats.count < n)).then(|| "nan".to_string()) } }; - // The written bands' names and the first one's class names, so a reopened window keeps - // them (a band keeping its number has no name to write). - if let Some(xml) = super::gdal_metadata_of(r, &plan.bands) { - e.push(ascii(42112, &xml)); + e.extend(geo_entries(r, geo, &plan.bands, nodata)); + } + e +} + +/// The full image's tags past its layout, in tag order: who holds the rights, where it lies, the +/// names of the bands written as samples (`written`) and the first one's class names, and +/// `nodata`, the value that marks pixels with none. +pub(super) fn geo_entries(r: &Raster, geo: &GeoKeys, written: &[usize], nodata: Option) -> Vec { + let mut e = Vec::new(); + if let Some(c) = r.copyright.as_deref().map(super::tag_text).filter(|c| !c.is_empty()) { + e.push(ascii(33432, &c)); + } + let gt = r.transform; + e.push(doubles(33550, &[gt.pixel_width.abs(), gt.pixel_height.abs(), 0.0])); + e.push(doubles(33922, &[0.0, 0.0, 0.0, gt.origin_x, gt.origin_y, 0.0])); + e.push(shorts(34735, &geo.keys)); + if let Some(d) = &geo.custom { + if !d.doubles.is_empty() { + e.push(doubles(34736, &d.doubles)); } - if let Some(nd) = nodata { - e.push(ascii(42113, &nd)); + if !d.ascii.is_empty() { + e.push(ascii(34737, &d.ascii)); } } + // The written bands' names and the first one's class names, so a reopened window keeps them (a + // band keeping its number has no name to write). + if let Some(xml) = super::gdal_metadata_of(r, written) { + e.push(ascii(42112, &xml)); + } + if let Some(nd) = nodata { + e.push(ascii(42113, &nd)); + } e } @@ -662,22 +712,130 @@ pub fn write_to(r: &Raster, band: usize, flavour: Flavour, out: } } - for p in &head.patches { - let Some(tiles) = places.get(p.level) else { continue }; - let mut bytes = Vec::with_capacity(tiles.len() * 8); - for (off, len) in tiles { + fill_tables(out, &head.patches, &places, big, pos, err)?; + Ok(CogInfo { big, levels: levels.iter().map(|l| (l.w, l.h)).collect(), bytes: pos }) +} + +/// Fills in the tile (or strip) tables of `patches` with where each piece of image `level` went in +/// the file and how long it is (`places`), then moves `out` back to the file's end, `end`. +fn fill_tables( + out: &mut W, + patches: &[Patch], + places: &[Vec<(u64, u64)>], + big: bool, + end: u64, + err: fn(String) -> DataError, +) -> Result<()> { + for p in patches { + let Some(pieces) = places.get(p.level) else { continue }; + let mut bytes = Vec::with_capacity(pieces.len() * 8); + for (off, len) in pieces { let v = if p.counts { *len } else { *off }; if big { bytes.extend_from_slice(&v.to_le_bytes()); } else { - bytes.extend_from_slice(&u32::try_from(v).map_err(err)?.to_le_bytes()); + bytes.extend_from_slice(&u32::try_from(v).map_err(|e| err(e.to_string()))?.to_le_bytes()); } } - out.seek(SeekFrom::Start(p.at)).map_err(err)?; - out.write_all(&bytes).map_err(err)?; + out.seek(SeekFrom::Start(p.at)).map_err(|e| err(e.to_string()))?; + out.write_all(&bytes).map_err(|e| err(e.to_string()))?; } - out.seek(SeekFrom::Start(pos)).map_err(err)?; - Ok(CogInfo { big, levels: levels.iter().map(|l| (l.w, l.h)).collect(), bytes: pos }) + out.seek(SeekFrom::Start(end)).map_err(|e| err(e.to_string()))?; + Ok(()) +} + +/// An untiled image for [`write_strips`]: how its pixels are laid out, and its other tags (from +/// [`geo_entries`], and any more), in any order. +pub(super) struct Strips { + pub w: u32, + pub h: u32, + /// Samples to a pixel, interleaved. + pub spp: u16, + pub sample: Sample, + /// PhotometricInterpretation: 1 for grey, 2 for RGB. + pub photometric: u16, + /// The ExtraSamples values of the samples past the grey or colour ones (none to leave it out). + pub extra: Vec, + /// Classes: no predictor, and the default Deflate effort (see the module's notes). + pub classes: bool, + pub tags: Vec, +} + +/// Uncompressed bytes a strip holds at most, as whole rows (one row when a row is longer): enough +/// for Deflate to find what repeats, small enough to hold one strip at a time. +const STRIP_BYTES: u64 = 1 << 20; + +/// Writes an untiled TIFF of one image into `out` (which must be empty: offsets are from its +/// start): its directory and tags, then its strips, each made by `rows` (which appends the +/// samples of a range of rows, interleaved and little-endian), run through the predictor and +/// compressed with Deflate before the next is made. Classic TIFF when the file can't pass 4 GiB, +/// BigTIFF otherwise. A cancelled run stops it within a strip. +pub(super) fn write_strips( + img: Strips, + out: &mut W, + mut rows: impl FnMut(Range, &mut Vec) -> Result<()>, +) -> Result<()> { + if img.w == 0 || img.h == 0 { + return Err(strip_err("the raster has no pixels to write")); + } + let spp = usize::from(img.spp); + let line = img.w as usize * spp * img.sample.bytes(); + let per_strip = (STRIP_BYTES / (line as u64).max(1)).clamp(1, u64::from(img.h.max(1))) as u32; + let strips = img.h.div_ceil(per_strip) as usize; + let predictor = predictor_of(img.sample, img.classes); + + // The worst a file can come to: every strip uncompressed (Deflate's expansion is under 0.1%), + // plus the headers, the strip tables and the other tags. + let raw = (line as u64).saturating_mul(u64::from(img.h)); + let tags: u64 = img.tags.iter().map(|e| e.data.len() as u64).sum(); + let worst = raw.saturating_add(raw / 512).saturating_add(strips as u64 * 64).saturating_add(tags).saturating_add(1 << 20); + let big = worst >= u64::from(u32::MAX); + + let (bits, format) = img.sample.tiff_type(); + let mut e = vec![ + long(256, img.w), + long(257, img.h), + shorts(258, &vec![bits; spp]), + shorts(259, &[8]), + shorts(262, &[img.photometric]), + table(273, strips, big, false), + shorts(277, &[img.spp]), + long(278, per_strip), + table(279, strips, big, true), + shorts(284, &[1]), + shorts(317, &[predictor]), + shorts(339, &vec![format; spp]), + ]; + if !img.extra.is_empty() { + e.push(shorts(338, &img.extra)); + } + e.extend(img.tags); + // A directory's entries go in tag order. + e.sort_by_key(|e| e.tag); + let head = build_head(&[e], big); + out.seek(SeekFrom::Start(0)).map_err(strip_err)?; + out.write_all(&head.bytes).map_err(strip_err)?; + let mut pos = head.bytes.len() as u64; + + let effort = if img.classes { DEFLATE_CLASSES } else { DEFLATE_FAST }; + let mut places = vec![Vec::with_capacity(strips)]; + let (mut raw, mut row) = (Vec::new(), Vec::new()); + for y in (0..img.h as usize).step_by(per_strip as usize) { + crate::progress::check()?; + raw.clear(); + rows(y..(y + per_strip as usize).min(img.h as usize), &mut raw)?; + predict_rows(&mut raw, line, spp, img.sample, predictor, &mut row); + let zipped = miniz_oxide::deflate::compress_to_vec_zlib(&raw, effort); + if !big && pos + zipped.len() as u64 > u64::from(u32::MAX) { + return Err(strip_err("the file is too large for classic TIFF")); + } + out.write_all(&zipped).map_err(strip_err)?; + if let Some(p) = places.first_mut() { + p.push((pos, zipped.len() as u64)); + } + pos += zipped.len() as u64; + } + fill_tables(out, &head.patches, &places, big, pos, strip_err) } /// The bands [`write`] and [`write_file`] write of `r` given `band`, from the same plan. @@ -1028,6 +1186,51 @@ mod tests { } } + /// The untiled writer's strips take the same predictors: floats (NaN, negative and very large + /// values) the floating-point one, RGB bytes differences, every band of `write_all`'s float + /// samples (colour, alpha and one more) the floating-point one, and a band of a few classes + /// none. Each image runs over more than one strip, and every value of every band reads back + /// exactly. + #[test] + fn untiled_strips_read_back_exactly_through_their_predictor() { + let (w, h) = (700u32, 600u32); + let n = (w * h) as usize; + let floats = |k: usize| -> Vec { + (0..n) + .map(|i| if i % 97 == k { f32::NAN } else { (i as f32 - 1000.0) * 3.7e-4 * if i % 5 == 0 { 1e30 } else { 1.0 } }) + .collect() + }; + let bytes = |k: usize| -> Vec { (0..n).map(|i| ((i * 7 + k * 31) % 256) as f32).collect() }; + let classes = || -> Vec { (0..n).map(|i| (i / 100 % 3) as f32).collect() }; + let five: Vec> = (0..5).map(floats).collect(); + let cases = [ + (geotiff::write(&utm(w, h, vec![floats(0)], ColorHint::Gray, "Float32", None), 0).unwrap(), 3, 1), + (geotiff::write(&utm(w, h, (0..3).map(bytes).collect(), ColorHint::Rgb { alpha: false }, "UInt8", None), 0).unwrap(), 2, 3), + (geotiff::write_all(&utm(w, h, five, ColorHint::Rgb { alpha: true }, "Float32", None)).unwrap(), 3, 5), + (geotiff::write(&utm(w, h, vec![classes()], ColorHint::Gray, "UInt8", None), 0).unwrap(), 1, 1), + ]; + for (i, (file, predictor, bands)) in cases.into_iter().enumerate() { + let mut dec = Decoder::new(Cursor::new(file.as_slice())).unwrap(); + assert_eq!(dec.find_tag_unsigned::(Tag::Predictor).unwrap(), Some(predictor), "case {i}"); + assert!(dec.find_tag_unsigned_vec::(Tag::StripOffsets).unwrap().unwrap().len() > 1, "case {i}"); + let back = geotiff::read(Cursor::new(file), "back").unwrap(); + assert_eq!(back.band_count(), bands, "case {i}"); + let want: Vec> = match i { + 0 => vec![floats(0)], + 1 => (0..3).map(bytes).collect(), + 2 => (0..5).map(floats).collect(), + _ => vec![classes()], + }; + for (b, data) in want.iter().enumerate() { + for (j, v) in data.iter().enumerate() { + let (x, y) = ((j % w as usize) as i64, (j / w as usize) as i64); + let got = back.bands[b].data[j]; + assert!(got == *v || (got.is_nan() && v.is_nan()), "case {i} band {b} at ({x}, {y}): {got} for {v}"); + } + } + } + } + #[test] fn sixteen_bit_and_grey_alpha_keep_their_type() { let (w, h) = (600u32, 10u32); diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index 81fb2bffa8..1d3573975b 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -1641,63 +1641,47 @@ pub fn written_bands(r: &Raster, band: usize) -> Vec { /// Writes a raster as a Deflate-compressed GeoTIFF: a colour raster's colour bands (RGB(A) bytes /// when they are 8-bit, else 32-bit floats, alpha kept), so it reads back in colour; otherwise /// `band` (0-based) as 32-bit floats. `band` is for rasters that aren't colour ones: a colour -/// raster always writes its colour bands, as its 8-bit form did before. A cancelled run stops -/// it (with [`DataError::Cancelled`]) within a strip. +/// raster always writes its colour bands, as its 8-bit form did before. The strips go through +/// the predictors and Deflate effort of the COG writer ([`super::cogwrite::write_strips`]). A +/// cancelled run stops it (with [`DataError::Cancelled`]) within a strip. pub fn write(r: &Raster, band: usize) -> Result> { - finish(encode(r, band, false)) + encode(r, band, false) } /// Writes every band of a raster, for a layer whose pixels the project keeps: as [`write`] does /// when that keeps them all, otherwise as 32-bit float samples (RGB first for a colour raster, /// and alpha after them when it has one), which [`read`] reads back as the same bands. pub fn write_all(r: &Raster) -> Result> { - finish(encode(r, 0, true)) + encode(r, 0, true) } -/// The encoder's answer, a cancel during it as [`DataError::Cancelled`]. -fn finish(encoded: Result>) -> Result> { - match encoded { - Ok(bytes) => Ok(bytes), - // The encoder's error for a cancel only says the write stopped. - Err(e) => crate::progress::check().and(Err(e)), - } -} - -/// The file's bytes, written by the encoder a strip's compressed output at a time: each write -/// fails once the run was cancelled. It counts no progress: the encoder doesn't say where a -/// strip ends. -struct Probed(W); - -impl std::io::Write for Probed { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - crate::progress::check().map_err(std::io::Error::other)?; - self.0.write(buf) - } - fn flush(&mut self) -> std::io::Result<()> { - self.0.flush() - } -} - -impl Seek for Probed { - fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { - self.0.seek(pos) - } +/// Writes `data` (samples interleaved as `img` lays them out, a row after another) as `img`'s +/// strips, at the end of `out`. +fn put( + img: super::cogwrite::Strips, + data: &[T], + le: fn(T) -> [u8; N], + out: &mut std::io::Cursor>, +) -> Result<()> { + let per_row = img.w as usize * usize::from(img.spp); + super::cogwrite::write_strips(img, out, |rows, buf| { + let s = data.get(rows.start * per_row..rows.end * per_row).ok_or_else(|| err("the samples don't fill the image"))?; + buf.extend(s.iter().flat_map(|v| le(*v))); + Ok(()) + }) } -/// [`write`] (or with `every`, [`write_all`]), a cancel failing as any write error does. +/// [`write`] (or with `every`, [`write_all`]). fn encode(r: &Raster, band: usize, every: bool) -> Result> { - use tiff::encoder::{Compression, DeflateLevel, TiffEncoder, colortype}; + use super::cogwrite::{Sample, Strips, few_classes, geo_entries}; // A scan's place is only where it was laid on the view: not coordinates to write into a file. r.require_coordinates().map_err(err)?; - let mut out = Probed(std::io::Cursor::new(Vec::new())); - let mut enc = TiffEncoder::new(&mut out).map_err(err)?.with_compression(Compression::Deflate(DeflateLevel::Balanced)); + let mut out = std::io::Cursor::new(Vec::new()); let gt = r.transform; if gt.pixel_height > 0.0 || gt.pixel_width < 0.0 { return Err(err("only north-up rasters can be exported for now")); } - let scale = [gt.pixel_width.abs(), gt.pixel_height.abs(), 0.0]; - let tie = [0.0, 0.0, 0.0, gt.origin_x, gt.origin_y, 0.0]; - let GeoKeys { keys, custom } = geo_keys(r)?; + let geo = geo_keys(r)?; let (w, h) = (r.width, r.height); // Every band holds width × height values; one that doesn't is refused rather than padded. if r.bands.iter().any(|b| b.data.len() != (w as usize) * (h as usize)) { @@ -1711,43 +1695,26 @@ fn encode(r: &Raster, band: usize, every: bool) -> Result> { .then(|| band.data.iter().map(|v| if r.is_valid(*v) { v.clamp(0.0, 255.0) as u8 } else { nodata.unwrap_or(0) }).collect()) }; let rgb = matches!(r.color, ColorHint::Rgb { .. }) && r.bands.len() >= 3; - // Each branch below names the bands it writes as samples (`$written`): their names, and the - // first one's class names (the red band's, for a colour file), go to GDAL's metadata tag, - // read back on open. - let copyright = r.copyright.as_deref().map(super::tag_text).filter(|c| !c.is_empty()); - macro_rules! tags { - ($img:expr, $written:expr) => {{ - let classes = super::gdal_metadata_of(r, $written); - let e = $img.encoder(); - // The fourth sample says what it is: unassociated alpha (2) or data (0, as RGB and - // near-infrared). The encoder writes no ExtraSamples tag, and without one a reader - // can't tell alpha from data. - if $written.len() == 4 && rgb { - let extra: u16 = if matches!(r.color, ColorHint::Rgb { alpha: true }) { 2 } else { 0 }; - e.write_tag(Tag::ExtraSamples, &[extra][..]).map_err(err)?; - } - e.write_tag(Tag::ModelPixelScaleTag, &scale[..]).map_err(err)?; - e.write_tag(Tag::ModelTiepointTag, &tie[..]).map_err(err)?; - e.write_tag(Tag::GeoKeyDirectoryTag, &keys[..]).map_err(err)?; - if let Some(d) = &custom { - if !d.doubles.is_empty() { - e.write_tag(Tag::Unknown(34736), &d.doubles[..]).map_err(err)?; - } - if !d.ascii.is_empty() { - e.write_tag(Tag::Unknown(34737), d.ascii.as_str()).map_err(err)?; - } - } - if let Some(xml) = &classes { - e.write_tag(Tag::Unknown(42112), xml.as_str()).map_err(err)?; - } - if let Some(c) = ©right { - e.write_tag(Tag::Copyright, c.as_str()).map_err(err)?; - } - if let Some(nd) = r.nodata { - e.write_tag(Tag::GdalNodata, nd.to_string().as_str()).map_err(err)?; - } - }}; - } + // The image each branch below writes: `written` names the bands it writes as samples, whose + // names, and the first one's class names (the red band's, for a colour file), go to GDAL's + // metadata tag, read back on open. Grey or RGB samples come first, then those `extra` says + // what they are (unassociated alpha, 2, or data, 0). A single band of classes takes no + // predictor. + let image = |written: &[usize], sample: Sample, extra: Vec| -> Result { + let colour = if rgb { 3 } else { 1 }; + Ok(Strips { + w, + h, + spp: u16::try_from(colour + extra.len()).map_err(|_| err("a TIFF holds at most 65,535 bands"))?, + sample, + photometric: if rgb { 2 } else { 1 }, + extra, + classes: matches!(written, [b] if r.bands.get(*b).is_some_and(few_classes)), + tags: geo_entries(r, &geo, written, r.nodata.map(|d| d.to_string())), + }) + }; + // The fourth sample of a colour file says what it is: alpha, or data (as RGB and near-infrared). + let fourth = || vec![if matches!(r.color, ColorHint::Rgb { alpha: true }) { 2 } else { 0 }]; // Without alpha or a no-data marker, byte samples cannot keep NaN holes (for example the // cells outside a reprojected RGBI image). Keep those samples as floats instead. let byte_holes = r.nodata.is_none() @@ -1757,13 +1724,10 @@ fn encode(r: &Raster, band: usize, every: bool) -> Result> { if every && written_bands(r, band).len() < all.len() { // More bands than colour (or one grey band) takes: each a 32-bit float sample, the first // past the colour (or grey) ones alpha when the raster says it is. - use tiff::tags::ExtraSamples; // Alpha follows the colour (or grey) samples it belongs to: RGB's is the fourth band, grey's // the second, so a colour hint on too few bands for colour marks none. let alpha = if rgb { matches!(r.color, ColorHint::Rgb { alpha: true }) } else { matches!(r.color, ColorHint::GrayAlpha) }; - let extras: Vec = (0..all.len() - if rgb { 3 } else { 1 }) - .map(|i| if i == 0 && alpha { ExtraSamples::UnassociatedAlpha } else { ExtraSamples::Unspecified }) - .collect(); + let extras: Vec = (0..all.len() - if rgb { 3 } else { 1 }).map(|i| if i == 0 && alpha { 2 } else { 0 }).collect(); // The samples interleaved, as large as the raster: reserved first, so a raster this // machine can't hold twice is an error, and a cancel is heard while they are copied. // Bytes when every band is whole bytes (and an empty cell can be written as no-data), @@ -1780,7 +1744,7 @@ fn encode(r: &Raster, band: usize, every: bool) -> Result> { && all.iter().all(|b| fits_bytes(r, *b)) && r.bands.iter().all(|b| b.data.iter().all(|v| in_bytes(v) && marked(v))); macro_rules! wide { - ($colour:ty, $sample:expr) => {{ + ($sample:expr, $kind:expr, $le:expr) => {{ let mut data = super::room(samples, w as usize, h as usize)?; for i in 0..n { if i % (1 << 20) == 0 { @@ -1789,20 +1753,15 @@ fn encode(r: &Raster, band: usize, every: bool) -> Result> { // Every band holds `n` values (checked above), so none is filled in. data.extend(r.bands.iter().map(|b| $sample(b.data.get(i).copied().unwrap_or(f32::NAN)))); } - let mut img = enc.new_image::<$colour>(w, h).map_err(err)?; - img.extra_samples(&extras).map_err(err)?; - tags!(img, &all[..]); - img.write_data(&data).map_err(err)?; + put(image(&all[..], $kind, extras)?, &data, $le, &mut out)?; }}; } if as_bytes { let nodata = r.nodata.map_or(0, |d| d as u8); let byte = |v: f32| if r.is_valid(v) { v.clamp(0.0, 255.0) as u8 } else { nodata }; - if rgb { wide!(colortype::RGB8, byte) } else { wide!(colortype::Gray8, byte) } - } else if rgb { - wide!(colortype::RGB32Float, |v: f32| v) + wide!(byte, Sample::Byte, u8::to_le_bytes) } else { - wide!(colortype::Gray32Float, |v: f32| v) + wide!(|v: f32| v, Sample::Float, f32::to_le_bytes) } } else if rgb && !byte_holes @@ -1816,17 +1775,13 @@ fn encode(r: &Raster, band: usize, every: bool) -> Result> { for i in 0..n { data.extend([red.get(i), green.get(i), blue.get(i), a.get(i)].map(|v| v.copied().unwrap_or(0))); } - let mut img = enc.new_image::(w, h).map_err(err)?; - tags!(img, &[0, 1, 2, 3]); - img.write_data(&data).map_err(err)?; + put(image(&[0, 1, 2, 3], Sample::Byte, fourth())?, &data, u8::to_le_bytes, &mut out)?; } else { let mut data = Vec::with_capacity(n * 3); for i in 0..n { data.extend([red.get(i), green.get(i), blue.get(i)].map(|v| v.copied().unwrap_or(0))); } - let mut img = enc.new_image::(w, h).map_err(err)?; - tags!(img, &[0, 1, 2]); - img.write_data(&data).map_err(err)?; + put(image(&[0, 1, 2], Sample::Byte, Vec::new())?, &data, u8::to_le_bytes, &mut out)?; } } else if rgb && let (Some(red), Some(green), Some(blue)) = (r.bands.first(), r.bands.get(1), r.bands.get(2)) { // Colour bands that aren't bytes (three embedding dimensions): kept as 32-bit floats, with @@ -1839,27 +1794,21 @@ fn encode(r: &Raster, band: usize, every: bool) -> Result> { for i in 0..n { data.extend([at(red, i), at(green, i), at(blue, i), at(alpha, i)]); } - let mut img = enc.new_image::(w, h).map_err(err)?; - tags!(img, &[0, 1, 2, 3]); - img.write_data(&data).map_err(err)?; + put(image(&[0, 1, 2, 3], Sample::Float, fourth())?, &data, f32::to_le_bytes, &mut out)?; } None => { let mut data = Vec::with_capacity(n * 3); for i in 0..n { data.extend([at(red, i), at(green, i), at(blue, i)]); } - let mut img = enc.new_image::(w, h).map_err(err)?; - tags!(img, &[0, 1, 2]); - img.write_data(&data).map_err(err)?; + put(image(&[0, 1, 2], Sample::Float, Vec::new())?, &data, f32::to_le_bytes, &mut out)?; } } } else { let b = r.bands.get(band).ok_or_else(|| err(format!("the raster has no band {}", band + 1)))?; - let mut img = enc.new_image::(w, h).map_err(err)?; - tags!(img, &[band]); - img.write_data(&b.data).map_err(err)?; + put(image(&[band], Sample::Float, Vec::new())?, &b.data, f32::to_le_bytes, &mut out)?; } - Ok(out.0.into_inner()) + Ok(out.into_inner()) } #[cfg(test)] diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 038e25327e..6a2cda227b 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -1182,16 +1182,16 @@ exporting it (`layer.addData`, `share.exportRaster`) give the full size as `over and the MCP server see it too. A GeoTIFF read in place that is added before its overview is in (the desktop app, the browser) answers `"reading": true`, sized by its header when that is in, and with `null` for its size and bands until then. -`share.exportRaster` writes an untiled GeoTIFF; with `"cog": true` it writes a cloud-optimised one: -512-pixel tiles compressed with Deflate after a predictor (the floating-point one for decimal -values, differences along the row for whole numbers, none for classes), overviews (each half the -size of the last, down to one tile), and every directory ahead of the tiles, so a reader fetches the header once and then only the -tiles it needs. Overviews are averaged, except for a palette raster or one with a few whole-number -classes, whose overviews take a pixel so that no class is invented; a palette keeps its colour -table, though not the table's transparency, which a TIFF colour table can't hold (GDAL drops it -too). The file is classic TIFF when it would fit in 4 GiB even uncompressed and BigTIFF when it -might not. Exported to a file it is written tile by tile, never held whole in memory (the browser -builds it in memory to download it). +`share.exportRaster` writes an untiled GeoTIFF, in strips of rows; with `"cog": true` it writes a +cloud-optimised one: 512-pixel tiles, overviews (each half the size of the last, down to one tile), +and every directory ahead of the tiles, so a reader fetches the header once and then only the tiles +it needs. Both compress with Deflate after a predictor: the floating-point one for decimal values, +differences along the row for whole numbers, none for classes. Overviews are averaged, except for a +palette raster or one with a few whole-number classes, whose overviews take a pixel so that no +class is invented; a palette keeps its colour table, though not the table's transparency, which a +TIFF colour table can't hold (GDAL drops it too). Either file is classic TIFF when it would fit in +4 GiB even uncompressed and BigTIFF when it might not. Exported to a file, a cloud-optimised one is +written tile by tile, never held whole in memory (the browser builds it in memory to download it). GeoTIFFs may be regular or cloud-optimised: grey, RGB(A), palette (coloured by the file's colour table, as land cover such as the NLCD is) or multiband, of any pixel type, tiled or striped, and compressed with Deflate, LZW, PackBits, JPEG or Zstandard. JPEG-compressed YCbCr images are shown From d5940293178e141f6564c10c71926f1840213d63 Mon Sep 17 00:00:00 2001 From: mavrick Date: Thu, 12 Nov 2026 18:45:05 -0600 Subject: [PATCH 2/4] fix(data): a cancel during the last strip stops the untiled write Also say which rasters' overviews take a pixel rather than an average: a palette, a band with class names, or an 8- or 16-bit band of a few whole-number values. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/raster/cogwrite.rs | 21 +++++++++++++++++++++ docs/user-guide/data.md | 11 ++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/crates/data/src/raster/cogwrite.rs b/crates/data/src/raster/cogwrite.rs index d1e61638d5..3f149bbe9b 100644 --- a/crates/data/src/raster/cogwrite.rs +++ b/crates/data/src/raster/cogwrite.rs @@ -835,6 +835,8 @@ pub(super) fn write_strips( } pos += zipped.len() as u64; } + // A cancel that came while the last strip was made stops the write too. + crate::progress::check()?; fill_tables(out, &head.patches, &places, big, pos, strip_err) } @@ -1384,4 +1386,23 @@ mod tests { write_file(&r, 0, &path).unwrap(); assert!(geotiff::write(&r, 0).is_ok()); } + + /// A cancel that comes while the untiled writer makes its last strip stops the write, rather + /// than the file being finished as if nothing had happened. + #[test] + fn a_cancel_during_the_last_strip_stops_the_untiled_write() { + use crate::progress::{install, tests::Counted}; + use std::sync::atomic::Ordering; + let probe = std::sync::Arc::new(Counted::default()); + let _g = install(Some(probe.clone())); + let img = Strips { w: 4, h: 3, spp: 1, sample: Sample::Byte, photometric: 1, extra: Vec::new(), classes: false, tags: Vec::new() }; + let res = write_strips(img, &mut Cursor::new(Vec::new()), |rows, buf| { + if rows.end == 3 { + probe.cancel.store(true, Ordering::Relaxed); + } + buf.extend(std::iter::repeat_n(7u8, rows.len() * 4)); + Ok(()) + }); + assert!(matches!(res, Err(DataError::Cancelled))); + } } diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 6a2cda227b..04735076b7 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -1187,11 +1187,12 @@ cloud-optimised one: 512-pixel tiles, overviews (each half the size of the last, and every directory ahead of the tiles, so a reader fetches the header once and then only the tiles it needs. Both compress with Deflate after a predictor: the floating-point one for decimal values, differences along the row for whole numbers, none for classes. Overviews are averaged, except for a -palette raster or one with a few whole-number classes, whose overviews take a pixel so that no -class is invented; a palette keeps its colour table, though not the table's transparency, which a -TIFF colour table can't hold (GDAL drops it too). Either file is classic TIFF when it would fit in -4 GiB even uncompressed and BigTIFF when it might not. Exported to a file, a cloud-optimised one is -written tile by tile, never held whole in memory (the browser builds it in memory to download it). +palette raster, a band with class names, or an 8- or 16-bit band of a few whole-number values, +whose overviews take a pixel so that no class is invented; a palette keeps its colour table, though +not the table's transparency, which a TIFF colour table can't hold (GDAL drops it too). Either file +is classic TIFF when it would fit in 4 GiB even uncompressed and BigTIFF when it might not. +Exported to a file, a cloud-optimised one is written tile by tile, never held whole in memory (the +browser builds it in memory to download it). GeoTIFFs may be regular or cloud-optimised: grey, RGB(A), palette (coloured by the file's colour table, as land cover such as the NLCD is) or multiband, of any pixel type, tiled or striped, and compressed with Deflate, LZW, PackBits, JPEG or Zstandard. JPEG-compressed YCbCr images are shown From 83e2bfd67335feccffbc70374686a31f1069116e Mon Sep 17 00:00:00 2001 From: mavrick Date: Thu, 12 Nov 2026 20:09:03 -0600 Subject: [PATCH 3/4] perf(data): the untiled GeoTIFF writer counts its strips as progress Co-Authored-By: Claude Opus 5.5 --- crates/data/src/raster/cogwrite.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/crates/data/src/raster/cogwrite.rs b/crates/data/src/raster/cogwrite.rs index 378e04479a..f42540bc30 100644 --- a/crates/data/src/raster/cogwrite.rs +++ b/crates/data/src/raster/cogwrite.rs @@ -769,7 +769,8 @@ const STRIP_BYTES: u64 = 1 << 20; /// start): its directory and tags, then its strips, each made by `rows` (which appends the /// samples of a range of rows, interleaved and little-endian), run through the predictor and /// compressed with Deflate before the next is made. Classic TIFF when the file can't pass 4 GiB, -/// BigTIFF otherwise. A cancelled run stops it within a strip. +/// BigTIFF otherwise. It counts the strips as its progress, and a cancelled run stops it within a +/// strip. pub(super) fn write_strips( img: Strips, out: &mut W, @@ -820,8 +821,9 @@ pub(super) fn write_strips( let effort = if img.classes { DEFLATE_CLASSES } else { DEFLATE_FAST }; let mut places = vec![Vec::with_capacity(strips)]; let (mut raw, mut row) = (Vec::new(), Vec::new()); + crate::progress::begin(strips); for y in (0..img.h as usize).step_by(per_strip as usize) { - crate::progress::check()?; + crate::progress::step(1)?; raw.clear(); rows(y..(y + per_strip as usize).min(img.h as usize), &mut raw)?; predict_rows(&mut raw, line, spp, img.sample, predictor, &mut row); @@ -1367,7 +1369,7 @@ mod tests { } /// A cancelled GeoTIFF or COG write stops with [`DataError::Cancelled`], the COG leaving no - /// file; without a probe both are written. + /// file; without a probe both are written, and the GeoTIFF writer counts its strips. #[test] fn a_cancelled_raster_write_stops() { use crate::progress::{install, tests::Counted}; @@ -1378,10 +1380,18 @@ mod tests { { let _g = install(Some(Counted::cancelling_after(2))); assert!(matches!(write_file(&r, 0, &path), Err(DataError::Cancelled))); - // Cancelled before it starts: the GeoTIFF writer counts nothing. + // Cancelled before its first strip. let _g = install(Some(Counted::cancelling_after(0))); assert!(matches!(geotiff::write(&r, 0), Err(DataError::Cancelled))); } + // The GeoTIFF writer counts its strips: 1100 rows of 4400 bytes, 238 to a strip. + let counted = std::sync::Arc::new(Counted::default()); + { + let _g = install(Some(counted.clone())); + assert!(geotiff::write(&r, 0).is_ok()); + } + use std::sync::atomic::Ordering; + assert_eq!((counted.done.load(Ordering::Relaxed), counted.total.load(Ordering::Relaxed)), (5, 5)); assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); write_file(&r, 0, &path).unwrap(); assert!(geotiff::write(&r, 0).is_ok()); From 51aac9c545c4709e33e4149391cad8f6af196547 Mon Sep 17 00:00:00 2001 From: mavrick Date: Thu, 12 Nov 2026 20:09:50 -0600 Subject: [PATCH 4/4] docs: only a single band of classes takes a pixel in a COG overview Co-Authored-By: Claude Opus 5.5 --- docs/user-guide/data.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 477c05199b..70b98b2ed1 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -1188,10 +1188,12 @@ cloud-optimised one: 512-pixel tiles, overviews (each half the size of the last, and every directory ahead of the tiles, so a reader fetches the header once and then only the tiles it needs. Both compress with Deflate after a predictor: the floating-point one for decimal values, differences along the row for whole numbers, none for classes. Overviews are averaged, except for a -palette raster, a band with class names, or an 8- or 16-bit band of a few whole-number values, -whose overviews take a pixel so that no class is invented; a palette keeps its colour table, though -not the table's transparency, which a TIFF colour table can't hold (GDAL drops it too). Either file -is classic TIFF when it would fit in 4 GiB even uncompressed and BigTIFF when it might not. +single band of classes (a palette raster's, a band with class names, or a band written as 8- or +16-bit whole numbers with a few distinct values), whose overviews take a pixel so that no class is +invented; colour and grey-with-alpha files are always averaged. A palette keeps its colour table, +though not the table's transparency, which a TIFF colour table can't hold (GDAL drops it too). +Either file is classic TIFF when it would fit in 4 GiB even uncompressed and BigTIFF when it might +not. Exported to a file, a cloud-optimised one is written tile by tile, never held whole in memory (the browser builds it in memory to download it). GeoTIFFs may be regular or cloud-optimised: grey, RGB(A), palette (coloured by the file's colour