diff --git a/apps/rustgis-python/python/rustgis/_embed.py b/apps/rustgis-python/python/rustgis/_embed.py index d2c04867de..573872f4f8 100644 --- a/apps/rustgis-python/python/rustgis/_embed.py +++ b/apps/rustgis-python/python/rustgis/_embed.py @@ -29,10 +29,12 @@ #: each change, and some setups cap a message near 10 MB, so this stays small. EMBED_LIMIT = 2 << 20 -# The extensions `layer.addDataBytes` reads from a file's contents. +# The extensions `layer.addDataBytes` reads from a file's contents. (Not `.hgt` or `.dt2`, never +# under EMBED_LIMIT; a `.dt1` cell is under it from 50° of latitude, where its spacing widens.) _EMBEDDABLE = { - ".tif", ".tiff", ".asc", ".geojson", ".json", ".csv", ".tsv", ".txt", ".zip", ".kml", ".kmz", - ".gpx", ".parquet", ".geoparquet", ".fgb", ".gpkg", ".mbtiles", ".xlsx", ".xls", ".ods", + ".tif", ".tiff", ".asc", ".dt0", ".dt1", ".geojson", ".json", ".csv", ".tsv", ".txt", ".zip", + ".kml", ".kmz", ".gpx", ".parquet", ".geoparquet", ".fgb", ".gpkg", ".mbtiles", ".xlsx", + ".xls", ".ods", } #: The page script that frames the app: an anywidget module, also run by the Qt host page. diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index a2c810a42e..ff3e1c3753 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -57,6 +57,7 @@ const DATA_EXTS: &[&str] = &[ "tif", "tiff", "asc", + // SRTM height tiles and DTED are added below (`ELEVATION_EXTENSIONS`). ]; const CANVAS_ID: &str = "rustgis_canvas"; const LOADING_ID: &str = "rustgis_loading"; @@ -1294,7 +1295,10 @@ fn services(inbox: Inbox, font_inbox: Inbox, notes: Notes, ctx: egui::Context, e } else if purpose == "picture" { rfd::AsyncFileDialog::new().add_filter("Pictures", &["png", "jpg", "jpeg"]) } else { - rfd::AsyncFileDialog::new().add_filter("Vector and raster data", DATA_EXTS) + rfd::AsyncFileDialog::new().add_filter( + "Vector and raster data", + &[DATA_EXTS, &rustgis_ui_egui::rustgis_engine::data::raster::ELEVATION_EXTENSIONS[..]].concat(), + ) }; // Data may come several files at once (photos gather in one layer). let many = matches!(purpose, "data" | "shapefile"); diff --git a/crates/analysis/src/warp.rs b/crates/analysis/src/warp.rs index ea1f7d0d8f..6ff71c1092 100644 --- a/crates/analysis/src/warp.rs +++ b/crates/analysis/src/warp.rs @@ -223,7 +223,7 @@ pub fn warp(r: &Raster, fit: &Fit, o: &Options) -> Result { let re = Reproject::new(o.points_crs, o.crs)?; let (w, h) = (f64::from(r.width), f64::from(r.height)); // An overview's pixels are bigger than the full image's, which the fit is in. - let (sx, sy) = r.overview.map_or((1.0, 1.0), |(fw, fh)| (f64::from(fw) / w, f64::from(fh) / h)); + let (sx, sy) = r.file_pixels(); let place = |c: f64, row: f64| fit.to_map(c * sx, row * sy).and_then(|(x, y)| re.forward(x, y)); let [min_x, min_y, max_x, max_y] = footprint(w, h, &place).ok_or("the image's edge can't be placed in the output coordinate system")?; @@ -1206,6 +1206,34 @@ mod tests { assert!(out.bands[0].data.iter().all(|v| *v == 10.0 || *v == -9999.0)); } + #[test] + fn a_picture_read_at_a_step_is_warped_by_the_step_exactly() { + // A 61-pixel-wide file read at every second pixel: 31 cells, each exactly two of the + // file's pixels from the corner (61 isn't a multiple of 2, so 61/31 ≈ 1.968 would put + // the far cells about one cell off). The fit is in the file's pixels; each cell holds + // the file column it was read from. + let (w, h) = (31u32, 4u32); + let data: Vec = (0..w * h).map(|i| 2.0 * (i % w) as f32).collect(); + let r = scan(w, h, vec![data], ColorHint::Gray, "Float32").read_at_step(2, (61, 7)); + assert_eq!(r.file_pixels(), (2.0, 2.0)); + let map = |c: f64, rw: f64| (500_000.0 + 10.0 * c, 4_000_000.0 - 10.0 * rw); + let fit = georef::fit(Kind::Affine, &gcps(61.0, 7.0, map)).unwrap(); + let out = warp(&r, &fit, &opts(Some(5.0), Resampling::Nearest)).unwrap(); + let mut checked = 0; + for col in 0..i64::from(out.width) { + let p = out.transform.to_world(col as f64 + 0.5, 1.5); + let Some((c, _)) = fit.to_pixel(p.x, p.y) else { continue }; + // Away from the cells' edges, where nearest could go either way. + if !(0.2..61.8).contains(&c) || (c % 2.0 - 1.0).abs() < 0.2 || c % 2.0 < 0.2 { + continue; + } + let v = out.value(0, col, 1).unwrap(); + assert_eq!(f64::from(v), 2.0 * (c / 2.0).floor(), "file column {c}"); + checked += 1; + } + assert!(checked > 40, "{checked}"); + } + #[test] fn hostile_fits_and_options_are_refused() { let r = scan(4, 4, vec![vec![1.0; 16]], ColorHint::Gray, "Float32"); diff --git a/crates/data/src/provenance.rs b/crates/data/src/provenance.rs index 62f39072c3..6e028264c8 100644 --- a/crates/data/src/provenance.rs +++ b/crates/data/src/provenance.rs @@ -231,6 +231,38 @@ pub fn read(path: impl AsRef) -> std::io::Result> { Ok(out) } +/// Reads as [`read`] does, but no more than `max` bytes: a file larger than that (one that grew +/// or was replaced after its size was checked) is an error, read no further than `max + 1` bytes. +pub fn read_at_most(path: impl AsRef, max: u64) -> std::io::Result> { + use std::io::Read; + let path = path.as_ref(); + let file = File::open(path)?; + let metadata = file.metadata().ok(); + let mut out = Vec::new(); + // Sized once: reading through `take` loses the size hint `File::read_to_end` uses, so a large + // file would be copied as the buffer grows. One byte more finds a file that grew. Space that + // can't be had now is only grown into. + if let Some(m) = &metadata { + let _ = out.try_reserve_exact(usize::try_from(m.len().min(max).saturating_add(1)).unwrap_or(0)); + } + let read = file.take(max.saturating_add(1)).read_to_end(&mut out); + let too_large = u64::try_from(out.len()).map_or(true, |n| n > max); + if read.is_err() || too_large { + if active() && metadata.as_ref().is_none_or(Metadata::is_file) { + observe(path, metadata.as_ref(), None); + } + read?; + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("the file grew past {max} bytes after its size was checked"), + )); + } + if active() { + bytes(path, metadata.as_ref(), &out); + } + Ok(out) +} + /// Reads UTF-8 text with the same input capture as a byte read. pub fn read_to_string(path: impl AsRef) -> std::io::Result { use std::io::Read; @@ -255,6 +287,21 @@ pub fn read_to_string(path: impl AsRef) -> std::io::Result { mod tests { use super::*; + #[test] + fn a_read_at_most_refuses_a_file_that_grew_past_its_checked_size() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("N00E000.hgt"); + std::fs::write(&path, [7u8; 10]).unwrap(); + assert_eq!(read_at_most(&path, 10).unwrap(), vec![7u8; 10]); + assert_eq!(read_at_most(&path, 12).unwrap().len(), 10); + let err = read_at_most(&path, 9).unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!(err.to_string().contains("grew past 9 bytes"), "{err}"); + let _capture = Capture::start(); + assert!(read_at_most(&path, 9).is_err()); + assert_eq!(read_at_most(&path, 10).unwrap().len(), 10); + } + /// An atomic pathname replacement cannot pair replacement metadata with the original bytes. #[cfg(unix)] #[test] diff --git a/crates/data/src/raster/dted.rs b/crates/data/src/raster/dted.rs new file mode 100644 index 0000000000..32312aa056 --- /dev/null +++ b/crates/data/src/raster/dted.rs @@ -0,0 +1,878 @@ +//! DTED (Digital Terrain Elevation Data) levels 0, 1 and 2 (`.dt0`, `.dt1`, `.dt2`), read +//! natively on the desktop and in the browser (#2076). +//! +//! Every fact here is from the public performance specification MIL-PRF-89020B (NIMA, 23 May +//! 2000, Distribution Statement A), cited by paragraph; character positions in it count from 1, +//! the offsets here from 0. +//! +//! - A file (3.10.7.4) is a User Header Label (UHL, 80 bytes, from byte 1), a Data Set +//! Identification record (DSI, 648 bytes, from byte 81), an Accuracy Description record (ACC, +//! 2700 bytes, from byte 729), then the data records from byte 3429. Each header starts with +//! its name as a recognition sentinel (3.12 c, d, e). +//! - UHL (3.12 c): longitude of origin `DDDMMSSH` at character 5, latitude of origin `DDDMMSSH` +//! at 13 (the lower-left, south-west, post of the data), longitude and latitude intervals in +//! tenths of seconds at 21 and 25 (four digits each), the number of longitude lines at 48 and of +//! latitude points at 52 (four digits each). +//! - DSI (3.12 d): the series designator `DTED0`–`DTED2` at character 60, the vertical datum +//! (`MSL` or `E96`) at 142, the horizontal datum (`WGS84`) at 145, the latitude and longitude +//! intervals in tenths of seconds at 274 and 278, and the numbers of latitude and longitude +//! lines at 282 and 286. +//! - Data records (3.12 f, 3.10.5) run west to east, one longitude line each: a sentinel byte +//! 252 octal (0xAA), a 3-byte data block count, a 2-byte longitude count and a 2-byte latitude +//! count, then one 2-byte height per latitude point from south to north, then a 4-byte checksum: +//! 12 + 2 × points bytes in all (2414 at level 1, 7214 at level 2). +//! - Numbers in a record are "fixed binary": signed-magnitude, big-endian (3.11.1, the note to +//! 3.12 f). The sign is the top bit and negatives are not complemented, so `0x8005` is −5. +//! - The checksum (3.10.6) is the sum of every byte of the record before it, each an unsigned +//! 8-bit value. +//! - A void is the null value, every bit set: −32767 in signed magnitude (3.10.9.1, 3.11.3.1). +//! - Heights are meters "as determined by the Earth Gravity Model (EGM) 1996" (3.12 f), on WGS +//! 84 horizontally. +//! - A cell is 1° × 1° between whole degrees (3.5.1) and adjacent cells share their edges +//! (3.5.2); the latitude interval is the same everywhere, while the longitude interval widens +//! with latitude in five zones (3.9.2, Tables I–III): from 0°, 50°, 70°, 75° and 80°, one, two, +//! three, four and six times the latitude interval, which is 30″ at level 0, 3″ at level 1 and +//! 1″ at level 2. +//! +//! Posts lie on the grid lines (true longitude = count × interval + origin, 3.12 f), so as a +//! raster each post is a cell centered on it, and the cells reach half a cell beyond the cell's +//! whole-degree lines ([`super::post_transform`]), as an SRTM tile's do. +//! +//! A column whose record doesn't start with the sentinel is damaged and read as no data, and the +//! columns are counted in a notice. A record whose longitude or latitude count disagrees with its +//! place in the file (a producer numbering them differently, from 1, say) is read by its place, +//! and the columns are counted in a notice. A column whose checksum doesn't match its heights is +//! read as stored, and counted in a notice too: GDAL's DTED driver documentation says some +//! producers write wrong checksums, and that GDAL ignores them by default. A complete cell cut +//! short is read as far as it goes, the missing columns as no data, and said; a partial cell cut +//! short is refused, as it can't be told from the magnetic-tape layout. +//! +//! The same documentation says some producers write heights in two's complement rather than +//! signed magnitude. Signed magnitude reads such a negative height as one below −16000 m, which no +//! terrain has, so a height decoding below −16000 (other than the void, every bit set) is decoded +//! as two's complement instead, and counted in a notice ([`decode_height`]). + +use super::{ColorHint, Crs, Raster, max_samples, post_transform, step_within, vertical_note, void_note}; +use crate::{DataError, Result}; + +/// The headers' length: the UHL, DSI and ACC (3.10.7.4). +const HEADERS: usize = 80 + 648 + 2700; +/// Where the DSI record starts. +const DSI: usize = 80; +/// Where the ACC record starts. +const ACC: usize = 80 + 648; +/// The data record's sentinel, 252 octal (3.12 f). +const SENTINEL: u8 = 0o252; +/// The no-data value: a void (3.10.9.1). +pub const VOID: i16 = -32767; +/// Most posts along a side: 3601, level 2's latitude points (3.10.7.4.2). +pub const MAX_POSTS: usize = 3601; +/// The largest DTED file read: the headers and 3601 records of 3601 points, with room for bytes +/// after them (said, not read). +pub const MAX_BYTES: u64 = (HEADERS + MAX_POSTS * (12 + 2 * MAX_POSTS)) as u64 + (1 << 20); + +/// The ASCII text of `bytes[at..at + len]`, if it is all there. +fn field(bytes: &[u8], at: usize, len: usize) -> Option<&str> { + std::str::from_utf8(bytes.get(at..at.checked_add(len)?)?).ok() +} + +/// A field of `len` digits as a number, without a sign or spaces. +fn digits(bytes: &[u8], at: usize, len: usize) -> Option { + let s = field(bytes, at, len)?; + if s.bytes().all(|b| b.is_ascii_digit()) { s.parse().ok() } else { None } +} + +/// An angle written `DDDMMSSH` (3.12 c) as signed degrees: `H` is `N` or `E` for positive, `S` +/// or `W` for negative. +fn angle(bytes: &[u8], at: usize) -> Option { + let (d, m, s) = (digits(bytes, at, 3)?, digits(bytes, at + 3, 2)?, digits(bytes, at + 5, 2)?); + if m >= 60 || s >= 60 { + return None; + } + let v = f64::from(d) + f64::from(m) / 60.0 + f64::from(s) / 3600.0; + match bytes.get(at + 7)?.to_ascii_uppercase() { + b'N' | b'E' => Some(v), + b'S' | b'W' => Some(-v), + _ => None, + } +} + +/// A big-endian signed-magnitude number of `bytes` (3.11.1): the top bit is the sign. +fn signed_magnitude(bytes: &[u8]) -> i64 { + let raw = bytes.iter().fold(0u64, |a, b| (a << 8) | u64::from(*b)); + let top = 1u64 << (bytes.len() * 8).saturating_sub(1).min(63); + let magnitude = (raw & !top) as i64; + if raw & top != 0 { -magnitude } else { magnitude } +} + +/// A height as a record stores it (3.11.1): signed magnitude, with the void (every bit set) as +/// [`VOID`]. A value below −16000 m that isn't the void is taken for two's complement, the way some +/// producers write heights, and decoded so; the `bool` says when. Such a producer's void (−32767, +/// `0x8001` in two's complement) reads as −1 in signed magnitude, a plausible height: it can't be +/// told apart, and is read as −1. +fn decode_height(raw: [u8; 2]) -> (i16, bool) { + let word = u16::from_be_bytes(raw); + if word == 0xFFFF { + return (VOID, false); + } + if twos_complement(word) { + return (i16::from_be_bytes(raw), true); + } + let magnitude = (word & 0x7FFF) as i16; + (if word & 0x8000 != 0 { -magnitude } else { magnitude }, false) +} + +/// Whether [`decode_height`] takes a stored word for two's complement: negative in signed +/// magnitude, below −16000, and not the void. Counted for columns skipped at a step without +/// decoding them. +fn twos_complement(word: u16) -> bool { + word != 0xFFFF && word & 0x8000 != 0 && word & 0x7FFF > 16000 +} + +/// The longitude interval MIL-PRF-89020B sets, in tenths of seconds, for DTED `level` (0–2) in +/// the cell whose south edge is `south` (3.9.2, Tables I–III): the latitude interval times one, +/// two, three, four or six, by the zone of the cell's edge nearer the equator. +pub fn spec_longitude_interval(level: u8, south: f64) -> Option { + let base = spec_latitude_interval(level)?; + let north = south + 1.0; + let equatorward = if south < 0.0 && north > 0.0 { 0.0 } else { south.abs().min(north.abs()) }; + let times = match equatorward { + e if e < 50.0 => 1, + e if e < 70.0 => 2, + e if e < 75.0 => 3, + e if e < 80.0 => 4, + _ => 6, + }; + Some(base * times) +} + +/// The latitude interval MIL-PRF-89020B sets, in tenths of seconds, for DTED `level`: 30″, 3″ +/// and 1″ (Tables I–III). +pub fn spec_latitude_interval(level: u8) -> Option { + match level { + 0 => Some(300), + 1 => Some(30), + 2 => Some(10), + _ => None, + } +} + +/// Reads a DTED file: `bytes` are the file, `file_name` its name (its extension gives the level +/// the spacing is checked against when the DSI doesn't name one). +/// +/// One step that doesn't look at Cancel: the file is already in memory and is read in a single +/// pass over its records. On the desktop it runs in Add Data's read-ahead, off the UI thread, and +/// a Cancel takes effect when it returns. It stays on one thread, as [`super::hgt::read`] does: a +/// cell is at most a quarter of the largest SRTM tile's size. +pub fn read(file_name: &str, bytes: &[u8]) -> Result { + read_limited(file_name, bytes, max_samples()) +} + +/// [`read`], holding at most `max` samples (the tests' small limit exercises the reduction). +fn read_limited(file_name: &str, bytes: &[u8], max: usize) -> Result { + let bad = |m: &str| DataError::Format(format!("DTED: {m}")); + if bytes.len() < HEADERS { + return Err(bad(if bytes.starts_with(b"UHL") { "the headers are cut short" } else { "not a DTED file (no UHL header)" })); + } + if !bytes.starts_with(b"UHL") { + return Err(bad("not a DTED file (no UHL header)")); + } + if field(bytes, DSI, 3) != Some("DSI") || field(bytes, ACC, 3) != Some("ACC") { + return Err(bad("the DSI and ACC records aren't where the UHL ends")); + } + // The UHL: origin, intervals and counts (3.12 c). + let west = + angle(bytes, 4).filter(|v| (-180.0..=180.0).contains(v)).ok_or_else(|| bad("the UHL's longitude of origin can't be read"))?; + let south = angle(bytes, 12).filter(|v| (-90.0..=90.0).contains(v)).ok_or_else(|| bad("the UHL's latitude of origin can't be read"))?; + let lon_tenths = digits(bytes, 20, 4).filter(|v| *v > 0).ok_or_else(|| bad("the UHL's longitude interval can't be read"))?; + let lat_tenths = digits(bytes, 24, 4).filter(|v| *v > 0).ok_or_else(|| bad("the UHL's latitude interval can't be read"))?; + let posts = |v: Option| v.map(|v| v as usize).filter(|v| (2..=MAX_POSTS).contains(v)); + let cols = posts(digits(bytes, 47, 4)).ok_or_else(|| bad("the UHL's number of longitude lines isn't 2 to 3601"))?; + let rows = posts(digits(bytes, 51, 4)).ok_or_else(|| bad("the UHL's number of latitude points isn't 2 to 3601"))?; + let (dx, dy) = (f64::from(lon_tenths) / 36_000.0, f64::from(lat_tenths) / 36_000.0); + // A cell doesn't cross whole degrees (3.5.1): its posts span a degree at most. + let (east, north) = (west + (cols - 1) as f64 * dx, south + (rows - 1) as f64 * dy); + if east - west > 1.0 + 1e-9 || north - south > 1.0 + 1e-9 || north > 90.0 + 1e-9 || east > 180.0 + 1e-9 { + return Err(bad("its posts span more than the one-degree cell a DTED file holds")); + } + let mut lost = Vec::new(); + // What the DSI says differently (3.12 d), and what the spec sets for the level (3.9.2). + let dsi = [ + (digits(bytes, DSI + 273, 4), lat_tenths, "latitude interval"), + (digits(bytes, DSI + 277, 4), lon_tenths, "longitude interval"), + (digits(bytes, DSI + 281, 4), rows as u32, "number of latitude lines"), + (digits(bytes, DSI + 285, 4), cols as u32, "number of longitude lines"), + ]; + let differ: Vec = + dsi.iter().filter_map(|(d, u, what)| d.filter(|d| d != u).map(|d| format!("{what} {d} (the UHL's is {u})"))).collect(); + if !differ.is_empty() { + lost.push(format!("its DSI record gives a different {}; the UHL's are used", differ.join(", "))); + } + // A partial cell (3.11.3) in the magnetic-tape layout: the UHL counts a full cell (3.12 c), but + // the DSI counts only the lines that have heights, and the records leave missing posts out, so + // they differ in length (3.11.3.2.2, 3.13.4 s, t). The CD-ROM layout always has the full grid + // in both, and is the one read here (3.11.3.2.1), so its file holds every record whole. A tape + // cell with a height on every line counts the whole cell in its DSI too, but leaving posts out + // makes its file shorter than the full grid's: refused as well, said as cut short or tape. + let (dsi_rows, dsi_cols) = (dsi[2].0.map(|n| n as usize), dsi[3].0.map(|n| n as usize)); + if let Some(percent) = digits(bytes, DSI + 289, 2).filter(|p| (1..=99).contains(p)) { + if dsi_cols.is_some_and(|n| n < cols) || dsi_rows.is_some_and(|n| n < rows) { + return Err(bad(&format!( + "a partial cell ({percent}% coverage) in the magnetic-tape layout, whose DSI counts only the {} of {cols} longitude lines and {} of {rows} latitude points with heights and whose records leave missing posts out; only the full-grid layout, as on CD-ROM, is read", + dsi_cols.unwrap_or(cols), + dsi_rows.unwrap_or(rows) + ))); + } + let full = HEADERS.saturating_add(cols.saturating_mul(12 + 2 * rows)); + if bytes.len() < full { + return Err(bad(&format!( + "a partial cell ({percent}% coverage) whose file is {} bytes shorter than its full grid of {cols} × {rows} posts: cut short, or in the magnetic-tape layout, which leaves missing posts out. A partial cell is read only whole, in the full-grid layout as on CD-ROM, since the two can't be told apart", + full - bytes.len() + ))); + } + } + // The level: the DSI's series designator (3.12 d), or the extension when it names none. + let ext = std::path::Path::new(file_name).extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + let named = match ext.as_str() { + "dt0" => Some(0u8), + "dt1" => Some(1), + "dt2" => Some(2), + _ => None, + }; + let designated = match field(bytes, DSI + 59, 5) { + Some("DTED0") => Some(0u8), + Some("DTED1") => Some(1), + Some("DTED2") => Some(2), + _ => None, + }; + if let (Some(d), Some(n)) = (designated, named) + && d != n + { + lost.push(format!("its DSI says DTED level {d} but the file is named .{ext}; the DSI's level is used")); + } + let level = designated.or(named); + if let Some(level) = level { + let tenths = |t: u32| format!("{}″", f64::from(t) / 10.0); + let want = (spec_latitude_interval(level), spec_longitude_interval(level, south.floor())); + if want != (Some(lat_tenths), Some(lon_tenths)) + && let (Some(lat), Some(lon)) = want + { + lost.push(format!( + "its spacing ({} of latitude by {} of longitude) isn't the {} by {} MIL-PRF-89020B sets for DTED level {level} at this latitude; the file's own spacing is used", + tenths(lat_tenths), + tenths(lon_tenths), + tenths(lat), + tenths(lon) + )); + } + } + match field(bytes, DSI + 144, 5) { + Some("WGS84") => {} + Some(other) if !other.trim().is_empty() => { + lost.push(format!("its horizontal datum is {:?}, not WGS84; it is read as WGS 84", other.trim())); + } + _ => lost.push("it doesn't name its horizontal datum; it is read as WGS 84, as MIL-PRF-89020B sets".into()), + } + let surface = match field(bytes, DSI + 141, 3) { + Some("E96") => "the EGM96 geoid (mean sea level)".to_string(), + Some("MSL") => "mean sea level (its vertical datum MSL)".to_string(), + Some(other) if !other.trim().is_empty() => format!("its vertical datum {:?}, taken for mean sea level", other.trim()), + _ => "mean sea level (it doesn't name its vertical datum; MIL-PRF-89020B sets EGM96)".to_string(), + }; + lost.push(vertical_note(&surface)); + + // The data records, one a longitude line, west to east (3.10.5, 3.12 f). + let record = 12 + 2 * rows; + let body = bytes.get(HEADERS..).unwrap_or_default(); + let present = (body.len() / record).min(cols); + if present == 0 { + return Err(bad("the file is cut short: it has no whole data record")); + } + let step = step_within(cols, rows, max).ok_or_else(|| bad("the cell is too large to open"))?; + let (w, h) = (cols.div_ceil(step), rows.div_ceil(step)); + let mut data = super::filled(w, h, f32::from(VOID))?; + let (mut damaged, mut misnumbered, mut mismatched, mut voids, mut twos) = (0usize, 0usize, 0usize, 0u64, 0u64); + let mut column = Vec::with_capacity(rows); + for (i, rec) in body.chunks_exact(record).take(cols).enumerate() { + let Some((head, (heights, sum))) = + rec.split_at_checked(8).and_then(|(head, rest)| Some((head, rest.split_at_checked(rest.len().checked_sub(4)?)?))) + else { + damaged += 1; + continue; + }; + let lon_count = signed_magnitude(head.get(4..6).unwrap_or_default()); + let lat_count = signed_magnitude(head.get(6..8).unwrap_or_default()); + if head.first() != Some(&SENTINEL) { + damaged += 1; + continue; + } + // Counts other than the record's place are said, and the record read by its place. + if lon_count != i as i64 || lat_count != 0 { + misnumbered += 1; + } + // A wrong checksum is counted, and the heights read as stored (see the module's notes). + let total: u64 = head.iter().chain(heights).map(|b| u64::from(*b)).sum(); + if signed_magnitude(sum) != total as i64 { + mismatched += 1; + } + let raws = heights.as_chunks::<2>().0.iter().take(rows); + if i % step != 0 { + // A column skipped at a step: only counted, for the notices speak of the whole file. + for word in raws.map(|raw| u16::from_be_bytes(*raw)) { + voids += u64::from(word == 0xFFFF); + twos += u64::from(twos_complement(word)); + } + continue; + } + column.clear(); + for raw in raws { + let (v, two) = decode_height(*raw); + voids += u64::from(v == VOID); + twos += u64::from(two); + column.push(v); + } + // Rows run north to south: output row `r` is the latitude point `rows - 1 - r × step`. + for r in 0..h { + if let Some(cell) = data.get_mut(r * w + i / step) { + *cell = f32::from(column.get(rows.saturating_sub(1 + r * step)).copied().unwrap_or(VOID)); + } + } + } + if damaged == present { + return Err(bad("no data record is whole: none starts with the 0xAA sentinel")); + } + let count = |n: usize| if n == 1 { format!("1 of its {cols} columns has") } else { format!("{n} of its {cols} columns have") }; + if present < cols { + lost.push(format!("the file is cut short: {} no heights, read as no data", count(cols - present))); + } else if let Some(extra) = body.len().checked_sub(cols * record).filter(|n| *n > 0) { + lost.push(format!("{extra} bytes after its last column are ignored")); + } + if damaged > 0 { + lost.push(format!("{} a damaged record header (not the 0xAA sentinel), read as no data", count(damaged))); + } + match misnumbered { + 0 => {} + 1 => lost.push(format!("1 of its {cols} columns' record counts don't match its place in the file; it is read by place")), + n => lost.push(format!("{n} of its {cols} columns' record counts don't match their place in the file; they are read by place")), + } + match mismatched { + 0 => {} + 1 => lost.push(format!( + "1 of its {cols} columns' checksum doesn't match its heights (some producers write them wrongly); its heights are read as stored" + )), + n => lost.push(format!( + "{n} of its {cols} columns' checksums don't match their heights (some producers write them wrongly); their heights are read as stored" + )), + } + match twos { + 0 => {} + 1 => lost.push( + "1 of its heights was written in two's complement, not the signed magnitude MIL-PRF-89020B sets; it is read as two's complement" + .into(), + ), + n => lost.push(format!( + "{n} of its heights were written in two's complement, not the signed magnitude MIL-PRF-89020B sets; they are read as two's complement" + )), + } + lost.extend(void_note(voids, (rows * cols) as u64, VOID)); + let name = std::path::Path::new(file_name).file_stem().and_then(|s| s.to_str()).unwrap_or("dted").to_string(); + let transform = post_transform(west, south, rows, dx, dy, step); + let mut r = Raster::new(name, w as u32, h as u32, vec![data], transform, Crs::LonLat, Some(f64::from(VOID)), ColorHint::Gray, "Int16")?; + if step > 1 { + // The file's own size, as the layer says it; each cell read stands for exactly `step` + // posts across and down (`step`, which `Raster::file_pixels` and the warp's scale take + // rather than this over the size read, since `step` needn't divide the posts). + r.overview = Some((cols as u32, rows as u32)); + r.step = step as u32; + } + r.lost = lost; + Ok(r) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A height in signed magnitude (3.11.1); `None` is a void (every bit set). + fn sm(v: Option) -> [u8; 2] { + match v { + None => [0xff, 0xff], + Some(v) if v < 0 => (0x8000 | v.unsigned_abs()).to_be_bytes(), + Some(v) => (v as u16).to_be_bytes(), + } + } + + /// A DTED file per MIL-PRF-89020B: origin `lon`/`lat` (`DDDMMSSH`), intervals in tenths of + /// seconds, `cols` × `rows` posts, the height of column `i` (west to east), point `j` (south + /// to north) `f(i, j)`. + fn dted( + lon: &str, + lat: &str, + (lon_t, lat_t): (u32, u32), + (cols, rows): (usize, usize), + f: impl Fn(usize, usize) -> Option, + ) -> Vec { + let uhl = format!("UHL1{lon}{lat}{lon_t:04}{lat_t:04}NA U {:12}{cols:04}{rows:04}0{:24}", "", ""); + assert_eq!(uhl.len(), 80); + let mut dsi = vec![b' '; 648]; + let mut put = |at: usize, s: &str| dsi[at - 1..at - 1 + s.len()].copy_from_slice(s.as_bytes()); + put(1, "DSI"); + put(4, "U"); + put(60, "DTED1"); + put(142, "E96"); + put(145, "WGS84"); + put(274, &format!("{lat_t:04}")); + put(278, &format!("{lon_t:04}")); + put(282, &format!("{rows:04}")); + put(286, &format!("{cols:04}")); + put(290, "00"); + let mut acc = vec![b' '; 2700]; + acc[..3].copy_from_slice(b"ACC"); + let mut b = uhl.into_bytes(); + b.extend(dsi); + b.extend(acc); + assert_eq!(b.len(), 3428); + for i in 0..cols { + let mut rec = vec![SENTINEL, 0, (i >> 8) as u8, i as u8]; + rec.extend((i as u16).to_be_bytes()); + rec.extend([0, 0]); + for j in 0..rows { + rec.extend(sm(f(i, j))); + } + let sum: u32 = rec.iter().map(|b| u32::from(*b)).sum(); + rec.extend(sum.to_be_bytes()); + b.extend(rec); + } + b + } + + /// The offset of column `i`'s record in a file of `rows` points a column. + fn at(i: usize, rows: usize) -> usize { + HEADERS + i * (12 + 2 * rows) + } + + #[test] + fn signed_magnitude_is_not_twos_complement() { + assert_eq!(signed_magnitude(&[0x80, 0x05]), -5); + assert_eq!(signed_magnitude(&[0x00, 0x05]), 5); + assert_eq!(signed_magnitude(&[0xff, 0xff]), -32767); + assert_eq!(signed_magnitude(&[0x80, 0x00]), 0); + assert_eq!(signed_magnitude(&[0x00, 0x01, 0x02, 0x03]), 0x010203); + assert_eq!(signed_magnitude(&[]), 0); + } + + #[test] + fn heights_below_minus_16000_are_read_as_twos_complement() { + assert_eq!(decode_height([0x80, 0x05]), (-5, false)); + assert_eq!(decode_height([0x00, 0x05]), (5, false)); + assert_eq!(decode_height([0xff, 0xff]), (VOID, false), "the void, not two's complement −1"); + assert_eq!(decode_height([0xbe, 0x80]), (-16000, false), "−16000 itself is signed magnitude"); + assert_eq!(decode_height([0xff, 0xfb]), (-5, true)); + assert_eq!(decode_height([0xc1, 0x80]), (-16000, true)); + // A two's-complement producer's void (0x8001) reads as −1: it can't be told apart. + assert_eq!(decode_height([0x80, 0x01]), (-1, false)); + } + + #[test] + fn a_twos_complement_height_is_read_and_said() { + let rows = 11; + let mut b = dted("0000000E", "0000000N", (3600, 3600), (11, rows), |i, j| Some((i + j) as i16)); + // Column 4, point 2 holds −5 as two's complement (0xFFFB); its checksum is made right. + let rec = at(4, rows); + b[rec + 8 + 2 * 2..rec + 8 + 2 * 2 + 2].copy_from_slice(&(-5i16).to_be_bytes()); + let sum: u32 = b[rec..rec + 8 + 2 * rows].iter().map(|v| u32::from(*v)).sum(); + b[rec + 8 + 2 * rows..rec + 12 + 2 * rows].copy_from_slice(&sum.to_be_bytes()); + let r = read("x.dt1", &b).unwrap(); + assert_eq!(r.value(0, 4, 10 - 2), Some(-5.0)); + assert!( + r.lost.iter().any(|l| l + == "1 of its heights was written in two's complement, not the signed magnitude MIL-PRF-89020B sets; it is read as two's complement"), + "{:?}", + r.lost + ); + assert!(!r.lost.iter().any(|l| l.contains("checksum")), "{:?}", r.lost); + } + + #[test] + fn the_dsi_names_the_level() { + // 11 × 11 posts of 3″ is no level's spacing: the notice names the level checked against. + let b = dted("0000000E", "0000000N", (30, 30), (11, 11), |_, _| Some(1)); + let r = read("x.dt2", &b).unwrap(); + assert!( + r.lost.iter().any(|l| l == "its DSI says DTED level 1 but the file is named .dt2; the DSI's level is used"), + "{:?}", + r.lost + ); + assert!(!r.lost.iter().any(|l| l.contains("isn't the")), "level 1's spacing is 3″ by 3″: {:?}", r.lost); + // Agreeing, nothing is said; without a designator the extension gives the level. + assert!(!read("x.dt1", &b).unwrap().lost.iter().any(|l| l.contains("DSI says"))); + let mut blank = b.clone(); + blank[DSI + 59..DSI + 64].copy_from_slice(b" "); + let r = read("x.dt2", &blank).unwrap(); + assert!(!r.lost.iter().any(|l| l.contains("DSI says")), "{:?}", r.lost); + assert!(r.lost.iter().any(|l| l.contains("isn't the 1″ by 1″") && l.contains("level 2")), "{:?}", r.lost); + } + + #[test] + fn the_spec_zones_set_the_longitude_interval() { + // Tables I–III, north and south alike. + for (level, lat, want) in [ + (1, 37.0, 30), + (1, 49.0, 30), + (1, 50.0, 60), + (1, -51.0, 60), + (1, -50.0, 30), + (1, 70.0, 90), + (1, 75.0, 120), + (1, 80.0, 180), + (1, -90.0, 180), + (0, 0.0, 300), + (0, 60.0, 600), + (2, 0.0, 10), + (2, 72.0, 30), + (2, 85.0, 60), + ] { + assert_eq!(spec_longitude_interval(level, lat), Some(want), "level {level} at {lat}"); + } + assert_eq!(spec_longitude_interval(3, 0.0), None); + } + + #[test] + fn reads_a_cell_in_place() { + // 11 × 11 posts of 360″ (0.1°), from 37°N, 122°W, as a level 1 file would have 1201. + let f = |i: usize, j: usize| match (i, j) { + (0, 0) => Some(-412), + (3, 2) => None, + (10, 10) => Some(4421), + _ => Some((i * 100 + j) as i16 - 500), + }; + let bytes = dted("1220000W", "0370000N", (3600, 3600), (11, 11), f); + let r = read("cell.dt1", &bytes).unwrap(); + assert_eq!((r.width, r.height, r.crs, r.nodata, r.data_type.as_str()), (11, 11, Crs::LonLat, Some(-32767.0), "Int16")); + let t = r.transform; + assert!((t.origin_x - (-122.05)).abs() < 1e-12 && (t.origin_y - 38.05).abs() < 1e-12, "{t:?}"); + assert!((t.pixel_width - 0.1).abs() < 1e-12 && (t.pixel_height + 0.1).abs() < 1e-12); + // Column 0, point 0 is the south-west post: the bottom-left cell. + assert_eq!(r.value(0, 0, 10), Some(-412.0), "a negative height in signed magnitude"); + assert_eq!(r.value(0, 10, 0), Some(4421.0), "the north-east post"); + assert_eq!(r.value(0, 3, 8), None, "a void"); + assert_eq!(r.value(0, 4, 5), Some(-95.0)); + let sw = t.to_world(0.5, 10.5); + assert!((sw.x + 122.0).abs() < 1e-9 && (sw.y - 37.0).abs() < 1e-9); + // Its spacing isn't level 1's, the vertical datum and the one void are said. + assert!(r.lost.iter().any(|l| l.contains("isn't the 3″ by 3″")), "{:?}", r.lost); + assert!(r.lost.iter().any(|l| l.contains("EGM96 geoid")), "{:?}", r.lost); + assert!(r.lost.iter().any(|l| l == "1 of its 121 heights is a void (-32767), read as no data"), "{:?}", r.lost); + assert_eq!(r.lost.len(), 3, "{:?}", r.lost); + } + + #[test] + fn a_full_level_1_cell_says_only_its_vertical_datum() { + let bytes = dted("0060000E", "0450000S", (30, 30), (1201, 1201), |i, j| Some(((i + j) % 300) as i16)); + let r = read("s45e006.dt1", &bytes).unwrap(); + assert_eq!((r.width, r.height), (1201, 1201)); + let d = 3.0 / 3600.0; + assert!((r.transform.origin_x - (6.0 - d / 2.0)).abs() < 1e-12); + assert!((r.transform.origin_y - (-44.0 + d / 2.0)).abs() < 1e-12); + assert_eq!(r.lost.len(), 1, "{:?}", r.lost); + assert!(r.lost[0].contains("EGM96"), "{:?}", r.lost); + } + + #[test] + fn high_latitudes_have_wider_longitude_spacing() { + // Level 1, zone II (50°–70°): 3″ by 6″, 601 longitude lines of 1201 points. + let bytes = dted("0100000E", "0600000N", (60, 30), (601, 1201), |i, j| Some((i % 50 + j % 50) as i16)); + let r = read("n60e010.dt1", &bytes).unwrap(); + assert_eq!((r.width, r.height), (601, 1201)); + let (dx, dy) = (6.0 / 3600.0, 3.0 / 3600.0); + assert!((r.transform.pixel_width - dx).abs() < 1e-15 && (r.transform.pixel_height + dy).abs() < 1e-15); + assert!((r.transform.origin_x - (10.0 - dx / 2.0)).abs() < 1e-12); + assert!((r.transform.origin_y - (61.0 + dy / 2.0)).abs() < 1e-12); + assert_eq!(r.value(0, 600, 1200), Some(0.0 + 0.0), "the south-east post: column 600, point 0"); + assert_eq!(r.value(0, 7, 1200 - 9), Some(16.0)); + assert_eq!(r.lost.len(), 1, "{:?}", r.lost); + // The same spacing at 40°N isn't what the spec sets there. + let r = read("n40e010.dt1", &dted("0100000E", "0400000N", (60, 30), (601, 1201), |_, _| Some(1))).unwrap(); + assert!(r.lost.iter().any(|l| l.contains("3″ of latitude by 6″ of longitude") && l.contains("3″ by 3″")), "{:?}", r.lost); + } + + #[test] + fn bad_columns_and_cut_files_are_said() { + let rows = 11; + let mut bytes = dted("0000000E", "0000000N", (3600, 3600), (11, rows), |i, j| Some((i + j) as i16)); + // Column 2's checksum is wrong; column 5's sentinel is. + let c2 = at(2, rows) + 8; + bytes[c2 + 1] ^= 1; + bytes[at(5, rows)] = 0xab; + let r = read("x.dt1", &bytes).unwrap(); + // Column 2's heights are read as stored: point 3's is 2 + 3 = 5, point 0's (the flipped + // byte's) 2 + 0 + 1 = 3. + assert_eq!(r.value(0, 2, 10 - 3), Some(5.0), "a column failing its checksum is read as stored"); + assert_eq!(r.value(0, 2, 10), Some(3.0)); + assert_eq!(r.value(0, 5, 3), None, "a damaged header is no data"); + assert_eq!(r.value(0, 4, 3), Some(11.0)); + assert!( + r.lost.iter().any(|l| l + == "1 of its 11 columns' checksum doesn't match its heights (some producers write them wrongly); its heights are read as stored"), + "{:?}", + r.lost + ); + assert!(r.lost.iter().any(|l| l.starts_with("1 of its 11 columns has a damaged record header")), "{:?}", r.lost); + assert!(!r.lost.iter().any(|l| l.contains("void")), "voids in bad columns aren't counted: {:?}", r.lost); + // Every checksum wrong: the cell still opens, read as stored, and the count says so. + let mut sums = dted("0000000E", "0000000N", (3600, 3600), (11, rows), |i, j| Some((i + j) as i16)); + for i in 0..11 { + sums[at(i, rows) + 12 + 2 * rows - 1] ^= 1; + } + let r = read("x.dt1", &sums).unwrap(); + assert_eq!(r.value(0, 7, 10), Some(7.0)); + assert!( + r.lost.iter().any(|l| l + == "11 of its 11 columns' checksums don't match their heights (some producers write them wrongly); their heights are read as stored"), + "{:?}", + r.lost + ); + // A wrong column number or latitude count is read by its place, and said. + let mut moved = dted("0000000E", "0000000N", (3600, 3600), (11, rows), |i, _| Some(i as i16)); + moved[at(7, rows) + 5] = 9; + moved[at(8, rows) + 7] = 1; + let r = read("x.dt2", &moved).unwrap(); + assert_eq!((r.value(0, 7, 4), r.value(0, 8, 4)), (Some(7.0), Some(8.0)), "read by place"); + assert!(!r.lost.iter().any(|l| l.contains("damaged record header")), "{:?}", r.lost); + assert!( + r.lost.iter().any(|l| l == "2 of its 11 columns' record counts don't match their place in the file; they are read by place"), + "{:?}", + r.lost + ); + // Cut in the middle of column 8: columns 8–10 are missing. + let cut = &bytes[..at(8, rows) + 7]; + let r = read("x.dt2", cut).unwrap(); + assert_eq!(r.value(0, 8, 0), None); + assert!(r.lost.iter().any(|l| l == "the file is cut short: 3 of its 11 columns have no heights, read as no data"), "{:?}", r.lost); + // Bytes past the last column are said. + let mut longer = dted("0000000E", "0000000N", (3600, 3600), (11, rows), |_, _| Some(1)); + longer.extend([0; 5]); + assert!(read("x.dt2", &longer).unwrap().lost.iter().any(|l| l == "5 bytes after its last column are ignored")); + // No whole record, or no good one, is refused. + assert!(read("x.dt2", &bytes[..HEADERS + 20]).unwrap_err().to_string().contains("no whole data record")); + let mut none = dted("0000000E", "0000000N", (3600, 3600), (2, rows), |_, _| Some(1)); + none[at(0, rows)] = 0; + none[at(1, rows)] = 0; + assert!(read("x.dt2", &none).unwrap_err().to_string().contains("none starts with the 0xAA sentinel")); + } + + #[test] + fn records_numbered_from_1_are_read_by_place() { + let rows = 11; + let mut b = dted("0000000E", "0000000N", (3600, 3600), (11, rows), |i, j| Some((i * 100 + j) as i16)); + // Every longitude count one more than its place, the checksums made right. + for i in 0..11 { + let rec = at(i, rows); + b[rec + 4..rec + 6].copy_from_slice(&(i as u16 + 1).to_be_bytes()); + let sum: u32 = b[rec..rec + 8 + 2 * rows].iter().map(|v| u32::from(*v)).sum(); + b[rec + 8 + 2 * rows..rec + 12 + 2 * rows].copy_from_slice(&sum.to_be_bytes()); + } + let r = read("x.dt1", &b).unwrap(); + assert_eq!(r.value(0, 0, 10), Some(0.0), "column 0, point 0"); + assert_eq!(r.value(0, 10, 0), Some(1010.0), "column 10, point 10"); + assert_eq!( + r.lost.iter().filter(|l| l.contains("record counts")).collect::>(), + ["11 of its 11 columns' record counts don't match their place in the file; they are read by place"] + ); + assert!(!r.lost.iter().any(|l| l.contains("damaged") || l.contains("checksum")), "{:?}", r.lost); + // One column out of place is said in the singular. + let mut one = dted("0000000E", "0000000N", (3600, 3600), (11, rows), |_, _| Some(1)); + one[at(3, rows) + 5] = 4; + let r = read("x.dt1", &one).unwrap(); + assert!( + r.lost.iter().any(|l| l == "1 of its 11 columns' record counts don't match its place in the file; it is read by place"), + "{:?}", + r.lost + ); + } + + #[test] + fn headers_are_checked_before_anything_is_held() { + let good = dted("0000000E", "0000000N", (3600, 3600), (11, 11), |_, _| Some(1)); + let with = |at: usize, s: &str| { + let mut b = good.clone(); + b[at..at + s.len()].copy_from_slice(s.as_bytes()); + b + }; + for (b, why) in [ + (with(0, "XHL"), "no UHL"), + (with(80, "XSI"), "DSI and ACC"), + (with(728, "XCC"), "DSI and ACC"), + (with(47, "9999"), "longitude lines"), + (with(51, "0001"), "latitude points"), + (with(47, "12x4"), "longitude lines"), + (with(20, "0000"), "longitude interval"), + (with(4, "1810000E"), "longitude of origin"), + (with(4, "0006000E"), "longitude of origin"), + (with(12, "0910000N"), "latitude of origin"), + (with(12, "0000000Q"), "latitude of origin"), + (with(20, "9999"), "more than the one-degree cell"), + (with(12, "0895959N"), "more than the one-degree cell"), + ] { + let e = read("x.dt1", &b).unwrap_err().to_string(); + assert!(e.contains(why), "{why}: {e}"); + } + assert!(read("x.dt1", &good[..100]).unwrap_err().to_string().contains("cut short")); + assert!(read("x.dt1", b"").is_err()); + } + + #[test] + fn a_dsi_that_disagrees_or_another_datum_is_said() { + let mut b = dted("0000000E", "0000000N", (3600, 3600), (11, 11), |_, _| Some(1)); + b[DSI + 277..DSI + 281].copy_from_slice(b"0030"); + b[DSI + 141..DSI + 144].copy_from_slice(b"MSL"); + b[DSI + 144..DSI + 149].copy_from_slice(b"WGS72"); + let r = read("x.dt1", &b).unwrap(); + assert!(r.lost.iter().any(|l| l.contains("longitude interval 30 (the UHL's is 3600)")), "{:?}", r.lost); + assert!(r.lost.iter().any(|l| l.contains("mean sea level (its vertical datum MSL)")), "{:?}", r.lost); + assert!(r.lost.iter().any(|l| l.contains("\"WGS72\"")), "{:?}", r.lost); + } + + #[test] + fn a_level_2_cell_past_the_budget_is_read_at_every_other_post() { + let bytes = dted("0000000E", "0000000N", (3600, 3600), (11, 11), |i, j| Some((i * 100 + j) as i16)); + let r = read_limited("x.dt2", &bytes, 100).unwrap(); + assert_eq!((r.width, r.height, r.overview, r.step), (6, 6, Some((11, 11)), 2)); + // Output (col 1, row 1) is column 2, point 10 - 2 = 8. + assert_eq!(r.value(0, 1, 1), Some(208.0)); + // Cells are two posts wide, each centered on the post kept: column 2, point 8 is at 0.2°E, + // 0.8°N. + let t = r.transform; + assert!((t.pixel_width - 0.2).abs() < 1e-12 && (t.origin_x + 0.1).abs() < 1e-12 && (t.origin_y - 1.1).abs() < 1e-12); + let kept = t.to_world(1.5, 1.5); + assert!((kept.x - 0.2).abs() < 1e-9 && (kept.y - 0.8).abs() < 1e-9, "{kept:?}"); + } + + #[test] + fn a_step_that_doesnt_divide_the_posts_keeps_the_scale_exact() { + // 9 × 13 posts at step 2, as a level 2 cell at 75°–80° (901 × 3601) is at a step: 5 × 7 + // cells, each exactly two posts across and down. + let (cols, rows) = (9, 13); + let bytes = dted("0000000E", "0000000N", (1800, 1800), (cols, rows), |i, j| Some((i * 100 + j) as i16)); + let r = read_limited("x.dt2", &bytes, 100).unwrap(); + assert_eq!((r.width, r.height, r.step), (5, 7, 2)); + // The layer says the file's own size; the scale is the step, exactly, though 9 and 13 + // aren't multiples of the size read (the warp and world files take `file_pixels`). + assert_eq!(r.overview, Some((cols as u32, rows as u32))); + assert_eq!(r.file_pixels(), (2.0, 2.0)); + // The kept posts are columns 0, 2, …, 8 and points 12, 10, …, 0, 0.05° apart: the extent + // reaches one post (half a cell) past the outer ones on every side. + let e = r.extent(); + for (got, want) in [(e.min_x, -0.05), (e.max_x, 0.45), (e.min_y, -0.05), (e.max_y, 0.65)] { + assert!((got - want).abs() < 1e-9, "{e:?}"); + } + assert!(((e.min_x + e.max_x) / 2.0 - 0.2).abs() < 1e-9 && ((e.min_y + e.max_y) / 2.0 - 0.3).abs() < 1e-9, "{e:?}"); + assert_eq!(r.value(0, 4, 0), Some(812.0), "column 8, point 12"); + assert_eq!(r.value(0, 4, 6), Some(800.0), "column 8, point 0"); + } + + #[test] + fn counts_at_a_step_are_the_whole_files() { + // Voids and two's-complement heights in kept and skipped columns alike. + let rows = 11; + let mut b = dted("0000000E", "0000000N", (3600, 3600), (11, rows), |i, j| if (i + j) % 4 == 0 { None } else { Some(1) }); + for (i, j) in [(1, 2), (4, 3), (7, 7)] { + let rec = at(i, rows); + b[rec + 8 + 2 * j..rec + 10 + 2 * j].copy_from_slice(&(-20i16).to_be_bytes()); + } + let whole = read("x.dt1", &b).unwrap(); + let stepped = read_limited("x.dt1", &b, 40).unwrap(); + assert_eq!(stepped.step, 2); + let notes = |r: &Raster| r.lost.iter().filter(|l| l.contains("void") || l.contains("two's")).cloned().collect::>(); + assert_eq!(notes(&whole), notes(&stepped)); + assert!(notes(&whole).iter().any(|l| l.starts_with("3 of its heights were written in two's complement")), "{:?}", whole.lost); + assert!(notes(&whole).iter().any(|l| l.starts_with("30 of its 121 heights are voids")), "{:?}", whole.lost); + } + + #[test] + fn a_partial_cell_is_read_on_its_full_grid_and_refused_in_the_tape_layout() { + let partial = |mut b: Vec| { + b[DSI + 289..DSI + 291].copy_from_slice(b"50"); + b + }; + // At 6′ spacing a whole cell is 11 × 11 posts. With the full grid (as on CD-ROM), a + // partial cell's missing posts are voids, read as no data and said. + let cdrom = partial(dted("1220000W", "0370000N", (3600, 3600), (11, 11), |i, _| (i < 5).then_some(100))); + let r = read("n37.dt1", &cdrom).unwrap(); + assert_eq!(r.value(0, 0, 0), Some(100.0)); + assert_eq!(r.value(0, 8, 0), None); + assert!(r.lost.iter().any(|l| l.contains("no data")), "{:?}", r.lost); + // On tape the UHL still counts the whole cell, but the DSI only its 6 longitude lines + // with heights: refused, not misplaced. + let mut tape = partial(dted("1220000W", "0370000N", (3600, 3600), (11, 11), |_, _| Some(100))); + tape[DSI + 285..DSI + 289].copy_from_slice(b"0006"); + let err = read("n37.dt1", &tape).unwrap_err().to_string(); + assert!(err.contains("partial cell (50% coverage) in the magnetic-tape layout"), "{err}"); + assert!(err.contains("only the 6 of 11 longitude lines"), "{err}"); + // On tape with a height on every line the DSI counts the whole cell too, but the record + // that starts 5 points up is 10 bytes short, so the file is: refused, as is a partial cell + // cut short, which can't be told from it. + let mut tape = cdrom.clone(); + tape[at(0, 11) + 7] = 5; + tape.drain(at(0, 11) + 8..at(0, 11) + 18); + let err = read("n37.dt1", &tape).unwrap_err().to_string(); + assert!( + err.contains("whose file is 10 bytes shorter than its full grid of 11 × 11 posts: cut short, or in the magnetic-tape layout"), + "{err}" + ); + let err = read("n37.dt1", &cdrom[..cdrom.len() - 3]).unwrap_err().to_string(); + assert!(err.contains("3 bytes shorter"), "{err}"); + // A complete cell cut short is still read as far as it goes, and a whole partial cell + // with a damaged record is read with the damage said. + let complete = dted("1220000W", "0370000N", (3600, 3600), (11, 11), |_, _| Some(100)); + let r = read("n37.dt1", &complete[..complete.len() - 3]).unwrap(); + assert!(r.lost.iter().any(|l| l.contains("the file is cut short")), "{:?}", r.lost); + let mut damaged = cdrom.clone(); + damaged[at(3, 11)] = 0; + let r = read("n37.dt1", &damaged).unwrap(); + assert!(r.lost.iter().any(|l| l.contains("damaged record header")), "{:?}", r.lost); + // A complete cell's file (indicator 00) whose DSI disagrees is read, the difference said. + let mut complete = dted("1220000W", "0370000N", (3600, 3600), (11, 11), |_, _| Some(100)); + complete[DSI + 285..DSI + 289].copy_from_slice(b"0006"); + let r = read("n37.dt1", &complete).unwrap(); + assert!(r.lost.iter().any(|l| l.contains("DSI record gives a different")), "{:?}", r.lost); + } + + #[test] + fn junk_is_refused() { + let good = + dted( + "1220000W", + "0370000N", + (3600, 3600), + (11, 11), + |i, j| if (i + j) % 5 == 0 { None } else { Some(-(i as i16) * 7 + j as i16) }, + ); + let mut seed = 0x89020bu64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + // Every prefix is read or refused, never a panic. + for cut in 0..=good.len() { + let _ = read("x.dt1", &good[..cut]); + } + // Random bytes, alone and after good headers, and flipped bytes in a good file. + for _ in 0..200 { + let n = (next() % 8000) as usize; + let junk: Vec = (0..n).map(|_| next() as u8).collect(); + assert!(read("x.dt1", &junk).is_err()); + let mut after = good[..HEADERS].to_vec(); + after.extend(&junk); + let _ = read("x.dt1", &after); + let mut flipped = good.clone(); + for _ in 0..4 { + let at = (next() % flipped.len() as u64) as usize; + flipped[at] = next() as u8; + } + let _ = read("x.dt1", &flipped); + } + } +} diff --git a/crates/data/src/raster/gdal.rs b/crates/data/src/raster/gdal.rs index e2a6b0875e..1a0966f8f6 100644 --- a/crates/data/src/raster/gdal.rs +++ b/crates/data/src/raster/gdal.rs @@ -1,5 +1,5 @@ //! GDAL, optionally, for rasters: formats the built-in readers don't know (JPEG 2000, ERDAS -//! Imagine, SRTM HGT, DTED, NetCDF, HDF, ENVI, SAGA…) read through GDAL's `gdal_translate` when it +//! Imagine, NetCDF, HDF, ENVI, SAGA…) read through GDAL's `gdal_translate` when it //! is installed. //! //! A file (or one of its subdatasets, such as a NetCDF variable) is converted once into a @@ -16,8 +16,8 @@ use crate::{DataError, Result}; /// Raster file types read through GDAL. (Not `.vrt`, which can point GDAL at any file on the /// machine, nor `.xyz`, more often a point cloud than a grid.) pub const EXTENSIONS: &[&str] = &[ - "jp2", "j2k", "img", "hgt", "dem", "dt0", "dt1", "dt2", "nc", "nc4", "hdf", "hdf5", "h5", "he5", "bil", "bsq", "bip", "envi", "grd", - "rst", "sdat", "kea", "ers", "ntf", "nitf", "ecw", "sid", "gsb", "bt", "rsw", + "jp2", "j2k", "img", "dem", "nc", "nc4", "hdf", "hdf5", "h5", "he5", "bil", "bsq", "bip", "envi", "grd", "rst", "sdat", "kea", "ers", + "ntf", "nitf", "ecw", "sid", "gsb", "bt", "rsw", ]; /// Most subdatasets listed from one file. @@ -39,10 +39,8 @@ fn drivers(src: &Path) -> Result<&'static [&'static str]> { "jp2" | "j2k" => &["JP2OpenJPEG", "JP2KAK", "JP2ECW", "JP2MrSID"], // ERDAS Imagine, or raw cells with an ENVI or EHdr `.hdr` header beside them. "img" => &["HFA", "ENVI", "EHdr"], - "hgt" => &["SRTMHGT"], // A USGS DEM, or a GTOPO30 tile. "dem" => &["USGSDEM", "EHdr"], - "dt0" | "dt1" | "dt2" => &["DTED"], // NetCDF-4 is HDF5, and either driver may be the one built. "nc" | "nc4" | "hdf5" | "h5" | "he5" => &["netCDF", "HDF5", "HDF5Image"], "hdf" => &["HDF4", "HDF4Image", "HDF5", "HDF5Image"], @@ -633,7 +631,9 @@ mod tests { #[test] fn names() { - assert!(is_gdal_raster_name("/d/N45E006.HGT")); + assert!(is_gdal_raster_name("/d/G.IMG")); + // SRTM tiles and DTED are read natively (`raster::hgt`, `raster::dted`). + assert!(!is_gdal_raster_name("/d/N45E006.HGT") && !is_gdal_raster_name("n45.dt1")); assert!(is_gdal_raster_name("air.nc")); assert!(!is_gdal_raster_name("a.tif") && !is_gdal_raster_name("a.vrt")); assert_eq!(short_name("NETCDF:\"/d/air.nc\":air"), "air"); @@ -845,7 +845,7 @@ mod tests { assert!(e.contains("not recognized"), "{e}"); } - /// Runs only where GDAL is installed: a NetCDF file's variables, an HGT tile. + /// Runs only where GDAL is installed: a NetCDF file's variables. #[test] fn raster_conversion_provenance_excludes_cache_files_when_gdal_is_installed() { let Some(translate) = program("gdal_translate") else { return }; diff --git a/crates/data/src/raster/hgt.rs b/crates/data/src/raster/hgt.rs new file mode 100644 index 0000000000..80bed7d3ee --- /dev/null +++ b/crates/data/src/raster/hgt.rs @@ -0,0 +1,375 @@ +//! SRTM height tiles (`.hgt`), read natively on the desktop and in the browser (#2076). +//! +//! The format, from NASA/USGS's SRTM documentation (the SRTM Topography readme +//! `SRTM_Topo.pdf`, and the user guide of the NASA SRTM Version 3.0 products, SRTMGL1/SRTMGL3, +//! at LP DAAC): +//! +//! - A file holds one 1° × 1° tile, named for its south-west corner: `N37W122.hgt` is the tile +//! from 37°N to 38°N and from 122°W to 121°W (`S` and `E` likewise; latitude in two digits, +//! longitude in three). +//! - It is a square grid of 16-bit signed, big-endian (Motorola order) heights in meters, with no +//! header: 1201 × 1201 samples at 3 arc-seconds, or 3601 × 3601 at 1 arc-second. GDAL's SRTMHGT +//! driver documentation adds 0.5 arc-second tiles (since GDAL 3.10), 7201 × 7201, read here too, +//! on the same grid rule. Its size alone says which, and nothing else is read: another size is +//! refused rather than guessed. A 7201 × 7201 tile holds more samples than the browser does, so +//! there it is read at a step ([`super::step_within`]). +//! - Rows run north to south, and samples within a row west to east. +//! - −32768 is a void (no data). +//! - Horizontal datum WGS 84; heights are above the EGM96 geoid (mean sea level), not the +//! ellipsoid. +//! - Samples are points on the grid lines, the first and last row and column lying exactly on +//! the tile's whole-degree edges, so that 1° holds 1201 (3601, 7201) of them and neighboring tiles +//! share an edge row and column. +//! +//! As a raster, each sample is a cell centered on its point: the raster's cells reach half a cell +//! beyond the whole-degree lines on every side, so a tile's extent is one degree plus one cell +//! wide ([`super::post_transform`]). + +use super::{ColorHint, Crs, Raster, max_samples, post_transform, step_within, vertical_note, void_note}; +use crate::{DataError, Result}; + +/// The no-data value of a tile: a void. +pub const VOID: i16 = i16::MIN; +/// The sides a tile can have: 3 arc-seconds (SRTM3), 1 arc-second (SRTM1) and 0.5 arc-second. +pub const SIDES: [usize; 3] = [1201, 3601, 7201]; + +/// Whether `len` bytes are a tile of one of [`SIDES`], and its side. +pub fn side_of(len: u64) -> Option { + SIDES.into_iter().find(|s| (s * s * 2) as u64 == len) +} + +/// The south-west corner (latitude, longitude, in whole degrees) of the tile a file name gives: +/// `N37W122.hgt` and `n37w122.hgt` are (37, −122); `S01E006.SRTMGL1.hgt` (the name NASA's +/// downloads have) is (−1, 6). `None` for a name that isn't a tile's, or one that lies outside +/// −90…90° and −180…180°. +pub fn tile_corner(file_name: &str) -> Option<(i32, i32)> { + let base = std::path::Path::new(file_name).file_name()?.to_str()?; + let tile = base.get(..7)?; + // The tile code ends the name or a `.` follows it (`None` when byte 7 is inside a character). + let rest = base.get(7..)?; + if !(rest.is_empty() || rest.starts_with('.')) { + return None; + } + let bytes = tile.as_bytes(); + let digits = |r: std::ops::Range| -> Option { + let s = tile.get(r)?; + if s.bytes().all(|b| b.is_ascii_digit()) { s.parse().ok() } else { None } + }; + let lat = digits(1..3)?; + let lon = digits(4..7)?; + let lat = match bytes.first()?.to_ascii_uppercase() { + b'N' => lat, + b'S' => -lat, + _ => return None, + }; + let lon = match bytes.get(3)?.to_ascii_uppercase() { + b'E' => lon, + b'W' => -lon, + _ => return None, + }; + // The tile spans a degree north and east of its corner. + ((-90..=89).contains(&lat) && (-180..=179).contains(&lon)).then_some((lat, lon)) +} + +/// Reads an SRTM tile: `bytes` are the file, `file_name` its name (which places it). +/// +/// One step that doesn't look at Cancel: the file is already in memory and is read in a single +/// pass over it. On the desktop it runs in Add Data's read-ahead, off the UI thread, and a Cancel +/// takes effect when it returns. It stays on one thread: turning big-endian words into numbers is +/// about 0.12 s for the largest (7201 × 7201) tile in a release build, less than reading its +/// 104 MB from disk, so threads would add little. +pub fn read(file_name: &str, bytes: &[u8]) -> Result { + read_limited(file_name, bytes, max_samples()) +} + +fn bad(m: String) -> DataError { + DataError::Format(format!("SRTM height tile: {m}")) +} + +/// Where a tile named `file_name` of `len` bytes lies and its side: its south-west corner +/// (latitude, longitude) and 1201, 3601 or 7201. Checked before a file is read, so a wrong one is +/// refused unread, saying why: a name that isn't a tile's, or a size that isn't one. +pub fn check(file_name: &str, len: u64) -> Result<(i32, i32, usize)> { + let (lat, lon) = tile_corner(file_name).ok_or_else(|| { + bad(format!( + "the file name {file_name:?} doesn't say where the tile is: it must start with its south-west corner, as N37W122.hgt does" + )) + })?; + let side = side_of(len).ok_or_else(|| { + bad(format!( + "{len} bytes isn't a tile's size: a tile is 1201 × 1201 heights (2,884,802 bytes), 3601 × 3601 (25,934,402 bytes) or 7201 × 7201 (103,708,802 bytes) ({})", + nearest_sizes(len) + )) + })?; + Ok((lat, lon, side)) +} + +/// How `len` bytes, no tile's size, compares with the tiles' sizes nearest it: smaller than the +/// smallest, the file may be cut short; else the tile it is larger than, and the one it is smaller +/// than, if any. +fn nearest_sizes(len: u64) -> String { + let bytes = |s: usize| (s * s * 2) as u64; + let below = SIDES.into_iter().rev().find(|s| bytes(*s) < len); + let above = SIDES.into_iter().find(|s| bytes(*s) > len); + match (below, above) { + (None, Some(s)) => format!("{} bytes fewer than a {s} × {s} tile; the file may be cut short", bytes(s) - len), + (Some(b), Some(a)) => { + format!("{} bytes more than a {b} × {b} tile, {} fewer than a {a} × {a} one", len - bytes(b), bytes(a) - len) + } + (Some(b), None) => format!("{} bytes more than a {b} × {b} tile", len - bytes(b)), + (None, None) => "no tile is near it".into(), + } +} + +/// [`read`] of a tile [`check`] has placed: `checked` is what it gave for the file's name and +/// size. The bytes are still held to that size (the file may have changed since), but the name +/// isn't parsed again. +pub fn read_checked(file_name: &str, checked: (i32, i32, usize), bytes: &[u8]) -> Result { + read_checked_limited(file_name, checked, bytes, max_samples()) +} + +/// [`read`], holding at most `max` samples (the tests' small limit exercises the reduction). +fn read_limited(file_name: &str, bytes: &[u8], max: usize) -> Result { + let checked = check(file_name, bytes.len() as u64)?; + read_checked_limited(file_name, checked, bytes, max) +} + +/// [`read_checked`], holding at most `max` samples. +fn read_checked_limited(file_name: &str, (lat, lon, side): (i32, i32, usize), bytes: &[u8], max: usize) -> Result { + if bytes.len() as u64 != (side * side * 2) as u64 { + return Err(bad(format!("the file is {} bytes now, not the {side} × {side} tile's it was", bytes.len()))); + } + let step = step_within(side, side, max).ok_or_else(|| bad("the tile is too large to open".into()))?; + let out = side.div_ceil(step); + let mut data = super::grid::(out, out)?; + // One pass: every sample's void counted, every `step`-th kept. + let mut voids = 0u64; + for (row, line) in bytes.as_chunks::<2>().0.chunks(side).enumerate() { + let keep = row % step == 0; + for (col, raw) in line.iter().enumerate() { + voids += u64::from(*raw == VOID.to_be_bytes()); + if keep && col % step == 0 { + data.push(f32::from(i16::from_be_bytes(*raw))); + } + } + } + let d = 1.0 / (side - 1) as f64; + let transform = post_transform(f64::from(lon), f64::from(lat), side, d, d, step); + let name = std::path::Path::new(file_name).file_stem().and_then(|s| s.to_str()).unwrap_or("srtm").to_string(); + let (w, h) = (out as u32, out as u32); + let mut r = Raster::new(name, w, h, vec![data], transform, Crs::LonLat, Some(f64::from(VOID)), ColorHint::Gray, "Int16")?; + if step > 1 { + // The file's own size; each cell read stands for exactly `step` samples (see `dted::read`). + r.overview = Some((side as u32, side as u32)); + r.step = step as u32; + } + r.lost.push(vertical_note("the EGM96 geoid (mean sea level)")); + r.lost.extend(void_note(voids, (side * side) as u64, VOID)); + Ok(r) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A 1201 × 1201 tile whose height at (row, col) is `f(row, col)`. + fn tile(f: impl Fn(usize, usize) -> i16) -> Vec { + let mut b = Vec::with_capacity(1201 * 1201 * 2); + for row in 0..1201 { + for col in 0..1201 { + b.extend_from_slice(&f(row, col).to_be_bytes()); + } + } + b + } + + #[test] + fn names_give_the_south_west_corner() { + assert_eq!(tile_corner("N37W122.hgt"), Some((37, -122))); + assert_eq!(tile_corner("/d/n37w122.HGT"), Some((37, -122))); + assert_eq!(tile_corner("S01E006.SRTMGL1.hgt"), Some((-1, 6))); + assert_eq!(tile_corner("S90W180.hgt"), Some((-90, -180))); + assert_eq!(tile_corner("N00E000.hgt"), Some((0, 0))); + for bad in [ + "N90E000.hgt", + "N00E180.hgt", + "S91E000.hgt", + "X37W122.hgt", + "N37Q122.hgt", + "N3xW122.hgt", + "N37W1222.hgt", + "srtm.hgt", + "N37W12.hgt", + "", + "N37W122x.hgt", + "N+7W122.hgt", + "ÑÑÑÑ.hgt", + "N37W122é.hgt", + "N37W122é", + ] { + assert_eq!(tile_corner(bad), None, "{bad}"); + } + } + + #[test] + fn reads_a_tile_in_place() { + let bytes = tile(|row, col| match (row, col) { + (0, 0) => 100, + (1200, 0) => -5, + (0, 1200) => 7, + (600, 600) => VOID, + (600, 601) => VOID, + _ => (row % 100) as i16, + }); + let d = 1.0 / 1200.0; + for (name, west, south) in + [("N37W122.hgt", -122.0, 37.0), ("S12E045.hgt", 45.0, -12.0), ("n00e000.hgt", 0.0, 0.0), ("S01W001.hgt", -1.0, -1.0)] + { + let r = read(name, &bytes).unwrap(); + assert_eq!((r.width, r.height, r.crs, r.nodata), (1201, 1201, Crs::LonLat, Some(-32768.0)), "{name}"); + let t = r.transform; + // The cells reach half a cell past the whole-degree lines. + assert!((t.origin_x - (west - d / 2.0)).abs() < 1e-12, "{name}: {t:?}"); + assert!((t.origin_y - (south + 1.0 + d / 2.0)).abs() < 1e-12, "{name}: {t:?}"); + assert!((t.pixel_width - d).abs() < 1e-15 && (t.pixel_height + d).abs() < 1e-15); + // The north-west sample is at the corner's whole degrees, rows run north to south. + let nw = t.to_world(0.5, 0.5); + assert!((nw.x - west).abs() < 1e-9 && (nw.y - (south + 1.0)).abs() < 1e-9); + assert_eq!(r.value(0, 0, 0), Some(100.0)); + assert_eq!(r.value(0, 0, 1200), Some(-5.0), "the south-west sample"); + assert_eq!(r.value(0, 1200, 0), Some(7.0), "the north-east sample"); + assert_eq!(r.value(0, 600, 600), None, "a void is no data"); + assert_eq!(r.overview, None); + assert_eq!(r.lost.len(), 2, "{:?}", r.lost); + assert!(r.lost[0].contains("EGM96 geoid"), "{:?}", r.lost); + assert_eq!(r.lost[1], "2 of its 1442401 heights are voids (-32768), read as no data"); + } + // Tiles at the poles and the antimeridian open, their edges half a cell past the lines. + for name in ["N89E179.hgt", "S90W180.hgt"] { + let r = read(name, &bytes).unwrap_or_else(|e| panic!("{name}: {e}")); + assert!(!r.extent().is_empty(), "{name}"); + } + // With no voids, nothing is said of them; the vertical reference still is. + let r = read("N00E000.hgt", &tile(|_, _| 1)).unwrap(); + assert_eq!(r.lost.len(), 1, "{:?}", r.lost); + assert!(r.lost[0].contains("EGM96")); + } + + #[test] + fn wrong_sizes_and_names_are_refused() { + let bytes = tile(|_, _| 3); + let e = read("srtm.hgt", &bytes).unwrap_err().to_string(); + assert!(e.contains("doesn't say where the tile is"), "{e}"); + // Smaller than the smallest tile, the file may be cut short. + let e = read("N37W122.hgt", &bytes[..bytes.len() - 2]).unwrap_err().to_string(); + assert!( + e.contains("isn't a tile's size") && e.ends_with("(2 bytes fewer than a 1201 × 1201 tile; the file may be cut short)"), + "{e}" + ); + // Between sizes, the nearest are given, not a cut. + let mut longer = bytes.clone(); + longer.extend_from_slice(&[0, 0]); + let e = read("N37W122.hgt", &longer).unwrap_err().to_string(); + assert!(e.contains("isn't a tile's size") && e.contains("7201 × 7201 (103,708,802 bytes)"), "{e}"); + assert!( + e.ends_with("(2 bytes more than a 1201 × 1201 tile, 23049598 fewer than a 3601 × 3601 one)") && !e.contains("cut short"), + "{e}" + ); + let e = check("N37W122.hgt", 2 * 3601 * 3601 + 2).unwrap_err().to_string(); + assert!( + e.ends_with("(2 bytes more than a 3601 × 3601 tile, 77774398 fewer than a 7201 × 7201 one)") && !e.contains("cut short"), + "{e}" + ); + let e = check("N37W122.hgt", 2 * 7201 * 7201 - 2).unwrap_err().to_string(); + assert!(e.contains("2 fewer than a 7201 × 7201 one") && !e.contains("cut short"), "{e}"); + let e = check("N37W122.hgt", 2 * 7201 * 7201 + 2).unwrap_err().to_string(); + assert!(e.ends_with("(2 bytes more than a 7201 × 7201 tile)") && !e.contains("cut short"), "{e}"); + let e = check("N37W122.hgt", 0).unwrap_err().to_string(); + assert!(e.ends_with("(2884802 bytes fewer than a 1201 × 1201 tile; the file may be cut short)"), "{e}"); + assert_eq!(check("N37W122.hgt", 2 * 3601 * 3601).unwrap(), (37, -122, 3601)); + // 1200 × 1200 or 1201 × 1200 would be guesses. + assert!(read("N37W122.hgt", &vec![0; 2 * 1200 * 1200]).is_err()); + assert!(read("N37W122.hgt", &vec![0; 2 * 1201 * 1200]).is_err()); + } + + #[test] + fn a_tile_past_the_budget_is_read_at_every_other_sample() { + let bytes = tile(|row, col| (row * 2 + col % 7) as i16); + let r = read_limited("N10E020.hgt", &bytes, 1201 * 1201 - 1).unwrap(); + assert_eq!((r.width, r.height, r.overview, r.step), (601, 601, Some((1201, 1201)), 2)); + assert_eq!(r.value(0, 3, 5), Some(26.0), "file row 10 × 2 + file column 6 % 7"); + // Cells are two samples wide, each centered on the sample kept: output (3, 5) is file + // column 6, row 10, at 20° + 6d east and 11° − 10d north. + let d = 1.0 / 1200.0; + assert!((r.transform.origin_x - (20.0 - d)).abs() < 1e-12); + assert!((r.transform.origin_y - (11.0 + d)).abs() < 1e-12); + assert!((r.transform.pixel_width - 2.0 * d).abs() < 1e-15); + let kept = r.transform.to_world(3.5, 5.5); + assert!((kept.x - (20.0 + 6.0 * d)).abs() < 1e-9 && (kept.y - (11.0 - 10.0 * d)).abs() < 1e-9, "{kept:?}"); + } + + #[test] + fn a_half_arc_second_tile_is_read_within_the_budget() { + // 7201 × 7201 at 0.5″: 51,854,401 samples, more than the browser holds, read at a step. + let side = 7201; + let mut bytes = vec![0u8; 2 * side * side]; + let mut put = |row: usize, col: usize, v: i16| { + let at = 2 * (row * side + col); + bytes[at..at + 2].copy_from_slice(&v.to_be_bytes()); + }; + put(0, 0, 11); + put(8 * 3, 8 * 5, 853); + put(7200, 7200, -4); + let r = read_limited("N45E006.hgt", &bytes, 1_000_000).unwrap(); + assert_eq!((r.width, r.height, r.overview, r.step), (901, 901, Some((7201, 7201)), 8)); + assert_eq!(r.value(0, 0, 0), Some(11.0)); + assert_eq!(r.value(0, 5, 3), Some(853.0)); + assert_eq!(r.value(0, 900, 900), Some(-4.0), "the south-east sample is kept"); + let d = 1.0 / 7200.0; + let t = r.transform; + assert!((t.pixel_width - 8.0 * d).abs() < 1e-15 && (t.pixel_height + 8.0 * d).abs() < 1e-15); + let kept = t.to_world(5.5, 3.5); + assert!((kept.x - (6.0 + 40.0 * d)).abs() < 1e-9 && (kept.y - (46.0 - 24.0 * d)).abs() < 1e-9, "{kept:?}"); + let se = t.to_world(900.5, 900.5); + assert!((se.x - 7.0).abs() < 1e-9 && (se.y - 45.0).abs() < 1e-9, "{se:?}"); + // Within the desktop's least budget it would be read whole. + assert_eq!(step_within(side, side, crate::raster::SAMPLES_FLOOR), Some(1)); + assert_eq!(check("N45E006.hgt", bytes.len() as u64).unwrap(), (45, 6, 7201)); + } + + #[test] + fn a_checked_tile_is_read_once_its_bytes_agree() { + let bytes = tile(|_, _| 9); + let checked = check("N37W122.hgt", bytes.len() as u64).unwrap(); + let r = read_checked("N37W122.hgt", checked, &bytes).unwrap(); + assert_eq!((r.width, r.value(0, 0, 0)), (1201, Some(9.0))); + // Bytes that no longer match the size checked are refused. + let e = read_checked("N37W122.hgt", checked, &bytes[..bytes.len() - 2]).unwrap_err().to_string(); + assert!(e.contains("not the 1201 × 1201 tile's it was"), "{e}"); + } + + #[test] + fn junk_is_refused() { + let bytes = tile(|r, c| (r ^ c) as i16); + // Prefixes, random bytes of a tile's size, and nothing never panic. + let mut seed = 0x2076u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + for _ in 0..64 { + let cut = (next() % bytes.len() as u64) as usize; + assert!(read("N37W122.hgt", &bytes[..cut]).is_err()); + } + let mut random = vec![0u8; bytes.len()]; + for _ in 0..4 { + random.iter_mut().for_each(|b| *b = next() as u8); + let r = read("N37W122.hgt", &random).unwrap(); + assert_eq!((r.width, r.height), (1201, 1201)); + } + assert!(read("N37W122.hgt", &[]).is_err()); + assert!(read("", &[]).is_err()); + } +} diff --git a/crates/data/src/raster/mod.rs b/crates/data/src/raster/mod.rs index 124158239e..7bd3ef0b89 100644 --- a/crates/data/src/raster/mod.rs +++ b/crates/data/src/raster/mod.rs @@ -10,6 +10,7 @@ pub mod ascii; pub mod cog; pub mod cogwrite; pub mod crs; +pub mod dted; pub(crate) mod epsg_axes; mod epsg_gap; mod esri; @@ -17,6 +18,7 @@ mod esri_datums; #[cfg(not(target_arch = "wasm32"))] pub mod gdal; pub mod geotiff; +pub mod hgt; mod image_grid; mod lerc; mod lzw; @@ -92,10 +94,66 @@ pub const SAMPLES_CEILING: usize = 1_000_000_000; pub fn max_samples() -> usize { crate::memory::scaled(crate::memory::physical(), SAMPLES_FLOOR, SAMPLES_CEILING, 8) } + +// Shared by the elevation readers ([`hgt`], [`dted`]) and pictures read at a step. + +/// The transform of a grid of posts (samples on grid lines) whose south-west post is at +/// (`west`, `south`), `cols` × `rows` of them `dx` and `dy` degrees apart, read every `step`-th +/// post from the north-west one: each post kept becomes a cell centered on it, `step` posts wide, +/// so the grid's edges lie half a cell beyond the outer posts kept. (At `step` 1, half a post's +/// spacing beyond the outer posts.) +fn post_transform(west: f64, south: f64, rows: usize, dx: f64, dy: f64, step: usize) -> GeoTransform { + let north = south + (rows.saturating_sub(1)) as f64 * dy; + let k = step.max(1) as f64; + let (cell_x, cell_y) = (dx * k, dy * k); + GeoTransform { origin_x: west - cell_x / 2.0, origin_y: north + cell_y / 2.0, pixel_width: cell_x, pixel_height: -cell_y } +} + +/// The step at which a `w` × `h` grid fits in `max` samples: 1, or the smallest power of two +/// whose every-`step`-th sample does ([`read_scan`] reduces pictures with it too). `None` +/// when none does, up to a step of 2^20. +fn step_within(w: usize, h: usize, max: usize) -> Option { + let mut k = 1usize; + while w.div_ceil(k).saturating_mul(h.div_ceil(k)) > max { + k = k.checked_mul(2)?; + if k > 1 << 20 { + return None; + } + } + Some(k) +} + +/// What a tile says of its heights' vertical reference, once a layer (a raster's `lost`): the +/// layer's coordinate system is horizontal only, so this is where it is said. +fn vertical_note(surface: &str) -> String { + format!( + "heights are meters above {surface}, not the WGS 84 ellipsoid; the layer's coordinate system doesn't carry a vertical datum, so compare it with ellipsoidal heights (GPS, many lidar files) only after a geoid correction" + ) +} + +/// What reading a grid's no-data posts says: `None` when there are none. +fn void_note(voids: u64, total: u64, value: i16) -> Option { + match voids { + 0 => None, + 1 => Some(format!("1 of its {total} heights is a void ({value}), read as no data")), + n => Some(format!("{n} of its {total} heights are voids ({value}), read as no data")), + } +} + /// Largest ASCII grid file read (1 GiB). pub const MAX_ASCII_BYTES: u64 = 1 << 30; -/// File extensions read as rasters. -pub const READ_EXTENSIONS: &[&str] = &["tif", "tiff", "asc", "png", "jpg", "jpeg"]; +/// File extensions read as rasters. SRTM height tiles (`.hgt`) and DTED (`.dt0`–`.dt2`) are +/// read natively ([`hgt`], [`dted`]), on the desktop and in the browser: the last four are +/// [`ELEVATION_EXTENSIONS`] (a test holds them to it). +pub const READ_EXTENSIONS: &[&str] = &["tif", "tiff", "asc", "png", "jpg", "jpeg", "hgt", "dt0", "dt1", "dt2"]; +/// The extensions of the elevation files read natively: SRTM height tiles and DTED levels 0–2. +pub const ELEVATION_EXTENSIONS: [&str; 4] = ["hgt", "dt0", "dt1", "dt2"]; + +/// Whether a file name is an SRTM height tile's or a DTED file's ([`ELEVATION_EXTENSIONS`]). +pub fn is_elevation_name(name: &str) -> bool { + let ext = Path::new(name).extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + ELEVATION_EXTENSIONS.contains(&ext.as_str()) +} /// Whether a file name is a raster this module reads. pub fn is_raster_name(name: &str) -> bool { @@ -996,8 +1054,9 @@ impl Raster { } /// How many of the file's pixels one pixel read spans, across and down: exactly `step` for - /// every `step`-th pixel, an overview's size over the file's, else 1. - fn file_pixels(&self) -> (f64, f64) { + /// every `step`-th pixel, an overview's size over the file's, else 1. (`overview` stays the + /// file's own size, which `step` needn't divide, so a reader at a step goes by `step`.) + pub fn file_pixels(&self) -> (f64, f64) { if self.step > 1 { return (f64::from(self.step), f64::from(self.step)); } @@ -1005,6 +1064,15 @@ impl Raster { (f64::from(full_w) / f64::from(self.width.max(1)), f64::from(full_h) / f64::from(self.height.max(1))) } + /// For other crates' tests: this raster as if read at every `step`-th pixel of a file of + /// `full` pixels, as the readers mark one read within the memory budget. + #[doc(hidden)] + pub fn read_at_step(mut self, step: u32, full: (u32, u32)) -> Raster { + self.step = step.max(1); + self.overview = Some(full); + self + } + /// A scan placed by its world file, in `crs`: georeferenced. The world file speaks of the /// file's pixels, so a scan read at a reduced size keeps its place with pixels as much /// larger. A rotated, skewed or mirrored one is resampled north up, as a rotated GeoTIFF is, @@ -1043,16 +1111,21 @@ impl Raster { old.stats = new.stats.clone(); *new = old; } - // A picture read at a reduced size still says so, as a turned GeoTIFF does: of the - // turned grid at the file's resolution, so the two sizes said are alike. - let full = |n: u32, s: f64| (f64::from(n) * s).round().clamp(1.0, f64::from(u32::MAX)) as u32; - r.overview = self.overview.map(|_| (full(ow, sx), full(oh, sy))); + // A picture read at a reduced size still says so, as a turned GeoTIFF does, with the + // size the whole file has turned at its own resolution (its corners turned, over a + // cell of its pixels' area, as `north_up` sizes a grid). + let [_, xc, xr, _, yc, yr] = wf.affine(); + let file_cell = (xc * yr - xr * yc).abs().sqrt(); + r.overview = self.overview.map(|(fw, fh)| turned_size([xc, xr, yc, yr], fw, fh)); + // Each turned cell is exactly `step` of the file's pixels across, unless the turned + // grid had to be made coarser to fit; then the sizes' ratio is the scale. + let exact = (r.transform.pixel_width.abs() / (f64::from(self.step) * file_cell) - 1.0).abs() < 1e-9; + r.step = if exact { self.step } else { 1 }; (r.source, r.lost, r.copyright) = (self.source, self.lost, self.copyright); (r.rounded, r.rounded_some) = (self.rounded, self.rounded_some); r }; r.georeferenced = true; - r.step = 1; Ok(r) } @@ -1205,6 +1278,20 @@ fn read_path_placing(path: &Path, world: bool) -> Result { let text = crate::provenance::read_to_string(path)?; read_ascii(path, &text, &name, sidecar_nodata)? } + // Elevation, read whole once its name and size are checked: an SRTM tile's size says its + // grid, and a DTED file is no larger than its largest cell. + e if ELEVATION_EXTENSIONS.contains(&e) => { + let file_name = path.file_name().and_then(|n| n.to_str()).unwrap_or_default(); + if ext == "hgt" { + let checked = hgt::check(file_name, meta.len())?; + hgt::read_checked(file_name, checked, &crate::provenance::read_at_most(path, meta.len())?)? + } else { + if meta.len() > dted::MAX_BYTES { + return Err(DataError::Format("DTED: the file is larger than any DTED cell".into())); + } + dted::read(file_name, &crate::provenance::read_at_most(path, meta.len())?)? + } + } #[cfg(not(target_arch = "wasm32"))] _ if gdal::is_gdal_raster_name(&path.to_string_lossy()) => { // Converted once to a GeoTIFF in the cache; the layer still names this file. @@ -1330,8 +1417,30 @@ fn read_path_with_placing(path: &Path, subdataset: Option<&str>, crs: Option<&st if (subdataset.is_some() || crs.is_some()) && gdal::is_gdal_raster_name(&path.to_string_lossy()) { return read_converted(path, &gdal::convert(path, subdataset, crs)?); } - let _ = (subdataset, crs); - read_path_placing(path, world) + let ignored = elevation_ignores(&path.to_string_lossy(), subdataset, crs); + let mut r = read_path_placing(path, world)?; + r.lost.extend(ignored); + Ok(r) +} + +/// What reading an SRTM tile or DTED file (`name`) with a subdataset (`"layer"`) or a coordinate +/// system (`"crs"`) says: they hold one band and give their own coordinates, so neither applies, +/// and the file is read without them. A command may give them, and so may a project saved when +/// these files went through GDAL. `None` for other files, or when neither is given. +pub fn elevation_ignores(name: &str, subdataset: Option<&str>, crs: Option<&str>) -> Option { + if !is_elevation_name(name) { + return None; + } + // A DTED file naming a datum other than WGS 84 says so as it is read (`dted::read`). + const PLACE: &str = "give their own coordinates, read as WGS 84 longitude/latitude (a DTED file naming another datum says so)"; + const BAND: &str = "hold one band of heights, with no layers to choose from"; + let given = |v: Option<&str>| v.is_some_and(|v| !v.trim().is_empty()); + Some(match (given(subdataset), given(crs)) { + (true, true) => format!("the \"layer\" and \"crs\" given are ignored: SRTM and DTED files {BAND}, and {PLACE}"), + (true, false) => format!("the \"layer\" given is ignored: SRTM and DTED files {BAND}"), + (false, true) => format!("the \"crs\" given is ignored: SRTM and DTED files {PLACE}"), + (false, false) => return None, + }) } /// Reads the GeoTIFF GDAL converted `path` to, named and sourced as `path` (and its subdataset). @@ -1443,13 +1552,9 @@ fn read_scan_limited(name: &str, mut source }; let (w, h) = (img.width(), img.height()); let ch = usize::from(img.color().channel_count()); - let mut k = 1u32; - while (w.div_ceil(k) as usize).saturating_mul(h.div_ceil(k) as usize).saturating_mul(ch) > max { - k = k.saturating_mul(2); - if k > 1 << 20 { - return Err(too_large()); - } - } + // The step an elevation tile is read at, the budget shared by the channels: a × ch > max + // exactly when a > ⌊max / ch⌋. + let k = step_within(w as usize, h as usize, max / ch.max(1)).and_then(|k| u32::try_from(k).ok()).ok_or_else(too_large)?; let (bands, color, data_type) = match &img { D::ImageLuma8(i) => (deinterleave(i.as_raw(), w, h, ch, k)?, ColorHint::Gray, "UInt8"), D::ImageLumaA8(i) => (deinterleave(i.as_raw(), w, h, ch, k)?, ColorHint::GrayAlpha, "UInt8"), @@ -1525,6 +1630,20 @@ fn rotate(r: &Raster, degrees: f64) -> Result { Raster::new(r.name.clone(), ow, oh, bands, transform, r.crs, None, r.color.clone(), r.data_type.clone()) } +/// The size, in cells of a pixel's area, of the north-up box around `w`×`h` pixels placed by +/// `[x per column, x per row, y per column, y per row]`, as [`north_up`] sizes its grid. +fn turned_size([xc, xr, yc, yr]: [f64; 4], w: u32, h: u32) -> (u32, u32) { + let cell = (xc * yr - xr * yc).abs().sqrt(); + let (fw, fh) = (f64::from(w), f64::from(h)); + let span = |a: f64, b: f64| (0.0f64).max(a).max(b).max(a + b) - (0.0f64).min(a).min(b).min(a + b); + // A thousandth of a cell of slack, so a size that is whole on paper isn't rounded up by noise. + let n = |len: f64| ((len / cell) - 1e-3).ceil().clamp(1.0, f64::from(u32::MAX)) as u32; + if !cell.is_normal() { + return (w.max(1), h.max(1)); + } + (n(span(fw * xc, fh * xr)), n(span(fw * yc, fh * yr))) +} + /// A `w`×`h` image placed by a rotated or skewed `affine` (GDAL's order: x, x per column, x per /// row, y, y per column, y per row), resampled (nearest pixel) to the north-up grid of square /// pixels over its bounding box: pixels of the same area, coarser if that would hold more than @@ -1625,6 +1744,9 @@ pub fn read_bytes(file_name: &str, bytes: &[u8]) -> Result { let text = std::str::from_utf8(bytes).map_err(|_| DataError::Format("not a text file".into()))?; ascii::read(text, &name, None) } + // An upload keeps its file name, which places an SRTM tile. + "hgt" => hgt::read(file_name, bytes), + e if ELEVATION_EXTENSIONS.contains(&e) => dted::read(file_name, bytes), other => Err(DataError::Unsupported(format!(".{other}"))), } } @@ -1799,6 +1921,16 @@ fn extent_of(transform: &GeoTransform, transformer: &Transformer, width: u32, he pub(crate) mod tests { use super::*; + #[test] + fn elevation_files_are_raster_names() { + for ext in ELEVATION_EXTENSIONS { + assert!(READ_EXTENSIONS.contains(&ext), "{ext}"); + assert!(is_raster_name(&format!("/d/N37W122.{}", ext.to_uppercase())), "{ext}"); + assert!(is_elevation_name(&format!("x.{ext}")), "{ext}"); + } + assert!(!is_elevation_name("x.tif") && !is_elevation_name("hgt") && !is_elevation_name("")); + } + /// A large band's statistics, counted on any number of threads, are those of one pass on /// one thread (#1252) to the bit: the sums in the band's order, the counts, range, histogram /// and distinct values whatever the runs. No-data, NaN, −0 and 0 (the first kept), byte-like @@ -2440,6 +2572,18 @@ mod scan_tests { use super::*; use std::io::Cursor; + #[test] + fn a_turned_size_is_the_grid_north_up_makes() { + let (w, h) = (40u32, 17u32); + let band = vec![1.0f32; (w * h) as usize]; + for degrees in [0.0f64, 30.0, 90.0, 135.0, 200.0, 271.0] { + let (sin, cos) = degrees.to_radians().sin_cos(); + let (xc, xr, yc, yr) = (2.5 * cos, 2.5 * sin, 2.5 * sin, -2.5 * cos); + let (_, ow, oh, _) = north_up(&[band.as_slice()], w, h, [100.0, xc, xr, 50.0, yc, yr], usize::MAX).unwrap(); + assert_eq!(turned_size([xc, xr, yc, yr], w, h), (ow, oh), "{degrees}°"); + } + } + fn tiff(tags: &[(u16, &[f64])]) -> Vec { use tiff::encoder::{TiffEncoder, colortype}; let mut out = Cursor::new(Vec::new()); diff --git a/crates/data/src/raster/world.rs b/crates/data/src/raster/world.rs index af2a5d9206..3f6033644a 100644 --- a/crates/data/src/raster/world.rs +++ b/crates/data/src/raster/world.rs @@ -372,11 +372,20 @@ mod tests { image::GrayImage::from_pixel(101, 3, image::Luma([9])) .write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png) .unwrap(); - let r = super::super::read_scan_limited("p", std::io::Cursor::new(png), 300).unwrap(); + let r = super::super::read_scan_limited("p", std::io::Cursor::new(png.clone()), 300).unwrap(); assert_eq!((r.width, r.height, r.step, r.overview), (51, 2, 2, Some((101, 3)))); let wf = WorldFile::parse("1\n0\n0\n-1\n0.5\n9.5\n").unwrap(); let r = r.placed_by_world_file(&wf, crate::raster::Crs::LonLat).map_err(|(_, e)| e).unwrap(); assert_eq!((r.transform.origin_x, r.transform.origin_y, r.transform.pixel_width, r.transform.pixel_height), (0.0, 10.0, 2.0, -2.0)); + assert_eq!((r.step, r.overview, r.file_pixels()), (2, Some((101, 3)), (2.0, 2.0))); + // Turned a quarter, it is resampled north up at the reduced size: 2 × 51 cells, each two of + // the file's pixels. The size said is the whole file's turned, 3 × 101, and a later warp + // still scales its cells by exactly 2 (not 101/51). + let r = super::super::read_scan_limited("p", std::io::Cursor::new(png), 300).unwrap(); + let wf = WorldFile::parse("0\n-1\n-1\n0\n11.5\n49.5\n").unwrap(); + let r = r.placed_by_world_file(&wf, crate::raster::Crs::LonLat).map_err(|(_, e)| e).unwrap(); + assert_eq!((r.width, r.height, r.step, r.overview), (2, 51, 2, Some((3, 101)))); + assert_eq!(r.file_pixels(), (2.0, 2.0)); } #[test] diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 95aaec9d67..bfe618c9db 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -1695,7 +1695,9 @@ fn add_data(s: &mut Session, p: &Value) -> CmdResult { return add_photos(s, &json!({"paths": [path], "name": name})); } if rustgis_data::raster::is_raster_name(path) { - return super::raster::add_path(s, path, ps(p, "name").map(str::trim).filter(|n| !n.is_empty()), scan); + // An SRTM tile or DTED file has no subdataset or coordinate system to choose: said. + let ignored = rustgis_data::raster::elevation_ignores(path, ps(p, "layer"), ps(p, "crs")); + return super::raster::add_path(s, path, ps(p, "name").map(str::trim).filter(|n| !n.is_empty()), scan, ignored); } // A Zarr store's folder: its first variable, or the one "layer" names. #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/engine/src/cmd/raster.rs b/crates/engine/src/cmd/raster.rs index 263e3ff04f..46b8e139a8 100644 --- a/crates/engine/src/cmd/raster.rs +++ b/crates/engine/src/cmd/raster.rs @@ -34,7 +34,7 @@ pub(crate) fn need_raster(s: &Session) -> Option<&'static str> { pub fn commands() -> Vec { vec![ C::new("layer.addDataBytes", "Add Data (from file contents)", "Home › Add", add_bytes).roles(&[]).effects(crate::policy::Effects::NONE) - .params(r#"{"fileName": "name.tif|.tiff|.asc|.geojson|.csv|.zip|.kml|.kmz|.gpx|.parquet|.fgb|.gpkg|.mbtiles|.jp2|.j2k|.dxf (JPEG 2000: desktop, through GDAL)", "data": base64, "zoom"?: bool, "layer"?: "one layer of a zip, KML or GPX by name", "name"?: text, "x"?, "y"?, "wkt"?, "crs"?, "delimiter"?, "paperSpace"?, "blocks"?, "stream"?, "as"? (as layer.addData)}"#), + .params(r#"{"fileName": "name.tif|.tiff|.asc|.hgt|.dt0|.dt1|.dt2|.geojson|.csv|.zip|.kml|.kmz|.gpx|.parquet|.fgb|.gpkg|.mbtiles|.jp2|.j2k|.dxf (JPEG 2000: desktop, through GDAL)", "data": base64, "zoom"?: bool, "layer"?: "one layer of a zip, KML or GPX by name", "name"?: text, "x"?, "y"?, "wkt"?, "crs"?, "delimiter"?, "paperSpace"?, "blocks"?, "stream"?, "as"? (as layer.addData)}"#), C::new("raster.symbology", "Symbology", "Style › Rendering", symbology).when(need_raster).roles(&[]).effects(crate::policy::Effects::NONE).params( r#"{"layer"?, "type": "stretch"|"rgb"|"classify"|"unique"|"colormap"|"shadedRelief", "band"?: 1.., "ramp"?, "invert"?: bool, "classes"?: 1..32, "method"?, "azimuth"?, "altitude"?, "zFactor"?, "shadow"?: 0..1 (shadedRelief: how dark the shadows the terrain casts are; left out, kept; 0 casts none), "sky"?: 0..1 (shadedRelief: how much the share of the sky each cell sees dims its ambient light; left out, kept; 0 leaves it whole), "fill"?: 0..1 (shadedRelief: a fill light from the side opposite the sun that lightens the slopes turned toward it; left out, kept; 0 has none), "ambient"?: 0..1|null (shadedRelief: the share of the colour a slope turned fully from the sun keeps; left out, kept; null, the default: 0.25, or 0.5 balanced), "balanced"?: bool (shadedRelief: flat ground keeps the ramp's colour and slopes facing the sun brighten, up to 1.3; left out, kept; false, the default, lights as before), "rock"?: {"from"?: 0..90, "to"?: 0..90, "color"?: #RRGGBB} | false (shadedRelief: steep slopes tinted toward a rock colour before the light, from the ground's own slope in degrees, without the Z factor: the ramp below from, the colour above to, blended between; left out, kept; an object's missing keys keep the current rock's, or 25, 40 and #B2ACA4; false or null removes it), "grey"?: bool (shadedRelief without its colour ramp)}"#, ), @@ -561,12 +561,14 @@ pub(crate) fn say_losses(s: &mut Session, losses: Vec<(String, String)>) { } } -/// Opens a raster file by path. -pub fn add_path(s: &mut Session, path: &str, name: Option<&str>, scan: bool) -> CmdResult { +/// Opens a raster file by path; `ignored` is said with what reading it lost (options given that +/// don't apply to it). +pub fn add_path(s: &mut Session, path: &str, name: Option<&str>, scan: bool, ignored: Option) -> CmdResult { let mut r = match super::reads::take(s, &read_key(path, scan)) { Some(r) => r, None => read_raster_as(path, scan), }?; + r.lost.extend(ignored); if scan { r = r.into_scan().map_err(|e| format!("{path}: {e}"))?; } @@ -691,6 +693,7 @@ pub(crate) fn add_named_bytes(s: &mut Session, p: &Value, name: String, bytes: V } if raster::is_raster_name(&name) { let mut r = raster::read_bytes(&name, &bytes).map_err(|e| format!("{name}: {e}"))?; + r.lost.extend(raster::elevation_ignores(&name, ps(p, "layer"), ps(p, "crs"))); if scan { r = r.into_scan().map_err(|e| format!("{name}: {e}"))?; } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 353e541141..6587a87bd4 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -38,6 +38,9 @@ mod service_logins; #[path = "tests/scopes_and_redaction.rs"] mod scopes_and_redaction; +#[path = "tests/elevation_tiles.rs"] +mod elevation_tiles; + /// Where the tests' plugins folder keeps what plugins remember: not the user's. fn plugin_settings_dir() -> std::path::PathBuf { rustgis_plugins::store::dir().unwrap().join("settings") diff --git a/crates/engine/src/tests/elevation_tiles.rs b/crates/engine/src/tests/elevation_tiles.rs new file mode 100644 index 0000000000..89096f558a --- /dev/null +++ b/crates/engine/src/tests/elevation_tiles.rs @@ -0,0 +1,132 @@ +//! SRTM height tiles and DTED, added natively by path and as uploads (#2076). +use super::*; + +/// A 1201 × 1201 SRTM tile: heights by row, one void. +fn hgt_tile() -> Vec { + let mut b = Vec::with_capacity(1201 * 1201 * 2); + for row in 0..1201usize { + for col in 0..1201usize { + let v: i16 = if (row, col) == (5, 5) { i16::MIN } else { (row % 300) as i16 }; + b.extend_from_slice(&v.to_be_bytes()); + } + } + b +} + +/// A DTED level 1 cell of 1201 × 1201 posts at 3″ from 6°E, 45°S (MIL-PRF-89020B: UHL, DSI, ACC, +/// then one record a longitude line). +fn dt1_cell() -> Vec { + let mut b = format!("UHL10060000E0450000S00300030NA U {:12}120112010{:24}", "", "").into_bytes(); + let mut dsi = vec![b' '; 648]; + dsi[..3].copy_from_slice(b"DSI"); + dsi[141..144].copy_from_slice(b"E96"); + dsi[144..149].copy_from_slice(b"WGS84"); + b.extend(dsi); + let mut acc = vec![b' '; 2700]; + acc[..3].copy_from_slice(b"ACC"); + b.extend(acc); + for i in 0..1201u16 { + let mut rec = vec![0xaa, 0, (i >> 8) as u8, i as u8]; + rec.extend(i.to_be_bytes()); + rec.extend([0, 0]); + for j in 0..1201u16 { + // Negative heights in signed magnitude. + let v = if j % 2 == 0 { j % 100 } else { 0x8000 | (j % 100) }; + rec.extend(v.to_be_bytes()); + } + let sum: u32 = rec.iter().map(|b| u32::from(*b)).sum(); + rec.extend(sum.to_be_bytes()); + b.extend(rec); + } + b +} + +/// The layer `result` names: its raster's extent and pixel size. +fn placed(s: &Session, result: &Value) -> (rustgis_data::Extent, u32, u32) { + let layer = s.map().layer(result["layer"].as_u64().unwrap() as u32).unwrap(); + let r = layer.raster.as_ref().unwrap().data.as_ref().unwrap(); + assert!(r.georeferenced); + assert_eq!(r.crs, rustgis_data::raster::Crs::LonLat); + (r.extent(), r.width, r.height) +} + +fn near(e: rustgis_data::Extent, [w, s, east, n]: [f64; 4]) -> bool { + [(e.min_x, w), (e.min_y, s), (e.max_x, east), (e.max_y, n)].iter().all(|(a, b)| (a - b).abs() < 1e-9) +} + +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn srtm_and_dted_are_added_by_path_with_their_extent() { + let dir = std::env::temp_dir().join(format!("rustgis-elevation-tiles-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let half = 1.5 / 3600.0; + for (name, bytes, bbox) in [ + ("N37W122.hgt", hgt_tile(), [-122.0 - half, 37.0 - half, -121.0 + half, 38.0 + half]), + ("S45E006.dt1", dt1_cell(), [6.0 - half, -45.0 - half, 7.0 + half, -44.0 + half]), + ] { + let path = dir.join(name); + std::fs::write(&path, bytes).unwrap(); + let mut s = Session::new(Project::default()); + let result = s.run("layer.addData", &json!({"path": path})).unwrap(); + let (extent, w, h) = placed(&s, &result); + assert_eq!((w, h), (1201, 1201), "{name}"); + assert!(near(extent, bbox), "{name}: {extent:?}"); + // The vertical reference is said once, as the layer is added. + assert_eq!(s.notices.iter().filter(|n| n.text.contains("EGM96 geoid")).count(), 1, "{name}: {:?}", s.notices); + let voids = s.notices.iter().filter(|n| n.text.contains("void")).count(); + assert_eq!(voids, usize::from(name.ends_with(".hgt")), "{name}: {:?}", s.notices); + } + // A tile's name places it: one that doesn't is refused, saying so, not handed to GDAL. + let unnamed = dir.join("srtm.hgt"); + std::fs::write(&unnamed, hgt_tile()).unwrap(); + let e = Session::new(Project::default()).run("layer.addData", &json!({"path": unnamed})).unwrap_err().to_string(); + assert!(e.contains("doesn't say where the tile is"), "{e}"); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn an_srtm_upload_keeps_its_name_and_place() { + use base64::Engine; + let data = base64::engine::general_purpose::STANDARD.encode(hgt_tile()); + let mut s = Session::new(Project::default()); + let result = s.run("layer.addDataBytes", &json!({"fileName": "S01E006.hgt", "data": data})).unwrap(); + let (extent, _, _) = placed(&s, &result); + let half = 1.5 / 3600.0; + assert!(near(extent, [6.0 - half, -1.0 - half, 7.0 + half, 0.0 + half]), "{extent:?}"); + assert!(s.notices.iter().any(|n| n.text.contains("1 of its 1442401 heights is a void")), "{:?}", s.notices); + // A cut-short upload is refused. + let cut = base64::engine::general_purpose::STANDARD.encode(&hgt_tile()[..1000]); + assert!(Session::new(Project::default()).run("layer.addDataBytes", &json!({"fileName": "S01E006.hgt", "data": cut})).is_err()); +} + +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn a_crs_or_subdataset_given_for_srtm_or_dted_is_said_and_ignored() { + use base64::Engine; + let dir = std::env::temp_dir().join(format!("rustgis-elevation-options-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let hgt = dir.join("N37W122.hgt"); + std::fs::write(&hgt, hgt_tile()).unwrap(); + let dt1 = dir.join("S45E006.dt1"); + std::fs::write(&dt1, dt1_cell()).unwrap(); + // As a project saved when these went through GDAL reopens them: read, the options said. + let r = rustgis_data::raster::read_path_with(&hgt, Some("band2"), Some("EPSG:32610")).unwrap(); + assert_eq!((r.width, r.crs), (1201, rustgis_data::raster::Crs::LonLat)); + let both = "the \"layer\" and \"crs\" given are ignored: SRTM and DTED files hold one band of heights, with no layers to choose from, and give their own coordinates, read as WGS 84 longitude/latitude (a DTED file naming another datum says so)"; + assert!(r.lost.iter().any(|l| l == both), "{:?}", r.lost); + let r = rustgis_data::raster::read_path_with(&dt1, None, Some("EPSG:4326")).unwrap(); + let crs = "the \"crs\" given is ignored: SRTM and DTED files give their own coordinates, read as WGS 84 longitude/latitude (a DTED file naming another datum says so)"; + assert!(r.lost.iter().any(|l| l == crs), "{:?}", r.lost); + let r = rustgis_data::raster::read_path_with(&dt1, None, None).unwrap(); + assert!(!r.lost.iter().any(|l| l.contains("ignored")), "{:?}", r.lost); + // From a command, by path and as an upload. + let mut s = Session::new(Project::default()); + s.run("layer.addData", &json!({"path": hgt, "layer": "x"})).unwrap(); + let layer = "the \"layer\" given is ignored: SRTM and DTED files hold one band of heights, with no layers to choose from"; + assert!(s.notices.iter().any(|n| n.text.ends_with(layer)), "{:?}", s.notices); + let data = base64::engine::general_purpose::STANDARD.encode(dt1_cell()); + let mut s = Session::new(Project::default()); + s.run("layer.addDataBytes", &json!({"fileName": "S45E006.dt1", "data": data, "crs": "EPSG:3857"})).unwrap(); + assert!(s.notices.iter().any(|n| n.text.ends_with(crs)), "{:?}", s.notices); + std::fs::remove_dir_all(dir).unwrap(); +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index dbe9edd120..85248b97fa 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -4094,6 +4094,8 @@ impl RustGisApp { "tiff", "asc", ][..], + // SRTM height tiles and DTED, read natively. + &rustgis_engine::data::raster::ELEVATION_EXTENSIONS[..], // Rasters GDAL reads (every one it is asked to read). #[cfg(not(target_arch = "wasm32"))] rustgis_engine::data::raster::gdal::EXTENSIONS, diff --git a/crates/ui-egui/src/panes.rs b/crates/ui-egui/src/panes.rs index 232528e578..6881e2a7ee 100644 --- a/crates/ui-egui/src/panes.rs +++ b/crates/ui-egui/src/panes.rs @@ -2786,6 +2786,8 @@ fn folder_browser(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens) { "rgislayout", ] .iter() + // SRTM height tiles and DTED. + .chain(rustgis_engine::data::raster::ELEVATION_EXTENSIONS.iter()) .any(|x| n.to_lowercase().ends_with(&format!(".{x}")))) }) .collect(); diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 8ed3c3b657..d68d9dea69 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -597,7 +597,9 @@ databases](data.md#sql-queries-and-duckdb-databases). | Scans with no coordinates | an untagged `.tif` or `.png`; any `.tif`, `.png` or `.jpg`/`.jpeg` with `"as": "scan"` | Add Data, or upload (`layer.addData`, `layer.addDataBytes`) | Grey or RGB(A), 8- or 16-bit pixels. An untagged TIFF or PNG opens as a scan by default; a JPEG needs `"as": "scan"` (otherwise it is a photo). `"as": "scan"` also discards a GeoTIFF's coordinates. A world file beside the picture (`.tfw`, `.jgw`, `.pgw`, the four-letter `.tifw`, `.jpgw`, `.pngw`, or `.wld`), with a `.prj` beside it for the coordinate system, places it instead (a JPEG too, which is then not a photo): a turned or skewed one is resampled north up; without a `.prj`, with one RustGIS can't use, or with a world file that can't be read or places the picture nowhere it can be drawn (UTM meters with a longitude/latitude `.prj`), it stays a scan and a notice says why. `"as": "scan"` passes the world file over. World files are read for files added by their path (an upload doesn't bring the files beside it). A project reopens a picture as it was added: one that opened as a scan stays one (add it again once its `.prj` is there), and one a world file placed still opens if its `.prj` or world file is gone, at a stand-in place, with a notice saying it has no coordinates now. A world file is compared with a GeoTIFF's own tags only when both are north up (a turned GeoTIFF is resampled as it is read). A GeoTIFF keeps its own tags, and a notice says when a world file places it differently. The scan initially covers the current view and is marked not georeferenced (`georeferenced: false`) until **Georeference** places it. Export and tools needing ground coordinates refuse it until then. PNG/JPEG decoding is bounded and may reduce the image; large TIFFs can be read at reduced resolution. | | ASCII grid | `.asc` (with an optional `.prj` on desktop) | Add Data, or upload | A single numeric band with square `CELLSIZE` or separate `DX` and `DY` pixel sizes. Desktop files use the neighbouring `.prj`; uploaded and downloaded grids have no sidecar and must have coordinates that fit WGS 84 longitude/latitude. A grid with no `.prj` is read as WGS 84, and a notice says so. | | Zarr | a store's folder (`.zarr`, desktop) or web address (desktop and browser), format 2 (consolidated) or 3 | Home › Add › Add Data › Zarr… (`layer.addZarr`), Add Data on a local folder, or a `?zarr=` link | A variable's latitude/longitude (or y/x) map at the steps picked of its other dimensions (time, level), read chunk by chunk in pure Rust (blosc, zlib, gzip, zstd, LZ4; sharded arrays too). See [Zarr stores](data.md#zarr-stores). | -| Rasters GDAL reads | `.img`, `.hgt`, `.dem`, `.dt0`–`.dt2`, `.nc`, `.nc4`, `.hdf`, `.hdf5`, `.h5`, `.he5`, `.bil`, `.bsq`, `.bip`, `.envi`, `.grd`, `.rst`, `.sdat`, `.kea`, `.ers`, `.ntf`, `.nitf`, `.ecw`, `.sid`, `.gsb`, `.bt`, `.rsw` | Add Data (desktop, with [GDAL](https://gdal.org): `gdal_translate`, `gdalinfo` and `gdalwarp`; the installed build must include the driver) | Converted once to a cached cloud-optimised GeoTIFF. NetCDF/HDF variables are subdatasets: the first is added, the others named in a notice; `"layer"` picks one. Without a declared coordinate system, longitude/latitude-like coordinates use WGS 84 (0–360° longitudes are moved to −180–180°) and a notice says so, every time the file is read, not only when it is added; otherwise give `"crs"`. JPEG 2000 is listed separately above. | +| SRTM height tile | `.hgt` | Add Data, or upload (desktop and browser) | One 1° tile of 16-bit heights in meters, 1201 × 1201 (3 arc-seconds), 3601 × 3601 (1 arc-second) or 7201 × 7201 (0.5 arc-second), read natively; a tile larger than the memory budget (a 0.5 arc-second tile in the browser) is read at every second sample or fewer, each kept sample still at its cell's center. The file name places it (`N37W122.hgt` is the tile whose south-west corner is 37°N, 122°W; an upload keeps its name), so a file whose name isn't a tile's, or whose size is neither grid's, is refused. Samples sit on the whole-degree lines, so each is a cell centered on its point and the layer reaches half a cell past the tile's edges. WGS 84; voids (−32768) are no data, and a notice counts them. Heights are above the EGM96 geoid (mean sea level), not the ellipsoid, and a notice says so once as the layer is added. SRTM and DTED files hold one band and give their own coordinates (read as WGS 84; a DTED file naming another datum says so), so a `"crs"` or `"layer"` given for one (or kept by a project saved when these files went through GDAL) is ignored, and a notice says so. | +| DTED | `.dt0`, `.dt1`, `.dt2` | Add Data, or upload (desktop and browser) | DTED levels 0, 1 and 2, read natively per MIL-PRF-89020B: the UHL header gives the south-west post, the spacing (wider in longitude at high latitudes) and the counts. Posts are cells centered on them, as for SRTM. Voids (−32767) are no data and counted in a notice. A column whose record lacks its 0xAA sentinel is read as no data, and a notice counts those columns; a record whose longitude or latitude count doesn't match its place in the file (a producer numbering from 1, say) is read by its place, and a notice counts those; a column whose checksum doesn't match its heights (some producers write them wrongly) is read as stored, and a notice counts those too. Heights some producers write in two's complement rather than signed magnitude (those that would read below −16000 m) are read as two's complement, and counted in a notice. A complete cell cut short opens with its missing columns as no data, and says so. A partial cell is read in the full-grid layout of CD-ROM files, its missing posts as voids; one in the magnetic-tape layout is refused when its DSI counts only the lines that have heights. A tape cell with a height on every line counts the whole cell too, but its file is shorter than the full grid's, so it's refused as well; a partial cell cut short is refused the same way, since the two can't be told apart. The level is the one the DSI record names (the extension's when it names none, and a notice says when the two differ). A spacing other than the specification's for the level and latitude, a DSI record that disagrees with the UHL, or a horizontal datum other than WGS 84 is said. The vertical datum (EGM96, or mean sea level) is said once, as for SRTM. | +| Rasters GDAL reads | `.img`, `.dem`, `.nc`, `.nc4`, `.hdf`, `.hdf5`, `.h5`, `.he5`, `.bil`, `.bsq`, `.bip`, `.envi`, `.grd`, `.rst`, `.sdat`, `.kea`, `.ers`, `.ntf`, `.nitf`, `.ecw`, `.sid`, `.gsb`, `.bt`, `.rsw` | Add Data (desktop, with [GDAL](https://gdal.org): `gdal_translate`, `gdalinfo` and `gdalwarp`; the installed build must include the driver) | Converted once to a cached cloud-optimised GeoTIFF. NetCDF/HDF variables are subdatasets: the first is added, the others named in a notice; `"layer"` picks one. Without a declared coordinate system, longitude/latitude-like coordinates use WGS 84 (0–360° longitudes are moved to −180–180°) and a notice says so, every time the file is read, not only when it is added; otherwise give `"crs"`. JPEG 2000 is listed separately above. | For GDAL subdatasets, the notice counts the other variables, names at most the first 20, and counts any names omitted from that list. diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index cf1c08ddfa..7bdcc330ee 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -26,8 +26,9 @@ Formats](data-formats.md). | DuckDB databases (`.duckdb`, `.ddb`) and SQL queries | Yes (desktop, when DuckDB is installed) | No | | GeoTIFF and cloud-optimised GeoTIFF (`.tif`, `.tiff`) | Yes | Yes | | ASCII grid (`.asc`, with an optional `.prj`) | Yes | No | +| SRTM height tiles (`.hgt`) and DTED (`.dt0`–`.dt2`) | Yes | No | | Unreferenced TIFF and PNG scans; JPEG with `"as": "scan"` | Yes | GeoTIFF after georeferencing | -| Through GDAL: JPEG 2000, ERDAS Imagine, SRTM HGT, DTED, NetCDF, HDF, ENVI, SAGA… | Yes (desktop, when GDAL is installed) | No | +| Through GDAL: JPEG 2000, ERDAS Imagine, NetCDF, HDF, ENVI, SAGA… | Yes (desktop, when GDAL is installed) | No | | Web tiles: XYZ/TMS/WMS rasters, Mapbox Vector Tiles, MapLibre styles, PMTiles and MBTiles archives | Yes | Yes (Imagery › Create Hosted Imagery) | | RustGIS project (`.rgis`; older `.rgisx` files open too) | Yes | Yes | diff --git a/schema/fixtures/elevation_tiles.rgis b/schema/fixtures/elevation_tiles.rgis new file mode 100644 index 0000000000..15e1ade407 --- /dev/null +++ b/schema/fixtures/elevation_tiles.rgis @@ -0,0 +1 @@ +{"requires":["shaded-relief-light"],"format":2,"name":"SRTM and DTED layers","maps":[{"name":"Map","layers":[{"id":1,"name":"N37W122","visible":true,"dataset":{"name":"dem","geometry_type":"None","fields":[],"features":[],"source":null},"renderer":{"Single":{"fill":[224,123,57,170],"stroke":[134,74,34,255],"stroke_width":0.7,"size":0.0,"marker":"Circle"}},"labels":{"enabled":false,"field":"","size":10.0,"color":[43,43,43,255],"halo":true,"bold":false,"min_scale":null,"max_scale":null,"closest":null,"farthest":null,"placement":"Auto"},"transparency":0.0,"min_scale":null,"max_scale":null,"definition_query":null,"selection":[],"expanded":true,"selectable":false,"graphics":false,"blend":"Normal","mask":null,"display_filters":[],"aggregation":null,"extrusion":null,"priority":0,"raster":{"path":"N37W122.hgt","file_name":null,"style":{"renderer":{"ShadedRelief":{"band":0,"ramp":"Terrain","azimuth":315.0,"altitude":45.0,"z_factor":2.0,"ambient":0.4,"balanced":true}},"stretch":{"PercentClip":{"low":0.5,"high":0.5}},"dra":false,"resampling":"Nearest","brightness":0.0,"contrast":0.0,"gamma":1.0,"rotation":0.0},"crs":"EPSG:4326"},"geometry_rev":0},{"id":2,"name":"S45E006","visible":true,"dataset":{"name":"dem","geometry_type":"None","fields":[],"features":[],"source":null},"renderer":{"Single":{"fill":[224,123,57,170],"stroke":[134,74,34,255],"stroke_width":0.7,"size":0.0,"marker":"Circle"}},"labels":{"enabled":false,"field":"","size":10.0,"color":[43,43,43,255],"halo":true,"bold":false,"min_scale":null,"max_scale":null,"closest":null,"farthest":null,"placement":"Auto"},"transparency":0.0,"min_scale":null,"max_scale":null,"definition_query":null,"selection":[],"expanded":true,"selectable":false,"graphics":false,"blend":"Normal","mask":null,"display_filters":[],"aggregation":null,"extrusion":null,"priority":0,"raster":{"path":"S45E006.dt1","file_name":null,"style":{"renderer":{"ShadedRelief":{"band":0,"ramp":"Terrain","azimuth":315.0,"altitude":45.0,"z_factor":2.0,"ambient":0.4,"balanced":true}},"stretch":{"PercentClip":{"low":0.5,"high":0.5}},"dra":false,"resampling":"Nearest","brightness":0.0,"contrast":0.0,"gamma":1.0,"rotation":0.0}},"geometry_rev":0}],"basemap":"Paper","camera":{"center":{"x":-4654314.293187932,"y":-2329385.2820357964},"res":64.69445270508425,"bearing":0.0,"pitch":0.0},"bookmarks":[],"graticule":true,"projection":"WebMercator","popup_mode":"Topmost","label_display":{"paused":false,"locked":false,"show_unplaced":false},"editing":{"relationships":[],"snapping":{"enabled":false,"tolerance_px":10.0,"vertex":true,"edge":true,"end":true},"topology":[],"settings":{"enabled":true,"show_vertices":true,"elevation":"None","ground_to_grid":null}},"swipe":null,"view":{"effects":{"stars":true,"sun_time":null,"weather":"None","vignette":false,"edges":false,"tilt_shift":false,"grain":false,"brightness":0.0,"contrast":0.0,"saturation":0.0,"sepia":0.0},"wireframe":false,"clip_distance":null,"terrain":null},"animation":[],"next_layer_id":2}],"active_map":0,"tables":[],"next_table_id":1,"gp_history":[],"layouts":[],"folders":[],"styles":[],"ramps":[],"tasks":[],"gp_env":{"extent":null,"cell_size":null},"models":[],"workspaces":[],"link_views":false,"jobs":[],"offline_areas":[],"reports":[],"presentations":[],"notebooks":[],"dl_models":[],"shared":[],"data_stores":[]}