From 8fc0d9e762e7010fd5b6fd96758ef423a3bfecb5 Mon Sep 17 00:00:00 2001 From: ro9er117911 <127029993+ro9er117911@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:51:52 +0800 Subject: [PATCH 1/4] fix(data): a Shapefile read as UTF-8 says how much of its text isn't UTF-8, and in which fields A .dbf read as UTF-8 (no language driver and no .cpg, UTF-8 named, or a code page RustGIS can't decode) went through dbase's UnicodeLossy, which replaces bytes that aren't UTF-8 with U+FFFD and says nothing. Big5 names from Taiwan's open data portal came in as U+FFFD with no notice. Count the text values and field names whose bytes aren't valid UTF-8, on the whole-file read (dbf_reader) and the read-in-place source (ShpSource, through Batch::not_utf8), and say how many and in which fields. The values read and the encoding chosen don't change. --- ROADMAP.md | 6 +- crates/data/src/dbf.rs | 126 ++++++++++++++ crates/data/src/export.rs | 6 + crates/data/src/export/tests.rs | 2 + crates/data/src/io.rs | 32 ++-- crates/data/src/source/geojson.rs | 1 + crates/data/src/source/gpkg_mem.rs | 1 + crates/data/src/source/lidar.rs | 1 + crates/data/src/source/mod.rs | 30 +++- crates/data/src/source/parquet.rs | 1 + crates/data/src/source/shp.rs | 45 +++-- crates/data/tests/shapefile_encoding.rs | 174 ++++++++++++++++++-- crates/engine/src/cmd/share.rs | 1 + crates/engine/src/cmd/workflows.rs | 1 + crates/engine/src/lib.rs | 6 + crates/engine/src/tests.rs | 67 ++++++++ crates/engine/tests/never_silent_reading.rs | 44 ++++- crates/map/src/lib.rs | 2 + crates/map/src/source.rs | 26 +++ crates/map/src/source/window.rs | 1 + crates/render/src/stream_tests.rs | 3 + crates/tiles/src/embeddings.rs | 2 + docs/user-guide/data-formats.md | 4 +- docs/user-guide/shapefile-encoding.md | 10 +- 24 files changed, 537 insertions(+), 55 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 36cb06ccc4..d961392248 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -75,9 +75,9 @@ A result is right, or RustGIS says what it lost or assumed each vertex's Z in DXF and unreadable coordinates, holes and parts in KML, GeoRSS, MapInfo and DXF. A fix makes its case `Said` _(open)_. - Lossy sample files in `never_silent_reading.rs` for the readers it doesn't cover yet: CityJSON, - CityGML and the binary formats (Shapefile, GeoPackage, FlatGeobuf, GeoParquet, OSM PBF, KMZ, - spreadsheets). It covers GeoJSON, TopoJSON, CSV, KML, GPX, GML, GeoRSS, DXF, LandXML and MapInfo; - exports are checked by `never_silent_roundtrip.rs` _(open)_. + CityGML and the binary formats (Shapefile beyond text that isn't UTF-8, GeoPackage, FlatGeobuf, + GeoParquet, OSM PBF, KMZ, spreadsheets). It covers GeoJSON, TopoJSON, CSV, KML, GPX, GML, GeoRSS, + DXF, LandXML and MapInfo; exports are checked by `never_silent_roundtrip.rs` _(open)_. - A `--strict` option for `rustgis-cli run` that exits non-zero when a loss is reported, for scheduled jobs. Notices carry a severity but not a kind today (`Notice` in `crates/engine/src/lib.rs`): start with an issue on which notices count as a loss. diff --git a/crates/data/src/dbf.rs b/crates/data/src/dbf.rs index 55492a986f..e40e2e0ef8 100644 --- a/crates/data/src/dbf.rs +++ b/crates/data/src/dbf.rs @@ -27,6 +27,16 @@ impl DbfEncoding { } } + /// Whether text is read as UTF-8, whose decoder puts U+FFFD in place of bytes that aren't + /// UTF-8 without a word: the read counts them itself ([`not_utf8`]). + pub(crate) fn is_utf8(&self) -> bool { + match self { + Self::Standard(encoding) => *encoding == rs::UTF_8, + Self::Oem(_) => false, + Self::Utf8 => true, + } + } + fn from_number(number: u16) -> Option { // DOS 437, 850, 852, 861 and 865 go to dbase's OEM adapter; DOS 866 and the other pages // decoded here keep encoding_rs, because dbase's yore adapter is strict for those and can @@ -159,6 +169,59 @@ fn page_name(page: u16) -> Option<&'static str> { }) } +/// The end of each note of text that isn't UTF-8: why a `.dbf`'s text was read as UTF-8, and what +/// names another encoding. It holds for a file that names no encoding, one that names UTF-8, and +/// one whose code page RustGIS can't decode. +pub(crate) const NO_OTHER_ENCODING: &str = + "no .cpg file or .dbf code page names another encoding RustGIS reads (a .cpg file containing 950 names Big5)"; + +/// What opening a `.dbf` read as UTF-8 says of its field names that aren't UTF-8 ([`not_utf8`]). +/// `None` when there are none. +pub(crate) fn not_utf8_names_note(names: usize) -> Option { + let what = match names { + 0 => return None, + 1 => "1 field name isn't valid UTF-8 and was".to_string(), + n => format!("{n} field names aren't valid UTF-8 and were"), + }; + Some(format!("Shapefile: {what} read with U+FFFD in place of the bytes that can't be read; {NO_OTHER_ENCODING}")) +} + +/// What a `.dbf` read as UTF-8 holds that isn't UTF-8: how many of its field names, and for each +/// column by its position how many of its text values. `dbase` reads them with U+FFFD in place of +/// those bytes and doesn't say so. They are counted on the bytes, since U+FFFD in the text read can +/// also be what the file holds. The values are the ones `dbase` decodes as text: a character (`C`) +/// column's, up to a NUL, in the rows the header counts that aren't marked deleted (a row the read +/// then leaves out as unreadable is counted too). It doesn't look at Cancel: it is one pass over a +/// file that is read whole, so of at most [`crate::io::MAX_BYTES`]. +pub(crate) fn not_utf8(dbf: &[u8]) -> (usize, Vec) { + let lossy = |text: &[u8]| std::str::from_utf8(text.split(|b| *b == 0).next().unwrap_or(&[])).is_err(); + let header_len = usize::from(u16::from_le_bytes([dbf.get(8).copied().unwrap_or(0), dbf.get(9).copied().unwrap_or(0)])); + let rows = dbf.get(4..8).and_then(|b| <[u8; 4]>::try_from(b).ok()).map_or(0, |b| u32::from_le_bytes(b) as usize); + let mut names = 0; + // Each column's width, and whether its values are text. + let mut columns = Vec::new(); + for start in (32..header_len.min(dbf.len())).step_by(32) { + let Some(d) = dbf.get(start..start + 32).filter(|d| d.first() != Some(&0x0D)) else { break }; + names += usize::from(d.get(..11).is_some_and(lossy)); + columns.push((usize::from(d.get(16).copied().unwrap_or(0)), d.get(11) == Some(&b'C'))); + } + let mut values = vec![0; columns.len()]; + let row_len = 1 + columns.iter().map(|(len, _)| len).sum::(); + for row in dbf.get(header_len..).unwrap_or(&[]).chunks_exact(row_len).take(rows) { + if row.first() == Some(&b'*') { + continue; + } + let mut at = 1; + for ((len, text), count) in columns.iter().zip(&mut values) { + if *text && row.get(at..at + len).is_some_and(lossy) { + *count += 1; + } + at += len; + } + } + (names, values) +} + /// Makes a `.dbf` header's last-update date (bytes 1–3: years since 1900, month, day) one `dbase` /// accepts, so a writer that leaves it zero doesn't make the file unreadable (#1272). Nothing reads /// the date, so a placeholder (1 January of its year) is all it takes; a real date is kept. @@ -479,4 +542,67 @@ mod tests { encoding(None, 0xFF, &mut notes); assert_eq!(notes, ["Shapefile: unsupported .dbf code page 0xFF; using UTF-8"]); } + + /// Text is read as UTF-8, so counted when it isn't, with no encoding named, with UTF-8 named + /// and with a code page that can't be decoded; a code page that is decoded isn't UTF-8. + #[test] + fn utf8_is_what_text_is_read_as_unless_another_encoding_is_named() { + let read = |cpg: Option<&[u8]>, ldid| encoding(cpg, ldid, &mut Vec::new()).is_utf8(); + assert!(read(None, 0) && read(None, 0xF0) && read(Some(b"UTF-8"), 0x03) && read(Some(b"65001"), 0) && read(None, 0x6C)); + assert!(!read(Some(b"950"), 0) && !read(None, 0x4F) && !read(None, 0x01) && !read(Some(b"1252"), 0xF0)); + } + + /// A `.dbf` of these columns (name, type, width) and rows (the deletion flag, then the values). + fn table(columns: &[(&[u8], u8, u8)], rows: &[&[u8]]) -> Vec { + let header_len = 32 + 32 * columns.len() + 1; + let row_len = 1 + columns.iter().map(|c| usize::from(c.2)).sum::(); + assert!(rows.iter().all(|row| row.len() == row_len), "rows of {row_len} bytes"); + let mut dbf = vec![0u8; header_len]; + dbf[4..8].copy_from_slice(&(rows.len() as u32).to_le_bytes()); + dbf[8..10].copy_from_slice(&(header_len as u16).to_le_bytes()); + for (k, (name, ty, len)) in columns.iter().enumerate() { + let at = 32 + 32 * k; + dbf[at..at + name.len()].copy_from_slice(name); + (dbf[at + 11], dbf[at + 16]) = (*ty, *len); + } + dbf[header_len - 1] = 0x0D; + rows.iter().for_each(|row| dbf.extend_from_slice(row)); + dbf + } + + /// Text that isn't UTF-8 is counted where `dbase` decodes text: the field names, and the + /// character values up to a NUL of the rows the header counts that aren't deleted. + #[test] + fn text_that_isnt_utf8_is_counted_in_names_and_character_values() { + let columns: &[(&[u8], u8, u8)] = &[(b"caf\xe9", b'C', 4), (b"day", b'D', 8), (b"n", b'N', 2), ("né".as_bytes(), b'C', 3)]; + let rows: &[&[u8]] = &[ + b" \xe9t\xe9 20260305 1abc", + // A date or a number that isn't one is said by its own notice. + b" abc \xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xe9 ", + b"*\xe9t\xe9 20260305 1\xe9bc", + // Bytes past a NUL aren't read; UTF-8 is. + b" a\0\xe9\xe9 \xc3\xa9 ", + ]; + let mut dbf = table(columns, rows); + assert_eq!(not_utf8(&dbf), (1, vec![1, 0, 0, 1])); + // A row past the header's count isn't read. + dbf.extend_from_slice(b" \xe9t\xe9 20260305 1\xe9bc"); + assert_eq!(not_utf8(&dbf), (1, vec![1, 0, 0, 1])); + // A file cut short counts the rows that are whole, and one without a header nothing. + let cut = dbf.len() - 2 * 18 - 1; + assert_eq!(not_utf8(&dbf[..cut]), (1, vec![1, 0, 0, 1])); + assert_eq!(not_utf8(&dbf[..cut - 18]), (1, vec![1, 0, 0, 0])); + assert_eq!(not_utf8(&dbf[..40]), (0, Vec::new())); + assert_eq!(not_utf8(&[]), (0, Vec::new())); + } + + #[test] + fn field_names_that_arent_utf8_are_said_with_their_count() { + assert_eq!(not_utf8_names_note(0), None); + let one = not_utf8_names_note(1).unwrap(); + assert!(one.starts_with("Shapefile: 1 field name isn't valid UTF-8 and was read with U+FFFD"), "{one}"); + let two = not_utf8_names_note(2).unwrap(); + assert!(two.starts_with("Shapefile: 2 field names aren't valid UTF-8 and were read with U+FFFD"), "{two}"); + assert!(two.ends_with(NO_OTHER_ENCODING), "{two}"); + } } diff --git a/crates/data/src/export.rs b/crates/data/src/export.rs index 91ab2a446b..6e41705a12 100644 --- a/crates/data/src/export.rs +++ b/crates/data/src/export.rs @@ -53,6 +53,9 @@ pub struct Exported { /// Per field of the layer, text values `` the source read as null /// ([`crate::source::Batch::null_text`], said by [`crate::source::null_text_note`]). pub null_text: Vec, + /// Per field of the layer, text values the source read as UTF-8 that aren't UTF-8 + /// ([`crate::source::Batch::not_utf8`], said by [`crate::source::not_utf8_note`]). + pub not_utf8: Vec, /// Per field of the layer, date-times a GeoPackage holds in UTC rather than at their own /// offset ([`crate::io::offset_dates`]; empty for the other formats). pub utc_dates: Vec, @@ -153,6 +156,7 @@ fn write_chunks_as( let mut rounded_dates = Vec::new(); let mut null_text = Vec::new(); let mut read_notes = Vec::new(); + let mut not_utf8 = Vec::new(); let mut utc_dates = Vec::new(); let mut fine_dates = Vec::new(); let mut written_values = Vec::new(); @@ -199,6 +203,7 @@ fn write_chunks_as( rounded_dates = batch.rounded_dates; null_text = batch.null_text; read_notes = batch.notes; + not_utf8 = batch.not_utf8; Ok(()) }; let mut table = None; @@ -251,6 +256,7 @@ fn write_chunks_as( unread_dates, rounded_dates, null_text, + not_utf8, utc_dates, fine_dates, written_values, diff --git a/crates/data/src/export/tests.rs b/crates/data/src/export/tests.rs index 293eee1861..79c4f16cfd 100644 --- a/crates/data/src/export/tests.rs +++ b/crates/data/src/export/tests.rs @@ -95,6 +95,7 @@ impl FeatureSource for Runs { rounded_dates: Vec::new(), null_text: Vec::new(), notes: Vec::new(), + not_utf8: Vec::new(), }) } fn visits_chunks(&self) -> bool { @@ -208,6 +209,7 @@ fn a_streamed_export_writes_what_a_whole_export_writes() { unread_dates: Vec::new(), rounded_dates: Vec::new(), null_text: Vec::new(), + not_utf8: Vec::new(), utc_dates, fine_dates: if ext == "gpkg" { crate::io::fine_dates(whole.features.iter(), &whole.fields) } else { Vec::new() }, written_values: if ext == "fgb" { crate::io::written_values(whole.features.iter(), &whole.fields) } else { Vec::new() }, diff --git a/crates/data/src/io.rs b/crates/data/src/io.rs index 85779fb61d..e6e4b0dc94 100644 --- a/crates/data/src/io.rs +++ b/crates/data/src/io.rs @@ -2875,6 +2875,9 @@ struct DbfRead { fields: Vec, decimals: Vec, report: ReadReport, + /// For each field, its text values that aren't UTF-8 in a table read as UTF-8 + /// ([`crate::dbf::not_utf8`]). + not_utf8: Vec, } /// Opens dBASE attributes with the Shapefile reader's code-page and date/number handling. @@ -2882,6 +2885,9 @@ fn dbf_reader(mut bytes: Vec, cpg: Option<&[u8]>) -> Result { use std::io::Cursor; let mut encoding_notes = Vec::new(); let encoding = crate::dbf::encoding(cpg, bytes.get(29).copied().unwrap_or(0), &mut encoding_notes); + // Counted before a date column is marked a character one below: its values aren't text. + let (names, not_utf8) = if encoding.is_utf8() { crate::dbf::not_utf8(&bytes) } else { (0, Vec::new()) }; + encoding_notes.extend(crate::dbf::not_utf8_names_note(names)); crate::dbf::readable_header_date(&mut bytes); let date_columns = crate::dbf::dates_as_text(&mut bytes); crate::dbf::floats_as_numeric(&mut bytes); @@ -2911,7 +2917,7 @@ fn dbf_reader(mut bytes: Vec, cpg: Option<&[u8]>) -> Result { }) .collect(); let fields = order.into_iter().map(|(n, t)| Field::new(n, t)).collect(); - Ok(DbfRead { reader: dbf, fields, decimals, report: ReadReport { messages: encoding_notes, ..Default::default() } }) + Ok(DbfRead { reader: dbf, fields, decimals, report: ReadReport { messages: encoding_notes, ..Default::default() }, not_utf8 }) } /// Reads a geometry-free dBASE table, checking cancellation and counting progress per record. @@ -2919,9 +2925,9 @@ fn dbf_reader(mut bytes: Vec, cpg: Option<&[u8]>) -> Result { /// same budget as delimited tables; archive members share the remaining allowance. /// The bounded file read and header decoding finish before cancellation can be observed. fn read_dbf(bytes: Vec, cpg: Option<&[u8]>, name: &str, budget: &mut usize) -> Result<(Dataset, ReadReport)> { - let DbfRead { mut reader, fields, decimals, mut report } = dbf_reader(bytes, cpg)?; + let DbfRead { mut reader, fields, decimals, mut report, not_utf8 } = dbf_reader(bytes, cpg)?; let mut ds = Dataset::new(name, GeometryType::None, fields); - let mut counts = DbfCounts::new(ds.fields.len()); + let mut counts = DbfCounts::new(ds.fields.len(), not_utf8); let allowance = *budget; let mut left = allowance; crate::progress::begin(reader.header().num_records as usize); @@ -2973,7 +2979,7 @@ fn read_dbf(bytes: Vec, cpg: Option<&[u8]>, name: &str, budget: &mut usize) /// Converts one record to typed values, sharing loss counts with the containing dataset. fn dbf_attrs(record: &shapefile::dbase::Record, fields: &[Field], counts: &mut DbfCounts) -> Vec { use shapefile::dbase::FieldValue; - let DbfCounts { unread_dates, nonfinite, null_text } = counts; + let DbfCounts { unread_dates, nonfinite, null_text, .. } = counts; fields .iter() .enumerate() @@ -3009,22 +3015,25 @@ fn dbf_attrs(record: &shapefile::dbase::Record, fields: &[Field], counts: &mut D } /// Values a `.dbf` read changed, counted for each field by its position: dates that weren't -/// ones, numbers that weren't finite, and text ``, all read as null. +/// ones, numbers that weren't finite, and text ``, all read as null; and text that wasn't +/// UTF-8 in a table read as UTF-8, counted before the read ([`crate::dbf::not_utf8`]). struct DbfCounts { unread_dates: Vec, nonfinite: Vec, null_text: Vec, + not_utf8: Vec, } impl DbfCounts { - fn new(fields: usize) -> Self { - DbfCounts { unread_dates: vec![0; fields], nonfinite: vec![0; fields], null_text: Vec::new() } + fn new(fields: usize, not_utf8: Vec) -> Self { + DbfCounts { unread_dates: vec![0; fields], nonfinite: vec![0; fields], null_text: Vec::new(), not_utf8 } } } -/// Reports date, numeric and `` cells read as null, identically for tables and Shapefiles. +/// Reports date, numeric and `` cells read as null, and text that isn't UTF-8, identically +/// for tables and Shapefiles. fn dbf_notices(ds: &Dataset, report: &mut ReadReport, counts: &DbfCounts) { - let DbfCounts { unread_dates, nonfinite, null_text } = counts; + let DbfCounts { unread_dates, nonfinite, null_text, not_utf8 } = counts; let name = &ds.name; report.unread_dates = crate::source::unread_dates_note(&ds.fields, unread_dates); report.messages.extend(report.unread_dates.iter().map(|n| format!("{name}: {n}"))); @@ -3038,6 +3047,7 @@ fn dbf_notices(ds: &Dataset, report: &mut ReadReport, counts: &DbfCounts) { report.messages.push(format!("{name}: {what} read as empty, in {}", named_fields(&ds.fields, nonfinite))); } report.messages.extend(crate::source::null_text_note(&ds.fields, null_text).map(|n| format!("{name}: {n}"))); + report.messages.extend(crate::source::not_utf8_note(&ds.fields, not_utf8).map(|n| format!("{name}: {n}"))); } /// The files of a Shapefile, read into memory. @@ -3055,7 +3065,7 @@ pub(crate) fn read_shapefile_parts(parts: ShapefileParts, name: &str, crs: Optio use shapefile::Shape; use std::io::Cursor; let fmt = |e: shapefile::Error| DataError::Format(format!("Shapefile: {e}")); - let DbfRead { reader: dbf, fields, decimals, mut report } = dbf_reader(parts.dbf, parts.cpg.as_deref())?; + let DbfRead { reader: dbf, fields, decimals, mut report, not_utf8 } = dbf_reader(parts.dbf, parts.cpg.as_deref())?; let shapes = match parts.shx { Some(shx) => shapefile::ShapeReader::with_shx(Cursor::new(parts.shp), Cursor::new(shx)), None => shapefile::ShapeReader::new(Cursor::new(parts.shp)), @@ -3074,7 +3084,7 @@ pub(crate) fn read_shapefile_parts(parts: ShapefileParts, name: &str, crs: Optio }), }; let (mut converted, mut unconverted, mut zm_shapes, mut not_numbers) = (0usize, 0usize, 0usize, 0usize); - let mut counts = DbfCounts::new(ds.fields.len()); + let mut counts = DbfCounts::new(ds.fields.len(), not_utf8); let mut source_kind = GeometryType::None; let pt = |x: f64, y: f64| Coord::new(x, y); for item in reader.iter_shapes_and_records() { diff --git a/crates/data/src/source/geojson.rs b/crates/data/src/source/geojson.rs index cfec9e4b8e..df29026f6c 100644 --- a/crates/data/src/source/geojson.rs +++ b/crates/data/src/source/geojson.rs @@ -334,6 +334,7 @@ impl FeatureSource for GeoJsonSource { rounded_dates: Vec::new(), null_text: Vec::new(), notes: Vec::new(), + not_utf8: Vec::new(), }) } } diff --git a/crates/data/src/source/gpkg_mem.rs b/crates/data/src/source/gpkg_mem.rs index 1672275ef7..d477ac7555 100644 --- a/crates/data/src/source/gpkg_mem.rs +++ b/crates/data/src/source/gpkg_mem.rs @@ -417,6 +417,7 @@ impl FeatureSource for GpkgMemSource { rounded_dates, null_text: Vec::new(), notes: Vec::new(), + not_utf8: Vec::new(), }) } diff --git a/crates/data/src/source/lidar.rs b/crates/data/src/source/lidar.rs index 197387b782..e3ec5a3f76 100644 --- a/crates/data/src/source/lidar.rs +++ b/crates/data/src/source/lidar.rs @@ -94,6 +94,7 @@ impl FeatureSource for LidarSource { rounded_dates: Vec::new(), null_text: Vec::new(), notes: Vec::new(), + not_utf8: Vec::new(), }) } } diff --git a/crates/data/src/source/mod.rs b/crates/data/src/source/mod.rs index c5fe5369fd..31caa328af 100644 --- a/crates/data/src/source/mod.rs +++ b/crates/data/src/source/mod.rs @@ -256,6 +256,9 @@ pub struct Batch { /// What else the read lost or assumed, one note per kind with its count (values that don't fit /// their field, rings without area): said as they are. pub notes: Vec, + /// Text values of a Shapefile read as UTF-8 that aren't UTF-8, so hold U+FFFD in place of + /// those bytes, counted for each field by its position. Said by [`not_utf8_note`]. + pub not_utf8: Vec, } /// Counts one more value of the field at `k` that wasn't a date, or one date-time rounded to the @@ -325,6 +328,26 @@ pub fn null_text_note(fields: &[crate::Field], counts: &[usize]) -> Option Option { + let total: usize = counts.iter().sum(); + if total == 0 { + return None; + } + let what = if total == 1 { + "1 text value isn't valid UTF-8 and was".to_string() + } else { + format!("{total} text values aren't valid UTF-8 and were") + }; + Some(format!( + "{what} read with U+FFFD in place of the bytes that can't be read, in {}; {}", + crate::io::named_fields(fields, counts), + crate::dbf::NO_OTHER_ENCODING + )) +} + /// What a read says of date-times it rounded to the microsecond ([`Batch::rounded_dates`]), naming /// the first five fields. `None` when there were none. pub fn rounded_dates_note(fields: &[crate::Field], counts: &[usize]) -> Option { @@ -650,7 +673,7 @@ pub fn materialize(src: &dyn FeatureSource) -> Result<(Dataset, ReadReport)> { fn materialize_within(src: &dyn FeatureSource, budget: u64) -> Result<(Dataset, ReadReport)> { let info = src.info(); let batch = if src.visits_chunks() { read_chunks(src, budget)? } else { read_at_once(src)? }; - let Batch { features, skipped, unread_dates, rounded_dates, null_text, notes, .. } = batch; + let Batch { features, skipped, unread_dates, rounded_dates, null_text, notes, not_utf8, .. } = batch; let mut ds = Dataset::new(info.name.clone(), info.geometry_type, info.fields.clone()); ds.crs = info.coordinate_system(); // A streamed Shapefile skips unconvertible shapes rather than retaining empty geometry. @@ -669,7 +692,10 @@ fn materialize_within(src: &dyn FeatureSource, budget: u64) -> Result<(Dataset, extrusion: None, }; let null_text = null_text_note(&info.fields, &null_text); - report.messages.extend(unread_dates.into_iter().chain(rounded).chain(null_text).chain(notes).map(|n| format!("{}: {n}", info.name))); + let not_utf8 = not_utf8_note(&info.fields, ¬_utf8); + report.messages.extend( + unread_dates.into_iter().chain(rounded).chain(null_text).chain(notes).chain(not_utf8).map(|n| format!("{}: {n}", info.name)), + ); if ds.geometry_type == GeometryType::None && let Some(g) = ds.features.iter().find_map(|f| f.geometry.as_ref()) { diff --git a/crates/data/src/source/parquet.rs b/crates/data/src/source/parquet.rs index 0093bbbd07..cbf5a32ba1 100644 --- a/crates/data/src/source/parquet.rs +++ b/crates/data/src/source/parquet.rs @@ -673,6 +673,7 @@ impl FeatureSource for ParquetSource { rounded_dates, null_text: Vec::new(), notes: Vec::new(), + not_utf8: Vec::new(), }) } diff --git a/crates/data/src/source/shp.rs b/crates/data/src/source/shp.rs index 133277ea45..ed7d1ebc1c 100644 --- a/crates/data/src/source/shp.rs +++ b/crates/data/src/source/shp.rs @@ -158,12 +158,9 @@ impl ShpSource { { notes.push("coordinates are not longitude/latitude; RustGIS reads them as WGS 84 and they may draw out of place".into()); } - let fields = table - .columns - .iter() - .zip(read_names(&dbf, table.columns.len(), &encoding.decoder())?) - .map(|(c, n)| Field::new(n, c.out)) - .collect(); + let (names, not_utf8) = read_names(&dbf, table.columns.len(), &encoding.decoder(), encoding.is_utf8())?; + notes.extend(crate::dbf::not_utf8_names_note(not_utf8)); + let fields = table.columns.iter().zip(names).map(|(c, n)| Field::new(n, c.out)).collect(); let mut info = SourceInfo { name: name.to_string(), format: "Shapefile", @@ -210,7 +207,8 @@ impl FeatureSource for ShpSource { let mut buf = Vec::new(); let mut row = vec![0u8; self.table.row_len]; let mut features = Vec::new(); - let (mut skipped, mut unread_dates, mut null_text) = (0, Vec::new(), Vec::new()); + let (mut skipped, mut unread_dates, mut null_text, mut not_utf8) = (0, Vec::new(), Vec::new(), Vec::new()); + let utf8 = self.encoding.is_utf8(); // An extent the file's system couldn't bound (#1925) has no boxes for the index to sample // by: every record is visited, tested, and the matches sampled as they come. let late = req.extent.is_some() && bounds.is_none(); @@ -269,7 +267,7 @@ impl FeatureSource for ShpSource { if late && !sample.wants() { return Ok(()); } - self.table.values(&row, &encoding, &mut unread_dates, &mut null_text) + self.table.values(&row, &encoding, &mut unread_dates, &mut null_text, utf8.then_some(&mut not_utf8)) } else { if late && !sample.wants() { return Ok(()); @@ -291,7 +289,7 @@ impl FeatureSource for ShpSource { } else { self.index.visit(bounds, limit, &mut each)? }; - Ok(Batch { features, matched, complete, skipped, unread_dates, rounded_dates: Vec::new(), null_text, notes: Vec::new() }) + Ok(Batch { features, matched, complete, skipped, unread_dates, rounded_dates: Vec::new(), null_text, notes: Vec::new(), not_utf8 }) } fn streams_chunks(&self) -> bool { true @@ -311,7 +309,8 @@ impl FeatureSource for ShpSource { let mut dbf = Cursor::open(&self.dbf)?; let mut buf = Vec::new(); let mut row = vec![0u8; self.table.row_len]; - let (mut skipped, mut unread_dates, mut null_text) = (0, Vec::new(), Vec::new()); + let (mut skipped, mut unread_dates, mut null_text, mut not_utf8) = (0, Vec::new(), Vec::new(), Vec::new()); + let utf8 = self.encoding.is_utf8(); // The same directory, ordering, extent rules and read-only scan fallback as sampled // reads, with no interaction row cap. The index visits records without materializing // their directory or the complete feature collection in memory. @@ -363,7 +362,7 @@ impl FeatureSource for ShpSource { if row.first() == Some(&b'*') { return Ok(()); } - self.table.values(&row, &encoding, &mut unread_dates, &mut null_text) + self.table.values(&row, &encoding, &mut unread_dates, &mut null_text, utf8.then_some(&mut not_utf8)) } else { vec![Value::Null; self.table.columns.len()] }; @@ -379,6 +378,7 @@ impl FeatureSource for ShpSource { rounded_dates: Vec::new(), null_text, notes: Vec::new(), + not_utf8, }) } } @@ -562,14 +562,16 @@ fn read_table(path: &Path) -> Result { impl Table { /// A row's values (after its deletion flag), counting in `unread_dates` the dates that - /// aren't ones ([`Batch::unread_dates`]) and in `null_text` the text `` read as null - /// ([`Batch::null_text`]). + /// aren't ones ([`Batch::unread_dates`]), in `null_text` the text `` read as null + /// ([`Batch::null_text`]) and in `not_utf8`, given when the table is read as UTF-8, the text + /// that isn't UTF-8 ([`Batch::not_utf8`]). fn values( &self, row: &[u8], encoding: &dyn shapefile::dbase::Encoding, unread_dates: &mut Vec, null_text: &mut Vec, + mut not_utf8: Option<&mut Vec>, ) -> Vec { let mut out = Vec::with_capacity(self.columns.len()); for (k, c) in self.columns.iter().enumerate() { @@ -591,6 +593,13 @@ impl Table { super::null_text_at(null_text, k); Value::Null } else { + // Counted on the bytes: U+FFFD in the text read can also be what the file holds. + if let Some(counts) = &mut not_utf8 + && matches!(v, Value::Text(_)) + && std::str::from_utf8(padded(b)).is_err() + { + super::unread_dates_by(counts, k, 1); + } v } }); @@ -599,20 +608,24 @@ impl Table { } } -/// The `.dbf`'s column names, in order (read again: the layout above keeps only offsets). -fn read_names(path: &Path, n: usize, encoding: &dyn shapefile::dbase::Encoding) -> Result> { +/// The `.dbf`'s column names, in order (read again: the layout above keeps only offsets), and, +/// when they are read as UTF-8 (`utf8`), how many of them aren't UTF-8 +/// ([`crate::dbf::not_utf8_names_note`]). +fn read_names(path: &Path, n: usize, encoding: &dyn shapefile::dbase::Encoding, utf8: bool) -> Result<(Vec, usize)> { let mut f = BufReader::new(crate::provenance::open(path)?); f.seek(SeekFrom::Start(32))?; let mut out = Vec::with_capacity(n); + let mut not_utf8 = 0; let mut d = [0u8; 32]; for _ in 0..n { f.read_exact(&mut d)?; let name = d.get(..11).unwrap_or(&[]); let end = name.iter().position(|b| *b == 0).unwrap_or(name.len()); let name = name.get(..end).unwrap_or(&[]); + not_utf8 += usize::from(utf8 && std::str::from_utf8(name).is_err()); out.push(encoding.decode(name).unwrap_or_else(|_| String::from_utf8_lossy(name)).trim().to_string()); } - Ok(out) + Ok((out, not_utf8)) } /// A text field's bytes up to the first NUL, without the padding after them: spaces before the diff --git a/crates/data/tests/shapefile_encoding.rs b/crates/data/tests/shapefile_encoding.rs index 9970d62e6d..78530f19c4 100644 --- a/crates/data/tests/shapefile_encoding.rs +++ b/crates/data/tests/shapefile_encoding.rs @@ -65,6 +65,135 @@ fn named_parts(field: &[u8], text: &[u8], ldid: u8, cpg: Option<&[u8]>, rows: us .collect() } +/// Why text was read as UTF-8, as the notes of text that isn't UTF-8 end. +const NO_OTHER_ENCODING: &str = + "no .cpg file or .dbf code page names another encoding RustGIS reads (a .cpg file containing 950 names Big5)"; + +/// What a read of `places` says of `values` text values of its field `field` that aren't UTF-8. +fn not_utf8_note(values: usize, field: &str) -> String { + let what = match values { + 1 => "1 text value isn't valid UTF-8 and was".to_string(), + n => format!("{n} text values aren't valid UTF-8 and were"), + }; + format!("places: {what} read with U+FFFD in place of the bytes that can't be read, in '{field}'; {NO_OTHER_ENCODING}") +} + +/// What opening a `.dbf` says of its one field name that isn't UTF-8. +fn name_not_utf8_note() -> String { + format!( + "Shapefile: 1 field name isn't valid UTF-8 and was read with U+FFFD in place of the bytes that can't be read; {NO_OTHER_ENCODING}" + ) +} + +/// Three points with two text fields: `name`, holding `first`, `second` and `Tainan` as these +/// bytes, and `note`, plain ASCII. +fn two_field_parts(first: &[u8], second: &[u8], ldid: u8, cpg: Option<&[u8]>) -> Vec<(String, Vec)> { + let fields = vec![Field::new("name", FieldType::Text), Field::new("note", FieldType::Text)]; + let mut ds = Dataset::new("places", GeometryType::Point, fields); + // Written as placeholders of the same lengths, which the bytes then replace in the `.dbf`. + let holders = ["A".repeat(first.len()), "B".repeat(second.len())]; + for (i, name) in [holders[0].as_str(), holders[1].as_str(), "Tainan"].into_iter().enumerate() { + let point = Geometry::Point(rustgis_data::Coord::new(120.0 + i as f64, 23.0)); + ds.push(Some(point), vec![Value::Text(name.into()), Value::Text("plain".into())]); + } + io::shapefile_parts(&ds) + .unwrap() + .0 + .into_iter() + .filter_map(|(ext, mut bytes)| { + if ext == "cpg" { + return cpg.map(|value| ("places.cpg".into(), value.to_vec())); + } + if ext == "dbf" { + bytes[29] = ldid; + for (holder, text) in holders.iter().zip([first, second]) { + let at = bytes.windows(holder.len()).position(|w| w == holder.as_bytes()).unwrap(); + bytes[at..at + text.len()].copy_from_slice(text); + } + } + Some((format!("places.{ext}"), bytes)) + }) + .collect() +} + +/// The `name` column of a read of [`two_field_parts`]. +fn names(ds: &Dataset) -> Vec { + (0..ds.features.len()).map(|row| ds.value(row, 0).clone()).collect() +} + +/// Text in a code page the file doesn't name (Big5 here) is read as UTF-8: the bytes that aren't +/// UTF-8 become U+FFFD, and every read says how many values that is and in which field. +#[test] +fn text_that_isnt_utf8_is_counted_by_field_when_no_encoding_is_named() { + let (taipei, _, _) = encoding_rs::BIG5.encode("臺北101"); + let (kaohsiung, _, _) = encoding_rs::BIG5.encode("高雄車站"); + let files = two_field_parts(&taipei, &kaohsiung, 0, None); + let read = [Value::Text("\u{fffd}O\u{fffd}_101".into()), Value::Text("\u{fffd}".repeat(8)), Value::Text("Tainan".into())]; + let said = [not_utf8_note(2, "name")]; + let layers = io::read_zip("places.zip", &rustgis_data::zip::write(&files).unwrap()).unwrap(); + assert_eq!(names(&layers[0].0), read); + assert_eq!(layers[0].1.messages, said, "zipped"); + // The `.dbf` on its own, a table. + let dbf = &files.iter().find(|(n, _)| n.ends_with(".dbf")).unwrap().1; + let (table, report) = io::read_bytes("places.dbf", dbf).unwrap(); + assert_eq!(names(&table), read); + assert_eq!(report.messages, said, "the .dbf alone"); + #[cfg(not(target_arch = "wasm32"))] + { + use rustgis_data::source::FeatureSource; + let dir = tempfile::tempdir().unwrap(); + for (name, bytes) in files { + std::fs::write(dir.path().join(name), bytes).unwrap(); + } + let path = dir.path().join("places.shp"); + let (ds, report) = io::read_path(&path).unwrap(); + assert_eq!(names(&ds), read); + assert_eq!(report.messages, said, "loose files"); + // Read in place: the field names are fine, so opening says nothing; a whole read counts. + let src = open_shp(&path, "places").unwrap(); + assert!(src.info().notes.is_empty(), "{:?}", src.info().notes); + let (ds, report) = rustgis_data::source::materialize(&src).unwrap(); + assert_eq!(names(&ds), read); + assert_eq!(report.messages, said, "read in place"); + } +} + +/// The same bytes with a `.cpg` naming their code page are read as written, and nothing is said. +/// Nor is it of UTF-8 text, which can hold U+FFFD itself. +#[test] +fn text_in_a_named_code_page_and_utf8_text_say_nothing_of_utf8() { + let (taipei, _, _) = encoding_rs::BIG5.encode("臺北101"); + let (kaohsiung, _, _) = encoding_rs::BIG5.encode("高雄車站"); + for (first, second, cpg, expected) in [ + (&taipei[..], &kaohsiung[..], Some(b"950".as_slice()), ["臺北101", "高雄車站"]), + ("臺北101".as_bytes(), "高雄車站".as_bytes(), None, ["臺北101", "高雄車站"]), + ("Caf\u{fffd}".as_bytes(), "\u{fffd}".as_bytes(), None, ["Caf\u{fffd}", "\u{fffd}"]), + ] { + let files = two_field_parts(first, second, 0, cpg); + let read = [Value::Text(expected[0].into()), Value::Text(expected[1].into()), Value::Text("Tainan".into())]; + let layers = io::read_zip("places.zip", &rustgis_data::zip::write(&files).unwrap()).unwrap(); + assert_eq!(names(&layers[0].0), read); + assert!(layers[0].1.messages.is_empty(), "{expected:?} zipped: {:?}", layers[0].1.messages); + #[cfg(not(target_arch = "wasm32"))] + { + use rustgis_data::source::FeatureSource; + let dir = tempfile::tempdir().unwrap(); + for (name, bytes) in files { + std::fs::write(dir.path().join(name), bytes).unwrap(); + } + let path = dir.path().join("places.shp"); + let (ds, report) = io::read_path(&path).unwrap(); + assert_eq!(names(&ds), read); + assert!(report.messages.is_empty(), "{expected:?} loose files: {:?}", report.messages); + let src = open_shp(&path, "places").unwrap(); + assert!(src.info().notes.is_empty(), "{expected:?}: {:?}", src.info().notes); + let (ds, report) = rustgis_data::source::materialize(&src).unwrap(); + assert_eq!(names(&ds), read); + assert!(report.messages.is_empty(), "{expected:?} read in place: {:?}", report.messages); + } + } +} + #[cfg(not(target_arch = "wasm32"))] #[test] fn loose_shapefile_honors_cpg_before_dbf_code_page() { @@ -352,18 +481,19 @@ fn dos_dbf_markers_decode_text_without_cpg_in_all_readers() { #[test] fn dos_support_preserves_other_encodings_and_lossy_utf8_fallback() { - for (cpg, ldid, text, expected) in [ - (Some(b"ISO-8859-2".as_slice()), 0xf0, b"\xa1".as_slice(), "Ą"), - (Some(b"CP932".as_slice()), 0xf0, b"\x93\x8c\x8b\x9e".as_slice(), "東京"), - (Some(b"UTF-8".as_slice()), 0xf0, b"Caf\xff".as_slice(), "Caf�"), + for (cpg, ldid, text, expected, replaced) in [ + (Some(b"ISO-8859-2".as_slice()), 0xf0, b"\xa1".as_slice(), "Ą", false), + (Some(b"CP932".as_slice()), 0xf0, b"\x93\x8c\x8b\x9e".as_slice(), "東京", false), + (Some(b"UTF-8".as_slice()), 0xf0, b"Caf\xff".as_slice(), "Caf�", true), // With no encoding metadata, preserve the old UTF-8 fallback's treatment of // field content: a leading U+FEFF is data, and invalid bytes are replaced. - (None, 0, b"\xef\xbb\xbfCaf\xff".as_slice(), "\u{feff}Caf�"), + (None, 0, b"\xef\xbb\xbfCaf\xff".as_slice(), "\u{feff}Caf�", true), ] { let files = parts(text, ldid, cpg); let layers = io::read_zip("places.zip", &rustgis_data::zip::write(&files).unwrap()).unwrap(); assert_eq!(layers[0].0.value(0, 0), &Value::Text(expected.into())); - assert!(layers[0].1.messages.is_empty(), "{:?}", layers[0].1.messages); + let said = if replaced { vec![not_utf8_note(1, "name")] } else { Vec::new() }; + assert_eq!(layers[0].1.messages, said, "{expected}"); } } @@ -377,10 +507,12 @@ fn utf8_metadata_preserves_leading_bom_in_fields_and_values() { let dbf = &mut files.iter_mut().find(|(n, _)| n.ends_with(".dbf")).unwrap().1; dbf[32..43].fill(0); dbf[32..39].copy_from_slice(b"\xef\xbb\xbfname"); + // The value's last byte isn't UTF-8: it is replaced, and the value counted. + let said = [not_utf8_note(1, "\u{feff}name")]; let layers = io::read_zip("places.zip", &rustgis_data::zip::write(&files).unwrap()).unwrap(); assert_eq!(layers[0].0.fields[0].name, "\u{feff}name"); assert_eq!(layers[0].0.value(0, 0), &Value::Text("\u{feff}Caf�".into())); - assert!(layers[0].1.messages.is_empty(), "{:?}", layers[0].1.messages); + assert_eq!(layers[0].1.messages, said); let dir = tempfile::tempdir().unwrap(); for (name, bytes) in files { std::fs::write(dir.path().join(name), bytes).unwrap(); @@ -389,7 +521,7 @@ fn utf8_metadata_preserves_leading_bom_in_fields_and_values() { let (ds, report) = io::read_path(&path).unwrap(); assert_eq!(ds.fields[0].name, "\u{feff}name"); assert_eq!(ds.value(0, 0), &Value::Text("\u{feff}Caf�".into())); - assert!(report.messages.is_empty(), "{:?}", report.messages); + assert_eq!(report.messages, said); let src = open_shp(&path, "places").unwrap(); assert_eq!(src.info().fields[0].name, "\u{feff}name"); assert_eq!(src.read(&Request::all()).unwrap().features[0].attrs, [Value::Text("\u{feff}Caf�".into())]); @@ -467,6 +599,8 @@ const UNSUPPORTED_LDIDS: &[(u8, &str)] = &[ ]; /// Three records make a limit-one read genuinely sampled; names and values use the same bytes. +/// `notes` is what a read of the whole file says. A source read in place says those of opening +/// when it opens, and the count of values (`places: …`) with them once it is read whole. fn check_ldid_readers(ldid: u8, text: &[u8], expected: &str, cpg: Option<&[u8]>, notes: &[&str]) { let files = named_parts(text, text, ldid, cpg, 3); let check = |ds: &Dataset, rows: usize| { @@ -496,7 +630,11 @@ fn check_ldid_readers(ldid: u8, text: &[u8], expected: &str, cpg: Option<&[u8]>, assert_eq!(report.messages, notes, "LDID {ldid:02X} loose-file notes"); let src = open_shp(&path, "places").unwrap(); assert_eq!(src.info().fields[0].name, expected, "LDID {ldid:02X} streamed field name"); - assert_eq!(src.info().notes, notes, "LDID {ldid:02X} streamed notes"); + let opening: Vec<&str> = notes.iter().copied().filter(|n| !n.starts_with("places: ")).collect(); + assert_eq!(src.info().notes, opening, "LDID {ldid:02X} streamed notes"); + let whole: Vec = + notes.iter().map(|n| if n.starts_with("places: ") { n.to_string() } else { format!("places: {n}") }).collect(); + assert_eq!(rustgis_data::source::materialize(&src).unwrap().1.messages, whole, "LDID {ldid:02X} whole read in place"); for (request, rows, complete) in [(Request { extent: None, limit: 1 }, 1, false), (Request::all(), 3, true)] { let batch = src.read(&request).unwrap(); assert_eq!(batch.matched, 3); @@ -532,11 +670,13 @@ fn every_supported_ldid_decodes_names_and_values_in_all_readers() { #[test] fn documented_unsupported_ldids_and_unknown_markers_keep_lossy_utf8_with_a_diagnostic() { + // The last byte isn't UTF-8: the name and the three values it is replaced in are counted. + let (name, values) = (name_not_utf8_note(), not_utf8_note(3, "é�")); for &(ldid, page) in UNSUPPORTED_LDIDS { let note = format!("Shapefile: unsupported .dbf code page 0x{ldid:02X} (code page {page}); using UTF-8"); - check_ldid_readers(ldid, b"\xc3\xa9\xff", "é�", None, &[¬e]); + check_ldid_readers(ldid, b"\xc3\xa9\xff", "é�", None, &[¬e, &name, &values]); } - check_ldid_readers(0xFF, b"\xc3\xa9\xff", "é�", None, &["Shapefile: unsupported .dbf code page 0xFF; using UTF-8"]); + check_ldid_readers(0xFF, b"\xc3\xa9\xff", "é�", None, &["Shapefile: unsupported .dbf code page 0xFF; using UTF-8", &name, &values]); } #[test] @@ -554,7 +694,13 @@ fn unusable_cpg_falls_back_to_each_supported_decoder() { for &(markers, text, expected) in SUPPORTED_LDIDS { check_ldid_readers(markers[0], text, expected, Some(b"not-an-encoding"), &[note]); } - check_ldid_readers(0xFF, b"\xc3\xa9\xff", "é�", Some(b""), &[note, "Shapefile: unsupported .dbf code page 0xFF; using UTF-8"]); + check_ldid_readers( + 0xFF, + b"\xc3\xa9\xff", + "é�", + Some(b""), + &[note, "Shapefile: unsupported .dbf code page 0xFF; using UTF-8", &name_not_utf8_note(), ¬_utf8_note(3, "é�")], + ); } #[test] @@ -564,7 +710,9 @@ fn esri_57_remains_windows_1252_rather_than_iso_8859_1() { #[test] fn malformed_multibyte_fields_replace_bytes_without_dropping_records() { - for (ldid, text) in [(0x13, b"X\x82".as_slice()), (0x4D, b"X\x81"), (0x4E, b"X\x81"), (0x4F, b"X\xa4"), (0xF0, b"X\xff")] { + for (ldid, text) in [(0x13, b"X\x82".as_slice()), (0x4D, b"X\x81"), (0x4E, b"X\x81"), (0x4F, b"X\xa4")] { check_ldid_readers(ldid, text, "X�", None, &[]); } + // Read as UTF-8, the name and the values with a byte replaced are counted. + check_ldid_readers(0xF0, b"X\xff", "X�", None, &[&name_not_utf8_note(), ¬_utf8_note(3, "X�")]); } diff --git a/crates/engine/src/cmd/share.rs b/crates/engine/src/cmd/share.rs index 5746c6d732..0e219874e9 100644 --- a/crates/engine/src/cmd/share.rs +++ b/crates/engine/src/cmd/share.rs @@ -724,6 +724,7 @@ fn write_layer(l: &rustgis_map::Layer, path: &str, asked: Option, gpkg: Gpk rustgis_data::source::null_text_note(&layer.fields, &e.null_text) .map(|n| format!("{path}: {}: {n}, so they're empty in the export", layer.name)), ); + dates.extend(rustgis_data::source::not_utf8_note(&layer.fields, &e.not_utf8).map(|n| format!("{path}: {}: {n}", layer.name))); dates.extend(rustgis_data::io::utc_dates_note(&layer.fields, &e.utc_dates, path)); dates.extend(rustgis_data::io::fine_dates_note(&layer.fields, &e.fine_dates, path)); if ext == "fgb" { diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index 073f05df51..0b730a4a3a 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -1947,6 +1947,7 @@ pub(crate) fn convert_work(s: &Session, p: &Value) -> Result Vec { + s.notices_since(mark).iter().map(|n| n.text.clone()).filter(|t| t.contains("valid UTF-8")).collect() + }; + let mark = s.notice_mark(); + for _ in 0..2 { + s.map_mut().sync_sources(&rustgis_geom::Extent::new(-180.0, -90.0, 180.0, 90.0), 1000.0, true); + s.notice_landed_rasters(); + } + let expected = if said { + vec![format!("big5: some text values in it aren't valid UTF-8 (Load All Features counts them all); so far, {note}")] + } else { + Vec::new() + }; + assert_eq!(not_utf8(&s, mark), expected, "{name}: view reads"); + // An export a chunk at a time, and a Data Interop conversion of the source, say it. + s.map_mut().layer_mut(id).unwrap().dataset.features.clear(); + for (command, params) in [ + ("share.exportLayer", json!({"layer": id, "path": dir.0.join(format!("{name}.csv"))})), + ("data.convert", json!({"layer": id, "format": "geojson", "path": dir.0.join(format!("{name}.geojson"))})), + ] { + let mark = s.notice_mark(); + s.run(command, ¶ms).unwrap(); + let notes = not_utf8(&s, mark); + assert_eq!(notes.len(), usize::from(said), "{name} {command}: {:?}", s.notices_since(mark)); + if said { + assert!(notes[0].ends_with(&format!("big5: {note}")), "{}", notes[0]); + } + } + } +} + /// Load All Features of a GeoPackage read in place says the date-times it read rounded to the /// microsecond; a clean one says nothing of them. #[test] diff --git a/crates/engine/tests/never_silent_reading.rs b/crates/engine/tests/never_silent_reading.rs index 8956f1cce9..47e326db9e 100644 --- a/crates/engine/tests/never_silent_reading.rs +++ b/crates/engine/tests/never_silent_reading.rs @@ -11,8 +11,9 @@ //! fails here until the case is moved to [`Expect::Said`], so the list only shrinks. Exports have //! their own test, `never_silent_roundtrip.rs`. //! -//! Text formats only so far, without CityJSON and CityGML; those and the binary ones (Shapefile, -//! GeoPackage, FlatGeobuf, GeoParquet, OSM PBF, KMZ, spreadsheets) are still to add (ROADMAP.md). +//! Text formats so far, without CityJSON and CityGML, and of Shapefile only text that isn't UTF-8; +//! CityJSON, CityGML and the binary ones (Shapefile's other losses, GeoPackage, FlatGeobuf, +//! GeoParquet, OSM PBF, KMZ, spreadsheets) are still to add (ROADMAP.md). use rustgis_engine::{Project, Session}; use serde_json::json; @@ -32,14 +33,14 @@ struct Case { name: &'static str, /// The file name, whose extension picks the reader. file: &'static str, - clean: String, - lossy: String, + clean: Vec, + lossy: Vec, /// Files beside it, the same in both (a MapInfo `.mid`). beside: Vec<(&'static str, String)>, expect: Expect, } -fn case(name: &'static str, file: &'static str, clean: impl Into, lossy: impl Into, expect: Expect) -> Case { +fn case(name: &'static str, file: &'static str, clean: impl Into>, lossy: impl Into>, expect: Expect) -> Case { Case { name, file, clean: clean.into(), lossy: lossy.into(), beside: Vec::new(), expect } } @@ -94,6 +95,29 @@ fn landxml(content: &str) -> String { format!(r#"{content}"#) } +/// A zipped Shapefile of one point whose text field `NAME` holds these bytes. It names no +/// encoding: it has no `.cpg`, and its `.dbf` no language driver. +fn shapefile_zip(name: &[u8]) -> Vec { + use rustgis_engine::data::{Coord, Dataset, Field, FieldType, Geometry, GeometryType, Value, io}; + // Written as a placeholder of the same length, which the bytes then replace in the `.dbf`. + let holder = "A".repeat(name.len()); + let mut ds = Dataset::new("a", GeometryType::Point, vec![Field::new("NAME", FieldType::Text)]); + ds.push(Some(Geometry::Point(Coord::new(121.5, 25.0))), vec![Value::Text(holder.clone())]); + let mut entries = Vec::new(); + for (ext, mut bytes) in io::shapefile_parts(&ds).unwrap().0 { + if ext == "cpg" { + continue; + } + if ext == "dbf" { + bytes[29] = 0; + let at = bytes.windows(holder.len()).position(|w| w == holder.as_bytes()).unwrap(); + bytes[at..at + name.len()].copy_from_slice(name); + } + entries.push((format!("a.{ext}"), bytes)); + } + rustgis_engine::data::zip::write(&entries).unwrap() +} + fn cases() -> Vec { let point = r#"{"type":"Point","coordinates":[1,2]}"#; let topo_line = |arcs: &str| format!(r#"{{"type":"Topology","objects":{{"o":{{"type":"LineString","arcs":[0]}}}},"arcs":[{arcs}]}}"#); @@ -320,6 +344,14 @@ fn cases() -> Vec { landxml(&format!(r#"{}"#, cg_points("1 1"))), Silent("the parcel is left out"), ), + // Shapefile. 臺北101 as UTF-8, and in Big5, which isn't UTF-8. + case( + "Shapefile: text in a code page it doesn't name", + "a.zip", + shapefile_zip("臺北101".as_bytes()), + shapefile_zip(b"\xbb\x4f\xa5\x5f101"), + Said("text value isn't valid UTF-8"), + ), ]; // MapInfo: the .mid beside each .mif holds one row per object. @@ -354,7 +386,7 @@ type Reading = Result<(Vec, bool), String>; /// Writes `file` with `content`, and `beside`, into `dir` (emptied first) and reads it. `Err` when /// the files can't be written: a failure of the test, never taken for a refusal. Each file of a /// case is read from the same `dir`, so notices naming its path are the same for both. -fn read(dir: &std::path::Path, file: &str, content: &str, beside: &[(&str, String)]) -> std::io::Result { +fn read(dir: &std::path::Path, file: &str, content: &[u8], beside: &[(&str, String)]) -> std::io::Result { let _ = std::fs::remove_dir_all(dir); std::fs::create_dir_all(dir)?; for (name, text) in beside { diff --git a/crates/map/src/lib.rs b/crates/map/src/lib.rs index 40ef228613..7db9584eef 100644 --- a/crates/map/src/lib.rs +++ b/crates/map/src/lib.rs @@ -650,6 +650,7 @@ impl Layer { source::Polled::Done(_, batch) => { src.saw_unread_dates(&batch.unread_dates, &batch.rounded_dates); src.saw_null_text(&batch.null_text); + src.saw_not_utf8(&batch.not_utf8); self.install(batch.features) } source::Polled::Failed(_) => self.install(Vec::new()), @@ -662,6 +663,7 @@ impl Layer { source::Polled::Done(_, batch) => { src.saw_unread_dates(&batch.unread_dates, &batch.rounded_dates); src.saw_null_text(&batch.null_text); + src.saw_not_utf8(&batch.not_utf8); self.install(batch.features) } source::Polled::Failed(_) => self.install(Vec::new()), diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index 56dbd512a4..06734085d7 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -111,6 +111,11 @@ pub struct SourceLayer { null_text: Option>, #[serde(skip)] null_text_said: bool, + /// Text values a view read found weren't UTF-8 ([`Batch::not_utf8`]), likewise. + #[serde(skip)] + not_utf8: Option>, + #[serde(skip)] + not_utf8_said: bool, #[serde(skip)] window: Option, /// Whole-source thinning decisions shared by render snapshots, independent of view rows. @@ -772,6 +777,23 @@ impl SourceLayer { rustgis_data::source::null_text_note(&fields, &counts) } + /// Keeps what view reads found of text values that aren't UTF-8, until + /// [`SourceLayer::take_not_utf8`] says it, as [`SourceLayer::saw_unread_dates`] does for dates. + pub fn saw_not_utf8(&mut self, counts: &[usize]) { + if !self.not_utf8_said && counts.iter().any(|n| *n > 0) { + rustgis_data::source::add_unread_dates(self.not_utf8.get_or_insert_with(Vec::new), counts); + } + } + + /// What view reads found of text values that aren't UTF-8 + /// ([`rustgis_data::source::not_utf8_note`]), once for the source. + pub fn take_not_utf8(&mut self) -> Option { + let fields = self.info().map(|i| i.fields.clone())?; + let counts = self.not_utf8.take()?; + self.not_utf8_said = true; + rustgis_data::source::not_utf8_note(&fields, &counts) + } + /// Whether another installed window needs its retained display expressions checked. pub fn take_display_window(&mut self) -> bool { if self.reads == self.display_read_said { @@ -1503,6 +1525,8 @@ impl SourceLayer { self.rounded_dates_said = false; self.null_text = None; self.null_text_said = false; + self.not_utf8 = None; + self.not_utf8_said = false; self.reset(); self.open() } @@ -1826,6 +1850,8 @@ impl SourceLayer { rounded_dates_said: _, null_text: _, null_text_said: _, + not_utf8: _, + not_utf8_said: _, gdal, duckdb, postgis, diff --git a/crates/map/src/source/window.rs b/crates/map/src/source/window.rs index 9f91f2bb3a..56c4e7a4d2 100644 --- a/crates/map/src/source/window.rs +++ b/crates/map/src/source/window.rs @@ -189,6 +189,7 @@ fn read_filtered( rounded_dates: summary.rounded_dates, null_text: summary.null_text, notes: summary.notes, + not_utf8: summary.not_utf8, }, areas, )) diff --git a/crates/render/src/stream_tests.rs b/crates/render/src/stream_tests.rs index aadc6232c8..16835df5c8 100644 --- a/crates/render/src/stream_tests.rs +++ b/crates/render/src/stream_tests.rs @@ -28,6 +28,7 @@ impl FeatureSource for Chunks { rounded_dates: Vec::new(), null_text: Vec::new(), notes: Vec::new(), + not_utf8: Vec::new(), }) } fn streams_chunks(&self) -> bool { @@ -111,6 +112,7 @@ fn complete_chunks_preserve_painter_order_filters_transparency_labels_and_select rounded_dates: Vec::new(), null_text: Vec::new(), notes: Vec::new(), + not_utf8: Vec::new(), }) } fn streams_chunks(&self) -> bool { @@ -863,6 +865,7 @@ fn pitched_mixed_sources_keep_windows_and_pending_on_the_owner() { rounded_dates: Vec::new(), null_text: Vec::new(), notes: Vec::new(), + not_utf8: Vec::new(), }) } } diff --git a/crates/tiles/src/embeddings.rs b/crates/tiles/src/embeddings.rs index 4fe377fd52..36b10132c6 100644 --- a/crates/tiles/src/embeddings.rs +++ b/crates/tiles/src/embeddings.rs @@ -928,6 +928,7 @@ mod tests { rounded_dates: Vec::new(), null_text: Vec::new(), notes: Vec::new(), + not_utf8: Vec::new(), }; let t = aef_tiles(&fields, &batch, 2021, [-123.0, 40.0, -122.0, 41.0]); assert_eq!(t.iter().map(|t| t.id.as_str()).collect::>(), vec!["near", "far"]); @@ -994,6 +995,7 @@ mod tests { rounded_dates: Vec::new(), null_text: Vec::new(), notes: Vec::new(), + not_utf8: Vec::new(), }; let f = aef_series(&fields, &batch, (2017, 2025), [-43.0, -20.9, -42.9, -20.8]); assert_eq!((f.series.len(), f.duplicates, f.dropped, f.complete), (2, 1, 0, false)); diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index e2209c926b..e94f9a3471 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -63,7 +63,9 @@ and common East Asian encodings are supported. The Esri `0x57` DBF marker is read as Windows-1252. An unreadable, empty, invalid, oversized or unsupported `.cpg` gives a notice and falls back to the DBF code page; an unsupported DBF code page gives a notice and falls back to UTF-8. Files without encoding metadata keep the UTF-8 default. Unsupported characters become -replacement characters. This applies to zipped files and large files read in place too. +replacement characters, and when text read as UTF-8 isn't valid UTF-8 (Big5 text with no `.cpg`, +say) a notice counts those values and names their fields. This applies to zipped files and large +files read in place too. See the [DBF language-driver support inventory](shapefile-encoding.md) for every recognized marker and the legacy code pages that still use the UTF-8 fallback. diff --git a/docs/user-guide/shapefile-encoding.md b/docs/user-guide/shapefile-encoding.md index 41dcb894ac..6d2e897a07 100644 --- a/docs/user-guide/shapefile-encoding.md +++ b/docs/user-guide/shapefile-encoding.md @@ -59,6 +59,10 @@ bytes become replacement characters, and the records remain available. An unusab adds its own diagnostic before falling back to the LDID. Supplying a valid `.cpg` bypasses the LDID and its unsupported-code-page diagnostic. +Whenever text is read as UTF-8 (the markers in this section, unknown markers, `00`, `F0` or a +UTF-8 `.cpg`), a notice counts the text values whose bytes aren't valid UTF-8 and names their +fields, and another counts such field names. A file whose text is all valid UTF-8 gives neither. + The existing transitive `yore` 1.4.0 package has CP737, CP857, CP860 and CP863 decoders, but RustGIS's current `dbase` adapter does not expose them. Making these available would require decoder integration and a coordinated direct dependency declaration; their presence @@ -82,6 +86,6 @@ tables or system programs are needed for the supported decoders above. `crates/data/tests/shapefile_encoding.rs` specifies fixed bytes and Unicode expectations independently of RustGIS's mapping. Its matrix covers every marker above, names and values, ZIP and loose-file imports, a genuinely sampled streamed read, a complete read, and ordered -chunk traversal. It also checks `.cpg` precedence, fallback diagnostics, and malformed -multibyte text without discarding records. Test files are generated locally; the tests do -not download data or call external converters. +chunk traversal. It also checks `.cpg` precedence, fallback diagnostics, the notices for text +that isn't valid UTF-8, and malformed multibyte text without discarding records. Test files are +generated locally; the tests do not download data or call external converters. From 2ec873feb1352bc8c294b7b7c23f6555197a445c Mon Sep 17 00:00:00 2001 From: ro9er117911 <127029993+ro9er117911@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:32:17 +0800 Subject: [PATCH 2/4] fix(data): the notice's cause names only the .cpg, which holds whenever a .dbf is read as UTF-8 A .cpg naming UTF-8 is used before the .dbf code page, so "no .cpg file or .dbf code page names another encoding" was false for a .cpg of UTF-8 beside a code page RustGIS reads (1252, say). "No .cpg file names another encoding RustGIS reads" holds in every case that is read as UTF-8. The read-in-place source counts through not_utf8_at, as null_text_at does. --- crates/data/src/dbf.rs | 8 ++++---- crates/data/src/source/mod.rs | 5 +++++ crates/data/src/source/shp.rs | 2 +- crates/data/tests/shapefile_encoding.rs | 3 +-- crates/engine/src/tests.rs | 2 +- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/data/src/dbf.rs b/crates/data/src/dbf.rs index e40e2e0ef8..97fac8cf40 100644 --- a/crates/data/src/dbf.rs +++ b/crates/data/src/dbf.rs @@ -170,10 +170,10 @@ fn page_name(page: u16) -> Option<&'static str> { } /// The end of each note of text that isn't UTF-8: why a `.dbf`'s text was read as UTF-8, and what -/// names another encoding. It holds for a file that names no encoding, one that names UTF-8, and -/// one whose code page RustGIS can't decode. -pub(crate) const NO_OTHER_ENCODING: &str = - "no .cpg file or .dbf code page names another encoding RustGIS reads (a .cpg file containing 950 names Big5)"; +/// names another encoding. A `.cpg` comes before the code page, so text is read as UTF-8 only when +/// no `.cpg` names another encoding RustGIS reads: the file names none, names UTF-8, or names a +/// code page RustGIS can't decode. +pub(crate) const NO_OTHER_ENCODING: &str = "no .cpg file names another encoding RustGIS reads (a .cpg file containing 950 names Big5)"; /// What opening a `.dbf` read as UTF-8 says of its field names that aren't UTF-8 ([`not_utf8`]). /// `None` when there are none. diff --git a/crates/data/src/source/mod.rs b/crates/data/src/source/mod.rs index 31caa328af..18bc959303 100644 --- a/crates/data/src/source/mod.rs +++ b/crates/data/src/source/mod.rs @@ -272,6 +272,11 @@ pub(crate) fn null_text_at(counts: &mut Vec, k: usize) { unread_dates_by(counts, k, 1); } +/// Counts one more text value that isn't UTF-8 in the field at `k` ([`Batch::not_utf8`]). +pub(crate) fn not_utf8_at(counts: &mut Vec, k: usize) { + unread_dates_by(counts, k, 1); +} + /// [`unread_date`], `n` more times. pub(crate) fn unread_dates_by(counts: &mut Vec, k: usize, n: usize) { if n == 0 { diff --git a/crates/data/src/source/shp.rs b/crates/data/src/source/shp.rs index ed7d1ebc1c..61a3d9343c 100644 --- a/crates/data/src/source/shp.rs +++ b/crates/data/src/source/shp.rs @@ -598,7 +598,7 @@ impl Table { && matches!(v, Value::Text(_)) && std::str::from_utf8(padded(b)).is_err() { - super::unread_dates_by(counts, k, 1); + super::not_utf8_at(counts, k); } v } diff --git a/crates/data/tests/shapefile_encoding.rs b/crates/data/tests/shapefile_encoding.rs index 78530f19c4..1d70daf94f 100644 --- a/crates/data/tests/shapefile_encoding.rs +++ b/crates/data/tests/shapefile_encoding.rs @@ -66,8 +66,7 @@ fn named_parts(field: &[u8], text: &[u8], ldid: u8, cpg: Option<&[u8]>, rows: us } /// Why text was read as UTF-8, as the notes of text that isn't UTF-8 end. -const NO_OTHER_ENCODING: &str = - "no .cpg file or .dbf code page names another encoding RustGIS reads (a .cpg file containing 950 names Big5)"; +const NO_OTHER_ENCODING: &str = "no .cpg file names another encoding RustGIS reads (a .cpg file containing 950 names Big5)"; /// What a read of `places` says of `values` text values of its field `field` that aren't UTF-8. fn not_utf8_note(values: usize, field: &str) -> String { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index d736ec82e8..25b7490896 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1343,7 +1343,7 @@ fn a_streamed_shapefile_says_text_that_isnt_utf8_once_and_in_its_exports() { let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_nanos()); let dir = Dir(std::env::temp_dir().join(format!("rustgis-streamed-not-utf8-{}-{nanos}", std::process::id()))); std::fs::create_dir_all(&dir.0).unwrap(); - let note = "1 text value isn't valid UTF-8 and was read with U+FFFD in place of the bytes that can't be read, in 'NAME'; no .cpg file or .dbf code page names another encoding RustGIS reads (a .cpg file containing 950 names Big5)"; + let note = "1 text value isn't valid UTF-8 and was read with U+FFFD in place of the bytes that can't be read, in 'NAME'; no .cpg file names another encoding RustGIS reads (a .cpg file containing 950 names Big5)"; // 臺北101 in Big5, and as many bytes of ASCII. for (name, second, said) in [("big5", &b"\xbb\x4f\xa5\x5f101"[..], true), ("clean", &b"Tainan1"[..], false)] { let mut ds = Dataset::new(name, GeometryType::Point, vec![Field::new("NAME", FieldType::Text)]); From 9e343c640c930a72aefbc351dbddc8a4aeef6c81 Mon Sep 17 00:00:00 2001 From: ro9er117911 <127029993+ro9er117911@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:50:49 +0800 Subject: [PATCH 3/4] fix(data): not_utf8_at is native only, as its one caller (the read-in-place Shapefile source) is --- crates/data/src/source/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/data/src/source/mod.rs b/crates/data/src/source/mod.rs index 18bc959303..33370a7422 100644 --- a/crates/data/src/source/mod.rs +++ b/crates/data/src/source/mod.rs @@ -273,6 +273,7 @@ pub(crate) fn null_text_at(counts: &mut Vec, k: usize) { } /// Counts one more text value that isn't UTF-8 in the field at `k` ([`Batch::not_utf8`]). +#[cfg(not(target_arch = "wasm32"))] pub(crate) fn not_utf8_at(counts: &mut Vec, k: usize) { unread_dates_by(counts, k, 1); } From 0e8e66b4ed927b2cc4575b6ed49f23997ffd9119 Mon Sep 17 00:00:00 2001 From: ro9er117911 <127029993+ro9er117911@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:26:22 +0800 Subject: [PATCH 4/4] fix(engine): the view-read notice of text that isn't UTF-8 keeps a translatable template It goes through NoticeText and notify_text, as source read failures do, so an interface in another language can translate it; the English text, and so the command line's, is the same. The streamed test reads the same opening a second time (the source reset, not reopened) and checks that the batch lands again with no second notice, so the once-per-opening guard is tested: without it, the test fails. --- crates/engine/src/lib.rs | 10 +++++++--- crates/engine/src/tests.rs | 14 ++++++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 54ef1ee7a5..9a823241b5 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -2279,6 +2279,7 @@ impl Session { // once, keeping the same counted notice channel as retained resident expressions. let mut landed = Vec::new(); let mut dates = Vec::new(); + let mut not_utf8 = Vec::new(); let mut source_errors = Vec::new(); let mut source_error_count = 0; for map in &mut self.project.maps { @@ -2319,9 +2320,9 @@ impl Session { )); } if let Some(note) = layer.source.as_mut().and_then(|source| source.take_not_utf8()) { - dates.push(format!( - "{}: some text values in it aren't valid UTF-8 (Load All Features counts them all); so far, {note}", - layer.name + not_utf8.push(notice_text::NoticeText::format( + "{name}: some text values in it aren't valid UTF-8 (Load All Features counts them all); so far, {note}", + [("name", layer.name.clone().into()), ("note", note.into())], )); } if layer.source.as_mut().is_some_and(|source| source.take_display_window()) { @@ -2335,6 +2336,9 @@ impl Session { for note in dates { self.notify(note, false); } + for note in not_utf8 { + self.notify_text(note, false); + } if let Some(message) = notice_text::NoticeText::source_errors(source_error_count, source_errors, source_error_count) { self.notify_text(message, true); } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 25b7490896..e7600b405b 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1368,10 +1368,16 @@ fn a_streamed_shapefile_says_text_that_isnt_utf8_once_and_in_its_exports() { s.notices_since(mark).iter().map(|n| n.text.clone()).filter(|t| t.contains("valid UTF-8")).collect() }; let mark = s.notice_mark(); - for _ in 0..2 { - s.map_mut().sync_sources(&rustgis_geom::Extent::new(-180.0, -90.0, 180.0, 90.0), 1000.0, true); - s.notice_landed_rasters(); - } + let view = rustgis_geom::Extent::new(-180.0, -90.0, 180.0, 90.0); + s.map_mut().sync_sources(&view, 1000.0, true); + s.notice_landed_rasters(); + // A second read of the same opening lands its batch again and says nothing more. + let layer = s.map_mut().layer_mut(id).unwrap(); + layer.dataset.features.clear(); + layer.source.as_mut().unwrap().reset(); + s.map_mut().sync_sources(&view, 1000.0, true); + s.notice_landed_rasters(); + assert!(!s.map().layer(id).unwrap().dataset.features.is_empty(), "{name}: the second read lands"); let expected = if said { vec![format!("big5: some text values in it aren't valid UTF-8 (Load All Features counts them all); so far, {note}")] } else {