Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ROADMAP.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,9 @@ A result is right, or RustGIS says what it lost or assumed
each vertex's Z in DXF and unreadable coordinates, holes and parts in KML, GeoRSS, MapInfo and
DXF. A fix makes its case `Said` _(open)_.
- Lossy sample files in `never_silent_reading.rs` for the readers it doesn't cover yet: CityJSON,
CityGML and the binary formats (Shapefile, GeoPackage, FlatGeobuf, GeoParquet, OSM PBF, KMZ,
spreadsheets). It covers GeoJSON, TopoJSON, CSV, KML, GPX, GML, GeoRSS, DXF, LandXML and MapInfo;
exports are checked by `never_silent_roundtrip.rs` _(open)_.
CityGML and the binary formats (Shapefile beyond text that isn't UTF-8, GeoPackage, FlatGeobuf,
GeoParquet, OSM PBF, KMZ, spreadsheets). It covers GeoJSON, TopoJSON, CSV, KML, GPX, GML, GeoRSS,
DXF, LandXML and MapInfo; exports are checked by `never_silent_roundtrip.rs` _(open)_.
- A `--strict` option for `rustgis-cli run` that exits non-zero when a loss is reported, for
scheduled jobs. Notices carry a severity but not a kind today (`Notice` in
`crates/engine/src/lib.rs`): start with an issue on which notices count as a loss.
Expand Down
126 changes: 126 additions & 0 deletions crates/data/src/dbf.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,16 @@ impl DbfEncoding {
}
}

/// Whether text is read as UTF-8, whose decoder puts U+FFFD in place of bytes that aren't
/// UTF-8 without a word: the read counts them itself ([`not_utf8`]).
pub(crate) fn is_utf8(&self) -> bool {
match self {
Self::Standard(encoding) => *encoding == rs::UTF_8,
Self::Oem(_) => false,
Self::Utf8 => true,
}
}

fn from_number(number: u16) -> Option<Self> {
// DOS 437, 850, 852, 861 and 865 go to dbase's OEM adapter; DOS 866 and the other pages
// decoded here keep encoding_rs, because dbase's yore adapter is strict for those and can
Expand Down Expand Up @@ -159,6 +169,59 @@ fn page_name(page: u16) -> Option<&'static str> {
})
}

/// The end of each note of text that isn't UTF-8: why a `.dbf`'s text was read as UTF-8, and what
/// names another encoding. A `.cpg` comes before the code page, so text is read as UTF-8 only when
/// no `.cpg` names another encoding RustGIS reads: the file names none, names UTF-8, or names a
/// code page RustGIS can't decode.
pub(crate) const NO_OTHER_ENCODING: &str = "no .cpg file names another encoding RustGIS reads (a .cpg file containing 950 names Big5)";

/// What opening a `.dbf` read as UTF-8 says of its field names that aren't UTF-8 ([`not_utf8`]).
/// `None` when there are none.
pub(crate) fn not_utf8_names_note(names: usize) -> Option<String> {
let what = match names {
0 => return None,
1 => "1 field name isn't valid UTF-8 and was".to_string(),
n => format!("{n} field names aren't valid UTF-8 and were"),
};
Some(format!("Shapefile: {what} read with U+FFFD in place of the bytes that can't be read; {NO_OTHER_ENCODING}"))
}

/// What a `.dbf` read as UTF-8 holds that isn't UTF-8: how many of its field names, and for each
/// column by its position how many of its text values. `dbase` reads them with U+FFFD in place of
/// those bytes and doesn't say so. They are counted on the bytes, since U+FFFD in the text read can
/// also be what the file holds. The values are the ones `dbase` decodes as text: a character (`C`)
/// column's, up to a NUL, in the rows the header counts that aren't marked deleted (a row the read
/// then leaves out as unreadable is counted too). It doesn't look at Cancel: it is one pass over a
/// file that is read whole, so of at most [`crate::io::MAX_BYTES`].
pub(crate) fn not_utf8(dbf: &[u8]) -> (usize, Vec<usize>) {
let lossy = |text: &[u8]| std::str::from_utf8(text.split(|b| *b == 0).next().unwrap_or(&[])).is_err();
let header_len = usize::from(u16::from_le_bytes([dbf.get(8).copied().unwrap_or(0), dbf.get(9).copied().unwrap_or(0)]));
let rows = dbf.get(4..8).and_then(|b| <[u8; 4]>::try_from(b).ok()).map_or(0, |b| u32::from_le_bytes(b) as usize);
let mut names = 0;
// Each column's width, and whether its values are text.
let mut columns = Vec::new();
for start in (32..header_len.min(dbf.len())).step_by(32) {
let Some(d) = dbf.get(start..start + 32).filter(|d| d.first() != Some(&0x0D)) else { break };
names += usize::from(d.get(..11).is_some_and(lossy));
columns.push((usize::from(d.get(16).copied().unwrap_or(0)), d.get(11) == Some(&b'C')));
}
let mut values = vec![0; columns.len()];
let row_len = 1 + columns.iter().map(|(len, _)| len).sum::<usize>();
for row in dbf.get(header_len..).unwrap_or(&[]).chunks_exact(row_len).take(rows) {
if row.first() == Some(&b'*') {
continue;
}
let mut at = 1;
for ((len, text), count) in columns.iter().zip(&mut values) {
if *text && row.get(at..at + len).is_some_and(lossy) {
*count += 1;
}
at += len;
}
}
(names, values)
}

/// Makes a `.dbf` header's last-update date (bytes 1–3: years since 1900, month, day) one `dbase`
/// accepts, so a writer that leaves it zero doesn't make the file unreadable (#1272). Nothing reads
/// the date, so a placeholder (1 January of its year) is all it takes; a real date is kept.
Expand Down Expand Up @@ -479,4 +542,67 @@ mod tests {
encoding(None, 0xFF, &mut notes);
assert_eq!(notes, ["Shapefile: unsupported .dbf code page 0xFF; using UTF-8"]);
}

/// Text is read as UTF-8, so counted when it isn't, with no encoding named, with UTF-8 named
/// and with a code page that can't be decoded; a code page that is decoded isn't UTF-8.
#[test]
fn utf8_is_what_text_is_read_as_unless_another_encoding_is_named() {
let read = |cpg: Option<&[u8]>, ldid| encoding(cpg, ldid, &mut Vec::new()).is_utf8();
assert!(read(None, 0) && read(None, 0xF0) && read(Some(b"UTF-8"), 0x03) && read(Some(b"65001"), 0) && read(None, 0x6C));
assert!(!read(Some(b"950"), 0) && !read(None, 0x4F) && !read(None, 0x01) && !read(Some(b"1252"), 0xF0));
}

/// A `.dbf` of these columns (name, type, width) and rows (the deletion flag, then the values).
fn table(columns: &[(&[u8], u8, u8)], rows: &[&[u8]]) -> Vec<u8> {
let header_len = 32 + 32 * columns.len() + 1;
let row_len = 1 + columns.iter().map(|c| usize::from(c.2)).sum::<usize>();
assert!(rows.iter().all(|row| row.len() == row_len), "rows of {row_len} bytes");
let mut dbf = vec![0u8; header_len];
dbf[4..8].copy_from_slice(&(rows.len() as u32).to_le_bytes());
dbf[8..10].copy_from_slice(&(header_len as u16).to_le_bytes());
for (k, (name, ty, len)) in columns.iter().enumerate() {
let at = 32 + 32 * k;
dbf[at..at + name.len()].copy_from_slice(name);
(dbf[at + 11], dbf[at + 16]) = (*ty, *len);
}
dbf[header_len - 1] = 0x0D;
rows.iter().for_each(|row| dbf.extend_from_slice(row));
dbf
}

/// Text that isn't UTF-8 is counted where `dbase` decodes text: the field names, and the
/// character values up to a NUL of the rows the header counts that aren't deleted.
#[test]
fn text_that_isnt_utf8_is_counted_in_names_and_character_values() {
let columns: &[(&[u8], u8, u8)] = &[(b"caf\xe9", b'C', 4), (b"day", b'D', 8), (b"n", b'N', 2), ("né".as_bytes(), b'C', 3)];
let rows: &[&[u8]] = &[
b" \xe9t\xe9 20260305 1abc",
// A date or a number that isn't one is said by its own notice.
b" abc \xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xe9 ",
b"*\xe9t\xe9 20260305 1\xe9bc",
// Bytes past a NUL aren't read; UTF-8 is.
b" a\0\xe9\xe9 \xc3\xa9 ",
];
let mut dbf = table(columns, rows);
assert_eq!(not_utf8(&dbf), (1, vec![1, 0, 0, 1]));
// A row past the header's count isn't read.
dbf.extend_from_slice(b" \xe9t\xe9 20260305 1\xe9bc");
assert_eq!(not_utf8(&dbf), (1, vec![1, 0, 0, 1]));
// A file cut short counts the rows that are whole, and one without a header nothing.
let cut = dbf.len() - 2 * 18 - 1;
assert_eq!(not_utf8(&dbf[..cut]), (1, vec![1, 0, 0, 1]));
assert_eq!(not_utf8(&dbf[..cut - 18]), (1, vec![1, 0, 0, 0]));
assert_eq!(not_utf8(&dbf[..40]), (0, Vec::new()));
assert_eq!(not_utf8(&[]), (0, Vec::new()));
}

#[test]
fn field_names_that_arent_utf8_are_said_with_their_count() {
assert_eq!(not_utf8_names_note(0), None);
let one = not_utf8_names_note(1).unwrap();
assert!(one.starts_with("Shapefile: 1 field name isn't valid UTF-8 and was read with U+FFFD"), "{one}");
let two = not_utf8_names_note(2).unwrap();
assert!(two.starts_with("Shapefile: 2 field names aren't valid UTF-8 and were read with U+FFFD"), "{two}");
assert!(two.ends_with(NO_OTHER_ENCODING), "{two}");
}
}
6 changes: 6 additions & 0 deletions crates/data/src/export.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ pub struct Exported {
/// Per field of the layer, text values `<Null>` the source read as null
/// ([`crate::source::Batch::null_text`], said by [`crate::source::null_text_note`]).
pub null_text: Vec<usize>,
/// Per field of the layer, text values the source read as UTF-8 that aren't UTF-8
/// ([`crate::source::Batch::not_utf8`], said by [`crate::source::not_utf8_note`]).
pub not_utf8: Vec<usize>,
/// Per field of the layer, date-times a GeoPackage holds in UTC rather than at their own
/// offset ([`crate::io::offset_dates`]; empty for the other formats).
pub utc_dates: Vec<usize>,
Expand Down Expand Up @@ -153,6 +156,7 @@ fn write_chunks_as(
let mut rounded_dates = Vec::new();
let mut null_text = Vec::new();
let mut read_notes = Vec::new();
let mut not_utf8 = Vec::new();
let mut utc_dates = Vec::new();
let mut fine_dates = Vec::new();
let mut written_values = Vec::new();
Expand Down Expand Up @@ -199,6 +203,7 @@ fn write_chunks_as(
rounded_dates = batch.rounded_dates;
null_text = batch.null_text;
read_notes = batch.notes;
not_utf8 = batch.not_utf8;
Ok(())
};
let mut table = None;
Expand Down Expand Up @@ -251,6 +256,7 @@ fn write_chunks_as(
unread_dates,
rounded_dates,
null_text,
not_utf8,
utc_dates,
fine_dates,
written_values,
Expand Down
2 changes: 2 additions & 0 deletions crates/data/src/export/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ impl FeatureSource for Runs {
rounded_dates: Vec::new(),
null_text: Vec::new(),
notes: Vec::new(),
not_utf8: Vec::new(),
})
}
fn visits_chunks(&self) -> bool {
Expand Down Expand Up @@ -208,6 +209,7 @@ fn a_streamed_export_writes_what_a_whole_export_writes() {
unread_dates: Vec::new(),
rounded_dates: Vec::new(),
null_text: Vec::new(),
not_utf8: Vec::new(),
utc_dates,
fine_dates: if ext == "gpkg" { crate::io::fine_dates(whole.features.iter(), &whole.fields) } else { Vec::new() },
written_values: if ext == "fgb" { crate::io::written_values(whole.features.iter(), &whole.fields) } else { Vec::new() },
Expand Down
32 changes: 21 additions & 11 deletions crates/data/src/io.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2875,13 +2875,19 @@ struct DbfRead {
fields: Vec<Field>,
decimals: Vec<u8>,
report: ReadReport,
/// For each field, its text values that aren't UTF-8 in a table read as UTF-8
/// ([`crate::dbf::not_utf8`]).
not_utf8: Vec<usize>,
}

/// Opens dBASE attributes with the Shapefile reader's code-page and date/number handling.
fn dbf_reader(mut bytes: Vec<u8>, cpg: Option<&[u8]>) -> Result<DbfRead> {
use std::io::Cursor;
let mut encoding_notes = Vec::new();
let encoding = crate::dbf::encoding(cpg, bytes.get(29).copied().unwrap_or(0), &mut encoding_notes);
// Counted before a date column is marked a character one below: its values aren't text.
let (names, not_utf8) = if encoding.is_utf8() { crate::dbf::not_utf8(&bytes) } else { (0, Vec::new()) };
encoding_notes.extend(crate::dbf::not_utf8_names_note(names));
crate::dbf::readable_header_date(&mut bytes);
let date_columns = crate::dbf::dates_as_text(&mut bytes);
crate::dbf::floats_as_numeric(&mut bytes);
Expand Down Expand Up @@ -2911,17 +2917,17 @@ fn dbf_reader(mut bytes: Vec<u8>, cpg: Option<&[u8]>) -> Result<DbfRead> {
})
.collect();
let fields = order.into_iter().map(|(n, t)| Field::new(n, t)).collect();
Ok(DbfRead { reader: dbf, fields, decimals, report: ReadReport { messages: encoding_notes, ..Default::default() } })
Ok(DbfRead { reader: dbf, fields, decimals, report: ReadReport { messages: encoding_notes, ..Default::default() }, not_utf8 })
}

/// Reads a geometry-free dBASE table, checking cancellation and counting progress per record.
/// Charges retained rows and cell allocations, with room for vector capacity, against the
/// same budget as delimited tables; archive members share the remaining allowance.
/// The bounded file read and header decoding finish before cancellation can be observed.
fn read_dbf(bytes: Vec<u8>, cpg: Option<&[u8]>, name: &str, budget: &mut usize) -> Result<(Dataset, ReadReport)> {
let DbfRead { mut reader, fields, decimals, mut report } = dbf_reader(bytes, cpg)?;
let DbfRead { mut reader, fields, decimals, mut report, not_utf8 } = dbf_reader(bytes, cpg)?;
let mut ds = Dataset::new(name, GeometryType::None, fields);
let mut counts = DbfCounts::new(ds.fields.len());
let mut counts = DbfCounts::new(ds.fields.len(), not_utf8);
let allowance = *budget;
let mut left = allowance;
crate::progress::begin(reader.header().num_records as usize);
Expand Down Expand Up @@ -2973,7 +2979,7 @@ fn read_dbf(bytes: Vec<u8>, cpg: Option<&[u8]>, name: &str, budget: &mut usize)
/// Converts one record to typed values, sharing loss counts with the containing dataset.
fn dbf_attrs(record: &shapefile::dbase::Record, fields: &[Field], counts: &mut DbfCounts) -> Vec<Value> {
use shapefile::dbase::FieldValue;
let DbfCounts { unread_dates, nonfinite, null_text } = counts;
let DbfCounts { unread_dates, nonfinite, null_text, .. } = counts;
fields
.iter()
.enumerate()
Expand Down Expand Up @@ -3009,22 +3015,25 @@ fn dbf_attrs(record: &shapefile::dbase::Record, fields: &[Field], counts: &mut D
}

/// Values a `.dbf` read changed, counted for each field by its position: dates that weren't
/// ones, numbers that weren't finite, and text `<Null>`, all read as null.
/// ones, numbers that weren't finite, and text `<Null>`, all read as null; and text that wasn't
/// UTF-8 in a table read as UTF-8, counted before the read ([`crate::dbf::not_utf8`]).
struct DbfCounts {
unread_dates: Vec<usize>,
nonfinite: Vec<usize>,
null_text: Vec<usize>,
not_utf8: Vec<usize>,
}

impl DbfCounts {
fn new(fields: usize) -> Self {
DbfCounts { unread_dates: vec![0; fields], nonfinite: vec![0; fields], null_text: Vec::new() }
fn new(fields: usize, not_utf8: Vec<usize>) -> Self {
DbfCounts { unread_dates: vec![0; fields], nonfinite: vec![0; fields], null_text: Vec::new(), not_utf8 }
}
}

/// Reports date, numeric and `<Null>` cells read as null, identically for tables and Shapefiles.
/// Reports date, numeric and `<Null>` cells read as null, and text that isn't UTF-8, identically
/// for tables and Shapefiles.
fn dbf_notices(ds: &Dataset, report: &mut ReadReport, counts: &DbfCounts) {
let DbfCounts { unread_dates, nonfinite, null_text } = counts;
let DbfCounts { unread_dates, nonfinite, null_text, not_utf8 } = counts;
let name = &ds.name;
report.unread_dates = crate::source::unread_dates_note(&ds.fields, unread_dates);
report.messages.extend(report.unread_dates.iter().map(|n| format!("{name}: {n}")));
Expand All @@ -3038,6 +3047,7 @@ fn dbf_notices(ds: &Dataset, report: &mut ReadReport, counts: &DbfCounts) {
report.messages.push(format!("{name}: {what} read as empty, in {}", named_fields(&ds.fields, nonfinite)));
}
report.messages.extend(crate::source::null_text_note(&ds.fields, null_text).map(|n| format!("{name}: {n}")));
report.messages.extend(crate::source::not_utf8_note(&ds.fields, not_utf8).map(|n| format!("{name}: {n}")));
}

/// The files of a Shapefile, read into memory.
Expand All @@ -3055,7 +3065,7 @@ pub(crate) fn read_shapefile_parts(parts: ShapefileParts, name: &str, crs: Optio
use shapefile::Shape;
use std::io::Cursor;
let fmt = |e: shapefile::Error| DataError::Format(format!("Shapefile: {e}"));
let DbfRead { reader: dbf, fields, decimals, mut report } = dbf_reader(parts.dbf, parts.cpg.as_deref())?;
let DbfRead { reader: dbf, fields, decimals, mut report, not_utf8 } = dbf_reader(parts.dbf, parts.cpg.as_deref())?;
let shapes = match parts.shx {
Some(shx) => shapefile::ShapeReader::with_shx(Cursor::new(parts.shp), Cursor::new(shx)),
None => shapefile::ShapeReader::new(Cursor::new(parts.shp)),
Expand All @@ -3074,7 +3084,7 @@ pub(crate) fn read_shapefile_parts(parts: ShapefileParts, name: &str, crs: Optio
}),
};
let (mut converted, mut unconverted, mut zm_shapes, mut not_numbers) = (0usize, 0usize, 0usize, 0usize);
let mut counts = DbfCounts::new(ds.fields.len());
let mut counts = DbfCounts::new(ds.fields.len(), not_utf8);
let mut source_kind = GeometryType::None;
let pt = |x: f64, y: f64| Coord::new(x, y);
for item in reader.iter_shapes_and_records() {
Expand Down
1 change: 1 addition & 0 deletions crates/data/src/source/geojson.rs
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,7 @@ impl FeatureSource for GeoJsonSource {
rounded_dates: Vec::new(),
null_text: Vec::new(),
notes: Vec::new(),
not_utf8: Vec::new(),
})
}
}
Expand Down
1 change: 1 addition & 0 deletions crates/data/src/source/gpkg_mem.rs
Original file line number Diff line number Diff line change
Expand Up @@ -417,6 +417,7 @@ impl FeatureSource for GpkgMemSource {
rounded_dates,
null_text: Vec::new(),
notes: Vec::new(),
not_utf8: Vec::new(),
})
}

Expand Down
1 change: 1 addition & 0 deletions crates/data/src/source/lidar.rs
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ impl FeatureSource for LidarSource {
rounded_dates: Vec::new(),
null_text: Vec::new(),
notes: Vec::new(),
not_utf8: Vec::new(),
})
}
}
Loading
Loading