From e1b744c73e594b6269ef5d65da7506cde7a99259 Mon Sep 17 00:00:00 2001 From: generalpawz Date: Fri, 6 Nov 2026 21:37:24 +0000 Subject: [PATCH 1/3] feat(parser): give event sinks the byte span of each data record A consumer of the event API could not say where a record it rejects sits in the file: a record the IFC model cannot represent was reported without a position (openbimrs/ifc#614, #616). The parser already computes the span for the reference check. A provided method hands it over without changing any sink that only implements `event`. Closes #18. --- CHANGELOG.md | 7 ++ openbim-step/src/parser.rs | 19 ++++- openbim-step/tests/partition_events.rs | 97 +++++++++++++++++++++++++- 3 files changed, 119 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 760e812..538d1ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and ## [Unreleased] +### Added + +- `EventSink::data_record(record, span)`: a provided method that gives a sink + each data record with its byte span, `#` through `;`. Its default forwards to + `event(Event::DataRecord(record))`, so existing sinks and closures keep + working (#18). + ## [0.11.3] - 2026-11-03 ### Changed diff --git a/openbim-step/src/parser.rs b/openbim-step/src/parser.rs index 2f11463..4f2c405 100644 --- a/openbim-step/src/parser.rs +++ b/openbim-step/src/parser.rs @@ -37,6 +37,19 @@ pub enum Event { pub trait EventSink { /// Receives one event. Events are delivered in source order. fn event(&mut self, event: Event); + + /// Receives one data record and its source span, from the `#` of its id + /// through its `;`. It is called in place of [`Self::event`] for + /// [`Event::DataRecord`], in the same position in source order. + /// + /// The default drops the span and forwards to [`Self::event`], so a sink + /// that does not need positions, such as a closure, implements only that. + /// The span is the one a reference diagnostic for the record reports. + #[inline] + fn data_record(&mut self, record: DataRecord, span: Span) { + let _ = span; + self.event(Event::DataRecord(record)); + } } impl EventSink for F @@ -617,14 +630,14 @@ impl<'a, S: Text<'a>> Parser<'a, S> { match self.parse_data_record(&id, token.span) { Ok(record) => { self.take_reals_without_point(); + let span = Span::new(start, self.last_end); if let Some(check) = &mut self.references { - let span = Span::new(start, self.last_end); check.record(&record, span, &mut self.diagnostics); } if let Some(spans) = &mut self.record_spans { - spans.push(Span::new(start, self.last_end)); + spans.push(span); } - sink.event(Event::DataRecord(record)); + sink.data_record(record, span); } Err(error) => self.recover_or_fail( error, diff --git a/openbim-step/tests/partition_events.rs b/openbim-step/tests/partition_events.rs index 3ee493d..bf75cb6 100644 --- a/openbim-step/tests/partition_events.rs +++ b/openbim-step/tests/partition_events.rs @@ -1,7 +1,10 @@ #![allow(missing_docs)] +use std::borrow::Cow; + use openbim_step::{ - data_record_spans, parse_events, partition_data_records, Event, EventSink, InstanceId, + data_record_spans, parse_events, parse_events_borrowed, parse_events_with, + partition_data_records, DataRecord, Event, EventSink, InstanceId, ParseOptions, Span, }; const INPUT: &[u8] = br#"ISO-10303-21; @@ -128,3 +131,95 @@ fn event_sink_receives_records_in_source_order_without_building_exchange() { ["start-header", "end-header", "start-data", "end-data"] ); } + +/// Keeps what the span-carrying method receives, and counts data records that +/// still arrive through `event`. +#[derive(Default)] +struct Positioned { + spans: Vec<(InstanceId, Span)>, + forwarded: usize, +} + +impl EventSink for Positioned { + fn event(&mut self, event: Event) { + if matches!(event, Event::DataRecord(_)) { + self.forwarded += 1; + } + } + + fn data_record(&mut self, record: DataRecord, span: Span) { + self.spans.push((record.id, span)); + } +} + +#[test] +fn a_sink_receives_each_data_record_with_the_span_of_its_bytes() { + let mut sink = Positioned::default(); + parse_events(INPUT, &mut sink).unwrap(); + assert_eq!(sink.forwarded, 0, "the override replaces the event"); + let spans: Vec = sink.spans.iter().map(|(_, span)| *span).collect(); + assert_eq!(spans, data_record_spans(INPUT).unwrap()); + let (id, span) = &sink.spans[0]; + assert_eq!(*id, InstanceId::from(1_u64)); + assert_eq!(&INPUT[span.start..span.end], b"#1=A('text ; #999= NOPE');"); +} + +/// The positions stay those of the original input past a recovered record, +/// the case the method exists for. +#[test] +fn record_spans_stay_exact_after_a_skipped_record() { + let input = b"ISO-10303-21;HEADER;FILE_DESCRIPTION((''),'2;1');\ + FILE_NAME('','',(),(),'','','');FILE_SCHEMA(('S'));ENDSEC;DATA;\n\ + #1=A(@);\n#2 = /* c */ B(1) ;\n#3=C(,);\n#4=D();\nENDSEC;END-ISO-10303-21;\n"; + let mut sink = Positioned::default(); + let diagnostics = parse_events_with(input, &mut sink, ParseOptions::lenient()).unwrap(); + assert_eq!(diagnostics.len(), 2); + let kept: Vec<&[u8]> = sink + .spans + .iter() + .map(|(_, span)| &input[span.start..span.end]) + .collect(); + assert_eq!(kept, [&b"#2 = /* c */ B(1) ;"[..], b"#4=D();"]); +} + +/// A sink that only implements `event` keeps receiving every record, whether +/// it is a type, a closure, or borrows its text. +#[test] +fn the_default_forwards_data_records_to_event() { + let mut typed = Collector::default(); + parse_events(INPUT, &mut typed).unwrap(); + assert_eq!(typed.data_ids.len(), 4); + + let mut owned = 0; + parse_events(INPUT, &mut |event: Event| { + owned += usize::from(matches!(event, Event::DataRecord(_))); + }) + .unwrap(); + assert_eq!(owned, 4); + + let mut borrowed = 0; + parse_events_borrowed( + INPUT, + &mut |event: Event>| { + borrowed += usize::from(matches!(event, Event::DataRecord(_))); + }, + ParseOptions::strict(), + ) + .unwrap(); + assert_eq!(borrowed, 4); +} + +/// The method is generic over the text type, like the trait. +#[test] +fn a_borrowing_sink_receives_the_same_spans() { + struct Spans(Vec); + impl<'a> EventSink> for Spans { + fn event(&mut self, _: Event>) {} + fn data_record(&mut self, _: DataRecord>, span: Span) { + self.0.push(span); + } + } + let mut sink = Spans(Vec::new()); + parse_events_borrowed(INPUT, &mut sink, ParseOptions::strict()).unwrap(); + assert_eq!(sink.0, data_record_spans(INPUT).unwrap()); +} From 71af179eeaa123c629b5e761a5ff971ea7f18af6 Mon Sep 17 00:00:00 2001 From: generalpawz Date: Fri, 6 Nov 2026 21:44:44 +0000 Subject: [PATCH 2/3] fix(recovery): stop before the next record when resynchronizing Recovery resynchronized at the next `;`, so a record missing its `;` took the following record with it, and no diagnostic named that record. A consumer that reserves skipped ids could reuse the id, and a conflicting duplicate went unreported (openbimrs/ifc#668). The scan now also stops before a `#id =` outside a literal, so the next record is parsed and kept, which recovers more than listing the swallowed ids would. It reads comments the way the scan does rather than through the lexer: the two disagree on a comment opener split by a control, and that made every `#` look ahead to the end of the file. Closes #20. --- CHANGELOG.md | 9 ++ openbim-step/src/lexer.rs | 2 +- openbim-step/src/parser.rs | 47 +++++++++- openbim-step/src/recovery.rs | 9 +- openbim-step/tests/parallel.rs | 34 +++++++ openbim-step/tests/references.rs | 151 ++++++++++++++++++++++++++++++- 6 files changed, 243 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 538d1ff..9efe6c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,15 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and `event(Event::DataRecord(record))`, so existing sinks and closures keep working (#18). +### Fixed + +- Lenient recovery stops before the next `#id =` outside a literal, so a record + missing its `;` no longer swallows the following record without a diagnostic + naming it. That record is now parsed and kept, and a conflict with it is + reported. Recovered output changes for such files: more records, the same or + more diagnostics, and shorter spans. A record missing its own `=`, or hidden + in text by an unbalanced quote, stays inside the skip's span (#20). + ## [0.11.3] - 2026-11-03 ### Changed diff --git a/openbim-step/src/lexer.rs b/openbim-step/src/lexer.rs index 5379d37..3cbf675 100644 --- a/openbim-step/src/lexer.rs +++ b/openbim-step/src/lexer.rs @@ -81,7 +81,7 @@ fn is_valid_binary(body: &[u8]) -> bool { matches!(body.first(), Some(b'0'..=b'3')) && body[1..].iter().all(u8::is_ascii_hexdigit) } -const fn is_ignored_control(byte: u8) -> bool { +pub(crate) const fn is_ignored_control(byte: u8) -> bool { matches!(byte, b'\t' | b'\n' | b'\r' | 0x0c) } diff --git a/openbim-step/src/parser.rs b/openbim-step/src/parser.rs index 4f2c405..8d8b4a5 100644 --- a/openbim-step/src/parser.rs +++ b/openbim-step/src/parser.rs @@ -8,7 +8,7 @@ use std::borrow::Cow; use std::collections::HashMap; use crate::escape; -use crate::lexer::{Lexer, Token}; +use crate::lexer::{is_ignored_control, Lexer, Token}; use crate::recovery::{Diagnostic, OnMalformed, ParseOptions, ParseOutcome}; use crate::references::ReferenceCheck; use crate::{ @@ -772,7 +772,9 @@ impl<'a, S: Text<'a>> Parser<'a, S> { /// mistaken for a record boundary. Scanning a literal's payload as code /// would let recovery fabricate records that were never in the source. /// A section terminator stops the scan *before* it is consumed, so - /// recovery can never swallow the end of `DATA`. + /// recovery can never swallow the end of `DATA`. So does the head of the + /// next record, so a record missing its `;` does not take the next one + /// with it. fn resync_from(&self, from: usize) -> usize { // Which literal the scanner is currently inside. STEP has three, and // all of them can contain bytes that look like record syntax. @@ -832,6 +834,7 @@ impl<'a, S: Text<'a>> Parser<'a, S> { .map_or(self.input.len(), |offset| position + 2 + offset + 2); } b';' => return position + 1, + b'#' if self.record_head_at(position) => return position, _ if self.section_end_at(position) => return position, _ => position += 1, }, @@ -840,6 +843,46 @@ impl<'a, S: Text<'a>> Parser<'a, S> { self.input.len() } + /// Whether a record head, `#digits` then `=`, starts at this `#`. Outside + /// a literal nothing else has that shape: a reference is followed by `,` + /// or `)`. + /// + /// Controls are allowed where the lexer allows them. Comments are read as + /// `resync_from` reads them, not through the lexer: the scan then skips + /// every byte this looks at past the next `#` too, which keeps recovery + /// linear on input built to make each `#` look ahead to the end of the file. + fn record_head_at(&self, position: usize) -> bool { + let input = self.input; + let mut cursor = position + 1; + while input.get(cursor).copied().is_some_and(is_ignored_control) { + cursor += 1; + } + if !input.get(cursor).is_some_and(u8::is_ascii_digit) { + return false; + } + while input + .get(cursor) + .is_some_and(|byte| byte.is_ascii_digit() || is_ignored_control(*byte)) + { + cursor += 1; + } + loop { + while input.get(cursor).is_some_and(u8::is_ascii_whitespace) { + cursor += 1; + } + if !input[cursor..].starts_with(b"/*") { + return input.get(cursor) == Some(&b'='); + } + let Some(offset) = input[cursor + 2..] + .windows(2) + .position(|window| window == b"*/") + else { + return false; + }; + cursor += 2 + offset + 2; + } + } + fn section_end_at(&self, position: usize) -> bool { let preceded_by_word = position .checked_sub(1) diff --git a/openbim-step/src/recovery.rs b/openbim-step/src/recovery.rs index 1a09480..0fe2360 100644 --- a/openbim-step/src/recovery.rs +++ b/openbim-step/src/recovery.rs @@ -225,8 +225,13 @@ impl Diagnostic { /// read, and for a real without a point. /// /// A skipped record's id is that of the record where the damage started. - /// Its span can cover further records, whose ids are not reported, and - /// the id is the token that was read, not a guarantee that no kept + /// Recovery stops before the next `#id =` outside a literal, so a record + /// missing its `;` does not take the next one with it. The span can still + /// cover further records whose ids are not reported: one whose own `=` is + /// missing, and one that an unbalanced quote turned into string text, + /// which cannot be told from a string that really holds record syntax. + /// + /// The id is the token that was read, not a guarantee that no kept /// record uses it. Compare it with a dangling reference's id after /// removing leading zeros. #[must_use] diff --git a/openbim-step/tests/parallel.rs b/openbim-step/tests/parallel.rs index 62d2e81..7766fd9 100644 --- a/openbim-step/tests/parallel.rs +++ b/openbim-step/tests/parallel.rs @@ -80,6 +80,40 @@ fn damaged_files_recover_and_diagnose_identically() { } } +/// Recovery can resume before a record head as well as past a `;`. The slices +/// must still land where the sequential parse does. +#[test] +fn records_missing_their_semicolon_recover_identically() { + for separator in ["\n", " ", ""] { + let mut records = String::new(); + for id in 1..=2_000 { + let end = if id % 7 == 0 { "" } else { ";" }; + write!(records, "#{id}=IFCPOINT(({id}.,1.)){end}{separator}") + .expect("writing to a String cannot fail"); + } + let input = format!("{HEADER}{records}\n{FOOTER}"); + let outcome = parse_with(input.as_bytes(), ParseOptions::lenient()).unwrap(); + assert_eq!(outcome.diagnostics.len(), 2_000 / 7); + assert_eq!(outcome.exchange.data.records.len(), 2_000 - 2_000 / 7); + assert_same(input.as_bytes(), &format!("separator {separator:?}")); + } +} + +/// Every `;` in turn removed from a file full of split-guess traps, including +/// those inside literals, where the removal changes nothing but the payload. +#[test] +fn removing_any_one_semicolon_recovers_identically() { + let input = file(40, true); + let data = HEADER.len(); + for position in data..input.len() - FOOTER.len() { + if input[position] == b';' { + let mut damaged = input.clone(); + damaged.remove(position); + assert_same(&damaged, &format!("without the ';' at {position}")); + } + } +} + #[test] fn structural_errors_are_the_sequential_errors() { let good = file(500, false); diff --git a/openbim-step/tests/references.rs b/openbim-step/tests/references.rs index 4c98f54..021dafe 100644 --- a/openbim-step/tests/references.rs +++ b/openbim-step/tests/references.rs @@ -221,10 +221,10 @@ fn a_skipped_record_without_an_id_has_no_instance() { } } -/// A record missing its `;` swallows the next one: one diagnostic, naming -/// only the first id, with a span over both. +/// A record missing its `;` no longer swallows the next one (#20): the next +/// record is kept, so its id cannot be reused and a conflict is reported. #[test] -fn a_skip_over_several_records_names_only_the_first() { +fn a_record_missing_its_semicolon_leaves_the_next_record_alone() { let input = exchange("#1=A(1)\n#2=B();\n#3=C();"); let outcome = parse_with(input.as_bytes(), ParseOptions::lenient()).unwrap(); assert_eq!( @@ -232,10 +232,153 @@ fn a_skip_over_several_records_names_only_the_first() { [(DiagnosticKind::SkippedRecord, "1".to_owned())] ); let span = outcome.diagnostics[0].span(); - assert!(input[span.start..span.end].contains("#2=B();")); + assert_eq!(&input[span.start..span.end], "#1=A(1)\n"); + let kept: Vec<&str> = outcome + .exchange + .data + .records + .iter() + .map(|record| record.id.as_str()) + .collect(); + assert_eq!(kept, ["2", "3"]); +} + +/// Each record missing its `;` gets its own diagnostic. +#[test] +fn consecutive_records_missing_their_semicolon_are_each_named() { + let input = exchange("#1=A(1)\n#2=B(2)\n#3=C();"); + let outcome = parse_with(input.as_bytes(), ParseOptions::lenient()).unwrap(); + assert_eq!( + kinds(&outcome.diagnostics), + [ + (DiagnosticKind::SkippedRecord, "1".to_owned()), + (DiagnosticKind::SkippedRecord, "2".to_owned()), + ] + ); assert_eq!(outcome.exchange.data.records.len(), 1); } +/// The damage that hid a conflicting duplicate (openbimrs/ifc#668): the record +/// after the skip is kept, so the reference check sees it. +#[test] +fn a_duplicate_after_a_missing_semicolon_is_reported() { + let (_, diagnostics) = run( + "#1=A(1);\n#30=B($)\n#1=A(2);", + ParseOptions::lenient().check_references(true), + ); + assert_eq!( + kinds(&diagnostics), + [ + (DiagnosticKind::SkippedRecord, "30".to_owned()), + (DiagnosticKind::DuplicateId, "1".to_owned()), + ] + ); +} + +/// The next record is found however it is written: on one line, without any +/// separator, and with what the lexer allows between `#id` and `=`. +#[test] +fn the_next_record_is_found_in_every_form_the_lexer_reads() { + for records in [ + "#1=A(1) #2=B();", + "#1=A(1)#2=B();", + "#1=A(1)\n#2 = B();", + "#1=A(1)\n#2\n=B();", + "#1=A(1)\n#2/* c */=B();", + "#1=A(1)\n#\n2=B();", + "#1=A(1)\r\n\t#2\t=\tB();", + ] { + let input = exchange(records); + let outcome = parse_with(input.as_bytes(), ParseOptions::lenient()).unwrap(); + assert_eq!( + kinds(&outcome.diagnostics), + [(DiagnosticKind::SkippedRecord, "1".to_owned())], + "{records:?}" + ); + let kept: Vec<&str> = outcome + .exchange + .data + .records + .iter() + .map(|record| record.id.as_str()) + .collect(); + assert_eq!(kept, ["2"], "{records:?}"); + } +} + +/// Record syntax in a literal is payload. Recovery must not read a record out +/// of it, and a reference is not a record head. +#[test] +fn recovery_does_not_read_a_record_head_out_of_a_literal_or_a_reference() { + for records in [ + "#1=A(@,'#2=B();');\n#3=C();", + "#1=A(@,'it''s #2=B();');\n#3=C();", + "#1=A(@,\"#2=B();\");\n#3=C();", + "#1=A(@) /* #2=B(); */\n#3=C();", + "#1=A(@,#2,(#4,#5),#6);\n#3=C();", + "#1=A(@,#2\n,#4);\n#3=C();", + ] { + let input = exchange(records); + let outcome = parse_with(input.as_bytes(), ParseOptions::lenient()).unwrap(); + assert_eq!( + kinds(&outcome.diagnostics), + [(DiagnosticKind::SkippedRecord, "1".to_owned())], + "{records:?}" + ); + let kept: Vec<&str> = outcome + .exchange + .data + .records + .iter() + .map(|record| record.id.as_str()) + .collect(); + assert_eq!(kept, ["3"], "{records:?}"); + } +} + +/// The limits `Diagnostic::instance` documents. A record without its `=` has +/// no head to stop at, and an unbalanced quote hides the next record in text. +/// Both stay inside the skip's span, so the bytes are still accounted for. +#[test] +fn records_recovery_cannot_tell_apart_stay_in_the_span() { + for (records, swallowed) in [ + ("#1=A(1)\n#2 B();\n#3=C();", "#2 B();"), + ("#1=A('x);\n#2=B('y);\n#3=C();", "#2=B("), + ] { + let input = exchange(records); + let outcome = parse_with(input.as_bytes(), ParseOptions::lenient()).unwrap(); + assert_eq!( + kinds(&outcome.diagnostics), + [(DiagnosticKind::SkippedRecord, "1".to_owned())], + "{records:?}" + ); + let span = outcome.diagnostics[0].span(); + assert!( + input[span.start..span.end].contains(swallowed), + "{records:?}" + ); + assert_eq!(outcome.exchange.data.records.len(), 1, "{records:?}"); + } +} + +/// Looking for the next record head must not read ahead from every `#`: each +/// of these repeats a shape that could make one `#` scan to the end of the file. +#[test] +fn looking_for_the_next_record_stays_linear_on_crafted_damage() { + const REPEATS: usize = 400_000; + for unit in ["#1 /\n* ", "#1 ", "#1\n", "#1 /**/ ", "# "] { + let input = exchange(&format!("#2=A(1 {}", unit.repeat(REPEATS))); + let started = std::time::Instant::now(); + let outcome = parse_with(input.as_bytes(), ParseOptions::lenient()).unwrap(); + assert_eq!(outcome.exchange.data.records.len(), 0, "{unit:?}"); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "{unit:?}: {:?}", + started.elapsed() + ); + } +} + /// One aggregation record forward-referencing many ids must stay linear: /// deduplicating missing ids by scanning a list would make this quadratic. #[test] From 1064776a0ab6706e7adbf5b29b27254577a342d8 Mon Sep 17 00:00:00 2001 From: generalpawz Date: Fri, 6 Nov 2026 21:45:23 +0000 Subject: [PATCH 3/3] chore(release): openbim-step 0.11.4 --- CHANGELOG.md | 5 ++++- Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9efe6c6..9c7c00d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and ## [Unreleased] +## [0.11.4] - 2026-11-06 + ### Added - `EventSink::data_record(record, span)`: a provided method that gives a sink @@ -485,7 +487,8 @@ slots) and are pinned in `tests/schema_graph.rs`. - Arbitrary-precision instance IDs, classic user-defined keywords, and legacy alphabet-selection decoding. - Incremental event parsing with bounded token buffering. -[Unreleased]: https://github.com/openbimrs/step/compare/v0.11.3...HEAD +[Unreleased]: https://github.com/openbimrs/step/compare/v0.11.4...HEAD +[0.11.4]: https://github.com/openbimrs/step/compare/v0.11.3...v0.11.4 [0.11.3]: https://github.com/openbimrs/step/compare/v0.11.2...v0.11.3 [0.11.2]: https://github.com/openbimrs/step/compare/v0.11.1...v0.11.2 [0.11.1]: https://github.com/openbimrs/step/compare/v0.11.0...v0.11.1 diff --git a/Cargo.lock b/Cargo.lock index 311d9ac..899559c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,7 +25,7 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "openbim-step" -version = "0.11.3" +version = "0.11.4" dependencies = [ "encoding_rs", "memchr", diff --git a/Cargo.toml b/Cargo.toml index a4053fb..2407ed2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["openbim-step"] resolver = "2" [workspace.package] -version = "0.11.3" +version = "0.11.4" edition = "2021" rust-version = "1.99" license = "AGPL-3.0-or-later OR LicenseRef-Commercial"