Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,24 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and

## [Unreleased]

## [0.11.4] - 2026-11-06

### Added

- `EventSink::data_record(record, span)`: a provided method that gives a sink
each data record with its byte span, `#` through `;`. Its default forwards to
`event(Event::DataRecord(record))`, so existing sinks and closures keep
working (#18).

### Fixed

- Lenient recovery stops before the next `#id =` outside a literal, so a record
missing its `;` no longer swallows the following record without a diagnostic
naming it. That record is now parsed and kept, and a conflict with it is
reported. Recovered output changes for such files: more records, the same or
more diagnostics, and shorter spans. A record missing its own `=`, or hidden
in text by an unbalanced quote, stays inside the skip's span (#20).

## [0.11.3] - 2026-11-03

### Changed
Expand Down Expand Up @@ -469,7 +487,8 @@ slots) and are pinned in `tests/schema_graph.rs`.
- Arbitrary-precision instance IDs, classic user-defined keywords, and legacy alphabet-selection decoding.
- Incremental event parsing with bounded token buffering.

[Unreleased]: https://github.com/openbimrs/step/compare/v0.11.3...HEAD
[Unreleased]: https://github.com/openbimrs/step/compare/v0.11.4...HEAD
[0.11.4]: https://github.com/openbimrs/step/compare/v0.11.3...v0.11.4
[0.11.3]: https://github.com/openbimrs/step/compare/v0.11.2...v0.11.3
[0.11.2]: https://github.com/openbimrs/step/compare/v0.11.1...v0.11.2
[0.11.1]: https://github.com/openbimrs/step/compare/v0.11.0...v0.11.1
Expand Down
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ members = ["openbim-step"]
resolver = "2"

[workspace.package]
version = "0.11.3"
version = "0.11.4"
edition = "2021"
rust-version = "1.99"
license = "AGPL-3.0-or-later OR LicenseRef-Commercial"
Expand Down
2 changes: 1 addition & 1 deletion openbim-step/src/lexer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ fn is_valid_binary(body: &[u8]) -> bool {
matches!(body.first(), Some(b'0'..=b'3')) && body[1..].iter().all(u8::is_ascii_hexdigit)
}

const fn is_ignored_control(byte: u8) -> bool {
pub(crate) const fn is_ignored_control(byte: u8) -> bool {
matches!(byte, b'\t' | b'\n' | b'\r' | 0x0c)
}

Expand Down
66 changes: 61 additions & 5 deletions openbim-step/src/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use std::borrow::Cow;
use std::collections::HashMap;

use crate::escape;
use crate::lexer::{Lexer, Token};
use crate::lexer::{is_ignored_control, Lexer, Token};
use crate::recovery::{Diagnostic, OnMalformed, ParseOptions, ParseOutcome};
use crate::references::ReferenceCheck;
use crate::{
Expand Down Expand Up @@ -37,6 +37,19 @@ pub enum Event<S = Str> {
pub trait EventSink<S = Str> {
/// Receives one event. Events are delivered in source order.
fn event(&mut self, event: Event<S>);

/// Receives one data record and its source span, from the `#` of its id
/// through its `;`. It is called in place of [`Self::event`] for
/// [`Event::DataRecord`], in the same position in source order.
///
/// The default drops the span and forwards to [`Self::event`], so a sink
/// that does not need positions, such as a closure, implements only that.
/// The span is the one a reference diagnostic for the record reports.
#[inline]
fn data_record(&mut self, record: DataRecord<S>, span: Span) {
let _ = span;
self.event(Event::DataRecord(record));
}
}

impl<S, F> EventSink<S> for F
Expand Down Expand Up @@ -617,14 +630,14 @@ impl<'a, S: Text<'a>> Parser<'a, S> {
match self.parse_data_record(&id, token.span) {
Ok(record) => {
self.take_reals_without_point();
let span = Span::new(start, self.last_end);
if let Some(check) = &mut self.references {
let span = Span::new(start, self.last_end);
check.record(&record, span, &mut self.diagnostics);
}
if let Some(spans) = &mut self.record_spans {
spans.push(Span::new(start, self.last_end));
spans.push(span);
}
sink.event(Event::DataRecord(record));
sink.data_record(record, span);
}
Err(error) => self.recover_or_fail(
error,
Expand Down Expand Up @@ -759,7 +772,9 @@ impl<'a, S: Text<'a>> Parser<'a, S> {
/// mistaken for a record boundary. Scanning a literal's payload as code
/// would let recovery fabricate records that were never in the source.
/// A section terminator stops the scan *before* it is consumed, so
/// recovery can never swallow the end of `DATA`.
/// recovery can never swallow the end of `DATA`. So does the head of the
/// next record, so a record missing its `;` does not take the next one
/// with it.
fn resync_from(&self, from: usize) -> usize {
// Which literal the scanner is currently inside. STEP has three, and
// all of them can contain bytes that look like record syntax.
Expand Down Expand Up @@ -819,6 +834,7 @@ impl<'a, S: Text<'a>> Parser<'a, S> {
.map_or(self.input.len(), |offset| position + 2 + offset + 2);
}
b';' => return position + 1,
b'#' if self.record_head_at(position) => return position,
_ if self.section_end_at(position) => return position,
_ => position += 1,
},
Expand All @@ -827,6 +843,46 @@ impl<'a, S: Text<'a>> Parser<'a, S> {
self.input.len()
}

/// Whether a record head, `#digits` then `=`, starts at this `#`. Outside
/// a literal nothing else has that shape: a reference is followed by `,`
/// or `)`.
///
/// Controls are allowed where the lexer allows them. Comments are read as
/// `resync_from` reads them, not through the lexer: the scan then skips
/// every byte this looks at past the next `#` too, which keeps recovery
/// linear on input built to make each `#` look ahead to the end of the file.
fn record_head_at(&self, position: usize) -> bool {
let input = self.input;
let mut cursor = position + 1;
while input.get(cursor).copied().is_some_and(is_ignored_control) {
cursor += 1;
}
if !input.get(cursor).is_some_and(u8::is_ascii_digit) {
return false;
}
while input
.get(cursor)
.is_some_and(|byte| byte.is_ascii_digit() || is_ignored_control(*byte))
{
cursor += 1;
}
loop {
while input.get(cursor).is_some_and(u8::is_ascii_whitespace) {
cursor += 1;
}
if !input[cursor..].starts_with(b"/*") {
return input.get(cursor) == Some(&b'=');
}
let Some(offset) = input[cursor + 2..]
.windows(2)
.position(|window| window == b"*/")
else {
return false;
};
cursor += 2 + offset + 2;
}
}

fn section_end_at(&self, position: usize) -> bool {
let preceded_by_word = position
.checked_sub(1)
Expand Down
9 changes: 7 additions & 2 deletions openbim-step/src/recovery.rs
Original file line number Diff line number Diff line change
Expand Up @@ -225,8 +225,13 @@ impl Diagnostic {
/// read, and for a real without a point.
///
/// A skipped record's id is that of the record where the damage started.
/// Its span can cover further records, whose ids are not reported, and
/// the id is the token that was read, not a guarantee that no kept
/// Recovery stops before the next `#id =` outside a literal, so a record
/// missing its `;` does not take the next one with it. The span can still
/// cover further records whose ids are not reported: one whose own `=` is
/// missing, and one that an unbalanced quote turned into string text,
/// which cannot be told from a string that really holds record syntax.
///
/// The id is the token that was read, not a guarantee that no kept
/// record uses it. Compare it with a dangling reference's id after
/// removing leading zeros.
#[must_use]
Expand Down
34 changes: 34 additions & 0 deletions openbim-step/tests/parallel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,40 @@ fn damaged_files_recover_and_diagnose_identically() {
}
}

/// Recovery can resume before a record head as well as past a `;`. The slices
/// must still land where the sequential parse does.
#[test]
fn records_missing_their_semicolon_recover_identically() {
for separator in ["\n", " ", ""] {
let mut records = String::new();
for id in 1..=2_000 {
let end = if id % 7 == 0 { "" } else { ";" };
write!(records, "#{id}=IFCPOINT(({id}.,1.)){end}{separator}")
.expect("writing to a String cannot fail");
}
let input = format!("{HEADER}{records}\n{FOOTER}");
let outcome = parse_with(input.as_bytes(), ParseOptions::lenient()).unwrap();
assert_eq!(outcome.diagnostics.len(), 2_000 / 7);
assert_eq!(outcome.exchange.data.records.len(), 2_000 - 2_000 / 7);
assert_same(input.as_bytes(), &format!("separator {separator:?}"));
}
}

/// Every `;` in turn removed from a file full of split-guess traps, including
/// those inside literals, where the removal changes nothing but the payload.
#[test]
fn removing_any_one_semicolon_recovers_identically() {
let input = file(40, true);
let data = HEADER.len();
for position in data..input.len() - FOOTER.len() {
if input[position] == b';' {
let mut damaged = input.clone();
damaged.remove(position);
assert_same(&damaged, &format!("without the ';' at {position}"));
}
}
}

#[test]
fn structural_errors_are_the_sequential_errors() {
let good = file(500, false);
Expand Down
97 changes: 96 additions & 1 deletion openbim-step/tests/partition_events.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
#![allow(missing_docs)]

use std::borrow::Cow;

use openbim_step::{
data_record_spans, parse_events, partition_data_records, Event, EventSink, InstanceId,
data_record_spans, parse_events, parse_events_borrowed, parse_events_with,
partition_data_records, DataRecord, Event, EventSink, InstanceId, ParseOptions, Span,
};

const INPUT: &[u8] = br#"ISO-10303-21;
Expand Down Expand Up @@ -128,3 +131,95 @@ fn event_sink_receives_records_in_source_order_without_building_exchange() {
["start-header", "end-header", "start-data", "end-data"]
);
}

/// Keeps what the span-carrying method receives, and counts data records that
/// still arrive through `event`.
#[derive(Default)]
struct Positioned {
spans: Vec<(InstanceId, Span)>,
forwarded: usize,
}

impl EventSink for Positioned {
fn event(&mut self, event: Event) {
if matches!(event, Event::DataRecord(_)) {
self.forwarded += 1;
}
}

fn data_record(&mut self, record: DataRecord, span: Span) {
self.spans.push((record.id, span));
}
}

#[test]
fn a_sink_receives_each_data_record_with_the_span_of_its_bytes() {
let mut sink = Positioned::default();
parse_events(INPUT, &mut sink).unwrap();
assert_eq!(sink.forwarded, 0, "the override replaces the event");
let spans: Vec<Span> = sink.spans.iter().map(|(_, span)| *span).collect();
assert_eq!(spans, data_record_spans(INPUT).unwrap());
let (id, span) = &sink.spans[0];
assert_eq!(*id, InstanceId::from(1_u64));
assert_eq!(&INPUT[span.start..span.end], b"#1=A('text ; #999= NOPE');");
}

/// The positions stay those of the original input past a recovered record,
/// the case the method exists for.
#[test]
fn record_spans_stay_exact_after_a_skipped_record() {
let input = b"ISO-10303-21;HEADER;FILE_DESCRIPTION((''),'2;1');\
FILE_NAME('','',(),(),'','','');FILE_SCHEMA(('S'));ENDSEC;DATA;\n\
#1=A(@);\n#2 = /* c */ B(1) ;\n#3=C(,);\n#4=D();\nENDSEC;END-ISO-10303-21;\n";
let mut sink = Positioned::default();
let diagnostics = parse_events_with(input, &mut sink, ParseOptions::lenient()).unwrap();
assert_eq!(diagnostics.len(), 2);
let kept: Vec<&[u8]> = sink
.spans
.iter()
.map(|(_, span)| &input[span.start..span.end])
.collect();
assert_eq!(kept, [&b"#2 = /* c */ B(1) ;"[..], b"#4=D();"]);
}

/// A sink that only implements `event` keeps receiving every record, whether
/// it is a type, a closure, or borrows its text.
#[test]
fn the_default_forwards_data_records_to_event() {
let mut typed = Collector::default();
parse_events(INPUT, &mut typed).unwrap();
assert_eq!(typed.data_ids.len(), 4);

let mut owned = 0;
parse_events(INPUT, &mut |event: Event| {
owned += usize::from(matches!(event, Event::DataRecord(_)));
})
.unwrap();
assert_eq!(owned, 4);

let mut borrowed = 0;
parse_events_borrowed(
INPUT,
&mut |event: Event<Cow<'_, str>>| {
borrowed += usize::from(matches!(event, Event::DataRecord(_)));
},
ParseOptions::strict(),
)
.unwrap();
assert_eq!(borrowed, 4);
}

/// The method is generic over the text type, like the trait.
#[test]
fn a_borrowing_sink_receives_the_same_spans() {
struct Spans(Vec<Span>);
impl<'a> EventSink<Cow<'a, str>> for Spans {
fn event(&mut self, _: Event<Cow<'a, str>>) {}
fn data_record(&mut self, _: DataRecord<Cow<'a, str>>, span: Span) {
self.0.push(span);
}
}
let mut sink = Spans(Vec::new());
parse_events_borrowed(INPUT, &mut sink, ParseOptions::strict()).unwrap();
assert_eq!(sink.0, data_record_spans(INPUT).unwrap());
}
Loading
Loading