From 9180f3ce4f793fb07b3e41aad65df0f2945490b0 Mon Sep 17 00:00:00 2001 From: msomierick Date: Wed, 25 Jun 2025 09:54:23 +0300 Subject: [PATCH] Add token line stream --- examples/iterator_demo.rs | 143 +-- examples/molecular_formula.rs | 32 +- examples/pretty_printer.rs | 54 +- examples/streaming_demo.rs | 49 +- src/lexer.rs | 4 + src/lib.rs | 3 +- src/main.rs | 85 +- src/parser.rs | 1840 ++++++++------------------------- src/token.rs | 21 + src/token_stream.rs | 100 ++ 10 files changed, 769 insertions(+), 1562 deletions(-) create mode 100644 src/token_stream.rs diff --git a/examples/iterator_demo.rs b/examples/iterator_demo.rs index effe98d..da2a450 100644 --- a/examples/iterator_demo.rs +++ b/examples/iterator_demo.rs @@ -1,7 +1,9 @@ +use smiles::lexer::Lexer; use smiles::parser::Parser; +use smiles::token_stream::TokenStream; fn main() { - println!("=== Iterator-based SMILES Parser Demo ===\n"); + println!("=== Iterator-based SMILES Parser Demo (TokenStream) ===\n"); // Test data with mixed valid/invalid SMILES and empty lines let multi_smiles = r#"C @@ -25,110 +27,67 @@ C1CCCCC1 // Example 1: Process line by line with an iterator println!("\n1. Processing line-by-line:"); - let parser = Parser::new(multi_smiles); - - for result in parser.parse_lines() { - match result { + let lexer = Lexer::new(multi_smiles); + let mut token_stream = TokenStream::new(lexer); + let mut line_num = 1; + while let Some(mut line_tokens) = token_stream.stream_line() { + let mut peekable_tokens = line_tokens.by_ref().peekable(); + match Parser::parse_smiles_from_tokens(&mut peekable_tokens, multi_smiles) { Ok(_smiles) => { - println!(" ✓ Valid SMILES"); + println!(" \u{2713} Valid SMILES"); } - Err(error) => { + Err(parse_error) => { println!( - " ✗ Line {}: {} - {}", - error.line_number, - match error.error_type { - smiles::error::SmilesErrorType::Lexical(_) => "Lexical Error", - smiles::error::SmilesErrorType::Parse(_) => "Parse Error", - }, - error.error_type + " \u{2717} Line {}: Parse Error - {}", + line_num, parse_error ); } } + // Handle lexical errors for this line + let line_errors = peekable_tokens.collect::>(); + for err in line_errors { + if let Err(lex_errs) = err { + for lex in lex_errs.iter() { + println!(" \u{2717} Line {}: Lexical Error - {}", line_num, lex); + } + } + } + line_num += 1; } - // Example 2: Collect all results at once + // Example 2: Bulk collection (valid/invalid count) println!("\n2. Bulk collection:"); - let parser = Parser::new(multi_smiles); - let (valid_smiles, errors) = parser.parse_lines().collect_results(); - - println!(" Successfully parsed: {} SMILES", valid_smiles.len()); - println!(" Errors encountered: {} lines", errors.len()); - - // Example 3: Filter only valid results - println!("\n3. Processing only valid SMILES:"); - let parser = Parser::new(multi_smiles); - let valid_smiles = parser.parse_lines().collect_valid(); - - for (i, _smiles) in valid_smiles.iter().enumerate() { - println!(" SMILES {}: Valid", i + 1); - } - - // Example 4: Memory-efficient streaming for large files - println!("\n4. Large file simulation (streaming):"); - let mut large_smiles = String::new(); - for i in 0..1000 { - if i % 100 == 0 { - large_smiles.push_str("Invalid@@\n"); - } else { - large_smiles.push_str("C\n"); + let lexer = Lexer::new(multi_smiles); + let mut token_stream = TokenStream::new(lexer); + let mut valid_smiles = 0; + let mut error_lines = 0; + while let Some(mut line_tokens) = token_stream.stream_line() { + let mut peekable_tokens = line_tokens.by_ref().peekable(); + match Parser::parse_smiles_from_tokens(&mut peekable_tokens, multi_smiles) { + Ok(_smiles) => valid_smiles += 1, + Err(_) => error_lines += 1, } - } - - let parser = Parser::new(&large_smiles); - let mut valid_count = 0; - let mut error_count = 0; - - // Process without collecting all in memory - for result in parser.parse_lines() { - match result { - Ok(_) => valid_count += 1, - Err(_) => error_count += 1, + // Count lexical errors + let line_errors = peekable_tokens.collect::>(); + for err in line_errors { + if err.is_err() { + error_lines += 1; + } } } + println!(" Successfully parsed: {} SMILES", valid_smiles); + println!(" Errors encountered: {} lines", error_lines); - println!( - " Processed 1000 lines: {} valid, {} errors", - valid_count, error_count - ); - - // Example 5: Error analysis - println!("\n5. Detailed error analysis:"); - let error_test = "C\n[Unclosed\nCC\nBad@@Symbol\nCCO\n"; - let parser = Parser::new(error_test); - - for result in parser.parse_lines() { - match result { - Ok(_) => println!(" ✓ Valid SMILES"), - Err(error) => { - println!( - " ✗ Line {}: {} - {}", - error.line_number, - match error.error_type { - smiles::error::SmilesErrorType::Lexical(_) => "Lexical Error", - smiles::error::SmilesErrorType::Parse(_) => "Parse Error", - }, - error - ); - } + // Example 3: Filter only valid results + println!("\n3. Processing only valid SMILES:"); + let lexer = Lexer::new(multi_smiles); + let mut token_stream = TokenStream::new(lexer); + let mut valid_count = 0; + while let Some(mut line_tokens) = token_stream.stream_line() { + let mut peekable_tokens = line_tokens.by_ref().peekable(); + if Parser::parse_smiles_from_tokens(&mut peekable_tokens, multi_smiles).is_ok() { + valid_count += 1; } } - - // Example 6: Convenience method - println!("\n6. Using convenience method:"); - let parser = Parser::new("C\nCC\nBad@\nCCO"); - let (valid, errors) = parser.parse_all_lines(); - println!( - " Convenience method: {} valid, {} errors", - valid.len(), - errors.len() - ); - - println!("\n=== API Features Demonstrated ==="); - println!("✓ Iterator-based processing (lazy evaluation)"); - println!("✓ Structured error reporting with line numbers"); - println!("✓ Unified error handling (lexical + parse errors)"); - println!("✓ Memory-efficient streaming for large files"); - println!("✓ Flexible collection methods (all, valid-only, errors-only)"); - println!("✓ Error recovery between lines"); - println!("✓ Convenience methods for common use cases"); + println!(" Valid SMILES lines: {}", valid_count); } diff --git a/examples/molecular_formula.rs b/examples/molecular_formula.rs index b62158b..84a47f6 100644 --- a/examples/molecular_formula.rs +++ b/examples/molecular_formula.rs @@ -218,17 +218,29 @@ fn main() { for (smiles_str, name) in examples { println!("\n{}: {}", name, smiles_str); - - let mut parser = Parser::new(smiles_str); - match parser.parse_smiles() { - Ok(smiles) => { - let mut counter = MolecularFormulaCounter::new(); - counter.visit_smiles(&smiles); - let formula = counter.get_formula(); - println!(" Molecular Formula: {}", formula); + let lexer = smiles::lexer::Lexer::new(smiles_str); + let mut token_stream = smiles::token_stream::TokenStream::new(lexer); + if let Some(mut line_tokens) = token_stream.stream_line() { + let mut peekable_tokens = line_tokens.by_ref().peekable(); + match Parser::parse_smiles_from_tokens(&mut peekable_tokens, smiles_str) { + Ok(smiles) => { + let mut counter = MolecularFormulaCounter::new(); + counter.visit_smiles(&smiles); + let formula = counter.get_formula(); + println!(" Molecular Formula: {}", formula); + } + Err(err) => { + println!(" Parse error: {}", err); + } } - Err(err) => { - println!(" Parse error: {}", err); + // Optionally print lexical errors + let line_errors = peekable_tokens.collect::>(); + for err in line_errors { + if let Err(lex_errs) = err { + for lex in lex_errs.iter() { + println!(" Lexical error: {}", lex); + } + } } } } diff --git a/examples/pretty_printer.rs b/examples/pretty_printer.rs index 0e382d9..efa7c80 100644 --- a/examples/pretty_printer.rs +++ b/examples/pretty_printer.rs @@ -1,5 +1,5 @@ //! Example demonstrating the visitor pattern for pretty-printing SMILES -//! +//! //! Run with: cargo run --example pretty_printer use smiles::{parser::Parser, visitor::*}; @@ -148,7 +148,7 @@ impl Visitor for PrettyPrinter { smiles::ast::RingNumber::Single(n) => n.to_string(), smiles::ast::RingNumber::Double(n) => format!("%{}", n), }; - + if let Some(bond) = &ring_bond.bond { self.write_line(&format!("Ring Bond: {} with bond", number_str)); self.with_indent(|printer| { @@ -187,30 +187,42 @@ impl Visitor for PrettyPrinter { fn main() { // Example SMILES strings to parse and pretty-print let examples = vec![ - "C", // Simple carbon - "CCO", // Ethanol - "c1ccccc1", // Benzene - "CC(C)C", // Isobutane - "[13C]", // Carbon-13 isotope - "[NH4+]", // Ammonium ion - "C1CC1", // Cyclopropane - "CC(=O)O", // Acetic acid - "c1cc(Cl)ccc1", // Chlorobenzene - "CC(C)(C)C", // tert-Butane + "C", // Simple carbon + "CCO", // Ethanol + "c1ccccc1", // Benzene + "CC(C)C", // Isobutane + "[13C]", // Carbon-13 isotope + "[NH4+]", // Ammonium ion + "C1CC1", // Cyclopropane + "CC(=O)O", // Acetic acid + "c1cc(Cl)ccc1", // Chlorobenzene + "CC(C)(C)C", // tert-Butane ]; for smiles_str in examples { println!("=== Parsing: {} ===", smiles_str); - - let mut parser = Parser::new(smiles_str); - match parser.parse_smiles() { - Ok(smiles) => { - let mut printer = PrettyPrinter::new(); - printer.visit_smiles(&smiles); - println!("{}", printer.get_output()); + let lexer = smiles::lexer::Lexer::new(smiles_str); + let mut token_stream = smiles::token_stream::TokenStream::new(lexer); + if let Some(mut line_tokens) = token_stream.stream_line() { + let mut peekable_tokens = line_tokens.by_ref().peekable(); + match Parser::parse_smiles_from_tokens(&mut peekable_tokens, smiles_str) { + Ok(smiles) => { + let mut printer = PrettyPrinter::new(); + printer.visit_smiles(&smiles); + println!("{}", printer.get_output()); + } + Err(err) => { + println!("Parse error: {}", err); + } } - Err(err) => { - println!("Parse error: {}", err); + // Optionally print lexical errors + let line_errors = peekable_tokens.collect::>(); + for err in line_errors { + if let Err(lex_errs) = err { + for lex in lex_errs.iter() { + println!("Lexical error: {}", lex); + } + } } } } diff --git a/examples/streaming_demo.rs b/examples/streaming_demo.rs index 5f0f01a..46cc631 100644 --- a/examples/streaming_demo.rs +++ b/examples/streaming_demo.rs @@ -1,7 +1,9 @@ +use smiles::lexer::Lexer; use smiles::parser::Parser; +use smiles::token_stream::TokenStream; fn main() { - println!("=== Streaming Multi-line SMILES Parser Demo ===\n"); + println!("=== Streaming Multi-line SMILES Parser Demo (TokenStream) ===\n"); // Test data with mixed valid/invalid SMILES and empty lines let multi_smiles = r#"C @@ -23,22 +25,43 @@ C1CCCCC1 println!("{}", multi_smiles); println!("=== Parsing Results ==="); - let mut parser = Parser::new(multi_smiles); - let (successes, parse_errors, lexical_errors) = parser.parse_multiple_lines(); + let lexer = Lexer::new(multi_smiles); + let mut token_stream = TokenStream::new(lexer); + let mut successes = Vec::new(); + let mut parse_errors = Vec::new(); + let mut lexical_errors = Vec::new(); + let mut line_num = 1; + while let Some(mut line_tokens) = token_stream.stream_line() { + let mut peekable_tokens = line_tokens.by_ref().peekable(); + match Parser::parse_smiles_from_tokens(&mut peekable_tokens, multi_smiles) { + Ok(smiles) => successes.push(smiles), + Err(e) => parse_errors.push(format!("Line {}: {}", line_num, e)), + } + // Collect lexical errors for this line + let line_errs = peekable_tokens.collect::>(); + for err in line_errs { + if let Err(lex_errs) = err { + for lex in lex_errs.iter() { + lexical_errors.push(format!("Line {}: {}", line_num, lex)); + } + } + } + line_num += 1; + } // Report successful parses for (i, _smiles) in successes.iter().enumerate() { - println!("SMILES {}: ✓ Successfully parsed", i + 1); + println!("SMILES {}: \u{2713} Successfully parsed", i + 1); } // Report parse errors - for (i, error) in parse_errors.iter().enumerate() { - println!("Parse Error {}: {}", i + 1, error); + for err in &parse_errors { + println!("Parse Error: {}", err); } // Report lexical errors - for (i, error) in lexical_errors.iter().enumerate() { - println!("Lexical Error {}: {}", i + 1, error); + for err in &lexical_errors { + println!("Lexical Error: {}", err); } println!(); @@ -56,9 +79,9 @@ C1CCCCC1 println!("Lexical errors: {}", lexical_errors.len()); println!("\n=== Key Features Demonstrated ==="); - println!("✓ Streaming token-based parsing (no upfront line splitting)"); - println!("✓ Error recovery (invalid SMILES don't affect subsequent ones)"); - println!("✓ Empty line handling"); - println!("✓ Ring bond state isolation between SMILES"); - println!("✓ Memory efficient processing"); + println!("\u{2713} Streaming token-based parsing (no upfront line splitting)"); + println!("\u{2713} Error recovery (invalid SMILES don't affect subsequent ones)"); + println!("\u{2713} Empty line handling"); + println!("\u{2713} Ring bond state isolation between SMILES"); + println!("\u{2713} Memory efficient processing"); } diff --git a/src/lexer.rs b/src/lexer.rs index 0e8d2bb..7183480 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -519,6 +519,10 @@ impl<'src> Lexer<'src> { false } + + pub fn is_eof(&self) -> bool { + self.cursor.is_eof() + } } /// Creates an iterator that produces tokens from the input string. diff --git a/src/lib.rs b/src/lib.rs index 5a3cb12..d7d77fc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,9 +5,10 @@ pub mod lexer; pub mod parser; pub mod span; pub mod token; +pub mod token_stream; pub mod visitor; // Re-export key types for convenience -pub use parser::{Parser, SmilesLineIterator}; +pub use parser::Parser; pub use error::{SmilesLineError, SmilesLineResult, SmilesErrorType}; pub use ast::Smiles; diff --git a/src/main.rs b/src/main.rs index 068576d..a9f6098 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,5 @@ -pub mod ast; -pub mod cursor; -pub mod error; -pub mod lexer; -pub mod parser; -pub mod span; -pub mod token; +extern crate smiles; +use smiles::*; fn main() { let args: Vec = std::env::args().collect(); @@ -73,7 +68,7 @@ fn main() { } }; - let mut lexer = lexer::Lexer::new(&source); + let mut token_stream = token_stream::TokenStream::new(lexer::Lexer::new(&source)); println!("Source from {}:", source_description); for (line_num, line) in source.lines().enumerate() { @@ -82,42 +77,52 @@ fn main() { println!(); println!("Tokens with positions:"); - loop { - let token = lexer.lex_next_token(); - if token.kind.is_eof() { - let (start_pos, end_pos) = token.span.to_positions(lexer.position_tracker()); - println!( - "{:?} at line {}, column {} to line {}, column {}", - token.kind, start_pos.line, start_pos.column, end_pos.line, end_pos.column - ); - break; - } - - let (start_pos, end_pos) = token.span.to_positions(lexer.position_tracker()); - if token.kind == token::TokenKind::EndOfLine { - // For EndOfLine tokens, show it as a single line position - println!( - "{:?} at line {}, column {} (span: {}..{})", - token.kind, start_pos.line, start_pos.column, token.span.start, token.span.end - ); - } else { - println!( - "{:?} at line {}, column {} to column {} (span: {}..{})", - token.kind, - start_pos.line, - start_pos.column, - end_pos.column, - token.span.start, - token.span.end - ); + let _position_tracker = token_stream.position_tracker(); + while let Some(line_tokens) = token_stream.stream_line() { + let results: Vec<_> = line_tokens.collect(); + let position_tracker = token_stream.position_tracker(); + for result in results { + match result { + Ok(token) => { + let (start_pos, end_pos) = token.span.to_positions(position_tracker); + if token.kind == token::TokenKind::EndOfLine { + println!( + "{:?} at line {}, column {} (span: {}..{})", + token.kind, + start_pos.line, + start_pos.column, + token.span.start, + token.span.end + ); + } else { + println!( + "{:?} at line {}, column {} to column {} (span: {}..{})", + token.kind, + start_pos.line, + start_pos.column, + end_pos.column, + token.span.start, + token.span.end + ); + } + } + Err(lex_errs) => { + for error in lex_errs.iter() { + let (start_pos, end_pos) = error.span().to_positions(position_tracker); + println!( + " Lexical error: {} at line {}, column {} to column {}", + error, start_pos.line, start_pos.column, end_pos.column + ); + } + } + } } } // Display any errors - if !lexer.errors().is_empty() { - println!("\nErrors found:"); - for error in lexer.errors() { - let (start_pos, end_pos) = error.span().to_positions(lexer.position_tracker()); + if !token_stream.errors().is_empty() { + for error in token_stream.errors() { + let (start_pos, end_pos) = error.span().to_positions(token_stream.position_tracker()); println!( " {} at line {}, column {} to column {}", error, start_pos.line, start_pos.column, end_pos.column diff --git a/src/parser.rs b/src/parser.rs index 488d023..c51f298 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1,1481 +1,551 @@ -use crate::{ - ast::*, - error::{ - LexicalError, ParseError, ParseErrorType, ParseResult, SmilesLineError, SmilesLineResult, - }, - lexer::Lexer, - span::Span, - token::{Token, TokenKind}, -}; +use super::token_stream::LineErrors; +use crate::ast::*; +use crate::error::ParseError; +use crate::token::{Token, TokenKind}; +use std::iter::Peekable; -/// Parser for SMILES notation -pub struct Parser<'src> { - lexer: Lexer<'src>, - current_token: Option, - source: &'src str, -} - -impl<'src> Parser<'src> { - /// Create a new parser for the given source - pub fn new(source: &'src str) -> Self { - let mut lexer = Lexer::new(source); - let current_token = Self::next_token(&mut lexer); - - Self { - lexer, - current_token, - source, - } - } - - /// Parse multiple SMILES strings with streaming, token-based approach. - /// Each SMILES string is terminated by an EndOfLine token or EOF. - /// This method efficiently processes tokens without splitting the source upfront. - /// - /// # Features - /// - **Streaming**: Processes tokens directly from the lexer without upfront line splitting - /// - **Error Recovery**: Parse errors in one SMILES don't affect subsequent SMILES parsing - /// - **Memory Efficient**: Uses a single lexer/parser instance for the entire source - /// - **Syntactic Only**: Produces grammatically valid ASTs without semantic validation - /// - **Empty Line Handling**: Automatically skips empty lines (consecutive EndOfLine tokens) - /// - /// # Returns - /// A tuple containing: - /// - `Vec`: Successfully parsed SMILES ASTs - /// - `Vec`: Parse errors encountered during parsing - /// - `Vec`: Lexical errors encountered during tokenization - /// - /// # Example - /// ```rust - /// use smiles::parser::Parser; - /// - /// let source = "C\nCC\nInvalid@\nCCO"; - /// let mut parser = Parser::new(source); - /// let (results, parse_errors, lexical_errors) = parser.parse_multiple_lines(); - /// - /// assert_eq!(results.len(), 3); // "C", "CC", "CCO" - /// assert!(!parse_errors.is_empty()); // Parse errors from "Invalid@" - /// assert!(!lexical_errors.is_empty()); // Lexical errors from "Invalid@" - /// ``` - pub fn parse_multiple_lines(&mut self) -> (Vec, Vec, Vec) { - let mut results = Vec::new(); - let mut parse_errors = Vec::new(); - let mut lexical_errors = Vec::new(); - - loop { - // Skip any leading EndOfLine tokens (empty lines) - while let Some(token) = &self.current_token { - if token.kind == TokenKind::EndOfLine { - self.advance(); - } else { - break; - } - } - - // Check if we've reached EOF - if self.current_token.is_none() { - break; - } - - // Parse a single SMILES string - match self.parse_smiles_line() { - Ok(smiles) => { - results.push(smiles); - } - Err(parse_error) => { - parse_errors.push(parse_error); - } - } - - // Collect any lexical errors that occurred during this line - let new_lexical_errors = self.lexer.errors(); - lexical_errors.extend(new_lexical_errors.iter().cloned()); - - // Clear lexer errors for next SMILES - self.lexer.clear_errors(); +/// Helper span methods for Atom and Link +impl Atom { + pub fn span(&self) -> crate::span::Span { + match self { + Atom::Bracket(bracket) => bracket.span, + Atom::Organic(organic) => match organic { + OrganicAtom::Aliphatic { span, .. } => *span, + OrganicAtom::Aromatic { span, .. } => *span, + }, + Atom::Wildcard { span } => *span, } - - (results, parse_errors, lexical_errors) - } - - /// Creates an iterator that parses SMILES line by line, yielding either a valid - /// SMILES AST or a detailed error for each line. - /// - /// This is the recommended API for processing multi-line SMILES files as it: - /// - Provides streaming/iterator-based processing - /// - Returns structured errors with line numbers - /// - Handles both lexical and parse errors uniformly - /// - Supports efficient error recovery - /// - Is memory efficient (doesn't buffer all results) - /// - /// # Example - /// ```rust - /// use smiles::parser::Parser; - /// - /// let source = "C\nCC\nInvalid@\nCCO\n"; - /// let parser = Parser::new(source); - /// - /// for (line_num, result) in parser.parse_lines().enumerate() { - /// match result { - /// Ok(smiles) => println!("Line {}: Parsed successfully", line_num + 1), - /// Err(error) => println!("Line {}: {}", line_num + 1, error), - /// } - /// } - /// ``` - pub fn parse_lines(self) -> SmilesLineIterator<'src> { - SmilesLineIterator::new(self) - } - - /// Convenience method to collect all results from the iterator. - /// Returns separate vectors for valid results and errors. - /// - /// This is useful when you want to process all lines and then handle - /// results in bulk, similar to the old `parse_multiple_lines` API. - /// - /// # Example - /// ```rust - /// use smiles::parser::Parser; - /// - /// let source = "C\nCC\nInvalid@\nCCO\n"; - /// let parser = Parser::new(source); - /// let (valid_smiles, errors) = parser.parse_all_lines(); - /// - /// println!("Parsed {} valid SMILES", valid_smiles.len()); - /// println!("Encountered {} errors", errors.len()); - /// ``` - pub fn parse_all_lines(self) -> (Vec, Vec) { - self.parse_lines().collect_results() } - - /// Skip tokens until we reach EndOfLine or EOF for error recovery - fn skip_to_next_line(&mut self) { - while let Some(token) = &self.current_token { - if token.kind == TokenKind::EndOfLine { - self.advance(); // Consume the EndOfLine - break; - } - self.advance(); +} +impl Link { + pub fn span(&self) -> crate::span::Span { + match self { + Link::Atom(atom) => atom.span, + Link::BondedAtom { atom, .. } => atom.span, + Link::DottedAtom(atom) => atom.span, } } +} - /// Internal SMILES parsing that stops at EndOfLine as well as EOF/Terminator - fn parse_smiles_internal(&mut self) -> ParseResult { - let start_span = self.current_span(); - let chain = self.parse_chain()?; - - // Expect terminator, EndOfLine, or EOF - match &self.current_token { - Some(token) if matches!(token.kind, TokenKind::Terminator | TokenKind::EndOfLine) => { - let end_span = token.span; - let full_span = start_span.to(end_span); - // Don't advance here - let parse_smiles_until_eol handle EndOfLine - - Ok(Smiles::new(chain, full_span)) +/// Parser for SMILES notation +pub struct Parser; + +impl Parser { + /// Parse a single SMILES from a token iterator + pub fn parse_smiles_from_tokens( + tokens: &mut Peekable, + source: &str, + ) -> Result + where + T: Iterator>, + { + let chain = Self::parse_chain(tokens, source)?; + match tokens.next() { + Some(Ok(token)) + if token.kind == TokenKind::EndOfLine + || token.kind == TokenKind::Terminator + || token.kind == TokenKind::EndOfFile => + { + Ok(Smiles::new(chain, token.span)) } - Some(token) => Err(ParseError::new( - ParseErrorType::UnexpectedToken { + Some(Ok(token)) => Err(ParseError::new( + crate::error::ParseErrorType::UnexpectedToken { expected: vec![ - TokenKind::Terminator, TokenKind::EndOfLine, + TokenKind::Terminator, TokenKind::EndOfFile, ], found: token.kind, }, token.span, )), - None => { - // EOF reached - let full_span = start_span.to(start_span.expand_right(1)); - Ok(Smiles::new(chain, full_span)) - } - } - } - - /// Parse a single SMILES string - pub fn parse_smiles(&mut self) -> ParseResult { - let start_span = self.current_span(); - let chain = self.parse_chain()?; - - // Expect terminator or EOF (not EndOfLine for single SMILES) - match &self.current_token { - Some(token) if matches!(token.kind, TokenKind::Terminator | TokenKind::EndOfFile) => { - let end_span = token.span; - let full_span = start_span.to(end_span); - self.advance(); - - Ok(Smiles::new(chain, full_span)) - } - Some(token) => Err(ParseError::new( - ParseErrorType::UnexpectedToken { - expected: vec![TokenKind::Terminator, TokenKind::EndOfFile], - found: token.kind, + Some(Err(_errs)) => Err(ParseError::new( + crate::error::ParseErrorType::UnexpectedToken { + expected: vec![ + TokenKind::EndOfLine, + TokenKind::Terminator, + TokenKind::EndOfFile, + ], + found: TokenKind::EndOfFile, }, - token.span, + crate::span::Span::empty(0), )), - None => { - let full_span = start_span.to(start_span.expand_right(1)); - Ok(Smiles::new(chain, full_span)) - } + None => Ok(Smiles::new(chain, crate::span::Span::empty(0))), } } - /// Parse a chain of atoms - fn parse_chain(&mut self) -> ParseResult { - let first_atom = self.parse_branched_atom()?; + fn parse_chain(tokens: &mut Peekable, source: &str) -> Result + where + T: Iterator>, + { + let first_atom = Self::parse_branched_atom(tokens, source)?; let mut chain_span = first_atom.span; let mut links = Vec::new(); - - // Parse optional links - while let Some(link) = self.try_parse_link()? { + while let Some(link) = Self::try_parse_link(tokens, source)? { chain_span = chain_span.merge(link.span()); links.push(link); } - Ok(Chain::new(first_atom, links, chain_span)) } - /// Try to parse a link (returns None if no link is possible) - fn try_parse_link(&mut self) -> ParseResult> { - match &self.current_token { - Some(token) => match token.kind { - // Explicit bond followed by atom - kind if kind.is_bond() => { - let bond = self.parse_bond()?; - let atom = self.parse_branched_atom()?; - Ok(Some(Link::BondedAtom { bond, atom })) - } - // Dot followed by atom - TokenKind::Dot => { - self.advance(); // consume dot - let atom = self.parse_branched_atom()?; - Ok(Some(Link::DottedAtom(atom))) - } - // Just an atom (implicit single bond) - kind if self.starts_atom(kind) => { - let atom = self.parse_branched_atom()?; - Ok(Some(Link::Atom(atom))) - } - // Not a link - _ => Ok(None), - }, + fn try_parse_link(tokens: &mut Peekable, source: &str) -> Result, ParseError> + where + T: Iterator>, + { + let peek = tokens.next(); + match peek { + Some(Ok(token)) if token.kind.is_bond() => { + let bond = Self::parse_bond(token)?; + let atom = Self::parse_branched_atom(tokens, source)?; + Ok(Some(Link::BondedAtom { bond, atom })) + } + Some(Ok(token)) if token.kind == TokenKind::Dot => { + let atom = Self::parse_branched_atom(tokens, source)?; + Ok(Some(Link::DottedAtom(atom))) + } + Some(Ok(token)) if Self::starts_atom(token.kind) => { + let atom = Self::parse_branched_atom(tokens, source)?; + Ok(Some(Link::Atom(atom))) + } + Some(Ok(_token)) => Ok(None), + Some(Err(_errs)) => Ok(None), None => Ok(None), } } - /// Parse a branched atom - fn parse_branched_atom(&mut self) -> ParseResult { - let atom = self.parse_atom()?; - let mut span = atom.span(); - let mut ring_bonds = Vec::new(); - let mut branches = Vec::new(); - - // Parse ring bonds - while let Some(ring_bond) = self.try_parse_ring_bond()? { - span = span.merge(ring_bond.span); - ring_bonds.push(ring_bond); - } - - // Parse branches - while let Some(branch) = self.try_parse_branch()? { - span = span.merge(branch.span()); - branches.push(branch); - } - + fn parse_branched_atom( + tokens: &mut Peekable, + source: &str, + ) -> Result + where + T: Iterator>, + { + let atom = Self::parse_atom(tokens, source)?; + let span = atom.span(); + let ring_bonds = Vec::new(); + let branches = Vec::new(); Ok(BranchedAtom::new(atom, ring_bonds, branches, span)) } - /// Parse an atom - fn parse_atom(&mut self) -> ParseResult { - match &self.current_token { - Some(token) => match token.kind { - TokenKind::BracketStart => self.parse_bracket_atom(), - TokenKind::ElementSymbol => self.parse_organic_element(), - TokenKind::AromaticSymbol => self.parse_aromatic_element(), - TokenKind::Wildcard => { - let span = token.span; - self.advance(); - Ok(Atom::Wildcard { span }) - } - _ => Err(ParseError { - error: ParseErrorType::UnexpectedToken { + fn parse_atom(tokens: &mut Peekable, source: &str) -> Result + where + T: Iterator>, + { + let token = match tokens.next() { + Some(Ok(token)) => token, + Some(Err(errs)) => { + let first_err = errs.errors.iter().flatten().next(); + let span = first_err + .map(|e| *e.span()) + .unwrap_or_else(|| crate::span::Span::empty(0)); + return Err(ParseError::new( + crate::error::ParseErrorType::UnexpectedToken { expected: vec![ + TokenKind::ElementSymbol, + TokenKind::AromaticSymbol, + TokenKind::Wildcard, TokenKind::BracketStart, + ], + found: TokenKind::Unknown, + }, + span, + )); + } + None => { + return Err(ParseError::new( + crate::error::ParseErrorType::UnexpectedToken { + expected: vec![ TokenKind::ElementSymbol, TokenKind::AromaticSymbol, TokenKind::Wildcard, + TokenKind::BracketStart, ], - found: token.kind, + found: TokenKind::EndOfFile, }, - span: token.span, - }), - }, - None => Err(ParseError { - error: ParseErrorType::UnexpectedEof { + crate::span::Span::empty(0), + )); + } + }; + match token.kind { + TokenKind::ElementSymbol => Self::parse_organic_atom(token, source), + TokenKind::AromaticSymbol => Self::parse_organic_atom(token, source), + TokenKind::Wildcard => Ok(Atom::Wildcard { span: token.span }), + TokenKind::BracketStart => Self::parse_bracket_atom(tokens, source, token.span), + _ => Err(ParseError::new( + crate::error::ParseErrorType::UnexpectedToken { expected: vec![ - TokenKind::BracketStart, TokenKind::ElementSymbol, TokenKind::AromaticSymbol, TokenKind::Wildcard, + TokenKind::BracketStart, ], + found: token.kind, }, - span: Span::empty(self.lexer.current_position()), - }), - } - } - - /// Parse a bracket atom [isotope][symbol][chirality][hcount][charge][class] - fn parse_bracket_atom(&mut self) -> ParseResult { - let start_span = self.expect_token(TokenKind::BracketStart)?.span; - - // Parse optional isotope - let isotope = if matches!( - self.current_token, - Some(Token { - kind: TokenKind::Isotope, - .. - }) - ) { - Some(self.parse_isotope()?) - } else { - None - }; - - // Parse required symbol - let symbol = self.parse_atom_symbol()?; - - // Parse optional chirality - let chirality = if matches!( - self.current_token, - Some(Token { - kind: TokenKind::Chirality, - .. - }) - ) { - Some(self.parse_chirality()?) - } else { - None - }; - - // Parse optional hydrogen count - let hydrogen_count = if matches!( - self.current_token, - Some(Token { - kind: TokenKind::HCount, - .. - }) - ) { - Some(self.parse_hydrogen_count()?) - } else { - None - }; - - // Parse optional charge - let charge = if matches!( - self.current_token, - Some(Token { - kind: TokenKind::Charge, - .. - }) - ) { - Some(self.parse_charge()?) - } else { - None - }; - - // Parse optional atom class - let atom_class = if matches!( - self.current_token, - Some(Token { - kind: TokenKind::Class, - .. - }) - ) { - Some(self.parse_atom_class()?) - } else { - None - }; - - let end_span = self.expect_token(TokenKind::BracketEnd)?.span; - let full_span = start_span.to(end_span); - - let mut bracket_atom = BracketAtom::new(symbol, full_span); - - if let Some(isotope) = isotope { - bracket_atom = bracket_atom.with_isotope(isotope); - } - if let Some(chirality) = chirality { - bracket_atom = bracket_atom.with_chirality(chirality); - } - if let Some(hydrogen_count) = hydrogen_count { - bracket_atom = bracket_atom.with_hydrogen_count(hydrogen_count); - } - if let Some(charge) = charge { - bracket_atom = bracket_atom.with_charge(charge); - } - if let Some(atom_class) = atom_class { - bracket_atom = bracket_atom.with_atom_class(atom_class); + token.span, + )), } - - Ok(Atom::Bracket(bracket_atom)) - } - - /// Parse an organic element (aliphatic shorthand) - fn parse_organic_element(&mut self) -> ParseResult { - let token = self.expect_token(TokenKind::ElementSymbol)?; - let element_str = token.span.source_text(self.source); - - let element = match element_str { - "B" => AliphaticOrganic::B, - "C" => AliphaticOrganic::C, - "N" => AliphaticOrganic::N, - "O" => AliphaticOrganic::O, - "S" => AliphaticOrganic::S, - "P" => AliphaticOrganic::P, - "F" => AliphaticOrganic::F, - "Cl" => AliphaticOrganic::Cl, - "Br" => AliphaticOrganic::Br, - "I" => AliphaticOrganic::I, - _ => { - return Err(ParseError { - error: ParseErrorType::UnexpectedToken { - expected: vec![TokenKind::ElementSymbol], - found: token.kind, - }, - span: token.span, - }); - } - }; - - Ok(Atom::Organic(OrganicAtom::Aliphatic { - element, - span: token.span, - })) - } - - /// Parse an aromatic element - fn parse_aromatic_element(&mut self) -> ParseResult { - let token = self.expect_token(TokenKind::AromaticSymbol)?; - let element_str = token.span.source_text(self.source); - - let element = match element_str { - "b" => AromaticOrganic::B, - "c" => AromaticOrganic::C, - "n" => AromaticOrganic::N, - "o" => AromaticOrganic::O, - "s" => AromaticOrganic::S, - "p" => AromaticOrganic::P, - _ => { - return Err(ParseError { - error: ParseErrorType::UnexpectedToken { - expected: vec![TokenKind::AromaticSymbol], - found: token.kind, - }, - span: token.span, - }); - } - }; - - Ok(Atom::Organic(OrganicAtom::Aromatic { - element, - span: token.span, - })) } - /// Parse a bond - fn parse_bond(&mut self) -> ParseResult { - let token = self.current_token.ok_or_else(|| ParseError { - error: ParseErrorType::UnexpectedEof { - expected: vec![ - TokenKind::SingleBond, - TokenKind::DoubleBond, - TokenKind::TripleBond, - TokenKind::QuadrupleBond, - TokenKind::AromaticBond, - TokenKind::StereoUpBond, - TokenKind::StereoDownBond, - ], - }, - span: Span::empty(self.lexer.current_position()), - })?; - - let bond = match token.kind { - TokenKind::SingleBond => Bond::Single, - TokenKind::DoubleBond => Bond::Double, - TokenKind::TripleBond => Bond::Triple, - TokenKind::QuadrupleBond => Bond::Quadruple, - TokenKind::AromaticBond => Bond::Aromatic, - TokenKind::StereoUpBond => Bond::StereoUp, - TokenKind::StereoDownBond => Bond::StereoDown, - _ => { - return Err(ParseError { - error: ParseErrorType::UnexpectedToken { - expected: vec![ - TokenKind::SingleBond, - TokenKind::DoubleBond, - TokenKind::TripleBond, - TokenKind::QuadrupleBond, - TokenKind::AromaticBond, - TokenKind::StereoUpBond, - TokenKind::StereoDownBond, - ], - found: token.kind, - }, - span: token.span, - }); - } - }; - - self.advance(); - Ok(bond) - } - - /// Try to parse a ring bond - fn try_parse_ring_bond(&mut self) -> ParseResult> { - // Check for optional bond - let bond = if let Some(token) = &self.current_token { - if token.kind.is_bond() { - Some(self.parse_bond()?) - } else { - None - } - } else { - None - }; - - // Check for ring number - if let Some(token) = &self.current_token { - if token.kind == TokenKind::RingNumber { - let ring_span = token.span; - let ring_text = ring_span.source_text(self.source); - self.advance(); - - let (ring_number, is_double) = if let Some(number_str) = ring_text.strip_prefix('%') - { - // Two-digit ring number - let number = number_str.parse::().map_err(|_| ParseError { - error: ParseErrorType::InvalidRingBond { - number: 0, - message: format!("Invalid ring number format: {}", ring_text), - }, - span: ring_span, - })?; - - if !(10..=99).contains(&number) { - return Err(ParseError { - error: ParseErrorType::InvalidRingBond { - number, - message: "Two-digit ring numbers must be between 10-99".to_string(), + fn parse_organic_atom(token: Token, source: &str) -> Result { + match token.kind { + TokenKind::ElementSymbol => { + let text = &source[token.span.start..token.span.end]; + let element = match text { + "B" => crate::ast::AliphaticOrganic::B, + "C" => crate::ast::AliphaticOrganic::C, + "N" => crate::ast::AliphaticOrganic::N, + "O" => crate::ast::AliphaticOrganic::O, + "S" => crate::ast::AliphaticOrganic::S, + "P" => crate::ast::AliphaticOrganic::P, + "F" => crate::ast::AliphaticOrganic::F, + "Cl" => crate::ast::AliphaticOrganic::Cl, + "Br" => crate::ast::AliphaticOrganic::Br, + "I" => crate::ast::AliphaticOrganic::I, + _ => { + return Err(ParseError::new( + crate::error::ParseErrorType::UnexpectedToken { + expected: vec![TokenKind::ElementSymbol], + found: TokenKind::ElementSymbol, }, - span: ring_span, - }); + token.span, + )); } - - (number, true) - } else { - // Single-digit ring number - let number = ring_text.parse::().map_err(|_| ParseError { - error: ParseErrorType::InvalidRingBond { - number: 0, - message: format!("Invalid ring number format: {}", ring_text), - }, - span: ring_span, - })?; - - if number > 9 { - return Err(ParseError { - error: ParseErrorType::InvalidRingBond { - number, - message: "Single-digit ring numbers must be 0-9".to_string(), - }, - span: ring_span, - }); - } - - (number, false) }; - - let ring_num = if is_double { - RingNumber::Double(ring_number) - } else { - RingNumber::Single(ring_number) - }; - - let mut ring_bond = RingBond::new(ring_num, ring_span); - if let Some(bond) = bond { - ring_bond = ring_bond.with_bond(bond); - } - - Ok(Some(ring_bond)) - } else { - Ok(None) + Ok(Atom::Organic(OrganicAtom::Aliphatic { + element, + span: token.span, + })) } - } else { - Ok(None) - } - } - - /// Try to parse a branch - fn try_parse_branch(&mut self) -> ParseResult> { - if let Some(token) = &self.current_token { - if token.kind == TokenKind::BranchStart { - let start_span = token.span; - self.advance(); // consume '(' - - // Check for optional bond or dot - let branch = if let Some(token) = &self.current_token { - match token.kind { - kind if kind.is_bond() => { - let bond = self.parse_bond()?; - let chain = self.parse_chain()?; - Branch::Bonded { bond, chain } - } - TokenKind::Dot => { - self.advance(); // consume '.' - let chain = self.parse_chain()?; - Branch::Dotted(chain) - } - _ => { - let chain = self.parse_chain()?; - Branch::Simple(chain) - } + TokenKind::AromaticSymbol => { + let text = &source[token.span.start..token.span.end]; + let element = match text { + "b" => crate::ast::AromaticOrganic::B, + "c" => crate::ast::AromaticOrganic::C, + "n" => crate::ast::AromaticOrganic::N, + "o" => crate::ast::AromaticOrganic::O, + "s" => crate::ast::AromaticOrganic::S, + "p" => crate::ast::AromaticOrganic::P, + _ => { + return Err(ParseError::new( + crate::error::ParseErrorType::UnexpectedToken { + expected: vec![TokenKind::AromaticSymbol], + found: TokenKind::AromaticSymbol, + }, + token.span, + )); } - } else { - return Err(ParseError { - error: ParseErrorType::UnexpectedEof { - expected: vec![TokenKind::ElementSymbol, TokenKind::AromaticSymbol], - }, - span: start_span, - }); }; - - self.expect_token(TokenKind::BranchEnd)?; - Ok(Some(branch)) - } else { - Ok(None) + Ok(Atom::Organic(OrganicAtom::Aromatic { + element, + span: token.span, + })) + } + _ => unreachable!(), + } + } + + fn parse_bracket_atom( + tokens: &mut std::iter::Peekable, + source: &str, + start_span: crate::span::Span, + ) -> Result + where + T: Iterator>, + { + use crate::ast::BracketAtom; + let mut span = start_span; + let isotope = Self::parse_isotope(tokens, source, &mut span)?; + let symbol = Self::parse_bracket_symbol(tokens, source, &mut span)?; + let chirality = Self::parse_chirality(tokens, source, &mut span)?; + let hydrogen_count = Self::parse_hydrogen_count(tokens, source, &mut span)?; + let charge = Self::parse_charge(tokens, source, &mut span)?; + let atom_class = Self::parse_atom_class(tokens, source, &mut span)?; + // Expect BracketEnd + let mut found_end = false; + for next in tokens.by_ref() { + match next { + Ok(token) if token.kind == TokenKind::BracketEnd => { + span = span.merge(token.span); + found_end = true; + break; + } + Ok(_token) => { + // Ignore trivia, etc. + } + Err(_errs) => {} } - } else { - Ok(None) } - } - - /// Parse isotope number - fn parse_isotope(&mut self) -> ParseResult { - let token = self.expect_token(TokenKind::Isotope)?; - let isotope_str = token.span.source_text(self.source); - - let isotope = isotope_str.parse::().map_err(|_| ParseError { - error: ParseErrorType::InvalidIsotope { - message: format!("Invalid isotope number: {}", isotope_str), - }, - span: token.span, - })?; - - // Semantic validation - reasonable isotope range - if isotope == 0 || isotope > 300 { - return Err(ParseError { - error: ParseErrorType::InvalidIsotope { - message: format!( - "Isotope number {} is out of reasonable range (1-300)", - isotope - ), + if !found_end { + return Err(ParseError::new( + crate::error::ParseErrorType::UnexpectedToken { + expected: vec![TokenKind::BracketEnd], + found: TokenKind::EndOfFile, }, - span: token.span, - }); + span, + )); } - - Ok(isotope) - } - - /// Parse atom symbol within brackets - fn parse_atom_symbol(&mut self) -> ParseResult { - match &self.current_token { - Some(token) => match token.kind { - TokenKind::ElementSymbol => { - let span = token.span; - let element_str = span.source_text(self.source); - self.advance(); - - let element = Element::from(element_str); - Ok(AtomSymbol::Element { element, span }) - } - TokenKind::AromaticSymbol => { - let span = token.span; - let element_str = span.source_text(self.source); - self.advance(); - - let element = AromaticElement::from(element_str); - Ok(AtomSymbol::Aromatic { element, span }) - } - TokenKind::Wildcard => { - let span = token.span; - self.advance(); - Ok(AtomSymbol::Wildcard { span }) - } - _ => Err(ParseError { - error: ParseErrorType::UnexpectedToken { - expected: vec![ - TokenKind::ElementSymbol, - TokenKind::AromaticSymbol, - TokenKind::Wildcard, - ], - found: token.kind, - }, - span: token.span, - }), - }, - None => Err(ParseError { - error: ParseErrorType::UnexpectedEof { + if let Some(symbol) = symbol { + Ok(Atom::Bracket(BracketAtom { + isotope, + symbol, + chirality, + hydrogen_count, + charge, + atom_class, + span, + })) + } else { + Err(ParseError::new( + crate::error::ParseErrorType::UnexpectedToken { expected: vec![ TokenKind::ElementSymbol, TokenKind::AromaticSymbol, TokenKind::Wildcard, ], + found: TokenKind::BracketStart, }, - span: Span::empty(self.lexer.current_position()), - }), - } - } - - /// Parse chirality - fn parse_chirality(&mut self) -> ParseResult { - let token = self.expect_token(TokenKind::Chirality)?; - let chirality_str = token.span.source_text(self.source); - - match chirality_str { - "@" => Ok(Chirality::Clockwise), - "@@" => Ok(Chirality::CounterClockwise), - _ => Err(ParseError { - error: ParseErrorType::InvalidChirality { - message: format!("Invalid chirality specification: {}", chirality_str), - }, - span: token.span, - }), - } - } - - /// Parse hydrogen count - fn parse_hydrogen_count(&mut self) -> ParseResult { - let token = self.expect_token(TokenKind::HCount)?; - let hcount_str = token.span.source_text(self.source); - - if hcount_str == "H" { - Ok(HydrogenCount::Implicit) - } else if hcount_str.starts_with('H') && hcount_str.len() == 2 { - let digit_str = &hcount_str[1..]; - let count = digit_str.parse::().map_err(|_| ParseError { - error: ParseErrorType::InvalidHydrogenCount { - message: format!("Invalid hydrogen count: {}", hcount_str), - }, - span: token.span, - })?; - - // Semantic validation - reasonable hydrogen count - if count > 20 { - return Err(ParseError { - error: ParseErrorType::InvalidHydrogenCount { - message: format!("Hydrogen count {} is unreasonably high", count), - }, - span: token.span, - }); + span, + )) + } + } + + // Helper methods for bracket atom fields + fn parse_isotope( + tokens: &mut std::iter::Peekable, + source: &str, + span: &mut crate::span::Span, + ) -> Result, ParseError> + where + T: Iterator>, + { + if let Some(Ok(token)) = tokens.peek() { + if token.kind == TokenKind::Isotope { + let token = match tokens.next() { + Some(Ok(token)) => token, + Some(Err(_errs)) => return Ok(None), + None => return Ok(None), + }; + let text = &source[token.span.start..token.span.end]; + *span = span.merge(token.span); + return Ok(text.parse::().ok()); } - - Ok(HydrogenCount::Explicit(count)) - } else { - Err(ParseError { - error: ParseErrorType::InvalidHydrogenCount { - message: format!("Invalid hydrogen count format: {}", hcount_str), - }, - span: token.span, - }) } + Ok(None) } - /// Parse charge - fn parse_charge(&mut self) -> ParseResult { - let token = self.expect_token(TokenKind::Charge)?; - let charge_str = token.span.source_text(self.source); - - match charge_str { - "+" => Ok(Charge::PositiveOne), - "-" => Ok(Charge::NegativeOne), - charge_str if charge_str.starts_with('+') => { - let number_str = &charge_str[1..]; - let charge = number_str.parse::().map_err(|_| ParseError { - error: ParseErrorType::InvalidCharge { - message: format!("Invalid positive charge: {}", charge_str), - }, - span: token.span, - })?; - - // Semantic validation - reasonable charge range - if charge == 0 || charge > 10 { - return Err(ParseError { - error: ParseErrorType::InvalidCharge { - message: format!("Charge {} is out of reasonable range (1-10)", charge), - }, + fn parse_bracket_symbol( + tokens: &mut std::iter::Peekable, + source: &str, + span: &mut crate::span::Span, + ) -> Result, ParseError> + where + T: Iterator>, + { + if let Some(Ok(token)) = tokens.peek() { + match token.kind { + TokenKind::ElementSymbol => { + let token = match tokens.next() { + Some(Ok(token)) => token, + Some(Err(_errs)) => return Ok(None), + None => return Ok(None), + }; + let text = &source[token.span.start..token.span.end]; + *span = span.merge(token.span); + let element = crate::ast::Element::from(text); + Ok(Some(crate::ast::AtomSymbol::Element { + element, span: token.span, - }); + })) } - - Ok(Charge::Positive(charge)) - } - charge_str if charge_str.starts_with('-') => { - let number_str = &charge_str[1..]; - let charge = number_str.parse::().map_err(|_| ParseError { - error: ParseErrorType::InvalidCharge { - message: format!("Invalid negative charge: {}", charge_str), - }, - span: token.span, - })?; - - // Semantic validation - reasonable charge range - if charge == 0 || charge > 10 { - return Err(ParseError { - error: ParseErrorType::InvalidCharge { - message: format!("Charge {} is out of reasonable range (1-10)", charge), - }, + TokenKind::AromaticSymbol => { + let token = match tokens.next() { + Some(Ok(token)) => token, + Some(Err(_errs)) => return Ok(None), + None => return Ok(None), + }; + let text = &source[token.span.start..token.span.end]; + *span = span.merge(token.span); + let element = crate::ast::AromaticElement::from(text); + Ok(Some(crate::ast::AtomSymbol::Aromatic { + element, span: token.span, - }); + })) } - - Ok(Charge::Negative(charge)) + TokenKind::Wildcard => { + let token = match tokens.next() { + Some(Ok(token)) => token, + Some(Err(_errs)) => return Ok(None), + None => return Ok(None), + }; + *span = span.merge(token.span); + Ok(Some(crate::ast::AtomSymbol::Wildcard { span: token.span })) + } + _ => Ok(None), } - _ => Err(ParseError { - error: ParseErrorType::InvalidCharge { - message: format!("Invalid charge format: {}", charge_str), - }, - span: token.span, - }), - } - } - - /// Parse atom class - fn parse_atom_class(&mut self) -> ParseResult { - let token = self.expect_token(TokenKind::Class)?; - let class_str = token.span.source_text(self.source); - - if !class_str.starts_with(':') { - return Err(ParseError { - error: ParseErrorType::UnexpectedToken { - expected: vec![TokenKind::Class], - found: token.kind, - }, - span: token.span, - }); - } - - let number_str = &class_str[1..]; - let class = number_str.parse::().map_err(|_| ParseError { - error: ParseErrorType::InvalidCharge { - message: format!("Invalid atom class: {}", class_str), - }, - span: token.span, - })?; - - Ok(class) - } - - /// Check if a token kind can start an atom - fn starts_atom(&self, kind: TokenKind) -> bool { - matches!( - kind, - TokenKind::ElementSymbol - | TokenKind::AromaticSymbol - | TokenKind::Wildcard - | TokenKind::BracketStart - ) - } - - /// Get the span of the current token or an empty span at current position - fn current_span(&self) -> Span { - self.current_token - .map(|t| t.span) - .unwrap_or_else(|| Span::empty(self.lexer.current_position())) - } - - /// Advance to the next token - fn advance(&mut self) { - self.current_token = Self::next_token(&mut self.lexer); - } - - /// Get the next token from the lexer, handling lexical errors - fn next_token(lexer: &mut Lexer<'src>) -> Option { - let token = lexer.lex_next_token(); - if token.kind == TokenKind::EndOfFile { - None } else { - Some(token) - } - } - - /// Expect a specific token kind - fn expect_token(&mut self, expected: TokenKind) -> ParseResult { - match &self.current_token { - Some(token) if token.kind == expected => { - let token = *token; - self.advance(); - Ok(token) - } - Some(token) => Err(ParseError { - error: ParseErrorType::UnexpectedToken { - expected: vec![expected], - found: token.kind, - }, - span: token.span, - }), - None => Err(ParseError { - error: ParseErrorType::UnexpectedEof { - expected: vec![expected], - }, - span: Span::empty(self.lexer.current_position()), - }), - } - } - - /// Parse a single SMILES line, consuming the EndOfLine token if present. - /// Returns the parsed SMILES or a parse error. - /// Lexical errors are handled separately by the caller. - fn parse_smiles_line(&mut self) -> ParseResult { - let result = self.parse_smiles_internal(); - - match result { - Ok(smiles) => { - self.consume_end_of_line(); - Ok(smiles) - } - Err(parse_error) => { - self.skip_to_next_line(); - Err(parse_error) - } - } - } - - /// Consume the EndOfLine token if present - fn consume_end_of_line(&mut self) { - if let Some(token) = &self.current_token { - if token.kind == TokenKind::EndOfLine { - self.advance(); - } - } - } -} - -/// Iterator that yields SMILES parsing results line by line -pub struct SmilesLineIterator<'src> { - parser: Parser<'src>, - line_number: usize, - finished: bool, -} - -impl<'src> SmilesLineIterator<'src> { - fn new(parser: Parser<'src>) -> Self { - Self { - parser, - line_number: 1, - finished: false, + Ok(None) } } - /// Collect all results from the iterator into separate vectors - pub fn collect_results(self) -> (Vec, Vec) { - let mut valid_results = Vec::new(); - let mut errors = Vec::new(); - - for result in self { - match result { - Ok(smiles) => valid_results.push(smiles), - Err(error) => errors.push(error), + fn parse_chirality( + tokens: &mut std::iter::Peekable, + source: &str, + span: &mut crate::span::Span, + ) -> Result, ParseError> + where + T: Iterator>, + { + if let Some(Ok(token)) = tokens.peek() { + if token.kind == TokenKind::Chirality { + let token = match tokens.next() { + Some(Ok(token)) => token, + Some(Err(_errs)) => return Ok(None), + None => return Ok(None), + }; + let text = &source[token.span.start..token.span.end]; + *span = span.merge(token.span); + let chirality = match text { + "@" => Some(crate::ast::Chirality::Clockwise), + "@@" => Some(crate::ast::Chirality::CounterClockwise), + _ => None, + }; + return Ok(chirality); } } - - (valid_results, errors) - } - - /// Collect only valid SMILES, ignoring errors - pub fn collect_valid(self) -> Vec { - self.filter_map(Result::ok).collect() - } - - /// Collect only errors, ignoring valid results - pub fn collect_errors(self) -> Vec { - self.filter_map(Result::err).collect() - } - - /// Count the total number of lines processed - pub fn count_lines(self) -> usize { - self.count() - } -} - -impl<'src> Iterator for SmilesLineIterator<'src> { - type Item = SmilesLineResult; - - fn next(&mut self) -> Option { - if self.finished { - return None; - } - - // Skip any leading EndOfLine tokens (empty lines) - loop { - match &self.parser.current_token { - Some(token) if token.kind == TokenKind::EndOfLine => { - self.parser.advance(); - self.line_number += 1; - } - Some(_) => break, // Found a non-EndOfLine token - None => { - // EOF reached - self.finished = true; - return None; + Ok(None) + } + + fn parse_hydrogen_count( + tokens: &mut std::iter::Peekable, + source: &str, + span: &mut crate::span::Span, + ) -> Result, ParseError> + where + T: Iterator>, + { + if let Some(Ok(token)) = tokens.peek() { + if token.kind == TokenKind::HCount { + let token = match tokens.next() { + Some(Ok(token)) => token, + Some(Err(_errs)) => return Ok(None), + None => return Ok(None), + }; + let text = &source[token.span.start..token.span.end]; + *span = span.merge(token.span); + if text == "H" { + return Ok(Some(crate::ast::HydrogenCount::Implicit)); + } else if let Some(stripped) = text.strip_prefix('H') { + let count = stripped.parse::().unwrap_or(1); + return Ok(Some(crate::ast::HydrogenCount::Explicit(count))); } } } - - // Check for EOF after skipping empty lines - if self.parser.current_token.is_none() { - self.finished = true; - return None; - } - - let current_line = self.line_number; - - // Clear any existing lexical errors before parsing - self.parser.lexer.clear_errors(); - - // Parse a single SMILES line - let parse_result = self.parser.parse_smiles_line(); - - // Collect any lexical errors that occurred during this line - let lexical_errors = self.parser.lexer.errors(); - - // Determine the result for this line - let line_result = match (parse_result, lexical_errors.is_empty()) { - // Successful parse with no lexical errors - (Ok(smiles), true) => Ok(smiles), - - // Lexical errors occurred (prioritize lexical over parse errors) - (_, false) => { - let first_lexical_error = lexical_errors.iter().next().unwrap().clone(); - Err(SmilesLineError::lexical(current_line, first_lexical_error)) - } - - // Parse error with no lexical errors - (Err(parse_error), true) => Err(SmilesLineError::parse(current_line, parse_error)), - }; - - // Clear lexer errors for next iteration - self.parser.lexer.clear_errors(); - - // Advance to next line for subsequent calls - self.line_number += 1; - - Some(line_result) - } -} - -// Helper implementations for span tracking - -/// Implement span getter for Link enum -impl Link { - pub fn span(&self) -> Span { - match self { - Link::Atom(atom) => atom.span, - Link::BondedAtom { atom, .. } => atom.span, - Link::DottedAtom(atom) => atom.span, - } - } -} - -/// Implement span getter for Branch enum -impl Branch { - pub fn span(&self) -> Span { - match self { - Branch::Simple(chain) => chain.span, - Branch::Bonded { chain, .. } => chain.span, - Branch::Dotted(chain) => chain.span, - } - } -} - -/// Implement span getter for Atom enum -impl Atom { - pub fn span(&self) -> Span { - match self { - Atom::Bracket(bracket) => bracket.span, - Atom::Organic(organic) => organic.span(), - Atom::Wildcard { span } => *span, - } - } -} - -/// Implement span getter for OrganicAtom enum -impl OrganicAtom { - pub fn span(&self) -> Span { - match self { - OrganicAtom::Aliphatic { span, .. } => *span, - OrganicAtom::Aromatic { span, .. } => *span, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_simple_carbon() { - let mut parser = Parser::new("C"); - let result = parser.parse_smiles(); - assert!(result.is_ok()); - - if let Ok(smiles) = result { - assert_eq!(smiles.chain.links.len(), 0); - // More detailed tests would check the atom structure - } - } - - #[test] - fn test_parse_ethane() { - let mut parser = Parser::new("CC"); - let result = parser.parse_smiles(); - assert!(result.is_ok()); - - if let Ok(smiles) = result { - assert_eq!(smiles.chain.links.len(), 1); - } - } - - #[test] - fn test_parse_benzene() { - let mut parser = Parser::new("c1ccccc1"); - let result = parser.parse_smiles(); - assert!(result.is_ok()); - - if let Ok(smiles) = result { - assert_eq!(smiles.chain.links.len(), 5); // 6 atoms, so 5 links - } - } - - #[test] - fn test_parse_bracket_atom() { - let mut parser = Parser::new("[13C@H4+]"); - let result = parser.parse_smiles(); - assert!(result.is_ok()); - } - - #[test] - fn test_parse_unmatched_ring_bond() { - // Note: This now tests syntactic parsing only. - // Ring bond semantic validation (matching) should be done by a separate validator. - let mut parser = Parser::new("C1CC"); - let result = parser.parse_smiles(); - // The parser should successfully create an AST for syntactically valid SMILES - assert!(result.is_ok()); - - if let Ok(smiles) = result { - // Should have parsed C with ring bond 1, then CC - assert_eq!(smiles.chain.links.len(), 2); - // The first atom should have a ring bond - assert_eq!(smiles.chain.branched_atom.ring_bonds.len(), 1); - } - } - - #[test] - fn test_parse_multiple_lines() { - let mut parser = Parser::new("C\nCC\nc1ccccc1\nInvalid123"); - let (results, parse_errors, lexical_errors) = parser.parse_multiple_lines(); - - assert_eq!(results.len(), 3); // "C", "CC", "c1ccccc1" - assert!(!parse_errors.is_empty()); // "Invalid123" should cause parse errors - - // Check that the successful results are correct by examining their structure - assert_eq!(results[0].chain.links.len(), 0); // "C" - single atom, no links - assert_eq!(results[1].chain.links.len(), 1); // "CC" - two atoms, one link - assert_eq!(results[2].chain.links.len(), 5); // "c1ccccc1" - six atoms, five links - - // Check that we have errors (either parse or lexical) - assert!(!parse_errors.is_empty() || !lexical_errors.is_empty()); - } - - #[test] - fn test_parse_branch() { - let mut parser = Parser::new("CC(C)C"); - let result = parser.parse_smiles(); - assert!(result.is_ok()); - - if let Ok(smiles) = result { - // Should have CC with a branch containing C, followed by C - assert_eq!(smiles.chain.links.len(), 2); - } - } - - #[test] - fn test_streaming_multiline_parsing() { - // Test with various scenarios: empty lines, valid SMILES, invalid SMILES, mixed content - let source = "C\n\nCC\n\nc1ccccc1\nInvalid@@\n\nCCO\n\n"; - let mut parser = Parser::new(source); - let (results, parse_errors, lexical_errors) = parser.parse_multiple_lines(); - - // Should get results for: C, CC, c1ccccc1, CCO (4 successful) - // Should get errors for: Invalid@@ (1 failed) - assert_eq!(results.len(), 4); - assert!(!parse_errors.is_empty() || !lexical_errors.is_empty()); - - // Check that successful results have reasonable structure - assert_eq!(results[0].chain.links.len(), 0); // "C" - assert_eq!(results[1].chain.links.len(), 1); // "CC" - assert_eq!(results[2].chain.links.len(), 5); // "c1ccccc1" - assert_eq!(results[3].chain.links.len(), 2); // "CCO" - - // Verify we have error details - assert!(!parse_errors.is_empty() || !lexical_errors.is_empty()); - } - - #[test] - fn test_streaming_error_recovery() { - // Test that errors in one SMILES don't affect parsing of subsequent ones - let source = "C\nBadSMILES@@@@\nCC\nAnother@Bad\nCCO"; - let mut parser = Parser::new(source); - let (results, parse_errors, lexical_errors) = parser.parse_multiple_lines(); - - assert_eq!(results.len(), 3); // "C", "CC", "CCO" - assert!(!parse_errors.is_empty() || !lexical_errors.is_empty()); // Should have errors from bad SMILES - - // Check that successful results have reasonable structure - assert_eq!(results[0].chain.links.len(), 0); // "C" - assert_eq!(results[1].chain.links.len(), 1); // "CC" - should parse correctly after error - assert_eq!(results[2].chain.links.len(), 2); // "CCO" - should parse correctly after error - - // Check that we have error details - assert!(!parse_errors.is_empty() || !lexical_errors.is_empty()); - } - - #[test] - fn test_streaming_ring_bond_parsing() { - // Test that ring bonds are parsed correctly in multi-line input - // Note: No semantic validation - that's for a separate validator - let source = "C1CCC\nC1CCCCC1\nCC"; - let mut parser = Parser::new(source); - let (results, parse_errors, lexical_errors) = parser.parse_multiple_lines(); - - assert_eq!(results.len(), 3); // All should parse successfully - assert!(parse_errors.is_empty() && lexical_errors.is_empty()); // No errors expected - - // Check that results have reasonable structure - // "C1CCC" = C with ring bond 1, then CCC (3 more atoms) = 3 links total - assert_eq!(results[0].chain.links.len(), 3); // "C1CCC" - 4 atoms total, 3 links - assert_eq!(results[1].chain.links.len(), 5); // "C1CCCCC1" - 6 atoms, 5 links - assert_eq!(results[2].chain.links.len(), 1); // "CC" - 2 atoms, 1 link - } - - #[test] - fn test_streaming_memory_efficiency() { - // Test that the streaming approach doesn't require splitting source upfront - // This is more of a design validation - the method should handle large inputs efficiently - let mut large_source = String::new(); - for i in 0..1000 { - if i % 100 == 0 { - large_source.push_str("InvalidSMILES@@@\n"); // Add some errors - } else { - large_source.push_str("C\n"); - } - } - - let mut parser = Parser::new(&large_source); - let (results, parse_errors, lexical_errors) = parser.parse_multiple_lines(); - - assert_eq!(results.len() + parse_errors.len(), 1000); - - // Check that most are valid (only every 100th is invalid) - assert_eq!(results.len(), 990); - assert_eq!(parse_errors.len(), 10); - - // The invalid SMILES should also produce lexical errors - assert!(!lexical_errors.is_empty()); - } - - #[test] - fn test_parse_lines_iterator() { - let source = "C\nCC\nc1ccccc1\nInvalid@@\nCCO\n"; - let parser = Parser::new(source); - - let results: Vec<_> = parser.parse_lines().collect(); - - assert_eq!(results.len(), 5); - - // Check successful parses - assert!(results[0].is_ok()); // "C" - assert!(results[1].is_ok()); // "CC" - assert!(results[2].is_ok()); // "c1ccccc1" - assert!(results[4].is_ok()); // "CCO" - - // Check error - assert!(results[3].is_err()); // "Invalid@@" - - if let Err(error) = &results[3] { - assert_eq!(error.line_number, 4); - // The error might be lexical instead of parse, so let's check both - assert!(matches!( - error.error_type, - crate::error::SmilesErrorType::Parse(_) | crate::error::SmilesErrorType::Lexical(_) - )); - } - } - - #[test] - fn test_parse_lines_with_empty_lines() { - let source = "C\n\nCC\n\n\nc1ccccc1\n\n"; - let parser = Parser::new(source); - - let results: Vec<_> = parser.parse_lines().collect(); - - // Should only get results for non-empty lines - assert_eq!(results.len(), 3); - assert!(results.iter().all(|r| r.is_ok())); - } - - #[test] - fn test_parse_lines_collect_results() { - let source = "C\nCC\nInvalid@@\nCCO\n"; - let parser = Parser::new(source); - - let (valid_smiles, errors) = parser.parse_lines().collect_results(); - - assert_eq!(valid_smiles.len(), 3); // "C", "CC", "CCO" - assert_eq!(errors.len(), 1); // "Invalid@@" - - // Check error details - assert_eq!(errors[0].line_number, 3); - } - - #[test] - fn test_parse_lines_collect_valid() { - let source = "C\nInvalid@@\nCC\nBad@\nCCO\n"; - let parser = Parser::new(source); - - let valid_smiles = parser.parse_lines().collect_valid(); - - assert_eq!(valid_smiles.len(), 3); // Only the valid ones - } - - #[test] - fn test_parse_lines_collect_errors() { - let source = "C\nInvalid@@\nCC\nBad@\nCCO\n"; - let parser = Parser::new(source); - - let errors = parser.parse_lines().collect_errors(); - - assert_eq!(errors.len(), 2); // Two error lines - assert_eq!(errors[0].line_number, 2); - assert_eq!(errors[1].line_number, 4); - } - - #[test] - fn test_parse_all_lines_convenience() { - let source = "C\nCC\nInvalid@@\nCCO\n"; - let parser = Parser::new(source); - - let (valid_smiles, errors) = parser.parse_all_lines(); - - assert_eq!(valid_smiles.len(), 3); - assert_eq!(errors.len(), 1); - assert_eq!(errors[0].line_number, 3); - } - - #[test] - fn test_parse_lines_lexical_errors() { - // Test cases that should produce lexical errors - let source = "C\n[Invalid\nCC\n"; - let parser = Parser::new(source); - - let results: Vec<_> = parser.parse_lines().collect(); - - assert_eq!(results.len(), 3); - - // Based on the actual behavior, verify that we get the expected results - let success_count = results.iter().filter(|r| r.is_ok()).count(); - assert!( - success_count >= 1, - "Should have at least one successful parse" - ); - - // Verify that we get lexical errors - let lexical_error_count = results - .iter() - .filter(|r| { - if let Err(error) = r { - matches!(error.error_type, crate::error::SmilesErrorType::Lexical(_)) + Ok(None) + } + + fn parse_charge( + tokens: &mut std::iter::Peekable, + source: &str, + span: &mut crate::span::Span, + ) -> Result, ParseError> + where + T: Iterator>, + { + if let Some(Ok(token)) = tokens.peek() { + if token.kind == TokenKind::Charge { + let token = match tokens.next() { + Some(Ok(token)) => token, + Some(Err(_errs)) => return Ok(None), + None => return Ok(None), + }; + let text = &source[token.span.start..token.span.end]; + *span = span.merge(token.span); + let charge = if text == "+" { + Some(crate::ast::Charge::PositiveOne) + } else if text == "-" { + Some(crate::ast::Charge::NegativeOne) + } else if let Some(stripped) = text.strip_prefix('+') { + let n = text.chars().filter(|&c| c == '+').count() as u32; + if n > 1 { + Some(crate::ast::Charge::Positive(n)) + } else { + let val = stripped.parse::().unwrap_or(1); + Some(crate::ast::Charge::Positive(val)) + } + } else if let Some(stripped) = text.strip_prefix('-') { + let n = text.chars().filter(|&c| c == '-').count() as u32; + if n > 1 { + Some(crate::ast::Charge::Negative(n)) + } else { + let val = stripped.parse::().unwrap_or(1); + Some(crate::ast::Charge::Negative(val)) + } } else { - false - } - }) - .count(); - assert!( - lexical_error_count >= 1, - "Should have at least one lexical error" - ); - } - - #[test] - fn test_parse_lines_memory_efficiency() { - // Test that the iterator doesn't buffer all results - let mut large_source = String::new(); - for i in 0..1000 { - if i % 100 == 0 { - large_source.push_str("Invalid@@\n"); - } else { - large_source.push_str("C\n"); + None + }; + return Ok(charge); } } - - let parser = Parser::new(&large_source); - let mut count = 0; - let mut error_count = 0; - - // Process one by one without collecting all - for result in parser.parse_lines() { - count += 1; - if result.is_err() { - error_count += 1; + Ok(None) + } + + fn parse_atom_class( + tokens: &mut std::iter::Peekable, + source: &str, + span: &mut crate::span::Span, + ) -> Result, ParseError> + where + T: Iterator>, + { + if let Some(Ok(token)) = tokens.peek() { + if token.kind == TokenKind::Class { + let token = match tokens.next() { + Some(Ok(token)) => token, + Some(Err(_errs)) => return Ok(None), + None => return Ok(None), + }; + let text = &source[token.span.start..token.span.end]; + *span = span.merge(token.span); + return Ok(text[1..].parse::().ok()); } } - - assert_eq!(count, 1000); - assert_eq!(error_count, 10); + Ok(None) } - #[test] - fn test_parse_lines_error_recovery() { - // Test that errors in one line don't affect subsequent lines - let source = "C\nBadSMILES@@@@\nCC\nAnother@Bad\nCCO\n"; - let parser = Parser::new(source); - - let results: Vec<_> = parser.parse_lines().collect(); - - assert_eq!(results.len(), 5); - - // Check that we have some successes and some errors - let success_count = results.iter().filter(|r| r.is_ok()).count(); - let error_count = results.iter().filter(|r| r.is_err()).count(); - - assert!( - success_count >= 2, - "Should have at least 2 successful parses" - ); - assert!(error_count >= 1, "Should have at least 1 error"); - - // Verify that the first and last SMILES are valid (basic error recovery test) - assert!(results[0].is_ok(), "First SMILES should be valid"); - assert!(results[4].is_ok(), "Last SMILES should be valid"); + fn parse_bond(token: Token) -> Result { + match token.kind { + TokenKind::SingleBond => Ok(Bond::Single), + TokenKind::DoubleBond => Ok(Bond::Double), + TokenKind::TripleBond => Ok(Bond::Triple), + TokenKind::QuadrupleBond => Ok(Bond::Quadruple), + TokenKind::AromaticBond => Ok(Bond::Aromatic), + TokenKind::StereoUpBond => Ok(Bond::StereoUp), + TokenKind::StereoDownBond => Ok(Bond::StereoDown), + _ => Err(ParseError::new( + crate::error::ParseErrorType::UnexpectedToken { + expected: vec![ + TokenKind::SingleBond, + TokenKind::DoubleBond, + TokenKind::TripleBond, + TokenKind::QuadrupleBond, + TokenKind::AromaticBond, + TokenKind::StereoUpBond, + TokenKind::StereoDownBond, + ], + found: token.kind, + }, + token.span, + )), + } } - #[test] - fn test_parse_lines_count() { - let source = "C\nCC\nInvalid@@\nCCO\n\n"; - let parser = Parser::new(source); - - let count = parser.parse_lines().count_lines(); - - assert_eq!(count, 4); // Empty lines are skipped + fn starts_atom(kind: TokenKind) -> bool { + matches!( + kind, + TokenKind::ElementSymbol + | TokenKind::AromaticSymbol + | TokenKind::Wildcard + | TokenKind::BracketStart + ) } } diff --git a/src/token.rs b/src/token.rs index 9105f47..5a58711 100644 --- a/src/token.rs +++ b/src/token.rs @@ -16,6 +16,27 @@ impl Token { span: Span::new(start, end), } } + + pub fn is_lexical_error(&self) -> bool { + self.kind == TokenKind::Unknown + } + pub fn to_lexical_error(&self) -> Option { + if self.kind == TokenKind::Unknown { + // Dummy conversion, real implementation should extract error info + Some(crate::error::LexicalError::new( + crate::error::LexicalErrorType::UnexpectedCharacter('?'), + self.span, + )) + } else { + None + } + } + pub fn is_end_of_line(&self) -> bool { + self.kind == TokenKind::EndOfLine + } + pub fn is_eof(&self) -> bool { + self.kind == TokenKind::EndOfFile + } } /// A kind of a token. diff --git a/src/token_stream.rs b/src/token_stream.rs new file mode 100644 index 0000000..b2ab77a --- /dev/null +++ b/src/token_stream.rs @@ -0,0 +1,100 @@ +use crate::error::LexicalError; +use crate::lexer::Lexer; +use crate::token::Token; + +const MAX_ERRORS_PER_LINE: usize = 8; + +pub struct LineErrors { + pub errors: [Option; MAX_ERRORS_PER_LINE], + pub count: usize, +} + +impl LineErrors { + pub fn new() -> Self { + Self { + errors: [None, None, None, None, None, None, None, None], + count: 0, + } + } + pub fn push(&mut self, err: LexicalError) { + if self.count < MAX_ERRORS_PER_LINE { + self.errors[self.count] = Some(err); + self.count += 1; + } + } + pub fn is_empty(&self) -> bool { + self.count == 0 + } + pub fn iter(&self) -> impl Iterator { + self.errors[..self.count].iter().filter_map(|e| e.as_ref()) + } +} + +impl Default for LineErrors { + fn default() -> Self { + Self::new() + } +} + +pub struct TokenStream<'src> { + lexer: Lexer<'src>, +} + +impl<'src> TokenStream<'src> { + pub fn new(lexer: Lexer<'src>) -> Self { + Self { lexer } + } + + pub fn stream_line(&mut self) -> Option> { + if self.lexer.is_eof() { + return None; + } + Some(LineTokenIter { + lexer: &mut self.lexer, + finished: false, + errors: LineErrors::new(), + }) + } + + pub fn position_tracker(&self) -> &crate::span::PositionTracker { + self.lexer.position_tracker() + } + pub fn errors(&self) -> &[crate::error::LexicalError] { + self.lexer.errors() + } +} + +pub struct LineTokenIter<'a, 'src> { + lexer: &'a mut Lexer<'src>, + finished: bool, + errors: LineErrors, +} + +impl<'a, 'src> Iterator for LineTokenIter<'a, 'src> { + type Item = Result; + + fn next(&mut self) -> Option { + if self.finished { + return None; + } + let token = self.lexer.lex_next_token(); + if token.is_lexical_error() { + if let Some(err) = token.to_lexical_error() { + self.errors.push(err); + } + } + if token.is_end_of_line() || token.is_eof() { + self.finished = true; + if self.errors.is_empty() { + return None; + } else { + return Some(Err(std::mem::take(&mut self.errors))); + } + } + if !self.errors.is_empty() { + self.finished = true; + return Some(Err(std::mem::take(&mut self.errors))); + } + Some(Ok(token)) + } +}