Add token line stream

examples/iterator_demo.rs

@@ -1,7 +1,9 @@
+use smiles::lexer::Lexer;
 use smiles::parser::Parser;
+use smiles::token_stream::TokenStream;
 
 fn main() {
-    println!("=== Iterator-based SMILES Parser Demo ===\n");
+    println!("=== Iterator-based SMILES Parser Demo (TokenStream) ===\n");
 
     // Test data with mixed valid/invalid SMILES and empty lines
     let multi_smiles = r#"C
@@ -25,110 +27,67 @@ C1CCCCC1
 
     // Example 1: Process line by line with an iterator
     println!("\n1. Processing line-by-line:");
-    let parser = Parser::new(multi_smiles);
-
-    for result in parser.parse_lines() {
-        match result {
+    let lexer = Lexer::new(multi_smiles);
+    let mut token_stream = TokenStream::new(lexer);
+    let mut line_num = 1;
+    while let Some(mut line_tokens) = token_stream.stream_line() {
+        let mut peekable_tokens = line_tokens.by_ref().peekable();
+        match Parser::parse_smiles_from_tokens(&mut peekable_tokens, multi_smiles) {
             Ok(_smiles) => {
-                println!("  ✓ Valid SMILES");
+                println!("  \u{2713} Valid SMILES");
             }
-            Err(error) => {
+            Err(parse_error) => {
                 println!(
-                    "  ✗ Line {}: {} - {}",
-                    error.line_number,
-                    match error.error_type {
-                        smiles::error::SmilesErrorType::Lexical(_) => "Lexical Error",
-                        smiles::error::SmilesErrorType::Parse(_) => "Parse Error",
-                    },
-                    error.error_type
+                    "  \u{2717} Line {}: Parse Error - {}",
+                    line_num, parse_error
                 );
             }
         }
+        // Handle lexical errors for this line
+        let line_errors = peekable_tokens.collect::<Vec<_>>();
+        for err in line_errors {
+            if let Err(lex_errs) = err {
+                for lex in lex_errs.iter() {
+                    println!("  \u{2717} Line {}: Lexical Error - {}", line_num, lex);
+                }
+            }
+        }
+        line_num += 1;
     }
 
-    // Example 2: Collect all results at once
+    // Example 2: Bulk collection (valid/invalid count)
     println!("\n2. Bulk collection:");
-    let parser = Parser::new(multi_smiles);
-    let (valid_smiles, errors) = parser.parse_lines().collect_results();
-
-    println!("  Successfully parsed: {} SMILES", valid_smiles.len());
-    println!("  Errors encountered: {} lines", errors.len());
-
-    // Example 3: Filter only valid results
-    println!("\n3. Processing only valid SMILES:");
-    let parser = Parser::new(multi_smiles);
-    let valid_smiles = parser.parse_lines().collect_valid();
-
-    for (i, _smiles) in valid_smiles.iter().enumerate() {
-        println!("  SMILES {}: Valid", i + 1);
-    }
-
-    // Example 4: Memory-efficient streaming for large files
-    println!("\n4. Large file simulation (streaming):");
-    let mut large_smiles = String::new();
-    for i in 0..1000 {
-        if i % 100 == 0 {
-            large_smiles.push_str("Invalid@@\n");
-        } else {
-            large_smiles.push_str("C\n");
+    let lexer = Lexer::new(multi_smiles);
+    let mut token_stream = TokenStream::new(lexer);
+    let mut valid_smiles = 0;
+    let mut error_lines = 0;
+    while let Some(mut line_tokens) = token_stream.stream_line() {
+        let mut peekable_tokens = line_tokens.by_ref().peekable();
+        match Parser::parse_smiles_from_tokens(&mut peekable_tokens, multi_smiles) {
+            Ok(_smiles) => valid_smiles += 1,
+            Err(_) => error_lines += 1,
         }
-    }
-
-    let parser = Parser::new(&large_smiles);
-    let mut valid_count = 0;
-    let mut error_count = 0;
-
-    // Process without collecting all in memory
-    for result in parser.parse_lines() {
-        match result {
-            Ok(_) => valid_count += 1,
-            Err(_) => error_count += 1,
+        // Count lexical errors
+        let line_errors = peekable_tokens.collect::<Vec<_>>();
+        for err in line_errors {
+            if err.is_err() {
+                error_lines += 1;
+            }
         }
     }
+    println!("  Successfully parsed: {} SMILES", valid_smiles);
+    println!("  Errors encountered: {} lines", error_lines);
 
-    println!(
-        "  Processed 1000 lines: {} valid, {} errors",
-        valid_count, error_count
-    );
-
-    // Example 5: Error analysis
-    println!("\n5. Detailed error analysis:");
-    let error_test = "C\n[Unclosed\nCC\nBad@@Symbol\nCCO\n";
-    let parser = Parser::new(error_test);
-
-    for result in parser.parse_lines() {
-        match result {
-            Ok(_) => println!("  ✓ Valid SMILES"),
-            Err(error) => {
-                println!(
-                    "  ✗ Line {}: {} - {}",
-                    error.line_number,
-                    match error.error_type {
-                        smiles::error::SmilesErrorType::Lexical(_) => "Lexical Error",
-                        smiles::error::SmilesErrorType::Parse(_) => "Parse Error",
-                    },
-                    error
-                );
-            }
+    // Example 3: Filter only valid results
+    println!("\n3. Processing only valid SMILES:");
+    let lexer = Lexer::new(multi_smiles);
+    let mut token_stream = TokenStream::new(lexer);
+    let mut valid_count = 0;
+    while let Some(mut line_tokens) = token_stream.stream_line() {
+        let mut peekable_tokens = line_tokens.by_ref().peekable();
+        if Parser::parse_smiles_from_tokens(&mut peekable_tokens, multi_smiles).is_ok() {
+            valid_count += 1;
         }
     }
-
-    // Example 6: Convenience method
-    println!("\n6. Using convenience method:");
-    let parser = Parser::new("C\nCC\nBad@\nCCO");
-    let (valid, errors) = parser.parse_all_lines();
-    println!(
-        "  Convenience method: {} valid, {} errors",
-        valid.len(),
-        errors.len()
-    );
-
-    println!("\n=== API Features Demonstrated ===");
-    println!("✓ Iterator-based processing (lazy evaluation)");
-    println!("✓ Structured error reporting with line numbers");
-    println!("✓ Unified error handling (lexical + parse errors)");
-    println!("✓ Memory-efficient streaming for large files");
-    println!("✓ Flexible collection methods (all, valid-only, errors-only)");
-    println!("✓ Error recovery between lines");
-    println!("✓ Convenience methods for common use cases");
+    println!("  Valid SMILES lines: {}", valid_count);
 }

examples/molecular_formula.rs

@@ -218,17 +218,29 @@ fn main() {
 
     for (smiles_str, name) in examples {
         println!("\n{}: {}", name, smiles_str);
-
-        let mut parser = Parser::new(smiles_str);
-        match parser.parse_smiles() {
-            Ok(smiles) => {
-                let mut counter = MolecularFormulaCounter::new();
-                counter.visit_smiles(&smiles);
-                let formula = counter.get_formula();
-                println!("  Molecular Formula: {}", formula);
+        let lexer = smiles::lexer::Lexer::new(smiles_str);
+        let mut token_stream = smiles::token_stream::TokenStream::new(lexer);
+        if let Some(mut line_tokens) = token_stream.stream_line() {
+            let mut peekable_tokens = line_tokens.by_ref().peekable();
+            match Parser::parse_smiles_from_tokens(&mut peekable_tokens, smiles_str) {
+                Ok(smiles) => {
+                    let mut counter = MolecularFormulaCounter::new();
+                    counter.visit_smiles(&smiles);
+                    let formula = counter.get_formula();
+                    println!("  Molecular Formula: {}", formula);
+                }
+                Err(err) => {
+                    println!("  Parse error: {}", err);
+                }
             }
-            Err(err) => {
-                println!("  Parse error: {}", err);
+            // Optionally print lexical errors
+            let line_errors = peekable_tokens.collect::<Vec<_>>();
+            for err in line_errors {
+                if let Err(lex_errs) = err {
+                    for lex in lex_errs.iter() {
+                        println!("  Lexical error: {}", lex);
+                    }
+                }
             }
         }
     }

examples/pretty_printer.rs

@@ -1,5 +1,5 @@
 //! Example demonstrating the visitor pattern for pretty-printing SMILES
-//! 
+//!
 //! Run with: cargo run --example pretty_printer
 
 use smiles::{parser::Parser, visitor::*};
@@ -148,7 +148,7 @@ impl Visitor for PrettyPrinter {
             smiles::ast::RingNumber::Single(n) => n.to_string(),
             smiles::ast::RingNumber::Double(n) => format!("%{}", n),
         };
-        
+
         if let Some(bond) = &ring_bond.bond {
             self.write_line(&format!("Ring Bond: {} with bond", number_str));
             self.with_indent(|printer| {
@@ -187,30 +187,42 @@ impl Visitor for PrettyPrinter {
 fn main() {
     // Example SMILES strings to parse and pretty-print
     let examples = vec![
-        "C",                    // Simple carbon
-        "CCO",                  // Ethanol
-        "c1ccccc1",            // Benzene
-        "CC(C)C",              // Isobutane
-        "[13C]",               // Carbon-13 isotope
-        "[NH4+]",              // Ammonium ion
-        "C1CC1",               // Cyclopropane
-        "CC(=O)O",             // Acetic acid
-        "c1cc(Cl)ccc1",        // Chlorobenzene
-        "CC(C)(C)C",           // tert-Butane
+        "C",            // Simple carbon
+        "CCO",          // Ethanol
+        "c1ccccc1",     // Benzene
+        "CC(C)C",       // Isobutane
+        "[13C]",        // Carbon-13 isotope
+        "[NH4+]",       // Ammonium ion
+        "C1CC1",        // Cyclopropane
+        "CC(=O)O",      // Acetic acid
+        "c1cc(Cl)ccc1", // Chlorobenzene
+        "CC(C)(C)C",    // tert-Butane
     ];
 
     for smiles_str in examples {
         println!("=== Parsing: {} ===", smiles_str);
-
-        let mut parser = Parser::new(smiles_str);
-        match parser.parse_smiles() {
-            Ok(smiles) => {
-                let mut printer = PrettyPrinter::new();
-                printer.visit_smiles(&smiles);
-                println!("{}", printer.get_output());
+        let lexer = smiles::lexer::Lexer::new(smiles_str);
+        let mut token_stream = smiles::token_stream::TokenStream::new(lexer);
+        if let Some(mut line_tokens) = token_stream.stream_line() {
+            let mut peekable_tokens = line_tokens.by_ref().peekable();
+            match Parser::parse_smiles_from_tokens(&mut peekable_tokens, smiles_str) {
+                Ok(smiles) => {
+                    let mut printer = PrettyPrinter::new();
+                    printer.visit_smiles(&smiles);
+                    println!("{}", printer.get_output());
+                }
+                Err(err) => {
+                    println!("Parse error: {}", err);
+                }
             }
-            Err(err) => {
-                println!("Parse error: {}", err);
+            // Optionally print lexical errors
+            let line_errors = peekable_tokens.collect::<Vec<_>>();
+            for err in line_errors {
+                if let Err(lex_errs) = err {
+                    for lex in lex_errs.iter() {
+                        println!("Lexical error: {}", lex);
+                    }
+                }
             }
         }
     }

examples/streaming_demo.rs

@@ -1,7 +1,9 @@
+use smiles::lexer::Lexer;
 use smiles::parser::Parser;
+use smiles::token_stream::TokenStream;
 
 fn main() {
-    println!("=== Streaming Multi-line SMILES Parser Demo ===\n");
+    println!("=== Streaming Multi-line SMILES Parser Demo (TokenStream) ===\n");
 
     // Test data with mixed valid/invalid SMILES and empty lines
     let multi_smiles = r#"C
@@ -23,22 +25,43 @@ C1CCCCC1
     println!("{}", multi_smiles);
     println!("=== Parsing Results ===");
 
-    let mut parser = Parser::new(multi_smiles);
-    let (successes, parse_errors, lexical_errors) = parser.parse_multiple_lines();
+    let lexer = Lexer::new(multi_smiles);
+    let mut token_stream = TokenStream::new(lexer);
+    let mut successes = Vec::new();
+    let mut parse_errors = Vec::new();
+    let mut lexical_errors = Vec::new();
+    let mut line_num = 1;
+    while let Some(mut line_tokens) = token_stream.stream_line() {
+        let mut peekable_tokens = line_tokens.by_ref().peekable();
+        match Parser::parse_smiles_from_tokens(&mut peekable_tokens, multi_smiles) {
+            Ok(smiles) => successes.push(smiles),
+            Err(e) => parse_errors.push(format!("Line {}: {}", line_num, e)),
+        }
+        // Collect lexical errors for this line
+        let line_errs = peekable_tokens.collect::<Vec<_>>();
+        for err in line_errs {
+            if let Err(lex_errs) = err {
+                for lex in lex_errs.iter() {
+                    lexical_errors.push(format!("Line {}: {}", line_num, lex));
+                }
+            }
+        }
+        line_num += 1;
+    }
 
     // Report successful parses
     for (i, _smiles) in successes.iter().enumerate() {
-        println!("SMILES {}: ✓ Successfully parsed", i + 1);
+        println!("SMILES {}: \u{2713} Successfully parsed", i + 1);
     }
 
     // Report parse errors
-    for (i, error) in parse_errors.iter().enumerate() {
-        println!("Parse Error {}: {}", i + 1, error);
+    for err in &parse_errors {
+        println!("Parse Error: {}", err);
     }
 
     // Report lexical errors
-    for (i, error) in lexical_errors.iter().enumerate() {
-        println!("Lexical Error {}: {}", i + 1, error);
+    for err in &lexical_errors {
+        println!("Lexical Error: {}", err);
     }
 
     println!();
@@ -56,9 +79,9 @@ C1CCCCC1
     println!("Lexical errors: {}", lexical_errors.len());
 
     println!("\n=== Key Features Demonstrated ===");
-    println!("✓ Streaming token-based parsing (no upfront line splitting)");
-    println!("✓ Error recovery (invalid SMILES don't affect subsequent ones)");
-    println!("✓ Empty line handling");
-    println!("✓ Ring bond state isolation between SMILES");
-    println!("✓ Memory efficient processing");
+    println!("\u{2713} Streaming token-based parsing (no upfront line splitting)");
+    println!("\u{2713} Error recovery (invalid SMILES don't affect subsequent ones)");
+    println!("\u{2713} Empty line handling");
+    println!("\u{2713} Ring bond state isolation between SMILES");
+    println!("\u{2713} Memory efficient processing");
 }

src/lexer.rs

@@ -519,6 +519,10 @@ impl<'src> Lexer<'src> {
 
         false
     }
+
+    pub fn is_eof(&self) -> bool {
+        self.cursor.is_eof()
+    }
 }
 
 /// Creates an iterator that produces tokens from the input string.

src/lib.rs

@@ -5,9 +5,10 @@ pub mod lexer;
 pub mod parser;
 pub mod span;
 pub mod token;
+pub mod token_stream;
 pub mod visitor;
 
 // Re-export key types for convenience
-pub use parser::{Parser, SmilesLineIterator};
+pub use parser::Parser;
 pub use error::{SmilesLineError, SmilesLineResult, SmilesErrorType};
 pub use ast::Smiles;