Implement Leo's optimization

compact-bocage/src/bocage.rs

@@ -175,6 +175,12 @@ impl Forest for Bocage {
         result
     }
 
+    #[inline]
+    fn leo_product(&mut self, left_factor: Self::NodeRef, right_factor: Self::NodeRef)
+            -> Self::NodeRef {
+        unimplemented!()
+    }
+
     #[inline]
     fn sum(&mut self, _lhs_sym: Symbol, _origin: u32) -> Self::NodeRef {
         let result = self.sum.unwrap();

gearley-default-grammar/Cargo.toml

@@ -9,6 +9,7 @@ edition = "2021"
 gearley-grammar = { path = "../gearley-grammar/" }
 gearley-vec2d = { path = "../gearley-vec2d/" }
 cfg = { path = "../../cfg/cfg/" }
+bit-vec = "0.8"
 bit-matrix = { workspace = true }
 cfg-symbol = { workspace = true }
 miniserde = { workspace = true }

gearley-default-grammar/src/lib.rs

@@ -4,6 +4,8 @@ use std::{cmp, iter};
 
 use bit_matrix::row::BitSlice;
 use bit_matrix::BitMatrix;
+use bit_vec::BitVec;
+use cfg::classify::recursive::RecursionKind;
 use cfg::classify::CfgClassifyExt;
 use cfg::predict_sets::{FirstSets, FollowSets, PredictSets};
 use cfg::symbol_bit_matrix::{CfgSymbolBitMatrixExt, Remap};
@@ -67,6 +69,8 @@ pub struct DefaultGrammar {
 
     scan_prediction_matrix: BitMatrix,
 
+    is_right_recursive: BitVec,
+
     // For the forest
     forest_info: ForestInfo,
 }
@@ -311,6 +315,7 @@ impl DefaultGrammar {
         result.populate_maps(maps);
         result.populate_grammar(&grammar);
         result.populate_nulling(nulling);
+        result.populate_leo(&grammar);
         trace!("result: {:?}", result);
         result
     }
@@ -609,6 +614,15 @@ impl DefaultGrammar {
             .nulling_intermediate_rules
             .extend(iter_nulling_intermediate);
     }
+
+    fn populate_leo(&mut self, grammar: &Cfg) {
+        self.is_right_recursive = BitVec::from_elem(self.num_syms(), false);
+        for rule in grammar.recursion().recursive_rules() {
+            if rule.recursion == RecursionKind::Right && rule.rule.lhs.usize() < self.size.syms {
+                self.is_right_recursive.set(rule.rule.lhs.usize(), true);
+            }
+        }
+    }
 }
 
 impl Grammar for DefaultGrammar {
@@ -714,6 +728,11 @@ impl Grammar for DefaultGrammar {
         self.columns[0].syms[dot as usize].unwrap()
     }
 
+    #[inline]
+    fn is_right_recursive(&self, sym: Symbol) -> bool {
+        self.is_right_recursive.get(sym.usize()).unwrap_or(false)
+    }
+
     #[inline]
     fn external_origin(&self, dot: Dot) -> ExternalOrigin {
         self.forest_info.eval.get(dot as usize).cloned().unwrap()

gearley-forest/src/forest.rs

@@ -19,6 +19,9 @@ pub trait Forest<S = Symbol> {
     fn product(&mut self, left_factor: Self::NodeRef, right_factor: Self::NodeRef)
         -> Self::NodeRef;
 
+    fn leo_product(&mut self, left_factor: Self::NodeRef, right_factor: Self::NodeRef)
+        -> Self::NodeRef;
+
     fn leaf(&mut self, token: S, pos: u32, value: u32) -> Self::NodeRef;
 
     fn nulling(&self, token: S) -> Self::NodeRef;

gearley-forest/src/null_forest.rs

@@ -28,6 +28,13 @@ impl<S: Copy> Forest<S> for NullForest {
     ) -> Self::NodeRef {
     }
     #[inline(always)]
+    fn leo_product(
+        &mut self,
+        _left_factor: Self::NodeRef,
+        _right_factor: Self::NodeRef,
+    ) -> Self::NodeRef {
+    }
+    #[inline(always)]
     fn sum(&mut self, _lhs_sym: S, _origin: u32) -> Self::NodeRef {
         ()
     }

gearley-grammar/src/lib.rs

@@ -55,6 +55,8 @@ pub trait Grammar {
 
     fn get_lhs(&self, dot: Dot) -> Symbol;
 
+    fn is_right_recursive(&self, sym: Symbol) -> bool;
+
     fn external_origin(&self, dot: Dot) -> ExternalOrigin;
 
     fn eliminated_nulling_intermediate(&self) -> &[NullingIntermediateRule];
@@ -150,6 +152,10 @@ where
         (**self).get_lhs(dot)
     }
 
+    fn is_right_recursive(&self, sym: Symbol) -> bool {
+        (**self).is_right_recursive(sym)
+    }
+
     fn external_origin(&self, dot: Dot) -> ExternalOrigin {
         (**self).external_origin(dot)
     }

gearley/tests/test_c.rs

@@ -1,7 +1,7 @@
 use cfg::Cfg;
 use cfg_examples::c::grammar as c_grammar;
 use cfg_load::CfgLoadExt;
-use compact_bocage::Bocage;
+use simple_bocage::Bocage;
 use gearley::{DefaultGrammar, Recognizer, RecognizerParseExt};
 
 const SYM_NAMES: [Option<&'static str>; 176] = [

gearley/tests/test_leo.rs

@@ -1,8 +1,44 @@
-use core::panic;
-
-use cfg::Cfg;
+use cfg::{Cfg, Symbol};
 use gearley::utils::RecognizerParseExt;
-use gearley::{DefaultGrammar, Forest, Recognizer};
+use gearley::{DefaultGrammar, Forest, PerfHint, Recognizer};
+
+struct NoLeoPerfHint;
+
+impl PerfHint for NoLeoPerfHint {
+    const LEO: bool = false;
+    const LOOKAHEAD: bool = true;
+    type Symbol = Symbol;
+}
+
+#[test]
+fn test_right_rec() {
+    let _ = env_logger::try_init();
+    let mut external = Cfg::new();
+    let [start, rep, rr, maybe] = external.sym();
+    external
+        .rule(start)
+        .rhs([rr, rep])
+        .rule(rr)
+        .rhs([rep, maybe])
+        .rhs([rep])
+        .rule(maybe)
+        .rhs([])
+        .rhs([rr]);
+    external.set_roots([start]);
+    let cfg = DefaultGrammar::from_grammar(external);
+    let mut rec = Recognizer::with_forest(&cfg, InstrBocage::new());
+    assert!(rec.parse(&[rep, rep, rep, rep, rep]).unwrap());
+    assert_eq!(
+        rec.into_forest(),
+        InstrBocage {
+            num_sums: 11,
+            num_leaves: 6,
+            num_products: 2,
+            num_leo_products: 8,
+            num_summands: 11
+        }
+    );
+}
 
 #[test]
 fn test_leo() {
@@ -11,21 +47,49 @@ fn test_leo() {
     let [start, rep, rr] = external.sym();
     external
         .rule(start)
-        .rhs([rr])
+        .rhs([rr, rep])
         .rule(rr)
         .rhs([rep, rr])
         .rhs([rep]);
     external.set_roots([start]);
     let cfg = DefaultGrammar::from_grammar(external);
     let mut rec = Recognizer::with_forest(&cfg, InstrBocage::new());
-    assert!(rec.parse(&[rep, rep, rep, rep]).unwrap());
+    assert!(rec.parse(&[rep, rep, rep, rep, rep]).unwrap());
+    assert_eq!(
+        rec.into_forest(),
+        InstrBocage {
+            num_sums: 11,
+            num_leaves: 6,
+            num_products: 2,
+            num_leo_products: 8,
+            num_summands: 11
+        }
+    );
+}
+
+#[test]
+fn test_no_leo() {
+    let _ = env_logger::try_init();
+    let mut external = Cfg::new();
+    let [start, rep, rr] = external.sym();
+    external
+        .rule(start)
+        .rhs([rr, rep])
+        .rule(rr)
+        .rhs([rep, rr])
+        .rhs([rep]);
+    external.set_roots([start]);
+    let cfg = DefaultGrammar::from_grammar(external);
+    let mut rec = Recognizer::with_forest_and_policy(&cfg, InstrBocage::new(), NoLeoPerfHint);
+    assert!(rec.parse(&[rep, rep, rep, rep, rep]).unwrap());
     assert_eq!(
         rec.into_forest(),
         InstrBocage {
-            num_sums: 18,
-            num_leaves: 5,
-            num_products: 10,
-            num_summands: 18
+            num_sums: 17,
+            num_leaves: 6,
+            num_products: 12,
+            num_leo_products: 0,
+            num_summands: 17
         }
     );
 }
@@ -35,6 +99,7 @@ struct InstrBocage {
     num_sums: usize,
     num_leaves: usize,
     num_products: usize,
+    num_leo_products: usize,
     num_summands: usize,
 }
 
@@ -68,6 +133,14 @@ impl Forest for InstrBocage {
         self.num_products += 1;
     }
 
+    fn leo_product(
+        &mut self,
+        left_factor: Self::NodeRef,
+        right_factor: Self::NodeRef,
+    ) -> Self::NodeRef {
+        self.num_leo_products += 1;
+    }
+
     fn push_summand(&mut self, item: gearley::Item<Self::NodeRef>) {
         self.num_summands += 1;
     }

simple-bocage/src/bocage.rs

@@ -143,6 +143,16 @@ impl Forest for Bocage {
         NodeHandle(self.graph.len() as u32 - 1)
     }
 
+    #[inline]
+    fn leo_product(&mut self, left_factor: Self::NodeRef, right_factor: Self::NodeRef)
+            -> Self::NodeRef {
+        self.graph.push(Node::LeoRule {
+            left_factor,
+            right_factor,
+        });
+        NodeHandle(self.graph.len() as u32 - 1)
+    }
+
     #[inline]
     fn sum(&mut self, lhs_sym: Symbol, _origin: u32) -> Self::NodeRef {
         let result = {

simple-bocage/src/node.rs

@@ -17,6 +17,10 @@ pub enum Node {
         action: u32,
         factors: NodeHandle,
     },
+    LeoRule {
+        left_factor: NodeHandle,
+        right_factor: NodeHandle,
+    },
     Rule {
         // 8 bytes.
         left_factor: NodeHandle,

simple-bocage/src/traverse.rs

@@ -81,6 +81,31 @@ impl Bocage {
                         results: AVec::new_in(&alloc),
                     });
                 }
+                (Node::LeoRule { left_factor, .. }, 0) => {
+                    work.child += 1;
+                    let parent = work.parent;
+                    work_stack.push(work);
+                    work_stack.push(WorkNode {
+                        node: left_factor,
+                        child: 0,
+                        parent,
+                        results: AVec::new_in(&alloc),
+                    });
+                }
+                (Node::LeoRule { right_factor, .. }, 1) => {
+                    work.child += 1;
+                    let parent = work.parent;
+                    work_stack.push(work);
+                    work_stack.push(WorkNode {
+                        node: right_factor,
+                        child: 0,
+                        parent,
+                        results: AVec::new_in(&alloc),
+                    });
+                }
+                (Node::LeoRule { .. }, _) => {
+                    // nothing to do
+                }
                 (Node::Rule { left_factor, .. }, 0) => {
                     work.child += 1;
                     let parent = work.parent;