Add an example using RTParserBuilder during CTParserBuilder

Author: ratmice

Cargo.toml

@@ -12,6 +12,7 @@ members=[
     "lrpar/examples/calc_parsetree",
     "lrpar/examples/start_states",
     "lrpar/examples/clone_param",
+    "lrpar/examples/inspect",
     "lrtable",
     "nimbleparse",
 ]

lrlex/src/lib/ctbuilder.rs

@@ -344,17 +344,18 @@ where
     ///      module name is `c_l` (i.e. the file's leaf name, minus its extension, with a prefix of
     ///      `_l`).
     pub fn build(mut self) -> Result<CTLexer, Box<dyn Error>> {
+        let lexerp = self
+            .lexer_path
+            .as_ref()
+            .expect("lexer_path must be specified before processing.");
         if let Some(ref lrcfg) = self.lrpar_config {
             let mut ctp = CTParserBuilder::<LexerTypesT>::new();
+            ctp = ctp.lexer_path(lexerp.to_owned());
             ctp = lrcfg(ctp);
             let map = ctp.build()?;
             self.rule_ids_map = Some(map.token_map().to_owned());
         }
 
-        let lexerp = self
-            .lexer_path
-            .as_ref()
-            .expect("lexer_path must be specified before processing.");
         let outp = self
             .output_path
             .as_ref()

lrpar/examples/inspect/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "inspect"
+description = "During the code generation step inspect the generated parser"
+version = "0.1.0"
+edition = "2021"
+license = "Apache-2.0/MIT"
+
+[[bin]]
+doc = false
+name = "inspect"
+
+[build-dependencies]
+cfgrammar = { path="../../../cfgrammar" }
+lrlex = { path="../../../lrlex" }
+lrpar = { path="../.." }
+num-traits.workspace = true
+
+[dependencies]
+cfgrammar = { path="../../../cfgrammar" }
+lrlex = { path="../../../lrlex" }
+lrpar = { path="../.." }

lrpar/examples/inspect/build.rs

@@ -0,0 +1,83 @@
+#![deny(rust_2018_idioms)]
+use cfgrammar::{yacc::YaccGrammar, Span};
+use lrlex::{CTLexerBuilder, DefaultLexerTypes, LRNonStreamingLexerDef, LexerDef};
+use lrpar::{LexerTypes, RTParserBuilder};
+use num_traits::ToPrimitive as _;
+use std::io::Read;
+
+pub fn set_rule_ids<LexerTypesT: LexerTypes<StorageT = u32>, LT: LexerDef<LexerTypesT>>(
+    lexerdef: &mut LT,
+    grm: &YaccGrammar,
+) -> (Option<Vec<Span>>, Option<Vec<Span>>)
+where
+    usize: num_traits::AsPrimitive<LexerTypesT::StorageT>,
+{
+    let rule_ids = grm
+        .tokens_map()
+        .iter()
+        .map(|(&n, &i)| (n, usize::from(i).to_u32().unwrap()))
+        .collect::<std::collections::HashMap<&str, u32>>();
+    let (missing_from_lexer, missing_from_parser) = lexerdef.set_rule_ids_spanned(&rule_ids);
+    let missing_from_lexer = missing_from_lexer.map(|tokens| {
+        tokens
+            .iter()
+            .map(|name| {
+                grm.token_span(*grm.tokens_map().get(name).unwrap())
+                    .expect("Given token should have a span")
+            })
+            .collect::<Vec<_>>()
+    });
+
+    let missing_from_parser =
+        missing_from_parser.map(|tokens| tokens.iter().map(|(_, span)| *span).collect::<Vec<_>>());
+    (missing_from_lexer, missing_from_parser)
+}
+
+fn main() {
+    // Since we're using both lrlex and lrpar, we use lrlex's `lrpar_config` convenience function
+    // that makes it easy to a) create a lexer and parser and b) link them together.
+    CTLexerBuilder::new()
+        .rust_edition(lrlex::RustEdition::Rust2021)
+        .lrpar_config(|ctp| {
+            ctp.rust_edition(lrpar::RustEdition::Rust2021)
+                .inspect(Box::new(move |_, recov, grm, stable, _, lexer_path| {
+                    let good_srcs = ["", "TODO: [Walk, relax]", "todo: [run, rest]"];
+                    let bad_srcs = [
+                        // Should start with TODO:
+                        "Frodo: [Breakfast, Elevenses]",
+                    ];
+                    let mut lexer_src = String::new();
+                    let _ = std::fs::File::open(lexer_path.unwrap())
+                        .unwrap()
+                        .read_to_string(&mut lexer_src);
+                    let mut lexerdef =
+                        LRNonStreamingLexerDef::<DefaultLexerTypes>::from_str(&lexer_src).unwrap();
+                    set_rule_ids(&mut lexerdef, grm);
+                    let pb = RTParserBuilder::new(grm, stable).recoverer(recov);
+                    for src in good_srcs {
+                        let lexer = lexerdef.lexer(src);
+                        let errs = pb.parse_noaction(&lexer);
+                        if !errs.is_empty() {
+                            return Err(format!("{:?} while parsing src: {}", errs, src).into());
+                        }
+                    }
+                    for src in bad_srcs {
+                        let lexer = lexerdef.lexer(src);
+                        let errs = pb.parse_noaction(&lexer);
+                        if errs.is_empty() {
+                            return Err(format!(
+                                "Parse of source '{src}' succeeded while expecting failure"
+                            )
+                            .into());
+                        }
+                    }
+                    Ok(())
+                }))
+                .grammar_in_src_dir("todo.y")
+                .unwrap()
+        })
+        .lexer_in_src_dir("todo.l")
+        .unwrap()
+        .build()
+        .unwrap();
+}

lrpar/examples/inspect/src/main.rs

@@ -0,0 +1,24 @@
+#![allow(clippy::unnecessary_wraps)]
+
+use std::io::{self, Read};
+
+use lrlex::lrlex_mod;
+use lrpar::lrpar_mod;
+
+lrlex_mod!("todo.l");
+lrpar_mod!("todo.y");
+
+// For the example within this bin see build.rs
+fn main() {
+    let lexerdef = todo_l::lexerdef();
+    let mut input = String::new();
+    let _ = io::stdin().read_to_string(&mut input).unwrap();
+    let lexer = lexerdef.lexer(&input);
+    let (node, errs) = todo_y::parse(&lexer);
+    if let Some(node) = node {
+        println!("{:?}", node);
+    }
+    for e in errs {
+        eprintln!("Error: {}", e);
+    }
+}

lrpar/examples/inspect/src/todo.l

@@ -0,0 +1,9 @@
+%grmtools{case_insensitive}
+%%
+\[       '['
+\]       ']'
+\:        ':'
+\,        ','
+todo     'TODO'
+[A-Z]+   'NAME'
+[\n\t\ ] ;

lrpar/examples/inspect/src/todo.y

@@ -0,0 +1,7 @@
+%grmtools{yacckind: Original(GenericParseTree)}
+%token NAME
+%token TODO
+%%
+Start: %empty | TODO ':' '[' items ']';
+items: item | items ',' item;
+item: NAME | %empty;

lrpar/src/lib/ctbuilder.rs

@@ -242,13 +242,17 @@ where
     show_warnings: bool,
     visibility: Visibility,
     rust_edition: RustEdition,
+    lexer_path: Option<PathBuf>,
     // We want lifetimes that imply the callback can't capture the header or the grammar.
-    header_callback: Option<
+    inspect_callback: Option<
         Box<
-            dyn for<'h, 'y> Fn(
-                &'h mut Header,
+            dyn for<'y> Fn(
+                &'y mut Header,
                 RecoveryKind,
                 &'y YaccGrammar<LexerTypesT::StorageT>,
+                &'y StateTable<LexerTypesT::StorageT>,
+                &'y StateGraph<LexerTypesT::StorageT>,
+                Option<PathBuf>,
             ) -> Result<(), Box<dyn Error>>,
         >,
     >,
@@ -296,7 +300,8 @@ where
             show_warnings: true,
             visibility: Visibility::Private,
             rust_edition: RustEdition::Rust2021,
-            header_callback: None,
+            inspect_callback: None,
+            lexer_path: None,
             phantom: PhantomData,
         }
     }
@@ -427,17 +432,27 @@ where
         self
     }
 
-    pub fn process_header(
+    /// Sets the path to the lexer sources this is for usage from within callbacks only, and
+    /// not used during the build process.
+    pub fn lexer_path(mut self, lexer_path: PathBuf) -> Self {
+        self.lexer_path = Some(lexer_path);
+        self
+    }
+
+    pub fn inspect(
         mut self,
         cb: Box<
             dyn for<'h, 'y> Fn(
                 &'h mut Header,
                 RecoveryKind,
                 &'y YaccGrammar<StorageT>,
+                &'y StateTable<StorageT>,
+                &'y StateGraph<StorageT>,
+                Option<PathBuf>,
             ) -> Result<(), Box<dyn Error>>,
         >,
     ) -> Self {
-        self.header_callback = Some(cb);
+        self.inspect_callback = Some(cb);
         self
     }
 
@@ -637,27 +652,6 @@ where
             }
         };
 
-        if let Some(cb) = &self.header_callback {
-            cb(
-                &mut header,
-                self.recoverer.expect("has a default value"),
-                &grm,
-            )?;
-        }
-
-        let unused_keys = header.unused();
-        if !unused_keys.is_empty() {
-            return Err(format!("Unused keys in header: {}", unused_keys.join(", ")).into());
-        }
-        let missing_keys = header.missing();
-        if !missing_keys.is_empty() {
-            return Err(format!(
-                "Required values were missing from the header: {}",
-                unused_keys.join(", ")
-            )
-            .into());
-        }
-
         let rule_ids = grm
             .tokens_map()
             .iter()
@@ -727,6 +721,31 @@ where
         fs::remove_file(outp).ok();
 
         let (sgraph, stable) = from_yacc(&grm, Minimiser::Pager)?;
+
+        if let Some(cb) = &self.inspect_callback {
+            cb(
+                &mut header,
+                self.recoverer.expect("has a default value"),
+                &grm,
+                &stable,
+                &sgraph,
+                self.lexer_path.clone(),
+            )?;
+        }
+
+        let unused_keys = header.unused();
+        if !unused_keys.is_empty() {
+            return Err(format!("Unused keys in header: {}", unused_keys.join(", ")).into());
+        }
+        let missing_keys = header.missing();
+        if !missing_keys.is_empty() {
+            return Err(format!(
+                "Required values were missing from the header: {}",
+                unused_keys.join(", ")
+            )
+            .into());
+        }
+
         if self.error_on_conflicts {
             if let Some(c) = stable.conflicts() {
                 match (grm.expect(), grm.expectrr()) {
@@ -853,7 +872,8 @@ where
             show_warnings: self.show_warnings,
             visibility: self.visibility.clone(),
             rust_edition: self.rust_edition,
-            header_callback: None,
+            inspect_callback: None,
+            lexer_path: self.lexer_path.clone(),
             phantom: PhantomData,
         };
         Ok(cl.build()?.rule_ids)
@@ -930,7 +950,7 @@ where
         // rustc forces a recompile, this will change this value, causing anything which depends on
         // this build of lrpar to be recompiled too.
         let Self {
-            // All variables except for `output_path`, `header_callback` and `phantom` should
+            // All variables except for `output_path`, `inspect_callback` and `phantom` should
             // be written into the cache.
             grammar_path,
             mod_name,
@@ -942,11 +962,14 @@ where
             show_warnings,
             visibility,
             rust_edition,
-            header_callback: _,
+            inspect_callback: _,
+            lexer_path,
             phantom: _,
         } = self;
         let build_time = env!("VERGEN_BUILD_TIMESTAMP");
         let grammar_path = grammar_path.as_ref().unwrap().to_string_lossy();
+        let empty_path = PathBuf::new();
+        let lexer_path = lexer_path.as_ref().unwrap_or(&empty_path).to_string_lossy();
         let mod_name = QuoteOption(mod_name.as_deref());
         let visibility = visibility.to_variant_tokens();
         let rust_edition = rust_edition.to_variant_tokens();
@@ -973,6 +996,7 @@ where
             RUST_EDITION = #rust_edition
             RULE_IDS_MAP = [#(#rule_map,)*]
             VISIBILITY = #visibility
+            LEX_PATH = #lexer_path
         };
         let cache_info_str = cache_info.to_string();
         quote!(#cache_info_str)