Add API for user defined grmtools section entries in GrammarAST

cfgrammar/src/lib/header.rs

@@ -6,15 +6,20 @@ use crate::{
     },
 };
 use regex::{Regex, RegexBuilder};
-use std::{collections::HashMap, error::Error, fmt, sync::LazyLock};
+use std::{
+    collections::{HashMap, HashSet},
+    error::Error,
+    fmt,
+    sync::LazyLock,
+};
 
 /// An error regarding the `%grmtools` header section.
 ///
 /// It could be any of:
 ///
 /// * An error during parsing the section.
 /// * An error resulting from a value in the section having an invalid value.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 #[doc(hidden)]
 pub struct HeaderError<T> {
     pub kind: HeaderErrorKind,
@@ -48,7 +53,7 @@ impl Spanned for HeaderError<Span> {
 
 // This is essentially a tuple that needs a newtype so we can implement `From` for it.
 // Thus we aren't worried about it being `pub`.
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 #[doc(hidden)]
 pub struct HeaderValue<T>(pub T, pub Value<T>);
 
@@ -166,7 +171,7 @@ pub static RE_CRATE_DOT: LazyLock<Regex> = LazyLock::new(|| {
 static RE_DIGITS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^[0-9]+").unwrap());
 static RE_STRING: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"^\"(\\.|[^"\\])*\""#).unwrap());
 #[doc(hidden)]
-pub static CRATE_KEY_MAP: LazyLock<HashMap<&'static str, &'static str>> = LazyLock::new(|| {
+pub static KEY_CRATE_MAP: LazyLock<HashMap<&'static str, &'static str>> = LazyLock::new(|| {
     let mut map = HashMap::new();
     let cfgrammar = ["yacckind"];
     let lrpar = ["recoverer", "test_files", "serialisation_format"];
@@ -199,6 +204,44 @@ pub static CRATE_KEY_MAP: LazyLock<HashMap<&'static str, &'static str>> = LazyLo
     map
 });
 
+#[doc(hidden)]
+pub static CFGRAMMAR_KEYS: LazyLock<HashSet<&'static str>> =
+    LazyLock::new(|| HashSet::from_iter(["cfgrammar.yacckind"]));
+
+#[doc(hidden)]
+pub static LRPAR_KEYS: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
+    HashSet::from_iter([
+        "lrpar.recoverer",
+        "lrpar.test_files",
+        "lrpar.serialisation_format",
+    ])
+});
+
+#[doc(hidden)]
+pub static LRLEX_KEYS: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
+    HashSet::from_iter([
+        "lrlex.lexerkind",
+        "lrlex.allow_wholeline_comments",
+        "lrlex.posix_escapes",
+    ])
+});
+
+#[doc(hidden)]
+pub static REGEX_KEYS: LazyLock<HashSet<&'static str>> = LazyLock::new(|| {
+    HashSet::from_iter([
+        "regex.case_insensitive",
+        "regex.dot_matches_new_line",
+        "regex.multi_line",
+        "regex.octal",
+        "regex.swap_greed",
+        "regex.ignore_whitespace",
+        "regex.unicode",
+        "regex.size_limit",
+        "regex.dfa_size_limit",
+        "regex.nest_limit",
+    ])
+});
+
 const MAGIC: &str = "%grmtools";
 
 fn add_duplicate_occurrence<T: Eq + PartialEq + Clone>(
@@ -355,10 +398,14 @@ impl<'input> GrmtoolsSectionParser<'input> {
                     let (key, key_loc, val, j) = match self.parse_key_value(i) {
                         Ok((key, key_loc, val, pos)) => {
                             let key = if !RE_CRATE_DOT.is_match(&key) {
-                                if let Some(crate_name) = CRATE_KEY_MAP.get(key.as_str()) {
+                                if let Some(crate_name) = KEY_CRATE_MAP.get(key.as_str()) {
                                     format!("{crate_name}.{key}")
                                 } else {
-                                    key
+                                    errs.push(HeaderError {
+                                        kind: HeaderErrorKind::IllegalName,
+                                        locations: vec![key_loc],
+                                    });
+                                    return Err(errs);
                                 }
                             } else {
                                 key
@@ -477,14 +524,6 @@ impl<'input> GrmtoolsSectionParser<'input> {
 #[doc(hidden)]
 pub type Header<T> = MarkMap<String, HeaderValue<T>>;
 
-impl TryFrom<YaccKind> for Value<Location> {
-    type Error = HeaderError<Location>;
-    fn try_from(kind: YaccKind) -> Result<Value<Location>, HeaderError<Location>> {
-        let from_loc = Location::Other("From<YaccKind>".to_string());
-        Ok(Value::Namespaced(format!("YaccKind::{kind:?}"), from_loc))
-    }
-}
-
 impl<T: Clone> TryFrom<&Value<T>> for YaccKind {
     type Error = HeaderError<T>;
     fn try_from(value: &Value<T>) -> Result<YaccKind, HeaderError<T>> {
@@ -588,7 +627,7 @@ mod test {
 
     #[test]
     fn test_header_duplicates() {
-        let src = "%grmtools {dupe, !dupe, dupe: test}";
+        let src = "%grmtools {test.dupe, !test.dupe, test.dupe: test}";
         for flag in [true, false] {
             let parser = GrmtoolsSectionParser::new(src, flag);
             let res = parser.parse();

cfgrammar/src/lib/markmap.rs

@@ -21,7 +21,7 @@ use std::fmt;
 ///
 /// Merge behaviors configure how the merge operator handles cases where both `MarkMaps` being merged
 /// contain a particular key.
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Clone)]
 #[doc(hidden)]
 pub struct MarkMap<K, V> {
     default_merge_behavior: MergeBehavior,
@@ -484,15 +484,15 @@ impl<K: Ord + Clone, V> MarkMap<K, V> {
     }
 
     /// Returns a `Vec` containing all the keys that are not marked as used.
-    pub fn unused(&self) -> Vec<K> {
-        let mut ret = Vec::new();
-        for (k, mark, v) in &self.contents {
+    pub fn unused(&self) -> impl Iterator<Item = (&K, &V)> {
+        self.contents.iter().filter_map(|(k, mark, v)| {
             let used_mark = Mark::Used.repr();
             if v.is_some() && mark & used_mark == 0 {
-                ret.push(k.to_owned())
+                Some((k, v.as_ref().unwrap()))
+            } else {
+                None
             }
-        }
-        ret
+        })
     }
 
     /// Returns a `Vec` containing all the keys that are marked as required,
@@ -546,11 +546,18 @@ impl<'a, K, V> Iterator for MarkMapIterRef<'a, K, V> {
     type Item = (&'a K, &'a V);
 
     fn next(&mut self) -> Option<Self::Item> {
-        if let Some((k, _, v)) = self.map.contents.get(self.pos) {
+        loop {
+            if self.pos >= self.map.contents.len() {
+                return None;
+            }
+            let pos = self.pos;
             self.pos += 1;
-            v.as_ref().map(|v| (k, v))
-        } else {
-            None
+            if self.map.contents[pos].2.is_some() {
+                return Some((
+                    &self.map.contents[pos].0,
+                    self.map.contents[pos].2.as_ref().unwrap(),
+                ));
+            }
         }
     }
 }
@@ -711,8 +718,8 @@ mod test {
             assert!(mm.insert("a", "test").is_none());
             mm.mark_used(&"a");
             assert_eq!(mm.get_mark(&"a"), Some(Mark::Used.repr()));
-            let empty: &[&String] = &[];
-            assert_eq!(mm.unused().as_slice(), empty);
+            let empty: &[(&&str, &&str)] = &[];
+            assert_eq!(mm.unused().collect::<Vec<(&&str, &&str)>>(), empty);
         }
 
         {
@@ -722,7 +729,10 @@ mod test {
             assert!(mm.insert("b", "unused").is_none());
             assert_eq!(mm.get_mark(&"a"), Some(Mark::Used.repr()));
             assert_eq!(mm.get_mark(&"b"), Some(0));
-            assert_eq!(mm.unused().as_slice(), &["b"]);
+            assert_eq!(
+                mm.unused().collect::<Vec<(&&str, &&str)>>(),
+                vec![(&"b", &"unused")]
+            );
         }
     }
 

cfgrammar/src/lib/mod.rs

@@ -68,6 +68,9 @@ pub mod yacc;
 pub use newlinecache::NewlineCache;
 pub use span::{Location, Span, Spanned};
 
+#[cfg(test)]
+pub mod test_utils;
+
 /// A type specifically for rule indices.
 pub use crate::idxnewtype::{PIdx, RIdx, SIdx, TIdx};