Merge pull request #542 from ratmice/cttest_grmtools_section

Add comma to grmtools section syntax

Author: ltratt

.buildbot.sh

@@ -15,30 +15,35 @@ export PATH=`pwd`/.cargo_install/bin/:$WASMTIME_HOME/bin:$PATH
 
 # Install wasmtime once debian trixie is stablized
 # we can likely just use rust-wasmtime.
+#
+# Needed for wasm32-wasip2
 touch .wasmtime_profile
 if [ "X`which wasmtime`" = "X" ]; then
     PROFILE=".wasmtime_profile" bash -c 'curl https://wasmtime.dev/install.sh -sSf | bash'
 fi
 . ./.wasmtime_profile
 
+# Needed for wasm32-unknown-unknown
 mkdir -p $NVM_DIR
 PROFILE=/dev/null bash -c 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.2/install.sh | bash'
 . "$NVM_DIR/nvm.sh"
 # Download and install Node.js:
 nvm install 22
 
-rustup target add wasm32-unknown-unknown
-rustup target add wasm32-wasip2
-cargo install wasm-bindgen-cli
-
 cargo fmt --all -- --check
 
 rustup toolchain install stable
 rustup default stable
 
 cargo test
 cargo test --release
+
+rustup target add wasm32-unknown-unknown
+cargo install wasm-bindgen-cli
 cargo test --target wasm32-unknown-unknown
+
+rustup target add wasm32-wasip2
+cargo install workspace_runner
 cargo test --target wasm32-wasip2
 
 cargo test --lib cfgrammar --features serde

.cargo/config.toml

@@ -1,5 +1,5 @@
 [target.wasm32-wasip2]
-runner = "wasmtime run --dir ."
+runner = "workspace_runner --target wasm32-wasip2 --"
 
 [target.wasm32-unknown-unknown]
 # Provided by the crate wasm-bindgen-cli.

README.md

@@ -46,7 +46,7 @@ lexer can be found in `src/calc.l`:
 and where the definitions for the parser can be found in `src/calc.y`:
 
 ```rust
-%grmtools{yacckind Grmtools}
+%grmtools{yacckind: Grmtools}
 %start Expr
 %avoid_insert "INT"
 %%

cfgrammar/src/lib/yacc/parser.rs

@@ -455,6 +455,14 @@ impl YaccParser {
                     let (key_end_pos, key) = self.parse_name(i)?;
                     i = self.parse_ws(key_end_pos, false)?;
                     if key == "yacckind" {
+                        if let Some(j) = self.lookahead_is(":", i) {
+                            i = self.parse_ws(j, false)?;
+                        } else {
+                            return Err(YaccGrammarError {
+                                kind: YaccGrammarErrorKind::InvalidGrmtoolsHeaderKey,
+                                spans: vec![Span::new(i, i)],
+                            });
+                        }
                         let val_end_pos = self.parse_yacckind(i, update_yacc_kind)?;
                         if let Some(orig) = yacc_kind_key_span {
                             let dupe = Span::new(key_start_pos, key_end_pos);
@@ -466,6 +474,12 @@ impl YaccParser {
                             yacc_kind_key_span = Some(Span::new(key_start_pos, key_end_pos));
                             i = self.parse_ws(val_end_pos, true)?;
                         }
+                        if let Some(j) = self.lookahead_is(",", i) {
+                            i = self.parse_ws(j, true)?;
+                            continue;
+                        } else {
+                            break;
+                        }
                     } else {
                         return Err(YaccGrammarError {
                             kind: YaccGrammarErrorKind::InvalidGrmtoolsHeaderKey,
@@ -2832,20 +2846,20 @@ B";
     #[test]
     fn test_grmtools_section_yacckinds() {
         let srcs = [
-            "%grmtools{yacckind Original(NoAction)}
+            "%grmtools{yacckind: Original(NoAction)}
              %%
              Start: ;",
-            "%grmtools{yacckind YaccKind::Original(GenericParseTree)}
+            "%grmtools{yacckind: YaccKind::Original(GenericParseTree)}
              %%
              Start: ;",
-            "%grmtools{yacckind YaccKind::Original(yaccoriginalactionkind::useraction)}
+            "%grmtools{yacckind: YaccKind::Original(yaccoriginalactionkind::useraction)}
              %actiontype ()
              %%
              Start: ;",
-            "%grmtools{yacckind Original(YACCOriginalActionKind::NoAction)}
+            "%grmtools{yacckind: Original(YACCOriginalActionKind::NoAction)}
              %%
              Start: ;",
-            "%grmtools{yacckind YaccKind::Grmtools}
+            "%grmtools{yacckind: YaccKind::Grmtools}
              %%
              Start -> () : ;",
         ];
@@ -2856,6 +2870,35 @@ B";
         }
     }
 
+    #[test]
+    fn test_grmtools_section_commas() {
+        // We can't actually test much here, because
+        // We don't have a second value to test.
+        //
+        // `RecoveryKind` seemed like an option for an additional value to allow,
+        // but that is part of `lrpar` which cfgrammar doesn't depend upon.
+        let src = r#"
+            %grmtools{
+                yacckind: YaccKind::Grmtools,
+            }
+            %%
+            Start -> () : ;
+        "#;
+        YaccParser::new(YaccKindResolver::NoDefault, src.to_string())
+            .parse()
+            .unwrap();
+        let src = r#"
+            %grmtools{
+                yacckind: YaccKind::Grmtools
+            }
+            %%
+            Start -> () : ;
+        "#;
+        YaccParser::new(YaccKindResolver::NoDefault, src.to_string())
+            .parse()
+            .unwrap();
+    }
+
     #[test]
     fn test_empty_production_spans_issue_473() {
         let empty_prod_conflicts = [

doc/src/errorrecovery.md

@@ -49,7 +49,7 @@ make use of error recovery.
 A simple calculator grammar looks as follows:
 
 ```rust,noplaypen
-%grmtools{yacckind Grmtools}
+%grmtools{yacckind: Grmtools}
 %start Expr
 %%
 Expr -> u64:
@@ -182,7 +182,7 @@ We thus change the grammar so that inserted integers prevent evaluation from
 occurring:
 
 ```rust,noplaypen
-%grmtools{yacckind Grmtools}
+%grmtools{yacckind: Grmtools}
 %start Expr
 %%
 Expr -> Result<u64, ()>:

doc/src/lexextensions.md

@@ -15,9 +15,9 @@ other flags should specify their value immediately after the flag name.
 
 ```
 %grmtools {
-    allow_wholeline_comments
-    !octal
-    size_limit 1024
+    allow_wholeline_comments,
+    !octal,
+    size_limit: 1024,
 }
 %%
 . "rule"
@@ -56,7 +56,7 @@ The following flags can change the behavior to match posix lex more closely.
 
 ```
 %grmtools {
-    !dot_matches_new_line
+    !dot_matches_new_line,
     posix_escapes
 }
 %%

doc/src/quickstart.md

@@ -101,7 +101,7 @@ is lexed, but no lexemes are created from it.
 
 Our initial version of calc.y looks as follows:
 ```rust,noplaypen
-%grmtools{yacckind Grmtools}
+%grmtools{yacckind: Grmtools}
 %start Expr
 %%
 Expr -> Result<u64, ()>:

doc/src/yaccextensions.md

@@ -11,7 +11,7 @@ But a default can be set or forced by using a `YaccKindResolver`.
 ## Example
 
 ```
-%grmtools{yacckind Grmtools}
+%grmtools{yacckind: Grmtools}
 %%
 Start: ;
 ```

lrlex/examples/calc_manual_lex/src/calc.y

@@ -1,4 +1,4 @@
-%grmtools{yacckind Grmtools}
+%grmtools{yacckind: Grmtools}
 %start Expr
 %avoid_insert "INT"
 %expect-unused Unmatched "UNMATCHED"

lrlex/src/lib/parser.rs

@@ -304,6 +304,15 @@ where
                             });
                         }
                         i = self.parse_spaces(i)?;
+                        if let Some(j) = self.lookahead_is(":", i) {
+                            i = j
+                        } else {
+                            return Err(LexBuildError {
+                                kind: LexErrorKind::InvalidGrmtoolsSectionValue,
+                                spans: vec![Span::new(i, i)],
+                            });
+                        }
+                        i = self.parse_spaces(i)?;
                         let j = self.parse_digits(i)?;
                         // This checks that the digits are valid numbers, but currently just returns `None`
                         // when the values are actually out of range for that type. This could be improved.
@@ -351,11 +360,18 @@ where
                     &mut grmtools_section_span_map,
                     &mut grmtools_section_lex_flags,
                 )?;
-                if i == self.src.len() {
-                    return Err(self.mk_error(LexErrorKind::PrematureEnd, i));
+                if let Some(j) = self.lookahead_is(",", i) {
+                    i = self.parse_ws(j)?;
+                    continue;
+                } else {
+                    break;
                 }
             }
-            i = self.lookahead_is("}", i).unwrap();
+            if let Some(j) = self.lookahead_is("}", i) {
+                i = j
+            } else {
+                return Err(self.mk_error(LexErrorKind::PrematureEnd, i));
+            }
             i = self.parse_ws(i)?;
         }
         grmtools_section_lex_flags.merge_from(&self.force_lex_flags);
@@ -864,7 +880,9 @@ mod test {
                 .expect_err("Parsed ok while expecting error");
             assert_eq!(errs.len(), 1);
             let e = &errs[0];
-            assert!(e.kind.is_same_kind(&kind));
+            if !e.kind.is_same_kind(&kind) {
+                panic!("expected {:?}.is_same_kind({:?})", &e.kind, &kind);
+            }
             assert_eq!(
                 e.spans()
                     .iter()
@@ -1864,7 +1882,7 @@ b "A"
     fn test_grmtools_section_fails() {
         let src = r#"
 %grmtools {
-  !unicode
+  !unicode,
   unicode
 }
 %%
@@ -1879,8 +1897,8 @@ b "A"
 
         let src = r#"
 %grmtools {
-  size_limit 5
-  size_limit 6
+  size_limit: 5,
+  size_limit: 6,
 }
 %%
 . "dot";
@@ -1894,7 +1912,7 @@ b "A"
 
         let src = r#"
 %grmtools {
-  !size_limit 5
+  !size_limit: 5,
 }
 %%
 . "dot"
@@ -1906,6 +1924,22 @@ b "A"
             3,
             3,
         );
+        // The following is missing comma separators for more than 2 elements
+        // This is to avoid parsing it as "key value" number of elements.
+        // However we actually error after the first element since the parser
+        // knows "case_insensitive" is a flag with no arguments.
+        let src = r#"
+%grmtools {unicode, case_insensitive posix_escapes allow_comments}
+%%
+. "dot"
+\n+ ;
+"#;
+        LRNonStreamingLexerDef::<DefaultLexerTypes<u8>>::from_str(src).expect_error_at_line_col(
+            src,
+            LexErrorKind::PrematureEnd,
+            2,
+            38,
+        );
     }
 
     #[test]

lrpar/cttests/src/calc_nodefault_yacckind.test

@@ -1,6 +1,6 @@
 name: Test basic user actions using the calculator grammar
 grammar: |
-    %grmtools {yacckind Original(UserAction)}
+    %grmtools {yacckind: Original(UserAction)}
     %start Expr
     %actiontype Result<u64, ()>
     %avoid_insert 'INT'

lrpar/cttests/src/calc_wasm.test

@@ -1,6 +1,6 @@
 name: Test running on wasm targets
 grammar: |
-    %grmtools {yacckind Grmtools}
+    %grmtools {yacckind: Grmtools}
     %start Expr
     %avoid_insert "INT"
     %expect-unused Unmatched "UNMATCHED"

lrpar/cttests/src/cgen_helper.rs

@@ -79,10 +79,10 @@ pub(crate) fn run_test_path<P: AsRef<Path>>(path: P) -> Result<(), Box<dyn std::
         // Create grammar files
         let base = path.file_stem().unwrap().to_str().unwrap();
         let mut pg = PathBuf::from(&out_dir);
-        pg.push(format!("{}.y.rs", base));
+        pg.push(format!("{}.test.y", base));
         fs::write(&pg, grm).unwrap();
         let mut pl = PathBuf::from(&out_dir);
-        pl.push(format!("{}.l.rs", base));
+        pl.push(format!("{}.test.l", base));
         fs::write(&pl, lex).unwrap();
 
         // Build parser and lexer