Create a new string scanner

This scanner is inspired by Julia's tree-sitter and gives us a much more exact way to extract strings, raw strings, and external commands. Also rename `string` to `string_literal`, and split off `raw_string_literal`, and add checks with Wall and Werror in CI.
tgross35 · Jan 7, 2024 · 459ada4 · 459ada4
1 parent 39963d7
commit 459ada4
Show file tree

Hide file tree

Showing 14 changed files with 5,173 additions and 5,002 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -42,6 +42,15 @@ jobs:
             echo '::warning::run `npm run gen` and commit the changes'
           fi
       - run: npm test
+      - name: Check C files with args
+        shell: bash
+        run: >
+          find src/ -name '*.c' ! -name "parser.c" |
+          xargs -IFNAME sh -c
+          'echo "\nCHECKING FILE FNAME" &&
+          clang FNAME -c -Wall -Werror --pedantic 
+          -Wno-format-pedantic
+          -o/dev/null'
 
   static-validation:
     runs-on: ubuntu-latest
@@ -59,4 +68,4 @@ jobs:
             just --list --unstable --justfile "$fname"
           done
       - name: Look for tests that contain errors
-        run: "! grep -r -E '(ERROR|MISSING|UNEXPECTED)' test"
+        run: "! grep -nr -C4 -E '(ERROR|MISSING|UNEXPECTED)' test"
diff --git a/grammar.js b/grammar.js
@@ -1,5 +1,7 @@
 // Main grammar for justfiles
 
+const ESCAPE_SEQUENCE = token(/\\[nrt"\\]/);
+
 // Comma separated list with at least one item
 function comma_sep1(item) {
   return seq(item, repeat(seq(",", item)));
@@ -23,13 +25,23 @@ function array(item) {
 
 module.exports = grammar({
   name: "just",
-  externals: ($) => [$._indent, $._dedent, $._newline],
+  externals: (
+    $,
+  ) => [
+    $._indent,
+    $._dedent,
+    $._newline,
+    $._string_start,
+    $._string_end,
+    $._string_body,
+    $._raw_string_start,
+    $._raw_string_end,
+    $._command_start,
+    $._command_end,
+  ],
   inline: (
     $,
   ) => [
-    $._string,
-    $._string_indented,
-    $._raw_string_indented,
     $._expression_braced,
     $._expression_recurse,
   ],
@@ -85,15 +97,15 @@ module.exports = grammar({
     export: ($) => seq("export", $.assignment),
 
     // import        : 'import' '?'? string?
-    import: ($) => seq("import", optional("?"), $.string),
+    import: ($) => seq("import", optional("?"), $._string),
 
     // module        : 'mod' '?'? string?
     module: ($) =>
       seq(
         "mod",
         optional("?"),
         field("mod_name", $.identifier),
-        optional($.string),
+        optional($._string),
       ),
 
     // setting       : 'set' 'dotenv-load' boolean?
@@ -108,7 +120,7 @@ module.exports = grammar({
           field(
             "right",
             optional(
-              seq(":=", choice($.boolean, $.string, array($.string))),
+              seq(":=", choice($.boolean, $._string, array($._string))),
             ),
           ),
           $.eol,
@@ -119,7 +131,7 @@ module.exports = grammar({
           ":=",
           field(
             "right",
-            array($.string),
+            array($._string),
           ),
           $.eol,
         ),
@@ -181,7 +193,7 @@ module.exports = grammar({
           $.function_call,
           $.external_command,
           $.identifier,
-          $.string,
+          $._string,
           seq("(", $.expression, ")"),
         ),
       ),
@@ -194,9 +206,6 @@ module.exports = grammar({
         ")",
       ),
 
-    external_command: ($) =>
-      choice(seq($._backticked), seq($._indented_backticked)),
-
     // sequence      : expression ',' sequence
     //               | expression ','?
     sequence: ($) => comma_sep1($.expression),
@@ -288,9 +297,6 @@ module.exports = grammar({
     // `# ...` comment
     comment: ($) => seq(/#.*/, $._newline),
 
-    // notinterpolation: ($) => /[^{][^{]\S*/,
-    notinterpolation: (_) => /[^\s{][^\s{]\S*/,
-
     // interpolation : '{{' expression '}}'
     interpolation: ($) => seq("{{", $.expression, "}}"),
 
@@ -300,23 +306,28 @@ module.exports = grammar({
     //               | INDENTED_STRING
     //               | RAW_STRING
     //               | INDENTED_RAW_STRING
-    string: ($) =>
-      choice(
-        $._string_indented,
-        $._raw_string_indented,
-        $._string,
-        // _raw_string, can't be written as a separate inline for osm reason
-        /'[^']*'/,
+    _string: ($) => choice($.raw_string_literal, $.string_literal),
+
+    string_literal: ($) =>
+      seq(
+        $._string_start,
+        repeat(choice($._string_body, $.escape_sequence)),
+        $._string_end,
+      ),
+
+    raw_string_literal: ($) =>
+      seq($._raw_string_start, optional($._string_body), $._raw_string_end),
+
+    external_command: ($) =>
+      seq(
+        $._command_start,
+        repeat(choice(prec(1, $.interpolation), $.command_body)),
+        $._command_end,
       ),
 
-    _raw_string_indented: (_) => seq("'''", repeat(/./), "'''"),
-    _string: ($) => seq('"', repeat(choice($.string_escape, /[^\\"]+/)), '"'),
-    _string_indented: ($) =>
-      seq('"""', repeat(choice($.string_escape, /[^\\"]+/)), '"""'),
-    string_escape: (_) => /\\[nrt"\\]/,
+    command_body: ($) => $._string_body,
 
-    _backticked: (_) => seq("`", repeat(/./), "`"),
-    _indented_backticked: (_) => seq("```", repeat(/./), "```"),
+    escape_sequence: (_) => ESCAPE_SEQUENCE,
 
     text: (_) => /.+/, //recipe TEXT, only matches in a recipe body
     // text: (_) => /\S+/, //recipe TEXT, only matches in a recipe body

diff --git a/queries-src/highlights.scm b/queries-src/highlights.scm
@@ -71,8 +71,11 @@
 (dependency_expression recipe: (identifier) @function)
 
 ; handle escape sequences
-(string (string_escape) @constant.character.escape)
-(string) @string
+(string_literal (escape_sequence) @constant.character.escape)
+[
+  (string_literal)
+  (raw_string_literal)
+] @string
 
 (comment) @comment.line
 
@@ -97,6 +100,6 @@
 
 ["," ":"] @punctuation.delimiter
 
-"`" @punctuation.special
+; "`" @punctuation.special
 
 (ERROR) @error
diff --git a/queries-src/indents.scm b/queries-src/indents.scm
@@ -3,5 +3,6 @@
 ; Better documentation is in https://docs.helix-editor.com/guides/indent.html
 
 (recipe) @indent @extend
-(string) @indent @extend
+(string_literal) @indent @extend
+(raw_string_literal) @indent @extend
 (external_command) @indent @extend
diff --git a/queries/just/highlights.scm b/queries/just/highlights.scm
@@ -73,8 +73,11 @@
 (dependency_expression recipe: (identifier) @function)
 
 ; handle escape sequences
-(string (string_escape) @string.escape)
-(string) @string
+(string_literal (escape_sequence) @string.escape)
+[
+  (string_literal)
+  (raw_string_literal)
+] @string
 
 (comment) @comment
 
@@ -99,6 +102,6 @@
 
 ["," ":"] @punctuation.delimiter
 
-"`" @punctuation.special
+; "`" @punctuation.special
 
 (ERROR) @error
diff --git a/queries/just/indents.scm b/queries/just/indents.scm
@@ -5,5 +5,6 @@
 ; Better documentation is in https://docs.helix-editor.com/guides/indent.html
 
 (recipe) @indent @extend
-(string) @indent @extend
+(string_literal) @indent @extend
+(raw_string_literal) @indent @extend
 (external_command) @indent @extend