Correctly handle EOF as a dedent

tgross35 · Jan 3, 2024 · bda81c9 · bda81c9
1 parent 13d96ff
commit bda81c9
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 20 deletions.
diff --git a/src/scanner.cc b/src/scanner.cc
@@ -82,23 +82,36 @@ void skip(TSLexer *lexer) {
   return lexer->advance(lexer, true);
 }
 
+// An EOF works as a dedent
+bool handle_eof(TSLexer *lexer, const bool *valid_symbols) {
+  assert(lexer->eof(lexer));
+  if (valid_symbols[DEDENT]) {
+      lexer->result_symbol = DEDENT;
+      return true;
+  }
+  return false;  
+}
+
 // This function is responsible for recognizing external tokens. It should
 // return true if a token was recognized, and false otherwise.
 bool tree_sitter_just_external_scanner_scan(void *payload, TSLexer *lexer,
                                             const bool *valid_symbols) {
-
-  if (!lexer->lookahead) {
-    lexer->mark_end(lexer);
-    return false;
-  }
-
   Scanner *state = static_cast<Scanner *>(payload);
   int32_t &lookahead = lexer->lookahead;
-  TSSymbol &result_symbol = lexer->result_symbol;
+  bool (*eof)(const TSLexer *) = lexer->eof;
   void (*mark_end)(TSLexer *) = lexer->mark_end;
   bool (*is_at_included_range_start)(const TSLexer *) =
       lexer->is_at_included_range_start;
 
+  if (eof(lexer)) {
+      return handle_eof(lexer, valid_symbols);
+  }
+
+  if (!lexer->lookahead) {
+    lexer->mark_end(lexer);
+    return false;
+  }
+
   // Handle backslash escaping for newlines
   if (valid_symbols[NEWLINE]) {
     bool escape = false;
@@ -112,8 +125,9 @@ bool tree_sitter_just_external_scanner_scan(void *payload, TSLexer *lexer,
       eol = true;
       skip(lexer);
     }
+
     if (eol && !escape) {
-      result_symbol = NEWLINE;
+      lexer->result_symbol = NEWLINE;
       return true;
     }
   }
@@ -130,22 +144,20 @@ bool tree_sitter_just_external_scanner_scan(void *payload, TSLexer *lexer,
         break;
       }
 
-      // if (lexer->eof(lexer)) {
-      //   if (valid_symbols[DEDENT]) {
-      //     return true;
-      //   }
-      // }
+      if (lexer->eof(lexer)) {
+        return handle_eof(lexer, valid_symbols);
+      }
     }
 
     uint32_t indent = lexer->get_column(lexer);
     if (indent > state->prev_indent && valid_symbols[INDENT] &&
         state->prev_indent == 0) {
-      result_symbol = INDENT;
+      lexer->result_symbol = INDENT;
       state->prev_indent = indent;
       return true;
     } else if (indent < state->prev_indent && valid_symbols[DEDENT] &&
                indent == 0) {
-      result_symbol = DEDENT;
+      lexer->result_symbol = DEDENT;
       state->prev_indent = indent;
       return true;
     }

diff --git a/test/corpus/recipes.txt b/test/corpus/recipes.txt
@@ -39,7 +39,7 @@ bar:
             (recipeline
               (notcomment)
               (NEWLINE))))
-        (MISSING DEDENT)))))
+        (DEDENT)))))
 
 ================================================================================
 quiet
@@ -66,7 +66,7 @@ quiet
             (recipeline
               (notcomment)
               (NEWLINE))))
-        (MISSING DEDENT)))))
+        (DEDENT)))))
 
 ================================================================================
 dependencies
@@ -98,7 +98,7 @@ foo: bar baz
             (recipeline
               (notcomment)
               (NEWLINE))))
-        (MISSING DEDENT)))))
+        (DEDENT)))))
 
 ================================================================================
 arguments
@@ -150,7 +150,7 @@ baz qux *quux:
             (recipeline
               (notcomment)
               (NEWLINE))))
-        (MISSING DEDENT)))))
+        (DEDENT)))))
 
 ================================================================================
 dependency expression
@@ -189,7 +189,7 @@ foo +bar: baz(bar)
             (recipeline
               (notcomment)
               (NEWLINE))))
-        (MISSING DEDENT)))))
+        (DEDENT)))))
 
 ================================================================================
 attribute FIXME(attributes): XFAIL no attribute support