fuzzer tests

.gitattributes

@@ -0,0 +1,2 @@
+# These test files are from the fuzzer
+test/highlight/timeout-* binary

.gitignore

@@ -3,3 +3,4 @@ node_modules
 build
 log.html
 *.wasm
+fuzzer-out/

README.md

@@ -85,6 +85,13 @@ foo := if true {
 }
 ```
 
+## Test Information
+
+The tests directory contains "corpus" tests that are checked for syntax, as
+well as "highlight" tests that check the result. The "highlight" test directory
+includes some test files generated by the fuzzer that aren't always human
+readable.
+
 ## TODO
 
 - [x] Implement a basic parser that is able to understand all features of Justfiles

justfile

@@ -1,12 +1,3 @@
-debug_env := env("DEBUG", "default")
-debug_arg := if debug_env == "1" {
-	"--debug-build"
-} else if debug_env == "default" {
-	""
-} else {
-	error("unrecognized DEBUG variable " + debug_env)
-}
-
 # List all recipes
 default:
 	just --list
@@ -23,7 +14,18 @@ format:
 
 # Generate the parser
 gen:
-	npx tree-sitter generate {{ debug_arg }}
+	#!/bin/sh
+	debug={{ env("DEBUG", "0") }}
+	if [ "$debug" = "1" ]; then
+		debug_arg="--debug-build"
+	elif [ "$debug" = "0" ]; then
+		debug_arg=""
+	else
+		echo "invalid debug argument $debug_arg"
+		exit 1
+	fi
+
+	npx tree-sitter generate "$debug_arg"
 	python3 build-flavored-queries.py
 
 alias t := test-ts
@@ -65,6 +67,56 @@ check-c:
 		-Wno-format-pedantic \
 		-o/dev/null'
 
+fuzz max-total-time="1200":
+	#!/bin/sh
+	set -eaux
+
+	out="fuzzer-out"
+	ts_source="$out/tree-sitter"
+
+	flags="-fsanitize=fuzzer,address,undefined"
+	flags="$flags -g -O1"
+	flags="$flags -Isrc/ -I$ts_source/lib/include"
+	flags="$flags -o $out/fuzzer"
+
+	mkdir -p "$out"
+
+	[ ! -d "$ts_source" ] &&
+		git clone https://github.com/tree-sitter/tree-sitter "$ts_source" \
+		--depth=1
+
+	make -C "$ts_source"
+
+	cat << EOF | clang $flags "src/scanner.c" "src/parser.c" "$ts_source/libtree-sitter.a" -x c -
+	#include <stdio.h>
+	#include <stdlib.h>
+	#include "tree_sitter/api.h"
+
+	TSLanguage *tree_sitter_just();
+
+	int LLVMFuzzerTestOneInput(const uint8_t *data, const size_t len) {
+	  TSParser *parser = ts_parser_new();
+	  ts_parser_set_language(parser, tree_sitter_just());
+
+	  // Build a syntax tree based on source code stored in a string.
+	  TSTree *tree = ts_parser_parse_string(
+	    parser,
+	    NULL,
+	    (const char *)data,
+	    len
+	  );
+	  // Free all of the heap-allocated memory.
+	  ts_tree_delete(tree);
+	  ts_parser_delete(parser);
+	  return 0;
+	}
+	EOF
+
+	fuzzer_flags="-artifact_prefix=$out/ -timeout=20 -max_total_time={{ max-total-time }}"
+	./fuzzer-out/fuzzer $fuzzer-flags
+
+
+
 # Verify that the `just` tool parses all files we are using
 verify-just-parsing:
 	#!/bin/sh

src/scanner.c

@@ -460,6 +460,7 @@ bool tree_sitter_just_external_scanner_scan(void *payload, TSLexer *lexer,
         skip(lexer);
         break;
       }
+      skip(lexer);
     }
 
     if (eof(lexer)) {

test/corpus/newlines.txt

@@ -0,0 +1,17 @@
+================================================================================
+trailing whitespace
+================================================================================
+
+a: #
+--------------------------------------------------------------------------------
+
+(source_file
+  (item
+    (eol))
+  (item
+    (recipe
+      (recipe_header
+        (identifier))
+      (eol
+        (comment)))))
+