Try parsing files from the Just repository as part of our tests

justfile

@@ -1,31 +1,46 @@
 src := justfile_directory() / "src"
 bindings := justfile_directory() / "bindings"
-ts_src := justfile_directory() / "repositories" / "tree-sitter"
 fuzzer := justfile_directory() / "fuzzer"
-include_args := "-Isrc/ -I" + ts_src + "/lib/include -Inode_modules/nan"
+target := justfile_directory() / "target"
+bin_dir := target / "bin"
+obj_dir := target / "obj"
+debug_out := bin_dir / "debug.out"
+fuzz_out := bin_dir / "fuzz.out"
+
+ts_path := justfile_directory() / "repositories" / "tree-sitter"
+ts_repo := "https://github.com/tree-sitter/tree-sitter"
+ts_sha := "1c55abb5308fe3891da545662e5df7ba28ade275" # v0.21.0
+
+just_path := justfile_directory() / "repositories" / "just"
+just_repo := "https://github.com/casey/just.git"
+just_sha := "a2ff42e6c37ba5c429d444f3a18d3633e59f9a34" # 1.24.0
+
+include_args := "-Isrc/ -I" + ts_path + "/lib/include -Inode_modules/nan"
 general_cflags := "-Wall -Werror --pedantic -Wno-format-pedantic"
 
 # FIXME: there are errors running with ASAN, we ideally want `,address` here
 fuzzer_flags := env("FUZZER_FLAGS", "-fsanitize=fuzzer,undefined")
 fuzz_time := env("FUZZ_TOTAL_TIME", "1200")
 
 # Source files needed to build a parser
-parser_sources := src + "/scanner.c " + src + "/parser.c " + ts_src + "/lib/src/lib.c"
-
-target := justfile_directory() / "target"
-bin_dir := target / "bin"
-obj_dir := target / "obj"
-debug_out := bin_dir / "debug.out"
-fuzz_out := bin_dir / "fuzz.out"
-
-ts_tag := "v0.20.9"
+parser_sources := src + "/scanner.c " + src + "/parser.c " + ts_path + "/lib/src/lib.c"
 
 base_cache_key := sha256_file(src / "scanner.c") + sha256_file(src / "parser.c") + sha256(parser_sources) + sha256(include_args) + sha256(general_cflags) + sha256(fuzzer_flags)
 
+verbose_flag := if env("CI", "") == "1" { "--verbose" } else { "" }
+
 # `timeout` is not available on all platforms, but perl often is. This needs a
 # bash shell.
-make_timeout_fn := '''function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; }'''
-verbose_flag := if env("CI", "") == "1" { "--verbose" } else { "" }
+make_timeout_fn := '''timeout () { perl -e 'alarm shift; exec @ARGV' "$@"; }'''
+
+# Files that should parse with errors but not crash
+errors_expected := '''
+	readme.just
+	# FIXME: we want this to parse
+	test.just
+	test/timeout-1aa6bf37e914715f4aa49e6cf693f7abf81aaf8e
+'''
+
 
 # List all recipes
 default:
@@ -60,7 +75,7 @@ setup *npm-args:
 	fi
 
 # Lint with more minimal dependencies that can be run during pre-commit
-_lint-min: tree-sitter-clone configure-compile-database
+_lint-min: _clone-repo-tree-sitter configure-compile-database
 	npm run lint:check
 	git ls-files '**.c' | grep -v 'parser\.c' | \
 		xargs -IFNAME sh -c 'echo "\nchecking file FNAME" && clang-tidy FNAME'
@@ -115,13 +130,14 @@ test *ts-test-args: gen
 
 	echo '\nRunning Cargo tests'
 
-	# FIXME: xfail CI on Windows because we are getting STATUS_DLL_NOT_FOUND
+	# FIXME: xfail Rust CI on Windows because we are getting STATUS_DLL_NOT_FOUND
 	{{ if os_family() + env("CI", "1") == "windows1" { "echo skipping tests on Windows" } else { "cargo test" } }}
 
 # Verify that tree-sitter can parse and highlight all files in the repo. Requires a tree-sitter configuration.
-test-parse-highlight:
+test-parse-highlight: _clone-repo-just
 	#!/bin/bash
-	set -eau
+	set -eauo pipefail
+
 	{{ if env("CI", "") == "1" { "set -x && echo running in CI" } else { "" } }}
 
 	{{ make_timeout_fn }}
@@ -130,6 +146,12 @@ test-parse-highlight:
 
 	# skip readme.just because it is broken but works for testing, and skip files
 	# from the fuzzer
+	base_files=$(git ls-files '*.just' '*justfile')
+	just_repo_files=$(git -C '{{ just_path }}' '*.just' '*justfile' |
+		awk '{print "{{ just_path }}" $0}')
+	error_files=
+
+
 	git ls-files '**.just' 'justfile' |
 		grep -v readme.just |
 		grep -v test.just |
@@ -153,7 +175,7 @@ test-parse-highlight:
 		exitcode=0
 		# Run with a timeout
 		timeout 10 npx tree-sitter parse --scope=source.just "$fname" > /dev/null ||
-			exitstat=$?
+			exitcode=$?
 
 		# Failed highlighting returns exit code 1, syntax errors are 2. We are
 		# looking for SIGBUS/SIGSEGV/SIGALRM (timeout), 135/138/142
@@ -255,7 +277,7 @@ pre-commit: _lint-min format-check
 # Install pre-commit hooks
 pre-commit-install:
 	#!/bin/sh
-	fname="{{justfile_directory()}}/.git/hooks/pre-commit"
+	fname="{{ justfile_directory() }}/.git/hooks/pre-commit"
 	touch "$fname"
 	chmod +x "$fname"
 
@@ -264,18 +286,31 @@ pre-commit-install:
 	just pre-commit
 	EOF
 
-# Download and build upstream tree-sitter
-tree-sitter-clone:
+# Clone or update a repo
+_clone-repo url path sha:
 	#!/bin/sh
 	set -eaux
 
-	if [ ! -d "{{ ts_src }}" ]; then
-		git clone https://github.com/tree-sitter/tree-sitter "{{ ts_src }}" \
-			--branch {{ ts_tag }} --depth=1
+	if [ ! -d '{{ path }}' ]; then
+		echo "Cloning {{ url }}"
+		git clone '{{ url }}' '{{ path }}' --depth=100
 	fi
 
+	actual_sha=$(git -C '{{ path }}' rev-parse HEAD)
+	if [ "$actual_sha" != "{{ sha }}" ]; then
+		echo "Updating {{ url }} to {{ sha }}"
+		git -C '{{ path }}' fetch
+		git -C '{{ path }}' reset --hard '{{ sha }}'
+	fi
+
+# Clone the tree-sitter repo
+_clone-repo-tree-sitter: (_clone-repo ts_repo ts_path ts_sha)
+
+# Clone the just repo
+_clone-repo-just: (_clone-repo just_repo just_path just_sha)
+
 # Build a simple debug executable
-debug-build: tree-sitter-clone _out-dirs
+debug-build: _clone-repo-tree-sitter _out-dirs
 	#!/bin/sh
 	set -eau
 
@@ -294,7 +329,7 @@ debug-run *file-names: debug-build
 	{{ debug_out }} {{file-names}}
 
 # Build and run the fuzzer
-fuzz *extra-args: (gen "--debug-build") tree-sitter-clone _out-dirs
+fuzz *extra-args: (gen "--debug-build") _clone-repo-tree-sitter _out-dirs
 	#!/bin/sh
 	set -eaux
 
@@ -321,7 +356,7 @@ fuzz *extra-args: (gen "--debug-build") tree-sitter-clone _out-dirs
 	fuzzer_flags="-artifact_prefix=$artifacts -timeout=20 -max_total_time={{ fuzz_time }} -jobs={{ num_cpus() }}"
 
 	echo "Starting fuzzing at $(date -u -Is)"
-	LD_LIBRARY_PATH="{{ts_src}}" "{{ fuzz_out }}" "$corpus" $fuzzer_flags {{ extra-args }}
+	LD_LIBRARY_PATH="{{ ts_path }}" "{{ fuzz_out }}" "$corpus" $fuzzer_flags {{ extra-args }}
 
 # Configure the database used by clang-format, clang-tidy, and language servers
 configure-compile-database: