diff --git a/justfile b/justfile index f5f2a23..7fa2426 100644 --- a/justfile +++ b/justfile @@ -1,8 +1,21 @@ src := justfile_directory() / "src" bindings := justfile_directory() / "bindings" -ts_src := justfile_directory() / "repositories" / "tree-sitter" fuzzer := justfile_directory() / "fuzzer" -include_args := "-Isrc/ -I" + ts_src + "/lib/include -Inode_modules/nan" +target := justfile_directory() / "target" +bin_dir := target / "bin" +obj_dir := target / "obj" +debug_out := bin_dir / "debug.out" +fuzz_out := bin_dir / "fuzz.out" + +ts_path := justfile_directory() / "repositories" / "tree-sitter" +ts_repo := "https://github.com/tree-sitter/tree-sitter" +ts_sha := "1c55abb5308fe3891da545662e5df7ba28ade275" # v0.21.0 + +just_path := justfile_directory() / "repositories" / "just" +just_repo := "https://github.com/casey/just.git" +just_sha := "a2ff42e6c37ba5c429d444f3a18d3633e59f9a34" # 1.24.0 + +include_args := "-Isrc/ -I" + ts_path + "/lib/include -Inode_modules/nan" general_cflags := "-Wall -Werror --pedantic -Wno-format-pedantic" # FIXME: there are errors running with ASAN, we ideally want `,address` here @@ -10,22 +23,24 @@ fuzzer_flags := env("FUZZER_FLAGS", "-fsanitize=fuzzer,undefined") fuzz_time := env("FUZZ_TOTAL_TIME", "1200") # Source files needed to build a parser -parser_sources := src + "/scanner.c " + src + "/parser.c " + ts_src + "/lib/src/lib.c" - -target := justfile_directory() / "target" -bin_dir := target / "bin" -obj_dir := target / "obj" -debug_out := bin_dir / "debug.out" -fuzz_out := bin_dir / "fuzz.out" - -ts_tag := "v0.20.9" +parser_sources := src + "/scanner.c " + src + "/parser.c " + ts_path + "/lib/src/lib.c" base_cache_key := sha256_file(src / "scanner.c") + sha256_file(src / "parser.c") + sha256(parser_sources) + sha256(include_args) + sha256(general_cflags) + sha256(fuzzer_flags) +verbose_flag := if env("CI", "") == "1" { "--verbose" } else { "" } + # `timeout` is not available on all platforms, but perl often is. This needs a # bash shell. -make_timeout_fn := '''function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; }''' -verbose_flag := if env("CI", "") == "1" { "--verbose" } else { "" } +make_timeout_fn := '''timeout () { perl -e 'alarm shift; exec @ARGV' "$@"; }''' + +# Files that should parse with errors but not crash +errors_expected := ''' + readme.just + # FIXME: we want this to parse + test.just + test/timeout-1aa6bf37e914715f4aa49e6cf693f7abf81aaf8e +''' + # List all recipes default: @@ -60,7 +75,7 @@ setup *npm-args: fi # Lint with more minimal dependencies that can be run during pre-commit -_lint-min: tree-sitter-clone configure-compile-database +_lint-min: _clone-repo-tree-sitter configure-compile-database npm run lint:check git ls-files '**.c' | grep -v 'parser\.c' | \ xargs -IFNAME sh -c 'echo "\nchecking file FNAME" && clang-tidy FNAME' @@ -115,13 +130,14 @@ test *ts-test-args: gen echo '\nRunning Cargo tests' - # FIXME: xfail CI on Windows because we are getting STATUS_DLL_NOT_FOUND + # FIXME: xfail Rust CI on Windows because we are getting STATUS_DLL_NOT_FOUND {{ if os_family() + env("CI", "1") == "windows1" { "echo skipping tests on Windows" } else { "cargo test" } }} # Verify that tree-sitter can parse and highlight all files in the repo. Requires a tree-sitter configuration. -test-parse-highlight: +test-parse-highlight: _clone-repo-just #!/bin/bash - set -eau + set -eauo pipefail + {{ if env("CI", "") == "1" { "set -x && echo running in CI" } else { "" } }} {{ make_timeout_fn }} @@ -130,6 +146,12 @@ test-parse-highlight: # skip readme.just because it is broken but works for testing, and skip files # from the fuzzer + base_files=$(git ls-files '*.just' '*justfile') + just_repo_files=$(git -C '{{ just_path }}' '*.just' '*justfile' | + awk '{print "{{ just_path }}" $0}') + error_files= + + git ls-files '**.just' 'justfile' | grep -v readme.just | grep -v test.just | @@ -153,7 +175,7 @@ test-parse-highlight: exitcode=0 # Run with a timeout timeout 10 npx tree-sitter parse --scope=source.just "$fname" > /dev/null || - exitstat=$? + exitcode=$? # Failed highlighting returns exit code 1, syntax errors are 2. We are # looking for SIGBUS/SIGSEGV/SIGALRM (timeout), 135/138/142 @@ -255,7 +277,7 @@ pre-commit: _lint-min format-check # Install pre-commit hooks pre-commit-install: #!/bin/sh - fname="{{justfile_directory()}}/.git/hooks/pre-commit" + fname="{{ justfile_directory() }}/.git/hooks/pre-commit" touch "$fname" chmod +x "$fname" @@ -264,18 +286,31 @@ pre-commit-install: just pre-commit EOF -# Download and build upstream tree-sitter -tree-sitter-clone: +# Clone or update a repo +_clone-repo url path sha: #!/bin/sh set -eaux - if [ ! -d "{{ ts_src }}" ]; then - git clone https://github.com/tree-sitter/tree-sitter "{{ ts_src }}" \ - --branch {{ ts_tag }} --depth=1 + if [ ! -d '{{ path }}' ]; then + echo "Cloning {{ url }}" + git clone '{{ url }}' '{{ path }}' --depth=100 fi + actual_sha=$(git -C '{{ path }}' rev-parse HEAD) + if [ "$actual_sha" != "{{ sha }}" ]; then + echo "Updating {{ url }} to {{ sha }}" + git -C '{{ path }}' fetch + git -C '{{ path }}' reset --hard '{{ sha }}' + fi + +# Clone the tree-sitter repo +_clone-repo-tree-sitter: (_clone-repo ts_repo ts_path ts_sha) + +# Clone the just repo +_clone-repo-just: (_clone-repo just_repo just_path just_sha) + # Build a simple debug executable -debug-build: tree-sitter-clone _out-dirs +debug-build: _clone-repo-tree-sitter _out-dirs #!/bin/sh set -eau @@ -294,7 +329,7 @@ debug-run *file-names: debug-build {{ debug_out }} {{file-names}} # Build and run the fuzzer -fuzz *extra-args: (gen "--debug-build") tree-sitter-clone _out-dirs +fuzz *extra-args: (gen "--debug-build") _clone-repo-tree-sitter _out-dirs #!/bin/sh set -eaux @@ -321,7 +356,7 @@ fuzz *extra-args: (gen "--debug-build") tree-sitter-clone _out-dirs fuzzer_flags="-artifact_prefix=$artifacts -timeout=20 -max_total_time={{ fuzz_time }} -jobs={{ num_cpus() }}" echo "Starting fuzzing at $(date -u -Is)" - LD_LIBRARY_PATH="{{ts_src}}" "{{ fuzz_out }}" "$corpus" $fuzzer_flags {{ extra-args }} + LD_LIBRARY_PATH="{{ ts_path }}" "{{ fuzz_out }}" "$corpus" $fuzzer_flags {{ extra-args }} # Configure the database used by clang-format, clang-tidy, and language servers configure-compile-database: