From c2a3d16768133985ab5a5151d1432073b3dcdbf4 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Fri, 23 Feb 2024 12:56:53 -0500 Subject: [PATCH] Try parsing files from the Just repository as part of our tests --- .github/workflows/ci.yaml | 11 - justfile | 213 +++++++++++------- test/highlight/recipes.just | 12 +- ...bf37e914715f4aa49e6cf693f7abf81aaf8e.just} | Bin 4 files changed, 134 insertions(+), 102 deletions(-) rename test/{timeout-1aa6bf37e914715f4aa49e6cf693f7abf81aaf8e => timeout-1aa6bf37e914715f4aa49e6cf693f7abf81aaf8e.just} (100%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a02f661..a34d8b6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -65,14 +65,3 @@ jobs: run: just --verbose test - name: Check if generated files are up to date (warn only) run: just --verbose ci-validate-generated-files 0 - - static-validation: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@v4 - - uses: taiki-e/install-action@just - - name: Verify Just can parse test files - run: just --verbose verify-just-parsing - - name: Look for tests that contain errors - run: just --verbose verify-no-error-tests diff --git a/justfile b/justfile index f5f2a23..1134824 100644 --- a/justfile +++ b/justfile @@ -1,8 +1,21 @@ src := justfile_directory() / "src" bindings := justfile_directory() / "bindings" -ts_src := justfile_directory() / "repositories" / "tree-sitter" fuzzer := justfile_directory() / "fuzzer" -include_args := "-Isrc/ -I" + ts_src + "/lib/include -Inode_modules/nan" +target := justfile_directory() / "target" +bin_dir := target / "bin" +obj_dir := target / "obj" +debug_out := bin_dir / "debug.out" +fuzz_out := bin_dir / "fuzz.out" + +ts_path := justfile_directory() / "repositories" / "tree-sitter" +ts_repo := "https://github.com/tree-sitter/tree-sitter" +ts_sha := "1c55abb5308fe3891da545662e5df7ba28ade275" # v0.21.0 + +just_path := justfile_directory() / "repositories" / "just" +just_repo := "https://github.com/casey/just.git" +just_sha := "a2ff42e6c37ba5c429d444f3a18d3633e59f9a34" # 1.24.0 + +include_args := "-Isrc/ -I" + ts_path + "/lib/include -Inode_modules/nan" general_cflags := "-Wall -Werror --pedantic -Wno-format-pedantic" # FIXME: there are errors running with ASAN, we ideally want `,address` here @@ -10,22 +23,30 @@ fuzzer_flags := env("FUZZER_FLAGS", "-fsanitize=fuzzer,undefined") fuzz_time := env("FUZZ_TOTAL_TIME", "1200") # Source files needed to build a parser -parser_sources := src + "/scanner.c " + src + "/parser.c " + ts_src + "/lib/src/lib.c" - -target := justfile_directory() / "target" -bin_dir := target / "bin" -obj_dir := target / "obj" -debug_out := bin_dir / "debug.out" -fuzz_out := bin_dir / "fuzz.out" - -ts_tag := "v0.20.9" +parser_sources := src + "/scanner.c " + src + "/parser.c " + ts_path + "/lib/src/lib.c" base_cache_key := sha256_file(src / "scanner.c") + sha256_file(src / "parser.c") + sha256(parser_sources) + sha256(include_args) + sha256(general_cflags) + sha256(fuzzer_flags) +verbose_flag := if env("CI", "") == "1" { "--verbose" } else { "" } + # `timeout` is not available on all platforms, but perl often is. This needs a # bash shell. -make_timeout_fn := '''function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; }''' -verbose_flag := if env("CI", "") == "1" { "--verbose" } else { "" } +make_timeout_fn := '''timeout () { perl -e 'alarm shift; exec @ARGV' "$@"; }''' + +# Files that should parse with errors but not crash +errors_expected := ''' + test/timeout-1aa6bf37e914715f4aa49e6cf693f7abf81aaf8e.just + + # FIXME: xfail files, these should parse correctly + test/test.just + repositories/just/examples/kitchen-sink.just + repositories/just/examples/powershell.just +''' + +# Files used for testing that Just might not understand +no_just_parsing := ''' + test/readme.just +''' # List all recipes default: @@ -60,7 +81,7 @@ setup *npm-args: fi # Lint with more minimal dependencies that can be run during pre-commit -_lint-min: tree-sitter-clone configure-compile-database +_lint-min: _clone-repo-tree-sitter configure-compile-database npm run lint:check git ls-files '**.c' | grep -v 'parser\.c' | \ xargs -IFNAME sh -c 'echo "\nchecking file FNAME" && clang-tidy FNAME' @@ -112,82 +133,91 @@ alias t := test test *ts-test-args: gen npx tree-sitter test {{ ts-test-args }} just {{ verbose_flag }} test-parse-highlight + just {{ verbose_flag }} verify-no-error-tests echo '\nRunning Cargo tests' - # FIXME: xfail CI on Windows because we are getting STATUS_DLL_NOT_FOUND + # FIXME: xfail Rust CI on Windows because we are getting STATUS_DLL_NOT_FOUND {{ if os_family() + env("CI", "1") == "windows1" { "echo skipping tests on Windows" } else { "cargo test" } }} # Verify that tree-sitter can parse and highlight all files in the repo. Requires a tree-sitter configuration. -test-parse-highlight: - #!/bin/bash - set -eau - {{ if env("CI", "") == "1" { "set -x && echo running in CI" } else { "" } }} +test-parse-highlight: _clone-repo-just + #!/usr/bin/env python3 + import re + import subprocess as sp - {{ make_timeout_fn }} + repo_files = sp.check_output( + ["git", "ls-files", "*.just", "*justfile", "*timeout", "*crash"], encoding="utf8", + ) + just_repo_files = sp.check_output( + ["git", "-C", "{{ just_path }}", "ls-files", "*.just", "*justfile"], + encoding="utf8", + ) - echo Checking all files in the repo... + files = set() + error_files = set() + skip_just = [] - # skip readme.just because it is broken but works for testing, and skip files - # from the fuzzer - git ls-files '**.just' 'justfile' | - grep -v readme.just | - grep -v test.just | - grep -vE '(timeout|crash)-.*' | - while read -r fname - do - echo "::group::Parse and highlight testing for $fname" + for line in repo_files.splitlines(): + files.add(f"{{ justfile_directory() }}/{line}") - timeout 10 npx tree-sitter parse "$fname" > /dev/null - timeout 10 npx tree-sitter highlight "$fname" > /dev/null + for line in just_repo_files.splitlines(): + files.add(f"{{ just_path }}/{line}") - echo "::endgroup::" - done + for line in """{{ errors_expected }}""".splitlines(): + line = re.sub("#.*", "", line).strip() + if len(line) == 0: + continue - # Parse files with expected errors - git ls-files test | grep -E '(timeout-.*|crash-.*)' | - while read -r fname - do - echo "::group::Parse and highlight testing for invalid source $fname" - - exitcode=0 - # Run with a timeout - timeout 10 npx tree-sitter parse --scope=source.just "$fname" > /dev/null || - exitstat=$? - - # Failed highlighting returns exit code 1, syntax errors are 2. We are - # looking for SIGBUS/SIGSEGV/SIGALRM (timeout), 135/138/142 - if [ "$exitcode" -gt 2 ]; then - echo "exited with code $exitcode" - exit "$exitcode" - fi - - # Try the same for highlighting - timeout 10 npx tree-sitter highlight --scope=source.just "$fname" > /dev/null || - exitcode=$? - if [ "$exitcode" -gt 2 ]; then - echo "exited with code $exitcode" - exit "$exitcode" - fi + fname = f"{{ justfile_directory() }}/{line}" + error_files.add(fname) + + for line in """{{ no_just_parsing }}""".splitlines(): + line = re.sub("#.*", "", line).strip() + if len(line) == 0: + continue + + fname = f"{{ justfile_directory() }}/{line}" + skip_just.append(fname) + + files -= error_files + + print("Checking parsing and highlighting...") + + for fname in files: + print(f"Checking '{fname}': parse'") + sp.check_output(["npx", "tree-sitter", "parse", "--quiet", fname], timeout=10) - echo "::endgroup::" - done + print(f"Checking '{fname}': highlight'") + sp.check_output(["npx", "tree-sitter", "highlight", "--quiet", fname], timeout=10) + + # Verify that the `just` tool parses all files we are using + if not fname in skip_just: + sp.check_output(["just", "--list", "--unstable", "--justfile", fname], timeout=10) + + print("Checking parsing and highlighting for invalid files...") + + for fname in error_files: + cmd = ["npx", "tree-sitter", "parse", "--quiet", fname] + try: + print(f"Checking invalid source '{fname}': parse'") + res = sp.check_output(cmd, timeout=10) + except sp.CalledProcessError as e: + if e.returncode != 1: # error code 1 is a highlight failure + print("command completed with non-1 exit status") + raise e + else: + print(i, "aaa") + raise AssertionError(f"failure expected but not found: {cmd} -> {res}") + + # Highlighting should always succeed + print(f"Checking invalid source '{fname}': highlight'") + sp.check_output(["npx", "tree-sitter", "highlight", "--quiet", fname], timeout=10) -# Verify that the `just` tool parses all files we are using -verify-just-parsing: - #!/bin/sh - set -eaux - - # skip readme.just because it is broken but works for testing - git ls-files '**.just' 'justfile' - grep -v readme.just | - grep -vE 'timeout-.*' | - grep -vE 'crash-.*' | - while read -r fname - do - echo "::notice:: checking Just parsing" - just --list --unstable --justfile "$fname" - done + print( + f"Successfully parsed {len(files) + len(error_files)} example files " + f"with {len(error_files)} expected failures" + ) # Make sure that no tests contain errors verify-no-error-tests: @@ -255,7 +285,7 @@ pre-commit: _lint-min format-check # Install pre-commit hooks pre-commit-install: #!/bin/sh - fname="{{justfile_directory()}}/.git/hooks/pre-commit" + fname="{{ justfile_directory() }}/.git/hooks/pre-commit" touch "$fname" chmod +x "$fname" @@ -264,18 +294,31 @@ pre-commit-install: just pre-commit EOF -# Download and build upstream tree-sitter -tree-sitter-clone: +# Clone or update a repo +_clone-repo url path sha: #!/bin/sh set -eaux - if [ ! -d "{{ ts_src }}" ]; then - git clone https://github.com/tree-sitter/tree-sitter "{{ ts_src }}" \ - --branch {{ ts_tag }} --depth=1 + if [ ! -d '{{ path }}' ]; then + echo "Cloning {{ url }}" + git clone '{{ url }}' '{{ path }}' --depth=100 + fi + + actual_sha=$(git -C '{{ path }}' rev-parse HEAD) + if [ "$actual_sha" != "{{ sha }}" ]; then + echo "Updating {{ url }} to {{ sha }}" + git -C '{{ path }}' fetch + git -C '{{ path }}' reset --hard '{{ sha }}' fi +# Clone the tree-sitter repo +_clone-repo-tree-sitter: (_clone-repo ts_repo ts_path ts_sha) + +# Clone the just repo +_clone-repo-just: (_clone-repo just_repo just_path just_sha) + # Build a simple debug executable -debug-build: tree-sitter-clone _out-dirs +debug-build: _clone-repo-tree-sitter _out-dirs #!/bin/sh set -eau @@ -294,7 +337,7 @@ debug-run *file-names: debug-build {{ debug_out }} {{file-names}} # Build and run the fuzzer -fuzz *extra-args: (gen "--debug-build") tree-sitter-clone _out-dirs +fuzz *extra-args: (gen "--debug-build") _clone-repo-tree-sitter _out-dirs #!/bin/sh set -eaux @@ -321,7 +364,7 @@ fuzz *extra-args: (gen "--debug-build") tree-sitter-clone _out-dirs fuzzer_flags="-artifact_prefix=$artifacts -timeout=20 -max_total_time={{ fuzz_time }} -jobs={{ num_cpus() }}" echo "Starting fuzzing at $(date -u -Is)" - LD_LIBRARY_PATH="{{ts_src}}" "{{ fuzz_out }}" "$corpus" $fuzzer_flags {{ extra-args }} + LD_LIBRARY_PATH="{{ ts_path }}" "{{ fuzz_out }}" "$corpus" $fuzzer_flags {{ extra-args }} # Configure the database used by clang-format, clang-tidy, and language servers configure-compile-database: diff --git a/test/highlight/recipes.just b/test/highlight/recipes.just index 275b387..ecea20e 100644 --- a/test/highlight/recipes.just +++ b/test/highlight/recipes.just @@ -74,12 +74,12 @@ fred: garply && (waldo "x") # ^^^ string # ^ punctuation.bracket -# foo -foo: - echo "foo" -# bar -bar: - echo "bar" +# plugh +plugh: + echo "plugh" +# xyzzy +xyzzy: + echo "xyzzy" # FIXME: can't test these because we can't place comments between [private] diff --git a/test/timeout-1aa6bf37e914715f4aa49e6cf693f7abf81aaf8e b/test/timeout-1aa6bf37e914715f4aa49e6cf693f7abf81aaf8e.just similarity index 100% rename from test/timeout-1aa6bf37e914715f4aa49e6cf693f7abf81aaf8e rename to test/timeout-1aa6bf37e914715f4aa49e6cf693f7abf81aaf8e.just