From 10bbe40680e82da56749f47e9630a1f145f0f1cc Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Wed, 10 Jan 2024 04:21:33 -0500 Subject: [PATCH] Attempt to get some sort of CI for NeoVim specifically --- .github/workflows/ci.yaml | 9 ++ justfile | 152 ++++++++++++++++++++++------------ lua/tree-sitter-just/init.lua | 1 + scripts/ci-nvim-init.vim | 38 +++++++++ 4 files changed, 147 insertions(+), 53 deletions(-) create mode 100644 scripts/ci-nvim-init.vim diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a34d8b6..e2b7d58 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -65,3 +65,12 @@ jobs: run: just --verbose test - name: Check if generated files are up to date (warn only) run: just --verbose ci-validate-generated-files 0 + + nvim: + name: Tests that use NeoVim + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: taiki-e/install-action@just + - run: just --verbose ci-setup-nvim diff --git a/justfile b/justfile index 5b0ab6f..e1bdcf7 100644 --- a/justfile +++ b/justfile @@ -7,11 +7,13 @@ obj_dir := target / "obj" debug_out := bin_dir / "debug.out" fuzz_out := bin_dir / "fuzz.out" -ts_path := justfile_directory() / "repositories" / "tree-sitter" +downloads_path := justfile_directory() / "repositories" + +ts_path := downloads_path / "tree-sitter" ts_repo := "https://github.com/tree-sitter/tree-sitter" ts_sha := "1c55abb5308fe3891da545662e5df7ba28ade275" # v0.21.0 -just_path := justfile_directory() / "repositories" / "just" +just_path := downloads_path / "just" just_repo := "https://github.com/casey/just.git" just_sha := "a2ff42e6c37ba5c429d444f3a18d3633e59f9a34" # 1.24.0 @@ -27,10 +29,17 @@ parser_sources := src + "/scanner.c " + src + "/parser.c " + ts_path + "/lib/src base_cache_key := sha256_file(src / "scanner.c") + sha256_file(src / "parser.c") + sha256(parser_sources) + sha256(include_args) + sha256(general_cflags) + sha256(fuzzer_flags) verbose_flag := if env("CI", "") == "1" { "--verbose" } else { "" } +run_if_installed := "just " + verbose_flag + "_run-if-installed " + +format_queries_url := "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/63ca90eaa3ce1cc668add8828a9e3d6728dbbdf1/scripts/format-queries.lua" +format_queries_sha := "a37344c87a0b9affa1d46b117e48442a205845776c4cdac31c57f591770cd522" +format_queries_fname := downloads_path / "format-queries.lua" -# `timeout` is not available on all platforms, but perl often is. This needs a -# bash shell. -make_timeout_fn := '''timeout () { perl -e 'alarm shift; exec @ARGV' "$@"; }''' +nvim_url := "https://github.com/neovim/neovim/releases/download/v0.9.5/nvim-linux64.tar.gz" +nvim_sha := "44ee395d9b5f8a14be8ec00d3b8ead34e18fe6461e40c9c8c50e6956d643b6ca" +nvim_default_path := downloads_path / "nvim" +nvim_download_fname := downloads_path / "nvim-linux64.tar.gz" +nvim_exe := `which nvim || echo {{ nvim_default_path }}` # Files that should parse with errors but not crash errors_expected := ''' @@ -51,6 +60,15 @@ no_just_parsing := ''' default: just --list +# Only run a command if the tool is installed +_run-if-installed CMD *ARGS="": + @if command -v {{ CMD }} > /dev/null; then \ + echo "Running '{{ CMD }} {{ ARGS }}'" 1>&2; \ + {{ CMD }} {{ ARGS }}; \ + else \ + echo "NOT FOUND: {{ CMD }}. Skipping check (this will get verified in CI)."; \ + fi + # Install needed packages and make sure tools are setup setup *npm-args: #!/bin/sh @@ -73,22 +91,20 @@ setup *npm-args: check_installed clang-tidy check_installed clang-format - if which npm > /dev/null; then - npm install --include=dev {{ npm-args }} - else - echo "npm not found: skipping install" - fi + {{ run_if_installed }} npm install --include=dev {{ npm-args }} # Lint with more minimal dependencies that can be run during pre-commit _lint-min: _clone-repo-tree-sitter configure-compile-database - npm run lint:check + @{{ run_if_installed }} npm run lint:check git ls-files '**.c' | grep -v 'parser\.c' | \ - xargs -IFNAME sh -c 'echo "\nchecking file FNAME" && clang-tidy FNAME' + xargs -IFNAME sh -c 'echo "\nchecking file FNAME" && \ + {{ run_if_installed }} clang-tidy FNAME' -# Run the linter for JS, C, Cargo, and Python. Requires clang-tidy, clippy, and ruff. +# Run the linter for JS, C, Cargo, and Python. Will skip tools that are not installed. lint: _lint-min - cargo clippy - ruff . + @{{ run_if_installed }} cargo clippy + @{{ run_if_installed }} ruff . + @{{ run_if_installed }} luacheck . _out-dirs: mkdir -p "{{ bin_dir }}" @@ -96,22 +112,25 @@ _out-dirs: alias fmt := format -# Autoformat code. Requires Cargo, clang-format, and black. -format: configure-compile-database +# Autoformat code. Requires Cargo, clang-format and black are optional +format: configure-compile-database \ + (_ensure-downloaded format_queries_url format_queries_sha format_queries_fname) npm run format:write - git ls-files '**.c' | grep -v 'parser\.c' | \ + git ls-files '**.c' | grep -Fv 'parser.c' | \ xargs -IFNAME sh -c \ - 'echo "\nformatting 'FNAME'" && clang-format -i FNAME --verbose' - cargo fmt - black . + 'echo "\nformatting 'FNAME'" && \ + {{ run_if_installed }} clang-format -i FNAME --verbose' + {{ run_if_installed }} cargo fmt + {{ run_if_installed }} black . # Check formatting without editing format-check: configure-compile-database npm run format:check - git ls-files '**.c' | grep -v 'parser\.c' | \ + git ls-files '**.c' | grep -Fv 'parser.c' | \ xargs -IFNAME sh -c \ - 'echo "\nchecking formatting for 'FNAME'" && clang-format FNAME | diff -up - FNAME' - cargo fmt --check + 'echo "\nchecking formatting for 'FNAME'" && \ + {{ run_if_installed }} clang-format FNAME | diff -up - FNAME' + {{ run_if_installed }} cargo fmt --check # Generate the parser gen *extra-args: @@ -121,10 +140,7 @@ gen *extra-args: # Run formatting only on generated files npx prettier --write src/ - # Only clang-format if it is available - which clang-format > /dev/null && \ - clang-format -i src/parser.c || \ - echo "skipping clang-format" + @{{ run_if_installed }} -i src/parser.c alias t := test @@ -137,7 +153,11 @@ test *ts-test-args: gen echo '\nRunning Cargo tests' # FIXME: xfail Rust CI on Windows because we are getting STATUS_DLL_NOT_FOUND - {{ if os_family() + env("CI", "1") == "windows1" { "echo skipping tests on Windows" } else { "cargo test" } }} + {{ if os_family() + env("CI", "1") == "windows1" { \ + "echo skipping tests on Windows" \ + } else { \ + "cargo test" \ + } }} # Verify that tree-sitter can parse and highlight all files in the repo. Requires a tree-sitter configuration. test-parse-highlight: _clone-repo-just @@ -275,30 +295,6 @@ configure-tree-sitter: f.truncate() -# Run lint and check formatting -ci-codestyle: lint format-check - -# Make sure that files have not changed -ci-validate-generated-files exit-code="1": - #!/bin/sh - set -eaux - - git tag ci-tmp-pre-updates - - just {{ verbose_flag }} gen - - failed=false - git diff ci-tmp-pre-updates --exit-code || failed=true - git tag -d ci-tmp-pre-updates - - if ! [ "$failed" = "false" ]; then - echo '::warning::Generated files are out of date!' - echo '::warning::run `just gen` and commit the changes' - - # We use an exit code so that we can use this as either a warning or error - exit {{ exit-code }} - fi - # Run a subset of CI checks before committing. pre-commit: _lint-min format-check @@ -337,6 +333,16 @@ _clone-repo-tree-sitter: (_clone-repo ts_repo ts_path ts_sha) # Clone the just repo _clone-repo-just: (_clone-repo just_repo just_path just_sha) +_ensure-downloaded URL SHA OUTPUT_FNAME: + #!/bin/sh + set -eau + # Exit if already installed + echo '{{ SHA }} {{ OUTPUT_FNAME }}' | shasum -c && exit 0 + + set -x + curl -L '{{ URL }}' -o '{{ OUTPUT_FNAME }}' + echo '{{ SHA }} {{ OUTPUT_FNAME }}' | shasum -c + # Build a simple debug executable debug-build: _clone-repo-tree-sitter _out-dirs #!/bin/sh @@ -412,3 +418,43 @@ configure-compile-database: with open("compile_commands.json", "w") as f: json.dump(results, f, indent=4) + +# Set up NeoVim for use in CI +ci-setup-nvim: + #!/bin/sh + set -eaux + + # Exit if installed + if ! command -v nvim; then + mkdir -p "{{ nvim_default_path }}" + just {{ verbose_flag }} _ensure-downloaded '{{ nvim_url }}' \ + '{{ nvim_sha }}' '{{ nvim_download_fname }}' + + tar -xzvf "{{ nvim_download_fname }}" -C "{{ nvim_default_path }}" + fi + + {{ nvim_exe }} --version + +# Run lint and check formatting +ci-codestyle: lint format-check + +# Make sure that files have not changed +ci-validate-generated-files exit-code="1": + #!/bin/sh + set -eaux + + git tag ci-tmp-pre-updates + + just {{ verbose_flag }} gen + + failed=false + git diff ci-tmp-pre-updates --exit-code || failed=true + git tag -d ci-tmp-pre-updates + + if ! [ "$failed" = "false" ]; then + echo '::warning::Generated files are out of date!' + echo '::warning::run `just gen` and commit the changes' + + # We use an exit code so that we can use this as either a warning or error + exit {{ exit-code }} + fi diff --git a/lua/tree-sitter-just/init.lua b/lua/tree-sitter-just/init.lua index e7355f8..5f0e42b 100644 --- a/lua/tree-sitter-just/init.lua +++ b/lua/tree-sitter-just/init.lua @@ -22,6 +22,7 @@ function M.setup(arg) parser_config.just = { install_info = { url = arg["local"] and join_paths( + -- luacheck: ignore vim.fn.stdpath("data"), "site", "pack", diff --git a/scripts/ci-nvim-init.vim b/scripts/ci-nvim-init.vim new file mode 100644 index 0000000..448bf7d --- /dev/null +++ b/scripts/ci-nvim-init.vim @@ -0,0 +1,38 @@ +" This script is used by CI" + +" Install vim-plug +let data_dir = has('nvim') ? stdpath('data') . '/site' : '~/.vim' +if empty(glob(data_dir . '/autoload/plug.vim')) + silent execute '!curl -fLo '.data_dir.'/autoload/plug.vim --create-dirs https://raw.githubusercontent.com/junegunn/vim-plug/master/plug.vim' + autocmd VimEnter * PlugInstall --sync | source $MYVIMRC +endif + +call plug#begin() + +" Install treesitter +Plug 'nvim-treesitter/nvim-treesitter', {'do': ':TSUpdate'} + +" Install our local workspace +require("nvim-treesitter.parsers").get_parser_configs().just = { + install_info = { + url = "${{ github.workspace }}/tree-sitter-just", -- local path or git repo + files = { "src/parser.c", "src/scanner.c" }, + branch = "main", + -- use_makefile = true -- this may be necessary on MacOS (try if you see compiler errors) + }, + maintainers = { "@IndianBoy42" }, +} + +call plug#end() + +lua <