From fcedb315f39dbcfc1de2a9047ada723188b6c6c2 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Wed, 10 Jan 2024 04:21:33 -0500 Subject: [PATCH] Attempt to get some sort of CI for NeoVim specifically --- .github/workflows/ci.yaml | 12 +- justfile | 222 +++++++++++++++++++++++++--------- lua/tree-sitter-just/init.lua | 1 + scripts/ci-nvim-init.vim | 38 ++++++ 4 files changed, 212 insertions(+), 61 deletions(-) create mode 100644 scripts/ci-nvim-init.vim diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a34d8b6..740b5de 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,7 +18,12 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 18 - - run: pip install ruff + - name: Install dependencies + run: | + pip install ruff black + sudo apt-get update + sudo apt-get install -y luarocks + sudo luarocks install luacheck - name: Get npm cache directory id: npm-cache-dir shell: bash @@ -58,6 +63,11 @@ jobs: path: ${{ steps.npm-cache-dir.outputs.dir }} key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} restore-keys: ${{ runner.os }}-node- + - name: Setup nvim (requires sudo) + run: | + printenv + JUST=$(which just) + sudo "$JUST" --verbose ci-setup-nvim - run: just setup --locked - name: Configure run: just --verbose configure-tree-sitter diff --git a/justfile b/justfile index 92dce72..acd57cb 100644 --- a/justfile +++ b/justfile @@ -7,30 +7,66 @@ obj_dir := target / "obj" debug_out := bin_dir / "debug.out" fuzz_out := bin_dir / "fuzz.out" -ts_path := justfile_directory() / "repositories" / "tree-sitter" +downloads_path := justfile_directory() / "repositories" + +ts_path := downloads_path / "tree-sitter" ts_repo := "https://github.com/tree-sitter/tree-sitter" ts_sha := "1c55abb5308fe3891da545662e5df7ba28ade275" # v0.21.0 -just_path := justfile_directory() / "repositories" / "just" +just_path := downloads_path / "just" just_repo := "https://github.com/casey/just.git" just_sha := "a2ff42e6c37ba5c429d444f3a18d3633e59f9a34" # 1.24.0 +nvim_ts_path := downloads_path / "nvim-treesitter" +nvim_ts_repo := "https://github.com/nvim-treesitter/nvim-treesitter.git" +nvim_ts_sha := "f197a15b0d1e8d555263af20add51450e5aaa1f0" # v0.9.2 + include_args := "-Isrc/ -I" + ts_path + "/lib/include -Inode_modules/nan" general_cflags := "-Wall -Werror --pedantic -Wno-format-pedantic" fuzzer_flags := env("FUZZER_FLAGS", "-fsanitize=fuzzer,address,undefined") fuzz_time := env("FUZZ_TOTAL_TIME", "1200") +in_ci := env("CI", "0") # Source files needed to build a parser parser_sources := src + "/scanner.c " + src + "/parser.c " + ts_path + "/lib/src/lib.c" base_cache_key := sha256_file(src / "scanner.c") + sha256_file(src / "parser.c") + sha256(parser_sources) + sha256(include_args) + sha256(general_cflags) + sha256(fuzzer_flags) -verbose_flag := if env("CI", "") == "1" { "--verbose" } else { "" } - -# `timeout` is not available on all platforms, but perl often is. This needs a -# bash shell. -make_timeout_fn := '''timeout () { perl -e 'alarm shift; exec @ARGV' "$@"; }''' +verbose_flag := if in_ci != "0" { "--verbose" } else { "" } +run_if_installed := "just " + verbose_flag + " _run-if-installed " + +format_queries_url := "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/63ca90eaa3ce1cc668add8828a9e3d6728dbbdf1/scripts/format-queries.lua" +format_queries_sha := "a37344c87a0b9affa1d46b117e48442a205845776c4cdac31c57f591770cd522" +format_queries_fname := downloads_path / "format-queries.lua" + +check_queries_url := "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/63ca90eaa3ce1cc668add8828a9e3d6728dbbdf1/scripts/check-queries.lua" +check_queries_sha := "e4d7888aae1656e2a103f5d868c4f155eca20e554605bd3e6d1c2e8c8e4541f8" +check_queries_fname := downloads_path / "check-queries.lua" + +nvim_tag := "v0.9.5" +nvim_fsfx := if os() == "linux" { + "linux64.tar.gz" + } else if os() == "macos" { + "macos.tar.gz" + } else if os() == "windows" { + "win64.zip" + } else { + error("unsupported platform") + } +nvim_sha := if os() == "linux" { + "44ee395d9b5f8a14be8ec00d3b8ead34e18fe6461e40c9c8c50e6956d643b6ca" + } else if os() == "macos" { + "19d2366e0d6da001583bd0b8a3db59f69ce3dda5fa41f3064c6778cef3edd34c" + } else if os() == "windows" { + "de6dc1f0edb45f5f225ee24ce80a4fcbc3a337932037e98ae143975fca2556bf" + } else { + error("unsupported platform") + } +nvim_url := "https://github.com/neovim/neovim/releases/download/" + nvim_tag + "/nvim-" + nvim_fsfx +nvim_default_path := downloads_path / "nvim" +nvim_install_path := "$HOME/.local" +nvim_download_fname := downloads_path / "nvim-linux64.tar.gz" # Files that should parse with errors but not crash errors_expected := ''' @@ -50,6 +86,23 @@ no_just_parsing := ''' default: just --list +# Only run a command if the tool is installed +_run-if-installed CMD *ARGS="": + #!/bin/sh + set -eau + + if command -v {{ CMD }} > /dev/null; then + echo "Running '{{ CMD }} {{ ARGS }}'" 1>&2 + {{ CMD }} {{ ARGS }} + else + if [ '{{ in_ci }}' != 0 ]; then + echo 'NOT FOUND: {{ CMD }} required for CI' + exit 1 + fi + + echo "NOT FOUND: {{ CMD }}. Skipping check (this will get verified in CI)." + fi + # Install needed packages and make sure tools are setup setup *npm-args: #!/bin/sh @@ -71,23 +124,22 @@ setup *npm-args: check_installed clang check_installed clang-tidy check_installed clang-format + check_installed nvim - if which npm > /dev/null; then - npm install --include=dev {{ npm-args }} - else - echo "npm not found: skipping install" - fi + {{ run_if_installed }} npm install --include=dev {{ npm-args }} # Lint with more minimal dependencies that can be run during pre-commit _lint-min: _clone-repo-tree-sitter configure-compile-database - npm run lint:check + @{{ run_if_installed }} npm run lint:check git ls-files '**.c' | grep -v 'parser\.c' | \ - xargs -IFNAME sh -c 'echo "\nchecking file FNAME" && clang-tidy FNAME' + xargs -IFNAME sh -c \ + 'echo "\nchecking file FNAME" && {{ run_if_installed }} clang-tidy FNAME' -# Run the linter for JS, C, Cargo, and Python. Requires clang-tidy, clippy, and ruff. +# Run the linter for JS, C, Cargo, and Python. Will skip tools that are not installed. lint: _lint-min - cargo clippy - ruff . + @{{ run_if_installed }} cargo clippy + @{{ run_if_installed }} ruff . + @{{ run_if_installed }} luacheck . _out-dirs: mkdir -p "{{ bin_dir }}" @@ -95,22 +147,26 @@ _out-dirs: alias fmt := format -# Autoformat code. Requires Cargo, clang-format, and black. -format: configure-compile-database +# Autoformat code. Requires Cargo, clang-format and black are optional +format: configure-compile-database \ + (_ensure-downloaded format_queries_url format_queries_sha format_queries_fname) npm run format:write - git ls-files '**.c' | grep -v 'parser\.c' | \ + git ls-files '**.c' | grep -Fv 'parser.c' | \ xargs -IFNAME sh -c \ - 'echo "\nformatting 'FNAME'" && clang-format -i FNAME --verbose' - cargo fmt - black . + 'echo "\nformatting 'FNAME'" && \ + {{ run_if_installed }} clang-format -i FNAME --verbose' + {{ run_if_installed }} cargo fmt + {{ run_if_installed }} black . # Check formatting without editing format-check: configure-compile-database npm run format:check - git ls-files '**.c' | grep -v 'parser\.c' | \ + git ls-files '**.c' | grep -Fv 'parser.c' | \ xargs -IFNAME sh -c \ - 'echo "\nchecking formatting for 'FNAME'" && clang-format FNAME | diff -up - FNAME' - cargo fmt --check + 'echo "\nchecking formatting for 'FNAME'" && \ + {{ run_if_installed }} clang-format FNAME | diff -up - FNAME' + {{ run_if_installed }} cargo fmt --check + {{ run_if_installed }} black . --check # Generate the parser gen *extra-args: @@ -120,10 +176,7 @@ gen *extra-args: # Run formatting only on generated files npx prettier --write src/ - # Only clang-format if it is available - which clang-format > /dev/null && \ - clang-format -i src/parser.c || \ - echo "skipping clang-format" + @{{ run_if_installed }} clang-format -i src/parser.c alias t := test @@ -133,10 +186,21 @@ test *ts-test-args: gen just {{ verbose_flag }} test-parse-highlight just {{ verbose_flag }} verify-no-error-tests + if command -v nvim > /dev/null; then \ + just {{ verbose_flag }} test-nvim; \ + else \ + echo "NeoVim not found; skipping tests"; \ + fi + echo '\nRunning Cargo tests' # FIXME: xfail Rust CI on Windows because we are getting STATUS_DLL_NOT_FOUND - {{ if os_family() + env("CI", "1") == "windows1" { "echo skipping tests on Windows" } else { "cargo test" } }} + {{ if os_family() + env("CI", "1") == "windows1" { \ + "echo skipping tests on Windows" \ + } else { \ + "cargo test" \ + } }} + # Verify that tree-sitter can parse and highlight all files in the repo. Requires a tree-sitter configuration. test-parse-highlight: _clone-repo-just @@ -242,6 +306,10 @@ test-parse-highlight: _clone-repo-just verify-no-error-tests: ! grep -nr -C4 -E '(ERROR|MISSING|UNEXPECTED)' test +test-nvim: _clone-repo-nvim-treesitter + @echo "Running NeoVim tests" + # FIXME: do something + # Helper to rebuild helix grammars hx-build: hx --grammar build @@ -259,7 +327,7 @@ configure-tree-sitter: else: shell = False - cfg_fname = r"""{{ config_directory() / "tree-sitter" / "config.json" }}""" + cfg_fname = r"""{ config_directory() / "tree-sitter" / "config.json" }}""" if not os.path.isfile(cfg_fname): sp.run(["npx", "tree-sitter", "init-config"], check=True, shell=shell) @@ -274,30 +342,6 @@ configure-tree-sitter: f.truncate() -# Run lint and check formatting -ci-codestyle: lint format-check - -# Make sure that files have not changed -ci-validate-generated-files exit-code="1": - #!/bin/sh - set -eaux - - git tag ci-tmp-pre-updates - - just {{ verbose_flag }} gen - - failed=false - git diff ci-tmp-pre-updates --exit-code || failed=true - git tag -d ci-tmp-pre-updates - - if ! [ "$failed" = "false" ]; then - echo '::warning::Generated files are out of date!' - echo '::warning::run `just gen` and commit the changes' - - # We use an exit code so that we can use this as either a warning or error - exit {{ exit-code }} - fi - # Run a subset of CI checks before committing. pre-commit: _lint-min format-check @@ -320,7 +364,7 @@ _clone-repo url path sha: if [ ! -d '{{ path }}' ]; then echo "Cloning {{ url }}" - git clone '{{ url }}' '{{ path }}' --depth=100 + git clone '{{ url }}' '{{ path }}' --depth=500 fi actual_sha=$(git -C '{{ path }}' rev-parse HEAD) @@ -330,11 +374,21 @@ _clone-repo url path sha: git -C '{{ path }}' reset --hard '{{ sha }}' fi -# Clone the tree-sitter repo +# Helpers to clone specific repos _clone-repo-tree-sitter: (_clone-repo ts_repo ts_path ts_sha) - -# Clone the just repo _clone-repo-just: (_clone-repo just_repo just_path just_sha) +_clone-repo-nvim-treesitter: (_clone-repo nvim_ts_repo nvim_ts_path nvim_ts_sha) + +_ensure-downloaded URL SHA OUTPUT_FNAME: + #!/bin/sh + set -eau + # Exit if already installed + echo '{{ SHA }} {{ OUTPUT_FNAME }}' | shasum -c && exit 0 + + set -x + mkdir -p '{{ parent_directory(OUTPUT_FNAME) }}' + curl -L '{{ URL }}' -o '{{ OUTPUT_FNAME }}' + echo '{{ SHA }} {{ OUTPUT_FNAME }}' | shasum -c # Build a simple debug executable debug-build: _clone-repo-tree-sitter _out-dirs @@ -411,3 +465,51 @@ configure-compile-database: with open("compile_commands.json", "w") as f: json.dump(results, f, indent=4) + +# Set up NeoVim for use in CI +ci-setup-nvim: + #!/bin/sh + set -eaux + + # Exit if installed + if ! command -v nvim; then + if [ {{ in_ci }} != '1' ]; then + echo "refusing to install nvim when not in CI" + exit 1 + fi + + mkdir -p "{{ nvim_install_path }}" + just {{ verbose_flag }} _ensure-downloaded '{{ nvim_url }}' \ + '{{ nvim_sha }}' '{{ nvim_download_fname }}' + + tar -vxzf "{{ nvim_download_fname }}" -C "{{ nvim_install_path }}" \ + --strip-components=1 + + ln -s "{{ nvim_install_path }}/bin/nvim" /usr/bin/nvim + fi + + nvim --version + +# Run lint and check formatting +ci-codestyle: lint format-check + +# Make sure that files have not changed +ci-validate-generated-files exit-code="1": + #!/bin/sh + set -eaux + + git tag ci-tmp-pre-updates + + just {{ verbose_flag }} gen + + failed=false + git diff ci-tmp-pre-updates --exit-code || failed=true + git tag -d ci-tmp-pre-updates + + if ! [ "$failed" = "false" ]; then + echo '::warning::Generated files are out of date!' + echo '::warning::run `just gen` and commit the changes' + + # We use an exit code so that we can use this as either a warning or error + exit {{ exit-code }} + fi diff --git a/lua/tree-sitter-just/init.lua b/lua/tree-sitter-just/init.lua index e7355f8..5f0e42b 100644 --- a/lua/tree-sitter-just/init.lua +++ b/lua/tree-sitter-just/init.lua @@ -22,6 +22,7 @@ function M.setup(arg) parser_config.just = { install_info = { url = arg["local"] and join_paths( + -- luacheck: ignore vim.fn.stdpath("data"), "site", "pack", diff --git a/scripts/ci-nvim-init.vim b/scripts/ci-nvim-init.vim new file mode 100644 index 0000000..448bf7d --- /dev/null +++ b/scripts/ci-nvim-init.vim @@ -0,0 +1,38 @@ +" This script is used by CI" + +" Install vim-plug +let data_dir = has('nvim') ? stdpath('data') . '/site' : '~/.vim' +if empty(glob(data_dir . '/autoload/plug.vim')) + silent execute '!curl -fLo '.data_dir.'/autoload/plug.vim --create-dirs https://raw.githubusercontent.com/junegunn/vim-plug/master/plug.vim' + autocmd VimEnter * PlugInstall --sync | source $MYVIMRC +endif + +call plug#begin() + +" Install treesitter +Plug 'nvim-treesitter/nvim-treesitter', {'do': ':TSUpdate'} + +" Install our local workspace +require("nvim-treesitter.parsers").get_parser_configs().just = { + install_info = { + url = "${{ github.workspace }}/tree-sitter-just", -- local path or git repo + files = { "src/parser.c", "src/scanner.c" }, + branch = "main", + -- use_makefile = true -- this may be necessary on MacOS (try if you see compiler errors) + }, + maintainers = { "@IndianBoy42" }, +} + +call plug#end() + +lua <