Attempt to get some sort of CI for NeoVim specifically

tgross35 · Mar 4, 2024 · 0acb5e8 · 0acb5e8
1 parent 51aa99c
commit 0acb5e8
Show file tree

Hide file tree

Showing 5 changed files with 8,496 additions and 10,518 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -18,7 +18,10 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: 18
-      - run: pip install ruff
+      - run: |
+          pip install ruff
+          sudo apt-get update
+          sudo apt-get install -y luacheck
       - name: Get npm cache directory
         id: npm-cache-dir
         shell: bash
@@ -58,6 +61,7 @@ jobs:
           path: ${{ steps.npm-cache-dir.outputs.dir }}
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
           restore-keys: ${{ runner.os }}-node-
+      - run: just --verbose ci-setup-nvim
       - run: just setup --locked
       - name: Configure
         run: just --verbose configure-tree-sitter

diff --git a/justfile b/justfile
@@ -7,30 +7,66 @@ obj_dir := target / "obj"
 debug_out := bin_dir / "debug.out"
 fuzz_out := bin_dir / "fuzz.out"
 
-ts_path := justfile_directory() / "repositories" / "tree-sitter"
+downloads_path := justfile_directory() / "repositories"
+
+ts_path := downloads_path / "tree-sitter"
 ts_repo := "https://github.com/tree-sitter/tree-sitter"
 ts_sha := "1c55abb5308fe3891da545662e5df7ba28ade275" # v0.21.0
 
-just_path := justfile_directory() / "repositories" / "just"
+just_path := downloads_path / "just"
 just_repo := "https://github.com/casey/just.git"
 just_sha := "a2ff42e6c37ba5c429d444f3a18d3633e59f9a34" # 1.24.0
 
+nvim_ts_path := downloads_path / "nvim-treesitter"
+nvim_ts_repo := "https://github.com/nvim-treesitter/nvim-treesitter.git"
+nvim_ts_sha := "f197a15b0d1e8d555263af20add51450e5aaa1f0" # v0.9.2
+
 include_args := "-Isrc/ -I" + ts_path + "/lib/include -Inode_modules/nan"
 general_cflags := "-Wall -Werror --pedantic -Wno-format-pedantic"
 
 fuzzer_flags := env("FUZZER_FLAGS", "-fsanitize=fuzzer,address,undefined")
 fuzz_time := env("FUZZ_TOTAL_TIME", "1200")
+in_ci := env("CI", "0")
 
 # Source files needed to build a parser
 parser_sources := src + "/scanner.c " + src + "/parser.c " + ts_path + "/lib/src/lib.c"
 
 base_cache_key := sha256_file(src / "scanner.c") + sha256_file(src / "parser.c") + sha256(parser_sources) + sha256(include_args) + sha256(general_cflags) + sha256(fuzzer_flags)
 
-verbose_flag := if env("CI", "") == "1" { "--verbose" } else { "" }
-
-# `timeout` is not available on all platforms, but perl often is. This needs a
-# bash shell.
-make_timeout_fn := '''timeout () { perl -e 'alarm shift; exec @ARGV' "$@"; }'''
+verbose_flag := if in_ci != "0" { "--verbose" } else { "" }
+run_if_installed := "just " + verbose_flag + " _run-if-installed "
+
+format_queries_url := "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/63ca90eaa3ce1cc668add8828a9e3d6728dbbdf1/scripts/format-queries.lua"
+format_queries_sha := "a37344c87a0b9affa1d46b117e48442a205845776c4cdac31c57f591770cd522"
+format_queries_fname := downloads_path / "format-queries.lua"
+
+check_queries_url := "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/63ca90eaa3ce1cc668add8828a9e3d6728dbbdf1/scripts/check-queries.lua"
+check_queries_sha := "e4d7888aae1656e2a103f5d868c4f155eca20e554605bd3e6d1c2e8c8e4541f8"
+check_queries_fname := downloads_path / "check-queries.lua"
+
+nvim_tag := "v0.9.5"
+nvim_fsfx := if os() == "linux" {
+		"linux64.tar.gz"
+	} else if os() == "macos" { 
+		"macos.tar.gz"
+	} else if os() == "windows" {
+		"win64.zip"
+	} else {
+		error("unsupported platform")
+	}
+nvim_sha := if os() == "linux" {
+		"44ee395d9b5f8a14be8ec00d3b8ead34e18fe6461e40c9c8c50e6956d643b6ca"
+	} else if os() == "macos" {
+		"19d2366e0d6da001583bd0b8a3db59f69ce3dda5fa41f3064c6778cef3edd34c"
+	} else if os() == "windows" {
+		"de6dc1f0edb45f5f225ee24ce80a4fcbc3a337932037e98ae143975fca2556bf"
+	} else {
+		error("unsupported platform")
+	}
+nvim_url := "https://github.com/neovim/neovim/releases/download/" + nvim_tag + "/nvim-" + nvim_fsfx
+nvim_default_path := downloads_path / "nvim" / "nvim-" + os() / "bin" / "nvim"
+nvim_download_fname := downloads_path / "nvim-linux64.tar.gz"
+nvim_exe := if `which nvim || echo ""` != "" { `which nvim` } else { nvim_default_path }
 
 # Files that should parse with errors but not crash
 errors_expected := '''
@@ -51,6 +87,21 @@ no_just_parsing := '''
 default:
 	just --list
 
+# Only run a command if the tool is installed
+_run-if-installed CMD *ARGS="":
+	#!/bin/sh
+	if command -v {{ CMD }} > /dev/null; then
+		echo "Running '{{ CMD }} {{ ARGS }}'" 1>&2
+		{{ CMD }} {{ ARGS }}
+	else
+		if [ '{{ in_ci }}' != 0 ]; then
+			echo 'NOT FOUND: {{ CMD }} required for CI'
+			exit 1
+		fi
+
+		echo "NOT FOUND: {{ CMD }}. Skipping check (this will get verified in CI)."
+	fi
+
 # Install needed packages and make sure tools are setup
 setup *npm-args:
 	#!/bin/sh
@@ -72,46 +123,49 @@ setup *npm-args:
 	check_installed clang
 	check_installed clang-tidy
 	check_installed clang-format
+	check_installed nvim
 
-	if which npm > /dev/null; then
-		npm install --include=dev {{ npm-args }}
-	else
-		echo "npm not found: skipping install"
-	fi
+	{{ run_if_installed }} npm install --include=dev {{ npm-args }}
 
 # Lint with more minimal dependencies that can be run during pre-commit
 _lint-min: _clone-repo-tree-sitter configure-compile-database
-	npm run lint:check
+	@{{ run_if_installed }} npm run lint:check
 	git ls-files '**.c' | grep -v 'parser\.c' | \
-		xargs -IFNAME sh -c 'echo "\nchecking file FNAME" && clang-tidy FNAME'
+		xargs -IFNAME sh -c 'echo "\nchecking file FNAME" && \
+		{{ run_if_installed }} clang-tidy FNAME'
 
-# Run the linter for JS, C, Cargo, and Python. Requires clang-tidy, clippy, and ruff.
+# Run the linter for JS, C, Cargo, and Python. Will skip tools that are not installed.
 lint: _lint-min
-	cargo clippy
-	ruff .
+	@{{ run_if_installed }} cargo clippy
+	@{{ run_if_installed }} ruff .
+	@{{ run_if_installed }} luacheck .
 
 _out-dirs:
 	mkdir -p "{{ bin_dir }}"
 	mkdir -p "{{ obj_dir }}"
 
 alias fmt := format
 
-# Autoformat code. Requires Cargo, clang-format, and black.
-format: configure-compile-database
+# Autoformat code. Requires Cargo, clang-format and black are optional
+format: configure-compile-database \
+	(_ensure-downloaded format_queries_url format_queries_sha format_queries_fname)
 	npm run format:write
-	git ls-files '**.c' | grep -v 'parser\.c' | \
+	git ls-files '**.c' | grep -Fv 'parser.c' | \
 		xargs -IFNAME sh -c \
-		'echo "\nformatting 'FNAME'" && clang-format -i FNAME --verbose'
-	cargo fmt
-	black .
+		'echo "\nformatting 'FNAME'" && \
+		{{ run_if_installed }} clang-format -i FNAME --verbose'
+	{{ run_if_installed }} cargo fmt
+	{{ run_if_installed }} black .
 
 # Check formatting without editing
 format-check: configure-compile-database
 	npm run format:check
-	git ls-files '**.c' | grep -v 'parser\.c' | \
+	git ls-files '**.c' | grep -Fv 'parser.c' | \
 		xargs -IFNAME sh -c \
-		'echo "\nchecking formatting for 'FNAME'" && clang-format FNAME | diff -up - FNAME'
-	cargo fmt --check
+		'echo "\nchecking formatting for 'FNAME'" && \
+		{{ run_if_installed }} clang-format FNAME | diff -up - FNAME'
+	{{ run_if_installed }} cargo fmt --check
+	{{ run_if_installed }} black . --check
 
 # Generate the parser
 gen *extra-args:
@@ -121,10 +175,7 @@ gen *extra-args:
 	# Run formatting only on generated files
 	npx prettier --write src/
 
-	# Only clang-format if it is available
-	which clang-format > /dev/null && \
-		clang-format -i src/parser.c || \
-		echo "skipping clang-format"
+	@{{ run_if_installed }} -i src/parser.c
 
 alias t := test
 
@@ -134,10 +185,21 @@ test *ts-test-args: gen
 	just {{ verbose_flag }} test-parse-highlight
 	just {{ verbose_flag }} verify-no-error-tests
 
+	if command -v nvim > /dev/null; then \
+		just {{ verbose_flag }} test-nvim; \
+	else \
+		echo "NeoVim not found; skipping tests"; \
+	fi
+
 	echo '\nRunning Cargo tests'
 
 	# FIXME: xfail Rust CI on Windows because we are getting STATUS_DLL_NOT_FOUND
-	{{ if os_family() + env("CI", "1") == "windows1" { "echo skipping tests on Windows" } else { "cargo test" } }}
+	{{ if os_family() + env("CI", "1") == "windows1" { \
+		"echo skipping tests on Windows" \
+		} else { \
+		"cargo test" \
+	} }}
+
 
 # Verify that tree-sitter can parse and highlight all files in the repo. Requires a tree-sitter configuration.
 test-parse-highlight: _clone-repo-just
@@ -243,6 +305,10 @@ test-parse-highlight: _clone-repo-just
 verify-no-error-tests:
 	! grep -nr -C4 -E '(ERROR|MISSING|UNEXPECTED)' test
 
+test-nvim: _clone-repo-nvim-treesitter
+	@echo "Running NeoVim tests"
+	# FIXME: do something
+
 # Helper to rebuild helix grammars
 hx-build:
 	hx --grammar build
@@ -275,30 +341,6 @@ configure-tree-sitter:
 
 		f.truncate()
 
-# Run lint and check formatting
-ci-codestyle: lint format-check
-
-# Make sure that files have not changed
-ci-validate-generated-files exit-code="1":
-	#!/bin/sh
-	set -eaux
-
-	git tag ci-tmp-pre-updates
-
-	just {{ verbose_flag }} gen
-
-	failed=false
-	git diff ci-tmp-pre-updates --exit-code || failed=true
-	git tag -d ci-tmp-pre-updates
-
-	if ! [ "$failed" = "false" ]; then
-		echo '::warning::Generated files are out of date!'
-		echo '::warning::run `just gen` and commit the changes'
-
-		# We use an exit code so that we can use this as either a warning or error
-		exit {{ exit-code }}
-	fi
-
 # Run a subset of CI checks before committing.
 pre-commit: _lint-min format-check
 
@@ -321,7 +363,7 @@ _clone-repo url path sha:
 
 	if [ ! -d '{{ path }}' ]; then
 		echo "Cloning {{ url }}"
-		git clone '{{ url }}' '{{ path }}' --depth=100
+		git clone '{{ url }}' '{{ path }}' --depth=500
 	fi
 
 	actual_sha=$(git -C '{{ path }}' rev-parse HEAD)
@@ -331,11 +373,20 @@ _clone-repo url path sha:
 		git -C '{{ path }}' reset --hard '{{ sha }}'
 	fi
 
-# Clone the tree-sitter repo
+# Helpers to clone specific repos
 _clone-repo-tree-sitter: (_clone-repo ts_repo ts_path ts_sha)
-
-# Clone the just repo
 _clone-repo-just: (_clone-repo just_repo just_path just_sha)
+_clone-repo-nvim-treesitter: (_clone-repo nvim_ts_repo nvim_ts_path nvim_ts_sha)
+
+_ensure-downloaded URL SHA OUTPUT_FNAME:
+	#!/bin/sh
+	set -eau
+	# Exit if already installed
+	echo '{{ SHA }}  {{ OUTPUT_FNAME }}' | shasum -c && exit 0
+
+	set -x
+	curl -L '{{ URL }}' -o '{{ OUTPUT_FNAME }}'
+	echo '{{ SHA }}  {{ OUTPUT_FNAME }}' | shasum -c
 
 # Build a simple debug executable
 debug-build: _clone-repo-tree-sitter _out-dirs
@@ -412,3 +463,43 @@ configure-compile-database:
 
 	with open("compile_commands.json", "w") as f:
 		json.dump(results, f, indent=4)
+
+# Set up NeoVim for use in CI
+ci-setup-nvim:
+	#!/bin/sh
+	set -eaux
+
+	# Exit if installed
+	if ! command -v nvdim; then
+		mkdir -p "{{ nvim_default_path }}"
+		just {{ verbose_flag }} _ensure-downloaded '{{ nvim_url }}' \
+			'{{ nvim_sha }}' '{{ nvim_download_fname }}'
+
+		tar -xzvf "{{ nvim_download_fname }}" -C "{{ nvim_default_path }}"
+	fi
+
+	{{ nvim_exe }} --version
+
+# Run lint and check formatting
+ci-codestyle: lint format-check
+
+# Make sure that files have not changed
+ci-validate-generated-files exit-code="1":
+	#!/bin/sh
+	set -eaux
+
+	git tag ci-tmp-pre-updates
+
+	just {{ verbose_flag }} gen
+
+	failed=false
+	git diff ci-tmp-pre-updates --exit-code || failed=true
+	git tag -d ci-tmp-pre-updates
+
+	if ! [ "$failed" = "false" ]; then
+		echo '::warning::Generated files are out of date!'
+		echo '::warning::run `just gen` and commit the changes'
+
+		# We use an exit code so that we can use this as either a warning or error
+		exit {{ exit-code }}
+	fi
diff --git a/lua/tree-sitter-just/init.lua b/lua/tree-sitter-just/init.lua
@@ -22,6 +22,7 @@ function M.setup(arg)
 	parser_config.just = {
 		install_info = {
 			url = arg["local"] and join_paths(
+				-- luacheck: ignore
 				vim.fn.stdpath("data"),
 				"site",
 				"pack",

diff --git a/scripts/ci-nvim-init.vim b/scripts/ci-nvim-init.vim
@@ -0,0 +1,38 @@
+" This script is used by CI"
+
+" Install vim-plug
+let data_dir = has('nvim') ? stdpath('data') . '/site' : '~/.vim'
+if empty(glob(data_dir . '/autoload/plug.vim'))
+  silent execute '!curl -fLo '.data_dir.'/autoload/plug.vim --create-dirs  https://raw.githubusercontent.com/junegunn/vim-plug/master/plug.vim'
+  autocmd VimEnter * PlugInstall --sync | source $MYVIMRC
+endif
+
+call plug#begin()
+
+" Install treesitter
+Plug 'nvim-treesitter/nvim-treesitter', {'do': ':TSUpdate'}
+
+" Install our local workspace
+require("nvim-treesitter.parsers").get_parser_configs().just = {
+  install_info = {
+    url = "${{ github.workspace }}/tree-sitter-just", -- local path or git repo
+    files = { "src/parser.c", "src/scanner.c" },
+    branch = "main",
+    -- use_makefile = true -- this may be necessary on MacOS (try if you see compiler errors)
+  },
+  maintainers = { "@IndianBoy42" },
+}
+
+call plug#end()
+
+lua <<ENDLUA
+
+require'nvim-treesitter.configs'.setup {
+  ensure_installed = {"just"},
+  highlight = {enable = true},
+  indent = {enable = true},
+}
+
+require"tree-sitter-just".setup {}
+
+ENDLUA