Skip to content

Commit

Permalink
Attempt to get some sort of CI for NeoVim specifically
Browse files Browse the repository at this point in the history
  • Loading branch information
@tgross35
tgross35 committed Mar 4, 2024
1 parent 43f2c5e commit fcedb31
Show file tree
Hide file tree
Showing 4 changed files with 212 additions and 61 deletions.
12 changes: 11 additions & 1 deletion .github/workflows/ci.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,12 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: 18
- run: pip install ruff
- name: Install dependencies
run: |
pip install ruff black
sudo apt-get update
sudo apt-get install -y luarocks
sudo luarocks install luacheck
- name: Get npm cache directory
id: npm-cache-dir
shell: bash
Expand Down Expand Up @@ -58,6 +63,11 @@ jobs:
path: ${{ steps.npm-cache-dir.outputs.dir }}
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
restore-keys: ${{ runner.os }}-node-
- name: Setup nvim (requires sudo)
run: |
printenv
JUST=$(which just)
sudo "$JUST" --verbose ci-setup-nvim
- run: just setup --locked
- name: Configure
run: just --verbose configure-tree-sitter
Expand Down
222 changes: 162 additions & 60 deletions justfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,30 +7,66 @@ obj_dir := target / "obj"
debug_out := bin_dir / "debug.out"
fuzz_out := bin_dir / "fuzz.out"

ts_path := justfile_directory() / "repositories" / "tree-sitter"
downloads_path := justfile_directory() / "repositories"

ts_path := downloads_path / "tree-sitter"
ts_repo := "https://github.com/tree-sitter/tree-sitter"
ts_sha := "1c55abb5308fe3891da545662e5df7ba28ade275" # v0.21.0

just_path := justfile_directory() / "repositories" / "just"
just_path := downloads_path / "just"
just_repo := "https://github.com/casey/just.git"
just_sha := "a2ff42e6c37ba5c429d444f3a18d3633e59f9a34" # 1.24.0

nvim_ts_path := downloads_path / "nvim-treesitter"
nvim_ts_repo := "https://github.com/nvim-treesitter/nvim-treesitter.git"
nvim_ts_sha := "f197a15b0d1e8d555263af20add51450e5aaa1f0" # v0.9.2

include_args := "-Isrc/ -I" + ts_path + "/lib/include -Inode_modules/nan"
general_cflags := "-Wall -Werror --pedantic -Wno-format-pedantic"

fuzzer_flags := env("FUZZER_FLAGS", "-fsanitize=fuzzer,address,undefined")
fuzz_time := env("FUZZ_TOTAL_TIME", "1200")
in_ci := env("CI", "0")

# Source files needed to build a parser
parser_sources := src + "/scanner.c " + src + "/parser.c " + ts_path + "/lib/src/lib.c"

base_cache_key := sha256_file(src / "scanner.c") + sha256_file(src / "parser.c") + sha256(parser_sources) + sha256(include_args) + sha256(general_cflags) + sha256(fuzzer_flags)

verbose_flag := if env("CI", "") == "1" { "--verbose" } else { "" }

# `timeout` is not available on all platforms, but perl often is. This needs a
# bash shell.
make_timeout_fn := '''timeout () { perl -e 'alarm shift; exec @ARGV' "$@"; }'''
verbose_flag := if in_ci != "0" { "--verbose" } else { "" }
run_if_installed := "just " + verbose_flag + " _run-if-installed "

format_queries_url := "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/63ca90eaa3ce1cc668add8828a9e3d6728dbbdf1/scripts/format-queries.lua"
format_queries_sha := "a37344c87a0b9affa1d46b117e48442a205845776c4cdac31c57f591770cd522"
format_queries_fname := downloads_path / "format-queries.lua"

check_queries_url := "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/63ca90eaa3ce1cc668add8828a9e3d6728dbbdf1/scripts/check-queries.lua"
check_queries_sha := "e4d7888aae1656e2a103f5d868c4f155eca20e554605bd3e6d1c2e8c8e4541f8"
check_queries_fname := downloads_path / "check-queries.lua"

nvim_tag := "v0.9.5"
nvim_fsfx := if os() == "linux" {
"linux64.tar.gz"
} else if os() == "macos" {
"macos.tar.gz"
} else if os() == "windows" {
"win64.zip"
} else {
error("unsupported platform")
}
nvim_sha := if os() == "linux" {
"44ee395d9b5f8a14be8ec00d3b8ead34e18fe6461e40c9c8c50e6956d643b6ca"
} else if os() == "macos" {
"19d2366e0d6da001583bd0b8a3db59f69ce3dda5fa41f3064c6778cef3edd34c"
} else if os() == "windows" {
"de6dc1f0edb45f5f225ee24ce80a4fcbc3a337932037e98ae143975fca2556bf"
} else {
error("unsupported platform")
}
nvim_url := "https://github.com/neovim/neovim/releases/download/" + nvim_tag + "/nvim-" + nvim_fsfx
nvim_default_path := downloads_path / "nvim"
nvim_install_path := "$HOME/.local"
nvim_download_fname := downloads_path / "nvim-linux64.tar.gz"

# Files that should parse with errors but not crash
errors_expected := '''
Expand All @@ -50,6 +86,23 @@ no_just_parsing := '''
default:
just --list

# Only run a command if the tool is installed
_run-if-installed CMD *ARGS="":
#!/bin/sh
set -eau

if command -v {{ CMD }} > /dev/null; then
echo "Running '{{ CMD }} {{ ARGS }}'" 1>&2
{{ CMD }} {{ ARGS }}
else
if [ '{{ in_ci }}' != 0 ]; then
echo 'NOT FOUND: {{ CMD }} required for CI'
exit 1
fi

echo "NOT FOUND: {{ CMD }}. Skipping check (this will get verified in CI)."
fi

# Install needed packages and make sure tools are setup
setup *npm-args:
#!/bin/sh
Expand All @@ -71,46 +124,49 @@ setup *npm-args:
check_installed clang
check_installed clang-tidy
check_installed clang-format
check_installed nvim

if which npm > /dev/null; then
npm install --include=dev {{ npm-args }}
else
echo "npm not found: skipping install"
fi
{{ run_if_installed }} npm install --include=dev {{ npm-args }}

# Lint with more minimal dependencies that can be run during pre-commit
_lint-min: _clone-repo-tree-sitter configure-compile-database
npm run lint:check
@{{ run_if_installed }} npm run lint:check
git ls-files '**.c' | grep -v 'parser\.c' | \
xargs -IFNAME sh -c 'echo "\nchecking file FNAME" && clang-tidy FNAME'
xargs -IFNAME sh -c \
'echo "\nchecking file FNAME" && {{ run_if_installed }} clang-tidy FNAME'

# Run the linter for JS, C, Cargo, and Python. Requires clang-tidy, clippy, and ruff.
# Run the linter for JS, C, Cargo, and Python. Will skip tools that are not installed.
lint: _lint-min
cargo clippy
ruff .
@{{ run_if_installed }} cargo clippy
@{{ run_if_installed }} ruff .
@{{ run_if_installed }} luacheck .

_out-dirs:
mkdir -p "{{ bin_dir }}"
mkdir -p "{{ obj_dir }}"

alias fmt := format

# Autoformat code. Requires Cargo, clang-format, and black.
format: configure-compile-database
# Autoformat code. Requires Cargo, clang-format and black are optional
format: configure-compile-database \
(_ensure-downloaded format_queries_url format_queries_sha format_queries_fname)
npm run format:write
git ls-files '**.c' | grep -v 'parser\.c' | \
git ls-files '**.c' | grep -Fv 'parser.c' | \
xargs -IFNAME sh -c \
'echo "\nformatting 'FNAME'" && clang-format -i FNAME --verbose'
cargo fmt
black .
'echo "\nformatting 'FNAME'" && \
{{ run_if_installed }} clang-format -i FNAME --verbose'
{{ run_if_installed }} cargo fmt
{{ run_if_installed }} black .

# Check formatting without editing
format-check: configure-compile-database
npm run format:check
git ls-files '**.c' | grep -v 'parser\.c' | \
git ls-files '**.c' | grep -Fv 'parser.c' | \
xargs -IFNAME sh -c \
'echo "\nchecking formatting for 'FNAME'" && clang-format FNAME | diff -up - FNAME'
cargo fmt --check
'echo "\nchecking formatting for 'FNAME'" && \
{{ run_if_installed }} clang-format FNAME | diff -up - FNAME'
{{ run_if_installed }} cargo fmt --check
{{ run_if_installed }} black . --check

# Generate the parser
gen *extra-args:
Expand All @@ -120,10 +176,7 @@ gen *extra-args:
# Run formatting only on generated files
npx prettier --write src/

# Only clang-format if it is available
which clang-format > /dev/null && \
clang-format -i src/parser.c || \
echo "skipping clang-format"
@{{ run_if_installed }} clang-format -i src/parser.c

alias t := test

Expand All @@ -133,10 +186,21 @@ test *ts-test-args: gen
just {{ verbose_flag }} test-parse-highlight
just {{ verbose_flag }} verify-no-error-tests

if command -v nvim > /dev/null; then \
just {{ verbose_flag }} test-nvim; \
else \
echo "NeoVim not found; skipping tests"; \
fi

echo '\nRunning Cargo tests'

# FIXME: xfail Rust CI on Windows because we are getting STATUS_DLL_NOT_FOUND
{{ if os_family() + env("CI", "1") == "windows1" { "echo skipping tests on Windows" } else { "cargo test" } }}
{{ if os_family() + env("CI", "1") == "windows1" { \
"echo skipping tests on Windows" \
} else { \
"cargo test" \
} }}


# Verify that tree-sitter can parse and highlight all files in the repo. Requires a tree-sitter configuration.
test-parse-highlight: _clone-repo-just
Expand Down Expand Up @@ -242,6 +306,10 @@ test-parse-highlight: _clone-repo-just
verify-no-error-tests:
! grep -nr -C4 -E '(ERROR|MISSING|UNEXPECTED)' test

test-nvim: _clone-repo-nvim-treesitter
@echo "Running NeoVim tests"
# FIXME: do something

# Helper to rebuild helix grammars
hx-build:
hx --grammar build
Expand All @@ -259,7 +327,7 @@ configure-tree-sitter:
else:
shell = False

cfg_fname = r"""{{ config_directory() / "tree-sitter" / "config.json" }}"""
cfg_fname = r"""{ config_directory() / "tree-sitter" / "config.json" }}"""
if not os.path.isfile(cfg_fname):
sp.run(["npx", "tree-sitter", "init-config"], check=True, shell=shell)

Expand All @@ -274,30 +342,6 @@ configure-tree-sitter:

f.truncate()

# Run lint and check formatting
ci-codestyle: lint format-check

# Make sure that files have not changed
ci-validate-generated-files exit-code="1":
#!/bin/sh
set -eaux

git tag ci-tmp-pre-updates

just {{ verbose_flag }} gen

failed=false
git diff ci-tmp-pre-updates --exit-code || failed=true
git tag -d ci-tmp-pre-updates

if ! [ "$failed" = "false" ]; then
echo '::warning::Generated files are out of date!'
echo '::warning::run `just gen` and commit the changes'

# We use an exit code so that we can use this as either a warning or error
exit {{ exit-code }}
fi

# Run a subset of CI checks before committing.
pre-commit: _lint-min format-check

Expand All @@ -320,7 +364,7 @@ _clone-repo url path sha:

if [ ! -d '{{ path }}' ]; then
echo "Cloning {{ url }}"
git clone '{{ url }}' '{{ path }}' --depth=100
git clone '{{ url }}' '{{ path }}' --depth=500
fi

actual_sha=$(git -C '{{ path }}' rev-parse HEAD)
Expand All @@ -330,11 +374,21 @@ _clone-repo url path sha:
git -C '{{ path }}' reset --hard '{{ sha }}'
fi

# Clone the tree-sitter repo
# Helpers to clone specific repos
_clone-repo-tree-sitter: (_clone-repo ts_repo ts_path ts_sha)

# Clone the just repo
_clone-repo-just: (_clone-repo just_repo just_path just_sha)
_clone-repo-nvim-treesitter: (_clone-repo nvim_ts_repo nvim_ts_path nvim_ts_sha)

_ensure-downloaded URL SHA OUTPUT_FNAME:
#!/bin/sh
set -eau
# Exit if already installed
echo '{{ SHA }} {{ OUTPUT_FNAME }}' | shasum -c && exit 0

set -x
mkdir -p '{{ parent_directory(OUTPUT_FNAME) }}'
curl -L '{{ URL }}' -o '{{ OUTPUT_FNAME }}'
echo '{{ SHA }} {{ OUTPUT_FNAME }}' | shasum -c

# Build a simple debug executable
debug-build: _clone-repo-tree-sitter _out-dirs
Expand Down Expand Up @@ -411,3 +465,51 @@ configure-compile-database:

with open("compile_commands.json", "w") as f:
json.dump(results, f, indent=4)

# Set up NeoVim for use in CI
ci-setup-nvim:
#!/bin/sh
set -eaux

# Exit if installed
if ! command -v nvim; then
if [ {{ in_ci }} != '1' ]; then
echo "refusing to install nvim when not in CI"
exit 1
fi

mkdir -p "{{ nvim_install_path }}"
just {{ verbose_flag }} _ensure-downloaded '{{ nvim_url }}' \
'{{ nvim_sha }}' '{{ nvim_download_fname }}'

tar -vxzf "{{ nvim_download_fname }}" -C "{{ nvim_install_path }}" \
--strip-components=1

ln -s "{{ nvim_install_path }}/bin/nvim" /usr/bin/nvim
fi

nvim --version

# Run lint and check formatting
ci-codestyle: lint format-check

# Make sure that files have not changed
ci-validate-generated-files exit-code="1":
#!/bin/sh
set -eaux

git tag ci-tmp-pre-updates

just {{ verbose_flag }} gen

failed=false
git diff ci-tmp-pre-updates --exit-code || failed=true
git tag -d ci-tmp-pre-updates

if ! [ "$failed" = "false" ]; then
echo '::warning::Generated files are out of date!'
echo '::warning::run `just gen` and commit the changes'

# We use an exit code so that we can use this as either a warning or error
exit {{ exit-code }}
fi
1 change: 1 addition & 0 deletions lua/tree-sitter-just/init.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ function M.setup(arg)
parser_config.just = {
install_info = {
url = arg["local"] and join_paths(
-- luacheck: ignore
vim.fn.stdpath("data"),
"site",
"pack",
Expand Down
Loading

0 comments on commit fcedb31

Please sign in to comment.