diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5998cf45..e9889aa4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -61,7 +61,20 @@ jobs: - name: Fetch Poky & Docs if: steps.cache-poky.outputs.cache-hit != 'true' - run: npm run fetch + run: npm run fetch:poky && npm run fetch:docs + + - name: Cache build Wasm files + id: cache-wasm + uses: actions/cache@v4 + with: + path: | + server/tree-sitter-bash.wasm + server/tree-sitter-bitbake.wasm + key: cache-${{ hashFiles('server/tree-sitter-bash.info', 'server/tree-sitter-bitbake.info') }} + + - name: Fetch Wasm files + if: steps.cache-wasm.outputs.cache-hit != 'true' + run: npm run fetch:wasm - name: Run Linter run: npm run lint diff --git a/.gitignore b/.gitignore index 6acf8817..3df032d6 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ coverage/ resources/ integration-tests/project-folder/build* **/*.vsix + +*.wasm diff --git a/README-DEVELOPER.md b/README-DEVELOPER.md index ee6f61a5..ef0feb4b 100644 --- a/README-DEVELOPER.md +++ b/README-DEVELOPER.md @@ -15,39 +15,48 @@ For more information regarding the Extension Marketplace, please see the [offici ## Manual installation -Manual installation takes place in two steps. The code must be installed via `npm` and subsequently built within Visual Studio Code. Before performing these steps, please ensure you have cloned this repository. +Once compiled, the extension can either be launched in VS Code's debug mode or built into a VSIX file and installed. -### Commands +### Compile -To install the dependencies: +1. Install the dependencies: ``` sh npm install +npm run fetch:wasm # Download and build Wasm dependencies +npm run fetch:docs # Download Yocto's doc. Required for hints on hover ``` -To compile the typescript files: +2. Compile the Typescript files: ``` sh npm run compile ``` -To clean up the project (This deletes node_modules): -``` sh -npm run clean -``` -For more commands, refer to the `script` section in the root `package.json`. -## Debugging +### Debug Press `F5` or navigate to the debug section on the left of the VS Code and select the client or server to to launch the debug client. -## Testing +### Install +1. Build the VSIX file: +``` sh +npm run package +``` +2. Install the VSIX file. See [VS Code's documentation](https://code.visualstudio.com/docs/editor/extension-marketplace#_install-from-a-vsix). -BitBake and Yocto docs are required for some features to work. They need to be fetched before testing and development: +### Clean up +This deletes the Wasm files, the Yocto's doc, Poky, node_modules, and the compiled files: ``` sh -npm run fetch:docs +npm run clean ``` -Similar for the command that fetches poky, it needs to be run before running the integration tests: + +## Testing + +To run all the tests: +1. See [compilation](#compile) steps. +2. Fetch Poky (required for the integration tests): ``` sh npm run fetch:poky ``` -A wrapper npm script allows running several kinds of tests. To run all tests, use: +2. Run all the tests: ``` sh +# Note the integration tests require 'npm run compile' to be executed every time the Typescript files are modified. npm test ``` All the tests mentionned are run in our GitHub CI. @@ -82,9 +91,13 @@ These tests allow running the BitBake extension in a live VSCode environment. See [the individual integration tests README](integration-tests/README.md). ## Tree-sitter -This extension uses tree-sitter to parse the documents. The `.wasm` file used for creating the parser is generated from latest release at [here](https://github.com/amaanq/tree-sitter-bitbake). +This extension uses [tree-sitter-bitbake](https://github.com/tree-sitter-grammars/tree-sitter-bitbake) and [tree-sitter-bash](https://github.com/tree-sitter/tree-sitter-bash) to parse the BitBake documents. They are installed with `npm run fetch:wasm`. The versions of tree-sitter-bitbake and tree-sitter-bash are documented in [server/tree-sitter-bitbake.info](server/tree-sitter-bitbake.info) and [server/tree-sitter-bash.info](server/tree-sitter-bash.info) respectively, along with the versions of the tree-sitter-cli that have to be used. + +To update the .info files with the latest versions of tree-sitter-bitbake and tree-sitter-bash, it is recommended to use the scripts [scripts/update-tree-sitter-bitbake-wasm.sh](scripts/update-tree-sitter-bitbake-wasm.sh) and [scripts/update-tree-sitter-bash-wasm.sh](scripts/update-tree-sitter-bash-wasm.sh). The GitHub workflow [update-tree-sitter-wasm-file.yml](.github/workflows/update-tree-sitter-wasm-file.yml) is already responsible for doing it automatically. + +After updating the .info files, it is required to call `npm run fetch:wasm` in order to rebuild the Wasm files. -For more information about the tree-sitter and its CLI, Check out the offical [site](https://tree-sitter.github.io/tree-sitter/) and [npm page](https://www.npmjs.com/package/tree-sitter-cli) +For more information about the tree-sitter and its CLI, Check out the official [site](https://tree-sitter.github.io/tree-sitter/) and [npm page](https://www.npmjs.com/package/tree-sitter-cli) ## Publishing diff --git a/package.json b/package.json index 70322e57..c4a4de81 100644 --- a/package.json +++ b/package.json @@ -870,14 +870,15 @@ }, "scripts": { "postinstall": "cd server && npm install && cd ../client && npm install && cd ..", - "fetch": "npm run fetch:poky && npm run fetch:docs", + "fetch": "npm run fetch:poky && npm run fetch:docs && npm run fetch:wasm", "fetch:docs": "sh scripts/fetch-docs.sh", "fetch:poky": "sh scripts/fetch-poky.sh", + "fetch:wasm": "bash ./scripts/build-tree-sitter-wasm.sh", "compile": "tsc -b", "watch": "tsc -b -w && npm run postcompile", "vscode:prepublish": "cd server && npm install --omit=dev && cd ../client && npm install --omit=dev && cd ..", "package": "vsce package", - "clean:server": "rm -fr ./server/node_modules ./server/out ./server/resources server/tsconfig.tsbuildinfo server/.vscode-test server/*.tgz", + "clean:server": "rm -fr ./server/node_modules ./server/out ./server/resources server/tsconfig.tsbuildinfo server/.vscode-test server/*.tgz ./server/tree-sitter-bash.wasm ./server/tree-sitter-bitbake.wasm", "clean:client": "rm -fr ./client/node_modules ./client/out client/tsconfig.tsbuildinfo client/.vscode-test", "clean:lib": "rm -fr ./lib/node_modules", "clean": "npm run clean:lib && npm run clean:server && npm run clean:client && rm -fr node_modules integration-tests/out integration-tests/project-folder/build* .vscode-test .eslintcache resources coverage ./out *.vsix", @@ -888,8 +889,8 @@ "test:watch": "jest --watchAll", "test:grammar": "vscode-tmgrammar-test ./client/test/grammars/test-cases/*.bb", "snap-grammar": "vscode-tmgrammar-snap ./client/test/grammars/snaps/*.bb -u", - "dev:vsix": "npm run clean && npm install && npm run fetch:docs && npm run compile && npm run package", - "dev:npm": "npm run clean && npm install && npm run fetch:docs && cd server && npm pack" + "dev:vsix": "npm run clean && npm install && npm run fetch:wasm && npm run fetch:docs && npm run compile && npm run package", + "dev:npm": "npm run clean && npm install && npm run fetch:wasm && npm run fetch:docs && cd server && npm pack" }, "devDependencies": { "@types/glob": "^8.1.0", @@ -919,3 +920,4 @@ "baseImageUrl": "https://github.com/yoctoproject/vscode-bitbake/raw/HEAD/" } } + diff --git a/scripts/build-tree-sitter-wasm.sh b/scripts/build-tree-sitter-wasm.sh new file mode 100644 index 00000000..9ac5b4d3 --- /dev/null +++ b/scripts/build-tree-sitter-wasm.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Inspired by bash-language-server under MIT license +# Reference: https://github.com/bash-lsp/bash-language-server/blob/8c42218c77a9451b308839f9a754abde901323d5/scripts/upgrade-tree-sitter.sh + +set -euox pipefail + +update-tree-sitter-wasm() { + local name=$1 + + cd "$(dirname "$(readlink -f "$0")")/.." + + cd server + info_file="$name.info" + api_url=$(sed -n '1p' "$info_file" | cut -d '"' -f 2) + owner_repo=$(echo "$api_url" | sed -E 's|https://api.github.com/repos/([^/]+/[^/]+).*|\1|') + commit_hash=$(echo "$api_url" | sed -E 's|.*/commits/([^/]+)|\1|') + tarball_url="https://github.com/$owner_repo/tarball/$commit_hash" + tree_sitter_cli_version=$(sed -n '2p' "$info_file" | cut -d '"' -f 2) + npm install -D "tree-sitter-cli@$tree_sitter_cli_version" "$tarball_url" + npx tree-sitter build --wasm "node_modules/$name" + npm uninstall tree-sitter-cli "$name" +} + +update-tree-sitter-wasm "tree-sitter-bash" +update-tree-sitter-wasm "tree-sitter-bitbake" diff --git a/scripts/update-tree-sitter-wasm.sh b/scripts/update-tree-sitter-wasm.sh index 310eea5a..e714ab1e 100755 --- a/scripts/update-tree-sitter-wasm.sh +++ b/scripts/update-tree-sitter-wasm.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -set -e # Inspired by bash-language-server under MIT license # Reference: https://github.com/bash-lsp/bash-language-server/blob/8c42218c77a9451b308839f9a754abde901323d5/scripts/upgrade-tree-sitter.sh @@ -14,11 +13,6 @@ update-tree-sitter-wasm() { cd "$(dirname "$(readlink -f "$0")")/.." cd server - npm install -D tree-sitter-cli "https://github.com/$repo_owner/$repo_name" - npx tree-sitter build-wasm "node_modules/$repo_name" - curl "https://api.github.com/repos/$repo_owner/$repo_name/commits/master" | jq .commit.url > "$info_file" - echo "tree-sitter-cli $(cat package.json | jq '.devDependencies["tree-sitter-cli"]')" >> "$info_file" - - npm uninstall tree-sitter-cli "$repo_name" + echo "tree-sitter-cli $(npm show tree-sitter-cli version)" >> "$info_file" } diff --git a/server/tree-sitter-bash.wasm b/server/tree-sitter-bash.wasm deleted file mode 100755 index 89280733..00000000 Binary files a/server/tree-sitter-bash.wasm and /dev/null differ diff --git a/server/tree-sitter-bitbake.info b/server/tree-sitter-bitbake.info index ddf918c5..0a0ec215 100644 --- a/server/tree-sitter-bitbake.info +++ b/server/tree-sitter-bitbake.info @@ -1,2 +1,2 @@ "https://api.github.com/repos/idillon-sfl/tree-sitter-bitbake/git/commits/c69337c5c8f374613fb13f2dc8738e7827353084" -tree-sitter-cli "^0.21.0" +tree-sitter-cli "^0.22.6" diff --git a/server/tree-sitter-bitbake.wasm b/server/tree-sitter-bitbake.wasm deleted file mode 100755 index d2652611..00000000 Binary files a/server/tree-sitter-bitbake.wasm and /dev/null differ