diff --git a/.github/Makefile b/.github/Makefile index 82cb516f6d4..e494748be53 100644 --- a/.github/Makefile +++ b/.github/Makefile @@ -6,13 +6,14 @@ all: workflows/nightly.yml \ workflows/release.yml \ workflows/testing.yml \ workflows/dryrun.yml \ + workflows/ls-nightly.yml \ workflows/tests.yml \ workflows/tests-managed-pg.yml \ workflows/tests-ha.yml \ workflows/tests-pg-versions.yml \ workflows/tests-patches.yml -workflows/%.yml: workflows.src/%.tpl.yml workflows.src/%.targets.yml workflows.src/build.inc.yml +workflows/%.yml: workflows.src/%.tpl.yml workflows.src/%.targets.yml workflows.src/build.inc.yml workflows.src/ls-build.inc.yml $(ROOT)/workflows.src/render.py $* $*.targets.yml workflows.src/tests.tpl.yml: workflows.src/tests.inc.yml diff --git a/.github/workflows.src/ls-build.inc.yml b/.github/workflows.src/ls-build.inc.yml new file mode 100644 index 00000000000..40c6e227495 --- /dev/null +++ b/.github/workflows.src/ls-build.inc.yml @@ -0,0 +1,443 @@ +<% macro workflow(targets, publications, subdist="", publish_all=False) %> + prep: + runs-on: ubuntu-latest + outputs: + branch: ${{ steps.whichver.outputs.branch }} +<% if subdist == "nightly" %> +<% for tgt in targets.linux + targets.macos %> + if_<< tgt.name.replace('-', '_') >>: ${{ steps.scm.outputs.if_<< tgt.name.replace('-', '_') >> }} +<% endfor %> +<% endif %> + steps: + - uses: actions/checkout@v4 + + - name: Determine package version + shell: bash + run: | + branch=${GITHUB_REF#refs/heads/} + echo branch="${branch}" >> $GITHUB_OUTPUT + id: whichver + +<% if subdist == "nightly" %> + - name: Determine SCM revision + id: scm + shell: bash + run: | + rev=$(git rev-parse HEAD) + jq_filter='.packages[] | select(.basename == "edgedb-server") | select(.architecture == $ARCH) | .version_details.metadata.scm_revision | . as $rev | select(($rev != null) and ($REV | startswith($rev)))' +<% for tgt in targets.linux %> + val=true +<% if tgt.family == "debian" %> + idx_file=<< tgt.platform_version >>.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/apt/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "<< tgt.arch >>" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing << tgt.name >>' + val=false + fi +<% elif tgt.family == "redhat" %> + idx_file=el<< tgt.platform_version >>.nightly.json + if [ ! -e "/tmp/$idx_file" ]; then + curl -s https://packages.edgedb.com/rpm/.jsonindexes/$idx_file > /tmp/$idx_file + fi + out=$(cat /tmp/$idx_file | jq -r --arg REV "$rev" --arg ARCH "<< tgt.arch >>" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing << tgt.name >>' + val=false + fi +<% elif tgt.family == "generic" %> + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/<< tgt.platform_version >>-unknown-linux-<< "{}".format(tgt.platform_libc) if tgt.platform_libc else "gnu" >>.nightly.json | jq -r --arg REV "$rev" --arg ARCH "<< tgt.arch >>" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing << tgt.name >>' + val=false + fi +<% endif %> + echo if_<< tgt.name.replace('-', '_') >>="$val" >> $GITHUB_OUTPUT +<% endfor %> +<% for tgt in targets.macos %> + val=true +<% if tgt.platform == "macos" %> + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/<< tgt.platform_version >>-apple-darwin.nightly.json | jq -r --arg REV "$rev" --arg ARCH "<< tgt.arch >>" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing << tgt.platform >>-<< tgt.platform_version >>' + val=false + fi +<% elif tgt.platform == "win" %> + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/<< tgt.platform_version >>-pc-windows-msvc.nightly.json | jq -r --arg REV "$rev" --arg ARCH "<< tgt.arch >>" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing << tgt.platform >>-<< tgt.platform_version >>' + val=false + fi +<% endif %> + echo if_<< tgt.name.replace('-', '_') >>="$val" >> $GITHUB_OUTPUT +<% endfor %> +<% endif %> + +<%- for tgt in targets.linux %> +<%- set plat_id = tgt.platform + ("{}".format(tgt.platform_libc) if tgt.platform_libc else "") + ("-{}".format(tgt.platform_version) if tgt.platform_version else "") %> + + build-<< tgt.name >>: + runs-on: << tgt.runs_on if tgt.runs_on else "ubuntu-latest" >> + needs: prep +<% if subdist == "nightly" %> + if: needs.prep.outputs.if_<< tgt.name.replace('-', '_') >> == 'true' +<% endif %> + + steps: + - name: Build + uses: edgedb/edgedb-pkg/integration/linux/build/<< plat_id >>@language-server + env: + SRC_REF: "${{ needs.prep.outputs.branch }}" + PKG_REVISION: "" + PACKAGE: edgedbpkg.edgedb-ls:EdgeDBLanguageServer + <%- if subdist != "" %> + PKG_SUBDIST: "<< subdist >>" + <%- endif %> + PKG_PLATFORM: "<< tgt.platform >>" + PKG_PLATFORM_VERSION: "<< tgt.platform_version >>" + EXTRA_OPTIMIZATIONS: "true" + <%- if subdist != "nightly" %> + BUILD_IS_RELEASE: "true" + <%- endif %> + <%- if tgt.family == "generic" %> + BUILD_GENERIC: true + <%- endif %> + <%- if tgt.platform_libc %> + PKG_PLATFORM_LIBC: "<< tgt.platform_libc >>" + <%- endif %> + METAPKG_GIT_CACHE: disabled + + - uses: actions/upload-artifact@v4 + with: + name: builds-<< tgt.name >> + path: artifacts/<< plat_id >> +<%- endfor %> + +<%- for tgt in targets.macos %> +<%- set plat_id = tgt.platform + ("{}".format(tgt.platform_libc) if tgt.platform_libc else "") + ("-{}".format(tgt.platform_version) if tgt.platform_version else "") %> + + build-<< tgt.name >>: + runs-on: << tgt.runs_on if tgt.runs_on else "macos-latest" >> + needs: prep +<% if subdist == "nightly" %> + if: needs.prep.outputs.if_<< tgt.name.replace('-', '_') >> == 'true' +<% endif %> + + steps: + - uses: actions/checkout@v4 + with: + repository: edgedb/edgedb-pkg + ref: master + path: edgedb-pkg + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248 + if: << 'false' if tgt.runs_on and 'self-hosted' in tgt.runs_on else 'true' >> + with: + components: "cargo,rustc,rust-std" + toolchain: "stable" + targets: "<< tgt.arch >>-apple-darwin" + + - name: Set up Python + uses: actions/setup-python@v5 + if: << 'false' if tgt.runs_on and 'self-hosted' in tgt.runs_on else 'true' >> + with: + python-version: "3.x" + + - name: Set up NodeJS + uses: actions/setup-node@v4 + if: << 'false' if tgt.runs_on and 'self-hosted' in tgt.runs_on else 'true' >> + with: + node-version: '20' + + - name: Install dependencies + if: << 'false' if tgt.runs_on and 'self-hosted' in tgt.runs_on else 'true' >> + run: | + env HOMEBREW_NO_AUTO_UPDATE=1 brew install libmagic + + - name: Build + env: + SRC_REF: "${{ needs.prep.outputs.branch }}" + PACKAGE: edgedbpkg.edgedb-ls:EdgeDBLanguageServer + <%- if subdist != "nightly" %> + BUILD_IS_RELEASE: "true" + <%- endif %> + PKG_REVISION: "" + <%- if subdist != "" %> + PKG_SUBDIST: "<< subdist >>" + <%- endif %> + PKG_PLATFORM: "<< tgt.platform >>" + PKG_PLATFORM_VERSION: "<< tgt.platform_version >>" + PKG_PLATFORM_ARCH: "<< tgt.arch if tgt.arch else '' >>" + EXTRA_OPTIMIZATIONS: "true" + METAPKG_GIT_CACHE: disabled + <%- if tgt.family == "generic" %> + BUILD_GENERIC: true + <%- endif %> + run: | + edgedb-pkg/integration/macos/build.sh + + - uses: actions/upload-artifact@v4 + with: + name: builds-<< tgt.name >> + path: artifacts/<< plat_id >> +<%- endfor %> + +<%- if publish_all %> + collect: + needs: + <%- for tgt in targets.linux + targets.macos %> + - build-<< tgt.name >> + <%- endfor %> + runs-on: ubuntu-latest + steps: + - run: echo 'All build+tests passed, ready to publish now!' +<%- endif %> + +<%- for tgt in targets.linux %> +<%- set plat_id = tgt.platform + ("{}".format(tgt.platform_libc) if tgt.platform_libc else "") + ("-{}".format(tgt.platform_version) if tgt.platform_version else "") %> +<%- for publish in publications %> + + publish<< publish.suffix>>-<< tgt.name >>: + needs: [<% if publish_all %>collect<% else %>test-<< tgt.name >><% endif %>] + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@v4 + with: + name: builds-<< tgt.name >> + path: artifacts/<< plat_id >> + + - name: Publish + uses: edgedb/edgedb-pkg/integration/linux/upload/linux-x86_64@master + env: + PACKAGE: edgedbpkg.edgedb-ls:EdgeDBLanguageServer + <%- if subdist != "" %> + PKG_SUBDIST: "<< subdist >>" + <%- endif %> + <%- if publish.server != "" %> + PACKAGE_SERVER: << publish.server >> + <%- endif %> + PKG_PLATFORM: "<< tgt.platform >>" + PKG_PLATFORM_VERSION: "<< tgt.platform_version >>" + PKG_PLATFORM_LIBC: "<< tgt.platform_libc >>" + PACKAGE_UPLOAD_SSH_KEY: "${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}" + + check-published<>-<< tgt.name >>: + needs: [publish<< publish.suffix >>-<< tgt.name >>] + runs-on: << tgt.runs_on if tgt.runs_on else "ubuntu-latest" >> + + steps: + - uses: actions/download-artifact@v4 + with: + name: builds-<< tgt.name >> + path: artifacts/<< plat_id >> + + - name: Describe + id: describe + uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master + with: + target: << plat_id >> + + - name: Test Published + uses: edgedb/edgedb-pkg/integration/linux/testpublished/<< plat_id >>@language-server + env: + PACKAGE: edgedbpkg.edgedb-ls:EdgeDBLanguageServer + PKG_NAME: "${{ steps.describe.outputs.name }}" + <%- if subdist != "" %> + PKG_SUBDIST: "<< subdist >>" + <%- endif %> + <%- if publish.server != "" %> + PACKAGE_SERVER: << publish.server >> + <%- endif %> + PKG_PLATFORM: "<< tgt.platform >>" + PKG_PLATFORM_VERSION: "<< tgt.platform_version >>" + PKG_INSTALL_REF: "${{ steps.describe.outputs.install-ref }}" + PKG_VERSION_SLOT: "${{ steps.describe.outputs.version-slot }}" + + outputs: + version-slot: ${{ steps.describe.outputs.version-slot }} + version-core: ${{ steps.describe.outputs.version-core }} + catalog-version: ${{ steps.describe.outputs.catalog-version }} +<%- endfor %> +<%- endfor %> + +<%- if publications %> +<%- for tgt in targets.macos %> +<%- set plat_id = tgt.platform + ("{}".format(tgt.platform_libc) if tgt.platform_libc else "") + ("-{}".format(tgt.platform_version) if tgt.platform_version else "") %> + + publish-<< tgt.name >>: + needs: [<% if publish_all %>collect<% else %>test-<< tgt.name >><% endif %>] + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@v4 + with: + name: builds-<< tgt.name >> + path: artifacts/<< plat_id >> + + - uses: actions/checkout@v4 + with: + repository: edgedb/edgedb-pkg + ref: master + path: edgedb-pkg + + - name: Describe + id: describe + uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master + with: + target: << plat_id >> + + - name: Publish + uses: edgedb/edgedb-pkg/integration/linux/upload/linux-x86_64@master + env: + PACKAGE: edgedbpkg.edgedb-ls:EdgeDBLanguageServer + <%- if subdist != "" %> + PKG_SUBDIST: "<< subdist >>" + <%- endif %> + PKG_PLATFORM: "<< tgt.platform >>" + PKG_PLATFORM_VERSION: "<< tgt.platform_version >>" + PACKAGE_UPLOAD_SSH_KEY: "${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}" +<%- endfor %> +<%- endif %> + +<%- set docker_tgts = targets.linux | selectattr("docker_arch") | list %> +<%- if docker_tgts and publications %> +<%- set pub_outputs = "needs.check-published-" + (docker_tgts|first)["name"] + ".outputs" %> + + publish-docker: + needs: + <%- for tgt in docker_tgts %> + - check-published-<< tgt.name >> + <%- endfor %> + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + repository: edgedb/edgedb-docker + ref: master + path: dockerfile + + - env: + VERSION_SLOT: "${{ << pub_outputs >>.version-slot }}" + VERSION_CORE: "${{ << pub_outputs >>.version-core }}" + PKG_SUBDIST: "<< subdist >>" + id: tags + run: | + set -e + + url='https://registry.hub.docker.com/v2/repositories/edgedb/edgedb/tags?page_size=100' + repo_tags=$( + while [ -n "$url" ]; do + resp=$(curl -L -s "$url") + url=$(echo "$resp" | jq -r ".next") + if [ "$url" = "null" ] || [ -z "$url" ]; then + break + fi + echo "$resp" | jq -r '."results"[]["name"]' + done | grep "^[[:digit:]]\+.*" | grep -v "alpha\|beta\|rc" || : + ) + + tags=( "$VERSION_CORE" ) + + top=$(printf "%s\n%s\n" "$VERSION_CORE" "$repo_tags" \ + | grep "^${VERSION_SLOT}[\.-]" \ + | sort --version-sort --reverse | head -n 1) + if [ "$top" == "$VERSION_CORE" ]; then + tags+=( "$VERSION_SLOT" ) + fi + + if [ -z "$PKG_SUBDIST" ]; then + top=$(printf "%s\n%s\n" "$VERSION_CORE" "$repo_tags" \ + | sort --version-sort --reverse | head -n 1) + if [ "$top" == "$VERSION_CORE" ]; then + tags+=( "latest" ) + fi + fi + + IFS=, + echo "tags=${tags[*]}" >> $GITHUB_OUTPUT + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 # v3 + + - name: Publish Docker Image (docker.io) + uses: elgohr/Publish-Docker-Github-Action@43dc228e327224b2eda11c8883232afd5b34943b # v5 + with: + name: edgedb/edgedb + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + <%- if subdist == "nightly" %> + tags: "nightly,nightly_${{ << pub_outputs >>.version-slot }}_cv${{ << pub_outputs >>.catalog-version }}" + <%- else %> + tags: "${{ steps.tags.outputs.tags }}" + <%- endif %> + workdir: dockerfile + buildargs: version=${{ << pub_outputs >>.version-slot }},exact_version=${{ << pub_outputs >>.version-core }}<% if subdist != "" %>,subdist=<< subdist >><% endif %> + platforms: << docker_tgts|map(attribute="docker_arch")|join(",") >> + + - name: Publish Docker Image (ghcr.io) + uses: elgohr/Publish-Docker-Github-Action@43dc228e327224b2eda11c8883232afd5b34943b # v5 + with: + registry: ghcr.io + name: ${{ github.repository }} + username: "edgedb-ci" + password: ${{ secrets.GITHUB_CI_BOT_TOKEN }} + <%- if subdist == "nightly" %> + tags: "nightly,nightly_${{ << pub_outputs >>.version-slot }}_cv${{ << pub_outputs >>.catalog-version }}" + <%- else %> + tags: "${{ steps.tags.outputs.tags }}" + <%- endif %> + workdir: dockerfile + buildargs: version=${{ << pub_outputs >>.version-slot }},exact_version=${{ << pub_outputs >>.version-core }}<% if subdist != "" %>,subdist=<< subdist >><% endif %> + platforms: << docker_tgts|map(attribute="docker_arch")|join(",") >> +<%- endif %> + + workflow-notifications: + if: failure() && github.event_name != 'pull_request' + name: Notify in Slack on failures + + needs: + - prep + <%- if publish_all %> + - collect + <%- else %> + <%- endif %> + <%- for tgt in targets.linux %> + - build-<< tgt.name >> + - test-<< tgt.name >> + <%- for publish in publications %> + - publish<< publish.suffix>>-<< tgt.name >> + - check-published<< publish.suffix>>-<< tgt.name >> + <%- endfor %> + <%- endfor %> + <%- for tgt in targets.macos %> + - build-<< tgt.name >> + - test-<< tgt.name >> + <%- for publish in publications %> + - publish<< publish.suffix>>-<< tgt.name >> + <%- endfor %> + <%- endfor %> + <%- if docker_tgts and publications %> + - publish-docker + <%- endif %> + runs-on: ubuntu-latest + permissions: + actions: 'read' + steps: + - name: Slack Workflow Notification + uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309 + with: + repo_token: ${{secrets.GITHUB_TOKEN}} + slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}} + name: 'Workflow notifications' + icon_emoji: ':hammer:' + include_jobs: 'on-failure' + +<%- endmacro %> diff --git a/.github/workflows.src/ls-build.targets.yml b/.github/workflows.src/ls-build.targets.yml new file mode 100644 index 00000000000..c465db04915 --- /dev/null +++ b/.github/workflows.src/ls-build.targets.yml @@ -0,0 +1,47 @@ +publications: + - name: prod + suffix: "" + server: sftp://uploader@package-upload.edgedb.net:22/ + +targets: + linux: + - name: linux-x86_64 + arch: x86_64 + platform: linux + platform_version: x86_64 + family: generic + runs_on: [self-hosted, linux, x64] +# - name: linux-aarch64 +# arch: aarch64 +# platform: linux +# platform_version: aarch64 +# family: generic +# runs_on: [self-hosted, linux, arm64] +# - name: linuxmusl-x86_64 +# arch: x86_64 +# platform: linux +# platform_version: x86_64 +# platform_libc: musl +# family: generic +# runs_on: [self-hosted, linux, x64] +# - name: linuxmusl-aarch64 +# arch: aarch64 +# platform: linux +# platform_version: aarch64 +# platform_libc: musl +# family: generic +# runs_on: [self-hosted, linux, arm64] + + macos: [] + # - name: macos-x86_64 + # arch: x86_64 + # platform: macos + # platform_version: x86_64 + # family: generic + # runs_on: [macos-13] + # - name: macos-aarch64 + # arch: aarch64 + # platform: macos + # platform_version: aarch64 + # family: generic + # runs_on: [macos-14] diff --git a/.github/workflows.src/ls-nightly.targets.yml b/.github/workflows.src/ls-nightly.targets.yml new file mode 120000 index 00000000000..2a2e9026830 --- /dev/null +++ b/.github/workflows.src/ls-nightly.targets.yml @@ -0,0 +1 @@ +ls-build.targets.yml \ No newline at end of file diff --git a/.github/workflows.src/ls-nightly.tpl.yml b/.github/workflows.src/ls-nightly.tpl.yml new file mode 100644 index 00000000000..b8a6a2be1da --- /dev/null +++ b/.github/workflows.src/ls-nightly.tpl.yml @@ -0,0 +1,14 @@ +<% from "ls-build.inc.yml" import workflow -%> +name: 'edgedb-ls: Build and Publish Nightly Packages' + +on: + schedule: + - cron: "0 1 * * *" + workflow_dispatch: + inputs: {} + push: + branches: + - nightly + +jobs: + <<- workflow(targets, publications, subdist="nightly") ->> diff --git a/.github/workflows/ls-nightly.yml b/.github/workflows/ls-nightly.yml new file mode 100644 index 00000000000..2948196e633 --- /dev/null +++ b/.github/workflows/ls-nightly.yml @@ -0,0 +1,154 @@ +name: 'edgedb-ls: Build and Publish Nightly Packages' + +on: + schedule: + - cron: "0 1 * * *" + workflow_dispatch: + inputs: {} + push: + branches: + - nightly + +jobs: + prep: + runs-on: ubuntu-latest + outputs: + branch: ${{ steps.whichver.outputs.branch }} + + + if_linux_x86_64: ${{ steps.scm.outputs.if_linux_x86_64 }} + + + steps: + - uses: actions/checkout@v4 + + - name: Determine package version + shell: bash + run: | + branch=${GITHUB_REF#refs/heads/} + echo branch="${branch}" >> $GITHUB_OUTPUT + id: whichver + + + - name: Determine SCM revision + id: scm + shell: bash + run: | + rev=$(git rev-parse HEAD) + jq_filter='.packages[] | select(.basename == "edgedb-server") | select(.architecture == $ARCH) | .version_details.metadata.scm_revision | . as $rev | select(($rev != null) and ($REV | startswith($rev)))' + + val=true + + out=$(curl -s https://packages.edgedb.com/archive/.jsonindexes/x86_64-unknown-linux-gnu.nightly.json | jq -r --arg REV "$rev" --arg ARCH "x86_64" "$jq_filter") + if [ -n "$out" ]; then + echo 'Skip rebuilding existing linux-x86_64' + val=false + fi + + echo if_linux_x86_64="$val" >> $GITHUB_OUTPUT + + + + + build-linux-x86_64: + runs-on: ['self-hosted', 'linux', 'x64'] + needs: prep + + if: needs.prep.outputs.if_linux_x86_64 == 'true' + + + steps: + - name: Build + uses: edgedb/edgedb-pkg/integration/linux/build/linux-x86_64@language-server + env: + SRC_REF: "${{ needs.prep.outputs.branch }}" + PKG_REVISION: "" + PACKAGE: edgedbpkg.edgedb-ls:EdgeDBLanguageServer + PKG_SUBDIST: "nightly" + PKG_PLATFORM: "linux" + PKG_PLATFORM_VERSION: "x86_64" + EXTRA_OPTIMIZATIONS: "true" + BUILD_GENERIC: true + METAPKG_GIT_CACHE: disabled + + - uses: actions/upload-artifact@v4 + with: + name: builds-linux-x86_64 + path: artifacts/linux-x86_64 + + publish-linux-x86_64: + needs: [test-linux-x86_64] + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@v4 + with: + name: builds-linux-x86_64 + path: artifacts/linux-x86_64 + + - name: Publish + uses: edgedb/edgedb-pkg/integration/linux/upload/linux-x86_64@master + env: + PACKAGE: edgedbpkg.edgedb-ls:EdgeDBLanguageServer + PKG_SUBDIST: "nightly" + PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/ + PKG_PLATFORM: "linux" + PKG_PLATFORM_VERSION: "x86_64" + PKG_PLATFORM_LIBC: "" + PACKAGE_UPLOAD_SSH_KEY: "${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}" + + check-published-linux-x86_64: + needs: [publish-linux-x86_64] + runs-on: ['self-hosted', 'linux', 'x64'] + + steps: + - uses: actions/download-artifact@v4 + with: + name: builds-linux-x86_64 + path: artifacts/linux-x86_64 + + - name: Describe + id: describe + uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master + with: + target: linux-x86_64 + + - name: Test Published + uses: edgedb/edgedb-pkg/integration/linux/testpublished/linux-x86_64@language-server + env: + PACKAGE: edgedbpkg.edgedb-ls:EdgeDBLanguageServer + PKG_NAME: "${{ steps.describe.outputs.name }}" + PKG_SUBDIST: "nightly" + PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/ + PKG_PLATFORM: "linux" + PKG_PLATFORM_VERSION: "x86_64" + PKG_INSTALL_REF: "${{ steps.describe.outputs.install-ref }}" + PKG_VERSION_SLOT: "${{ steps.describe.outputs.version-slot }}" + + outputs: + version-slot: ${{ steps.describe.outputs.version-slot }} + version-core: ${{ steps.describe.outputs.version-core }} + catalog-version: ${{ steps.describe.outputs.catalog-version }} + + workflow-notifications: + if: failure() && github.event_name != 'pull_request' + name: Notify in Slack on failures + + needs: + - prep + - build-linux-x86_64 + - test-linux-x86_64 + - publish-linux-x86_64 + - check-published-linux-x86_64 + runs-on: ubuntu-latest + permissions: + actions: 'read' + steps: + - name: Slack Workflow Notification + uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309 + with: + repo_token: ${{secrets.GITHUB_TOKEN}} + slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}} + name: 'Workflow notifications' + icon_emoji: ':hammer:' + include_jobs: 'on-failure' diff --git a/edb/edgeql-parser/src/parser.rs b/edb/edgeql-parser/src/parser.rs index 3ca96ca4ed5..3842206e41c 100644 --- a/edb/edgeql-parser/src/parser.rs +++ b/edb/edgeql-parser/src/parser.rs @@ -558,9 +558,9 @@ fn injection_cost(kind: &Kind) -> u16 { // Manual keyword tweaks to encourage some error messages and discourage others. Keyword(keywords::Keyword( - "delete" | "update" | "migration" | "role" | "global" | "administer", + "delete" | "update" | "migration" | "role" | "global" | "administer" | "future" | "database", )) => 100, - Keyword(keywords::Keyword("insert")) => 20, + Keyword(keywords::Keyword("insert" | "module" | "extension" | "branch")) => 20, Keyword(keywords::Keyword("select" | "property" | "type")) => 10, Keyword(_) => 15, diff --git a/edb/errors/base.py b/edb/errors/base.py index caeb46d7002..ddaa6a7e66c 100644 --- a/edb/errors/base.py +++ b/edb/errors/base.py @@ -181,6 +181,14 @@ def line(self): def col(self): return int(self._attrs.get(FIELD_COLUMN_START, -1)) + @property + def line_end(self): + return int(self._attrs.get(FIELD_LINE_END, -1)) + + @property + def col_end(self): + return int(self._attrs.get(FIELD_COLUMN_END, -1)) + @property def position(self): return int(self._attrs.get(FIELD_POSITION_START, -1)) diff --git a/edb/language_server/__init__.py b/edb/language_server/__init__.py new file mode 100644 index 00000000000..9eab314cb6a --- /dev/null +++ b/edb/language_server/__init__.py @@ -0,0 +1,17 @@ +# +# This source file is part of the EdgeDB open source project. +# +# Copyright 2008-present MagicStack Inc. and the EdgeDB authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/edb/language_server/main.py b/edb/language_server/main.py new file mode 100644 index 00000000000..e971ab1ab23 --- /dev/null +++ b/edb/language_server/main.py @@ -0,0 +1,87 @@ +# +# This source file is part of the EdgeDB open source project. +# +# Copyright 2008-present MagicStack Inc. and the EdgeDB authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from lsprotocol import types as lsp_types + + +from edb.edgeql import parser as qlparser + +from . import parsing as ls_parsing +from . import server as ls_server + + +def main(): + ls = ls_server.EdgeDBLanguageServer() + + @ls.feature( + lsp_types.INITIALIZE, + ) + def init(_params: lsp_types.InitializeParams): + ls.show_message_log('Starting') + qlparser.preload_spec() + ls.show_message_log('Started') + + @ls.feature(lsp_types.TEXT_DOCUMENT_DID_OPEN) + def text_document_did_open(params: lsp_types.DidOpenTextDocumentParams): + document_updated(ls, params.text_document.uri) + + @ls.feature(lsp_types.TEXT_DOCUMENT_DID_CHANGE) + def text_document_did_change(params: lsp_types.DidChangeTextDocumentParams): + document_updated(ls, params.text_document.uri) + + @ls.feature( + lsp_types.TEXT_DOCUMENT_COMPLETION, + lsp_types.CompletionOptions(trigger_characters=[',']), + ) + def completions(params: lsp_types.CompletionParams): + items = [] + + document = ls.workspace.get_text_document(params.text_document.uri) + + if item := ls_parsing.parse_and_suggest(document, params.position): + items.append(item) + + return lsp_types.CompletionList(is_incomplete=False, items=items) + + ls.start_io() + + +def document_updated(ls: ls_server.EdgeDBLanguageServer, doc_uri: str): + # each call to this function should yield in exactly one publish_diagnostics + # for this document + + document = ls.workspace.get_text_document(doc_uri) + ql_ast = ls_parsing.parse(document, ls) + if diagnostics := ql_ast.error: + ls.publish_diagnostics(document.uri, diagnostics, document.version) + return + assert ql_ast.ok + + try: + if isinstance(ql_ast.ok, list): + diagnostics = ls_server.compile(ls, ql_ast.ok) + ls.publish_diagnostics(document.uri, diagnostics, document.version) + else: + ls.publish_diagnostics(document.uri, [], document.version) + except BaseException as e: + ls.show_message_log(f'Internal error: {e}') + ls.publish_diagnostics(document.uri, [], document.version) + + +if __name__ == '__main__': + main() diff --git a/edb/language_server/parsing.py b/edb/language_server/parsing.py new file mode 100644 index 00000000000..d4e2981feda --- /dev/null +++ b/edb/language_server/parsing.py @@ -0,0 +1,149 @@ +# +# This source file is part of the EdgeDB open source project. +# +# Copyright 2008-present MagicStack Inc. and the EdgeDB authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, List, Tuple, Optional, TypeVar, Generic +from dataclasses import dataclass + +from pygls.server import LanguageServer +from pygls.workspace import TextDocument +from lsprotocol import types as lsp_types + + +from edb.edgeql import ast as qlast +from edb.edgeql import tokenizer +from edb.edgeql import parser as qlparser +from edb.edgeql.parser.grammar import tokens as qltokens +import edb._edgeql_parser as rust_parser + + +T = TypeVar('T', covariant=True) +E = TypeVar('E', covariant=True) + + +@dataclass(kw_only=True, slots=True) +class Result(Generic[T, E]): + ok: Optional[T] = None + error: Optional[E] = None + + +def parse( + doc: TextDocument, ls: LanguageServer +) -> Result[List[qlast.Base] | qlast.Schema, List[lsp_types.Diagnostic]]: + sdl = doc.filename.endswith('.esdl') if doc.filename else False + + source, result, productions = _parse_inner(doc.source, sdl) + + if result.errors: + diagnostics = [] + for error in result.errors: + message, span, hint, details = error + + if details: + message += f"\n{details}" + if hint: + message += f"\nHint: {hint}" + (start, end) = tokenizer.inflate_span(source.text(), span) + assert end + + diagnostics.append( + lsp_types.Diagnostic( + range=lsp_types.Range( + start=lsp_types.Position( + line=start.line - 1, + character=start.column - 1, + ), + end=lsp_types.Position( + line=end.line - 1, + character=end.column - 1, + ), + ), + severity=lsp_types.DiagnosticSeverity.Error, + message=message, + ) + ) + + return Result(error=diagnostics) + + # parsing successful + assert isinstance(result.out, rust_parser.CSTNode) + + ast = qlparser._cst_to_ast( + result.out, productions, source, doc.filename + ).val + if sdl: + assert isinstance(ast, qlast.Schema), ast + else: + assert isinstance(ast, list), ast + return Result(ok=ast) + + +def parse_and_suggest( + doc: TextDocument, position: lsp_types.Position +) -> Optional[lsp_types.CompletionItem]: + sdl = doc.filename.endswith('.esdl') if doc.filename else False + + source, result, _productions = _parse_inner(doc.source, sdl) + for error in result.errors: + message: str + message, span, _hint, _details = error + if not message.startswith('Missing keyword '): + continue + (start, end) = tokenizer.inflate_span(source.text(), span) + + if not _position_in_span(position, (start, end)): + continue + + keyword = message.removeprefix('Missing keyword \'')[:-1] + + return lsp_types.CompletionItem( + label=keyword, + kind=lsp_types.CompletionItemKind.Keyword, + ) + return None + + +def _position_in_span(pos: lsp_types.Position, span: Tuple[Any, Any]): + start, end = span + + if pos.line < start.line - 1: + return False + if pos.line > end.line - 1: + return False + if pos.line == start.line - 1 and pos.character < start.column - 1: + return False + if pos.line == end.line - 1 and pos.character > end.column - 1: + return False + return True + + +def _parse_inner( + source_str: str, sdl: bool +) -> Tuple[tokenizer.Source, rust_parser.ParserResult, Any]: + try: + source = tokenizer.Source.from_string(source_str) + except Exception as e: + # TODO + print(e) + raise AssertionError(e) + + start_t = qltokens.T_STARTSDLDOCUMENT if sdl else qltokens.T_STARTBLOCK + start_t_name = start_t.__name__[2:] + tokens = source.tokens() + + result, productions = rust_parser.parse(start_t_name, tokens) + return source, result, productions diff --git a/edb/language_server/server.py b/edb/language_server/server.py new file mode 100644 index 00000000000..ee6f6fb3ee2 --- /dev/null +++ b/edb/language_server/server.py @@ -0,0 +1,166 @@ +# +# This source file is part of the EdgeDB open source project. +# +# Copyright 2008-present MagicStack Inc. and the EdgeDB authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional, List +import dataclasses +import pathlib +import os + +from pygls.server import LanguageServer +from pygls import uris as pygls_uris +from lsprotocol import types as lsp_types + + +from edb import errors + +from edb.edgeql import ast as qlast +from edb.edgeql import compiler as qlcompiler + +from edb.schema import schema as s_schema +from edb.schema import std as s_std +from edb.schema import ddl as s_ddl + +from . import parsing as ls_parsing + + +@dataclasses.dataclass(kw_only=True, slots=True) +class State: + schema: Optional[s_schema.Schema] = None + + std_schema: Optional[s_schema.Schema] = None + + +class EdgeDBLanguageServer(LanguageServer): + state: State + + def __init__(self): + super().__init__('EdgeDB Language Server', 'v0.1') + self.state = State() + + +def compile( + ls: EdgeDBLanguageServer, stmts: List[qlast.Base] +) -> List[lsp_types.Diagnostic]: + diagnostics: List[lsp_types.Diagnostic] = [] + + if not stmts: + return diagnostics + + schema = _get_schema(ls) + if not schema: + return diagnostics + + for ql_stmt in stmts: + + try: + if isinstance(ql_stmt, qlast.DDLCommand): + schema, _delta = s_ddl.delta_and_schema_from_ddl( + ql_stmt, schema=schema, modaliases={None: 'default'} + ) + + elif isinstance(ql_stmt, (qlast.Command, qlast.Query)): + ir_stmt = qlcompiler.compile_ast_to_ir(ql_stmt, schema) + ls.show_message_log(f'IR: {ir_stmt}') + + else: + ls.show_message_log(f'skip compile of {ql_stmt}') + except errors.EdgeDBError as error: + diagnostics.append(_convert_error(error)) + return diagnostics + + +def _convert_error(error: errors.EdgeDBError) -> lsp_types.Diagnostic: + return lsp_types.Diagnostic( + range=lsp_types.Range( + start=lsp_types.Position( + line=error.line - 1, + character=error.col - 1, + ), + end=lsp_types.Position( + line=error.line_end - 1, + character=error.col_end - 1, + ), + ), + severity=lsp_types.DiagnosticSeverity.Error, + message=error.args[0], + ) + + +def _get_schema(ls: EdgeDBLanguageServer) -> Optional[s_schema.Schema]: + + if ls.state.schema: + return ls.state.schema + + # discover dbschema/ folders + if len(ls.workspace.folders) != 1: + + if len(ls.workspace.folders) > 1: + ls.show_message_log( + "WARNING: workspaces with multiple root folders " + "are not supported" + ) + return None + + workspace: lsp_types.WorkspaceFolder = next( + iter(ls.workspace.folders.values()) + ) + workspace_path = pathlib.Path(pygls_uris.to_fs_path(workspace.uri)) + + dbschema = workspace_path / 'dbschema' + + # read and parse .esdl files + sdl = qlast.Schema(declarations=[]) + for entry in os.listdir(dbschema): + if not entry.endswith('.esdl'): + continue + doc = ls.workspace.get_text_document(f'dbschema/{entry}') + + res = ls_parsing.parse(doc, ls) + if diagnostics := res.error: + ls.publish_diagnostics(doc.uri, diagnostics, doc.version) + else: + if isinstance(res.ok, qlast.Schema): + sdl.declarations.extend(res.ok.declarations) + else: + # TODO: complain that .esdl contains non-SDL syntax + pass + + # apply SDL to std schema + std_schema = _load_std_schema(ls.state) + schema = s_ddl.apply_sdl( + sdl, + base_schema=std_schema, + current_schema=std_schema, + ) + + ls.state.schema = schema + return ls.state.schema + + +def _load_std_schema(state: State) -> s_schema.Schema: + if state.std_schema is not None: + return state.std_schema + + schema: s_schema.Schema = s_schema.EMPTY_SCHEMA + for modname in [*s_schema.STD_SOURCES, *s_schema.TESTMODE_SOURCES]: + schema = s_std.load_std_module(schema, modname) + schema, _ = s_std.make_schema_version(schema) + schema, _ = s_std.make_global_schema_version(schema) + + state.std_schema = schema + return state.std_schema diff --git a/edb/pgsql/parser/libpg_query b/edb/pgsql/parser/libpg_query index 1097b2c33e5..c773fdd7100 160000 --- a/edb/pgsql/parser/libpg_query +++ b/edb/pgsql/parser/libpg_query @@ -1 +1 @@ -Subproject commit 1097b2c33e54a37c0d2c0f2d498c7d1cf967eae9 +Subproject commit c773fdd7100c175d0bfbb8be3a79d1f46b370f46 diff --git a/pyproject.toml b/pyproject.toml index ab00b164bcb..2f5ce357bac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ [project.scripts] edgedb-server = "edb.server.main:main" +edgedb-ls = "edb.language_server.main:main" edb = "edb.tools.edb:edbcommands" edgedb = "edb.cli:rustcli" @@ -84,6 +85,10 @@ docs = [ 'sphinx_code_tabs~=0.5.3', ] +language-server = [ + 'pygls~=1.3.1', +] + [build-system] requires = [ "Cython (>=0.29.32, <0.30.0)",