From 70b528c7229602ca0f5a23d3c105274730180c71 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 7 Aug 2024 17:47:06 +0200 Subject: [PATCH 1/3] Add inputs to workflow, allowing to trigger on specific version --- .github/workflows/build.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc1dcbd..0680227 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,6 +16,14 @@ on: inputs: extension_name: type: string + duckdb_version: + type: string + workflow_call: + inputs: + extension_name: + type: string + duckdb_version: + type: string jobs: prepare: @@ -57,7 +65,7 @@ jobs: uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main if: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_NAME != '' }} with: - duckdb_version: v1.0.0 + duckdb_version: ${{ inputs.duckdb_version || 'v1.0.0' }} exclude_archs: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_EXCLUDE_PLATFORMS }} extra_toolchains: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_REQUIRES_TOOLCHAINS }} custom_toolchain_script: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_CUSTOM_TOOLCHAIN_SCRIPT == 'true'}} @@ -85,7 +93,7 @@ jobs: secrets: inherit with: deploy_latest: true - duckdb_version: v1.0.0 + duckdb_version: ${{ inputs.duckdb_version || 'v1.0.0' }} exclude_archs: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_EXCLUDE_PLATFORMS }} extension_name: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_NAME }} repository: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_GITHUB }} From 20508638351170a71f9c9b5dc3ec1a13394c616f Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 28 Aug 2024 13:06:05 +0200 Subject: [PATCH 2/3] Use ref_next when targeting a release different than DUCKDB_STABLE_RELEASE --- .github/workflows/build.yml | 4 ++++ scripts/build.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0680227..d442218 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,6 +37,9 @@ jobs: COMMUNITY_EXTENSION_CUSTOM_TOOLCHAIN_SCRIPT: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_CUSTOM_TOOLCHAIN_SCRIPT }} COMMUNITY_EXTENSION_VCPKG_URL: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_VCPKG_URL == '' && 'https://github.com/microsoft/vcpkg.git' || steps.parse.outputs.COMMUNITY_EXTENSION_VCPKG_URL }} COMMUNITY_EXTENSION_VCPKG_COMMIT: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_VCPKG_COMMIT == '' && 'a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6' || steps.parse.outputs.COMMUNITY_EXTENSION_VCPKG_COMMIT }} + env: + DUCKDB_LATEST_STABLE: 'v1.0.0' + DUCKDB_VERSION: ${{ inputs.duckdb_version || 'v1.0.0' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -56,6 +59,7 @@ jobs: ALL_CHANGED_FILES: ${{ github.event_name == 'workflow_dispatch' && format('extensions/{0}/description.yml', inputs.extension_name) || steps.changed-files.outputs.all_changed_files }} run: | pip install pyyaml + # scripts/build.py takes DUCKDB_VERSION from environment python scripts/build.py cat env.sh >> $GITHUB_OUTPUT echo `cat $GITHUB_OUTPUT` diff --git a/scripts/build.py b/scripts/build.py index 4b4fa67..a83a1ee 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -32,7 +32,11 @@ with open('env.sh', 'w+') as hdl: hdl.write(f"COMMUNITY_EXTENSION_GITHUB={desc['repo']['github']}\n") - hdl.write(f"COMMUNITY_EXTENSION_REF={desc['repo']['ref']}\n") + extension_ref = desc['repo']['ref'] + if os.environ['DUCKDB_VERSION'] != os.environ['DUCKDB_LATEST_STABLE']: + if 'ref_next' in desc['repo']: + extension_ref = desc['repo']['ref_next'] + hdl.write(f"COMMUNITY_EXTENSION_REF={extension_ref}\n") hdl.write(f"COMMUNITY_EXTENSION_NAME={desc['extension']['name']}\n") excluded_platforms = desc['extension'].get('excluded_platforms') requires_toolchains = desc['extension'].get('requires_toolchains') From ff641739c95abc5ac23f4be8a06a6c56486c4b77 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 28 Aug 2024 22:05:55 +0200 Subject: [PATCH 3/3] Add workflow to test all extensions at once --- .github/workflows/build.yml | 11 +++++++++- .github/workflows/test_all.yml | 37 ++++++++++++++++++++++++++++++++++ scripts/get_extension_list.sh | 9 +++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test_all.yml create mode 100755 scripts/get_extension_list.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d442218..29e547c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,12 +18,20 @@ on: type: string duckdb_version: type: string + deploy: + type: string + more_excluded: + type: string workflow_call: inputs: extension_name: type: string duckdb_version: type: string + deploy: + type: string + more_excluded: + type: string jobs: prepare: @@ -32,7 +40,7 @@ jobs: COMMUNITY_EXTENSION_GITHUB: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_GITHUB }} COMMUNITY_EXTENSION_REF: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_REF }} COMMUNITY_EXTENSION_DEPLOY: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_DEPLOY }} - COMMUNITY_EXTENSION_EXCLUDE_PLATFORMS: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_EXCLUDE_PLATFORMS }} + COMMUNITY_EXTENSION_EXCLUDE_PLATFORMS: ${{ inputs.more_excluded }}${{ steps.parse.outputs.COMMUNITY_EXTENSION_EXCLUDE_PLATFORMS }} COMMUNITY_EXTENSION_REQUIRES_TOOLCHAINS: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_REQUIRES_TOOLCHAINS }} COMMUNITY_EXTENSION_CUSTOM_TOOLCHAIN_SCRIPT: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_CUSTOM_TOOLCHAIN_SCRIPT }} COMMUNITY_EXTENSION_VCPKG_URL: ${{ steps.parse.outputs.COMMUNITY_EXTENSION_VCPKG_URL == '' && 'https://github.com/microsoft/vcpkg.git' || steps.parse.outputs.COMMUNITY_EXTENSION_VCPKG_URL }} @@ -80,6 +88,7 @@ jobs: vcpkg_commit: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_VCPKG_COMMIT}} doc_test: + if: ${{ inputs.deploy != 'false' }} needs: - prepare - build diff --git a/.github/workflows/test_all.yml b/.github/workflows/test_all.yml new file mode 100644 index 0000000..af68c4f --- /dev/null +++ b/.github/workflows/test_all.yml @@ -0,0 +1,37 @@ +name: Test all extensions +on: + workflow_dispatch: + inputs: + duckdb_version: + type: string + +jobs: + collect_extensions: + outputs: + COMMUNITY_EXTENSION_LIST: ${{ steps.generate_list.outputs.EXTENSION_LIST }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Generate extension list + id: generate_list + run: | + ./scripts/get_extension_list.sh + cat extension_list + cat extension_list >> $GITHUB_OUTPUT + + test_all: + needs: + - collect_extensions + strategy: + fail-fast: false + matrix: + extension_name: ${{ fromJson(needs.collect_extensions.outputs.COMMUNITY_EXTENSION_LIST) }} + uses: ./.github/workflows/build.yml + with: + extension_name: ${{ matrix.extension_name }} + duckdb_version: ${{ inputs.duckdb_version }} + deploy: 'false' + more_excluded: 'windows_amd64_rtools;windows_amd64;wasm_threads;wasm_mvp;linux_arm64;linux_amd64;osx_arm64' diff --git a/scripts/get_extension_list.sh b/scripts/get_extension_list.sh new file mode 100755 index 0000000..e7c23c4 --- /dev/null +++ b/scripts/get_extension_list.sh @@ -0,0 +1,9 @@ +set -eo pipefail + +echo -n "EXTENSION_LIST=[" > extension_list +for extension_folder in extensions/*; +do + extension_name=$(basename -- $extension_folder) + echo -n "'$extension_name'," >> extension_list +done +echo "]" >> extension_list