From 6d51cc5779e402cdf79d37997580477cabfdc506 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Fri, 6 Oct 2023 14:06:36 +0200 Subject: [PATCH] update to new version of extension workflow --- .../workflows/MainDistributionPipeline.yml | 24 ++-- .github/workflows/_extension_deploy.yml | 121 ++++++++++++++++++ scripts/extension-upload.sh | 51 ++++++++ 3 files changed, 187 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/_extension_deploy.yml create mode 100755 scripts/extension-upload.sh diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 8d8c6cb..de05311 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -12,14 +12,20 @@ concurrency: cancel-in-progress: true jobs: - duckdb-main-pipeline: - name: DuckDB Main Pipeline - uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@v0.9.0 + duckdb-stable: + name: Build extension binaries + uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@main with: - duckdb_version: v0.9.0 + duckdb_version: main extension_name: quack - extension_ref: ${{ github.ref }} - # Secrets are used to pass S3 credentials to the extension-upload script for distributing. - # Note that for security, switching to either explicitly passing secrets or moving the workflow into the extension - # repository can be considered. - secrets: inherit \ No newline at end of file + + duckdb-stable-deploy: + name: Deploy extension binaries + needs: duckdb-stable + uses: ./.github/workflows/_extension_deploy.yml + secrets: inherit + with: + duckdb_version: main + extension_name: quack + deploy_latest: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feature' }} + deploy_versioned: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feature' }} \ No newline at end of file diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml new file mode 100644 index 0000000..f8c7542 --- /dev/null +++ b/.github/workflows/_extension_deploy.yml @@ -0,0 +1,121 @@ +# +# Reusable workflow that deploys the artifacts produced by github.com/duckdb/duckdb/.github/workflows/_extension_distribution.yml +# +# note: this workflow needs to be located in the extension repository, as it requires secrets to be passed to the +# deploy script. However, it should generally not be necessary to modify this workflow in your extension repository, as +# this workflow can be configured to use a custom deploy script. + + +name: Extension Deployment +on: + workflow_call: + inputs: + # The name of the extension + extension_name: + required: true + type: string + # DuckDB version to build against + duckdb_version: + required: true + type: string + # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' + exclude_archs: + required: false + type: string + default: "" + # Whether to upload this deployment as the latest. This may overwrite a previous deployment. + deploy_latest: + required: false + type: boolean + default: false + # Whether to upload this deployment under a versioned path. These will not be deleted automatically + deploy_versioned: + required: false + type: boolean + default: false + # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times + artifact_postfix: + required: false + type: string + default: "" + # Override the default deploy script with a custom script + deploy_script: + required: false + type: string + default: "./scripts/extension-upload.sh" + # Override the default matrix parse script with a custom script + matrix_parse_script: + required: false + type: string + default: "./duckdb/scripts/modify_distribution_matrix.py" + +jobs: + generate_matrix: + name: Generate matrix + runs-on: ubuntu-latest + outputs: + deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Checkout DuckDB to version + run: | + cd duckdb + git checkout ${{ inputs.duckdb_version }} + + - id: parse-matrices + run: | + python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + deploy_matrix="`cat deploy_matrix.json`" + echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT + echo `cat $GITHUB_OUTPUT` + + deploy: + name: Deploy + runs-on: ubuntu-latest + needs: generate_matrix + if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }} + strategy: + matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Checkout DuckDB to version + run: | + cd duckdb + git checkout ${{ inputs.duckdb_version }} + + - uses: actions/download-artifact@v2 + with: + name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} + path: | + /tmp/extension + + - name: Deploy + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} + BUCKET_NAME: ${{ secrets.S3_BUCKET }} + DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }} + run: | + pwd + python3 -m pip install pip awscli + git config --global --add safe.directory '*' + cd duckdb + git fetch --tags + export DUCKDB_VERSION=`git tag --points-at HEAD` + export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} + cd .. + git fetch --tags + export EXT_VERSION=`git tag --points-at HEAD` + export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`} + ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.set_latest || 'true' && 'false'}} $BUCKET_NAME ${{inputs.set_versioned || 'true' && 'false'}} \ No newline at end of file diff --git a/scripts/extension-upload.sh b/scripts/extension-upload.sh new file mode 100755 index 0000000..3eb3b67 --- /dev/null +++ b/scripts/extension-upload.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Extension upload script + +# Usage: ./extension-upload-oote.sh +# : Name of the extension +# : Version (commit / version tag) of the extension +# : Version (commit / version tag) of DuckDB +# : Architecture target of the extension binary +# : S3 bucket to upload to +# : Set this as the latest version ("true" / "false", default: "false") +# : Set this as a versioned version that will prevent its deletion + +set -e + +ext="/tmp/extension/$1.duckdb_extension" + +script_dir="$(dirname "$(readlink -f "$0")")" + +# Abort if AWS key is not set +if [ -z "$AWS_ACCESS_KEY_ID" ]; then + echo "No AWS key found, skipping.." + exit 0 +fi + +# (Optionally) Sign binary +if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then + echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem + $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext > $ext.hash + openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign + cat $ext.sign >> $ext +fi + +set -e + +# compress extension binary +gzip < "${ext}" > "$ext.gz" + +# upload versioned version +if [[ $7 = 'true' ]]; then + aws s3 cp $ext.gz s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read +fi + +# upload to latest version +if [[ $6 = 'true' ]]; then + aws s3 cp $ext.gz s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read +fi + +if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then + rm private.pem +fi \ No newline at end of file