Merge pull request #46 from carlopi/buildanddeploywasm

Build and deploy DuckDB-Wasm extensions

.github/workflows/MainDistributionPipeline.yml

@@ -14,7 +14,7 @@ concurrency:
 jobs:
   duckdb-stable-build:
     name: Build extension binaries
-    uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@v0.9.2
+    uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@6812703823d1d66566bc7eaac2b6e4b273c85333
     with:
       vcpkg_commit: a42af01b72c28a8e1d7b48107b33e4f286a55ef6
       duckdb_version: v0.9.2

.github/workflows/_extension_deploy.yml

@@ -68,7 +68,9 @@ jobs:
 
       - id: parse-matrices
         run: |
-          python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty 
+          cat ./duckdb/.github/config/distribution_matrix.json > distribution_matrix.json
+          grep wasm distribution_matrix.json || (head -n -1 ./duckdb/.github/config/distribution_matrix.json > distribution_matrix.json && echo ',"wasm":{"include":[{"duckdb_arch":"wasm_mvp","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_eh","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_threads","vcpkg_triplet":"wasm32-emscripten"}]}}' >> distribution_matrix.json)
+          python3 ${{ inputs.matrix_parse_script }} --input distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty
           deploy_matrix="`cat deploy_matrix.json`"
           echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT
           echo `cat $GITHUB_OUTPUT`
@@ -94,7 +96,7 @@ jobs:
 
       - uses: actions/download-artifact@v2
         with:
-          name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}
+          name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}}
           path: |
             /tmp/extension
 
@@ -118,4 +120,4 @@ jobs:
           git fetch --tags
           export EXT_VERSION=`git tag --points-at HEAD`
           export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`}
-          ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}}
+          ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}}

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all clean format debug release duckdb_debug duckdb_release pull update
+.PHONY: all clean format debug release duckdb_debug duckdb_release pull update wasm_mvp wasm_eh wasm_threads
 
 all: release
 
@@ -31,6 +31,8 @@ ifeq ($(GEN),ninja)
 	GENERATOR=-G "Ninja" -DFORCE_COLORED_OUTPUT=1
 endif
 
+EXT_NAME=quack
+
 #### Configuration for this extension
 EXTENSION_NAME=QUACK
 EXTENSION_FLAGS=\
@@ -109,3 +111,23 @@ clean:
 	rm -rf testext
 	cd duckdb && make clean
 	cd duckdb && make clean-python
+
+WASM_LINK_TIME_FLAGS=
+
+wasm_mvp:
+	mkdir -p build/wasm_mvp
+	emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_mvp -DCMAKE_CXX_FLAGS="-DDUCKDB_CUSTOM_PLATFORM=wasm_mvp" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake
+	emmake make -j8 -Cbuild/wasm_mvp
+	cd build/wasm_mvp/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS)
+
+wasm_eh:
+	mkdir -p build/wasm_eh
+	emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_eh -DCMAKE_CXX_FLAGS="-fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_eh" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake
+	emmake make -j8 -Cbuild/wasm_eh
+	cd build/wasm_eh/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS)
+
+wasm_threads:
+	mkdir -p ./build/wasm_threads
+	emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_threads -DCMAKE_CXX_FLAGS="-fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_threads" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake
+	emmake make -j8 -Cbuild/wasm_threads
+	cd build/wasm_threads/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS)

scripts/bootstrap-template.py

@@ -50,6 +50,7 @@ def replace_everywhere(to_find, to_replace):
     for path in files_to_search:
         replace(path, to_find, to_replace)
         replace(path, to_find.capitalize(), to_camel_case(to_replace))
+        replace(path, to_find.upper(), to_replace.upper())
 
     replace("./CMakeLists.txt", to_find, to_replace)
     replace("./Makefile", to_find, to_replace)

scripts/extension-upload.sh

@@ -2,7 +2,7 @@
 
 # Extension upload script
 
-# Usage: ./extension-upload-oote.sh <name> <extension_version> <duckdb_version> <architecture> <s3_bucket> <copy_to_latest> <copy_to_versioned>
+# Usage: ./extension-upload.sh <name> <extension_version> <duckdb_version> <architecture> <s3_bucket> <copy_to_latest> <copy_to_versioned>
 # <name>                : Name of the extension
 # <extension_version>   : Version (commit / version tag) of the extension
 # <duckdb_version>      : Version (commit / version tag) of DuckDB
@@ -13,39 +13,78 @@
 
 set -e
 
-ext="/tmp/extension/$1.duckdb_extension"
+if [[ $4 == wasm* ]]; then
+  ext="/tmp/extension/$1.duckdb_extension.wasm"
+else
+  ext="/tmp/extension/$1.duckdb_extension"
+fi
+
+echo $ext
 
 script_dir="$(dirname "$(readlink -f "$0")")"
 
-# Abort if AWS key is not set
-if [ -z "$AWS_ACCESS_KEY_ID" ]; then
-    echo "No AWS key found, skipping.."
-    exit 0
+# calculate SHA256 hash of extension binary
+cat $ext > $ext.append
+
+if [[ $4 == wasm* ]]; then
+  # 0 for custom section
+  # 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256)
+  # [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02]
+  echo -n -e '\x00' >> $ext.append
+  echo -n -e '\x93\x02' >> $ext.append
+  # 10 in hex = 16 in decimal, lenght of name, 1 byte
+  echo -n -e '\x10' >> $ext.append
+  echo -n -e 'duckdb_signature' >> $ext.append
+  # the name of the WebAssembly custom section, 16 bytes
+  # 100 in hex, 256 in decimal
+  # [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)],
+  # for a grand total of 2 bytes
+  echo -n -e '\x80\x02' >> $ext.append
 fi
 
 # (Optionally) Sign binary
 if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then
   echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem
-  $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext > $ext.hash
+  $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext.append > $ext.hash
   openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign
-  cat $ext.sign >> $ext
+  rm -f private.pem
 fi
 
-set -e
+# Signature is always there, potentially defaulting to 256 zeros
+truncate -s 256 $ext.sign
+
+# append signature to extension binary
+cat $ext.sign >> $ext.append
 
 # compress extension binary
-gzip < "${ext}" > "$ext.gz"
+if [[ $4 == wasm_* ]]; then
+  gzip < $ext.append > "$ext.compressed"
+else
+  brotli < $ext.append > "$ext.compressed"
+fi
+
+set -e
+
+# Abort if AWS key is not set
+if [ -z "$AWS_ACCESS_KEY_ID" ]; then
+    echo "No AWS key found, skipping.."
+    exit 0
+fi
 
 # upload versioned version
 if [[ $7 = 'true' ]]; then
-  aws s3 cp $ext.gz s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read
+  if [[ $4 == wasm* ]]; then
+    aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$1/$2/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm"
+  else
+    aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read
+  fi
 fi
 
 # upload to latest version
 if [[ $6 = 'true' ]]; then
-  aws s3 cp $ext.gz s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read
+  if [[ $4 == wasm* ]]; then
+    aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm"
+  else
+    aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read
+  fi
 fi
-
-if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then
-  rm private.pem
-fi