From 95ac97ea97ce32bb88dbdbecca5812e0f59b1ac5 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Mon, 25 Nov 2024 06:49:32 +0100 Subject: [PATCH 01/99] sdk self check, disable wasm-opt temporarily --- .buildconfig | 4 +- .github/workflows/build_wasm_postgres.yml | 1 - cibuild.sh | 102 +++++--- cibuild/linkexport.sh | 0 cibuild/linkimports.sh | 0 cibuild/pg-16.x.sh | 0 cibuild/pg-git.sh | 0 cibuild/pgbuild.sh | 0 cibuild/pglite-ts.sh | 0 cibuild/sdk.sh | 107 +++++++++ patches/exports/pglite | 148 ++++++------ patches/imports/pgcore | 280 +++++++++++----------- patches/imports/plpgsql | 52 +++- 13 files changed, 426 insertions(+), 268 deletions(-) mode change 100644 => 100755 cibuild/linkexport.sh mode change 100644 => 100755 cibuild/linkimports.sh mode change 100644 => 100755 cibuild/pg-16.x.sh mode change 100644 => 100755 cibuild/pg-git.sh mode change 100644 => 100755 cibuild/pgbuild.sh mode change 100644 => 100755 cibuild/pglite-ts.sh create mode 100755 cibuild/sdk.sh diff --git a/.buildconfig b/.buildconfig index 94087d6ef..80848b3b3 100644 --- a/.buildconfig +++ b/.buildconfig @@ -1,3 +1,5 @@ PG_VERSION=16.4 -SDK_VERSION=3.1.72.0bi +SDK_VERSION=3.1.72.3bi +WASI_SDK_VERSION=24.0.4 +SDKROOT=/opt/python-wasm-sdk diff --git a/.github/workflows/build_wasm_postgres.yml b/.github/workflows/build_wasm_postgres.yml index 3571f1804..56a3a501f 100644 --- a/.github/workflows/build_wasm_postgres.yml +++ b/.github/workflows/build_wasm_postgres.yml @@ -15,7 +15,6 @@ jobs: env: BUILD_CONFIG_FILE: .buildconfig SDK_ARCHIVE: python3.13-wasm-sdk-Ubuntu-22.04.tar.lz4 - WASI_SDK_VERSION: 24.0.3 WASI_SDK_ARCHIVE: python3.13-wasi-sdk-Ubuntu-22.04.tar.lz4 SDKROOT: /opt/python-wasm-sdk SYS_PYTHON: /usr/bin/python3 diff --git a/cibuild.sh b/cibuild.sh index 85ab0501c..c037f78e5 100755 --- a/cibuild.sh +++ b/cibuild.sh @@ -2,16 +2,16 @@ # data transfer zone this is == (wire query size + result size ) + 2 # expressed in EMSDK MB -export CMA_MB=${CMA_MB:-64} +export CMA_MB=${CMA_MB:-32} export CI=${CI:-false} -if $CI -then - . .buildconfig -fi +chmod +x ./extra/*.sh cibuild/*.sh + +. .buildconfig + +export PG_VERSION SDK_VERSION WASI_SDK_VERSION SDKROOT -export PG_VERSION=${PG_VERSION:-16.4} export WORKSPACE=${GITHUB_WORKSPACE:-$(pwd)} export PGROOT=${PGROOT:-/tmp/pglite} export WEBROOT=${WEBROOT:-/tmp/web} @@ -19,14 +19,31 @@ export DEBUG=${DEBUG:-false} export PGDATA=${PGROOT}/base export PGUSER=${PGUSER:-postgres} export PGPATCH=${WORKSPACE}/patches -export TOTAL_MEMORY=${TOTAL_MEMORY:-128MB} +export TOTAL_MEMORY=${TOTAL_MEMORY:-512MB} export WASI=${WASI:-false} +# 72 - 144228352 +# -sINITIAL_HEAP not compatible with IMPORTED_MEMORY (which is enabled indirectly via SHARED_MEMORY, RELOCATABLE, ASYNCIFY_LAZY_LOAD_CODE) + +export MEMORY="-sINITIAL_MEMORY=128MB -sMAXIMUM_MEMORY=${TOTAL_MEMORY} -sSTACK_SIZE=2MB -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH" +# -sGLOBAL_BASE=${CMA_MB}MB +# export MEMORY="-sTOTAL_MEMORY=${TOTAL_MEMORY} -sSTACK_SIZE=4MB -sGLOBAL_BASE=${CMA_MB}MB -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH" +export PYDK_CFLAGS="-Wno-missing-prototypes" + # exit on error EOE=false -mkdir -p /tmp/sdk + + +if ./cibuild/sdk.sh +then + echo "sdk check passed (emscripten)" +else + echo sdk failed + exit 44 +fi + # the default is a user writeable path. if mkdir -p ${PGROOT}/sdk @@ -75,10 +92,17 @@ then echo "Wasi build (experimental)" . /opt/python-wasm-sdk/wasm32-wasi-shell.sh + if [ -f ${WORKSPACE}/sdk-fix.tar ] + then + pushd $WASI_SDK_DIR + tar xf ${WORKSPACE}/sdk-fix.tar + popd + fi + else if which emcc then - echo "emcc found in PATH=$PATH" + echo "emcc found in PATH=$PATH (please set PREFIX)" else . /opt/python-wasm-sdk/wasm32-bi-emscripten-shell.sh fi @@ -92,9 +116,9 @@ else node : $(which node) $($(which node) -v) PNPM : $(which pnpm) + PREFIX=$PREFIX " - # custom code for node/web builds that modify pg main/tools behaviour # this used by both node/linkweb build stages @@ -132,7 +156,7 @@ then then echo "wasm-objdump found" else - WRAPPER=$(which wasm-objdump) + WRAPPER=$(command -v wasm-objdump) WASIFILE=$(realpath ${WRAPPER}.wasi) if $WRAPPER -h $WASIFILE | grep -q 'file format wasm 0x1' then @@ -219,10 +243,17 @@ END #define PGDEBUG 0 #endif END + cat >> ${PG_DEBUG_HEADER} </dev/null - chmod +x ./extra/*.sh fi fi echo "======================= ${extra_ext} : $(pwd) ===================" + if [ -f ./extra/${extra_ext}.sh ] + then + ./extra/${extra_ext}.sh || exit 400 + else + echo " + + WARNING: Current source tree has not support ./extra/${extra_ext}.sh + for building ${extra_ext} + + + +" + fi - ./extra/${extra_ext}.sh || exit 400 python3 cibuild/pack_extension.py done @@ -451,6 +503,7 @@ do __________________________ enabled extensions (dlfcn)_____________________________ " + cp -f ${WORKSPACE}/extra/*.tar.gz ${WEBROOT}/ cp -vf ${WEBROOT}/*.tar.gz ${PGLITE}/release/ echo " __________________________________________________________________________________ @@ -470,19 +523,6 @@ ________________________________________________________________________________ mv $packed /tmp/sdk/pg${PG_VERSION}-${packed} - # for repl demo -# mkdir -p /tmp/web/pglite - - #cp -r ${PGLITE}/dist ${WEBROOT}/pglite/ - #cp -r ${PGLITE}/examples ${WEBROOT}/pglite/ - -# for dir in /tmp/web ${WEBROOT}/pglite/examples -# do -# pushd "$dir" -# cp ${PGLITE}/dist/postgres.data ./ -# popd -# done - echo "