From 70db6fe85d775297150dafbf6094b3ca30b9b963 Mon Sep 17 00:00:00 2001 From: Alexey Masterov Date: Mon, 6 Jan 2025 13:11:52 +0100 Subject: [PATCH] Add a concept for the test for extensions --- compute/compute-node.Dockerfile | 1 + docker-compose/collect_versions.sh | 55 ---------------- .../ext-src/pgvector-src/test-upgrade.sh | 4 ++ .../ext-src/plv8-src/test-upgrade.sh | 4 ++ .../postgresql-unit-src/test-upgrade.sh | 4 ++ docker-compose/test_extensions_upgrade.sh | 64 +++++++++++++++++++ 6 files changed, 77 insertions(+), 55 deletions(-) delete mode 100755 docker-compose/collect_versions.sh create mode 100755 docker-compose/ext-src/pgvector-src/test-upgrade.sh create mode 100755 docker-compose/ext-src/plv8-src/test-upgrade.sh create mode 100755 docker-compose/ext-src/postgresql-unit-src/test-upgrade.sh create mode 100755 docker-compose/test_extensions_upgrade.sh diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index f91caff609de..a9ca30e92f22 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1462,6 +1462,7 @@ ENV PATH=/usr/local/pgsql/bin:$PATH ENV PGHOST=compute ENV PGPORT=55433 ENV PGUSER=cloud_admin +ENV PGPASSWORD=cloud_admin ENV PGDATABASE=postgres ######################################################################################### # diff --git a/docker-compose/collect_versions.sh b/docker-compose/collect_versions.sh deleted file mode 100755 index db1aa273043c..000000000000 --- a/docker-compose/collect_versions.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -set -eux -o pipefail -export TAG=6659 -export PGUSER=cloud_admin -export PGPASSWORD=cloud_admin -export PGHOST=127.0.0.1 -export PGPORT=55433 -export PGDATABASE=postgres -docker volume prune -f -function wait_for_ready { - while ! docker compose logs compute_is_ready | grep -q "accepting connections"; do - sleep 1 - done -} -EXTENSIONS='[ -{"extname": "plv8", "extdir": "plv8-src"}, -{"extname": "vector", "extdir": "pgvector-src"} -]' -EXTNAMES=$(echo ${EXTENSIONS} | jq -r '.[].extname' | paste -sd ' ' -) -TAG=6660 docker compose up --build -d -wait_for_ready -for ext in $EXTNAMES; do - echo "CREATE EXTENSION IF NOT EXISTS ${ext};" -done | psql -X -v ON_ERROR_STOP=1 -query="select json_object_agg(extname,extversion) from pg_extension where extname in ('${EXTNAMES// /','}')" -new_vers=$(psql -Aqt -c "$query" ) -echo $new_vers -docker compose down -TAG=6659 docker compose --profile test-extensions up --build -d --force-recreate -wait_for_ready -for ext in $EXTNAMES; do - echo "CREATE EXTENSION IF NOT EXISTS ${ext};" -done | psql -X -v ON_ERROR_STOP=1 -query="select pge.extname from pg_extension pge join (select key as extname, value as extversion from json_each_text('${new_vers}')) x on pge.extname=x.extname and pge.extversion <> x.extversion" -#query="select * from pg_extension" -echo $query -exts=$(psql -Aqt -c "$query") -if [ -z "${exts}" ]; then - echo "No extensions were upgraded" -else - for ext in ${exts}; do - echo Testing ${ext}... - EXTDIR=$(echo ${EXTENSIONS} | jq -r '.[] | select(.extname=="'${ext}'") | .extdir') - docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions bash -c "cd /ext-src/${EXTDIR} && make installcheck" - TAG=6659 docker compose down compute - COMPUTE_TAG=6660 TAG=6659 docker compose up -d --build compute - wait_for_ready - docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions bash -c "cd /ext-src/${EXTDIR} && /usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --inputdir=./ --bindir='/usr/local/pgsql/bin' --inputdir=test --use-existing --dbname=contrib_regression bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector_type" - psql -d contrib_regression -c '\dx' - psql -d contrib_regression -c "alter extension ${ext} update" - psql -d contrib_regression -c '\dx' - docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions bash -c "cd /ext-src/${EXTDIR} && /usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --inputdir=./ --bindir='/usr/local/pgsql/bin' --inputdir=test --use-existing --dbname=contrib_regression bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector_type" - done -fi -docker compose --profile test-extensions down \ No newline at end of file diff --git a/docker-compose/ext-src/pgvector-src/test-upgrade.sh b/docker-compose/ext-src/pgvector-src/test-upgrade.sh new file mode 100755 index 000000000000..51d2450c61b0 --- /dev/null +++ b/docker-compose/ext-src/pgvector-src/test-upgrade.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -ex +cd "$(dirname ${0})" +/usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --inputdir=./ --bindir='/usr/local/pgsql/bin' --inputdir=test --use-existing --dbname=contrib_regression bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector_type \ No newline at end of file diff --git a/docker-compose/ext-src/plv8-src/test-upgrade.sh b/docker-compose/ext-src/plv8-src/test-upgrade.sh new file mode 100755 index 000000000000..38cc10a7f30e --- /dev/null +++ b/docker-compose/ext-src/plv8-src/test-upgrade.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -ex +cd "$(dirname ${0})" +/usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --inputdir=./ --bindir='/usr/local/pgsql/bin' --use-existing --dbname=contrib_regression plv8 plv8-errors scalar_args inline json startup_pre startup varparam json_conv jsonb_conv window guc es6 arraybuffer composites currentresource startup_perms bytea find_function_perms memory_limits reset show array_spread regression dialect bigint procedure \ No newline at end of file diff --git a/docker-compose/ext-src/postgresql-unit-src/test-upgrade.sh b/docker-compose/ext-src/postgresql-unit-src/test-upgrade.sh new file mode 100755 index 000000000000..60c32655c195 --- /dev/null +++ b/docker-compose/ext-src/postgresql-unit-src/test-upgrade.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -ex +cd "$(dirname ${0})" +/usr/local/pgsql/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress --inputdir=./ --bindir='/usr/local/pgsql/bin' --use-existing --dbname=contrib_regression extension tables unit binary unicode prefix units time temperature functions language_functions round derived compare aggregate iec custom crosstab convert \ No newline at end of file diff --git a/docker-compose/test_extensions_upgrade.sh b/docker-compose/test_extensions_upgrade.sh new file mode 100755 index 000000000000..fae2884f6425 --- /dev/null +++ b/docker-compose/test_extensions_upgrade.sh @@ -0,0 +1,64 @@ +#!/bin/bash +set -eux -o pipefail +if [ -z ${OLDTAG+x} ] || [ -z ${NEWTAG+x} ] || [ -z "${OLDTAG}" ] || [ -z "${NEWTAG}" ]; then + echo OLDTAG and NEWTAG must be defined + exit 1 +fi +export PGUSER=cloud_admin +export PGPASSWORD=cloud_admin +export PGHOST=127.0.0.1 +export PGPORT=55433 +export PGDATABASE=postgres +docker volume prune -f +function wait_for_ready { + while ! docker compose logs compute_is_ready | grep -q "accepting connections"; do + sleep 1 + done +} +function create_extensions() { + for ext in ${1}; do + echo "CREATE EXTENSION IF NOT EXISTS ${ext};" + done | psql -X -v ON_ERROR_STOP=1 +} +EXTENSIONS='[ +{"extname": "plv8", "extdir": "plv8-src"}, +{"extname": "vector", "extdir": "pgvector-src"}, +{"extname": "unit", "extdir": "postgresql-unit-src"} +]' +EXTNAMES=$(echo ${EXTENSIONS} | jq -r '.[].extname' | paste -sd ' ' -) +TAG=${NEWTAG} docker compose up --build -d +wait_for_ready +create_extensions "${EXTNAMES}" +query="select json_object_agg(extname,extversion) from pg_extension where extname in ('${EXTNAMES// /','}')" +new_vers=$(psql -Aqt -c "$query" ) +echo $new_vers +docker compose down +TAG=${OLDTAG} docker compose --profile test-extensions up --build -d --force-recreate +wait_for_ready +# XXX this is about to be included into the image, for test only +docker compose cp ext-src neon-test-extensions:/ +for ext in $EXTNAMES; do + echo "CREATE EXTENSION IF NOT EXISTS ${ext};" +done | psql -X -v ON_ERROR_STOP=1 +query="select pge.extname from pg_extension pge join (select key as extname, value as extversion from json_each_text('${new_vers}')) x on pge.extname=x.extname and pge.extversion <> x.extversion" +echo $query +exts=$(psql -Aqt -c "$query") +if [ -z "${exts}" ]; then + echo "No extensions were upgraded" +else + psql -c "CREATE DATABASE contrib_regression" + export PGDATABASE=contrib_regression + create_extensions "${exts}" + TAG=${OLDTAG} docker compose down compute compute_is_ready + COMPUTE_TAG=${NEWTAG} TAG=${OLDTAG} docker compose up -d --build compute compute_is_ready + wait_for_ready + for ext in ${exts}; do + echo Testing ${ext}... + EXTDIR=$(echo ${EXTENSIONS} | jq -r '.[] | select(.extname=="'${ext}'") | .extdir') + psql -d contrib_regression -c "\dx ${ext}" + docker compose exec -e PGPASSWORD=cloud_admin neon-test-extensions sh -c /ext-src/${EXTDIR}/test-upgrade.sh + psql -d contrib_regression -c "alter extension ${ext} update" + psql -d contrib_regression -c "\dx ${ext}" + done +fi +docker compose --profile test-extensions down \ No newline at end of file