Tests on PostgreSQL Versions #1016

Workflow file for this run

.github/workflows/tests-pg-versions.yml at dafa2d8

	name: Tests on PostgreSQL Versions

	on:
	schedule:
	- cron: "0 3 * * *"
	workflow_dispatch:
	inputs: {}
	push:
	branches:
	- pg-test

	jobs:
	build:
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	submodules: false

	- uses: actions/checkout@v4
	with:
	fetch-depth: 50
	submodules: true

	- name: Set up Python
	uses: actions/setup-python@v5
	id: setup-python
	with:
	python-version: '3.12.2'
	cache: 'pip'
	cache-dependency-path: \|
	pyproject.toml

	# The below is technically a lie as we are technically not
	# inside a virtual env, but there is really no reason to bother
	# actually creating and activating one as below works just fine.
	- name: Export $VIRTUAL_ENV
	run: \|
	venv="$(python -c 'import sys; sys.stdout.write(sys.prefix)')"
	echo "VIRTUAL_ENV=${venv}" >> $GITHUB_ENV

	- name: Set up uv cache
	uses: actions/cache@v4
	with:
	path: ~/.cache/uv
	key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}

	- name: Cached requirements.txt
	uses: actions/cache@v4
	id: requirements-cache
	with:
	path: requirements.txt
	key: edb-requirements-${{ hashFiles('pyproject.toml') }}

	- name: Compute requirements.txt
	if: steps.requirements-cache.outputs.cache-hit != 'true'
	run: \|
	python -m pip install pip-tools
	pip-compile --no-strip-extras --all-build-deps --extra test \
	--output-file requirements.txt pyproject.toml

	- name: Install Python dependencies
	run: \|
	python -c "import sys; print(sys.prefix)"
	python -m pip install uv && uv pip install -U -r requirements.txt

	- name: Compute cache keys
	run: \|
	mkdir -p .tmp
	python setup.py -q ci_helper --type cli > .tmp/edgedbcli_git_rev.txt
	python setup.py -q ci_helper --type rust >.tmp/rust_cache_key.txt
	python setup.py -q ci_helper --type ext >.tmp/ext_cache_key.txt
	python setup.py -q ci_helper --type parsers >.tmp/parsers_cache_key.txt
	python setup.py -q ci_helper --type postgres >.tmp/postgres_git_rev.txt
	python setup.py -q ci_helper --type libpg_query >.tmp/libpg_query_git_rev.txt
	echo 'f8cd94309eaccbfba5dea7835b88c78377608a37' >.tmp/stolon_git_rev.txt
	python setup.py -q ci_helper --type bootstrap >.tmp/bootstrap_cache_key.txt
	echo EDGEDBCLI_GIT_REV=$(cat .tmp/edgedbcli_git_rev.txt) >> $GITHUB_ENV
	echo POSTGRES_GIT_REV=$(cat .tmp/postgres_git_rev.txt) >> $GITHUB_ENV
	echo LIBPG_QUERY_GIT_REV=$(cat .tmp/libpg_query_git_rev.txt) >> $GITHUB_ENV
	echo STOLON_GIT_REV=$(cat .tmp/stolon_git_rev.txt) >> $GITHUB_ENV
	echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV
	echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV

	- name: Upload shared artifacts
	uses: actions/upload-artifact@v4
	with:
	name: shared-artifacts
	path: .tmp
	retention-days: 1

	# Restore binary cache

	- name: Handle cached EdgeDB CLI binaries
	uses: actions/cache@v4
	id: cli-cache
	with:
	path: build/cli
	key: edb-cli-v3-${{ env.EDGEDBCLI_GIT_REV }}

	- name: Handle cached Rust extensions
	uses: actions/cache@v4
	id: rust-cache
	with:
	path: build/rust_extensions
	key: edb-rust-v3-${{ hashFiles('.tmp/rust_cache_key.txt') }}
	restore-keys: \|
	edb-rust-v3-

	- name: Handle cached Cython extensions
	uses: actions/cache@v4
	id: ext-cache
	with:
	path: build/extensions
	key: edb-ext-v5-${{ hashFiles('.tmp/ext_cache_key.txt') }}

	- name: Handle cached PostgreSQL build
	uses: actions/cache@v4
	id: postgres-cache
	with:
	path: build/postgres/install
	key: edb-postgres-v2-${{ env.POSTGRES_GIT_REV }}

	- name: Handle cached Stolon build
	uses: actions/cache@v4
	id: stolon-cache
	with:
	path: build/stolon/bin
	key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}

	- name: Handle cached libpg_query build
	uses: actions/cache@v4
	id: libpg-query-cache
	with:
	path: edb/pgsql/parser/libpg_query/libpg_query.a
	key: edb-libpg_query-v1-${{ env.LIBPG_QUERY_GIT_REV }}

	# Install system dependencies for building

	- name: Install system deps
	if: \|
	steps.cli-cache.outputs.cache-hit != 'true' \|\|
	steps.rust-cache.outputs.cache-hit != 'true' \|\|
	steps.ext-cache.outputs.cache-hit != 'true' \|\|
	steps.stolon-cache.outputs.cache-hit != 'true' \|\|
	steps.postgres-cache.outputs.cache-hit != 'true'
	run: \|
	sudo apt-get update
	sudo apt-get install -y uuid-dev libreadline-dev bison flex

	- name: Install rust toolchain
	if: \|
	steps.cli-cache.outputs.cache-hit != 'true' \|\|
	steps.rust-cache.outputs.cache-hit != 'true'
	uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248
	with:
	components: "cargo,rustc,rust-std"
	toolchain: "stable"

	# Build EdgeDB CLI

	- name: Handle EdgeDB CLI build cache
	uses: actions/cache@v4
	if: steps.cli-cache.outputs.cache-hit != 'true'
	with:
	path: ${{ env.BUILD_TEMP }}/rust/cli
	key: edb-cli-build-v7-${{ env.EDGEDBCLI_GIT_REV }}
	restore-keys: \|
	edb-cli-build-v7-

	- name: Build EdgeDB CLI
	env:
	CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/cli/cargo_home
	CACHE_HIT: ${{ steps.cli-cache.outputs.cache-hit }}
	run: \|
	if [[ "$CACHE_HIT" == "true" ]]; then
	cp -v build/cli/bin/edgedb edb/cli/edgedb
	else
	python setup.py -v build_cli
	fi

	# Build Rust extensions

	- name: Handle Rust extensions build cache
	uses: actions/cache@v4
	if: steps.rust-cache.outputs.cache-hit != 'true'
	with:
	path: ${{ env.BUILD_TEMP }}/rust/extensions
	key: edb-rust-build-v1-${{ hashFiles('.tmp/rust_cache_key.txt') }}
	restore-keys: \|
	edb-rust-build-v1-

	- name: Build Rust extensions
	env:
	CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home
	CACHE_HIT: ${{ steps.rust-cache.outputs.cache-hit }}
	run: \|
	if [[ "$CACHE_HIT" != "true" ]]; then
	rm -rf ${BUILD_LIB}
	mkdir -p build/rust_extensions
	rsync -av ./build/rust_extensions/ ${BUILD_LIB}/
	python setup.py -v build_rust
	rsync -av ${BUILD_LIB}/ build/rust_extensions/
	rm -rf ${BUILD_LIB}
	fi
	rsync -av ./build/rust_extensions/edb/ ./edb/

	# Build libpg_query

	- name: Build libpg_query
	if: \|
	steps.libpg-query-cache.outputs.cache-hit != 'true' &&
	steps.ext-cache.outputs.cache-hit != 'true'
	run: \|
	python setup.py build_libpg_query

	# Build extensions

	- name: Handle Cython extensions build cache
	uses: actions/cache@v4
	if: steps.ext-cache.outputs.cache-hit != 'true'
	with:
	path: ${{ env.BUILD_TEMP }}/edb
	key: edb-ext-build-v3-${{ hashFiles('.tmp/ext_cache_key.txt') }}

	- name: Build Cython extensions
	env:
	CACHE_HIT: ${{ steps.ext-cache.outputs.cache-hit }}
	BUILD_EXT_MODE: py-only
	run: \|
	if [[ "$CACHE_HIT" != "true" ]]; then
	rm -rf ${BUILD_LIB}
	mkdir -p ./build/extensions
	rsync -av ./build/extensions/ ${BUILD_LIB}/
	BUILD_EXT_MODE=py-only python setup.py -v build_ext
	rsync -av ${BUILD_LIB}/ ./build/extensions/
	rm -rf ${BUILD_LIB}
	fi
	rsync -av ./build/extensions/edb/ ./edb/

	# Build parsers

	- name: Handle compiled parsers cache
	uses: actions/cache@v4
	id: parsers-cache
	with:
	path: build/lib
	key: edb-parsers-v3-${{ hashFiles('.tmp/parsers_cache_key.txt') }}
	restore-keys: \|
	edb-parsers-v3-

	- name: Build parsers
	env:
	CACHE_HIT: ${{ steps.parsers-cache.outputs.cache-hit }}
	run: \|
	if [[ "$CACHE_HIT" != "true" ]]; then
	rm -rf ${BUILD_LIB}
	mkdir -p ./build/lib
	rsync -av ./build/lib/ ${BUILD_LIB}/
	python setup.py -v build_parsers
	rsync -av ${BUILD_LIB}/ ./build/lib/
	rm -rf ${BUILD_LIB}
	fi
	rsync -av ./build/lib/edb/ ./edb/

	# Build PostgreSQL

	- name: Build PostgreSQL
	env:
	CACHE_HIT: ${{ steps.postgres-cache.outputs.cache-hit }}
	run: \|
	if [[ "$CACHE_HIT" == "true" ]]; then
	cp build/postgres/install/stamp build/postgres/
	else
	python setup.py build_postgres
	cp build/postgres/stamp build/postgres/install/
	fi

	# Build Stolon

	- name: Set up Go
	if: steps.stolon-cache.outputs.cache-hit != 'true'
	uses: actions/setup-go@v2
	with:
	go-version: 1.16

	- uses: actions/checkout@v4
	if: steps.stolon-cache.outputs.cache-hit != 'true'
	with:
	repository: edgedb/stolon
	path: build/stolon
	ref: ${{ env.STOLON_GIT_REV }}
	fetch-depth: 0
	submodules: false

	- name: Build Stolon
	if: steps.stolon-cache.outputs.cache-hit != 'true'
	run: \|
	mkdir -p build/stolon/bin/
	curl -fsSL https://releases.hashicorp.com/consul/1.10.1/consul_1.10.1_linux_amd64.zip \| zcat > build/stolon/bin/consul
	chmod +x build/stolon/bin/consul
	cd build/stolon && make

	# Install edgedb-server and populate egg-info

	- name: Install edgedb-server
	env:
	BUILD_EXT_MODE: skip
	run: \|
	# --no-build-isolation because we have explicitly installed all deps
	# and don't want them to be reinstalled in an "isolated env".
	pip install --no-build-isolation --no-deps -e .[test,docs]

	# Refresh the bootstrap cache

	- name: Handle bootstrap cache
	uses: actions/cache@v4
	id: bootstrap-cache
	with:
	path: build/cache
	key: edb-bootstrap-v2-${{ hashFiles('.tmp/bootstrap_cache_key.txt') }}
	restore-keys: \|
	edb-bootstrap-v2-

	- name: Bootstrap EdgeDB Server
	if: steps.bootstrap-cache.outputs.cache-hit != 'true'
	run: \|
	edb server --bootstrap-only

	test:
	runs-on: ubuntu-latest
	needs: build
	strategy:
	fail-fast: false
	matrix:
	postgres-version: [ 16 ]
	single-mode:
	- ''
	- 'NOCREATEDB NOCREATEROLE'
	- 'CREATEDB NOCREATEROLE'
	multi-tenant-mode: [ '' ]
	include:
	- postgres-version: 14
	single-mode: ''
	multi-tenant-mode: ''
	- postgres-version: 15
	single-mode: ''
	multi-tenant-mode: ''
	- postgres-version: 16
	single-mode: ''
	multi-tenant-mode: 'remote-compiler'
	- postgres-version: 16
	single-mode: ''
	multi-tenant-mode: 'multi-tenant'
	services:
	postgres:
	image: pgvector/pgvector:pg${{ matrix.postgres-version }}
	env:
	POSTGRES_PASSWORD: postgres
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	--name postgres
	ports:
	- 5432:5432

	steps:
	- name: Trust pgvector extension
	uses: docker://docker
	with:
	args: docker exec postgres sed -i $a\trusted=true /usr/share/postgresql/${{ matrix.postgres-version }}/extension/vector.control
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	submodules: false

	- uses: actions/checkout@v4
	with:
	fetch-depth: 50
	submodules: true

	- name: Set up Python
	uses: actions/setup-python@v5
	id: setup-python
	with:
	python-version: '3.12.2'
	cache: 'pip'
	cache-dependency-path: \|
	pyproject.toml

	# The below is technically a lie as we are technically not
	# inside a virtual env, but there is really no reason to bother
	# actually creating and activating one as below works just fine.
	- name: Export $VIRTUAL_ENV
	run: \|
	venv="$(python -c 'import sys; sys.stdout.write(sys.prefix)')"
	echo "VIRTUAL_ENV=${venv}" >> $GITHUB_ENV

	- name: Set up uv cache
	uses: actions/cache@v4
	with:
	path: ~/.cache/uv
	key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}

	- name: Download requirements.txt
	uses: actions/cache@v4
	with:
	path: requirements.txt
	key: edb-requirements-${{ hashFiles('pyproject.toml') }}

	- name: Install Python dependencies
	run: python -m pip install uv && uv pip install -U -r requirements.txt

	# Restore the artifacts and environment variables

	- name: Download shared artifacts
	uses: actions/download-artifact@v4
	with:
	name: shared-artifacts
	path: .tmp

	- name: Set environment variables
	run: \|
	echo EDGEDBCLI_GIT_REV=$(cat .tmp/edgedbcli_git_rev.txt) >> $GITHUB_ENV
	echo POSTGRES_GIT_REV=$(cat .tmp/postgres_git_rev.txt) >> $GITHUB_ENV
	echo STOLON_GIT_REV=$(cat .tmp/stolon_git_rev.txt) >> $GITHUB_ENV
	echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV
	echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV

	# Restore build cache

	- name: Restore cached EdgeDB CLI binaries
	uses: actions/cache@v4
	id: cli-cache
	with:
	path: build/cli
	key: edb-cli-v3-${{ env.EDGEDBCLI_GIT_REV }}

	- name: Restore cached Rust extensions
	uses: actions/cache@v4
	id: rust-cache
	with:
	path: build/rust_extensions
	key: edb-rust-v3-${{ hashFiles('.tmp/rust_cache_key.txt') }}

	- name: Restore cached Cython extensions
	uses: actions/cache@v4
	id: ext-cache
	with:
	path: build/extensions
	key: edb-ext-v5-${{ hashFiles('.tmp/ext_cache_key.txt') }}

	- name: Restore compiled parsers cache
	uses: actions/cache@v4
	id: parsers-cache
	with:
	path: build/lib
	key: edb-parsers-v3-${{ hashFiles('.tmp/parsers_cache_key.txt') }}

	- name: Restore cached PostgreSQL build
	uses: actions/cache@v4
	id: postgres-cache
	with:
	path: build/postgres/install
	key: edb-postgres-v2-${{ env.POSTGRES_GIT_REV }}

	- name: Restore cached Stolon build
	uses: actions/cache@v4
	id: stolon-cache
	with:
	path: build/stolon/bin
	key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}

	- name: Restore bootstrap cache
	uses: actions/cache@v4
	id: bootstrap-cache
	with:
	path: build/cache
	key: edb-bootstrap-v2-${{ hashFiles('.tmp/bootstrap_cache_key.txt') }}

	- name: Stop if we cannot retrieve the cache
	if: \|
	steps.cli-cache.outputs.cache-hit != 'true' \|\|
	steps.rust-cache.outputs.cache-hit != 'true' \|\|
	steps.ext-cache.outputs.cache-hit != 'true' \|\|
	steps.parsers-cache.outputs.cache-hit != 'true' \|\|
	steps.postgres-cache.outputs.cache-hit != 'true' \|\|
	steps.stolon-cache.outputs.cache-hit != 'true' \|\|
	steps.bootstrap-cache.outputs.cache-hit != 'true'
	run: \|
	echo ::error::Cannot retrieve build cache.
	exit 1

	- name: Restore cache into the source tree
	run: \|
	cp -v build/cli/bin/edgedb edb/cli/edgedb
	rsync -av ./build/rust_extensions/edb/ ./edb/
	rsync -av ./build/extensions/edb/ ./edb/
	rsync -av ./build/lib/edb/ ./edb/
	cp build/postgres/install/stamp build/postgres/

	- name: Install edgedb-server
	env:
	BUILD_EXT_MODE: skip
	run: \|
	# --no-build-isolation because we have explicitly installed all deps
	# and don't want them to be reinstalled in an "isolated env".
	pip install --no-build-isolation --no-deps -e .[test,docs]

	# Run the test

	- name: Setup single mode role and database
	if: ${{ matrix.single-mode }}
	shell: python
	run: \|
	import asyncio
	import subprocess

	from edb.server.pgcluster import get_pg_bin_dir

	async def main():
	psql = await get_pg_bin_dir() / "psql"
	dsn = "postgres://postgres:postgres@localhost/postgres"

	script = """\
	CREATE ROLE singles;
	ALTER ROLE singles WITH LOGIN PASSWORD 'test' NOSUPERUSER
	${{ matrix.single-mode }};
	CREATE DATABASE singles OWNER singles;
	REVOKE ALL ON DATABASE singles FROM PUBLIC;
	GRANT CONNECT ON DATABASE singles TO singles;
	GRANT ALL ON DATABASE singles TO singles;
	"""

	subprocess.run(
	[str(psql), dsn],
	check=True,
	text=True,
	input=script,
	)

	asyncio.run(main())

	- name: Test
	env:
	EDGEDB_TEST_POSTGRES_VERSION: ${{ matrix.postgres-version }}
	run: \|
	if [[ "${{ matrix.single-mode }}" ]]; then
	export EDGEDB_TEST_BACKEND_DSN=postgres://singles:test@localhost/singles
	else
	export EDGEDB_TEST_BACKEND_DSN=postgres://postgres:postgres@localhost/postgres
	fi
	if [[ "${{ matrix.multi-tenant-mode }}" == "remote-compiler" ]]; then
	export EDGEDB_TEST_REMOTE_COMPILER=localhost:5660
	export _EDGEDB_SERVER_COMPILER_POOL_SECRET=secret
	__EDGEDB_DEVMODE=1 edgedb-server compiler --pool-size 2 &
	fi
	edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN --testmode
	if [[ "${{ matrix.multi-tenant-mode }}" == "multi-tenant" ]]; then
	export EDGEDB_SERVER_MULTITENANT_CONFIG_FILE=/tmp/edb.mt.json
	echo "{\"localhost\":{\"instance-name\":\"localtest\",\"backend-dsn\":\"$EDGEDB_TEST_BACKEND_DSN\",\"admin\":true,\"max-backend-connections\":10}}" > /tmp/edb.mt.json
	fi
	if [[ "${{ matrix.single-mode }}" == "NOCREATEDB" ]]; then
	edb test -j1 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN
	else
	edb test -j2 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN
	fi


	workflow-notifications:
	if: failure() && github.event_name != 'pull_request'
	name: Notify in Slack on failures
	needs:
	- build
	- test
	runs-on: ubuntu-latest
	permissions:
	actions: 'read'
	steps:
	- name: Slack Workflow Notification
	uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309
	with:
	repo_token: ${{secrets.GITHUB_TOKEN}}
	slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}
	name: 'Workflow notifications'
	icon_emoji: ':hammer:'
	include_jobs: 'on-failure'

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Tests on PostgreSQL Versions #1016

Workflow file

Tests on PostgreSQL Versions #1016

Jobs

Run details

Workflow file for this run