diff --git a/.github/composite-actions/install-extensions/action.yml b/.github/composite-actions/install-extensions/action.yml index 6af170a61c7..ab945d9f8ee 100644 --- a/.github/composite-actions/install-extensions/action.yml +++ b/.github/composite-actions/install-extensions/action.yml @@ -8,7 +8,7 @@ inputs: migration_mode: description: 'Database migration mode' required: no - default: "single-db" + default: "multi-db" parallel_query_mode: description: 'Postgres Parallel Query Mode' required: no diff --git a/.github/composite-actions/run-pg-upgrade/action.yml b/.github/composite-actions/run-pg-upgrade/action.yml index 53f40d73191..98666d7d5cd 100644 --- a/.github/composite-actions/run-pg-upgrade/action.yml +++ b/.github/composite-actions/run-pg-upgrade/action.yml @@ -4,7 +4,7 @@ inputs: migration_mode: description: 'Database migration mode' required: no - default: 'single-db' + default: 'multi-db' pg_old_dir: description: "Previous postgres intall directory" required: false diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 583f5e4925a..563911493de 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -82,7 +82,7 @@ jobs: if: always() && steps.install-extensions.outcome == 'success' run: | sudo ~/psql/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -f .github/scripts/cleanup_babelfish_database.sql - sudo ~/psql/bin/psql -v ON_ERROR_STOP=1 -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -v migration_mode="single-db" -v tsql_port=1433 -v parallel_query_mode=false -f .github/scripts/create_extension.sql + sudo ~/psql/bin/psql -v ON_ERROR_STOP=1 -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -v migration_mode="multi-db" -v tsql_port=1433 -v parallel_query_mode=false -f .github/scripts/create_extension.sql sqlcmd -S localhost -U "jdbc_user" -P 12345678 -Q "SELECT @@version GO" - name: Run Python Tests diff --git a/.github/workflows/jdbc-tests-multi-db-mode.yml b/.github/workflows/jdbc-tests-multi-db-mode.yml new file mode 100644 index 00000000000..44d0cc1fdc9 --- /dev/null +++ b/.github/workflows/jdbc-tests-multi-db-mode.yml @@ -0,0 +1,174 @@ +name: JDBC Tests for multi-db mode +on: [push, pull_request] + +jobs: + run-babelfish-jdbc-tests: + env: + INSTALL_DIR: psql + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + id: checkout + + - name: Install Dependencies + id: install-dependencies + if: always() + uses: ./.github/composite-actions/install-dependencies + + - name: Build Modified Postgres + id: build-modified-postgres + if: always() && steps.install-dependencies.outcome == 'success' + uses: ./.github/composite-actions/build-modified-postgres + with: + install_dir: 'psql' + code_coverage: 'yes' + + - name: Compile ANTLR + id: compile-antlr + if: always() && steps.build-modified-postgres.outcome == 'success' + uses: ./.github/composite-actions/compile-antlr + + - name: Build Extensions + id: build-extensions + if: always() && steps.compile-antlr.outcome == 'success' + uses: ./.github/composite-actions/build-extensions + + - name: Build tds_fdw Extension + id: build-tds_fdw-extension + if: always() && steps.build-extensions.outcome == 'success' + uses: ./.github/composite-actions/build-tds_fdw-extension + + - name: Build vector Extension + id: build-vector-extension + if: always() && steps.build-tds_fdw-extension.outcome == 'success' + uses: ./.github/composite-actions/build-vector-extension + + - name: Build PostGIS Extension + id: build-postgis-extension + if: always() && steps.build-vector-extension.outcome == 'success' + uses: ./.github/composite-actions/build-postgis-extension + + - name: Install Extensions + id: install-extensions + if: always() && steps.build-postgis-extension.outcome == 'success' + uses: ./.github/composite-actions/install-extensions + with: + wal_level: logical + + - name: Run JDBC Tests + id: jdbc + if: always() && steps.install-extensions.outcome == 'success' + timeout-minutes: 60 + uses: ./.github/composite-actions/run-jdbc-tests + + - name: Start secondary server + id: start-secondary + if: always() && steps.jdbc.outcome == 'success' + uses: ./.github/composite-actions/install-extensions + with: + psql_port: 5433 + tsql_port: 8199 + wal_level: logical + + - name: Setup Publication and Subscription + id: setup-pub-sub + if: always() && steps.start-secondary.outcome == 'success' + run: | + ~/${{env.INSTALL_DIR}}/bin/psql -v ON_ERROR_STOP=1 -d jdbc_testdb -U runner -c "CREATE PUBLICATION my_pub;" + ~/${{env.INSTALL_DIR}}/bin/psql -v ON_ERROR_STOP=1 -d jdbc_testdb -U runner -p 5433 -c "CREATE SUBSCRIPTION my_sub CONNECTION 'host=localhost port=5432 user=jdbc_user dbname=jdbc_testdb password=12345678' PUBLICATION my_pub;" + + - name: Run Replication Tests + id: replication + if: always() && steps.setup-pub-sub.outcome == 'success' + timeout-minutes: 60 + uses: ./.github/composite-actions/run-jdbc-tests + with: + input_dir: 'replication' + + - name: Cleanup babelfish database + id: cleanup + if: always() && steps.replication.outcome == 'success' + run: | + sudo ~/psql/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -f .github/scripts/cleanup_babelfish_database.sql + + - name: Upload Log + if: always() && (steps.jdbc.outcome == 'failure' || steps.replication.outcome == 'failure') + uses: actions/upload-artifact@v2 + with: + name: postgres-log-jdbc + path: | + ~/psql/data/logfile + ~/psql/data_5433/logfile + + # The test summary files contain paths with ':' characters, which is not allowed with the upload-artifact actions + - name: Rename Test Summary Files + id: test-file-rename + if: always() && (steps.jdbc.outcome == 'failure' || steps.replication.outcome == 'failure') + run: | + cd test/JDBC/Info + timestamp=`ls -Art | tail -n 1` + cd $timestamp + mv $timestamp.diff ../output-diff.diff + mv "$timestamp"_runSummary.log ../run-summary.log + cd .. + # get the replication output diff as well if it is present + dir_count=`ls | wc -l` + if [[ $dir_count -eq 2 ]];then + timestamp=`ls -rt | tail -n 2 | sort -r | tail -n 1` + cd $timestamp + mv $timestamp.diff ../replication-output-diff.diff + mv "$timestamp"_runSummary.log ../replication-run-summary.log + + - name: Upload Run Summary + if: always() && steps.test-file-rename.outcome == 'success' + uses: actions/upload-artifact@v2 + with: + name: run-summary.log + path: | + test/JDBC/Info/run-summary.log + test/JDBC/Info/replication-run-summary.log + + - name: Upload Output Diff + if: always() && (steps.jdbc.outcome == 'failure' || steps.replication.outcome == 'failure') + uses: actions/upload-artifact@v2 + with: + name: jdbc-output-diff.diff + path: | + test/JDBC/Info/output-diff.diff + test/JDBC/Info/replication-output-diff.diff + + - name: Check and upload coredumps + if: always() && (steps.jdbc.outcome == 'failure' || steps.replication.outcome == 'failure') + uses: ./.github/composite-actions/upload-coredump + + - name: Generate Code Coverage + id: generate-code-coverage + if: always() && (steps.jdbc.outcome == 'success' && steps.replication.outcome == 'success') + run: | + export PG_CONFIG=~/psql/bin/pg_config + export PG_SRC=~/work/postgresql_modified_for_babelfish + export cmake=$(which cmake) + cd contrib + for ext in babelfishpg_common babelfishpg_money babelfishpg_tds babelfishpg_tsql + do + cd $ext + /usr/bin/lcov --gcov-tool /usr/bin/gcov -q --no-external -c -d . -d ./ -o lcov_test.info + cd .. + done + shell: bash + + - name: Summarize code coverage + id: code-coverage-summary + if: always() && steps.generate-code-coverage.outcome == 'success' + run: | + cd contrib/ + lcov -a babelfishpg_tsql/lcov_test.info -a babelfishpg_tds/lcov_test.info -a babelfishpg_common/lcov_test.info -a babelfishpg_money/lcov_test.info -o jdbc-lcov.info + lcov --list jdbc-lcov.info + + - name: Upload Coverage Report for Babelfish Extensions + if: always() && steps.code-coverage-summary.outcome == 'success' + uses: actions/upload-artifact@v3 + with: + name: coverage-babelfish-extensions-jdbc + path: contrib/jdbc-lcov.info + retention-days: 1 diff --git a/.github/workflows/jdbc-tests.yml b/.github/workflows/jdbc-tests.yml index 464fccdb88e..bab108f861c 100644 --- a/.github/workflows/jdbc-tests.yml +++ b/.github/workflows/jdbc-tests.yml @@ -1,4 +1,4 @@ -name: JDBC Tests +name: JDBC Tests for single-db mode on: [workflow_call] jobs: @@ -54,6 +54,7 @@ jobs: uses: ./.github/composite-actions/install-extensions with: wal_level: logical + migration_mode: "single-db" - name: Run JDBC Tests id: jdbc diff --git a/.github/workflows/major-version-upgrade.yml b/.github/workflows/major-version-upgrade.yml index 4b3dc0e891c..278d68d9cce 100644 --- a/.github/workflows/major-version-upgrade.yml +++ b/.github/workflows/major-version-upgrade.yml @@ -123,7 +123,7 @@ jobs: } > pg_hba.conf ~/${{env.OLD_INSTALL_DIR}}/bin/pg_ctl -c -D ~/${{env.OLD_INSTALL_DIR}}/data -l ~/${{env.OLD_INSTALL_DIR}}/data/logfile restart cd ~/work/babelfish_extensions/babelfish_extensions/ - sudo ~/${{env.OLD_INSTALL_DIR}}/bin/psql -v ON_ERROR_STOP=1 -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -v migration_mode="single-db" -v tsql_port="1433" -f .github/scripts/create_extension.sql + sudo ~/${{env.OLD_INSTALL_DIR}}/bin/psql -v ON_ERROR_STOP=1 -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -v migration_mode="multi-db" -v tsql_port="1433" -f .github/scripts/create_extension.sql sqlcmd -S localhost -U jdbc_user -P 12345678 -Q "SELECT @@version GO" shell: bash