Skip to content

Commit

Permalink
Workflow tp
Browse files Browse the repository at this point in the history
  • Loading branch information
@KushaalShroff
KushaalShroff committed May 21, 2024
1 parent 4bf4948 commit 7ec68e4
Show file tree
Hide file tree
Showing 14 changed files with 4 additions and 1,691 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/dotnet-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ jobs:
if: always() && steps.install-extensions.outcome == 'success'
uses: ./.github/composite-actions/install-and-run-dotnet

- name: Check and upload coredumps
if: always() && ( steps.upgrade-and-verify.outcome == 'failure' || steps.setup-base-version.outcome == 'failure' )
uses: ./.github/composite-actions/upload-coredump

- name: Generate Code Coverage
id: generate-code-coverage
if: always() && steps.run-dotnet-tests.outcome == 'success'
Expand Down
112 changes: 0 additions & 112 deletions .github/workflows/isolation-tests.yml
View file
Original file line number Diff line number Diff line change
@@ -1,112 +0,0 @@
name: Babelfish Smoke Tests
on: [workflow_call]

jobs:
isolation-tests:
name: Isolation-Tests
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
id: checkout

- name: Install Dependencies
id: install-dependencies
if: always()
uses: ./.github/composite-actions/install-dependencies

- name: Build Modified Postgres
id: build-modified-postgres
if: always() && steps.install-dependencies.outcome == 'success'
uses: ./.github/composite-actions/build-modified-postgres
with:
install_dir: 'psql'
code_coverage: 'yes'

- name: Compile ANTLR
id: compile-antlr
if: always() && steps.build-modified-postgres.outcome == 'success'
uses: ./.github/composite-actions/compile-antlr

- name: Build Extensions
id: build-extensions
if: always() && steps.compile-antlr.outcome == 'success'
uses: ./.github/composite-actions/build-extensions

- name: Build PostGIS Extension
id: build-postgis-extension
if: always() && steps.build-extensions.outcome == 'success'
uses: ./.github/composite-actions/build-postgis-extension

- name: Install Extensions
id: install-extensions
if: always() && steps.build-postgis-extension.outcome == 'success'
uses: ./.github/composite-actions/install-extensions

- name: Install Python
uses: actions/setup-python@v2
with:
python-version: 3.7

- name: Configure Python environment
run: |
cd ~
curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list
cd ~/work/babelfish_extensions/babelfish_extensions/test/python
sudo ACCEPT_EULA=Y apt-get install -y msodbcsql17 python3-dev
pip3 install pyodbc pymssql pytest pytest-xdist antlr4-python3-runtime==4.9.3
- name: Generate .spec file parser
run: |
cd ~/work/babelfish_extensions/babelfish_extensions/test/python/isolationtest/
java -Xmx500M -cp /usr/local/lib/antlr-4.9.3-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 ./parser/*.g4 -visitor -no-listener
- name: Run Isolation tests
id: run-isolation-tests
run: |
cd test/python
compareWithFile=true \
driver=pyodbc \
runInParallel=false \
testName=all \
provider="ODBC Driver 17 for SQL Server" \
fileGenerator_URL=localhost \
fileGenerator_port=1433 \
fileGenerator_databaseName=master \
fileGenerator_user=jdbc_user \
fileGenerator_password=12345678 \
inputFilesPath=./input/isolation \
runIsolationTests=true \
stepTimeLimit=30 \
pytest -s --tb=long -q .
- name: Generate Code Coverage
id: generate-code-coverage
if: always() && steps.run-isolation-tests.outcome == 'success'
run: |
export PG_CONFIG=~/psql/bin/pg_config
export PG_SRC=~/work/postgresql_modified_for_babelfish
export cmake=$(which cmake)
cd contrib
for ext in babelfishpg_common babelfishpg_money babelfishpg_tds babelfishpg_tsql
do
cd $ext
/usr/bin/lcov --gcov-tool /usr/bin/gcov -q --no-external -c -d . -d ./ -o lcov_test.info
cd ..
done
shell: bash

- name: Summarize code coverage
id: code-coverage-summary
if: always() && steps.generate-code-coverage.outcome == 'success'
run: |
cd contrib/
lcov -a babelfishpg_tsql/lcov_test.info -a babelfishpg_tds/lcov_test.info -a babelfishpg_common/lcov_test.info -a babelfishpg_money/lcov_test.info -o isolation-lcov.info
lcov --list isolation-lcov.info
- name: Upload Coverage Report for Babelfish Extensions
if: always() && steps.code-coverage-summary.outcome == 'success'
uses: actions/upload-artifact@v3
with:
name: coverage-babelfish-extensions-isolation
path: contrib/isolation-lcov.info
retention-days: 1
146 changes: 0 additions & 146 deletions .github/workflows/jdbc-tests-with-non-default-server-collation.yml
View file
Original file line number Diff line number Diff line change
@@ -1,146 +0,0 @@
name: JDBC Tests with Non Default Server Collation
on: [push, pull_request]

jobs:
run-babelfish-jdbc-tests:
env:
INSTALL_DIR: psql
SERVER_COLLATION_NAME: chinese_prc_ci_as

runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
id: checkout

- name: Install Dependencies
id: install-dependencies
if: always()
uses: ./.github/composite-actions/install-dependencies

- name: Build Modified Postgres
id: build-modified-postgres
if: always() && steps.install-dependencies.outcome == 'success'
uses: ./.github/composite-actions/build-modified-postgres

- name: Compile ANTLR
id: compile-antlr
if: always() && steps.build-modified-postgres.outcome == 'success'
uses: ./.github/composite-actions/compile-antlr

- name: Build Extensions
id: build-extensions
if: always() && steps.compile-antlr.outcome == 'success'
uses: ./.github/composite-actions/build-extensions

- name: Build tds_fdw Extension
id: build-tds_fdw-extension
if: always() && steps.build-extensions.outcome == 'success'
uses: ./.github/composite-actions/build-tds_fdw-extension

- name: Build vector Extension
id: build-vector-extension
if: always() && steps.build-tds_fdw-extension.outcome == 'success'
uses: ./.github/composite-actions/build-vector-extension

- name: Build PostGIS Extension
id: build-postgis-extension
if: always() && steps.build-vector-extension.outcome == 'success'
uses: ./.github/composite-actions/build-postgis-extension

- name: Install Extensions
id: install-extensions
if: always() && steps.build-postgis-extension.outcome == 'success'
uses: ./.github/composite-actions/install-extensions
with:
server_collation_name: ${{ env.SERVER_COLLATION_NAME }}
wal_level: logical

- name: Run JDBC Tests
id: jdbc
if: always() && steps.install-extensions.outcome == 'success'
timeout-minutes: 60
run: |
export PATH=~/${{env.INSTALL_DIR}}/bin:$PATH
export PG_SRC=~/work/babelfish_extensions/postgresql_modified_for_babelfish
cd test/JDBC/
# set env variable serverCollationName to current server collation name
export serverCollationName=${{ env.SERVER_COLLATION_NAME }}
mvn test
# reset env variable
unset serverCollationName
- name: Start secondary server
id: start-secondary
if: always() && steps.jdbc.outcome == 'success'
uses: ./.github/composite-actions/install-extensions
with:
psql_port: 5433
tsql_port: 8199
wal_level: logical

- name: Setup Publication and Subscription
id: setup-pub-sub
if: always() && steps.start-secondary.outcome == 'success'
run: |
~/${{env.INSTALL_DIR}}/bin/psql -v ON_ERROR_STOP=1 -d jdbc_testdb -U runner -c "CREATE PUBLICATION my_pub;"
~/${{env.INSTALL_DIR}}/bin/psql -v ON_ERROR_STOP=1 -d jdbc_testdb -U runner -p 5433 -c "CREATE SUBSCRIPTION my_sub CONNECTION 'host=localhost port=5432 user=jdbc_user dbname=jdbc_testdb password=12345678' PUBLICATION my_pub;"
- name: Run Replication Tests
id: replication
if: always() && steps.setup-pub-sub.outcome == 'success'
timeout-minutes: 60
uses: ./.github/composite-actions/run-jdbc-tests
with:
input_dir: 'replication'

- name: Cleanup babelfish database
id: cleanup
if: always() && steps.replication.outcome == 'success'
run: |
sudo ~/psql/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -f .github/scripts/cleanup_babelfish_database.sql
- name: Upload Log
if: always() && (steps.jdbc.outcome == 'failure' || steps.replication.outcome == 'failure')
uses: actions/upload-artifact@v2
with:
name: postgres-log
path: |
~/psql/data/logfile
~/psql/data_5433/logfile
# The test summary files contain paths with ':' characters, which is not allowed with the upload-artifact actions
- name: Rename Test Summary Files
id: test-file-rename
if: always() && (steps.jdbc.outcome == 'failure' || steps.replication.outcome == 'failure')
run: |
cd test/JDBC/Info
timestamp=`ls -Art | tail -n 1`
cd $timestamp
mv $timestamp.diff ../output-diff.diff
mv "$timestamp"_runSummary.log ../run-summary.log
cd ..
# get the replication output diff as well if it is present
dir_count=`ls | wc -l`
if [[ $dir_count -eq 2 ]];then
timestamp=`ls -rt | tail -n 2 | sort -r | tail -n 1`
cd $timestamp
mv $timestamp.diff ../replication-output-diff.diff
mv "$timestamp"_runSummary.log ../replication-run-summary.log
- name: Upload Run Summary
if: always() && steps.test-file-rename.outcome == 'success'
uses: actions/upload-artifact@v2
with:
name: run-summary.log
path: test/JDBC/Info/run-summary.log

- name: Upload Output Diff
if: always() && (steps.jdbc.outcome == 'failure' || steps.replication.outcome == 'failure')
uses: actions/upload-artifact@v2
with:
name: output-diff.diff
path: test/JDBC/Info/output-diff.diff

- name: Check and upload coredumps
if: always() && (steps.jdbc.outcome == 'failure' || steps.replication.outcome == 'failure')
uses: ./.github/composite-actions/upload-coredump
109 changes: 0 additions & 109 deletions .github/workflows/jdbc-tests-with-parallel-query.yml
View file
Original file line number Diff line number Diff line change
@@ -1,109 +0,0 @@
name: JDBC Tests With Parallel Query
on: [push, pull_request]

jobs:
run-babelfish-jdbc-tests-with-parallel-query-mode:
env:
INSTALL_DIR: psql
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
id: checkout

- name: Install Dependencies
id: install-dependencies
if: always()
uses: ./.github/composite-actions/install-dependencies

- name: Build Modified Postgres
id: build-modified-postgres
if: always() && steps.install-dependencies.outcome == 'success'
uses: ./.github/composite-actions/build-modified-postgres

- name: Compile ANTLR
id: compile-antlr
if: always() && steps.build-modified-postgres.outcome == 'success'
uses: ./.github/composite-actions/compile-antlr

- name: Build Extensions
id: build-extensions
if: always() && steps.compile-antlr.outcome == 'success'
uses: ./.github/composite-actions/build-extensions

- name: Build tds_fdw Extension
id: build-tds_fdw-extension
if: always() && steps.build-extensions.outcome == 'success'
uses: ./.github/composite-actions/build-tds_fdw-extension

- name: Build vector Extension
id: build-vector-extension
if: always() && steps.build-tds_fdw-extension.outcome == 'success'
uses: ./.github/composite-actions/build-vector-extension

- name: Build PostGIS Extension
id: build-postgis-extension
if: always() && steps.build-vector-extension.outcome == 'success'
uses: ./.github/composite-actions/build-postgis-extension

- name: Install Extensions
id: install-extensions
if: always() && steps.build-postgis-extension.outcome == 'success'
uses: ./.github/composite-actions/install-extensions
with:
parallel_query_mode: true

- name: Run JDBC Tests
id: jdbc
if: always() && steps.install-extensions.outcome == 'success'
timeout-minutes: 120
run: |
export PATH=~/${{env.INSTALL_DIR}}/bin:$PATH
export PG_SRC=~/work/babelfish_extensions/postgresql_modified_for_babelfish
cd test/JDBC/
# set env variable isParallelQueryMode to true to let jdbc know not to run tests of file parallel_query_jdbc_schedule
export isParallelQueryMode=true
mvn test
# reset env variable
unset isParallelQueryMode
- name: Cleanup babelfish database
id: cleanup
if: always() && steps.install-extensions.outcome == 'success'
run: |
sudo ~/psql/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="jdbc_testdb" -f .github/scripts/cleanup_babelfish_database.sql
- name: Upload Log
if: always() && steps.jdbc.outcome == 'failure'
uses: actions/upload-artifact@v2
with:
name: postgres-log
path: ~/psql/data/logfile

# The test summary files contain paths with ':' characters, which is not allowed with the upload-artifact actions
- name: Rename Test Summary Files
id: test-file-rename
if: always() && steps.jdbc.outcome == 'failure'
run: |
cd test/JDBC/Info
timestamp=`ls -Art | tail -n 1`
cd $timestamp
mv $timestamp.diff ../output-diff.diff
mv "$timestamp"_runSummary.log ../run-summary.log
- name: Upload Run Summary
if: always() && steps.test-file-rename == 'success'
uses: actions/upload-artifact@v2
with:
name: run-summary.log
path: test/JDBC/Info/run-summary.log

- name: Upload Output Diff
if: always() && steps.jdbc.outcome == 'failure'
uses: actions/upload-artifact@v2
with:
name: output-diff.diff
path: test/JDBC/Info/output-diff.diff

- name: Check and upload coredumps
if: always() && steps.jdbc.outcome == 'failure'
uses: ./.github/composite-actions/upload-coredump
Loading

0 comments on commit 7ec68e4

Please sign in to comment.