Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/BABEL_5_X_DEV' into jira-babel…
Browse files Browse the repository at this point in the history
…-4307
  • Loading branch information
manisha-deshpande committed Mar 6, 2025
2 parents 2bcd46f + 948d8b9 commit 0cc9089
Show file tree
Hide file tree
Showing 294 changed files with 12,318 additions and 2,060 deletions.
4 changes: 2 additions & 2 deletions .github/composite-actions/install-extensions/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,9 @@ runs:
if [[ ${{ inputs.server_collation_name }} != "default" ]]; then
sudo echo "babelfishpg_tsql.server_collation_name = '${{ inputs.server_collation_name }}'" >> postgresql.conf
fi
~/${{inputs.install_dir}}/bin/pg_ctl -c -D ~/${{inputs.install_dir}}/$DATADIR/ -l logfile start
~/${{inputs.install_dir}}/bin/pg_ctl -c -D ~/${{inputs.install_dir}}/$DATADIR/ -l ~/${{inputs.install_dir}}/$DATADIR/logfile start
cd ~/work/babelfish_extensions/babelfish_extensions/
sudo ~/${{inputs.install_dir}}/bin/psql -v ON_ERROR_STOP=1 -d postgres -U runner -p ${{inputs.psql_port}} -v user="jdbc_user" -v db="babelfish_db" -v migration_mode=${{inputs.migration_mode}} -v tsql_port=${{inputs.tsql_port}} -v parallel_query_mode=${{inputs.parallel_query_mode}} -f .github/scripts/create_extension.sql
~/${{inputs.install_dir}}/bin/pg_ctl -c -D ~/${{inputs.install_dir}}/$DATADIR/ -l logfile restart
~/${{inputs.install_dir}}/bin/pg_ctl -c -D ~/${{inputs.install_dir}}/$DATADIR/ -l ~/${{inputs.install_dir}}/$DATADIR/logfile restart
sqlcmd -S localhost,${{inputs.tsql_port}} -U "jdbc_user" -P 12345678 -Q "SELECT @@version GO"
shell: bash
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,6 @@ runs:
tar_dir: ${{ matrix.upgrade-path.last_version }}
run: |
cd test/JDBC/
export isUpgradeTestMode=true
if [[ ${{ inputs.server_collation_name }} != "default" ]]; then
export serverCollationName=${{ inputs.server_collation_name }}
fi
Expand Down
1 change: 0 additions & 1 deletion .github/composite-actions/run-verify-tests/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ runs:
touch dummy_schedule
echo "all" > dummy_schedule
export scheduleFile=dummy_schedule
export isUpgradeTestMode=true
if [[ ${{ inputs.server_collation_name }} != "default" ]]; then
export serverCollationName=${{ inputs.server_collation_name }}
fi
Expand Down
2 changes: 0 additions & 2 deletions .github/composite-actions/setup-base-version/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,6 @@ runs:
export serverCollationName=${{ inputs.server_collation_name }}
fi
if [[ "$migr_mode" == "multi-db" ]];then
export isUpgradeTestMode=true
base_dir=${{ matrix.upgrade-path.path[0] }}
if [[ "$base_dir" == *"latest"* || ${{ inputs.dump_restore }} == 'true' ]]; then
base_dir="latest"
Expand All @@ -210,7 +209,6 @@ runs:
exit 1
fi
done
export isUpgradeTestMode=true
export inputFilesPath=input
for filename in $(grep -v "^ignore.*\|^#.*\|^cmd.*\|^all.*\|^$" upgrade/$base_dir/schedule); do
sed -i "s/\b$filename[ ]*\b$/$filename-vu-prepare/g" upgrade/$base_dir/schedule
Expand Down
2 changes: 1 addition & 1 deletion .github/composite-actions/setup-dump-restore-ca/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ runs:
is_final_ver=false; [[ i -eq $LEN-1 ]] && is_final_ver=true
pg_old_dir=$(echo psql$(awk -F. '{print $1}' <<< $previous_installed_version))
pg_new_dir=$(echo psql$(awk -F. '{print $1}' <<< $dump_restore_version))
if [[ "$dump_restore_version" == 'target.latest' || "$dump_restore_version" -ge 16.7 ]]
if [[ "$dump_restore_version" == 'target.latest' || "$dump_restore_version" -ge 16.8 ]]
then
antlr_version=4.13.2
else
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ runs:
is_final_ver=false; [[ i -eq $LEN-1 ]] && is_final_ver=true
pg_old_dir=$(echo psql$(awk -F. '{print $1}' <<< $previous_installed_version))
pg_new_dir=$(echo psql$(awk -F. '{print $1}' <<< $upgrade_version))
if [[ "$upgrade_version" == 'target.latest' || "$upgrade_version" -ge 16.7 ]]
if [[ "$upgrade_version" == 'target.latest' || "$upgrade_version" -ge 16.8 ]]
then
antlr_version=4.13.2
else
Expand Down
32 changes: 10 additions & 22 deletions .github/configuration/upgrade-test-configuration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ upgrade-version: [{
{
upgrade-path: [
{
version: '14.16',
version: '14.17',
upgrade-type: null
},
{
Expand All @@ -97,7 +97,7 @@ upgrade-version: [{
{
upgrade-path: [
{
version: '14.17',
version: '14.18',
upgrade-type: null
},
{
Expand Down Expand Up @@ -133,7 +133,7 @@ upgrade-version: [{
{
upgrade-path: [
{
version: '15.11',
version: '15.12',
upgrade-type: null
},
{
Expand Down Expand Up @@ -169,19 +169,7 @@ upgrade-version: [{
{
upgrade-path: [
{
version: 17.2,
upgrade-type: null
},
{
version: target.latest,
upgrade-type: minor
}
]
},
{
upgrade-path: [
{
version: 17.3,
version: 17.4,
upgrade-type: null
},
{
Expand All @@ -201,7 +189,7 @@ upgrade-version: [{
upgrade-type: major
},
{
version: '15.11',
version: '15.12',
upgrade-type: major
},
{
Expand All @@ -227,7 +215,7 @@ upgrade-version: [{
{
upgrade-path: [
{
version: '15.11',
version: '15.12',
upgrade-type: null
},
{
Expand All @@ -240,7 +228,7 @@ upgrade-version: [{
{
upgrade-path: [
{
version: '15.11',
version: '15.12',
upgrade-type: null
},
{
Expand All @@ -257,7 +245,7 @@ upgrade-version: [{
upgrade-type: null
},
{
version: '15.11',
version: '15.12',
upgrade-type: minor
},
{
Expand All @@ -270,7 +258,7 @@ upgrade-version: [{
{
upgrade-path: [
{
version: 16.7,
version: 16.8,
upgrade-type: null,
},
{
Expand All @@ -296,7 +284,7 @@ upgrade-version: [{
{
upgrade-path: [
{
version: 16.8,
version: 16.9,
upgrade-type: null
},
{
Expand Down
65 changes: 65 additions & 0 deletions .github/scripts/scan-warnings.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
#!/usr/bin/bash
LOG_FILE=~/psql/data/logfile
# File to store the current warnings
CURRENT_WARNINGS="actual-leak-warnings.out"

# Function to extract and classify warnings
extract_and_classify_warnings() {
grep "WARNING:" "$LOG_FILE" |
awk -F"WARNING: " '{print $2}' |
sort |
while read -r line; do
case "$line" in
*"snapshot"*"still active"*)
echo "SNAPSHOT_ACTIVE_WARNING: $line"
;;
*"resource was not closed"*)
echo "LEAK_WARNING: $line"
;;
*"leak"*)
echo "LEAK_WARNING: $line"
;;
*"role"*"has not been granted membership"*|*"permission denied"*|*"transaction left non-empty SPI stack"*|*"cast will be ignored because the source data type is a domain"*|*"Query parsing failed using SLL parser mode but succeeded with LL mode"*|*"Using the TDS Foreign data wrapper (tds_fdw) as provider"*|*"Ignoring @provstr argument value"*|*"Product version setting by babelfishpg_tds.product_version GUC will have no effect on @@VERSION"*|*"no privileges could be revoked for"*|*"could not convert locale name"*|*"you don't own a lock of type"*|*"precision reduced to maximum allowed"*|*"WITH TIME ZONE precision reduced to maximum allowed"*|*"no privileges were granted for"*|*"parameter cannot be null"*|*"babelfishpg_tds.product_version cannot be set"*|*"could not convert locale name"*|*"This function has been deprecated and will no longer drop all users."*|*"cannot add relations to publication:"*|*"NUMERIC or DECIMAL type is cast to BIGINT"*)
# These warnings are expected and will be ignored
;;
*)
echo "OTHER_WARNING: $line"
;;
esac
done
}

# Extract and classify warnings
extract_and_classify_warnings > "$CURRENT_WARNINGS"
ERROR_FOUND=false
# Check for other warning
OTHER_WARNING_COUNT=$(grep -c "OTHER_WARNING:" "$CURRENT_WARNINGS" || echo 0)

if [[ "$OTHER_WARNING_COUNT" -ge 1 ]]; then
echo "Error: $OTHER_WARNING_COUNT unexpected warning type(s) detected."
grep "OTHER_WARNING:" "$CURRENT_WARNINGS"
ERROR_FOUND=true
fi
# Count warnings
SNAPSHOT_ACTIVE_COUNT=$(grep -c "SNAPSHOT_ACTIVE_WARNING:" "$CURRENT_WARNINGS")
LEAK_COUNT=$(grep -c "LEAK_WARNING:" "$CURRENT_WARNINGS")

if [[ "$SNAPSHOT_ACTIVE_COUNT" -ne 44 ]]; then
echo "Error: Expected 44 'snapshot ... still active' warnings, but found $SNAPSHOT_ACTIVE_COUNT"
ERROR_FOUND=true
fi

if [[ "$LEAK_COUNT" -ne 350 ]]; then
echo "Error: Expected 324 leak warnings, but found $LEAK_COUNT"
ERROR_FOUND=true
fi

if [[ "$ERROR_FOUND" = true ]]; then
echo "Unexpected warning counts detected. Please investigate."
echo "Snapshot 'still active' warnings: $SNAPSHOT_ACTIVE_COUNT"
echo "Leak warnings: $LEAK_COUNT"
exit 1
fi
echo "Warning counts are as expected:"
echo "Snapshot 'still active' warnings: $SNAPSHOT_ACTIVE_COUNT"
echo "Leak warnings: $LEAK_COUNT"
25 changes: 11 additions & 14 deletions .github/template/version-branch-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@
'14.15':
engine_branch: BABEL_2_11_STABLE__PG_14_15
extension_branch: BABEL_2_11_STABLE
'14.16':
engine_branch: BABEL_2_12_STABLE__PG_14_16
extension_branch: BABEL_2_12_STABLE
'14.17':
engine_branch: BABEL_2_12_STABLE__PG_14_17
extension_branch: BABEL_2_12_STABLE
'14.18':
engine_branch: BABEL_2_X_DEV__PG_14_X
extension_branch: BABEL_2_X_DEV
'15.2':
Expand All @@ -79,10 +79,10 @@
'15.10':
engine_branch: BABEL_3_8_STABLE__PG_15_10
extension_branch: BABEL_3_8_STABLE
'15.11':
engine_branch: BABEL_3_9_STABLE__PG_15_11
extension_branch: BABEL_3_9_STABLE
'15.12':
engine_branch: BABEL_3_9_STABLE__PG_15_12
extension_branch: BABEL_3_9_STABLE
'15.13':
engine_branch: BABEL_3_X_DEV__PG_15_X
extension_branch: BABEL_3_X_DEV
'16.1':
Expand All @@ -100,17 +100,14 @@
'16.6':
engine_branch: BABEL_4_4_STABLE__PG_16_6
extension_branch: BABEL_4_4_STABLE
'16.7':
engine_branch: BABEL_4_5_STABLE__PG_16_7
extension_branch: BABEL_4_5_STABLE
'16.8':
engine_branch: BABEL_4_5_STABLE__PG_16_8
extension_branch: BABEL_4_5_STABLE
'16.9':
engine_branch: BABEL_4_X_DEV__PG_16_X
extension_branch: BABEL_4_X_DEV
'17.2':
engine_branch: BABEL_5_0_STABLE__PG_17_2
extension_branch: BABEL_5_0_STABLE
'17.3':
engine_branch: BABEL_5_1_STABLE__PG_17_3
'17.4':
engine_branch: BABEL_5_1_STABLE__PG_17_4
extension_branch: BABEL_5_1_STABLE
'source.latest':
engine_branch: latest
Expand Down
14 changes: 11 additions & 3 deletions .github/workflows/code-coverage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,18 @@ jobs:
if: always() && steps.install-extensions.outcome == 'success'
timeout-minutes: 60
uses: ./.github/composite-actions/run-jdbc-tests

- name: Drop and re-create Babelfish database
id: re-install-extensions-1
if: always() && steps.install-extensions.outcome == 'success'
run: |
sudo ~/psql/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="babelfish_db" -f .github/scripts/cleanup_babelfish_database.sql
sudo ~/psql/bin/psql -v ON_ERROR_STOP=1 -d postgres -U runner -v user="jdbc_user" -v db="babelfish_db" -v migration_mode="multi-db" -v tsql_port=1433 -v parallel_query_mode=false -f .github/scripts/create_extension.sql
sqlcmd -S localhost -U "jdbc_user" -P 12345678 -Q "SELECT @@version GO"
- name: Run Dotnet Tests
id: install-and-run-dotnet
if: always() && steps.install-extensions.outcome == 'success'
if: always() && steps.re-install-extensions-1.outcome == 'success'
uses: ./.github/composite-actions/install-and-run-dotnet

- name: Run ODBC Tests
Expand All @@ -78,7 +86,7 @@ jobs:
uses: ./.github/composite-actions/install-and-run-odbc

- name: Drop and re-create Babelfish database
id: re-install-extensions
id: re-install-extensions-2
if: always() && steps.install-extensions.outcome == 'success'
run: |
sudo ~/psql/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="babelfish_db" -f .github/scripts/cleanup_babelfish_database.sql
Expand All @@ -87,7 +95,7 @@ jobs:
- name: Run Python Tests
id: install-and-run-python
if: always() && steps.re-install-extensions.outcome == 'success'
if: always() && steps.re-install-extensions-2.outcome == 'success'
uses: ./.github/composite-actions/install-and-run-python

- name: Generate code coverage HTML report
Expand Down
11 changes: 10 additions & 1 deletion .github/workflows/jdbc-tests-pgaudit-enable.yml
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,18 @@ jobs:
timeout-minutes: 60
uses: ./.github/composite-actions/run-jdbc-tests

- name: Drop and re-create Babelfish database with pgaudit installed
id: re-install-extensions
if: always() && steps.install-extensions.outcome == 'success'
run: |
sudo ~/psql/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="babelfish_db" -f .github/scripts/cleanup_babelfish_database.sql
sudo ~/psql/bin/psql -v ON_ERROR_STOP=1 -d postgres -U runner -v user="jdbc_user" -v db="babelfish_db" -v migration_mode="multi-db" -v tsql_port=1433 -v parallel_query_mode=false -f .github/scripts/create_extension.sql
sudo PGPASSWORD=12345678 ~/${{ env.INSTALL_DIR }}/bin/psql -v ON_ERROR_STOP=1 -h localhost -d babelfish_db -U jdbc_user -c "CREATE EXTENSION pgaudit;"
sqlcmd -S localhost -U "jdbc_user" -P 12345678 -Q "SELECT @@version GO"
- name: Run Dotnet Tests
id: run-dotnet-tests
if: always() && steps.install-extensions.outcome == 'success'
if: always() && steps.re-install-extensions.outcome == 'success'
uses: ./.github/composite-actions/install-and-run-dotnet

- name: Start secondary server
Expand Down
14 changes: 14 additions & 0 deletions .github/workflows/jdbc-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,20 @@ jobs:
~/psql/data/logfile
~/psql/data_5433/logfile
- name: Check for unexpected warnings
if: always()
id: check-warnings
run: |
bash ./.github/scripts/scan-warnings.sh
- name: Upload Unexpected Warning
if: always() && steps.check-warnings.outcome == 'failure'
uses: actions/upload-artifact@v4
with:
name: unexpected_warnings.log
path: |
actual-leak-warnings.out
# The test summary files contain paths with ':' characters, which is not allowed with the upload-artifact actions
- name: Rename Test Summary Files
id: test-file-rename
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/minor-version-upgrade.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ on: [push, pull_request]
jobs:
extension-tests:
env:
ENGINE_VER_FROM: BABEL_5_0_STABLE__PG_17_2
EXTENSION_VER_FROM: BABEL_5_0_STABLE
ENGINE_VER_FROM: BABEL_5_1_STABLE__PG_17_4
EXTENSION_VER_FROM: BABEL_5_1_STABLE
INSTALL_DIR: psql

name: Build and test
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pg_dump-restore-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ jobs:
install_dir: ${{ steps.find-branch.outputs.base-dir }}
migration_mode: 'multi-db'
dump_restore: 'true'
antlr_version: ${{ (steps.read-base-and-final-version.outputs.base-version == 'source.latest' || fromJSON(steps.read-base-and-final-version.outputs.base-version) >= 16.7 ) && '4.13.2' || '4.9.3' }}
antlr_version: ${{ (steps.read-base-and-final-version.outputs.base-version == 'source.latest' || fromJSON(steps.read-base-and-final-version.outputs.base-version) >= 16.8 ) && '4.13.2' || '4.9.3' }}

- name: Setup Dump Restore Composite Action
id: setup-dump-restore-ca
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/upgrade-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ jobs:
install_dir: ${{ steps.find-branch.outputs.base-dir }}
migration_mode: 'multi-db'
server_collation_name: ${{ matrix.upgrade-path.server_collation_name }}
antlr_version: ${{ (steps.read-base-and-final-version.outputs.base-version == 'source.latest' || fromJSON(steps.read-base-and-final-version.outputs.base-version) >= 16.7 ) && '4.13.2' || '4.9.3' }}
antlr_version: ${{ (steps.read-base-and-final-version.outputs.base-version == 'source.latest' || fromJSON(steps.read-base-and-final-version.outputs.base-version) >= 16.8 ) && '4.13.2' || '4.9.3' }}

- name: Setup Upgrade Version Composite Action
id: setup-upgrade-ver-ca
Expand Down
Loading

0 comments on commit 0cc9089

Please sign in to comment.