diff --git a/.github/composite-actions/build-modified-postgres/action.yml b/.github/composite-actions/build-modified-postgres/action.yml index cbd7f086b94..26ac521394d 100644 --- a/.github/composite-actions/build-modified-postgres/action.yml +++ b/.github/composite-actions/build-modified-postgres/action.yml @@ -24,11 +24,11 @@ inputs: runs: using: "composite" steps: - - name: Checkout, Build, and Install the Modified PostgreSQL Instance and Run Tests + - name: Checkout Modified PostgreSQL for Babelfish run: | cd .. rm -rf postgresql_modified_for_babelfish - + if [[ $GITHUB_EVENT_NAME == "pull_request" ]]; then if [[ ${{inputs.engine_branch}} == "latest" ]]; then ENGINE_BRANCH=$GITHUB_HEAD_REF @@ -44,10 +44,47 @@ runs: fi REPOSITORY_OWNER=$GITHUB_REPOSITORY_OWNER fi - $GITHUB_WORKSPACE/.github/scripts/clone_engine_repo "$REPOSITORY_OWNER" "$ENGINE_BRANCH" cd postgresql_modified_for_babelfish - git rev-parse HEAD + rm -rf ~/.ccache + if [[ ${{inputs.tap_tests}} == "yes" ]]; then + echo "CRESTORE_KEY=$(git rev-parse --short HEAD)-tapTest" >> $GITHUB_ENV + elif [[ ${{inputs.code_coverage}} == "yes" ]]; then + echo "CRESTORE_KEY=$(git rev-parse --short HEAD)-coverage" >> $GITHUB_ENV + elif [[ ${{inputs.release_mode}} == "yes" ]]; then + echo "CRESTORE_KEY=$(git rev-parse --short HEAD)-release" >> $GITHUB_ENV + else + echo "CRESTORE_KEY=$(git rev-parse --short HEAD)-default" >> $GITHUB_ENV + fi + shell: bash + + - uses: actions/cache/restore@v4 + id: restore-ccache + if: ${{ github.event_name == 'pull_request' }} || ${{ inputs.code_coverage == 'yes' }} + with: + path: + ~/.ccache + key: + random-random + restore-keys: + ccache-${{ env.CRESTORE_KEY }} + + - name: Save cache if cache hit fails in pull requests + if: ${{ github.event_name == 'pull_request' }} + run: | + if [[ '${{ steps.restore-ccache.outputs.cache-matched-key }}' == '' ]]; then + echo "SAVE_CCACHE=1" >> $GITHUB_ENV + elif [[ ${{ 'steps.restore-ccache.outputs.cache-matched-key' }} != '' ]]; then + echo "SAVE_CCACHE=" >> $GITHUB_ENV + fi + shell: bash + + - name: Build and Install the Modified PostgreSQL Instance and Run Tests + run: | + echo "${{env.SAVE_CCACHE}}" + echo "${{ steps.restore-ccache.outputs.cache-matched-key }}" + cd .. + cd postgresql_modified_for_babelfish if [[ ${{inputs.tap_tests}} == "yes" ]]; then ./configure CC='ccache gcc' --prefix=$HOME/${{ inputs.install_dir }}/ --with-python PYTHON=/usr/bin/python3.8 --enable-cassert CFLAGS="-ggdb" --with-libxml --with-uuid=ossp --with-icu --enable-tap-tests --with-gssapi elif [[ ${{inputs.code_coverage}} == "yes" ]]; then diff --git a/.github/composite-actions/dump-restore-util/action.yml b/.github/composite-actions/dump-restore-util/action.yml index 5d83d289722..fb73b5aeebe 100644 --- a/.github/composite-actions/dump-restore-util/action.yml +++ b/.github/composite-actions/dump-restore-util/action.yml @@ -327,6 +327,10 @@ runs: sqlcmd -S localhost -U jdbc_user -P 12345678 -Q "SELECT @@version GO" shell: bash + - name: Save cache + if: always() + uses: ./.github/composite-actions/save-ccache + - name: Run Verify Tests if: always() && steps.run-pg_dump-restore.outcome == 'success' && inputs.is_final_ver == 'true' uses: ./.github/composite-actions/run-verify-tests diff --git a/.github/composite-actions/install-dependencies/action.yml b/.github/composite-actions/install-dependencies/action.yml index de0912e518e..8f32d6f0d5c 100644 --- a/.github/composite-actions/install-dependencies/action.yml +++ b/.github/composite-actions/install-dependencies/action.yml @@ -15,14 +15,4 @@ runs: sudo /usr/sbin/update-ccache-symlinks echo 'export PATH="/usr/lib/ccache:$PATH"' | tee -a ~/.bashrc source ~/.bashrc && echo $PATH - echo "NOW=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV shell: bash - - - name: Restore ccache - id: cache-compiler - uses: actions/cache@v3 - with: - path: ~/.ccache - key: ccache-${{ runner.os }}-${{ env.NOW }} - restore-keys: | - ccache-${{ runner.os }} diff --git a/.github/composite-actions/minor-version-upgrade-util/action.yml b/.github/composite-actions/minor-version-upgrade-util/action.yml index 00a4d5e79d5..66767d0452a 100644 --- a/.github/composite-actions/minor-version-upgrade-util/action.yml +++ b/.github/composite-actions/minor-version-upgrade-util/action.yml @@ -48,6 +48,10 @@ runs: install_dir: ${{ inputs.install_dir }} extension_branch: ${{ inputs.extension_branch }} + - name: Save cache + if: always() && steps.build-extensions-newer == 'success' + uses: ./.github/composite-actions/save-ccache + # Not created and used composite action update-extensions here since, in the previous step it has # checked out a branch/tag which may not have the updated update-extension composite action - name: Update extensions diff --git a/.github/composite-actions/save-ccache/action.yml b/.github/composite-actions/save-ccache/action.yml new file mode 100644 index 00000000000..2409fbc9036 --- /dev/null +++ b/.github/composite-actions/save-ccache/action.yml @@ -0,0 +1,39 @@ +name: 'Save ccache on push' + +runs: + using: "composite" + steps: + + - name: Setup new cache key + if: always() + run: | + echo "CCACHE_KEY=${{env.CRESTORE_KEY}}-$(git rev-parse --short HEAD)" >> $GITHUB_ENV + echo "Event : ${{ github.event_name }}" + echo "Save ccache : ${{ env.SAVE_CCACHE }}" + if [[ ${{ github.event_name != 'pull_request' || env.SAVE_CCACHE == 1 }} == true ]]; then + echo "Cache to be Saved" + echo "SAVE_CACHE_HERE=1" >> $GITHUB_ENV + else + echo "Cache shouldn't be Saved" + echo "SAVE_CACHE_HERE=0" >> $GITHUB_ENV + fi + shell: bash + + - name: Save ccache + uses: actions/cache/save@v4 + if: env.SAVE_CACHE_HERE == 1 + with: + path: + ~/.ccache + key: + ccache-${{ env.CCACHE_KEY }} + + - name: Clean ccache directory and unset env variables + if: always() + shell: bash + run: | + rm -rf ~/.ccache + echo "CCACHE_KEY=" >> $GITHUB_ENV + echo "SAVE_CCACHE=" >> $GITHUB_ENV + echo "SAVE_CACHE_HERE=" >> $GITHUB_ENV + echo "CRESTORE_KEY=" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/composite-actions/setup-base-version/action.yml b/.github/composite-actions/setup-base-version/action.yml index 628518778c2..9d6f6b96126 100644 --- a/.github/composite-actions/setup-base-version/action.yml +++ b/.github/composite-actions/setup-base-version/action.yml @@ -221,6 +221,10 @@ runs: - uses: actions/checkout@v2 + - name: Save cache + if: always() && steps.jdbc-upgrade-tests.outcome == 'success' + uses: ./.github/composite-actions/save-ccache + - name: Install Python id: install-python if: ${{ matrix.upgrade-path.path[0] == 'source_latest' && steps.jdbc-upgrade-tests.outcome == 'success' }} diff --git a/.github/composite-actions/setup-new-version/action.yml b/.github/composite-actions/setup-new-version/action.yml index 2138d1cf418..aaa1c0adf0a 100644 --- a/.github/composite-actions/setup-new-version/action.yml +++ b/.github/composite-actions/setup-new-version/action.yml @@ -64,6 +64,10 @@ runs: with: install_dir: ${{ inputs.pg_new_dir }} + - name: Save cache + if: always() && steps.build-postgis-extension == 'success' + uses: ./.github/composite-actions/save-ccache + - name: Setup new data directory id: setup-new-datadir if: always() && steps.build-postgis-extension.outcome == 'success' diff --git a/.github/composite-actions/upload-coredump/action.yml b/.github/composite-actions/upload-coredump/action.yml index d7540cce7a9..04fd12f12d4 100644 --- a/.github/composite-actions/upload-coredump/action.yml +++ b/.github/composite-actions/upload-coredump/action.yml @@ -14,7 +14,7 @@ runs: - name: Upload Coredumps - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: coredumps diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 8d8c4f1edae..c039fb6b2e5 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -119,28 +119,28 @@ jobs: lcov --list lcov.info - name: Upload Coverage Report for babelfishpg_tsql extension if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage_tsql_${{github.ref_name}} path: contrib/babelfishpg_tsql/coverage/ - name: Upload Coverage Report for babelfishpg_tds extension if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage_tds_${{github.ref_name}} path: contrib/babelfishpg_tds/coverage/ - name: Upload Coverage Report for babelfishpg_common extension if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage_common_${{github.ref_name}} path: contrib/babelfishpg_common/coverage/ - name: Upload Coverage Report for babelfishpg_money extension if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage_money_${{github.ref_name}} path: contrib/babelfishpg_money/coverage/ @@ -163,7 +163,7 @@ jobs: - name: Upload CSV report with latest coverage numbers if: (github.event_name == 'schedule') - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: csv_${{github.ref_name}} path: contrib/${{github.ref_name}}.csv diff --git a/.github/workflows/dotnet-tests.yml b/.github/workflows/dotnet-tests.yml index 32d3936118c..476d6e1f626 100644 --- a/.github/workflows/dotnet-tests.yml +++ b/.github/workflows/dotnet-tests.yml @@ -40,6 +40,10 @@ jobs: id: install-extensions if: always() && steps.build-postgis-extension.outcome == 'success' uses: ./.github/composite-actions/install-extensions + + - name: Save cache + if: always() && steps.install-extensions.outcome == 'success' + uses: ./.github/composite-actions/save-ccache - name: Run Dotnet Tests id: run-dotnet-tests @@ -72,7 +76,7 @@ jobs: - name: Upload Coverage Report for Babelfish Extensions if: always() && steps.code-coverage-summary.outcome == 'success' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage-babelfish-extensions-dotnet path: contrib/dotnet-lcov.info diff --git a/.github/workflows/isolation-tests.yml b/.github/workflows/isolation-tests.yml index af8c7d3916a..024cbc03b4b 100644 --- a/.github/workflows/isolation-tests.yml +++ b/.github/workflows/isolation-tests.yml @@ -105,7 +105,7 @@ jobs: - name: Upload Coverage Report for Babelfish Extensions if: always() && steps.code-coverage-summary.outcome == 'success' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage-babelfish-extensions-isolation path: contrib/isolation-lcov.info diff --git a/.github/workflows/jdbc-tests-single-db-mode.yml b/.github/workflows/jdbc-tests-single-db-mode.yml index 8f470d66b5a..eb36a0d9d24 100644 --- a/.github/workflows/jdbc-tests-single-db-mode.yml +++ b/.github/workflows/jdbc-tests-single-db-mode.yml @@ -102,6 +102,10 @@ jobs: ~/psql/data/logfile ~/psql/data_5433/logfile + - name: Save cache + if: always() && steps.replication.outcome == 'success' + uses: ./.github/composite-actions/save-ccache + # The test summary files contain paths with ':' characters, which is not allowed with the upload-artifact actions - name: Rename Test Summary Files id: test-file-rename diff --git a/.github/workflows/jdbc-tests.yml b/.github/workflows/jdbc-tests.yml index 77e71dd0cbe..26b5b1b3c4d 100644 --- a/.github/workflows/jdbc-tests.yml +++ b/.github/workflows/jdbc-tests.yml @@ -167,7 +167,7 @@ jobs: - name: Upload Coverage Report for Babelfish Extensions if: always() && steps.code-coverage-summary.outcome == 'success' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage-babelfish-extensions-jdbc path: contrib/jdbc-lcov.info diff --git a/.github/workflows/major-version-upgrade.yml b/.github/workflows/major-version-upgrade.yml index e98cedce619..7dd3f2dfcb6 100644 --- a/.github/workflows/major-version-upgrade.yml +++ b/.github/workflows/major-version-upgrade.yml @@ -20,18 +20,11 @@ jobs: - name: Build Modified Postgres using ${{env.ENGINE_BRANCH_FROM}} id: build-modified-postgres-old + uses: ./.github/composite-actions/build-modified-postgres if: always() && steps.install-dependencies.outcome == 'success' - run: | - cd .. - git clone --branch ${{env.ENGINE_BRANCH_FROM}} https://github.com/babelfish-for-postgresql/postgresql_modified_for_babelfish.git - cd postgresql_modified_for_babelfish - ./configure --prefix=$HOME/${{env.OLD_INSTALL_DIR}} --with-python PYTHON=/usr/bin/python3.8 --enable-cassert CFLAGS="-ggdb" --with-libxml --with-uuid=ossp --with-icu - make clean - make -j 4 - make install - make check - cd contrib && make && sudo make install - shell: bash + with: + engine_branch: ${{ env.ENGINE_BRANCH_FROM }} + install_dir: ${{ env.OLD_INSTALL_DIR }} - name: Compile ANTLR id: compile-antlr @@ -130,6 +123,10 @@ jobs: - uses: actions/checkout@v2 + - name: Save cache + if: always() && steps.build-extensions-old.outcome == 'success' + uses: ./.github/composite-actions/save-ccache + - name: Build Modified Postgres using latest version id: build-modified-postgres-new if: always() && steps.install-extensions-old.outcome == 'success' @@ -172,6 +169,10 @@ jobs: uses: ./.github/composite-actions/build-postgis-extension with: install_dir: ${{env.NEW_INSTALL_DIR}} + + - name: Save cache + if: always() && steps.build-postgis-extension.outcome == 'success' + uses: ./.github/composite-actions/save-ccache - name: Setup new data directory id: setup-new-datadir diff --git a/.github/workflows/minor-version-upgrade.yml b/.github/workflows/minor-version-upgrade.yml index 3adc33bf2fc..a5d41afda81 100644 --- a/.github/workflows/minor-version-upgrade.yml +++ b/.github/workflows/minor-version-upgrade.yml @@ -111,6 +111,10 @@ jobs: - uses: actions/checkout@v2 + - name: Save cache + if: always() && steps.build-extensions-older.outcome == 'success' + uses: ./.github/composite-actions/save-ccache + - name: Build and run tests for Postgres engine using latest engine id: build-modified-postgres-newer if: always() && steps.install-extensions-older.outcome == 'success' diff --git a/.github/workflows/odbc-tests.yml b/.github/workflows/odbc-tests.yml index 83da6ec34b8..89996b6f2b2 100644 --- a/.github/workflows/odbc-tests.yml +++ b/.github/workflows/odbc-tests.yml @@ -72,7 +72,7 @@ jobs: - name: Upload Coverage Report for Babelfish Extensions if: always() && steps.code-coverage-summary.outcome == 'success' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage-babelfish-extensions-odbc path: contrib/odbc-lcov.info diff --git a/.github/workflows/pr-code-coverage.yml b/.github/workflows/pr-code-coverage.yml index 476c2d7eb44..886e29c3a18 100644 --- a/.github/workflows/pr-code-coverage.yml +++ b/.github/workflows/pr-code-coverage.yml @@ -37,31 +37,31 @@ jobs: run: | sudo apt-get install lcov - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 id: download-jdbc-coverage with: name:  coverage-babelfish-extensions-jdbc path: contrib/ - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 id: download-dotnet-coverage with: name: coverage-babelfish-extensions-dotnet path: contrib/ - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 id: download-odbc-coverage with: name: coverage-babelfish-extensions-odbc path: contrib/ - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 id: download-python-coverage with: name: coverage-babelfish-extensions-python path: contrib/ - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 id: download-isolation-coverage with: name: coverage-babelfish-extensions-isolation diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index f9e1c8d5cd4..63f8efa8095 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -72,7 +72,7 @@ jobs: - name: Upload Coverage Report for Babelfish Extensions if: always() && steps.code-coverage-summary.outcome == 'success' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage-babelfish-extensions-python path: contrib/python-lcov.info diff --git a/.github/workflows/tap-tests.yml b/.github/workflows/tap-tests.yml index e0bc7704dc8..c4b024af1c3 100644 --- a/.github/workflows/tap-tests.yml +++ b/.github/workflows/tap-tests.yml @@ -45,7 +45,8 @@ jobs: with: engine_branch: ${{env.ENGINE_BRANCH_16}} install_dir: ${{env.INSTALL_DIR_16}} - + + - name: Build Modified Postgres using ${{env.ENGINE_BRANCH_OLD}} id: build-modified-postgres-old if: always() && steps.build-modified-postgres-16.outcome == 'success' @@ -100,6 +101,8 @@ jobs: sudo make USE_PGXS=1 PG_CONFIG=~/psql_source/bin/pg_config install shell: bash + + - name: Build Extensions using ${{env.EXTENSION_BRANCH_OLD}} id: build-extensions-old if: always() && steps.build-postgis-extension-old.outcome == 'success' @@ -110,6 +113,10 @@ jobs: - uses: actions/checkout@v2 + - name: Save cache + if: always() && steps.build-extensions-old.outcome == 'success' + uses: ./.github/composite-actions/save-ccache + - name: Build Modified Postgres using latest version id: build-modified-postgres-new if: always() && steps.build-extensions-old.outcome == 'success' @@ -117,6 +124,8 @@ jobs: with: tap_tests: 'yes' install_dir: ${{env.NEW_INSTALL_DIR}} + + - name: Compile new ANTLR id: compile-new-antlr @@ -139,6 +148,10 @@ jobs: with: install_dir: ${{env.NEW_INSTALL_DIR}} + - name: Save cache + if: always() && steps.build-extensions-old.outcome == 'success' + uses: ./.github/composite-actions/save-ccache + - name: Run TAP Tests id: tap if: always() && steps.build-postgis-extension.outcome == 'success'