From 736be170df454e8010d5bfc1b3f623a6436d8538 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Mon, 20 Jan 2025 15:43:35 +0100 Subject: [PATCH 1/3] Reapply "DMD-33 exasol CI delete" This reverts commit 4b2b3641b05990f1adfb2d144380ffdcdb03b988. --- .../build-php-table-backend-utils.yml | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/.github/workflows/build-php-table-backend-utils.yml b/.github/workflows/build-php-table-backend-utils.yml index 5b618c3c1..7bc7486b3 100644 --- a/.github/workflows/build-php-table-backend-utils.yml +++ b/.github/workflows/build-php-table-backend-utils.yml @@ -51,10 +51,6 @@ env: TERADATA_PASSWORD: ${{ secrets.TERADATA_PASSWORD }} TERADATA_PORT: 1025 TERADATA_DATABASE: ci_table_utils - # Exasol - EXASOL_HOST: exasol:8563 - EXASOL_USERNAME: sys - EXASOL_PASSWORD: exasol # Bigquery BQ_KEY_FILE: ${{ secrets.BQ_KEY_FILE }} @@ -126,7 +122,7 @@ jobs: strategy: fail-fast: false matrix: - suite: [exasol, snowflake, bigquery] + suite: [snowflake, bigquery] steps: - name: Checkout @@ -155,14 +151,6 @@ jobs: docker pull $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG docker tag $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $LOCAL_IMAGE:latest # - # Before tests, start services - - - name: Exasol start - if: ${{ matrix.suite == 'exasol' }} - run: | - docker compose up -d exasol - sleep 60 - # # TESTS # - @@ -172,9 +160,3 @@ jobs: DOCKER_BUILDKIT: 1 run: | docker compose run ci-php-table-backend-utils composer tests-${{ matrix.suite }} - # - # After tests, stop services - - - name: Exasol stop - if: ${{ always() && matrix.suite == 'exasol' }} - run: docker compose stop exasol From 34b2acfab63061a22014910a8148e3207babcdaa Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Mon, 20 Jan 2025 15:43:35 +0100 Subject: [PATCH 2/3] Revert "DMD-33 exasol CI delete" This reverts commit 953cd8bc5b9f68344a5579e9e4303300f2bcf34e. --- .../workflows/build-php-db-import-export.yml | 93 ++++++++++++++++++- .../build-php-table-backend-utils.yml | 20 +++- .github/workflows/main.yml | 2 + 3 files changed, 113 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-php-db-import-export.yml b/.github/workflows/build-php-db-import-export.yml index 835ad9814..2319cd5d8 100644 --- a/.github/workflows/build-php-db-import-export.yml +++ b/.github/workflows/build-php-db-import-export.yml @@ -25,6 +25,10 @@ on: required: true ABS_TERADATA_PASSWORD: required: true + EXASOL_PASSWORD: + required: true + EXA_SAAS_TOKEN: + required: true OAUTH_TOKEN_GITHUB: required: true GCS_CREDENTIALS: @@ -74,6 +78,14 @@ env: ABS_TERADATA_PASSWORD: ${{ secrets.ABS_TERADATA_PASSWORD }} ABS_TERADATA_PORT: 1025 ABS_TERADATA_DATABASE: ci_ielib_abs + # Exasol + EXASOL_HOST: mbgghigkizhshorgb53ivhkrsu.clusters.exasol.com:8563 + EXASOL_USERNAME: devel + EXASOL_PASSWORD: ${{ secrets.EXASOL_PASSWORD }} + EXA_SAAS_DB_ID: 5ThvKt2NQEqTf-QVEBcNeg + EXA_SAAS_HOST: https://cloud.exasol.com + EXA_SAAS_USER_ID: org_W0xrKhY2DEnGwmSO + EXA_SAAS_TOKEN: ${{ secrets.EXA_SAAS_TOKEN }} # Bigquery BQ_KEY_FILE: ${{ secrets.BQ_KEY_FILE }} BQ_BUCKET_NAME: ie-ci-files-bucket @@ -469,6 +481,84 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ env.S3_AWS_SECRET_ACCESS_KEY }} run: docker compose run ci-php-db-import-export composer tests-${{ matrix.suite }} + pre-exasol-tests: + runs-on: ubuntu-latest + needs: build + concurrency: exasol-tests + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Exasol start + run: | + bash ./provisioning/scripts/exasolRun.sh -w || exit 1 + + exasol-tests: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + needs: [load-s3, pre-exasol-tests] + continue-on-error: true + strategy: + fail-fast: false + matrix: + suite: + - exasol + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_CREDENTIALS_ROLE_TO_ASSUME }} + aws-region: ${{ env.AWS_CREDENTIALS_REGION }} + + - + name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - + name: Pull image + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: ${{ env.ECR_REPOSITORY }} + IMAGE_TAG: ci-php-db-import-export-${{ github.sha }} + LOCAL_IMAGE: ${{ env.LOCAL_IMAGE }} + run: | + docker pull $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + docker tag $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $LOCAL_IMAGE:latest + # + # TESTS + # + - + name: Tests ${{ matrix.suite }} + env: + SUITE: ${{ matrix.suite }} + AWS_ACCESS_KEY_ID: ${{ env.S3_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ env.S3_AWS_SECRET_ACCESS_KEY }} + run: docker compose run ci-php-db-import-export composer tests-${{ matrix.suite }} + + post-exasol-tests: + runs-on: ubuntu-latest + needs: exasol-tests + if: | + always() && (needs.exasol-tests.result != 'skipped') + concurrency: exasol-tests + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Exasol stop + if: ${{ always() }} + run: | + bash ./provisioning/scripts/exasolRun.sh -p || exit 1 + # # Clean up uploaded data clean-s3: @@ -476,12 +566,13 @@ jobs: permissions: id-token: write contents: read - needs: [testsS3, testsAll] + needs: [testsS3, testsAll, exasol-tests] if: | always() && ( (needs.testsS3.result != 'skipped') || (needs.testsAll.result != 'skipped') + || (needs.exasol-tests.result != 'skipped') ) steps: - diff --git a/.github/workflows/build-php-table-backend-utils.yml b/.github/workflows/build-php-table-backend-utils.yml index 7bc7486b3..5b618c3c1 100644 --- a/.github/workflows/build-php-table-backend-utils.yml +++ b/.github/workflows/build-php-table-backend-utils.yml @@ -51,6 +51,10 @@ env: TERADATA_PASSWORD: ${{ secrets.TERADATA_PASSWORD }} TERADATA_PORT: 1025 TERADATA_DATABASE: ci_table_utils + # Exasol + EXASOL_HOST: exasol:8563 + EXASOL_USERNAME: sys + EXASOL_PASSWORD: exasol # Bigquery BQ_KEY_FILE: ${{ secrets.BQ_KEY_FILE }} @@ -122,7 +126,7 @@ jobs: strategy: fail-fast: false matrix: - suite: [snowflake, bigquery] + suite: [exasol, snowflake, bigquery] steps: - name: Checkout @@ -151,6 +155,14 @@ jobs: docker pull $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG docker tag $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $LOCAL_IMAGE:latest # + # Before tests, start services + - + name: Exasol start + if: ${{ matrix.suite == 'exasol' }} + run: | + docker compose up -d exasol + sleep 60 + # # TESTS # - @@ -160,3 +172,9 @@ jobs: DOCKER_BUILDKIT: 1 run: | docker compose run ci-php-table-backend-utils composer tests-${{ matrix.suite }} + # + # After tests, stop services + - + name: Exasol stop + if: ${{ always() && matrix.suite == 'exasol' }} + run: docker compose stop exasol diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f3b8f815b..743c5222c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -123,6 +123,8 @@ jobs: AZURE_CLIENT_SECRET: ${{ secrets.IE_AZURE_CLIENT_SECRET }} TERADATA_PASSWORD: ${{ secrets.IE_TERADATA_PASSWORD }} ABS_TERADATA_PASSWORD: ${{ secrets.IE_ABS_TERADATA_PASSWORD }} + EXASOL_PASSWORD: ${{ secrets.IE_EXASOL_PASSWORD }} + EXA_SAAS_TOKEN: ${{ secrets.IE_EXA_SAAS_TOKEN }} BQ_KEY_FILE: ${{ secrets.IE_BQ_KEY_FILE }} OAUTH_TOKEN_GITHUB: ${{ secrets.IE_OAUTH_TOKEN_GITHUB }} GCS_CREDENTIALS: ${{ secrets.IE_GCS_CREDENTIALS }} From bf30d04e9bed7500493b8ba04c0ecab21cfd526b Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Mon, 20 Jan 2025 15:45:13 +0100 Subject: [PATCH 3/3] reactivate exasol credentials --- .github/workflows/build-php-db-import-export.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-php-db-import-export.yml b/.github/workflows/build-php-db-import-export.yml index 2319cd5d8..ffb8be880 100644 --- a/.github/workflows/build-php-db-import-export.yml +++ b/.github/workflows/build-php-db-import-export.yml @@ -79,10 +79,10 @@ env: ABS_TERADATA_PORT: 1025 ABS_TERADATA_DATABASE: ci_ielib_abs # Exasol - EXASOL_HOST: mbgghigkizhshorgb53ivhkrsu.clusters.exasol.com:8563 - EXASOL_USERNAME: devel + EXASOL_HOST: 6kdmpm4k3rczxomktqwtt4mz4e.clusters.exasol.com:8563 + EXASOL_USERNAME: ci_storage_backend_ie EXASOL_PASSWORD: ${{ secrets.EXASOL_PASSWORD }} - EXA_SAAS_DB_ID: 5ThvKt2NQEqTf-QVEBcNeg + EXA_SAAS_DB_ID: O7N7G6VbS6acYhvQ1wYOgQ EXA_SAAS_HOST: https://cloud.exasol.com EXA_SAAS_USER_ID: org_W0xrKhY2DEnGwmSO EXA_SAAS_TOKEN: ${{ secrets.EXA_SAAS_TOKEN }}