diff --git a/.github/workflows/dev-branch-integration-tests.yml b/.github/workflows/dev-branch-integration-tests.yml index 62a1da210..ace39b778 100644 --- a/.github/workflows/dev-branch-integration-tests.yml +++ b/.github/workflows/dev-branch-integration-tests.yml @@ -68,6 +68,11 @@ jobs: REDSHIFT_TEST_DBNAME: ${{ vars.REDSHIFT_TEST_DBNAME }} REDSHIFT_TEST_USER: ${{ vars.REDSHIFT_TEST_USER }} REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_PROFILE }} + REDSHIFT_TEST_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_ACCESS_KEY_ID }} + REDSHIFT_TEST_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} DBT_TEST_USER_1: ${{ vars.REDSHIFT_TEST_USER_1 }} DBT_TEST_USER_2: ${{ vars.REDSHIFT_TEST_USER_2 }} DBT_TEST_USER_3: ${{ vars.REDSHIFT_TEST_USER_3 }} @@ -104,6 +109,13 @@ jobs: dbt-common-branch: ${{ inputs.dbt-common-branch }} dbt-core-branch: ${{ inputs.dbt-core-branch }} + - name: Create AWS IAM profile + run: | + aws configure set aws_access_key_id $REDSHIFT_TEST_ACCESS_KEY_ID + aws configure set aws_secret_access_key $REDSHIFT_TEST_SECRET_ACCESS_KEY + aws configure set region $REDSHIFT_TEST_REGION + aws configure set output json + - name: "Run integration tests" shell: bash run: hatch run integration-tests:all diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml deleted file mode 100644 index 3fac0b0db..000000000 --- a/.github/workflows/integration.yml +++ /dev/null @@ -1,259 +0,0 @@ -# **what?** -# This workflow runs all integration tests for supported OS -# and python versions and core adapters. If triggered by PR, -# the workflow will only run tests for adapters related -# to code changes. Use the `test all` and `test ${adapter}` -# label to run all or additional tests. Use `ok to test` -# label to mark PRs from forked repositories that are safe -# to run integration tests for. Requires secrets to run -# against different warehouses. - -# **why?** -# This checks the functionality of dbt from a user's perspective -# and attempts to catch functional regressions. - -# **when?** -# This workflow will run on every push to a protected branch -# and when manually triggered. It will also run for all PRs, including -# PRs from forks. The workflow will be skipped until there is a label -# to mark the PR as safe to run. - -name: Adapter Integration Tests - -on: - # pushes to release branches - push: - branches: - - "main" - - "develop" - - "*.latest" - - "releases/*" - # all PRs, important to note that `pull_request_target` workflows - # will run in the context of the target branch of a PR - pull_request_target: - # manual trigger - workflow_dispatch: - inputs: - dbt-core-branch: - description: "branch of dbt-core to use in dev-requirements.txt" - required: false - type: string - -# explicitly turn off permissions for `GITHUB_TOKEN` -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -# sets default shell to bash, for all operating systems -defaults: - run: - shell: bash - -jobs: - # generate test metadata about what files changed and the testing matrix to use - test-metadata: - # run if not a PR from a forked repository or has a label to mark as safe to test - if: >- - github.event_name != 'pull_request_target' || - github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'ok to test') - runs-on: ubuntu-latest - - outputs: - matrix: ${{ steps.generate-matrix.outputs.result }} - - steps: - - name: Check out the repository (non-PR) - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - ref: ${{ github.event.pull_request.head.sha }} - - - name: Check if relevant files changed - if: github.event_name == 'pull_request_target' - # https://github.com/marketplace/actions/paths-changes-filter - # For each filter, it sets output variable named by the filter to the text: - # 'true' - if any of changed files matches any of filter rules - # 'false' - if none of changed files matches any of filter rules - # also, returns: - # `changes` - JSON array with names of all filters matching any of the changed files - uses: dorny/paths-filter@v2 - id: get-changes - with: - token: ${{ secrets.GITHUB_TOKEN }} - filters: | - redshift: - - 'dbt/**' - - 'tests/**' - - 'dev-requirements.txt' - - - name: Generate integration test matrix - id: generate-matrix - uses: actions/github-script@v6 - env: - CHANGES: ${{ steps.get-changes.outputs.changes }} - with: - script: | - const script = require('./.github/scripts/integration-test-matrix.js') - const matrix = script({ context }) - console.log(matrix) - return matrix - test: - name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }} - - # run if not a PR from a forked repository or has a label to mark as safe to test - # also checks that the matrix generated is not empty - if: >- - needs.test-metadata.outputs.matrix && - fromJSON( needs.test-metadata.outputs.matrix ).include[0] && - ( - github.event_name != 'pull_request_target' || - github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'ok to test') - ) - runs-on: ${{ matrix.os }} - - needs: test-metadata - - strategy: - fail-fast: false - max-parallel: 3 - matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }} - - env: - TOXENV: integration-${{ matrix.adapter }} - PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" - DBT_INVOCATION_ENV: github-actions - DD_CIVISIBILITY_AGENTLESS_ENABLED: true - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_SITE: datadoghq.com - DD_ENV: ci - DD_SERVICE: ${{ github.event.repository.name }} - - steps: - - name: Check out the repository - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - - # explicity checkout the branch for the PR, - # this is necessary for the `pull_request_target` event - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - ref: ${{ github.event.pull_request.head.sha }} - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - - name: Update dev_requirements.txt - if: inputs.dbt-core-branch != '' - run: | - pip install bumpversion - ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} - - - name: Create AWS IAM profile - run: | - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region $AWS_REGION - aws configure set output json - env: - AWS_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} - - - name: Run tox (redshift) - if: matrix.adapter == 'redshift' - env: - REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} - REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} - REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} - REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} - REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} - REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} - REDSHIFT_TEST_IAM_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_PROFILE }} - REDSHIFT_TEST_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_ACCESS_KEY_ID }} - REDSHIFT_TEST_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_SECRET_ACCESS_KEY }} - DBT_TEST_USER_1: dbt_test_user_1 - DBT_TEST_USER_2: dbt_test_user_2 - DBT_TEST_USER_3: dbt_test_user_3 - run: tox -- --ddtrace - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: logs - path: ./logs - - - name: Get current date - if: always() - id: date - run: | - echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv - path: integration_results.csv - - require-label-comment: - runs-on: ubuntu-latest - - needs: test - - permissions: - pull-requests: write - - steps: - - name: Needs permission PR comment - if: >- - needs.test.result == 'skipped' && - github.event_name == 'pull_request_target' && - github.event.pull_request.head.repo.full_name != github.repository - uses: unsplash/comment-on-pr@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - msg: | - "You do not have permissions to run integration tests, @dbt-labs/core "\ - "needs to label this PR with `ok to test` in order to run integration tests!" - check_for_duplicate_msg: true - - post-failure: - runs-on: ubuntu-latest - needs: test - if: ${{ failure() }} - - steps: - - name: Posting scheduled run failures - uses: ravsamhq/notify-slack-action@v2 - if: ${{ github.event_name == 'schedule' }} - with: - notification_title: 'Redshift nightly integration test failed' - status: ${{ job.status }} - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }}