integration.yml

# **what?**
# This workflow runs all integration tests for supported OS
# and python versions and core adapters. If triggered by PR,
# the workflow will only run tests for adapters related
# to code changes. Use the `test all` and `test ${adapter}`
# label to run all or additional tests. Use `ok to test`
# label to mark PRs from forked repositories that are safe
# to run integration tests for. Requires secrets to run
# against different warehouses.

# **why?**
# This checks the functionality of dbt from a user's perspective
# and attempts to catch functional regressions.

# **when?**
# This workflow will run on every push to a protected branch
# and when manually triggered. It will also run for all PRs, including
# PRs from forks. The workflow will be skipped until there is a label
# to mark the PR as safe to run.

name: Adapter Integration Tests

on:
  # pushes to release branches
  push:
    branches:
      - "main"
      - "develop"
      - "*.latest"
      - "releases/*"
  # all PRs, important to note that `pull_request_target` workflows
  # will run in the context of the target branch of a PR
  pull_request_target:
  # manual trigger
  workflow_dispatch:
    inputs:
      dbt-core-branch:
        description: "branch of dbt-core to use in dev-requirements.txt"
        required: false
        type: string

# explicitly turn off permissions for `GITHUB_TOKEN`
permissions: read-all

# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
  cancel-in-progress: true

# sets default shell to bash, for all operating systems
defaults:
  run:
    shell: bash

jobs:
  # generate test metadata about what files changed and the testing matrix to use
  test-metadata:
    # run if not a PR from a forked repository or has a label to mark as safe to test
    if: >-
      github.event_name != 'pull_request_target' ||
      github.event.pull_request.head.repo.full_name == github.repository ||
      contains(github.event.pull_request.labels.*.name, 'ok to test')
    runs-on: ubuntu-latest

    outputs:
      matrix: ${{ steps.generate-matrix.outputs.result }}
      run-python-tests: ${{ steps.filter.outputs.bigquery-python }}

    steps:
      - name: Check out the repository (non-PR)
        if: github.event_name != 'pull_request_target'
        uses: actions/checkout@v4
        with:
          persist-credentials: false

      - name: Check out the repository (PR)
        if: github.event_name == 'pull_request_target'
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Check if relevant files changed
        if: github.event_name == 'pull_request_target'
        # https://github.com/marketplace/actions/paths-changes-filter
        # For each filter, it sets output variable named by the filter to the text:
        #  'true' - if any of changed files matches any of filter rules
        #  'false' - if none of changed files matches any of filter rules
        # also, returns:
        #  `changes` - JSON array with names of all filters matching any of the changed files
        uses: dorny/paths-filter@v3
        id: get-changes
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          filters: |
            bigquery:
              - 'dbt/**'
              - 'tests/**'
              - 'dev-requirements.txt'

      - name: Generate integration test matrix
        id: generate-matrix
        uses: actions/github-script@v7
        env:
          CHANGES: ${{ steps.get-changes.outputs.changes }}
        with:
          script: |
            const script = require('./.github/scripts/integration-test-matrix.js')
            const matrix = script({ context })
            console.log(matrix)
            return matrix
  test:
    name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }}

    # run if not a PR from a forked repository or has a label to mark as safe to test
    # also checks that the matrix generated is not empty
    if: >-
      needs.test-metadata.outputs.matrix &&
      fromJSON( needs.test-metadata.outputs.matrix ).include[0] &&
      (
        github.event_name != 'pull_request_target' ||
        github.event.pull_request.head.repo.full_name == github.repository ||
        contains(github.event.pull_request.labels.*.name, 'ok to test')
      )
    runs-on: ${{ matrix.os }}

    needs: test-metadata

    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }}

    env:
      TOXENV: integration-${{ matrix.adapter }}
      PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
      DBT_INVOCATION_ENV: github-actions
      DD_CIVISIBILITY_AGENTLESS_ENABLED: true
      DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
      DD_SITE: datadoghq.com
      DD_ENV: ci
      DD_SERVICE: ${{ github.event.repository.name }}

    steps:
      - name: Check out the repository
        if: github.event_name != 'pull_request_target'
        uses: actions/checkout@v4
        with:
          persist-credentials: false

      # explicity checkout the branch for the PR,
      # this is necessary for the `pull_request_target` event
      - name: Check out the repository (PR)
        if: github.event_name == 'pull_request_target'
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install python dependencies
        run: |
          python -m pip install --user --upgrade pip
          python -m pip install tox
          python -m pip --version
          tox --version

      - name: Update dev_requirements.txt
        if: inputs.dbt-core-branch != ''
        run: |
          pip install bumpversion
          ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}

      - name: Run tox (bigquery)
        if: matrix.adapter == 'bigquery'
        env:
          BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }}
          BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }}
          BIGQUERY_TEST_NO_ACCESS_DATABASE: ${{ secrets.BIGQUERY_TEST_NO_ACCESS_DATABASE }}
          DBT_TEST_USER_1: group:buildbot@dbtlabs.com
          DBT_TEST_USER_2: group:engineering-core-team@dbtlabs.com
          DBT_TEST_USER_3: serviceAccount:dbt-integration-test-user@dbt-test-env.iam.gserviceaccount.com
          DATAPROC_REGION: us-central1
          DATAPROC_CLUSTER_NAME: dbt-test-1
          GCS_BUCKET: dbt-ci
        run: tox -- --ddtrace

      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: logs
          path: ./logs

      - name: Get current date
        if: always()
        id: date
        run: |
          echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts

      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv
          path: integration_results.csv

  # python integration tests are slow so we only run them seperately and for a single OS / python version
  test-python:
    name: "test-python"
    needs: test-metadata
    runs-on: ubuntu-latest
    if: >-
      needs.test-metadata.outputs.matrix &&
      fromJSON( needs.test-metadata.outputs.matrix ).include[0] &&
      (
        github.event_name != 'pull_request_target' ||
        github.event.pull_request.head.repo.full_name == github.repository ||
        contains(github.event.pull_request.labels.*.name, 'ok to test')
      )

    steps:
      - name: Check out the repository
        if: github.event_name != 'pull_request_target'
        uses: actions/checkout@v4
        with:
          persist-credentials: false

      # explicitly checkout the branch for the PR,
      # this is necessary for the `pull_request_target` event
      - name: Check out the repository (PR)
        if: github.event_name == 'pull_request_target'
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Set up Python 3.8
        uses: actions/setup-python@v5
        with:
          python-version: "3.8"

      - name: Install python dependencies
        run: |
          python -m pip install --user --upgrade pip
          python -m pip install tox
          python -m pip --version
          tox --version

      - name: Run tox (python models)
        env:
          BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }}
          BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }}
          BIGQUERY_TEST_NO_ACCESS_DATABASE: ${{ secrets.BIGQUERY_TEST_NO_ACCESS_DATABASE }}
          DBT_TEST_USER_1: group:buildbot@dbtlabs.com
          DBT_TEST_USER_2: group:engineering-core-team@dbtlabs.com
          DBT_TEST_USER_3: serviceAccount:dbt-integration-test-user@dbt-test-env.iam.gserviceaccount.com
          DATAPROC_REGION: us-central1
          DATAPROC_CLUSTER_NAME: dbt-test-1
          GCS_BUCKET: dbt-ci
        run: tox -e python-tests -- --ddtrace

  require-label-comment:
    runs-on: ubuntu-latest

    needs:
      - test
      - test-python

    permissions:
      pull-requests: write

    steps:
      - name: Needs permission PR comment
        if: >-
          needs.test.result == 'skipped' &&
          github.event_name == 'pull_request_target' &&
          github.event.pull_request.head.repo.full_name != github.repository
        uses: unsplash/comment-on-pr@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          msg: |
            "You do not have permissions to run integration tests, @dbt-labs/core "\
            "needs to label this PR with `ok to test` in order to run integration tests!"
          check_for_duplicate_msg: true