Migrate Off Circle CI / To Github Actions + dagger.io

.changes/unreleased/Under the Hood-20230929-161218.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add Github action for integration testing
+time: 2023-09-29T16:12:18.968755+02:00
+custom:
+  Author: JCZuurmond
+  Issue: "719"

.github/scripts/update_dbt_core_branch.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+set -e
+
+git_branch=$1
+target_req_file="dev-requirements.txt"
+core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g"
+postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g"
+tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g"
+if [[ "$OSTYPE" == darwin* ]]; then
+ # mac ships with a different version of sed that requires a delimiter arg
+ sed -i "" "$core_req_sed_pattern" $target_req_file
+ sed -i "" "$postgres_req_sed_pattern" $target_req_file
+ sed -i "" "$tests_req_sed_pattern" $target_req_file
+else
+ sed -i "$core_req_sed_pattern" $target_req_file
+ sed -i "$postgres_req_sed_pattern" $target_req_file
+ sed -i "$tests_req_sed_pattern" $target_req_file
+fi
+core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2)
+bumpversion --allow-dirty --new-version "$core_version" major

.github/scripts/update_release_branch.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -e
+set -e
+
+release_branch=$1
+target_req_file=".github/workflows/nightly-release.yml"
+if [[ "$OSTYPE" == darwin* ]]; then
+ # mac ships with a different version of sed that requires a delimiter arg
+ sed -i "" "s|[0-9].[0-9].latest|$release_branch|" $target_req_file
+else
+ sed -i "s|[0-9].[0-9].latest|$release_branch|" $target_req_file
+fi

.github/workflows/integration.yml

@@ -0,0 +1,254 @@
+# **what?**
+# Runs integration tests.
+
+# **why?**
+# Ensure code runs as expected.
+
+# **when?**
+# This will run for all PRs, when code is pushed to a release
+# branch, and when manually triggered.
+
+name: Integration tests
+
+on:
+  push:
+    branches:
+      - "main"
+      - "*.latest"
+      - "releases/*"
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      dbt-core-branch:
+        description: "branch of dbt-core to use in dev-requirements.txt"
+        required: false
+        type: string
+
+# explicitly turn off permissions for `GITHUB_TOKEN`
+permissions: read-all
+
+# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  # generate test metadata about what files changed and the testing matrix to use
+  test-metadata:
+    # run if not a PR from a forked repository or has a label to mark as safe to test
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository ||
+      contains(github.event.pull_request.labels.*.name, 'ok to test')
+    runs-on: ubuntu-latest
+
+    outputs:
+      matrix: ${{ steps.generate-matrix.outputs.result }}
+      run-python-tests: ${{ steps.filter.outputs.bigquery-python }}
+
+    steps:
+      - name: Check out the repository (non-PR)
+        if: github.event_name != 'pull_request'
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+
+      - name: Check out the repository (PR)
+        if: github.event_name == 'pull_request'
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Check if relevant files changed
+        if: github.event_name == 'pull_request'
+        # https://github.com/marketplace/actions/paths-changes-filter
+        # For each filter, it sets output variable named by the filter to the text:
+        #  'true' - if any of changed files matches any of filter rules
+        #  'false' - if none of changed files matches any of filter rules
+        # also, returns:
+        #  `changes` - JSON array with names of all filters matching any of the changed files
+        uses: dorny/paths-filter@v2
+        id: get-changes
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          filters: |
+            spark:
+              - 'dbt/**'
+              - 'tests/**'
+              - 'dev-requirements.txt'
+
+  local-tests:
+    name: test spark local against python ${{ matrix.python-version }}
+
+    # run if not a PR from a forked repository or has a label to mark as safe to test
+    # also checks that the matrix generated is not empty
+    if: >-
+      (
+        github.event_name != 'pull_request' ||
+        github.event.pull_request.head.repo.full_name == github.repository ||
+        contains(github.event.pull_request.labels.*.name, 'ok to test')
+      )
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - "3.8"
+        test:
+          - "spark-thrift"
+          - "spark-session"
+
+    env:
+      PYTEST_ADDOPTS: "-v --color=yes --csv test_results.csv"
+      DBT_INVOCATION_ENV: github-actions
+      DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+      DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+      DD_SITE: datadoghq.com
+      DD_ENV: ci
+      DD_SERVICE: ${{ github.event.repository.name }}
+
+    steps:
+      - name: Check out the repository
+        if: github.event_name != 'pull_request'
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+
+      # explicity checkout the branch for the PR,
+      # this is necessary for the `pull_request` event
+      - name: Check out the repository (PR)
+        if: github.event_name == 'pull_request'
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install python dependencies
+        run: |
+          python -m pip install --user --upgrade pip
+          python -m pip install tox
+          python -m pip --version
+          tox --version
+
+      - name: Update dev_requirements.txt
+        if: inputs.dbt-core-branch != ''
+        run: |
+          pip install bumpversion
+          ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}
+
+      - uses: isbang/[email protected]
+        if: ${{ matrix.test == 'spark-thrift'}}
+        with:
+          compose-file: "./docker-compose.yml"
+
+      - name: Run tox for Spark ${{ matrix.test }}
+        run: tox -e integration-${{ matrix.test }}
+
+  databricks-tests:
+    name: run ${{ matrix.test }} against python ${{ matrix.python-version }}
+    # run if not a PR from a forked repository or has a label to mark as safe to test
+    # also checks that the matrix generated is not empty
+    if: >-
+      (
+        github.event_name != 'pull_request' ||
+        github.event.pull_request.head.repo.full_name == github.repository ||
+        contains(github.event.pull_request.labels.*.name, 'ok to test')
+      )
+
+    runs-on: ubuntu-latest
+    container:
+      image: "fishtownanalytics/test-container:10"
+      options: --user root --platform linux/amd64
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - "3.8"
+        test:
+          - "spark-databricks-odbc-sql-endpoint"
+          - "spark-databricks-odbc-cluster"
+          - "spark-databricks-http"
+
+    env:
+      PYTEST_ADDOPTS: "-v --color=yes --csv test_results.csv"
+      DBT_INVOCATION_ENV: github-actions
+      DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+      DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+      DD_SITE: datadoghq.com
+      DD_ENV: ci
+      DD_SERVICE: ${{ github.event.repository.name }}
+      DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }}
+      DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }}
+      DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }}
+      DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }}
+      DBT_DATABRICKS_USER: ${{ secrets.DBT_DATABRICKS_USERNAME }}
+      DBT_TEST_USER_1: "[email protected]"
+      DBT_TEST_USER_2: "[email protected]"
+      DBT_TEST_USER_3: "[email protected]"
+      ODBC_DRIVER: "Simba"
+    steps:
+      - name: Check out the repository
+        if: github.event_name != 'pull_request'
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+
+      # explicity checkout the branch for the PR,
+      # this is necessary for the `pull_request` event
+      - name: Check out the repository (PR)
+        if: github.event_name == 'pull_request'
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Install python dependencies
+        run: |
+          python -m pip install --user --upgrade pip
+          python -m pip install tox
+          python -m pip --version
+          tox --version
+
+      - name: Update dev_requirements.txt
+        if: inputs.dbt-core-branch != ''
+        run: |
+          pip install bumpversion
+          ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}
+
+      - name: Configure ODBC
+        if: ${{ matrix.test != 'spark-databricks-http' }}
+        shell: bash
+        run: |
+          apt-get update && apt-get install -y --no-install-recommends \
+          g++ \
+          unixodbc-dev \
+          unzip
+
+          apt-get install  -y curl
+          rm -rf /tmp && mkdir /tmp
+
+          curl -OL "https://databricks.com/wp-content/uploads/drivers-2020/SimbaSparkODBC-2.6.16.1019-Debian-64bit.zip" \
+
+          unzip SimbaSparkODBC-2.6.16.1019-Debian-64bit.zip -d /tmp/ \
+          && dpkg -i /tmp/SimbaSparkODBC-*/*.deb \
+          && echo "[Simba]\nDriver = /opt/simba/spark/lib/64/libsparkodbc_sb64.so" >> /etc/odbcinst.ini \
+          && rm -rf /tmp \
+          && dpkg -l | grep Simba # confirm that the driver is installed
+
+          ldd /opt/simba/spark/lib/64/libsparkodbc_sb64.so
+
+
+      - name: Run tox for Spark ${{ matrix.test }}
+        run: tox -e integration-${{ matrix.test }}

docker/Dockerfile

@@ -2,8 +2,8 @@ ARG OPENJDK_VERSION=8
 FROM eclipse-temurin:${OPENJDK_VERSION}-jre
 
 ARG BUILD_DATE
-ARG SPARK_VERSION=3.3.2
-ARG HADOOP_VERSION=3
+ARG SPARK_VERSION=3.1.3
+ARG HADOOP_VERSION=3.2
 
 LABEL org.label-schema.name="Apache Spark ${SPARK_VERSION}" \
       org.label-schema.build-date=$BUILD_DATE \
@@ -14,7 +14,7 @@ ENV PATH="/usr/spark/bin:/usr/spark/sbin:${PATH}"
 
 RUN apt-get update && \
     apt-get install -y wget netcat procps libpostgresql-jdbc-java && \
-    wget -q "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
+    wget -q "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
     tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
     rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
     mv "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" /usr/spark && \

tox.ini

@@ -56,23 +56,22 @@ deps =
 
 
 [testenv:integration-spark-thrift]
+description = run integration tests against a Spark thrift server
 allowlist_externals =
     /bin/bash
-basepython = python3.8
-commands = /bin/bash -c '{envpython} -m pytest -v --profile apache_spark {posargs} -n4 tests/functional/adapter/*'
 passenv =
     DBT_*
     PYTEST_ADDOPTS
 deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/dev-requirements.txt
     -e.
+commands = /bin/bash -c '{envpython} -m pytest -v --profile apache_spark {posargs} -n4 tests/functional/adapter/*'
 
 [testenv:integration-spark-session]
+description = run integration tests against a Spark session
 allowlist_externals =
     /bin/bash
-basepython = python3.10
-commands = /bin/bash -c '{envpython} -m pytest -v --profile spark_session {posargs} -n4 tests/functional/adapter/*'
 passenv =
     DBT_*
     PYTEST_*
@@ -81,3 +80,4 @@ deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/dev-requirements.txt
     -e.[session]
+commands = /bin/bash -c '{envpython} -m pytest -v --profile spark_session {posargs} -n4 tests/functional/adapter/*'