dbt_slim_ci.yml

# .github/workflows/app.yaml
name: dbt slim ci (in beta)

on:
  workflow_dispatch:
  pull_request:
    paths-ignore:
      - 'scripts/**'
      - 'Pipfile'
      - '.gitignore'
      - '.github/**'

concurrency:
  group:  ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  dbt-test:
    runs-on: [ self-hosted, linux, spellbook ]
    timeout-minutes: 90

    steps:
      - name: Check out repository code
        uses: actions/checkout@v2

      - name: Set environment variables
        run: |
          echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-8)" >> $GITHUB_ENV

      - name: Add git_sha to schema
        run: "/runner/change_schema.sh git_$GIT_SHA"

      - name: Get latest manifest
        run: "aws s3 cp s3://manifest-spellbook/manifest.json manifest.json"

      - name: dbt dependencies
        run: "dbt deps"

      - name: dbt compile to create manifest to compare to
        run: "dbt compile"

      - name: dbt seed
        run: "dbt seed --select state:modified --state ."

      - name: dbt run initial model(s)
        run: "dbt -x run --select state:modified --exclude tag:prod_exclude --defer --state ."

      - name: dbt test initial model(s)
        run: "dbt test --select state:new state:modified --exclude tag:prod_exclude --defer --state ."

      - name: Set environment variable for incremental model count
        run: |
          echo "INC_MODEL_COUNT=$(echo dbt ls --select state:modified,config.materialized:incremental --state . --resource-type model  | wc -l)" >> $GITHUB_ENV

      - name: dbt run incremental model(s) if applicable
        if: env.INC_MODEL_COUNT > 0
        run: "dbt run --select state:modified,config.materialized:incremental --exclude tag:prod_exclude --defer --state ."

      - name: dbt test incremental model(s) if applicable
        if: env.INC_MODEL_COUNT > 0
        run: "dbt test --select state:modified,config.materialized:incremental --exclude tag:prod_exclude --defer --state ."

      - name: Run DuneSQL Check
        run: "/runner/dunesql_check.py --schema test_schema --pr_schema git_$GIT_SHA"