Metrics package (#1)

* dbt-client compatibility changes * Create helper macros for primary and secondary aggregation * average rename * Add comment discussing difference between primary and secondary calc macro dispatch strategy * Hide original file from dbt * Swap out dynamic macro dispatch for hardcoded list that actually works * Move `how` to PoP macro as that’s the only place it happens * Move debug macros, change helpers to use consistent signature * Pull get_metric out to its own macro * Swap in a proper table * Update dbt_project.yml, add slack metric def * Change metric calculation to support arbitrary calendar properties * Protect against not execute * Metrics namespacing * Access relations without using ref * Import utils * Count * if no expression provided * Remove redundant files and add to gitignore * Remove redundant aggregate references in secondary calcs * whitespace management and remove obsolete TODOs * Remove redundant metric_name arg (available on the main metric object) * Moving stuff around * Pull through date columns needed for secondary calcs * Give aliases to calculations, protect against pulling a whole table for a count(*) * Everythig uses refs now, but it is very bad * Rename metrics file * Splitting into smaller files * add todo * Forgot to bring through calculation alias * validate metrics queries make sense (legal grains, aggregates) * Moving stuff around, run legality tests * Protect against missing key * Use joiner for prettier error message * Goodbye debug file * Actually rename debug file * swap one todo for another * Swap out loop for fancy one-liner * Add builtin calendar * swap out fancy one-liner for a good-old-fashioned loop * Add integration tests project * Protect against missing meta configs * Remove duplicate average key * Add defaults to metric call * Write README * warnings about experimental behaviour * add secondary calcs shoutout * Broader utils support * Cleanup todos, protect against whitespace sql agg * readme tweaks * Update get_metric_sql.sql * Cross-db support * Add ci, circle for now * Update get_metric_sql.sql * Revert "Add ci, circle for now" This reverts commit a7e8940. * Set up end to end testing with GHA (#3) * first pass at setting up postgres integration tests * try number 2 * try number 3 * try number 4 * try number 5 * try number 6 * try number 7 * add snowflake integration tests * use snowflake target * add rest of 'em * try again please * fix target arg * fix syntax * use json for bigquery keyfile * fix env var * rename workflow * fix bigquery * more fix bigquery * more fix bigquery * re run * test * asdf * aha! * Remove star macro * Remove debug command * Properly remove star macro * Add comment expanding on problem * Add support for min aggregate (#4) * fix a bunch of badly named stuff * meta accessing isn't dependent on index anymore * Update readme to contain info on secondary calcs * tweak TOC builder file * Update create-table-of-contents.yml * Update README.md * Auto update table of contents * Update README.md * Auto update table of contents Co-authored-by: Jeremy Cohen <[email protected]> Co-authored-by: Kyle Wigley <[email protected]> Co-authored-by: joellabes <[email protected]>
dbt-labs · Feb 9, 2022 · 0b03e6b · 0b03e6b
1 parent 5b4303b
commit 0b03e6b
Show file tree

Hide file tree

Showing 38 changed files with 1,109 additions and 348 deletions.
diff --git a/.github/actions/end-to-end-test/action.yml b/.github/actions/end-to-end-test/action.yml
@@ -0,0 +1,35 @@
+name: "End to end testing"
+description: "Set up profile and run dbt with test project"
+inputs:
+  dbt-project:
+    description: "Location of test project"
+    required: false
+    default: "integration_tests"
+  dbt-target:
+    description: "Name of target to use when running dbt"
+    required: true
+  database-adapter-package:
+    description: "Name of database adapter to install"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Install python dependencies
+      shell: bash
+      run: |
+        pip install --user --upgrade pip
+        pip --version
+        pip install --pre ${{ inputs.database-adapter-package }}
+
+    - name: Setup dbt profile
+      shell: bash
+      run: |
+        mkdir -p $HOME/.dbt
+        cp ${{ github.action_path }}/sample.profiles.yml $HOME/.dbt/profiles.yml
+
+    - name: Run dbt
+      shell: bash
+      run: |
+        cd ${{ inputs.dbt-project }}
+        dbt deps --target ${{ inputs.dbt-target }}
+        dbt build --target ${{ inputs.dbt-target }} --full-refresh
diff --git a/.github/actions/end-to-end-test/sample.profiles.yml b/.github/actions/end-to-end-test/sample.profiles.yml
@@ -0,0 +1,48 @@
+# HEY! This file is used in the dbt-utils integrations tests with GHA.
+# You should __NEVER__ check credentials into version control. Thanks for reading :)
+
+config:
+  send_anonymous_usage_stats: False
+  use_colors: True
+
+dbt_metrics_integration_tests:
+  target: postgres
+  outputs:
+    postgres:
+      type: postgres
+      host: "{{ env_var('POSTGRES_TEST_HOST') }}"
+      user: "{{ env_var('POSTGRES_TEST_USER') }}"
+      pass: "{{ env_var('POSTGRES_TEST_PASSWORD') }}"
+      port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}"
+      dbname: "{{ env_var('POSTGRES_TEST_DB') }}"
+      schema: dbt_metrics_integration_tests
+      threads: 5
+
+    redshift:
+      type: redshift
+      host: "{{ env_var('REDSHIFT_TEST_HOST') }}"
+      user: "{{ env_var('REDSHIFT_TEST_USER') }}"
+      pass: "{{ env_var('REDSHIFT_TEST_PASS') }}"
+      dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}"
+      port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}"
+      schema: dbt_metrics_integration_tests
+      threads: 5
+
+    bigquery:
+      type: bigquery
+      method: service-account
+      keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}"
+      project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}"
+      schema: dbt_metrics_integration_tests
+      threads: 10
+
+    snowflake:
+      type: snowflake
+      account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}"
+      user: "{{ env_var('SNOWFLAKE_TEST_USER') }}"
+      password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}"
+      role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}"
+      database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}"
+      warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}"
+      schema: dbt_metrics_integration_tests
+      threads: 10
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,125 @@
+name: Continuous Integration
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+
+jobs:
+  postgres:
+    runs-on: ubuntu-latest
+
+    # set up env vars so that we can use them to start an instance of postgres
+    env:
+      POSTGRES_TEST_USER: postgres
+      POSTGRES_TEST_PASSWORD: postgres
+      POSTGRES_TEST_DB: gha_test
+      POSTGRES_TEST_PORT: 5432
+      POSTGRES_TEST_HOST: localhost
+
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: ${{ env.POSTGRES_TEST_USER }}
+          POSTGRES_PASSWORD: ${{ env.POSTGRES_TEST_PASSWORD }}
+          POSTGRES_DB: ${{ env.POSTGRES_TEST_DB }}
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - uses: ./.github/actions/end-to-end-test
+        with:
+          dbt-target: postgres
+          database-adapter-package: dbt-postgres
+
+  snowflake:
+    needs: postgres
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - uses: ./.github/actions/end-to-end-test
+        env:
+          SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
+          SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
+          SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
+          SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
+          SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
+          SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
+        with:
+          dbt-target: snowflake
+          database-adapter-package: dbt-snowflake
+
+  redshift:
+    needs: postgres
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - uses: ./.github/actions/end-to-end-test
+        env:
+          REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
+          REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
+          REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
+          REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
+          REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
+        with:
+          dbt-target: redshift
+          database-adapter-package: dbt-redshift
+
+  bigquery:
+    needs: postgres
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Set up service key file
+        id: keyfile
+        env:
+          BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }}
+        run: |
+          mkdir -p $HOME/.dbt
+          KEYFILE_PATH=$HOME/.dbt/bigquery-service-key.json
+          echo $BIGQUERY_TEST_SERVICE_ACCOUNT_JSON > $KEYFILE_PATH
+          echo ::set-output name=path::$KEYFILE_PATH
+
+      - uses: ./.github/actions/end-to-end-test
+        env:
+          BIGQUERY_SERVICE_KEY_PATH: ${{ steps.keyfile.outputs.path }}
+          BIGQUERY_TEST_DATABASE: ${{ secrets.BIGQUERY_TEST_DATABASE }}
+        with:
+          dbt-target: bigquery
+          database-adapter-package: dbt-bigquery
diff --git a/.github/workflows/create-table-of-contents.yml b/.github/workflows/create-table-of-contents.yml
@@ -0,0 +1,22 @@
+name: Update table of contents
+
+# Controls when the workflow will run
+on:
+  push:
+    branches: '*'
+    paths: ['README.md']
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          curl https://raw.githubusercontent.com/ekalinin/github-markdown-toc/master/gh-md-toc -o gh-md-toc
+          chmod a+x gh-md-toc
+          ./gh-md-toc --insert --no-backup README.md
+          rm ./gh-md-toc
+      - uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: Auto update table of contents
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,6 @@
 
 target/
 dbt_modules/
+dbt_packages/
 logs/
+.DS_Store