Metrics package

.github/actions/end-to-end-test/action.yml

@@ -0,0 +1,35 @@
+name: "End to end testing"
+description: "Set up profile and run dbt with test project"
+inputs:
+  dbt-project:
+    description: "Location of test project"
+    required: false
+    default: "integration_tests"
+  dbt-target:
+    description: "Name of target to use when running dbt"
+    required: true
+  database-adapter-package:
+    description: "Name of database adapter to install"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Install python dependencies
+      shell: bash
+      run: |
+        pip install --user --upgrade pip
+        pip --version
+        pip install --pre ${{ inputs.database-adapter-package }}
+
+    - name: Setup dbt profile
+      shell: bash
+      run: |
+        mkdir -p $HOME/.dbt
+        cp ${{ github.action_path }}/sample.profiles.yml $HOME/.dbt/profiles.yml
+
+    - name: Run dbt
+      shell: bash
+      run: |
+        cd ${{ inputs.dbt-project }}
+        dbt deps --target ${{ inputs.dbt-target }}
+        dbt build --target ${{ inputs.dbt-target }} --full-refresh

.github/actions/end-to-end-test/sample.profiles.yml

@@ -0,0 +1,48 @@
+# HEY! This file is used in the dbt-utils integrations tests with GHA.
+# You should __NEVER__ check credentials into version control. Thanks for reading :)
+
+config:
+  send_anonymous_usage_stats: False
+  use_colors: True
+
+dbt_metrics_integration_tests:
+  target: postgres
+  outputs:
+    postgres:
+      type: postgres
+      host: "{{ env_var('POSTGRES_TEST_HOST') }}"
+      user: "{{ env_var('POSTGRES_TEST_USER') }}"
+      pass: "{{ env_var('POSTGRES_TEST_PASSWORD') }}"
+      port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}"
+      dbname: "{{ env_var('POSTGRES_TEST_DB') }}"
+      schema: dbt_metrics_integration_tests
+      threads: 5
+
+    redshift:
+      type: redshift
+      host: "{{ env_var('REDSHIFT_TEST_HOST') }}"
+      user: "{{ env_var('REDSHIFT_TEST_USER') }}"
+      pass: "{{ env_var('REDSHIFT_TEST_PASS') }}"
+      dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}"
+      port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}"
+      schema: dbt_metrics_integration_tests
+      threads: 5
+
+    bigquery:
+      type: bigquery
+      method: service-account
+      keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}"
+      project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}"
+      schema: dbt_metrics_integration_tests
+      threads: 10
+
+    snowflake:
+      type: snowflake
+      account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}"
+      user: "{{ env_var('SNOWFLAKE_TEST_USER') }}"
+      password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}"
+      role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}"
+      database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}"
+      warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}"
+      schema: dbt_metrics_integration_tests
+      threads: 10

.github/workflows/ci.yml

@@ -0,0 +1,125 @@
+name: Continuous Integration
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+
+jobs:
+  postgres:
+    runs-on: ubuntu-latest
+
+    # set up env vars so that we can use them to start an instance of postgres
+    env:
+      POSTGRES_TEST_USER: postgres
+      POSTGRES_TEST_PASSWORD: postgres
+      POSTGRES_TEST_DB: gha_test
+      POSTGRES_TEST_PORT: 5432
+      POSTGRES_TEST_HOST: localhost
+
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: ${{ env.POSTGRES_TEST_USER }}
+          POSTGRES_PASSWORD: ${{ env.POSTGRES_TEST_PASSWORD }}
+          POSTGRES_DB: ${{ env.POSTGRES_TEST_DB }}
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - uses: ./.github/actions/end-to-end-test
+        with:
+          dbt-target: postgres
+          database-adapter-package: dbt-postgres
+
+  snowflake:
+    needs: postgres
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - uses: ./.github/actions/end-to-end-test
+        env:
+          SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
+          SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
+          SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
+          SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
+          SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
+          SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
+        with:
+          dbt-target: snowflake
+          database-adapter-package: dbt-snowflake
+
+  redshift:
+    needs: postgres
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - uses: ./.github/actions/end-to-end-test
+        env:
+          REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
+          REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
+          REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
+          REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
+          REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
+        with:
+          dbt-target: redshift
+          database-adapter-package: dbt-redshift
+
+  bigquery:
+    needs: postgres
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Set up service key file
+        id: keyfile
+        env:
+          BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }}
+        run: |
+          mkdir -p $HOME/.dbt
+          KEYFILE_PATH=$HOME/.dbt/bigquery-service-key.json
+          echo $BIGQUERY_TEST_SERVICE_ACCOUNT_JSON > $KEYFILE_PATH
+          echo ::set-output name=path::$KEYFILE_PATH
+
+      - uses: ./.github/actions/end-to-end-test
+        env:
+          BIGQUERY_SERVICE_KEY_PATH: ${{ steps.keyfile.outputs.path }}
+          BIGQUERY_TEST_DATABASE: ${{ secrets.BIGQUERY_TEST_DATABASE }}
+        with:
+          dbt-target: bigquery
+          database-adapter-package: dbt-bigquery

.gitignore

@@ -1,4 +1,6 @@
 
 target/
 dbt_modules/
+dbt_packages/
 logs/
+.DS_Store

README.md

@@ -1,2 +1,103 @@
+# dbt_metrics
 
-This repo calculates metrics.
+## About
+This dbt package generates queries based on [metrics](https://docs.getdbt.com/docs/building-a-dbt-project/metrics), introduced to dbt Core in v1.0. 
+
+## :warning: A note on `ref`s
+To enable the dynamic referencing of models necessary for macro queries through the dbt Server, queries generated by this package do not participate in the DAG and `ref`'d nodes will not necessarily be built before they are accessed. Refer to the docs on [forcing dependencies](https://docs.getdbt.com/reference/dbt-jinja-functions/ref#forcing-dependencies) for more details.
+
+## Usage
+Access metrics like any other macro: 
+```
+select * 
+from {{ metrics.metric(
+    metric_name='new_customers',
+    grain='week',
+    dimensions=['plan', 'country'],
+    secondary_calcs=[
+        {
+            "type": "period_to_date", 
+            "aggregate": "sum", 
+            "period": "year", 
+            "alias": "ytd_sum"
+        },
+        {
+            "type": "period_over_period",
+            "lag": 1,
+            "how": "ratio",
+        },
+        {
+            "type": "rolling",
+            "window": 3,
+            "aggregate": "average"
+        }
+    ]
+) }}
+```
+
+## Secondary calculations
+_Documentation tk once the terminology has stabilised_
+
+## Customisation
+Most behaviour in the package can be overridden or customised.
+
+### Calendar 
+The package comes with a basic calendar table, running between 2010-01-01 and 2029-12-31 inclusive. You can replace it with any custom calendar table which meets the following requirements:
+- non-ephemeral (i.e. materialized as a table or view)
+- contains a `date_day` column. 
+- It should additionally contain the following columns: `date_week`, `date_month`, `date_quarter`, `date_year`, or equivalents. 
+- Additional date columns need to be prefixed with `date_`, e.g. `date_4_5_4_month` for a 4-5-4 retail calendar date set. Dimensions can have any name (see [dimensions on calendar tables](#dimensions-on-calendar-tables)).
+
+To do this, set the value of the `dbt_metrics_calendar_model` variable in your `dbt_project.yml` file: 
+```
+config-version: 2
+[...]
+vars:
+    dbt_metrics_calendar_model: ref('my_custom_table')
+```
+
+### Time Grains 
+The package protects against nonsensical secondary calculations, such as a month-to-date aggregate of data which has been rolled up to the quarter. If you customise your calendar (for example by adding a [4-5-4 retail calendar](https://nrf.com/resources/4-5-4-calendar) month), you will need to override the `get_grain_order()` macro. In that case, you might remove `month` and replace it with `month_4_5_4`. All date columns must be prefixed with `date_` in the table, but this is not necessary in the model config.
+
+### Custom aggregations 
+To create a custom primary aggregation (as exposed through the `type` config of a metric), create a macro of the form `metric_my_aggregate(expression)`, then override the `aggregate_primary_metric(aggregate, expression)` macro to add it to the dispatch list. The package also protects against nonsensical secondary calculations such as an average of an average; you will need to override the `get_metric_allowlist()` macro to both add your new aggregate to to the existing aggregations' allowlists, and to make an allowlist for your new aggregation:
+```
+    {% do return ({
+        "average": ['max', 'min'],
+        "count": ['max', 'min', 'average', 'my_new_aggregate'],
+        [...]
+        "my_new_aggregate": ['max', 'min', 'sum', 'average', 'my_new_aggregate']
+    }) %}
+```
+
+To create a custom secondary aggregation (as exposed through the `secondary_calcs` parameter in the `metric` macro), create a macro of the form `metric_secondary_calculations_my_calculation(metric_name, dims, config)`, then override the `metric_secondary_calculations(metric_name, dims, config)` macro to add it to the dispatch list. 
+
+### Secondary calculation column aliases
+Aliases can be passed into the `calcs` object. If no alias is provided, one will be automatically generated. To modify the existing alias logic, or add support for a custom secondary calculation, override `secondary_calculation_alias(calc_config, grain)`.
+
+## 🧪 Experimental behaviour 
+:warning: This behaviour is subject to change in future versions of dbt Core and this package.
+
+### Dimensions on calendar tables
+You may want to aggregate metrics by a dimension in your custom calendar table, for example `is_weekend`. _In addition to_ the primary `dimensions` list, add the following `meta` properties to your metric, with the model's dimensions first and the calendar's dimensions second:
+```
+      meta: 
+        dimensions: 
+          - type: model
+            columns:
+              - plan
+              - country
+          - type: calendar
+            columns: 
+              - is_weekend
+```
+You can then access the additional dimensions as normal:
+```
+select * 
+from {{ metrics.metric(
+    metric_name='new_customers',
+    grain='week',
+    dimensions=['plan', 'country', 'is_weekend'],
+    secondary_calcs=[]
+) }}
+```

dbt_project.yml

@@ -7,19 +7,19 @@ version: '1.0.0'
 config-version: 2
 
 # This setting configures which "profile" dbt uses for this project.
-profile: 'default'
+profile: 'user'
 
 # These configurations specify where dbt should look for different types of files.
 # The `source-paths` config, for example, states that models in this project can be
 # found in the "models/" directory. You probably won't need to change these!
 model-paths: ["models"]
-analysis-paths: ["analysis"]
+analysis-paths: ["analyses"]
 test-paths: ["tests"]
-seed-paths: ["data"]
+seed-paths: ["seeds"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
 
 target-path: "target"  # directory which will store compiled SQL files
 clean-targets:         # directories to be removed by `dbt clean`
   - "target"
-  - "dbt_modules"
+  - "dbt_packages"

integration_tests/.gitignore

@@ -0,0 +1,4 @@
+
+target/
+dbt_packages/
+logs/

integration_tests/README.md

@@ -0,0 +1,16 @@
+Welcome to your new dbt project!
+
+### Using the starter project
+
+Try running the following commands:
+
+- dbt run
+- dbt test
+
+### Resources:
+
+- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
+- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
+- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
+- Find [dbt events](https://events.getdbt.com) near you
+- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

integration_tests/dbt_project.yml

@@ -0,0 +1,22 @@
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: "dbt_metrics_integration_tests"
+version: "1.0.0"
+config-version: 2
+
+# This setting configures which "profile" dbt uses for this project.
+profile: "dbt_metrics_integration_tests"
+
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+target-path: "target"
+clean-targets:
+  - "target"
+  - "dbt_packages"
+  - "logs"