data transformation and modelling

dbt/.gitignore

@@ -1,6 +1,6 @@
 # you shouldn't commit these into source control
 # these are the default directory names, adjust/add to fit your needs
 target/
-dbt_packages/
 logs/
-analyses/
+analyses/
+.user.yml

dbt/dbt_packages/dbt_utils/.circleci/config.yml

@@ -0,0 +1,102 @@
+
+version: 2.1
+
+jobs:
+
+  integration-postgres:
+    docker:
+      - image: cimg/python:3.9
+      - image: cimg/postgres:9.6
+        environment:
+          POSTGRES_USER: root
+    environment:
+      POSTGRES_TEST_HOST: localhost
+      POSTGRES_TEST_USER: root
+      POSTGRES_TEST_PASS: ''
+      POSTGRES_TEST_PORT: 5432
+      POSTGRES_TEST_DBNAME: circle_test
+
+    steps:
+      - checkout
+      - run: pip install --pre dbt-postgres -r dev-requirements.txt
+      - run:
+          name: "Run OG Tests - Postgres"
+          command: ./run_test.sh postgres
+      - store_artifacts:
+          path: integration_tests/logs
+      - store_artifacts:
+          path: integration_tests/target
+
+    # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass    
+    resource_class: large
+
+  integration-redshift:
+    docker:
+      - image: cimg/python:3.9
+    steps:
+      - checkout
+      - run: pip install --pre dbt-redshift -r dev-requirements.txt
+      - run:
+          name: "Run OG Tests - Redshift"
+          command: ./run_test.sh redshift
+      - store_artifacts:
+          path: integration_tests/logs
+      - store_artifacts:
+          path: integration_tests/target
+    # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass    
+    resource_class: large
+
+  integration-snowflake:
+    docker:
+      - image: cimg/python:3.9
+    steps:
+      - checkout
+      - run: pip install --pre dbt-snowflake -r dev-requirements.txt
+      - run:
+          name: "Run OG Tests - Snowflake"
+          command: ./run_test.sh snowflake
+      - store_artifacts:
+          path: integration_tests/logs          
+      - store_artifacts:
+          path: integration_tests/target
+    # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass    
+    resource_class: large
+
+  integration-bigquery:
+    environment:
+      BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
+    docker:
+      - image: cimg/python:3.9
+    steps:
+      - checkout
+      - run: pip install --pre dbt-bigquery -r dev-requirements.txt
+      - run:
+          name: "Set up credentials"
+          command: echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json
+      - run:
+          name: "Run OG Tests - BigQuery"
+          command: ./run_test.sh bigquery
+      - store_artifacts:
+          path: integration_tests/logs
+      - store_artifacts:
+          path: integration_tests/target
+    # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass    
+    resource_class: large
+
+workflows:
+  version: 2
+  test-all:
+    jobs:
+      - integration-postgres
+      - integration-redshift:
+          context: profile-redshift
+          requires:
+            - integration-postgres
+      - integration-snowflake:
+          context: profile-snowflake
+          requires:
+            - integration-postgres
+      - integration-bigquery:
+          context: profile-bigquery
+          requires:
+            - integration-postgres

dbt/dbt_packages/dbt_utils/.github/CODEOWNERS

@@ -0,0 +1 @@
+*       @clrcrl

dbt/dbt_packages/dbt_utils/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,60 @@
+---
+name: Bug report
+about: Report a bug or an issue you've found with this package
+title: ''
+labels: bug, triage
+assignees: ''
+
+---
+
+### Describe the bug
+<!---
+A clear and concise description of what the bug is. You can also use the issue title to do this
+--->
+
+### Steps to reproduce
+<!---
+In as much detail as possible, please provide steps to reproduce the issue. Sample data that triggers the issue, example model code, etc is all very helpful here.
+--->
+
+### Expected results
+<!---
+A clear and concise description of what you expected to happen.
+--->
+
+### Actual results
+<!---
+A clear and concise description of what you expected to happen.
+--->
+
+### Screenshots and log output
+<!---
+If applicable, add screenshots or log output to help explain your problem.
+--->
+
+### System information
+**The contents of your `packages.yml` file:**
+
+**Which database are you using dbt with?**
+- [ ] postgres
+- [ ] redshift
+- [ ] bigquery
+- [ ] snowflake
+- [ ] other (specify: ____________)
+
+
+**The output of `dbt --version`:**
+```
+<output goes here>
+```
+
+
+### Additional context
+<!---
+Add any other context about the problem here. For example, if you think you know which line of code is causing the issue.
+--->
+
+### Are you interested in contributing the fix?
+<!---
+Let us know if you want to contribute the fix, and whether would need a hand getting started
+--->

dbt/dbt_packages/dbt_utils/.github/ISSUE_TEMPLATE/dbt_minor_release.md

@@ -0,0 +1,29 @@
+---
+name: dbt Minor Release Follow-Up
+about: A checklist of tasks to complete after a minor release is made to dbt
+title: 'dbt Minor Release Follow up for dbt v0.x.0'
+labels:
+assignees: ''
+---
+
+<!---
+This template is to be used once a new dbt minor release is available on pypi.
+In the future, we will consider doing pre-releases.
+-->
+
+First, check if this is a breaking change
+- [ ] Increase the upper bound of the `require-dbt-version` config in the `dbt_project.yml`
+- [ ] Increase the upper bound of the dbt version in `run_test.sh`
+- [ ] Create a PR against the `main` branch to see if tests pass
+
+If test pass, this is _not_ a breaking change. You should:
+- [ ] Merge into `main`
+- [ ] Create a patch release
+
+If tests fail, this _is_ a breaking change. You'll need to create a minor release:
+- [ ] Change the PR base to be against the next `dev` branch.
+- [ ] Increase the lower bound to the current dbt minor version in both the `dbt_project.yml` and `run_test.sh` files
+- [ ] Fix any errors
+- [ ] Merge `dev` into `main`
+- [ ] Create a minor release
+- [ ] Once the release is available on hub, [create a new issue](https://github.com/dbt-labs/dbt-utils/issues/new/choose) using the "dbt-utils Minor Release Checklist" template

dbt/dbt_packages/dbt_utils/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,25 @@
+---
+name: Feature request
+about: Suggest an idea for this package
+title: ''
+labels: enhancement, triage
+assignees: ''
+
+---
+
+### Describe the feature
+A clear and concise description of what you want to happen.
+
+### Describe alternatives you've considered
+A clear and concise description of any alternative solutions or features you've considered.
+
+### Additional context
+Is this feature database-specific? Which database(s) is/are relevant? Please include any other relevant context here.
+
+### Who will this benefit?
+What kind of use case will this feature be useful for? Please be specific and provide examples, this will help us prioritize properly.
+
+### Are you interested in contributing this feature?
+<!---
+Let us know if you want to contribute the feature, and whether would need a hand getting started
+--->

dbt/dbt_packages/dbt_utils/.github/ISSUE_TEMPLATE/utils_minor_release.md

@@ -0,0 +1,38 @@
+---
+name: dbt-utils Minor Release Follow-up
+about: A checklist of tasks to complete after making a dbt-utils minor release
+title: 'dbt Minor Release Follow up for dbt-utils v0.x.0'
+labels:
+assignees: ''
+---
+
+<!---
+This template is to be used once a new dbt-utils release is available on hub.
+In the future, we will consider automating this.
+-->
+
+## Process for each dependent package
+First, check if this is a breaking change
+- [ ] Increase the upper bound of the `dbt-utils` `version:` config in the `packages.yml` of the dependent package.
+- [ ] Push to a new branch to see if tests pass, or test locally.
+
+If this is _not_ a breaking change:
+- [ ] Create a patch release
+
+If this _is_ a breaking change:
+- [ ] Fix any breaking changes
+- [ ] Increase the lower bound to the current dbt-utils minor version
+- [ ] Create a minor release for the package
+
+## Checklist of dependent packages
+| Package                                                                      | PR     | Release     |
+|------------------------------------------------------------------------------|--------|-------------|
+| [audit-helper](https://github.com/dbt-labs/dbt-audit-helper)                 | [PR]() | [Release]() |
+| [codegen](https://github.com/dbt-labs/dbt-codegen)                           | [PR]() | [Release]() |
+| [redshift](https://github.com/dbt-labs/redshift)                             | [PR]() | [Release]() |
+| [event-logging](https://github.com/dbt-labs/dbt-event-logging)               | [PR]() | [Release]() |
+| [snowplow](https://github.com/dbt-labs/snowplow)                             | [PR]() | [Release]() |
+| [external-tables](https://github.com/dbt-labs/dbt-external-tables)           | [PR]() | [Release]() |
+| [segment](https://github.com/dbt-labs/segment)                               | [PR]() | [Release]() |
+| [facebook-ads](https://github.com/dbt-labs/facebook-ads)                     | [PR]() | [Release]() |
+| [stitch-utils](https://github.com/dbt-labs/stitch-utils)                     | [PR]() | [Release]() |

dbt/dbt_packages/dbt_utils/.github/pull_request_template.md

@@ -0,0 +1,29 @@
+resolves #
+
+This is a:
+- [ ] documentation update
+- [ ] bug fix with no breaking changes
+- [ ] new functionality
+- [ ] a breaking change
+
+All pull requests from community contributors should target the `main` branch (default).
+
+## Description & motivation
+<!---
+Describe your changes, and why you're making them.
+-->
+
+## Checklist
+- [ ] This code is associated with an Issue which has been triaged and [accepted for development](https://docs.getdbt.com/docs/contributing/oss-expectations#pull-requests). 
+- [ ] I have verified that these changes work locally on the following warehouses (Note: it's okay if you do not have access to all warehouses, this helps us understand what has been covered)
+    - [ ] BigQuery
+    - [ ] Postgres
+    - [ ] Redshift
+    - [ ] Snowflake
+- [ ] I followed guidelines to ensure that my changes will work on "non-core" adapters by:
+    - [ ] dispatching any new macro(s) so non-core adapters can also use them (e.g. [the `star()` source](https://github.com/dbt-labs/dbt-utils/blob/main/macros/sql/star.sql))
+    - [ ] using the `limit_zero()` macro in place of the literal string: `limit 0`
+    - [ ] using `dbt.type_*` macros instead of explicit datatypes (e.g. `dbt.type_timestamp()` instead of `TIMESTAMP`
+- [ ] I have updated the README.md (if applicable)
+- [ ] I have added tests & descriptions to my models (and macros if applicable)
+- [ ] I have added an entry to CHANGELOG.md

dbt/dbt_packages/dbt_utils/.github/workflows/create-table-of-contents.yml

@@ -0,0 +1,24 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Update table of contents
+
+# Controls when the workflow will run
+on:
+  push:
+    branches: [main]
+    paths: ['README.md']
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v3
+      - run: |
+          curl https://raw.githubusercontent.com/ekalinin/github-markdown-toc/master/gh-md-toc -o gh-md-toc
+          chmod a+x gh-md-toc
+          ./gh-md-toc --insert --no-backup README.md
+          rm ./gh-md-toc
+      - uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: Auto update table of contents

dbt/dbt_packages/dbt_utils/.gitignore

@@ -0,0 +1,9 @@
+
+target/
+dbt_modules/
+dbt_packages/
+logs/
+venv/
+env/
+test.env
+__pycache__