From b272d1efaa607555b254a5ff1e3a151a2216ea3c Mon Sep 17 00:00:00 2001 From: Taylor Dunlap Date: Thu, 27 Jun 2024 17:09:26 +0000 Subject: [PATCH 1/5] Adding lint models --- .github/workflows/lint_models.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/lint_models.yml diff --git a/.github/workflows/lint_models.yml b/.github/workflows/lint_models.yml new file mode 100644 index 0000000..a875a88 --- /dev/null +++ b/.github/workflows/lint_models.yml @@ -0,0 +1,16 @@ +name: Lint Models + +on: [pull_request] + +jobs: + lint-models: + runs-on: ubuntu-latest + steps: + - uses: "actions/checkout@v2" + - uses: "actions/setup-python@v2" + with: + python-version: "3.8" + - name: Install SQLFluff + run: "pip install sqlfluff==0.3.6" + - name: Lint models + run: "sqlfluff lint models" \ No newline at end of file From 09dc61a238b43ad73a564df0d65c392ed9d53f79 Mon Sep 17 00:00:00 2001 From: Taylor Dunlap Date: Thu, 27 Jun 2024 17:09:40 +0000 Subject: [PATCH 2/5] making change --- models/marts/core/dim_customers.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/dim_customers.sql b/models/marts/core/dim_customers.sql index e86796a..0fe2169 100644 --- a/models/marts/core/dim_customers.sql +++ b/models/marts/core/dim_customers.sql @@ -13,7 +13,7 @@ with customer as ( nation as ( select * from {{ ref('stg_tpch_nations') }} -), +), region as ( select * from {{ ref('stg_tpch_regions') }} From 231db2f87c9155a98d9606e6cbd08c11562f0005 Mon Sep 17 00:00:00 2001 From: Taylor Dunlap Date: Thu, 27 Jun 2024 17:13:16 +0000 Subject: [PATCH 3/5] changing file --- .github/workflows/lint_models.yml | 55 ++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/.github/workflows/lint_models.yml b/.github/workflows/lint_models.yml index a875a88..f98151c 100644 --- a/.github/workflows/lint_models.yml +++ b/.github/workflows/lint_models.yml @@ -1,16 +1,53 @@ -name: Lint Models +name: SQLFluff -on: [pull_request] +on: + - pull_request jobs: lint-models: runs-on: ubuntu-latest steps: - - uses: "actions/checkout@v2" - - uses: "actions/setup-python@v2" + - name: checkout + uses: actions/checkout@v2 + - name: Install Python + uses: "actions/setup-python@v2" with: - python-version: "3.8" - - name: Install SQLFluff - run: "pip install sqlfluff==0.3.6" - - name: Lint models - run: "sqlfluff lint models" \ No newline at end of file + python-version: "3.7" + - name: install sqlfluff + run: "pip install sqlfluff" + - name: Get changed files + id: get_file_changes + uses: trilom/file-changes-action@v1.2.4 + with: + output: ' ' + + - name: Get changed .sql files in /models to lint + id: get_files_to_lint + shell: bash -l {0} + run: | + # Set the command in the $() brackets as an output to use in later steps + echo "::set-output name=lintees::$( + # Issue where grep regular expressions don't work as expected on the + # Github Actions shell, check dbt/models/ folder + echo \ + $(echo ${{ steps.get_file_changes.outputs.files_modified }} | + tr -s ' ' '\n' | + grep -E '^dbt/models.*[.]sql$' | + tr -s '\n' ' ') \ + $(echo ${{ steps.get_file_changes.outputs.files_added }} | + tr -s ' ' '\n' | + grep -E '^dbt/models.*[.]sql$' | + tr -s '\n' ' ') + )" + + - name: Lint dbt models + id: sqlfluff_json + if: steps.get_files_to_lint.outputs.lintees != '' + shell: bash -l {0} + run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json + - name: Annotate + uses: yuzutech/annotations-action@v0.3.0 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" + title: "SQLFluff Lint" + input: "./annotations.json" \ No newline at end of file From 89bd7306e89641fce0618ff57eb6aa08f9584f0f Mon Sep 17 00:00:00 2001 From: Taylor Dunlap Date: Thu, 27 Jun 2024 17:14:58 +0000 Subject: [PATCH 4/5] making change --- models/marts/core/dim_parts.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/core/dim_parts.sql b/models/marts/core/dim_parts.sql index 14b9a95..163ad0c 100644 --- a/models/marts/core/dim_parts.sql +++ b/models/marts/core/dim_parts.sql @@ -12,7 +12,7 @@ with part as ( final as ( select part_key, - manufacturer, + manufacturer , name, brand as brand, type, From fff40bc2e17381b224b1d6d5db72380c78222190 Mon Sep 17 00:00:00 2001 From: Taylor Dunlap Date: Tue, 13 Aug 2024 23:08:14 +0000 Subject: [PATCH 5/5] changing files --- macros/create_schema.sql | 5 +++++ models/staging/tpch/stg_tpch_nations.sql | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 macros/create_schema.sql diff --git a/macros/create_schema.sql b/macros/create_schema.sql new file mode 100644 index 0000000..c9be6d1 --- /dev/null +++ b/macros/create_schema.sql @@ -0,0 +1,5 @@ +{% macro create_schema(relation) %} +{%- call statement('create_schema') -%} + create schema if not exists {{ relation.without_identifier() }} with managed access + {% endcall %} +{% endmacro %} \ No newline at end of file diff --git a/models/staging/tpch/stg_tpch_nations.sql b/models/staging/tpch/stg_tpch_nations.sql index 58ae737..f852f3b 100644 --- a/models/staging/tpch/stg_tpch_nations.sql +++ b/models/staging/tpch/stg_tpch_nations.sql @@ -1,3 +1,8 @@ +{{ + config( + schema='managed_access_test' + ) +}} with source as ( select * from {{ source('tpch', 'nation') }} @@ -11,7 +16,8 @@ renamed as ( n_nationkey as nation_key, n_name as name, n_regionkey as region_key, - n_comment as comment + n_comment as comment, + '{{env_var('DBT_ENV_TYPE')}}' as env_type from source