From 546f132aff48afcf466191510240bd7e053bf31c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 29 Aug 2023 16:42:34 -0400 Subject: [PATCH 01/15] parameterize where clause, add option to supply list of relations --- dbt/include/snowflake/macros/catalog.sql | 149 +++++++++++++---------- 1 file changed, 87 insertions(+), 62 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index cf96df25d..b75a4bc2d 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -1,72 +1,97 @@ -{% macro snowflake__get_catalog(information_schema, schemas) -%} - {% set query %} - with tables as ( - - select - table_catalog as "table_database", - table_schema as "table_schema", - table_name as "table_name", - table_type as "table_type", - comment as "table_comment", - - -- note: this is the _role_ that owns the table - table_owner as "table_owner", - - 'Clustering Key' as "stats:clustering_key:label", - clustering_key as "stats:clustering_key:value", - 'The key used to cluster this table' as "stats:clustering_key:description", - (clustering_key is not null) as "stats:clustering_key:include", - - 'Row Count' as "stats:row_count:label", - row_count as "stats:row_count:value", - 'An approximate count of rows in this table' as "stats:row_count:description", - (row_count is not null) as "stats:row_count:include", - - 'Approximate Size' as "stats:bytes:label", - bytes as "stats:bytes:value", - 'Approximate size of the table as reported by Snowflake' as "stats:bytes:description", - (bytes is not null) as "stats:bytes:include", - - 'Last Modified' as "stats:last_modified:label", - to_varchar(convert_timezone('UTC', last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value", - 'The timestamp for last update/change' as "stats:last_modified:description", - (last_altered is not null and table_type='BASE TABLE') as "stats:last_modified:include" - - from {{ information_schema }}.tables - where ( - {%- for schema in schemas -%} - upper("table_schema") = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) +{% macro snowflake__get_catalog(information_schema, schemas, relations) -%} + {% set query %} + with tables as ( - ), + select + table_catalog as "table_database", + table_schema as "table_schema", + table_name as "table_name", + table_type as "table_type", + comment as "table_comment", - columns as ( + -- note: this is the _role_ that owns the table + table_owner as "table_owner", - select - table_catalog as "table_database", - table_schema as "table_schema", - table_name as "table_name", + 'Clustering Key' as "stats:clustering_key:label", + clustering_key as "stats:clustering_key:value", + 'The key used to cluster this table' as "stats:clustering_key:description", + (clustering_key is not null) as "stats:clustering_key:include", - column_name as "column_name", - ordinal_position as "column_index", - data_type as "column_type", - comment as "column_comment" + 'Row Count' as "stats:row_count:label", + row_count as "stats:row_count:value", + 'An approximate count of rows in this table' as "stats:row_count:description", + (row_count is not null) as "stats:row_count:include", - from {{ information_schema }}.columns - where ( - {%- for schema in schemas -%} - upper("table_schema") = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - ) + 'Approximate Size' as "stats:bytes:label", + bytes as "stats:bytes:value", + 'Approximate size of the table as reported by Snowflake' as "stats:bytes:description", + (bytes is not null) as "stats:bytes:include", + + 'Last Modified' as "stats:last_modified:label", + to_varchar(convert_timezone('UTC', last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value", + 'The timestamp for last update/change' as "stats:last_modified:description", + (last_altered is not null and table_type='BASE TABLE') as "stats:last_modified:include" + + from {{ information_schema }}.tables + {{ snowflake__get_catalog_where_clause(schemas, relations) }} + + ), + + columns as ( + + select + table_catalog as "table_database", + table_schema as "table_schema", + table_name as "table_name", + + column_name as "column_name", + ordinal_position as "column_index", + data_type as "column_type", + comment as "column_comment" + + from {{ information_schema }}.columns + {{ snowflake__get_catalog_where_clause(schemas, relations) }} + ) - select * - from tables - join columns using ("table_database", "table_schema", "table_name") - order by "column_index" + select * + from tables + join columns using ("table_database", "table_schema", "table_name") + order by "column_index" {%- endset -%} - {{ return(run_query(query)) }} + {{ return(run_query(query)) }} {%- endmacro %} + + +{% snowflake__get_catalog_where_clause(schemas, relations) %} + + {% if schemas is not none %} + where ( + {%- for schema in schemas -%} + upper("table_schema") = upper('{{ schema }}') + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) + + {% elif relations is not none %} + where ( + {%- for relation in relations -%} + ( + upper("table_schema") = upper('{{ relation.schema }}') + {% if relation.identifier is not none %} + and upper("table_name") = upper('{{ relation.identifier }}') + {% endif %} + ) + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) + + {% else %} + {% do exceptions.raise_compiler_error( + '`get_catalog` requires a list of schemas or a list of relations.' + ) %} + + {% endif %} + +{% endmacro %} From ac21429e627a0868d4fea22a99a9c736a13235c3 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 29 Aug 2023 16:48:46 -0400 Subject: [PATCH 02/15] parameterize where clause, add option to supply list of relations --- .changes/unreleased/Features-20230829-152412.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20230829-152412.yaml diff --git a/.changes/unreleased/Features-20230829-152412.yaml b/.changes/unreleased/Features-20230829-152412.yaml new file mode 100644 index 000000000..7ff5a15fa --- /dev/null +++ b/.changes/unreleased/Features-20230829-152412.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support limiting get_catalog by object name +time: 2023-08-29T15:24:12.649104-04:00 +custom: + Author: mikealfare + Issue: "4997" From b1a2f7d016bf8f25e84bcb65ad8beeb7f7e975d9 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 29 Aug 2023 16:53:26 -0400 Subject: [PATCH 03/15] revert whitespace fix --- dbt/include/snowflake/macros/catalog.sql | 82 ++++++++++++------------ 1 file changed, 42 insertions(+), 40 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index b75a4bc2d..b977bc167 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -1,57 +1,59 @@ {% macro snowflake__get_catalog(information_schema, schemas, relations) -%} + {% set query %} {% set query %} + with tables as ( with tables as ( - select - table_catalog as "table_database", - table_schema as "table_schema", - table_name as "table_name", - table_type as "table_type", - comment as "table_comment", + select + table_catalog as "table_database", + table_schema as "table_schema", + table_name as "table_name", + table_type as "table_type", + comment as "table_comment", - -- note: this is the _role_ that owns the table - table_owner as "table_owner", + -- note: this is the _role_ that owns the table + table_owner as "table_owner", - 'Clustering Key' as "stats:clustering_key:label", - clustering_key as "stats:clustering_key:value", - 'The key used to cluster this table' as "stats:clustering_key:description", - (clustering_key is not null) as "stats:clustering_key:include", + 'Clustering Key' as "stats:clustering_key:label", + clustering_key as "stats:clustering_key:value", + 'The key used to cluster this table' as "stats:clustering_key:description", + (clustering_key is not null) as "stats:clustering_key:include", - 'Row Count' as "stats:row_count:label", - row_count as "stats:row_count:value", - 'An approximate count of rows in this table' as "stats:row_count:description", - (row_count is not null) as "stats:row_count:include", + 'Row Count' as "stats:row_count:label", + row_count as "stats:row_count:value", + 'An approximate count of rows in this table' as "stats:row_count:description", + (row_count is not null) as "stats:row_count:include", - 'Approximate Size' as "stats:bytes:label", - bytes as "stats:bytes:value", - 'Approximate size of the table as reported by Snowflake' as "stats:bytes:description", - (bytes is not null) as "stats:bytes:include", + 'Approximate Size' as "stats:bytes:label", + bytes as "stats:bytes:value", + 'Approximate size of the table as reported by Snowflake' as "stats:bytes:description", + (bytes is not null) as "stats:bytes:include", - 'Last Modified' as "stats:last_modified:label", - to_varchar(convert_timezone('UTC', last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value", - 'The timestamp for last update/change' as "stats:last_modified:description", - (last_altered is not null and table_type='BASE TABLE') as "stats:last_modified:include" + 'Last Modified' as "stats:last_modified:label", + to_varchar(convert_timezone('UTC', last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value", + 'The timestamp for last update/change' as "stats:last_modified:description", + (last_altered is not null and table_type='BASE TABLE') as "stats:last_modified:include" - from {{ information_schema }}.tables - {{ snowflake__get_catalog_where_clause(schemas, relations) }} + from {{ information_schema }}.tables + {{ snowflake__get_catalog_where_clause(schemas, relations) }} - ), + ), - columns as ( + columns as ( - select - table_catalog as "table_database", - table_schema as "table_schema", - table_name as "table_name", + select + table_catalog as "table_database", + table_schema as "table_schema", + table_name as "table_name", - column_name as "column_name", - ordinal_position as "column_index", - data_type as "column_type", - comment as "column_comment" + column_name as "column_name", + ordinal_position as "column_index", + data_type as "column_type", + comment as "column_comment" - from {{ information_schema }}.columns - {{ snowflake__get_catalog_where_clause(schemas, relations) }} - ) + from {{ information_schema }}.columns + {{ snowflake__get_catalog_where_clause(schemas, relations) }} + ) select * from tables @@ -59,7 +61,7 @@ order by "column_index" {%- endset -%} - {{ return(run_query(query)) }} + {{ return(run_query(query)) }} {%- endmacro %} From cad05559dc60ea4b859c5b608c6af4d6a7fd956b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 29 Aug 2023 16:54:23 -0400 Subject: [PATCH 04/15] revert whitespace fix --- dbt/include/snowflake/macros/catalog.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index b977bc167..d426c115a 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -1,8 +1,6 @@ {% macro snowflake__get_catalog(information_schema, schemas, relations) -%} {% set query %} - {% set query %} with tables as ( - with tables as ( select table_catalog as "table_database", From 9fb0839505364297d3871df383459571930ea836 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 29 Aug 2023 16:54:51 -0400 Subject: [PATCH 05/15] revert whitespace fix --- dbt/include/snowflake/macros/catalog.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index d426c115a..61077fd46 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -53,10 +53,10 @@ {{ snowflake__get_catalog_where_clause(schemas, relations) }} ) - select * - from tables - join columns using ("table_database", "table_schema", "table_name") - order by "column_index" + select * + from tables + join columns using ("table_database", "table_schema", "table_name") + order by "column_index" {%- endset -%} {{ return(run_query(query)) }} From 6e4e0b2170da287a086219f0af5b248a948251a4 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 29 Aug 2023 17:21:18 -0400 Subject: [PATCH 06/15] fix missing macro keyword --- dbt/include/snowflake/macros/catalog.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index 61077fd46..7170c9c4b 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -64,7 +64,7 @@ {%- endmacro %} -{% snowflake__get_catalog_where_clause(schemas, relations) %} +{% macro snowflake__get_catalog_where_clause(schemas, relations) %} {% if schemas is not none %} where ( From 673e1c52482add1db47b02a83bd7e84dc1c8229f Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 13 Sep 2023 19:45:19 -0400 Subject: [PATCH 07/15] point to the dev branch on core, revert before pushing to main --- dbt/include/snowflake/macros/catalog.sql | 20 ++++++++++++++------ dev-requirements.txt | 4 ++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index 7170c9c4b..de19e6a69 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -1,4 +1,7 @@ {% macro snowflake__get_catalog(information_schema, schemas, relations) -%} +{#- +-- `relations` is a dictionary with schema names as keys and an iterable (likely list) of relation identifiers as values +-#} {% set query %} with tables as ( @@ -65,6 +68,9 @@ {% macro snowflake__get_catalog_where_clause(schemas, relations) %} +{#- +-- `relations` is a dictionary with schema names as keys and an iterable (likely list) of relation identifiers as values +-#} {% if schemas is not none %} where ( @@ -76,12 +82,14 @@ {% elif relations is not none %} where ( - {%- for relation in relations -%} + {%- for schema_name, relations in relations.items() -%} ( - upper("table_schema") = upper('{{ relation.schema }}') - {% if relation.identifier is not none %} - and upper("table_name") = upper('{{ relation.identifier }}') - {% endif %} + upper("table_schema") = upper('{{ schema_name }}') + and upper("table_name") in ( + {%- for relation_name in relations -%} + upper('{{ relation_name }}') + {%- if not loop.last -%}, {% endif -%} + {%- endfor -%} ) {%- if not loop.last %} or {% endif -%} {%- endfor -%} @@ -89,7 +97,7 @@ {% else %} {% do exceptions.raise_compiler_error( - '`get_catalog` requires a list of schemas or a list of relations.' + '`get_catalog` requires a list of schema names or a dict of relation names by schema name.' ) %} {% endif %} diff --git a/dev-requirements.txt b/dev-requirements.txt index 2ccb4e980..5fe21c3ce 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@paw/get-catalog-relations#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@paw/get-catalog-relations#egg=dbt-tests-adapter&subdirectory=tests/adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor From 9bdacf908a1ba514dbc9cd22fe02af6980f84977 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 15 Sep 2023 15:03:49 -0400 Subject: [PATCH 08/15] add new macro get_catalog_relations, update get_catalog to share common logic with get_catalog_relations --- dbt/include/snowflake/macros/catalog.sql | 204 ++++++++++++----------- 1 file changed, 111 insertions(+), 93 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index de19e6a69..fca30a9b0 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -1,105 +1,123 @@ -{% macro snowflake__get_catalog(information_schema, schemas, relations) -%} -{#- --- `relations` is a dictionary with schema names as keys and an iterable (likely list) of relation identifiers as values --#} - {% set query %} - with tables as ( - - select - table_catalog as "table_database", - table_schema as "table_schema", - table_name as "table_name", - table_type as "table_type", - comment as "table_comment", - - -- note: this is the _role_ that owns the table - table_owner as "table_owner", - - 'Clustering Key' as "stats:clustering_key:label", - clustering_key as "stats:clustering_key:value", - 'The key used to cluster this table' as "stats:clustering_key:description", - (clustering_key is not null) as "stats:clustering_key:include", - - 'Row Count' as "stats:row_count:label", - row_count as "stats:row_count:value", - 'An approximate count of rows in this table' as "stats:row_count:description", - (row_count is not null) as "stats:row_count:include", - - 'Approximate Size' as "stats:bytes:label", - bytes as "stats:bytes:value", - 'Approximate size of the table as reported by Snowflake' as "stats:bytes:description", - (bytes is not null) as "stats:bytes:include", - - 'Last Modified' as "stats:last_modified:label", - to_varchar(convert_timezone('UTC', last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value", - 'The timestamp for last update/change' as "stats:last_modified:description", - (last_altered is not null and table_type='BASE TABLE') as "stats:last_modified:include" - - from {{ information_schema }}.tables - {{ snowflake__get_catalog_where_clause(schemas, relations) }} - - ), - - columns as ( - - select - table_catalog as "table_database", - table_schema as "table_schema", - table_name as "table_name", - - column_name as "column_name", - ordinal_position as "column_index", - data_type as "column_type", - comment as "column_comment" - - from {{ information_schema }}.columns - {{ snowflake__get_catalog_where_clause(schemas, relations) }} - ) - - select * - from tables - join columns using ("table_database", "table_schema", "table_name") - order by "column_index" +{% macro snowflake__get_catalog(information_schema, schemas) -%} + + {% set query %} + with tables as ( + {{ snowflake__get_catalog_tables_sql(information_schema) }} + {{ snowflake__get_catalog_schemas_where_clause_sql(schemas) }} + ), + columns as ( + {{ snowflake__get_catalog_columns_sql(information_schema) }} + {{ snowflake__get_catalog_schemas_where_clause_sql(schemas) }} + ) + {{ snowflake__get_catalog_results_sql() }} {%- endset -%} - {{ return(run_query(query)) }} + {{ return(run_query(query)) }} {%- endmacro %} -{% macro snowflake__get_catalog_where_clause(schemas, relations) %} -{#- --- `relations` is a dictionary with schema names as keys and an iterable (likely list) of relation identifiers as values --#} +{% macro snowflake__get_catalog_relations(information_schema, relations) -%} - {% if schemas is not none %} - where ( - {%- for schema in schemas -%} - upper("table_schema") = upper('{{ schema }}') - {%- if not loop.last %} or {% endif -%} - {%- endfor -%} + {% set query %} + with tables as ( + {{ snowflake__get_catalog_tables_sql(information_schema) }} + {{ snowflake__get_catalog_relations_where_clause_sql(schemas) }} + ), + columns as ( + {{ snowflake__get_catalog_columns_sql(information_schema) }} + {{ snowflake__get_catalog_relations_where_clause_sql(schemas) }} ) + {{ snowflake__get_catalog_results_sql() }} + {%- endset -%} - {% elif relations is not none %} - where ( - {%- for schema_name, relations in relations.items() -%} - ( - upper("table_schema") = upper('{{ schema_name }}') - and upper("table_name") in ( - {%- for relation_name in relations -%} - upper('{{ relation_name }}') - {%- if not loop.last -%}, {% endif -%} - {%- endfor -%} - ) - {%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) + {{ return(run_query(query)) }} + +{%- endmacro %} + + +{% macro snowflake__get_catalog_tables_sql(information_schema) -%} + select + table_catalog as "table_database", + table_schema as "table_schema", + table_name as "table_name", + table_type as "table_type", + comment as "table_comment", + + -- note: this is the _role_ that owns the table + table_owner as "table_owner", + + 'Clustering Key' as "stats:clustering_key:label", + clustering_key as "stats:clustering_key:value", + 'The key used to cluster this table' as "stats:clustering_key:description", + (clustering_key is not null) as "stats:clustering_key:include", + + 'Row Count' as "stats:row_count:label", + row_count as "stats:row_count:value", + 'An approximate count of rows in this table' as "stats:row_count:description", + (row_count is not null) as "stats:row_count:include", + + 'Approximate Size' as "stats:bytes:label", + bytes as "stats:bytes:value", + 'Approximate size of the table as reported by Snowflake' as "stats:bytes:description", + (bytes is not null) as "stats:bytes:include", + + 'Last Modified' as "stats:last_modified:label", + to_varchar(convert_timezone('UTC', last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value", + 'The timestamp for last update/change' as "stats:last_modified:description", + (last_altered is not null and table_type='BASE TABLE') as "stats:last_modified:include" + from {{ information_schema }}.tables +{%- endmacro %} + + +{% macro snowflake__get_catalog_columns_sql(information_schema) -%} + select + table_catalog as "table_database", + table_schema as "table_schema", + table_name as "table_name", - {% else %} - {% do exceptions.raise_compiler_error( - '`get_catalog` requires a list of schema names or a dict of relation names by schema name.' - ) %} + column_name as "column_name", + ordinal_position as "column_index", + data_type as "column_type", + comment as "column_comment" + from {{ information_schema }}.columns +{%- endmacro %} + + +{% macro snowflake__get_catalog_results_sql() -%} + select * + from tables + join columns using ("table_database", "table_schema", "table_name") + order by "column_index" +{%- endmacro %} + + +{% macro snowflake__get_catalog_schemas_where_clause_sql(schema) -%} + where ({%- for schema in schemas -%} + upper("table_schema") = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + {%- endfor -%}) +{%- endmacro %} - {% endif %} -{% endmacro %} +{% macro snowflake__get_catalog_relations_where_clause_sql(relations) -%} + where ( + {%- for relation in relations -%} + {% if relation.schema and relation.identifier %} + ( + upper("table_schema") = upper('{{ relation.schema }}') + and upper("table_name") = upper('{{ relation.identifier }}') + ) + {% elif relation.schema %} + ( + upper("table_schema") = upper('{{ relation.schema }}') + ) + {% else %} + {% do exceptions.raise_compiler_error( + '`get_catalog_relations` requires a list of relations, each with a schema' + ) %} + {% endif %} + + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) +{%- endmacro %} From e624ff1afd6df5a3fc8edb070aeb8957a633c49d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 15 Sep 2023 16:21:55 -0400 Subject: [PATCH 09/15] fixed reference in get_catalog_relations, added original dict version of relations temporarily for testing --- dbt/include/snowflake/macros/catalog.sql | 26 ++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index fca30a9b0..65fe37ea9 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -22,11 +22,11 @@ {% set query %} with tables as ( {{ snowflake__get_catalog_tables_sql(information_schema) }} - {{ snowflake__get_catalog_relations_where_clause_sql(schemas) }} + {{ snowflake__get_catalog_relations_dict_where_clause_sql(relations) }} ), columns as ( {{ snowflake__get_catalog_columns_sql(information_schema) }} - {{ snowflake__get_catalog_relations_where_clause_sql(schemas) }} + {{ snowflake__get_catalog_relations_dict_where_clause_sql(relations) }} ) {{ snowflake__get_catalog_results_sql() }} {%- endset -%} @@ -121,3 +121,25 @@ {%- endfor -%} ) {%- endmacro %} + + +{% macro snowflake__get_catalog_relations_dict_where_clause_sql(relations_by_schema) -%} + where ( + {%- for schema, relations in relations_by_schema.items() -%} + {% if relations %} + {% for relation in relations %} + ( + upper("table_schema") = upper('{{ schema }}') + and upper("table_name") = upper('{{ relation.identifier }}') + ) + {% endfor %} + {% else %} + ( + upper("table_schema") = upper('{{ schema }}') + ) + {% endif %} + + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) +{%- endmacro %} From de3393281a0a752ab6957633301bb33c6d89412b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 19 Sep 2023 16:47:04 -0400 Subject: [PATCH 10/15] remove dict version of relations based get_catalog, point to List[BaseRelation] version --- dbt/include/snowflake/macros/catalog.sql | 26 ++---------------------- 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index 65fe37ea9..97170964e 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -22,11 +22,11 @@ {% set query %} with tables as ( {{ snowflake__get_catalog_tables_sql(information_schema) }} - {{ snowflake__get_catalog_relations_dict_where_clause_sql(relations) }} + {{ snowflake__get_catalog_relations_where_clause_sql(relations) }} ), columns as ( {{ snowflake__get_catalog_columns_sql(information_schema) }} - {{ snowflake__get_catalog_relations_dict_where_clause_sql(relations) }} + {{ snowflake__get_catalog_relations_where_clause_sql(relations) }} ) {{ snowflake__get_catalog_results_sql() }} {%- endset -%} @@ -121,25 +121,3 @@ {%- endfor -%} ) {%- endmacro %} - - -{% macro snowflake__get_catalog_relations_dict_where_clause_sql(relations_by_schema) -%} - where ( - {%- for schema, relations in relations_by_schema.items() -%} - {% if relations %} - {% for relation in relations %} - ( - upper("table_schema") = upper('{{ schema }}') - and upper("table_name") = upper('{{ relation.identifier }}') - ) - {% endfor %} - {% else %} - ( - upper("table_schema") = upper('{{ schema }}') - ) - {% endif %} - - {%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) -{%- endmacro %} From 26db9498354be9e6fb99d9842fbcbe8dc37c65e6 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 3 Oct 2023 18:18:57 -0400 Subject: [PATCH 11/15] point dev reqs back to main on core --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index b94cdad56..c696a773c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@paw/get-catalog-relations#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@paw/get-catalog-relations#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor From 3b7dbff82011da8773b38d5b455cdec3e6cb811c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 3 Oct 2023 23:17:34 -0400 Subject: [PATCH 12/15] fix typo in schemas argument --- dbt/include/snowflake/macros/catalog.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/snowflake/macros/catalog.sql b/dbt/include/snowflake/macros/catalog.sql index 97170964e..eaa7582e9 100644 --- a/dbt/include/snowflake/macros/catalog.sql +++ b/dbt/include/snowflake/macros/catalog.sql @@ -92,7 +92,7 @@ {%- endmacro %} -{% macro snowflake__get_catalog_schemas_where_clause_sql(schema) -%} +{% macro snowflake__get_catalog_schemas_where_clause_sql(schemas) -%} where ({%- for schema in schemas -%} upper("table_schema") = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} {%- endfor -%}) From 08212d2acf256a70b44c853abb0ca72c4bd305fd Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 3 Oct 2023 23:50:46 -0400 Subject: [PATCH 13/15] add feature flag to turn on relation filtering for get_catalog --- dbt/adapters/snowflake/impl.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 671906242..2143ba55d 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -3,7 +3,7 @@ import agate -from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport # type: ignore +from dbt.adapters.base.impl import AdapterConfig, AdapterFeature, ConstraintSupport # type: ignore from dbt.adapters.base.meta import available from dbt.adapters.sql import SQLAdapter # type: ignore from dbt.adapters.sql.impl import ( @@ -256,6 +256,10 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str): def valid_incremental_strategies(self): return ["append", "merge", "delete+insert"] + @classmethod + def has_feature(cls, feature: AdapterFeature) -> bool: + return feature in [AdapterFeature.CatalogByRelations] + def debug_query(self): """Override for DebugTask method""" self.execute("select 1 as id") From d908ff804093d9769771af27b29c1784161bd933 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 9 Oct 2023 16:35:33 -0400 Subject: [PATCH 14/15] update changelog to point to the PR instead of a broken url --- .changes/unreleased/Features-20230829-152412.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Features-20230829-152412.yaml b/.changes/unreleased/Features-20230829-152412.yaml index 7ff5a15fa..364f755f3 100644 --- a/.changes/unreleased/Features-20230829-152412.yaml +++ b/.changes/unreleased/Features-20230829-152412.yaml @@ -3,4 +3,4 @@ body: Support limiting get_catalog by object name time: 2023-08-29T15:24:12.649104-04:00 custom: Author: mikealfare - Issue: "4997" + Issue: "758" From 423535e1a218c8d2900202486523719a4834cd8a Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Wed, 11 Oct 2023 16:09:47 -0400 Subject: [PATCH 15/15] Support changes to dbt-core capability system --- dbt/adapters/snowflake/impl.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 2143ba55d..3f8124a9a 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -3,8 +3,9 @@ import agate -from dbt.adapters.base.impl import AdapterConfig, AdapterFeature, ConstraintSupport # type: ignore +from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport # type: ignore from dbt.adapters.base.meta import available +from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability from dbt.adapters.sql import SQLAdapter # type: ignore from dbt.adapters.sql.impl import ( LIST_SCHEMAS_MACRO_NAME, @@ -49,6 +50,10 @@ class SnowflakeAdapter(SQLAdapter): ConstraintType.foreign_key: ConstraintSupport.NOT_ENFORCED, } + _capabilities = CapabilityDict( + {Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full)} + ) + @classmethod def date_function(cls): return "CURRENT_TIMESTAMP()" @@ -256,10 +261,6 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str): def valid_incremental_strategies(self): return ["append", "merge", "delete+insert"] - @classmethod - def has_feature(cls, feature: AdapterFeature) -> bool: - return feature in [AdapterFeature.CatalogByRelations] - def debug_query(self): """Override for DebugTask method""" self.execute("select 1 as id")