BigQuery check if schema already exists before trying to create it (d…

…bt-labs#220) * fix check schema * fix when we do not need to update the schema --------- Co-authored-by: Thomas van Latum <[email protected]>
thomas-vl · Aug 30, 2023 · 9a4d4da · 9a4d4da
1 parent bc33d37
commit 9a4d4da
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 9 deletions.
diff --git a/macros/plugins/bigquery/create_external_schema.sql b/macros/plugins/bigquery/create_external_schema.sql
@@ -7,9 +7,21 @@
         {%- endif -%}
     {%- endset -%}
 
-    {%- set ddl -%}
-        create schema if not exists {{ fqn }}
-    {%- endset -%}
+    {% set schema_exists_query %}
+        select * from {{ source_node.database }}.INFORMATION_SCHEMA.SCHEMATA where schema_name = '{{ source_node.schema }}' limit 1
+    {% endset %}
+    {% if execute %}
+        {% set schema_exists = run_query(schema_exists_query)|length > 0 %}
+    {% else %}
+        {% set schema_exists = false %}
+    {% endif %}  
 
-    {{ return(ddl) }}
+    {%- if not schema_exists -%}
+        {%- set ddl -%}
+            create schema if not exists {{ fqn }}
+        {%- endset -%}
+        {{ return(ddl) }}
+    {%- else -%}
+        {{ return('') }}
+    {% endif %} 
 {%- endmacro -%}
diff --git a/macros/plugins/bigquery/get_external_build_plan.sql b/macros/plugins/bigquery/get_external_build_plan.sql
@@ -11,10 +11,16 @@
     {% set create_or_replace = (old_relation is none or var('ext_full_refresh', false)) %}
 
     {% if create_or_replace %}
-        {% set build_plan = build_plan + [
-            dbt_external_tables.create_external_schema(source_node),
-            dbt_external_tables.create_external_table(source_node)
-        ] %}
+        {% if not dbt_external_tables.create_external_schema(source_node)|length %}
+            {% set build_plan = build_plan + [
+                dbt_external_tables.create_external_table(source_node)
+            ] %}
+        {% else %}
+            {% set build_plan = build_plan + [
+                dbt_external_tables.create_external_schema(source_node),
+                dbt_external_tables.create_external_table(source_node)
+            ] %}
+        {% endif %}
     {% else %}
         {% set build_plan = build_plan + dbt_external_tables.refresh_external_table(source_node) %}
     {% endif %}

diff --git a/run_test.sh b/run_test.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-echo "Setting up virtual environment"
+echo "Setting up virtual environment for dbt-$1"
 VENV="venv/bin/activate"
 
 if [[ ! -f $VENV ]]; then