From 0308a097e0f2d67d1499e3074dd212252caf9d87 Mon Sep 17 00:00:00 2001
From: Thomas van Latum <thomas.v.latum@gmail.com>
Date: Mon, 21 Aug 2023 10:57:49 +0200
Subject: [PATCH] Add table expiry

---
 .../models/plugins/bigquery/bigquery_external.yml      | 10 ++++++++++
 macros/plugins/bigquery/create_external_table.sql      |  6 +++++-
 sample_sources/bigquery.yml                            |  9 +++++++++
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/integration_tests/models/plugins/bigquery/bigquery_external.yml b/integration_tests/models/plugins/bigquery/bigquery_external.yml
index 3b81230b..278bd0ed 100644
--- a/integration_tests/models/plugins/bigquery/bigquery_external.yml
+++ b/integration_tests/models/plugins/bigquery/bigquery_external.yml
@@ -67,6 +67,16 @@ sources:
         columns: *cols-of-the-people
         tests: *equal-to-the-people
 
+      - name: people_csv_schema_auto_detect_expiration
+        external:
+          hours_to_expiration: 24
+          location: 'gs://dbt-external-tables-testing/csv/*'
+          options:
+            format: csv
+            skip_leading_rows: 1
+            hive_partition_uri_prefix: 'gs://dbt-external-tables-testing/csv'
+        tests: *equal-to-the-people
+
 #      - name: people_json_unpartitioned
 #        external: &json-people
 #          location: 'gs://dbt-external-tables-testing/json/*'
diff --git a/macros/plugins/bigquery/create_external_table.sql b/macros/plugins/bigquery/create_external_table.sql
index af0970cf..375f2717 100644
--- a/macros/plugins/bigquery/create_external_table.sql
+++ b/macros/plugins/bigquery/create_external_table.sql
@@ -4,7 +4,8 @@
     {%- set external = source_node.external -%}
     {%- set partitions = external.partitions -%}
     {%- set options = external.options -%}
-    
+    {%- set hours_to_expiration = external.get('hours_to_expiration') -%}
+
     {%- set uris = [] -%}
     {%- if options is mapping and options.get('uris', none) -%}
         {%- set uris = external.options.get('uris') -%}
@@ -38,5 +39,8 @@
                 {%- endif -%}
             {%- endfor -%}
             {%- endif -%}
+            {%- if hours_to_expiration -%}
+                , expiration_timestamp = TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL {{hours_to_expiration}} hour)
+            {%- endif -%}
         )
 {% endmacro %}
diff --git a/sample_sources/bigquery.yml b/sample_sources/bigquery.yml
index cfe18cde..970c69ee 100644
--- a/sample_sources/bigquery.yml
+++ b/sample_sources/bigquery.yml
@@ -58,3 +58,12 @@ sources:
               - 'gs://bucket_a/path/*'
               - 'gs://bucket_b/path/*'
               - 'gs://bucket_c/more/specific/path/file.csv'
+
+      #  you can use BigQuery table expiry if you want to automatically delete tables after a period of time
+      - name: table_expiry
+        external:
+          location: 'gs://bucket/path/*'
+          hours_to_expiration: 24
+          options:
+            format: csv
+            skip_leading_rows: 1
\ No newline at end of file