Merge pull request #112 from alanmcruickshank/ac/specify_stage

Run specific stages
brooklyn-data · Mar 30, 2022 · 79109a9 · 79109a9
2 parents ea64592 + fdb6a50
commit 79109a9
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 4 deletions.
diff --git a/macros/upload_artifacts.sql b/macros/upload_artifacts.sql
@@ -2,8 +2,9 @@
 
 {% set src_dbt_artifacts = source('dbt_artifacts', 'artifacts') %}
 
+{# All uploads are prefixed by the invocation_id in the stage to isolate parallel jobs from one another #}
 {% set remove_query %}
-    remove @{{ src_dbt_artifacts }} pattern='.*.json.gz';
+    remove @{{ src_dbt_artifacts }} pattern='.*\/{{ invocation_id }}\/.*\.json.gz';
 {% endset %}
 
 {% do log("Clearing existing files from Stage: " ~ remove_query, info=True) %}
@@ -14,7 +15,7 @@
     {% set file = filename ~ '.json' %}
 
     {% set put_query %}
-        put file://{{ prefix }}{{ file }} @{{ src_dbt_artifacts }} auto_compress=true;
+        put file://{{ prefix }}{{ file }} @{{ src_dbt_artifacts }}/{{ invocation_id }} auto_compress=true;
     {% endset %}
 
     {% do log("Uploading " ~ file ~ " to Stage: " ~ put_query, info=True) %}

diff --git a/macros/upload_artifacts_v2.sql b/macros/upload_artifacts_v2.sql
@@ -10,8 +10,9 @@
 {% set src_results_nodes = source('dbt_artifacts', 'dbt_run_results_nodes') %}
 {% set src_manifest_nodes = source('dbt_artifacts', 'dbt_manifest_nodes') %}
 
+{# All uploads are prefixed by the invocation_id in the stage to isolate parallel jobs from one another #}
 {% set remove_query %}
-    remove @{{ artifact_stage }} pattern='.*.json.gz';
+    remove @{{ artifact_stage }} pattern='.*\/{{ invocation_id }}\/.*\.json.gz';
 {% endset %}
 
 {% set results_query %}
@@ -191,7 +192,7 @@
     {% set file = filename ~ '.json' %}
 
     {% set put_query %}
-        put file://{{ prefix }}{{ file }} @{{ artifact_stage }} auto_compress=true;
+        put file://{{ prefix }}{{ file }} @{{ artifact_stage }}/{{ invocation_id }} auto_compress=true;
     {% endset %}
 
     {% do log("Uploading " ~ file ~ " to Stage: " ~ put_query, info=True) %}