Add option to use most recently created / updated model (#19)

* Modularise ref macro

* Enable selection of last updated table, modularise macro

* Add prefer_recent tests

* Add prefer_recent docs

* Apply singular tests fix

* Bump version

* Fix whitespace

* Clarify prefer_recent support

README.md

@@ -78,6 +78,7 @@ When using env schemas, you also need to add the `is_upstream_prod` parameter to
 - `upstream_prod_enabled`: Disables the package when False. Defaults to True.
 - `upstream_prod_disabled_targets`: List of targets where the package should be disabled.
 - `upstream_prod_fallback`: Whether to fall back to the default target when a model can't be found in prod. Defaults to False.
+- `upstream_prod_prefer_recent`: Whether to use dev relations that were updated more recently than prod; particularly useful when working on multiple large / slow models at once. Only supported in Snowflake & BigQuery. Defaults to False.
 
 **Example**
 
@@ -88,6 +89,7 @@ I use Snowflake and each developer has a separate database with identically-name
 vars:
   upstream_prod_database: <prod_db> # replace with your prod db
   upstream_prod_fallback: True
+  upstream_prod_prefer_recent: True
   upstream_prod_disabled_targets:
     - ci
     - prod
@@ -112,10 +114,11 @@ In your `macros` directory, create a file called `ref.sql` with the following co
     enabled=var("upstream_prod_enabled", True),
     fallback=var("upstream_prod_fallback", False),
     env_schemas=var("upstream_prod_env_schemas", False),
-    version=None
+    version=None,
+    prefer_recent=var("upstream_prod_prefer_recent", False)
 ) %}
 
-  {% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}
+  {% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version, prefer_recent)) %}
 
 {% endmacro %}
 ```

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'upstream_prod'
-version: '0.6.2'
+version: '0.7.0'
 config-version: 2
 
 require-dbt-version: [">=1.5.0", "<2.0.0"]

integration_tests/_template/macros/ref.sql

@@ -5,7 +5,8 @@
     enabled=var("upstream_prod_enabled", True),
     fallback=var("upstream_prod_fallback", False),
     env_schemas=var("upstream_prod_env_schemas", False),
-    version=None
+    version=None,
+    prefer_recent=var("upstream_prod_prefer_recent", False)
 ) %}
 
     {% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}

integration_tests/_template/models/production/dev_newer.sql

@@ -0,0 +1,11 @@
+select
+    source_target,
+    source_database,
+    source_schema,
+    source_model,
+    '{{ target.name }}' as this_target,
+    '{{ this.database }}' as this_database,
+    '{{ this.schema }}' as this_schema,
+    '{{ this.name }}' as this_model
+from
+    {{ ref('stg__dev_newer') }}

integration_tests/_template/models/staging/stg__dev_newer.sql

@@ -0,0 +1,5 @@
+select 
+    '{{ target.name }}' as source_target,
+    '{{ this.database }}' as source_database,
+    '{{ this.schema }}' as source_schema,
+    '{{ this.name }}' as source_model

integration_tests/_template/run_tests.sh

@@ -8,8 +8,8 @@ dbt run-operation create_test_db --args '{db: upstream__dev_db}'
 
 # Create staging model in appropriate envs
 echo "\nBUILDING STAGING MODELS\n"
-dbt build -s stg__defer_prod stg__defer_vers --target prod
-dbt build -s stg__dev_fallback
+dbt build -s stg__defer_prod stg__defer_vers stg__dev_newer --target prod
+dbt build -s stg__dev_fallback stg__dev_newer
 
 # Build & test downstream models
 echo "\nBUILDING DOWNSTREAM MODELS\n"

integration_tests/dev_db/dbt_project.yml

@@ -22,3 +22,4 @@ models:
 vars:
   upstream_prod_database: upstream__prod_db
   upstream_prod_fallback: true
+  upstream_prod_prefer_recent: true

integration_tests/dev_db/macros/ref.sql

@@ -5,7 +5,8 @@
     enabled=var("upstream_prod_enabled", True),
     fallback=var("upstream_prod_fallback", False),
     env_schemas=var("upstream_prod_env_schemas", False),
-    version=None
+    version=None,
+    prefer_recent=var("upstream_prod_prefer_recent", False)
 ) %}
 
     {% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}

integration_tests/dev_db/models/production/_dev_db_models.yml

@@ -161,6 +161,44 @@ models:
               values: ['dev_fallback']
 
 
+  - name: dev_newer
+    columns:
+      # Source
+      - name: source_target
+        tests:
+          - accepted_values:
+              values: ['dev']
+      - name: source_database
+        tests:
+          - accepted_values:
+              values: ['upstream__dev_db']
+      - name: source_schema
+        tests:
+          - accepted_values:
+              values: ['prod_stg']
+      - name: source_model
+        tests:
+          - accepted_values:
+              values: ['stg__dev_newer']
+      # This
+      - name: this_target
+        tests:
+          - accepted_values:
+              values: ['dev']
+      - name: this_database
+        tests:
+          - accepted_values:
+              values: ['upstream__dev_db']
+      - name: this_schema
+        tests:
+          - accepted_values:
+              values: ['prod']
+      - name: this_model
+        tests:
+          - accepted_values:
+              values: ['dev_newer']
+
+
   - name: ephem
     columns:
       # Source

integration_tests/dev_db/models/production/dev_newer.sql

@@ -0,0 +1,11 @@
+select
+    source_target,
+    source_database,
+    source_schema,
+    source_model,
+    '{{ target.name }}' as this_target,
+    '{{ this.database }}' as this_database,
+    '{{ this.schema }}' as this_schema,
+    '{{ this.name }}' as this_model
+from
+    {{ ref('stg__dev_newer') }}

integration_tests/dev_db/models/staging/stg__dev_newer.sql

@@ -0,0 +1,5 @@
+select 
+    '{{ target.name }}' as source_target,
+    '{{ this.database }}' as source_database,
+    '{{ this.schema }}' as source_schema,
+    '{{ this.name }}' as source_model

integration_tests/dev_db/run_tests.sh

@@ -8,8 +8,8 @@ dbt run-operation create_test_db --args '{db: upstream__dev_db}'
 
 # Create staging model in appropriate envs
 echo "\nBUILDING STAGING MODELS\n"
-dbt build -s stg__defer_prod stg__defer_vers --target prod
-dbt build -s stg__dev_fallback
+dbt build -s stg__defer_prod stg__defer_vers stg__dev_newer --target prod
+dbt build -s stg__dev_fallback stg__dev_newer
 
 # Build & test downstream models
 echo "\nBUILDING DOWNSTREAM MODELS\n"

integration_tests/dev_db_dev_sch/dbt_project.yml

@@ -23,3 +23,4 @@ vars:
   upstream_prod_database: upstream__prod_db
   upstream_prod_schema: prod
   upstream_prod_fallback: true
+  upstream_prod_prefer_recent: true

integration_tests/dev_db_dev_sch/macros/ref.sql

@@ -5,7 +5,8 @@
     enabled=var("upstream_prod_enabled", True),
     fallback=var("upstream_prod_fallback", False),
     env_schemas=var("upstream_prod_env_schemas", False),
-    version=None
+    version=None,
+    prefer_recent=var("upstream_prod_prefer_recent", False)
 ) %}
 
     {% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}

integration_tests/dev_db_dev_sch/models/production/_dev_db_dev_sch_models.yml

@@ -161,6 +161,44 @@ models:
               values: ['dev_fallback']
 
 
+  - name: dev_newer
+    columns:
+      # Source
+      - name: source_target
+        tests:
+          - accepted_values:
+              values: ['dev']
+      - name: source_database
+        tests:
+          - accepted_values:
+              values: ['upstream__dev_db']
+      - name: source_schema
+        tests:
+          - accepted_values:
+              values: ['dev_stg']
+      - name: source_model
+        tests:
+          - accepted_values:
+              values: ['stg__dev_newer']
+      # This
+      - name: this_target
+        tests:
+          - accepted_values:
+              values: ['dev']
+      - name: this_database
+        tests:
+          - accepted_values:
+              values: ['upstream__dev_db']
+      - name: this_schema
+        tests:
+          - accepted_values:
+              values: ['dev']
+      - name: this_model
+        tests:
+          - accepted_values:
+              values: ['dev_newer']
+
+
   - name: ephem
     columns:
       # Source

integration_tests/dev_db_dev_sch/models/production/dev_newer.sql

@@ -0,0 +1,11 @@
+select
+    source_target,
+    source_database,
+    source_schema,
+    source_model,
+    '{{ target.name }}' as this_target,
+    '{{ this.database }}' as this_database,
+    '{{ this.schema }}' as this_schema,
+    '{{ this.name }}' as this_model
+from
+    {{ ref('stg__dev_newer') }}

integration_tests/dev_db_dev_sch/models/staging/stg__dev_newer.sql

@@ -0,0 +1,5 @@
+select 
+    '{{ target.name }}' as source_target,
+    '{{ this.database }}' as source_database,
+    '{{ this.schema }}' as source_schema,
+    '{{ this.name }}' as source_model

integration_tests/dev_db_dev_sch/run_tests.sh

@@ -8,8 +8,8 @@ dbt run-operation create_test_db --args '{db: upstream__dev_db}'
 
 # Create staging model in appropriate envs
 echo "\nBUILDING STAGING MODELS\n"
-dbt build -s stg__defer_prod stg__defer_vers --target prod
-dbt build -s stg__dev_fallback
+dbt build -s stg__defer_prod stg__defer_vers stg__dev_newer --target prod
+dbt build -s stg__dev_fallback stg__dev_newer
 
 # Build & test downstream models
 echo "\nBUILDING DOWNSTREAM MODELS\n"

integration_tests/dev_db_env_sch/dbt_project.yml

@@ -23,3 +23,4 @@ vars:
   upstream_prod_database: upstream__prod_db
   upstream_prod_fallback: true
   upstream_prod_env_schemas: true
+  upstream_prod_prefer_recent: true

integration_tests/dev_db_env_sch/macros/ref.sql

@@ -5,7 +5,8 @@
     enabled=var("upstream_prod_enabled", True),
     fallback=var("upstream_prod_fallback", False),
     env_schemas=var("upstream_prod_env_schemas", False),
-    version=None
+    version=None,
+    prefer_recent=var("upstream_prod_prefer_recent", False)
 ) %}
 
     {% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}

integration_tests/dev_db_env_sch/models/production/_dev_db_env_sch_models.yml

@@ -161,6 +161,44 @@ models:
               values: ['dev_fallback']
 
 
+  - name: dev_newer
+    columns:
+      # Source
+      - name: source_target
+        tests:
+          - accepted_values:
+              values: ['dev']
+      - name: source_database
+        tests:
+          - accepted_values:
+              values: ['upstream__dev_db']
+      - name: source_schema
+        tests:
+          - accepted_values:
+              values: ['dev']
+      - name: source_model
+        tests:
+          - accepted_values:
+              values: ['stg__dev_newer']
+      # This
+      - name: this_target
+        tests:
+          - accepted_values:
+              values: ['dev']
+      - name: this_database
+        tests:
+          - accepted_values:
+              values: ['upstream__dev_db']
+      - name: this_schema
+        tests:
+          - accepted_values:
+              values: ['dev']
+      - name: this_model
+        tests:
+          - accepted_values:
+              values: ['dev_newer']
+
+
   - name: ephem
     columns:
       # Source

integration_tests/dev_db_env_sch/models/production/dev_newer.sql

@@ -0,0 +1,11 @@
+select
+    source_target,
+    source_database,
+    source_schema,
+    source_model,
+    '{{ target.name }}' as this_target,
+    '{{ this.database }}' as this_database,
+    '{{ this.schema }}' as this_schema,
+    '{{ this.name }}' as this_model
+from
+    {{ ref('stg__dev_newer') }}

integration_tests/dev_db_env_sch/models/staging/stg__dev_newer.sql

@@ -0,0 +1,5 @@
+select 
+    '{{ target.name }}' as source_target,
+    '{{ this.database }}' as source_database,
+    '{{ this.schema }}' as source_schema,
+    '{{ this.name }}' as source_model

integration_tests/dev_db_env_sch/run_tests.sh

@@ -8,8 +8,8 @@ dbt run-operation create_test_db --args '{db: upstream__dev_db}'
 
 # Create staging model in appropriate envs
 echo "\nBUILDING STAGING MODELS\n"
-dbt build -s stg__defer_prod stg__defer_vers --target prod
-dbt build -s stg__dev_fallback
+dbt build -s stg__defer_prod stg__defer_vers stg__dev_newer --target prod
+dbt build -s stg__dev_fallback stg__dev_newer
 
 # Build & test downstream models
 echo "\nBUILDING DOWNSTREAM MODELS\n"

integration_tests/dev_sch/dbt_project.yml

@@ -22,3 +22,4 @@ models:
 vars:
   upstream_prod_schema: prod  
   upstream_prod_fallback: true
+  upstream_prod_prefer_recent: true

integration_tests/dev_sch/macros/ref.sql

@@ -5,7 +5,8 @@
     enabled=var("upstream_prod_enabled", True),
     fallback=var("upstream_prod_fallback", False),
     env_schemas=var("upstream_prod_env_schemas", False),
-    version=None
+    version=None,
+    prefer_recent=var("upstream_prod_prefer_recent", False)
 ) %}
 
     {% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}