From d0c0de8fa6e0086f4042020658ba8f2d540124c0 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 18:04:29 -0400 Subject: [PATCH] remove partial parsing tests, they do not use any dbt-postgres or dbt-adapters functionality --- tests/functional/partial_parsing/fixtures.py | 1228 ----------------- .../partial_parsing/test_file_diff.py | 63 - .../partial_parsing/test_partial_parsing.py | 824 ----------- .../test_pp_disabled_config.py | 224 --- .../partial_parsing/test_pp_docs.py | 257 ---- .../partial_parsing/test_pp_groups.py | 155 --- .../partial_parsing/test_pp_metrics.py | 85 -- .../partial_parsing/test_pp_vars.py | 398 ------ .../partial_parsing/test_versioned_models.py | 128 -- 9 files changed, 3362 deletions(-) delete mode 100644 tests/functional/partial_parsing/fixtures.py delete mode 100644 tests/functional/partial_parsing/test_file_diff.py delete mode 100644 tests/functional/partial_parsing/test_partial_parsing.py delete mode 100644 tests/functional/partial_parsing/test_pp_disabled_config.py delete mode 100644 tests/functional/partial_parsing/test_pp_docs.py delete mode 100644 tests/functional/partial_parsing/test_pp_groups.py delete mode 100644 tests/functional/partial_parsing/test_pp_metrics.py delete mode 100644 tests/functional/partial_parsing/test_pp_vars.py delete mode 100644 tests/functional/partial_parsing/test_versioned_models.py diff --git a/tests/functional/partial_parsing/fixtures.py b/tests/functional/partial_parsing/fixtures.py deleted file mode 100644 index f76d90ad..00000000 --- a/tests/functional/partial_parsing/fixtures.py +++ /dev/null @@ -1,1228 +0,0 @@ -local_dependency__dbt_project_yml = """ - -name: 'local_dep' -version: '1.0' -config-version: 2 - -profile: 'default' - -model-paths: ["models"] -analysis-paths: ["analyses"] -test-paths: ["tests"] -seed-paths: ["seeds"] -macro-paths: ["macros"] - -require-dbt-version: '>=0.1.0' - -target-path: "target" # directory which will store compiled SQL files -clean-targets: # directories to be removed by `dbt clean` - - "target" - - "dbt_packages" - - -seeds: - quote_columns: False - -""" - -local_dependency__models__schema_yml = """ -sources: - - name: seed_source - schema: "{{ var('schema_override', target.schema) }}" - tables: - - name: "seed" - columns: - - name: id - data_tests: - - unique - -""" - -local_dependency__models__model_to_import_sql = """ -select * from {{ ref('seed') }} - -""" - -local_dependency__macros__dep_macro_sql = """ -{% macro some_overridden_macro() -%} -100 -{%- endmacro %} - -""" - -local_dependency__seeds__seed_csv = """id -1 -""" - -empty_schema_with_version_yml = """ - -""" - -schema_sources5_yml = """ - -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - data_tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - name: first_name - - name: last_name - - name: email - -seeds: - - name: rad_customers - description: "Raw customer data" - columns: - - name: id - data_tests: - - unique - - not_null - - name: first_name - - name: last_name - - name: email - - -""" - -my_macro2_sql = """ -{% macro do_something(foo2, bar2) %} - - select - 'foo' as foo2, - 'var' as bar2 - -{% endmacro %} - -""" - -raw_customers_csv = """id,first_name,last_name,email -1,Michael,Perez,mperez0@chronoengine.com -2,Shawn,Mccoy,smccoy1@reddit.com -3,Kathleen,Payne,kpayne2@cargocollective.com -4,Jimmy,Cooper,jcooper3@cargocollective.com -5,Katherine,Rice,krice4@typepad.com -6,Sarah,Ryan,sryan5@gnu.org -7,Martin,Mcdonald,mmcdonald6@opera.com -8,Frank,Robinson,frobinson7@wunderground.com -9,Jennifer,Franklin,jfranklin8@mail.ru -10,Henry,Welch,hwelch9@list-manage.com -""" - -model_three_disabled2_sql = """ -- Disabled model -{{ config(materialized='table', enabled=False) }} - -with source_data as ( - - select 1 as id - union all - select null as id - -) - -select * -from source_data - -""" - -schema_sources4_yml = """ - -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - data_tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - every_value_is_blue - - name: first_name - - name: last_name - - name: email - -seeds: - - name: raw_customers - description: "Raw customer data" - columns: - - name: id - data_tests: - - unique - - not_null - - name: first_name - - name: last_name - - name: email - - -""" - -env_var_schema_yml = """ - -models: - - name: model_one - config: - materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" - -""" - -my_test_sql = """ -select - * from {{ ref('customers') }} where first_name = '{{ macro_something() }}' - -""" - -empty_schema_yml = """ - -""" - -schema_models_c_yml = """ - -sources: - - name: seed_source - description: "This is a source override" - overrides: local_dep - schema: "{{ var('schema_override', target.schema) }}" - tables: - - name: "seed" - columns: - - name: id - data_tests: - - unique - - not_null - -""" - -env_var_sources_yml = """ -sources: - - name: seed_sources - schema: "{{ target.schema }}" - database: "{{ env_var('ENV_VAR_DATABASE') }}" - tables: - - name: raw_customers - columns: - - name: id - data_tests: - - not_null: - severity: "{{ env_var('ENV_VAR_SEVERITY') }}" - - unique - - name: first_name - - name: last_name - - name: email - - - -""" - -generic_test_edited_sql = """ -{% test is_odd(model, column_name) %} - -with validation as ( - - select - {{ column_name }} as odd_field2 - - from {{ model }} - -), - -validation_errors as ( - - select - odd_field2 - - from validation - -- if this is true, then odd_field is actually even! - where (odd_field2 % 2) = 0 - -) - -select * -from validation_errors - -{% endtest %} -""" - -schema_sources1_yml = """ -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - data_tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - name: first_name - - name: last_name - - name: email - - - -""" - -schema_sources3_yml = """ - -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - data_tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - name: first_name - - name: last_name - - name: email - -exposures: - - name: proxy_for_dashboard - description: "This is for the XXX dashboard" - type: "dashboard" - owner: - name: "Dashboard Tester" - email: "tester@dashboard.com" - depends_on: - - ref("model_one") - - source("seed_sources", "raw_customers") - - -""" - -my_analysis_sql = """ -select * from customers - -""" - -schema_sources2_yml = """ - -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - data_tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - name: first_name - - name: last_name - - name: email - -exposures: - - name: proxy_for_dashboard - description: "This is for the XXX dashboard" - type: "dashboard" - owner: - name: "Dashboard Tester" - email: "tester@dashboard.com" - depends_on: - - ref("model_one") - - ref("raw_customers") - - source("seed_sources", "raw_customers") - - -""" - -model_color_sql = """ -select 'blue' as fun - -""" - -my_metric_yml = """ -metrics: - - name: new_customers - label: New Customers - model: customers - description: "The number of paid customers who are using the product" - type: simple - type_params: - measure: - name: customers - filter: "{{ Dimension('id__loves_dbt') }} is true" - +meta: - is_okr: True - tags: - - okrs - - - -""" - -env_var_schema2_yml = """ - -models: - - name: model_one - config: - materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" - data_tests: - - check_color: - column_name: fun - color: "env_var('ENV_VAR_COLOR')" - - -""" - -gsm_override_sql = """ -- custom macro -{% macro generate_schema_name(schema_name, node) %} - - {{ schema_name }}_{{ target.schema }} - -{% endmacro %} - -""" - -model_four1_sql = """ -select * from {{ ref('model_three') }} - -""" - -model_one_sql = """ -select 1 as fun - -""" - -metricflow_time_spine_sql = """ -SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day -""" - -env_var_schema3_yml = """ - -models: - - name: model_one - config: - materialized: "{{ env_var('TEST_SCHEMA_VAR') }}" - data_tests: - - check_color: - column_name: fun - color: "env_var('ENV_VAR_COLOR')" - -exposures: - - name: proxy_for_dashboard - description: "This is for the XXX dashboard" - type: "dashboard" - owner: - name: "{{ env_var('ENV_VAR_OWNER') }}" - email: "tester@dashboard.com" - depends_on: - - ref("model_color") - - source("seed_sources", "raw_customers") - -""" - -people_semantic_models_yml = """ -version: 2 - -semantic_models: - - name: semantic_people - model: ref('people') - dimensions: - - name: favorite_color - type: categorical - - name: created_at - type: TIME - type_params: - time_granularity: day - measures: - - name: years_tenure - agg: SUM - expr: tenure - - name: people - agg: count - expr: id - entities: - - name: id - type: primary - defaults: - agg_time_dimension: created_at -""" - -env_var_metrics_yml = """ - -metrics: - - - name: number_of_people - description: Total count of people - label: "Number of people" - type: simple - type_params: - measure: people - meta: - my_meta: '{{ env_var("ENV_VAR_METRICS") }}' - - - name: collective_tenure - description: Total number of years of team experience - label: "Collective tenure" - type: simple - type_params: - measure: - name: years_tenure - filter: "{{ Dimension('id__loves_dbt') }} is true" - -""" - -customers_sql = """ -with source as ( - - select * from {{ source('seed_sources', 'raw_customers') }} - -), - -renamed as ( - - select - id as customer_id, - first_name, - last_name, - email - - from source - -) - -select * from renamed - -""" - -model_four2_sql = """ -select fun from {{ ref('model_one') }} - -""" - -env_var_model_sql = """ -select '{{ env_var('ENV_VAR_TEST') }}' as vartest - -""" - -env_var_model_one_sql = """ -select 'blue' as fun - -""" - -custom_schema_tests2_sql = """ -{% test type_one(model) %} - - select * from ( - - select * from {{ model }} - union all - select * from {{ ref('model_b') }} - - ) as Foo - -{% endtest %} - -{% test type_two(model) %} - - {{ config(severity = "ERROR") }} - - select * from {{ model }} - -{% endtest %} - -""" - -metric_model_a_sql = """ -{% - set metric_list = [ - metric('number_of_people'), - metric('collective_tenure') - ] -%} - -{% if not execute %} - - {% set metric_names = [] %} - {% for m in metric_list %} - {% do metric_names.append(m.metric_name) %} - {% endfor %} - - -- this config does nothing, but it lets us check these values - {{ config(metric_names = metric_names) }} - -{% endif %} - - -select 1 as fun - -""" - -model_b_sql = """ -select 1 as notfun - -""" - -customers2_md = """ -{% docs customer_table %} - -LOTS of customer data - -{% enddocs %} - -""" - -custom_schema_tests1_sql = """ -{% test type_one(model) %} - - select * from ( - - select * from {{ model }} - union all - select * from {{ ref('model_b') }} - - ) as Foo - -{% endtest %} - -{% test type_two(model) %} - - {{ config(severity = "WARN") }} - - select * from {{ model }} - -{% endtest %} - -""" - -people_metrics_yml = """ - -metrics: - - - name: number_of_people - description: Total count of people - label: "Number of people" - type: simple - type_params: - measure: people - meta: - my_meta: 'testing' - - - name: collective_tenure - description: Total number of years of team experience - label: "Collective tenure" - type: simple - type_params: - measure: - name: years_tenure - filter: "{{ Dimension('id__loves_dbt') }} is true" - -""" - -people_sql = """ -select 1 as id, 'Drew' as first_name, 'Banin' as last_name, 'yellow' as favorite_color, true as loves_dbt, 5 as tenure, current_timestamp as created_at -union all -select 1 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, true as loves_dbt, 4 as tenure, current_timestamp as created_at - -""" - -orders_sql = """ -select 1 as id, 101 as user_id, 'pending' as status - -""" - -orders_downstream_sql = """ -select * from {{ ref('orders') }} - -""" - -model_a_sql = """ -select 1 as fun - -""" - -model_three_disabled_sql = """ -{{ config(materialized='table', enabled=False) }} - -with source_data as ( - - select 1 as id - union all - select null as id - -) - -select * -from source_data - -""" - -models_schema2b_yml = """ - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - columns: - - name: id - data_tests: - - not_null - -""" - -env_var_macros_yml = """ -macros: - - name: do_something - description: "This is a test macro" - meta: - some_key: "{{ env_var('ENV_VAR_SOME_KEY') }}" - - -""" - -models_schema4_yml = """ - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - config: - enabled: false - columns: - - name: id - data_tests: - - unique - -""" - -model_two_sql = """ -select 1 as notfun - -""" - -generic_test_schema_yml = """ - -models: - - name: orders - description: "Some order data" - columns: - - name: id - data_tests: - - unique - - is_odd - -""" - -customers1_md = """ -{% docs customer_table %} - -This table contains customer data - -{% enddocs %} - -""" - -model_three_modified_sql = """ -{{ config(materialized='table') }} - -with source_data as ( - - {#- This is model three #} - - select 1 as id - union all - select null as id - -) - -select * -from source_data - -""" - -macros_yml = """ -macros: - - name: do_something - description: "This is a test macro" - -""" - -test_color_sql = """ -{% test check_color(model, column_name, color) %} - - select * - from {{ model }} - where {{ column_name }} = '{{ color }}' - -{% endtest %} - -""" - -models_schema2_yml = """ - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - columns: - - name: id - data_tests: - - unique - -""" - -gsm_override2_sql = """ -- custom macro xxxx -{% macro generate_schema_name(schema_name, node) %} - - {{ schema_name }}_{{ target.schema }} - -{% endmacro %} - -""" - -models_schema3_yml = """ - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - data_tests: - - unique -macros: - - name: do_something - description: "This is a test macro" - -""" - -generic_test_sql = """ -{% test is_odd(model, column_name) %} - -with validation as ( - - select - {{ column_name }} as odd_field - - from {{ model }} - -), - -validation_errors as ( - - select - odd_field - - from validation - -- if this is true, then odd_field is actually even! - where (odd_field % 2) = 0 - -) - -select * -from validation_errors - -{% endtest %} -""" - -env_var_model_test_yml = """ -models: - - name: model_color - columns: - - name: fun - data_tests: - - unique: - enabled: "{{ env_var('ENV_VAR_ENABLED', True) }}" - -""" - -model_three_sql = """ -{{ config(materialized='table') }} - -with source_data as ( - - select 1 as id - union all - select null as id - -) - -select * -from source_data - -""" - -ref_override2_sql = """ -- Macro to override ref xxxx -{% macro ref(modelname) %} -{% do return(builtins.ref(modelname)) %} -{% endmacro %} - -""" - -models_schema1_yml = """ - -models: - - name: model_one - description: "The first model" - -""" - -macros_schema_yml = """ - - -models: - - name: model_a - data_tests: - - type_one - - type_two - -""" - -models_versions_schema_yml = """ - -models: - - name: model_one - description: "The first model" - versions: - - v: 1 - - v: 2 -""" - -models_versions_defined_in_schema_yml = """ - -models: - - name: model_one - description: "The first model" - versions: - - v: 1 - - v: 2 - defined_in: model_one_different -""" - -models_versions_updated_schema_yml = """ - -models: - - name: model_one - latest_version: 1 - description: "The first model" - versions: - - v: 1 - - v: 2 - defined_in: model_one_different -""" - -my_macro_sql = """ -{% macro do_something(foo2, bar2) %} - - select - '{{ foo2 }}' as foo2, - '{{ bar2 }}' as bar2 - -{% endmacro %} - -""" - -snapshot_sql = """ -{% snapshot orders_snapshot %} - -{{ - config( - target_schema=schema, - strategy='check', - unique_key='id', - check_cols=['status'], - ) -}} - -select * from {{ ref('orders') }} - -{% endsnapshot %} - -{% snapshot orders2_snapshot %} - -{{ - config( - target_schema=schema, - strategy='check', - unique_key='id', - check_cols=['order_date'], - ) -}} - -select * from {{ ref('orders') }} - -{% endsnapshot %} - -""" - -models_schema4b_yml = """ - -models: - - name: model_one - description: "The first model" - - name: model_three - description: "The third model" - config: - enabled: true - columns: - - name: id - data_tests: - - unique - -""" - -test_macro_sql = """ -{% macro macro_something() %} - - {% do return('macro_something') %} - -{% endmacro %} - -""" - -people_metrics2_yml = """ - -metrics: - - - name: number_of_people - description: Total count of people - label: "Number of people" - type: simple - type_params: - measure: people - meta: - my_meta: 'replaced' - - - name: collective_tenure - description: Total number of years of team experience - label: "Collective tenure" - type: simple - type_params: - measure: - name: years_tenure - filter: "{{ Dimension('id__loves_dbt') }} is true" - -""" - -generic_schema_yml = """ - -models: - - name: orders - description: "Some order data" - columns: - - name: id - data_tests: - - unique - -""" - - -groups_schema_yml_one_group = """ - -groups: - - name: test_group - owner: - name: test_group_owner - -models: - - name: orders - description: "Some order data" -""" - - -groups_schema_yml_two_groups = """ - -groups: - - name: test_group - owner: - name: test_group_owner - - name: test_group2 - owner: - name: test_group_owner2 - -models: - - name: orders - description: "Some order data" -""" - - -groups_schema_yml_two_groups_private_orders_valid_access = """ - -groups: - - name: test_group - owner: - name: test_group_owner - - name: test_group2 - owner: - name: test_group_owner2 - -models: - - name: orders - group: test_group - access: private - description: "Some order data" - - name: orders_downstream - group: test_group - description: "Some order data" -""" - -groups_schema_yml_two_groups_private_orders_invalid_access = """ - -groups: - - name: test_group - owner: - name: test_group_owner - - name: test_group2 - owner: - name: test_group_owner2 - -models: - - name: orders - group: test_group2 - access: private - description: "Some order data" - - name: orders_downstream - group: test_group - description: "Some order data" -""" - -groups_schema_yml_one_group_model_in_group2 = """ - -groups: - - name: test_group - owner: - name: test_group_owner - -models: - - name: orders - description: "Some order data" - config: - group: test_group2 -""" - -groups_schema_yml_two_groups_edited = """ - -groups: - - name: test_group - owner: - name: test_group_owner - - name: test_group2_edited - owner: - name: test_group_owner2 - -models: - - name: orders - description: "Some order data" -""" - - -snapshot2_sql = """ -- add a comment -{% snapshot orders_snapshot %} - -{{ - config( - target_schema=schema, - strategy='check', - unique_key='id', - check_cols=['status'], - ) -}} - -select * from {{ ref('orders') }} - -{% endsnapshot %} - -{% snapshot orders2_snapshot %} - -{{ - config( - target_schema=schema, - strategy='check', - unique_key='id', - check_cols=['order_date'], - ) -}} - -select * from {{ ref('orders') }} - -{% endsnapshot %} - -""" - -sources_tests2_sql = """ - -{% test every_value_is_blue(model, column_name) %} - - select * - from {{ model }} - where {{ column_name }} != 99 - -{% endtest %} - - -""" - -people_metrics3_yml = """ - -metrics: - - - name: number_of_people - description: Total count of people - label: "Number of people" - type: simple - type_params: - measure: people - meta: - my_meta: 'replaced' - -""" - -ref_override_sql = """ -- Macro to override ref -{% macro ref(modelname) %} -{% do return(builtins.ref(modelname)) %} -{% endmacro %} - -""" - -test_macro2_sql = """ -{% macro macro_something() %} - - {% do return('some_name') %} - -{% endmacro %} - -""" - -env_var_macro_sql = """ -{% macro do_something(foo2, bar2) %} - - select - '{{ foo2 }}' as foo2, - '{{ bar2 }}' as bar2 - -{% endmacro %} - -""" - -sources_tests1_sql = """ - -{% test every_value_is_blue(model, column_name) %} - - select * - from {{ model }} - where {{ column_name }} = 9999 - -{% endtest %} - - -""" diff --git a/tests/functional/partial_parsing/test_file_diff.py b/tests/functional/partial_parsing/test_file_diff.py deleted file mode 100644 index c7e34780..00000000 --- a/tests/functional/partial_parsing/test_file_diff.py +++ /dev/null @@ -1,63 +0,0 @@ -import os - -from dbt.tests.util import run_dbt, write_artifact, write_file -import pytest - -from tests.functional.partial_parsing.fixtures import model_one_sql, model_two_sql - - -first_file_diff = { - "deleted": [], - "changed": [], - "added": [{"path": "models/model_one.sql", "content": "select 1 as fun"}], -} - - -second_file_diff = { - "deleted": [], - "changed": [], - "added": [{"path": "models/model_two.sql", "content": "select 123 as notfun"}], -} - - -class TestFileDiffPaths: - def test_file_diffs(self, project): - os.environ["DBT_PP_FILE_DIFF_TEST"] = "true" - - run_dbt(["deps"]) - run_dbt(["seed"]) - - # We start with an empty project - results = run_dbt() - - write_artifact(first_file_diff, "file_diff.json") - results = run_dbt() - assert len(results) == 1 - - write_artifact(second_file_diff, "file_diff.json") - results = run_dbt() - assert len(results) == 2 - - -class TestFileDiffs: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - def test_no_file_diffs(self, project): - # We start with a project with one model - manifest = run_dbt(["parse"]) - assert len(manifest.nodes) == 1 - - # add a model file - write_file(model_two_sql, project.project_root, "models", "model_two.sql") - - # parse without computing a file diff - manifest = run_dbt(["--partial-parse", "--no-partial-parse-file-diff", "parse"]) - assert len(manifest.nodes) == 1 - - # default behaviour - parse with computing a file diff - manifest = run_dbt(["--partial-parse", "parse"]) - assert len(manifest.nodes) == 2 diff --git a/tests/functional/partial_parsing/test_partial_parsing.py b/tests/functional/partial_parsing/test_partial_parsing.py deleted file mode 100644 index eb09dd32..00000000 --- a/tests/functional/partial_parsing/test_partial_parsing.py +++ /dev/null @@ -1,824 +0,0 @@ -import os -import re -from unittest import mock - -from dbt.tests.fixtures.project import write_project_files -from dbt.tests.util import ( - get_manifest, - rename_dir, - rm_file, - write_file, -) -from dbt.contracts.files import ParseFileType -from dbt.contracts.results import TestStatus -from dbt.plugins.manifest import ModelNodeArgs, PluginNodes -from dbt_common.exceptions import CompilationError -import pytest - -from tests.functional.partial_parsing.fixtures import ( - custom_schema_tests1_sql, - custom_schema_tests2_sql, - customers_sql, - customers1_md, - customers2_md, - empty_schema_with_version_yml, - empty_schema_yml, - generic_schema_yml, - generic_test_edited_sql, - generic_test_schema_yml, - generic_test_sql, - gsm_override_sql, - gsm_override2_sql, - local_dependency__dbt_project_yml, - local_dependency__macros__dep_macro_sql, - local_dependency__models__model_to_import_sql, - local_dependency__models__schema_yml, - local_dependency__seeds__seed_csv, - macros_schema_yml, - macros_yml, - model_a_sql, - model_b_sql, - model_four1_sql, - model_four2_sql, - model_one_sql, - model_three_disabled_sql, - model_three_disabled2_sql, - model_three_modified_sql, - model_three_sql, - model_two_sql, - models_schema1_yml, - models_schema2_yml, - models_schema2b_yml, - models_schema3_yml, - models_schema4_yml, - models_schema4b_yml, - my_analysis_sql, - my_macro_sql, - my_macro2_sql, - my_test_sql, - orders_sql, - raw_customers_csv, - ref_override_sql, - ref_override2_sql, - schema_models_c_yml, - schema_sources1_yml, - schema_sources2_yml, - schema_sources3_yml, - schema_sources4_yml, - schema_sources5_yml, - snapshot_sql, - snapshot2_sql, - sources_tests1_sql, - sources_tests2_sql, - test_macro_sql, - test_macro2_sql, -) -from tests.functional.utils import ( - run_dbt, - run_dbt_and_capture, - up_one, -) - - -os.environ["DBT_PP_TEST"] = "true" - - -def normalize(path): - return os.path.normcase(os.path.normpath(path)) - - -class TestModels: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - def test_pp_models(self, project): - # initial run - # run_dbt(['clean']) - results = run_dbt(["run"]) - assert len(results) == 1 - - # add a model file - write_file(model_two_sql, project.project_root, "models", "model_two.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - - # add a schema file - write_file(models_schema1_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert "model.test.model_one" in manifest.nodes - model_one_node = manifest.nodes["model.test.model_one"] - assert model_one_node.description == "The first model" - assert model_one_node.patch_path == "test://" + normalize("models/schema.yml") - - # add a model and a schema file (with a test) at the same time - write_file(models_schema2_yml, project.project_root, "models", "schema.yml") - write_file(model_three_sql, project.project_root, "models", "model_three.sql") - results = run_dbt(["--partial-parse", "test"], expect_pass=False) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - project_files = [f for f in manifest.files if f.startswith("test://")] - assert len(project_files) == 4 - model_3_file_id = "test://" + normalize("models/model_three.sql") - assert model_3_file_id in manifest.files - model_three_file = manifest.files[model_3_file_id] - assert model_three_file.parse_file_type == ParseFileType.Model - assert type(model_three_file).__name__ == "SourceFile" - model_three_node = manifest.nodes[model_three_file.nodes[0]] - schema_file_id = "test://" + normalize("models/schema.yml") - assert model_three_node.patch_path == schema_file_id - assert model_three_node.description == "The third model" - schema_file = manifest.files[schema_file_id] - assert type(schema_file).__name__ == "SchemaSourceFile" - assert len(schema_file.data_tests) == 1 - tests = schema_file.get_all_test_ids() - assert tests == ["test.test.unique_model_three_id.6776ac8160"] - unique_test_id = tests[0] - assert unique_test_id in manifest.nodes - - # modify model sql file, ensure description still there - write_file(model_three_modified_sql, project.project_root, "models", "model_three.sql") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - model_id = "model.test.model_three" - assert model_id in manifest.nodes - model_three_node = manifest.nodes[model_id] - assert model_three_node.description == "The third model" - - # Change the model 3 test from unique to not_null - write_file(models_schema2b_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "test"], expect_pass=False) - manifest = get_manifest(project.project_root) - schema_file_id = "test://" + normalize("models/schema.yml") - schema_file = manifest.files[schema_file_id] - tests = schema_file.get_all_test_ids() - assert tests == ["test.test.not_null_model_three_id.3162ce0a6f"] - not_null_test_id = tests[0] - assert not_null_test_id in manifest.nodes.keys() - assert unique_test_id not in manifest.nodes.keys() - assert len(results) == 1 - - # go back to previous version of schema file, removing patch, test, and model for model three - write_file(models_schema1_yml, project.project_root, "models", "schema.yml") - rm_file(project.project_root, "models", "model_three.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - - # remove schema file, still have 3 models - write_file(model_three_sql, project.project_root, "models", "model_three.sql") - rm_file(project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - manifest = get_manifest(project.project_root) - schema_file_id = "test://" + normalize("models/schema.yml") - assert schema_file_id not in manifest.files - project_files = [f for f in manifest.files if f.startswith("test://")] - assert len(project_files) == 3 - - # Put schema file back and remove a model - # referred to in schema file - write_file(models_schema2_yml, project.project_root, "models", "schema.yml") - rm_file(project.project_root, "models", "model_three.sql") - with pytest.raises(CompilationError): - results = run_dbt(["--partial-parse", "--warn-error", "run"]) - - # Put model back again - write_file(model_three_sql, project.project_root, "models", "model_three.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Add model four refing model three - write_file(model_four1_sql, project.project_root, "models", "model_four.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 4 - - # Remove model_three and change model_four to ref model_one - # and change schema file to remove model_three - rm_file(project.project_root, "models", "model_three.sql") - write_file(model_four2_sql, project.project_root, "models", "model_four.sql") - write_file(models_schema1_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Remove model four, put back model three, put back schema file - write_file(model_three_sql, project.project_root, "models", "model_three.sql") - write_file(models_schema2_yml, project.project_root, "models", "schema.yml") - rm_file(project.project_root, "models", "model_four.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # disable model three in the schema file - write_file(models_schema4_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - - # update enabled config to be true for model three in the schema file - write_file(models_schema4b_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # disable model three in the schema file again - write_file(models_schema4_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - - # remove disabled config for model three in the schema file to check it gets enabled - write_file(models_schema4b_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Add a macro - write_file(my_macro_sql, project.project_root, "macros", "my_macro.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - manifest = get_manifest(project.project_root) - macro_id = "macro.test.do_something" - assert macro_id in manifest.macros - - # Modify the macro - write_file(my_macro2_sql, project.project_root, "macros", "my_macro.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Add a macro patch - write_file(models_schema3_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Remove the macro - rm_file(project.project_root, "macros", "my_macro.sql") - with pytest.raises(CompilationError): - results = run_dbt(["--partial-parse", "--warn-error", "run"]) - - # put back macro file, got back to schema file with no macro - # add separate macro patch schema file - write_file(models_schema2_yml, project.project_root, "models", "schema.yml") - write_file(my_macro_sql, project.project_root, "macros", "my_macro.sql") - write_file(macros_yml, project.project_root, "macros", "macros.yml") - results = run_dbt(["--partial-parse", "run"]) - - # delete macro and schema file - rm_file(project.project_root, "macros", "my_macro.sql") - rm_file(project.project_root, "macros", "macros.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Add an empty schema file - write_file(empty_schema_yml, project.project_root, "models", "eschema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Add version to empty schema file - write_file(empty_schema_with_version_yml, project.project_root, "models", "eschema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # Disable model_three - write_file(model_three_disabled_sql, project.project_root, "models", "model_three.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - model_id = "model.test.model_three" - assert model_id in manifest.disabled - assert model_id not in manifest.nodes - - # Edit disabled model three - write_file(model_three_disabled2_sql, project.project_root, "models", "model_three.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - model_id = "model.test.model_three" - assert model_id in manifest.disabled - assert model_id not in manifest.nodes - - # Remove disabled from model three - write_file(model_three_sql, project.project_root, "models", "model_three.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - manifest = get_manifest(project.project_root) - model_id = "model.test.model_three" - assert model_id in manifest.nodes - assert model_id not in manifest.disabled - - -class TestSources: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - def test_pp_sources(self, project): - # initial run - write_file(raw_customers_csv, project.project_root, "seeds", "raw_customers.csv") - write_file(sources_tests1_sql, project.project_root, "macros", "tests.sql") - results = run_dbt(["run"]) - assert len(results) == 1 - - # Partial parse running 'seed' - run_dbt(["--partial-parse", "seed"]) - manifest = get_manifest(project.project_root) - seed_file_id = "test://" + normalize("seeds/raw_customers.csv") - assert seed_file_id in manifest.files - - # Add another seed file - write_file(raw_customers_csv, project.project_root, "seeds", "more_customers.csv") - run_dbt(["--partial-parse", "run"]) - seed_file_id = "test://" + normalize("seeds/more_customers.csv") - manifest = get_manifest(project.project_root) - assert seed_file_id in manifest.files - seed_id = "seed.test.more_customers" - assert seed_id in manifest.nodes - - # Remove seed file and add a schema files with a source referring to raw_customers - rm_file(project.project_root, "seeds", "more_customers.csv") - write_file(schema_sources1_yml, project.project_root, "models", "sources.yml") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - assert len(manifest.sources) == 1 - file_id = "test://" + normalize("models/sources.yml") - assert file_id in manifest.files - - # add a model referring to raw_customers source - write_file(customers_sql, project.project_root, "models", "customers.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - - # remove sources schema file - rm_file(project.project_root, "models", "sources.yml") - with pytest.raises(CompilationError): - results = run_dbt(["--partial-parse", "run"]) - - # put back sources and add an exposures file - write_file(schema_sources2_yml, project.project_root, "models", "sources.yml") - results = run_dbt(["--partial-parse", "run"]) - - # remove seed referenced in exposures file - rm_file(project.project_root, "seeds", "raw_customers.csv") - with pytest.raises(CompilationError): - results = run_dbt(["--partial-parse", "run"]) - - # put back seed and remove depends_on from exposure - write_file(raw_customers_csv, project.project_root, "seeds", "raw_customers.csv") - write_file(schema_sources3_yml, project.project_root, "models", "sources.yml") - results = run_dbt(["--partial-parse", "run"]) - - # Add seed config with test to schema.yml, remove exposure - write_file(schema_sources4_yml, project.project_root, "models", "sources.yml") - results = run_dbt(["--partial-parse", "run"]) - - # Change seed name to wrong name - write_file(schema_sources5_yml, project.project_root, "models", "sources.yml") - with pytest.raises(CompilationError): - results = run_dbt(["--partial-parse", "--warn-error", "run"]) - - # Put back seed name to right name - write_file(schema_sources4_yml, project.project_root, "models", "sources.yml") - results = run_dbt(["--partial-parse", "run"]) - - # Add docs file customers.md - write_file(customers1_md, project.project_root, "models", "customers.md") - results = run_dbt(["--partial-parse", "run"]) - - # Change docs file customers.md - write_file(customers2_md, project.project_root, "models", "customers.md") - results = run_dbt(["--partial-parse", "run"]) - - # Delete docs file - rm_file(project.project_root, "models", "customers.md") - results = run_dbt(["--partial-parse", "run"]) - - # Add a data test - write_file(test_macro_sql, project.project_root, "macros", "test-macro.sql") - write_file(my_test_sql, project.project_root, "tests", "my_test.sql") - results = run_dbt(["--partial-parse", "test"]) - manifest = get_manifest(project.project_root) - assert len(manifest.nodes) == 9 - test_id = "test.test.my_test" - assert test_id in manifest.nodes - - # Change macro that data test depends on - write_file(test_macro2_sql, project.project_root, "macros", "test-macro.sql") - results = run_dbt(["--partial-parse", "test"]) - manifest = get_manifest(project.project_root) - - # Add an analysis - write_file(my_analysis_sql, project.project_root, "analyses", "my_analysis.sql") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - - # Remove data test - rm_file(project.project_root, "tests", "my_test.sql") - results = run_dbt(["--partial-parse", "test"]) - manifest = get_manifest(project.project_root) - assert len(manifest.nodes) == 9 - - # Remove analysis - rm_file(project.project_root, "analyses", "my_analysis.sql") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - assert len(manifest.nodes) == 8 - - # Change source test - write_file(sources_tests2_sql, project.project_root, "macros", "tests.sql") - results = run_dbt(["--partial-parse", "run"]) - - -class TestPartialParsingDependency: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root): - local_dependency_files = { - "dbt_project.yml": local_dependency__dbt_project_yml, - "models": { - "schema.yml": local_dependency__models__schema_yml, - "model_to_import.sql": local_dependency__models__model_to_import_sql, - }, - "macros": {"dep_macro.sql": local_dependency__macros__dep_macro_sql}, - "seeds": {"seed.csv": local_dependency__seeds__seed_csv}, - } - write_project_files(project_root, "local_dependency", local_dependency_files) - - @pytest.fixture(scope="class") - def packages(self): - return {"packages": [{"local": "local_dependency"}]} - - def test_parsing_with_dependency(self, project): - run_dbt(["clean"]) - run_dbt(["deps"]) - run_dbt(["seed"]) - run_dbt(["run"]) - - # Add a source override - write_file(schema_models_c_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert len(manifest.sources) == 1 - source_id = "source.local_dep.seed_source.seed" - assert source_id in manifest.sources - # We have 1 root model, 1 local_dep model, 1 local_dep seed, 1 local_dep source test, 2 root source tests - assert len(manifest.nodes) == 5 - test_id = "test.local_dep.source_unique_seed_source_seed_id.afa94935ed" - assert test_id in manifest.nodes - - # Remove a source override - rm_file(project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - assert len(manifest.sources) == 1 - - -class TestNestedMacros: - @pytest.fixture(scope="class") - def models(self): - return { - "model_a.sql": model_a_sql, - "model_b.sql": model_b_sql, - "schema.yml": macros_schema_yml, - } - - @pytest.fixture(scope="class") - def macros(self): - return { - "custom_schema_tests.sql": custom_schema_tests1_sql, - } - - def test_nested_macros(self, project): - results = run_dbt() - assert len(results) == 2 - manifest = get_manifest(project.project_root) - macro_child_map = manifest.build_macro_child_map() - macro_unique_id = "macro.test.test_type_two" - assert macro_unique_id in macro_child_map - - results = run_dbt(["test"], expect_pass=False) - results = sorted(results, key=lambda r: r.node.name) - assert len(results) == 2 - # type_one_model_a_ - assert results[0].status == TestStatus.Fail - assert re.search(r"union all", results[0].node.compiled_code) - # type_two_model_a_ - assert results[1].status == TestStatus.Warn - assert results[1].node.config.severity == "WARN" - - write_file( - custom_schema_tests2_sql, project.project_root, "macros", "custom_schema_tests.sql" - ) - results = run_dbt(["--partial-parse", "test"], expect_pass=False) - manifest = get_manifest(project.project_root) - test_node_id = "test.test.type_two_model_a_.842bc6c2a7" - assert test_node_id in manifest.nodes - results = sorted(results, key=lambda r: r.node.name) - assert len(results) == 2 - # type_two_model_a_ - assert results[1].status == TestStatus.Fail - assert results[1].node.config.severity == "ERROR" - - -class TestSkipMacros: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - "eschema.yml": empty_schema_yml, - } - - def test_skip_macros(self, project): - # initial run so we have a msgpack file - # includes empty_schema file for bug #4850 - results = run_dbt() - - # add a new ref override macro - write_file(ref_override_sql, project.project_root, "macros", "ref_override.sql") - results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) - assert "Starting full parse." in log_output - - # modify a ref override macro - write_file(ref_override2_sql, project.project_root, "macros", "ref_override.sql") - results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) - assert "Starting full parse." in log_output - - # remove a ref override macro - rm_file(project.project_root, "macros", "ref_override.sql") - results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) - assert "Starting full parse." in log_output - - # custom generate_schema_name macro - write_file(gsm_override_sql, project.project_root, "macros", "gsm_override.sql") - results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) - assert "Starting full parse." in log_output - - # change generate_schema_name macro - write_file(gsm_override2_sql, project.project_root, "macros", "gsm_override.sql") - results, log_output = run_dbt_and_capture(["--partial-parse", "run"]) - assert "Starting full parse." in log_output - - -class TestSnapshots: - @pytest.fixture(scope="class") - def models(self): - return { - "orders.sql": orders_sql, - } - - def test_pp_snapshots(self, project): - # initial run - results = run_dbt() - assert len(results) == 1 - - # add snapshot - write_file(snapshot_sql, project.project_root, "snapshots", "snapshot.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - snapshot_id = "snapshot.test.orders_snapshot" - assert snapshot_id in manifest.nodes - snapshot2_id = "snapshot.test.orders2_snapshot" - assert snapshot2_id in manifest.nodes - - # run snapshot - results = run_dbt(["--partial-parse", "snapshot"]) - assert len(results) == 2 - - # modify snapshot - write_file(snapshot2_sql, project.project_root, "snapshots", "snapshot.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 - - # delete snapshot - rm_file(project.project_root, "snapshots", "snapshot.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 - - -class TestTests: - @pytest.fixture(scope="class") - def models(self): - return { - "orders.sql": orders_sql, - "schema.yml": generic_schema_yml, - } - - @pytest.fixture(scope="class") - def tests(self): - # Make sure "generic" directory is created - return {"generic": {"readme.md": ""}} - - def test_pp_generic_tests(self, project): - # initial run - results = run_dbt() - assert len(results) == 1 - manifest = get_manifest(project.project_root) - expected_nodes = ["model.test.orders", "test.test.unique_orders_id.1360ecc70e"] - assert expected_nodes == list(manifest.nodes.keys()) - - # add generic test in test-path - write_file(generic_test_sql, project.project_root, "tests", "generic", "generic_test.sql") - write_file(generic_test_schema_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - test_id = "test.test.is_odd_orders_id.82834fdc5b" - assert test_id in manifest.nodes - expected_nodes = [ - "model.test.orders", - "test.test.unique_orders_id.1360ecc70e", - "test.test.is_odd_orders_id.82834fdc5b", - ] - assert expected_nodes == list(manifest.nodes.keys()) - - # edit generic test in test-path - write_file( - generic_test_edited_sql, project.project_root, "tests", "generic", "generic_test.sql" - ) - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - test_id = "test.test.is_odd_orders_id.82834fdc5b" - assert test_id in manifest.nodes - expected_nodes = [ - "model.test.orders", - "test.test.unique_orders_id.1360ecc70e", - "test.test.is_odd_orders_id.82834fdc5b", - ] - assert expected_nodes == list(manifest.nodes.keys()) - - -class TestExternalModels: - @pytest.fixture(scope="class") - def external_model_node(self): - return ModelNodeArgs( - name="external_model", - package_name="external", - identifier="test_identifier", - schema="test_schema", - ) - - @pytest.fixture(scope="class") - def external_model_node_versioned(self): - return ModelNodeArgs( - name="external_model_versioned", - package_name="external", - identifier="test_identifier_v1", - schema="test_schema", - version=1, - ) - - @pytest.fixture(scope="class") - def external_model_node_depends_on(self): - return ModelNodeArgs( - name="external_model_depends_on", - package_name="external", - identifier="test_identifier_depends_on", - schema="test_schema", - depends_on_nodes=["model.external.external_model_depends_on_parent"], - ) - - @pytest.fixture(scope="class") - def external_model_node_depends_on_parent(self): - return ModelNodeArgs( - name="external_model_depends_on_parent", - package_name="external", - identifier="test_identifier_depends_on_parent", - schema="test_schema", - ) - - @pytest.fixture(scope="class") - def models(self): - return {"model_one.sql": model_one_sql} - - @mock.patch("dbt.plugins.get_plugin_manager") - def test_pp_external_models( - self, - get_plugin_manager, - project, - external_model_node, - external_model_node_versioned, - external_model_node_depends_on, - external_model_node_depends_on_parent, - ): - # initial plugin - one external model - external_nodes = PluginNodes() - external_nodes.add_model(external_model_node) - get_plugin_manager.return_value.get_nodes.return_value = external_nodes - - # initial parse - manifest = run_dbt(["parse"]) - assert len(manifest.nodes) == 2 - assert set(manifest.nodes.keys()) == { - "model.external.external_model", - "model.test.model_one", - } - assert len(manifest.external_node_unique_ids) == 1 - assert manifest.external_node_unique_ids == ["model.external.external_model"] - - # add a model file - write_file(model_two_sql, project.project_root, "models", "model_two.sql") - manifest = run_dbt(["--partial-parse", "parse"]) - assert len(manifest.nodes) == 3 - - # add an external model - external_nodes.add_model(external_model_node_versioned) - manifest = run_dbt(["--partial-parse", "parse"]) - assert len(manifest.nodes) == 4 - assert len(manifest.external_node_unique_ids) == 2 - - # add a model file that depends on external model - write_file( - "SELECT * FROM {{ref('external', 'external_model')}}", - project.project_root, - "models", - "model_depends_on_external.sql", - ) - manifest = run_dbt(["--partial-parse", "parse"]) - assert len(manifest.nodes) == 5 - assert len(manifest.external_node_unique_ids) == 2 - - # remove a model file that depends on external model - rm_file(project.project_root, "models", "model_depends_on_external.sql") - manifest = run_dbt(["--partial-parse", "parse"]) - assert len(manifest.nodes) == 4 - - # add an external node with depends on - external_nodes.add_model(external_model_node_depends_on) - external_nodes.add_model(external_model_node_depends_on_parent) - manifest = run_dbt(["--partial-parse", "parse"]) - assert len(manifest.nodes) == 6 - assert len(manifest.external_node_unique_ids) == 4 - - # skip files parsing - ensure no issues - run_dbt(["--partial-parse", "parse"]) - assert len(manifest.nodes) == 6 - assert len(manifest.external_node_unique_ids) == 4 - - -class TestPortablePartialParsing: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - @pytest.fixture(scope="class") - def packages(self): - return {"packages": [{"local": "local_dependency"}]} - - @pytest.fixture(scope="class") - def local_dependency_files(self): - return { - "dbt_project.yml": local_dependency__dbt_project_yml, - "models": { - "schema.yml": local_dependency__models__schema_yml, - "model_to_import.sql": local_dependency__models__model_to_import_sql, - }, - "macros": {"dep_macro.sql": local_dependency__macros__dep_macro_sql}, - "seeds": {"seed.csv": local_dependency__seeds__seed_csv}, - } - - def rename_project_root(self, project, new_project_root): - with up_one(new_project_root): - rename_dir(project.project_root, new_project_root) - project.project_root = new_project_root - # flags.project_dir is set during the project test fixture, and is persisted across run_dbt calls, - # so it needs to be reset between invocations - # flags.set_from_args(Namespace(PROJECT_DIR=new_project_root), None) - - @pytest.fixture(scope="class", autouse=True) - def initial_run_and_rename_project_dir(self, project, local_dependency_files): - initial_project_root = project.project_root - renamed_project_root = os.path.join(project.project_root.dirname, "renamed_project_dir") - - write_project_files(project.project_root, "local_dependency", local_dependency_files) - - # initial run - run_dbt(["deps"]) - assert len(run_dbt(["seed"])) == 1 - assert len(run_dbt(["run"])) == 2 - - self.rename_project_root(project, renamed_project_root) - yield - self.rename_project_root(project, initial_project_root) - - def test_pp_renamed_project_dir_unchanged_project_contents(self, project): - # partial parse same project in new absolute dir location, using partial_parse.msgpack created in previous dir - run_dbt(["deps"]) - assert len(run_dbt(["--partial-parse", "seed"])) == 1 - assert len(run_dbt(["--partial-parse", "run"])) == 2 - - def test_pp_renamed_project_dir_changed_project_contents(self, project): - write_file(model_two_sql, project.project_root, "models", "model_two.sql") - - # partial parse changed project in new absolute dir location, using partial_parse.msgpack created in previous dir - run_dbt(["deps"]) - len(run_dbt(["--partial-parse", "seed"])) == 1 - len(run_dbt(["--partial-parse", "run"])) == 3 diff --git a/tests/functional/partial_parsing/test_pp_disabled_config.py b/tests/functional/partial_parsing/test_pp_disabled_config.py deleted file mode 100644 index 8a4ece9d..00000000 --- a/tests/functional/partial_parsing/test_pp_disabled_config.py +++ /dev/null @@ -1,224 +0,0 @@ -from dbt.tests.util import get_manifest, run_dbt, write_file -import pytest - - -model_one_sql = """ -select 1 as fun -""" - -metricflow_time_spine_sql = """ -SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day -""" - -schema1_yml = """ -version: 2 - -models: - - name: model_one - -semantic_models: - - name: semantic_people - model: ref('model_one') - dimensions: - - name: created_at - type: TIME - type_params: - time_granularity: day - measures: - - name: people - agg: count - expr: fun - entities: - - name: fun - type: primary - defaults: - agg_time_dimension: created_at - -metrics: - - - name: number_of_people - label: "Number of people" - description: Total count of people - type: simple - type_params: - measure: people - meta: - my_meta: 'testing' - -exposures: - - name: proxy_for_dashboard - description: "My Exposure" - type: "dashboard" - owner: - name: "Dashboard Tester" - email: "tester@dashboard.com" - depends_on: - - ref("model_one") -""" - -schema2_yml = """ -version: 2 - -models: - - name: model_one - -semantic_models: - - name: semantic_people - model: ref('model_one') - dimensions: - - name: created_at - type: TIME - type_params: - time_granularity: day - measures: - - name: people - agg: count - expr: fun - entities: - - name: fun - type: primary - defaults: - agg_time_dimension: created_at - -metrics: - - - name: number_of_people - label: "Number of people" - description: Total count of people - config: - enabled: false - type: simple - type_params: - measure: people - meta: - my_meta: 'testing' - -exposures: - - name: proxy_for_dashboard - description: "My Exposure" - config: - enabled: false - type: "dashboard" - owner: - name: "Dashboard Tester" - email: "tester@dashboard.com" - depends_on: - - ref("model_one") -""" - -schema3_yml = """ -version: 2 - -models: - - name: model_one - -semantic_models: - - name: semantic_people - model: ref('model_one') - dimensions: - - name: created_at - type: TIME - type_params: - time_granularity: day - measures: - - name: people - agg: count - expr: fun - entities: - - name: fun - type: primary - defaults: - agg_time_dimension: created_at - -metrics: - - - name: number_of_people - label: "Number of people" - description: Total count of people - type: simple - type_params: - measure: people - meta: - my_meta: 'testing' -""" - -schema4_yml = """ -version: 2 - -models: - - name: model_one - -exposures: - - name: proxy_for_dashboard - description: "My Exposure" - config: - enabled: false - type: "dashboard" - owner: - name: "Dashboard Tester" - email: "tester@dashboard.com" - depends_on: - - ref("model_one") -""" - - -class TestDisabled: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - "metricflow_time_spine.sql": metricflow_time_spine_sql, - "schema.yml": schema1_yml, - } - - def test_pp_disabled(self, project): - expected_exposure = "exposure.test.proxy_for_dashboard" - expected_metric = "metric.test.number_of_people" - - run_dbt(["seed"]) - manifest = run_dbt(["parse"]) - - assert expected_exposure in manifest.exposures - assert expected_metric in manifest.metrics - assert expected_exposure not in manifest.disabled - assert expected_metric not in manifest.disabled - - # Update schema file with disabled metric and exposure - write_file(schema2_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert expected_exposure not in manifest.exposures - assert expected_metric not in manifest.metrics - assert expected_exposure in manifest.disabled - assert expected_metric in manifest.disabled - - # Update schema file with enabled metric and exposure - write_file(schema1_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert expected_exposure in manifest.exposures - assert expected_metric in manifest.metrics - assert expected_exposure not in manifest.disabled - assert expected_metric not in manifest.disabled - - # Update schema file - remove exposure, enable metric - write_file(schema3_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert expected_exposure not in manifest.exposures - assert expected_metric in manifest.metrics - assert expected_exposure not in manifest.disabled - assert expected_metric not in manifest.disabled - - # Update schema file - add back exposure, remove metric - write_file(schema4_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert expected_exposure not in manifest.exposures - assert expected_metric not in manifest.metrics - assert expected_exposure in manifest.disabled - assert expected_metric not in manifest.disabled diff --git a/tests/functional/partial_parsing/test_pp_docs.py b/tests/functional/partial_parsing/test_pp_docs.py deleted file mode 100644 index 5df08d4d..00000000 --- a/tests/functional/partial_parsing/test_pp_docs.py +++ /dev/null @@ -1,257 +0,0 @@ -from dbt.tests.util import get_manifest, rm_file, run_dbt, write_file -import pytest - - -model_one_sql = """ -select 1 as fun -""" - -raw_customers_csv = """id,first_name,last_name,email -1,Michael,Perez,mperez0@chronoengine.com -2,Shawn,Mccoy,smccoy1@reddit.com -3,Kathleen,Payne,kpayne2@cargocollective.com -4,Jimmy,Cooper,jcooper3@cargocollective.com -5,Katherine,Rice,krice4@typepad.com -6,Sarah,Ryan,sryan5@gnu.org -7,Martin,Mcdonald,mmcdonald6@opera.com -8,Frank,Robinson,frobinson7@wunderground.com -9,Jennifer,Franklin,jfranklin8@mail.ru -10,Henry,Welch,hwelch9@list-manage.com -""" - -my_macro_sql = """ -{% macro my_macro(something) %} - - select - '{{ something }}' as something2 - -{% endmacro %} - -""" - -customers1_md = """ -{% docs customer_table %} - -This table contains customer data - -{% enddocs %} -""" - -customers2_md = """ -{% docs customer_table %} - -LOTS of customer data - -{% enddocs %} - -""" - -schema1_yml = """ -version: 2 - -models: - - name: model_one - description: "{{ doc('customer_table') }}" -""" - -schema2_yml = """ -version: 2 - -models: - - name: model_one - description: "{{ doc('customer_table') }}" - -macros: - - name: my_macro - description: "{{ doc('customer_table') }}" - -sources: - - name: seed_sources - description: "{{ doc('customer_table') }}" - schema: "{{ target.schema }}" - tables: - - name: raw_customers - columns: - - name: id - data_tests: - - not_null: - severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" - - unique - - name: first_name - - name: last_name - - name: email - -exposures: - - name: proxy_for_dashboard - description: "{{ doc('customer_table') }}" - type: "dashboard" - owner: - name: "Dashboard Tester" - email: "tester@dashboard.com" - depends_on: - - ref("model_one") - - ref("raw_customers") - - source("seed_sources", "raw_customers") -""" - - -class TestDocs: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - @pytest.fixture(scope="class") - def seeds(self): - return { - "raw_customers.csv": raw_customers_csv, - } - - @pytest.fixture(scope="class") - def macros(self): - return { - "my_macro.sql": my_macro_sql, - } - - def test_pp_docs(self, project): - run_dbt(["seed"]) - results = run_dbt(["run"]) - assert len(results) == 1 - - # Add docs file customers.md - write_file(customers1_md, project.project_root, "models", "customers.md") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - assert len(manifest.docs) == 2 - - # Add schema file with 'docs' description - write_file(schema1_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - assert len(manifest.docs) == 2 - doc_id = "doc.test.customer_table" - assert doc_id in manifest.docs - doc = manifest.docs[doc_id] - doc_file_id = doc.file_id - assert doc_file_id in manifest.files - source_file = manifest.files[doc_file_id] - assert len(source_file.nodes) == 1 - model_one_id = "model.test.model_one" - assert model_one_id in source_file.nodes - model_node = manifest.nodes[model_one_id] - assert model_node.description == "This table contains customer data" - - # Update the doc file - write_file(customers2_md, project.project_root, "models", "customers.md") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - assert len(manifest.docs) == 2 - assert model_one_id in manifest.nodes - model_node = manifest.nodes[model_one_id] - assert "LOTS" in model_node.description - - # Add a macro patch, source and exposure with doc - write_file(schema2_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - doc_file = manifest.files[doc_file_id] - expected_nodes = [ - "model.test.model_one", - "source.test.seed_sources.raw_customers", - "macro.test.my_macro", - "exposure.test.proxy_for_dashboard", - ] - assert expected_nodes == doc_file.nodes - source_id = "source.test.seed_sources.raw_customers" - assert manifest.sources[source_id].source_description == "LOTS of customer data" - macro_id = "macro.test.my_macro" - assert manifest.macros[macro_id].description == "LOTS of customer data" - exposure_id = "exposure.test.proxy_for_dashboard" - assert manifest.exposures[exposure_id].description == "LOTS of customer data" - - # update the doc file again - write_file(customers1_md, project.project_root, "models", "customers.md") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - source_file = manifest.files[doc_file_id] - assert model_one_id in source_file.nodes - model_node = manifest.nodes[model_one_id] - assert model_node.description == "This table contains customer data" - assert ( - manifest.sources[source_id].source_description == "This table contains customer data" - ) - assert manifest.macros[macro_id].description == "This table contains customer data" - assert manifest.exposures[exposure_id].description == "This table contains customer data" - - # check that _lock is working - with manifest._lock: - assert manifest._lock - - -my_model_yml = """ -version: 2 -models: - - name: my_model - columns: - - name: id - description: "{{ doc('whatever') }}" -""" - -my_model_no_description_yml = """ -version: 2 -models: - - name: my_model - columns: - - name: id -""" - -my_model_md = """ -{% docs whatever %} - cool stuff -{% enddocs %} -""" - - -class TestDocsRemoveReplace: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": "select 1 as id", - "my_model.yml": my_model_yml, - "my_model.md": my_model_md, - } - - def test_remove_replace(self, project): - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - doc_id = "doc.test.whatever" - assert doc_id in manifest.docs - doc = manifest.docs[doc_id] - doc_file = manifest.files[doc.file_id] - - model_id = "model.test.my_model" - assert model_id in manifest.nodes - - assert doc_file.nodes == [model_id] - - model = manifest.nodes[model_id] - model_file_id = model.file_id - assert model_file_id in manifest.files - - # remove the doc file - rm_file(project.project_root, "models", "my_model.md") - # remove description from schema file - write_file(my_model_no_description_yml, project.project_root, "models", "my_model.yml") - run_dbt(["parse"]) - manifest = get_manifest(project.project_root) - assert doc_id not in manifest.docs - # The bug was that the file still existed in manifest.files - assert doc.file_id not in manifest.files - - # put back the doc file - write_file(my_model_md, project.project_root, "models", "my_model.md") - # put back the description in the schema file - write_file(my_model_yml, project.project_root, "models", "my_model.yml") - run_dbt(["parse"]) diff --git a/tests/functional/partial_parsing/test_pp_groups.py b/tests/functional/partial_parsing/test_pp_groups.py deleted file mode 100644 index f7577683..00000000 --- a/tests/functional/partial_parsing/test_pp_groups.py +++ /dev/null @@ -1,155 +0,0 @@ -from dbt.exceptions import ParsingError -from dbt.tests.util import get_manifest, run_dbt, write_file -import pytest - -from tests.functional.partial_parsing.fixtures import ( - groups_schema_yml_one_group, - groups_schema_yml_one_group_model_in_group2, - groups_schema_yml_two_groups, - groups_schema_yml_two_groups_edited, - groups_schema_yml_two_groups_private_orders_invalid_access, - groups_schema_yml_two_groups_private_orders_valid_access, - orders_downstream_sql, - orders_sql, -) - - -class TestGroups: - @pytest.fixture(scope="class") - def models(self): - return { - "orders.sql": orders_sql, - "orders_downstream.sql": orders_downstream_sql, - "schema.yml": groups_schema_yml_one_group, - } - - def test_pp_groups(self, project): - # initial run - results = run_dbt() - assert len(results) == 2 - manifest = get_manifest(project.project_root) - expected_nodes = ["model.test.orders", "model.test.orders_downstream"] - expected_groups = ["group.test.test_group"] - assert expected_nodes == sorted(list(manifest.nodes.keys())) - assert expected_groups == sorted(list(manifest.groups.keys())) - - # add group to schema - write_file(groups_schema_yml_two_groups, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - expected_nodes = ["model.test.orders", "model.test.orders_downstream"] - expected_groups = ["group.test.test_group", "group.test.test_group2"] - assert expected_nodes == sorted(list(manifest.nodes.keys())) - assert expected_groups == sorted(list(manifest.groups.keys())) - - # edit group in schema - write_file( - groups_schema_yml_two_groups_edited, project.project_root, "models", "schema.yml" - ) - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - expected_nodes = ["model.test.orders", "model.test.orders_downstream"] - expected_groups = ["group.test.test_group", "group.test.test_group2_edited"] - assert expected_nodes == sorted(list(manifest.nodes.keys())) - assert expected_groups == sorted(list(manifest.groups.keys())) - - # delete group in schema - write_file(groups_schema_yml_one_group, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - expected_nodes = ["model.test.orders", "model.test.orders_downstream"] - expected_groups = ["group.test.test_group"] - assert expected_nodes == sorted(list(manifest.nodes.keys())) - assert expected_groups == sorted(list(manifest.groups.keys())) - - # add back second group - write_file(groups_schema_yml_two_groups, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - - # remove second group with model still configured to second group - write_file( - groups_schema_yml_one_group_model_in_group2, - project.project_root, - "models", - "schema.yml", - ) - with pytest.raises(ParsingError): - results = run_dbt(["--partial-parse", "run"]) - - # add back second group, make orders private with valid ref - write_file( - groups_schema_yml_two_groups_private_orders_valid_access, - project.project_root, - "models", - "schema.yml", - ) - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - - write_file( - groups_schema_yml_two_groups_private_orders_invalid_access, - project.project_root, - "models", - "schema.yml", - ) - with pytest.raises(ParsingError): - results = run_dbt(["--partial-parse", "run"]) - - -my_model_c = """ -select * from {{ ref("my_model_a") }} union all -select * from {{ ref("my_model_b") }} -""" - -models_yml = """ -models: - - name: my_model_a - - name: my_model_b - - name: my_model_c -""" - -models_and_groups_yml = """ -groups: - - name: sales_analytics - owner: - name: Sales Analytics - email: sales@jaffleshop.com - -models: - - name: my_model_a - access: private - group: sales_analytics - - name: my_model_b - access: private - group: sales_analytics - - name: my_model_c - access: private - group: sales_analytics -""" - - -class TestAddingModelsToNewGroups: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_a.sql": "select 1 as id", - "my_model_b.sql": "select 2 as id", - "my_model_c.sql": my_model_c, - "models.yml": models_yml, - } - - def test_adding_models_to_new_groups(self, project): - run_dbt(["compile"]) - # This tests that the correct patch is added to my_model_c. The bug - # was that it was using the old patch, so model_c didn't have the - # correct group and access. - write_file(models_and_groups_yml, project.project_root, "models", "models.yml") - run_dbt(["compile"]) - manifest = get_manifest(project.project_root) - model_c_node = manifest.nodes["model.test.my_model_c"] - assert model_c_node.group == "sales_analytics" - assert model_c_node.access == "private" diff --git a/tests/functional/partial_parsing/test_pp_metrics.py b/tests/functional/partial_parsing/test_pp_metrics.py deleted file mode 100644 index bc688451..00000000 --- a/tests/functional/partial_parsing/test_pp_metrics.py +++ /dev/null @@ -1,85 +0,0 @@ -from dbt.tests.util import get_manifest, run_dbt, write_file -from dbt_common.exceptions import CompilationError -import pytest - -from tests.functional.partial_parsing.fixtures import ( - metric_model_a_sql, - metricflow_time_spine_sql, - people_metrics_yml, - people_metrics2_yml, - people_metrics3_yml, - people_semantic_models_yml, - people_sql, -) - - -class TestMetrics: - @pytest.fixture(scope="class") - def models(self): - return { - "people.sql": people_sql, - "metricflow_time_spine.sql": metricflow_time_spine_sql, - } - - def test_metrics(self, project): - # initial run - results = run_dbt(["run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert len(manifest.nodes) == 2 - - # Add metrics yaml file (and necessary semantic models yaml) - write_file( - people_semantic_models_yml, - project.project_root, - "models", - "people_semantic_models.yml", - ) - write_file(people_metrics_yml, project.project_root, "models", "people_metrics.yml") - results = run_dbt(["run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - assert len(manifest.metrics) == 2 - metric_people_id = "metric.test.number_of_people" - metric_people = manifest.metrics[metric_people_id] - expected_meta = {"my_meta": "testing"} - assert metric_people.meta == expected_meta - - # TODO: Bring back when we resolving `depends_on_nodes` - # metric_tenure_id = "metric.test.collective_tenure" - # metric_tenure = manifest.metrics[metric_tenure_id] - # assert metric_people.refs == [RefArgs(name="people")] - # assert metric_tenure.refs == [RefArgs(name="people")] - # expected_depends_on_nodes = ["model.test.people"] - # assert metric_people.depends_on.nodes == expected_depends_on_nodes - - # Change metrics yaml files - write_file(people_metrics2_yml, project.project_root, "models", "people_metrics.yml") - results = run_dbt(["run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - metric_people = manifest.metrics[metric_people_id] - expected_meta = {"my_meta": "replaced"} - assert metric_people.meta == expected_meta - # TODO: Bring back when we resolving `depends_on_nodes` - # expected_depends_on_nodes = ["model.test.people"] - # assert metric_people.depends_on.nodes == expected_depends_on_nodes - - # Add model referring to metric - write_file(metric_model_a_sql, project.project_root, "models", "metric_model_a.sql") - results = run_dbt(["run"]) - manifest = get_manifest(project.project_root) - # TODO: Bring back when we resolving `depends_on_nodes` - # model_a = manifest.nodes["model.test.metric_model_a"] - # expected_depends_on_nodes = [ - # "metric.test.number_of_people", - # "metric.test.collective_tenure", - # ] - # assert model_a.depends_on.nodes == expected_depends_on_nodes - - # Then delete a metric - write_file(people_metrics3_yml, project.project_root, "models", "people_metrics.yml") - with pytest.raises(CompilationError): - # We use "parse" here and not "run" because we're checking that the CompilationError - # occurs at parse time, not compilation - results = run_dbt(["parse"]) diff --git a/tests/functional/partial_parsing/test_pp_vars.py b/tests/functional/partial_parsing/test_pp_vars.py deleted file mode 100644 index c903cdea..00000000 --- a/tests/functional/partial_parsing/test_pp_vars.py +++ /dev/null @@ -1,398 +0,0 @@ -import os -from pathlib import Path - -from dbt.adapters.exceptions import FailedToConnectError -from dbt.constants import SECRET_ENV_PREFIX -from dbt.exceptions import ParsingError -from dbt.tests.util import get_manifest, write_file -import pytest - -from tests.functional.partial_parsing.fixtures import ( - env_var_macro_sql, - env_var_macros_yml, - env_var_metrics_yml, - env_var_model_one_sql, - env_var_model_sql, - env_var_model_test_yml, - env_var_schema_yml, - env_var_schema2_yml, - env_var_schema3_yml, - env_var_sources_yml, - metricflow_time_spine_sql, - model_color_sql, - model_one_sql, - people_semantic_models_yml, - people_sql, - raw_customers_csv, - test_color_sql, -) -from tests.functional.utils import run_dbt, run_dbt_and_capture - - -os.environ["DBT_PP_TEST"] = "true" - - -class TestEnvVars: - @pytest.fixture(scope="class") - def models(self): - return { - "model_color.sql": model_color_sql, - } - - def test_env_vars_models(self, project): - # initial run - results = run_dbt(["run"]) - assert len(results) == 1 - - # copy a file with an env_var call without an env_var - write_file(env_var_model_sql, project.project_root, "models", "env_var_model.sql") - with pytest.raises(ParsingError): - results = run_dbt(["--partial-parse", "run"]) - - # set the env var - os.environ["ENV_VAR_TEST"] = "TestingEnvVars" - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - expected_env_vars = {"ENV_VAR_TEST": "TestingEnvVars"} - assert expected_env_vars == manifest.env_vars - model_id = "model.test.env_var_model" - model = manifest.nodes[model_id] - model_created_at = model.created_at - - # change the env var - os.environ["ENV_VAR_TEST"] = "second" - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 2 - manifest = get_manifest(project.project_root) - expected_env_vars = {"ENV_VAR_TEST": "second"} - assert expected_env_vars == manifest.env_vars - assert model_created_at != manifest.nodes[model_id].created_at - - # set an env_var in a schema file - write_file(env_var_schema_yml, project.project_root, "models", "schema.yml") - write_file(env_var_model_one_sql, project.project_root, "models", "model_one.sql") - with pytest.raises(ParsingError): - results = run_dbt(["--partial-parse", "run"]) - - # actually set the env_var - os.environ["TEST_SCHEMA_VAR"] = "view" - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - expected_env_vars = {"ENV_VAR_TEST": "second", "TEST_SCHEMA_VAR": "view"} - assert expected_env_vars == manifest.env_vars - - # env vars in a source - os.environ["ENV_VAR_DATABASE"] = "dbt" - os.environ["ENV_VAR_SEVERITY"] = "warn" - write_file(raw_customers_csv, project.project_root, "seeds", "raw_customers.csv") - write_file(env_var_sources_yml, project.project_root, "models", "sources.yml") - run_dbt(["--partial-parse", "seed"]) - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - manifest = get_manifest(project.project_root) - expected_env_vars = { - "ENV_VAR_TEST": "second", - "TEST_SCHEMA_VAR": "view", - "ENV_VAR_DATABASE": "dbt", - "ENV_VAR_SEVERITY": "warn", - } - assert expected_env_vars == manifest.env_vars - assert len(manifest.sources) == 1 - source_id = "source.test.seed_sources.raw_customers" - source = manifest.sources[source_id] - assert source.database == "dbt" - schema_file = manifest.files[source.file_id] - test_id = "test.test.source_not_null_seed_sources_raw_customers_id.e39ee7bf0d" - test_node = manifest.nodes[test_id] - assert test_node.config.severity == "WARN" - - # Change severity env var - os.environ["ENV_VAR_SEVERITY"] = "error" - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - expected_env_vars = { - "ENV_VAR_TEST": "second", - "TEST_SCHEMA_VAR": "view", - "ENV_VAR_DATABASE": "dbt", - "ENV_VAR_SEVERITY": "error", - } - assert expected_env_vars == manifest.env_vars - source_id = "source.test.seed_sources.raw_customers" - source = manifest.sources[source_id] - schema_file = manifest.files[source.file_id] - expected_schema_file_env_vars = { - "sources": {"seed_sources": ["ENV_VAR_DATABASE", "ENV_VAR_SEVERITY"]} - } - assert expected_schema_file_env_vars == schema_file.env_vars - test_node = manifest.nodes[test_id] - assert test_node.config.severity == "ERROR" - - # Change database env var - os.environ["ENV_VAR_DATABASE"] = "test_dbt" - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - expected_env_vars = { - "ENV_VAR_TEST": "second", - "TEST_SCHEMA_VAR": "view", - "ENV_VAR_DATABASE": "test_dbt", - "ENV_VAR_SEVERITY": "error", - } - assert expected_env_vars == manifest.env_vars - source = manifest.sources[source_id] - assert source.database == "test_dbt" - - # Delete database env var - del os.environ["ENV_VAR_DATABASE"] - with pytest.raises(ParsingError): - results = run_dbt(["--partial-parse", "run"]) - os.environ["ENV_VAR_DATABASE"] = "test_dbt" - - # Add generic test with test kwarg that's rendered late (no curly brackets) - os.environ["ENV_VAR_DATABASE"] = "dbt" - write_file(test_color_sql, project.project_root, "macros", "test_color.sql") - results = run_dbt(["--partial-parse", "run"]) - # Add source test using test_color and an env_var for color - write_file(env_var_schema2_yml, project.project_root, "models/schema.yml") - with pytest.raises(ParsingError): - results = run_dbt(["--partial-parse", "run"]) - os.environ["ENV_VAR_COLOR"] = "green" - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - test_color_id = "test.test.check_color_model_one_env_var_ENV_VAR_COLOR___fun.89638de387" - test_node = manifest.nodes[test_color_id] - # kwarg was rendered but not changed (it will be rendered again when compiled) - assert test_node.test_metadata.kwargs["color"] == "env_var('ENV_VAR_COLOR')" - results = run_dbt(["--partial-parse", "test"]) - - # Add an exposure with an env_var - os.environ["ENV_VAR_OWNER"] = "John Doe" - write_file(env_var_schema3_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - expected_env_vars = { - "ENV_VAR_TEST": "second", - "TEST_SCHEMA_VAR": "view", - "ENV_VAR_DATABASE": "dbt", - "ENV_VAR_SEVERITY": "error", - "ENV_VAR_COLOR": "green", - "ENV_VAR_OWNER": "John Doe", - } - assert expected_env_vars == manifest.env_vars - exposure = list(manifest.exposures.values())[0] - schema_file = manifest.files[exposure.file_id] - expected_sf_env_vars = { - "models": {"model_one": ["TEST_SCHEMA_VAR", "ENV_VAR_COLOR"]}, - "exposures": {"proxy_for_dashboard": ["ENV_VAR_OWNER"]}, - } - assert expected_sf_env_vars == schema_file.env_vars - - # add a macro and a macro schema file - os.environ["ENV_VAR_SOME_KEY"] = "toodles" - write_file(env_var_macro_sql, project.project_root, "macros", "env_var_macro.sql") - write_file(env_var_macros_yml, project.project_root, "macros", "env_var_macros.yml") - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - expected_env_vars = { - "ENV_VAR_TEST": "second", - "TEST_SCHEMA_VAR": "view", - "ENV_VAR_DATABASE": "dbt", - "ENV_VAR_SEVERITY": "error", - "ENV_VAR_COLOR": "green", - "ENV_VAR_OWNER": "John Doe", - "ENV_VAR_SOME_KEY": "toodles", - } - assert expected_env_vars == manifest.env_vars - macro_id = "macro.test.do_something" - macro = manifest.macros[macro_id] - assert macro.meta == {"some_key": "toodles"} - # change the env var - os.environ["ENV_VAR_SOME_KEY"] = "dumdedum" - results = run_dbt(["--partial-parse", "run"]) - manifest = get_manifest(project.project_root) - macro = manifest.macros[macro_id] - assert macro.meta == {"some_key": "dumdedum"} - - # Add a schema file with a test on model_color and env_var in test enabled config - write_file(env_var_model_test_yml, project.project_root, "models", "schema.yml") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - manifest = get_manifest(project.project_root) - model_color = manifest.nodes["model.test.model_color"] - schema_file = manifest.files[model_color.patch_path] - expected_env_vars = { - "models": { - "model_one": ["TEST_SCHEMA_VAR", "ENV_VAR_COLOR"], - "model_color": ["ENV_VAR_ENABLED"], - }, - "exposures": {"proxy_for_dashboard": ["ENV_VAR_OWNER"]}, - } - assert expected_env_vars == schema_file.env_vars - - # Add a metrics file with env_vars - os.environ["ENV_VAR_METRICS"] = "TeStInG" - write_file(people_sql, project.project_root, "models", "people.sql") - write_file( - metricflow_time_spine_sql, project.project_root, "models", "metricflow_time_spine.sql" - ) - write_file( - people_semantic_models_yml, project.project_root, "models", "semantic_models.yml" - ) - write_file(env_var_metrics_yml, project.project_root, "models", "metrics.yml") - results = run_dbt(["run"]) - manifest = get_manifest(project.project_root) - assert "ENV_VAR_METRICS" in manifest.env_vars - assert manifest.env_vars["ENV_VAR_METRICS"] == "TeStInG" - metric_node = manifest.metrics["metric.test.number_of_people"] - assert metric_node.meta == {"my_meta": "TeStInG"} - - # Change metrics env var - os.environ["ENV_VAR_METRICS"] = "Changed!" - results = run_dbt(["run"]) - manifest = get_manifest(project.project_root) - metric_node = manifest.metrics["metric.test.number_of_people"] - assert metric_node.meta == {"my_meta": "Changed!"} - - # delete the env vars to cleanup - del os.environ["ENV_VAR_TEST"] - del os.environ["ENV_VAR_SEVERITY"] - del os.environ["ENV_VAR_DATABASE"] - del os.environ["TEST_SCHEMA_VAR"] - del os.environ["ENV_VAR_COLOR"] - del os.environ["ENV_VAR_SOME_KEY"] - del os.environ["ENV_VAR_OWNER"] - del os.environ["ENV_VAR_METRICS"] - - -class TestProjectEnvVars: - @pytest.fixture(scope="class") - def project_config_update(self): - # Need to set the environment variable here initially because - # the project fixture loads the config. - os.environ["ENV_VAR_NAME"] = "Jane Smith" - return {"models": {"+meta": {"meta_name": "{{ env_var('ENV_VAR_NAME') }}"}}} - - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - def test_project_env_vars(self, project): - # Initial run - results = run_dbt(["run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - state_check = manifest.state_check - model_id = "model.test.model_one" - model = manifest.nodes[model_id] - assert model.config.meta["meta_name"] == "Jane Smith" - env_vars_hash_checksum = state_check.project_env_vars_hash.checksum - - # Change the environment variable - os.environ["ENV_VAR_NAME"] = "Jane Doe" - results = run_dbt(["run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - model = manifest.nodes[model_id] - assert model.config.meta["meta_name"] == "Jane Doe" - assert env_vars_hash_checksum != manifest.state_check.project_env_vars_hash.checksum - - # cleanup - del os.environ["ENV_VAR_NAME"] - - -class TestProfileEnvVars: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - @pytest.fixture(scope="class") - def dbt_profile_target(self): - # Need to set these here because the base integration test class - # calls 'load_config' before the tests are run. - # Note: only the specified profile is rendered, so there's no - # point it setting env_vars in non-used profiles. - os.environ["ENV_VAR_USER"] = "root" - os.environ["ENV_VAR_PASS"] = "password" - return { - "type": "postgres", - "threads": 4, - "host": "localhost", - "port": 5432, - "user": "{{ env_var('ENV_VAR_USER') }}", - "pass": "{{ env_var('ENV_VAR_PASS') }}", - "dbname": "dbt", - } - - def test_profile_env_vars(self, project, logs_dir): - # Initial run - os.environ["ENV_VAR_USER"] = "root" - os.environ["ENV_VAR_PASS"] = "password" - - run_dbt(["run"]) - manifest = get_manifest(project.project_root) - env_vars_checksum = manifest.state_check.profile_env_vars_hash.checksum - - # Change env_vars, the user doesn't exist, this should fail - os.environ["ENV_VAR_USER"] = "fake_user" - - # N.B. run_dbt_and_capture won't work here because FailedToConnectError ends the test entirely - with pytest.raises(FailedToConnectError): - run_dbt(["run"], expect_pass=False) - - log_output = Path(logs_dir, "dbt.log").read_text() - assert "env vars used in profiles.yml have changed" in log_output - - manifest = get_manifest(project.project_root) - assert env_vars_checksum != manifest.state_check.profile_env_vars_hash.checksum - - -class TestProfileSecretEnvVars: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one.sql": model_one_sql, - } - - @property - def dbt_profile_target(self): - # Need to set these here because the base integration test class - # calls 'load_config' before the tests are run. - # Note: only the specified profile is rendered, so there's no - # point in setting env_vars in non-used profiles. - - # user is secret and password is not. postgres on macos doesn't care if the password - # changes so we have to change the user. related: https://github.com/dbt-labs/dbt-core/pull/4250 - os.environ[SECRET_ENV_PREFIX + "USER"] = "root" - os.environ["ENV_VAR_PASS"] = "password" - return { - "type": "postgres", - "threads": 4, - "host": "localhost", - "port": 5432, - "user": "{{ env_var('DBT_ENV_SECRET_USER') }}", - "pass": "{{ env_var('ENV_VAR_PASS') }}", - "dbname": "dbt", - } - - def test_profile_secret_env_vars(self, project): - # Initial run - os.environ[SECRET_ENV_PREFIX + "USER"] = "root" - os.environ["ENV_VAR_PASS"] = "password" - - results = run_dbt(["run"]) - manifest = get_manifest(project.project_root) - env_vars_checksum = manifest.state_check.profile_env_vars_hash.checksum - - # Change a secret var, it shouldn't register because we shouldn't save secrets. - os.environ[SECRET_ENV_PREFIX + "USER"] = "fake_user" - # we just want to see if the manifest has included - # the secret in the hash of environment variables. - (results, log_output) = run_dbt_and_capture(["run"], expect_pass=True) - # I020 is the event code for "env vars used in profiles.yml have changed" - assert not ("I020" in log_output) - manifest = get_manifest(project.project_root) - assert env_vars_checksum == manifest.state_check.profile_env_vars_hash.checksum diff --git a/tests/functional/partial_parsing/test_versioned_models.py b/tests/functional/partial_parsing/test_versioned_models.py deleted file mode 100644 index d725c671..00000000 --- a/tests/functional/partial_parsing/test_versioned_models.py +++ /dev/null @@ -1,128 +0,0 @@ -import pathlib - -from dbt.exceptions import DuplicateVersionedUnversionedError -from dbt.tests.util import ( - get_manifest, - read_file, - rm_file, - run_dbt, - write_file, -) -import pytest - - -model_one_sql = """ -select 1 as fun -""" - -model_one_downstream_sql = """ -select fun from {{ ref('model_one') }} -""" - -models_versions_schema_yml = """ - -models: - - name: model_one - description: "The first model" - versions: - - v: 1 - - v: 2 -""" - -models_versions_defined_in_schema_yml = """ -models: - - name: model_one - description: "The first model" - versions: - - v: 1 - - v: 2 - defined_in: model_one_different -""" - -models_versions_updated_schema_yml = """ -models: - - name: model_one - latest_version: 1 - description: "The first model" - versions: - - v: 1 - - v: 2 - defined_in: model_one_different -""" - -model_two_sql = """ -select 1 as notfun -""" - - -class TestVersionedModels: - @pytest.fixture(scope="class") - def models(self): - return { - "model_one_v1.sql": model_one_sql, - "model_one.sql": model_one_sql, - "model_one_downstream.sql": model_one_downstream_sql, - "schema.yml": models_versions_schema_yml, - } - - def test_pp_versioned_models(self, project): - results = run_dbt(["run"]) - assert len(results) == 3 - - manifest = get_manifest(project.project_root) - model_one_node = manifest.nodes["model.test.model_one.v1"] - assert not model_one_node.is_latest_version - model_two_node = manifest.nodes["model.test.model_one.v2"] - assert model_two_node.is_latest_version - # assert unpinned ref points to latest version - model_one_downstream_node = manifest.nodes["model.test.model_one_downstream"] - assert model_one_downstream_node.depends_on.nodes == ["model.test.model_one.v2"] - - # update schema.yml block - model_one is now 'defined_in: model_one_different' - rm_file(project.project_root, "models", "model_one.sql") - write_file(model_one_sql, project.project_root, "models", "model_one_different.sql") - write_file( - models_versions_defined_in_schema_yml, project.project_root, "models", "schema.yml" - ) - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - - # update versions schema.yml block - latest_version from 2 to 1 - write_file( - models_versions_updated_schema_yml, project.project_root, "models", "schema.yml" - ) - # This is where the test was failings in a CI run with: - # relation \"test..._test_partial_parsing.model_one_downstream\" does not exist - # because in core/dbt/include/global_project/macros/materializations/models/view/view.sql - # "existing_relation" didn't actually exist by the time it gets to the rename of the - # existing relation. - (pathlib.Path(project.project_root) / "log_output").mkdir(parents=True, exist_ok=True) - results = run_dbt( - ["--partial-parse", "--log-format-file", "json", "--log-path", "log_output", "run"] - ) - assert len(results) == 3 - - manifest = get_manifest(project.project_root) - model_one_node = manifest.nodes["model.test.model_one.v1"] - assert model_one_node.is_latest_version - model_two_node = manifest.nodes["model.test.model_one.v2"] - assert not model_two_node.is_latest_version - # assert unpinned ref points to latest version - model_one_downstream_node = manifest.nodes["model.test.model_one_downstream"] - assert model_one_downstream_node.depends_on.nodes == ["model.test.model_one.v1"] - - # assert unpinned ref to latest-not-max version yields an "FYI" info-level log - log_output = read_file("log_output", "dbt.log").replace("\n", " ").replace("\\n", " ") - assert "UnpinnedRefNewVersionAvailable" in log_output - - # update versioned model - write_file(model_two_sql, project.project_root, "models", "model_one_different.sql") - results = run_dbt(["--partial-parse", "run"]) - assert len(results) == 3 - manifest = get_manifest(project.project_root) - assert len(manifest.nodes) == 3 - - # create a new model_one in model_one.sql and re-parse - write_file(model_one_sql, project.project_root, "models", "model_one.sql") - with pytest.raises(DuplicateVersionedUnversionedError): - run_dbt(["parse"])