From e699f5d04286cf57f02e304af272f4bd88fc9745 Mon Sep 17 00:00:00 2001
From: Chenyu Li <chenyu.li@dbtlabs.com>
Date: Tue, 11 Jun 2024 14:41:11 -0700
Subject: [PATCH] create a happy path project fixture (#10291)

---
 core/dbt/tests/fixtures/project.py            | 41 +++++++++----
 core/dbt/tests/util.py                        |  1 -
 .../adapter/basic/test_adapter_methods.py     | 11 ----
 tests/functional/fixtures/__init__.py         |  0
 .../functional/fixtures/happy_path_fixture.py | 32 +++++++++++
 .../happy_path_project/analyses/a.sql         |  1 +
 .../happy_path_project/dbt_project.yml        | 17 ++++++
 .../happy_path_project/macros/macro_stuff.sql |  7 +++
 .../happy_path_project/models/docs.md         |  3 +
 .../happy_path_project/models/ephemeral.sql   |  5 ++
 .../happy_path_project/models/incremental.sql | 12 ++++
 .../fixtures/happy_path_project/models/m.yml  |  7 +++
 .../models/metricflow_time_spine.sql          |  2 +
 .../happy_path_project/models/outer.sql       |  1 +
 .../happy_path_project/models/schema.yml      | 15 +++++
 .../fixtures/happy_path_project/models/sm.yml | 18 ++++++
 .../fixtures/happy_path_project/models/sq.yml | 14 +++++
 .../happy_path_project/models/sub/inner.sql   |  1 +
 .../happy_path_project/seeds/seed.csv         |  2 +
 .../happy_path_project/snapshots/snapshot.sql | 12 ++++
 .../fixtures/happy_path_project/tests/t.sql   |  1 +
 tests/functional/list/test_list.py            | 57 +++++++------------
 tests/functional/test_selection/fixtures.py   | 12 ----
 .../test_selection_expansion.py               |  6 +-
 24 files changed, 199 insertions(+), 79 deletions(-)
 create mode 100644 tests/functional/fixtures/__init__.py
 create mode 100644 tests/functional/fixtures/happy_path_fixture.py
 create mode 100644 tests/functional/fixtures/happy_path_project/analyses/a.sql
 create mode 100644 tests/functional/fixtures/happy_path_project/dbt_project.yml
 create mode 100644 tests/functional/fixtures/happy_path_project/macros/macro_stuff.sql
 create mode 100644 tests/functional/fixtures/happy_path_project/models/docs.md
 create mode 100644 tests/functional/fixtures/happy_path_project/models/ephemeral.sql
 create mode 100644 tests/functional/fixtures/happy_path_project/models/incremental.sql
 create mode 100644 tests/functional/fixtures/happy_path_project/models/m.yml
 create mode 100644 tests/functional/fixtures/happy_path_project/models/metricflow_time_spine.sql
 create mode 100644 tests/functional/fixtures/happy_path_project/models/outer.sql
 create mode 100644 tests/functional/fixtures/happy_path_project/models/schema.yml
 create mode 100644 tests/functional/fixtures/happy_path_project/models/sm.yml
 create mode 100644 tests/functional/fixtures/happy_path_project/models/sq.yml
 create mode 100644 tests/functional/fixtures/happy_path_project/models/sub/inner.sql
 create mode 100644 tests/functional/fixtures/happy_path_project/seeds/seed.csv
 create mode 100644 tests/functional/fixtures/happy_path_project/snapshots/snapshot.sql
 create mode 100644 tests/functional/fixtures/happy_path_project/tests/t.sql

diff --git a/core/dbt/tests/fixtures/project.py b/core/dbt/tests/fixtures/project.py
index 5395247a74c..daacef0f553 100644
--- a/core/dbt/tests/fixtures/project.py
+++ b/core/dbt/tests/fixtures/project.py
@@ -282,8 +282,8 @@ def adapter(
     project_root,
     profiles_root,
     profiles_yml,
-    dbt_project_yml,
     clean_up_logging,
+    dbt_project_yml,
 ):
     # The profiles.yml and dbt_project.yml should already be written out
     args = Namespace(
@@ -385,7 +385,20 @@ def analyses():
 
 # Write out the files provided by models, macros, properties, snapshots, seeds, tests, analyses
 @pytest.fixture(scope="class")
-def project_files(project_root, models, macros, snapshots, properties, seeds, tests, analyses):
+def project_files(
+    project_root,
+    models,
+    macros,
+    snapshots,
+    properties,
+    seeds,
+    tests,
+    analyses,
+    selectors_yml,
+    dependencies_yml,
+    packages_yml,
+    dbt_project_yml,
+):
     write_project_files(project_root, "models", {**models, **properties})
     write_project_files(project_root, "macros", macros)
     write_project_files(project_root, "snapshots", snapshots)
@@ -515,12 +528,8 @@ def initialization(environment) -> None:
     enable_test_caching()
 
 
-# This is the main fixture that is used in all functional tests. It pulls in the other
-# fixtures that are necessary to set up a dbt project, and saves some of the information
-# in a TestProjInfo class, which it returns, so that individual test cases do not have
-# to pull in the other fixtures individually to access their information.
 @pytest.fixture(scope="class")
-def project(
+def project_setup(
     initialization,
     clean_up_logging,
     project_root,
@@ -528,12 +537,7 @@ def project(
     request,
     unique_schema,
     profiles_yml,
-    dbt_project_yml,
-    packages_yml,
-    dependencies_yml,
-    selectors_yml,
     adapter,
-    project_files,
     shared_data_dir,
     test_data_dir,
     logs_dir,
@@ -587,3 +591,16 @@ def project(
         pass
     os.chdir(orig_cwd)
     cleanup_event_logger()
+
+
+# This is the main fixture that is used in all functional tests. It pulls in the other
+# fixtures that are necessary to set up a dbt project, and saves some of the information
+# in a TestProjInfo class, which it returns, so that individual test cases do not have
+# to pull in the other fixtures individually to access their information.
+# The order of arguments here determine which steps runs first.
+@pytest.fixture(scope="class")
+def project(
+    project_setup: TestProjInfo,
+    project_files,
+):
+    return project_setup
diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py
index d5c5e49d2aa..cb2fbeccca8 100644
--- a/core/dbt/tests/util.py
+++ b/core/dbt/tests/util.py
@@ -90,7 +90,6 @@ def run_dbt(
         args.extend(["--project-dir", project_dir])
     if profiles_dir and "--profiles-dir" not in args:
         args.extend(["--profiles-dir", profiles_dir])
-
     dbt = dbtRunner()
     res = dbt.invoke(args)
 
diff --git a/tests/functional/adapter/basic/test_adapter_methods.py b/tests/functional/adapter/basic/test_adapter_methods.py
index 51d987f07de..62f70b4f0f1 100644
--- a/tests/functional/adapter/basic/test_adapter_methods.py
+++ b/tests/functional/adapter/basic/test_adapter_methods.py
@@ -1,6 +1,5 @@
 import pytest
 
-from dbt.tests.fixtures.project import write_project_files
 from dbt.tests.util import check_relations_equal, run_dbt
 
 tests__get_columns_in_relation_sql = """
@@ -73,16 +72,6 @@ def models(self):
             "model.sql": models__model_sql,
         }
 
-    @pytest.fixture(scope="class")
-    def project_files(
-        self,
-        project_root,
-        tests,
-        models,
-    ):
-        write_project_files(project_root, "tests", tests)
-        write_project_files(project_root, "models", models)
-
     @pytest.fixture(scope="class")
     def project_config_update(self):
         return {
diff --git a/tests/functional/fixtures/__init__.py b/tests/functional/fixtures/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/functional/fixtures/happy_path_fixture.py b/tests/functional/fixtures/happy_path_fixture.py
new file mode 100644
index 00000000000..3f365f36413
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_fixture.py
@@ -0,0 +1,32 @@
+import os
+from distutils.dir_util import copy_tree
+
+import pytest
+
+
+def delete_files_in_directory(directory_path):
+    try:
+        with os.scandir(directory_path) as entries:
+            for entry in entries:
+                if entry.is_file():
+                    os.unlink(entry.path)
+        print("All files deleted successfully.")
+    except OSError:
+        print("Error occurred while deleting files.")
+
+
+@pytest.fixture(scope="class")
+def happy_path_project_files(project_root):
+    # copy fixture files to the project root
+    delete_files_in_directory(project_root)
+    copy_tree(
+        os.path.dirname(os.path.realpath(__file__)) + "/happy_path_project", str(project_root)
+    )
+
+
+# We do project_setup first because it will write out a dbt_project.yml.
+# This file will be overwritten by the files in happy_path_project later on.
+@pytest.fixture(scope="class")
+def happy_path_project(project_setup, happy_path_project_files):
+    # A fixture that gives functional test the project living in happy_path_project
+    return project_setup
diff --git a/tests/functional/fixtures/happy_path_project/analyses/a.sql b/tests/functional/fixtures/happy_path_project/analyses/a.sql
new file mode 100644
index 00000000000..1f7d87c55a9
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/analyses/a.sql
@@ -0,0 +1 @@
+select 4 as id
diff --git a/tests/functional/fixtures/happy_path_project/dbt_project.yml b/tests/functional/fixtures/happy_path_project/dbt_project.yml
new file mode 100644
index 00000000000..716ab611ef3
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/dbt_project.yml
@@ -0,0 +1,17 @@
+analysis-paths:
+- analyses
+config-version: 2
+flags:
+  send_anonymous_usage_stats: false
+macro-paths:
+- macros
+name: test
+profile: test
+seed-paths:
+- seeds
+seeds:
+  quote_columns: false
+snapshot-paths:
+- snapshots
+test-paths:
+- tests
diff --git a/tests/functional/fixtures/happy_path_project/macros/macro_stuff.sql b/tests/functional/fixtures/happy_path_project/macros/macro_stuff.sql
new file mode 100644
index 00000000000..2ac8c3b572f
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/macros/macro_stuff.sql
@@ -0,0 +1,7 @@
+{% macro cool_macro() %}
+  wow!
+{% endmacro %}
+
+{% macro other_cool_macro(a, b) %}
+  cool!
+{% endmacro %}
diff --git a/tests/functional/fixtures/happy_path_project/models/docs.md b/tests/functional/fixtures/happy_path_project/models/docs.md
new file mode 100644
index 00000000000..3f547f37fd9
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/models/docs.md
@@ -0,0 +1,3 @@
+{% docs my_docs %}
+  some docs
+{% enddocs %}
diff --git a/tests/functional/fixtures/happy_path_project/models/ephemeral.sql b/tests/functional/fixtures/happy_path_project/models/ephemeral.sql
new file mode 100644
index 00000000000..4f9208f28a2
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/models/ephemeral.sql
@@ -0,0 +1,5 @@
+{{ config(materialized='ephemeral') }}
+
+select
+  1 as id,
+  {{ dbt.date_trunc('day', dbt.current_timestamp()) }} as created_at
diff --git a/tests/functional/fixtures/happy_path_project/models/incremental.sql b/tests/functional/fixtures/happy_path_project/models/incremental.sql
new file mode 100644
index 00000000000..2bb15542da9
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/models/incremental.sql
@@ -0,0 +1,12 @@
+{{
+  config(
+    materialized = "incremental",
+    incremental_strategy = "delete+insert",
+  )
+}}
+
+select * from {{ ref('seed') }}
+
+{% if is_incremental() %}
+    where a > (select max(a) from {{this}})
+{% endif %}
diff --git a/tests/functional/fixtures/happy_path_project/models/m.yml b/tests/functional/fixtures/happy_path_project/models/m.yml
new file mode 100644
index 00000000000..38e5be96af2
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/models/m.yml
@@ -0,0 +1,7 @@
+metrics:
+  - name: total_outer
+    type: simple
+    description: The total count of outer
+    label: Total Outer
+    type_params:
+      measure: total_outer_count
diff --git a/tests/functional/fixtures/happy_path_project/models/metricflow_time_spine.sql b/tests/functional/fixtures/happy_path_project/models/metricflow_time_spine.sql
new file mode 100644
index 00000000000..b672fabcad2
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/models/metricflow_time_spine.sql
@@ -0,0 +1,2 @@
+select
+  {{ dbt.date_trunc('day', dbt.current_timestamp()) }} as date_day
diff --git a/tests/functional/fixtures/happy_path_project/models/outer.sql b/tests/functional/fixtures/happy_path_project/models/outer.sql
new file mode 100644
index 00000000000..85bbabe21e3
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/models/outer.sql
@@ -0,0 +1 @@
+select * from {{ ref('ephemeral') }}
diff --git a/tests/functional/fixtures/happy_path_project/models/schema.yml b/tests/functional/fixtures/happy_path_project/models/schema.yml
new file mode 100644
index 00000000000..ef6addf5b91
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/models/schema.yml
@@ -0,0 +1,15 @@
+version: 2
+models:
+  - name: outer
+    description: The outer table
+    columns:
+      - name: id
+        description: The id value
+        data_tests:
+          - unique
+          - not_null
+
+sources:
+  - name: my_source
+    tables:
+      - name: my_table
diff --git a/tests/functional/fixtures/happy_path_project/models/sm.yml b/tests/functional/fixtures/happy_path_project/models/sm.yml
new file mode 100644
index 00000000000..6db38e3ae4c
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/models/sm.yml
@@ -0,0 +1,18 @@
+semantic_models:
+  - name: my_sm
+    model: ref('outer')
+    defaults:
+      agg_time_dimension: created_at
+    entities:
+      - name: my_entity
+        type: primary
+        expr: id
+    dimensions:
+      - name: created_at
+        type: time
+        type_params:
+          time_granularity: day
+    measures:
+      - name: total_outer_count
+        agg: count
+        expr: 1
diff --git a/tests/functional/fixtures/happy_path_project/models/sq.yml b/tests/functional/fixtures/happy_path_project/models/sq.yml
new file mode 100644
index 00000000000..e28e3805a3f
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/models/sq.yml
@@ -0,0 +1,14 @@
+saved_queries:
+  - name: my_saved_query
+    label: My Saved Query
+    query_params:
+        metrics:
+            - total_outer
+        group_by:
+            - "Dimension('my_entity__created_at')"
+    exports:
+        - name: my_export
+          config:
+            alias: my_export_alias
+            export_as: table
+            schema: my_export_schema_name
diff --git a/tests/functional/fixtures/happy_path_project/models/sub/inner.sql b/tests/functional/fixtures/happy_path_project/models/sub/inner.sql
new file mode 100644
index 00000000000..a90004d480d
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/models/sub/inner.sql
@@ -0,0 +1 @@
+select * from {{ ref('outer') }}
diff --git a/tests/functional/fixtures/happy_path_project/seeds/seed.csv b/tests/functional/fixtures/happy_path_project/seeds/seed.csv
new file mode 100644
index 00000000000..cfa20f81071
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/seeds/seed.csv
@@ -0,0 +1,2 @@
+a,b
+1,2
diff --git a/tests/functional/fixtures/happy_path_project/snapshots/snapshot.sql b/tests/functional/fixtures/happy_path_project/snapshots/snapshot.sql
new file mode 100644
index 00000000000..60d803dfbb4
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/snapshots/snapshot.sql
@@ -0,0 +1,12 @@
+{% snapshot my_snapshot %}
+    {{
+        config(
+            target_database=var('target_database', database),
+            target_schema=schema,
+            unique_key='id',
+            strategy='timestamp',
+            updated_at='updated_at',
+        )
+    }}
+    select * from {{database}}.{{schema}}.seed
+{% endsnapshot %}
diff --git a/tests/functional/fixtures/happy_path_project/tests/t.sql b/tests/functional/fixtures/happy_path_project/tests/t.sql
new file mode 100644
index 00000000000..9757b1b7ae9
--- /dev/null
+++ b/tests/functional/fixtures/happy_path_project/tests/t.sql
@@ -0,0 +1 @@
+select 1 as id limit 0
diff --git a/tests/functional/list/test_list.py b/tests/functional/list/test_list.py
index 9b51b6798c9..8d462e258b0 100644
--- a/tests/functional/list/test_list.py
+++ b/tests/functional/list/test_list.py
@@ -1,20 +1,10 @@
 import json
 import os
 
-import pytest
-
 from dbt.tests.util import run_dbt
-from tests.functional.list.fixtures import (  # noqa: F401
-    analyses,
-    macros,
-    metrics,
-    models,
-    project_files,
-    saved_queries,
-    seeds,
-    semantic_models,
-    snapshots,
-    tests,
+from tests.functional.fixtures.happy_path_fixture import (  # noqa: F401
+    happy_path_project,
+    happy_path_project_files,
 )
 
 
@@ -22,21 +12,7 @@ class TestList:
     def dir(self, value):
         return os.path.normpath(value)
 
-    @pytest.fixture(scope="class")
-    def project_config_update(self):
-        return {
-            "config-version": 2,
-            "analysis-paths": [self.dir("analyses")],
-            "snapshot-paths": [self.dir("snapshots")],
-            "macro-paths": [self.dir("macros")],
-            "seed-paths": [self.dir("seeds")],
-            "test-paths": [self.dir("tests")],
-            "seeds": {
-                "quote_columns": False,
-            },
-        }
-
-    def test_packages_install_path_does_not_exist(self, project):
+    def test_packages_install_path_does_not_exist(self, happy_path_project):  # noqa: F811
         run_dbt(["list"])
         packages_install_path = "dbt_packages"
 
@@ -47,7 +23,6 @@ def run_dbt_ls(self, args=None, expect_pass=True):
         full_args = ["ls"]
         if args is not None:
             full_args += args
-
         result = run_dbt(args=full_args, expect_pass=expect_pass)
 
         return result
@@ -67,7 +42,7 @@ def expect_given_output(self, args, expectations):
                 else:
                     assert got == expected
 
-    def expect_snapshot_output(self, project):
+    def expect_snapshot_output(self, happy_path_project):  # noqa: F811
         expectations = {
             "name": "my_snapshot",
             "selector": "test.snapshot.my_snapshot",
@@ -86,8 +61,8 @@ def expect_snapshot_output(self, project):
                     "quoting": {},
                     "column_types": {},
                     "persist_docs": {},
-                    "target_database": project.database,
-                    "target_schema": project.test_schema,
+                    "target_database": happy_path_project.database,
+                    "target_schema": happy_path_project.test_schema,
                     "unique_key": "id",
                     "strategy": "timestamp",
                     "updated_at": "updated_at",
@@ -735,10 +710,14 @@ def expect_resource_type_env_var(self):
         }
         del os.environ["DBT_EXCLUDE_RESOURCE_TYPES"]
 
-    def expect_selected_keys(self, project):
+    def expect_selected_keys(self, happy_path_project):  # noqa: F811
         """Expect selected fields of the the selected model"""
         expectations = [
-            {"database": project.database, "schema": project.test_schema, "alias": "inner"}
+            {
+                "database": happy_path_project.database,
+                "schema": happy_path_project.test_schema,
+                "alias": "inner",
+            }
         ]
         results = self.run_dbt_ls(
             [
@@ -759,7 +738,9 @@ def expect_selected_keys(self, project):
 
         """Expect selected fields when --output-keys given multiple times
         """
-        expectations = [{"database": project.database, "schema": project.test_schema}]
+        expectations = [
+            {"database": happy_path_project.database, "schema": happy_path_project.test_schema}
+        ]
         results = self.run_dbt_ls(
             [
                 "--model",
@@ -821,8 +802,8 @@ def expect_selected_keys(self, project):
         for got, expected in zip(results, expectations):
             self.assert_json_equal(got, expected)
 
-    def test_ls(self, project):
-        self.expect_snapshot_output(project)
+    def test_ls(self, happy_path_project):  # noqa: F811
+        self.expect_snapshot_output(happy_path_project)
         self.expect_analyses_output()
         self.expect_model_output()
         self.expect_source_output()
@@ -832,7 +813,7 @@ def test_ls(self, project):
         self.expect_resource_type_multiple()
         self.expect_resource_type_env_var()
         self.expect_all_output()
-        self.expect_selected_keys(project)
+        self.expect_selected_keys(happy_path_project)
 
 
 def normalize(path):
diff --git a/tests/functional/test_selection/fixtures.py b/tests/functional/test_selection/fixtures.py
index 37cce1bbae9..84f4c532e5f 100644
--- a/tests/functional/test_selection/fixtures.py
+++ b/tests/functional/test_selection/fixtures.py
@@ -1,7 +1,5 @@
 import pytest
 
-from dbt.tests.fixtures.project import write_project_files
-
 tests__cf_a_b_sql = """
 select * from {{ ref('model_a') }}
 cross join {{ ref('model_b') }}
@@ -84,13 +82,3 @@ def models():
         "model_b.sql": models__model_b_sql,
         "model_a.sql": models__model_a_sql,
     }
-
-
-@pytest.fixture(scope="class")
-def project_files(
-    project_root,
-    tests,
-    models,
-):
-    write_project_files(project_root, "tests", tests)
-    write_project_files(project_root, "models", models)
diff --git a/tests/functional/test_selection/test_selection_expansion.py b/tests/functional/test_selection/test_selection_expansion.py
index e42d6196328..d163588303c 100644
--- a/tests/functional/test_selection/test_selection_expansion.py
+++ b/tests/functional/test_selection/test_selection_expansion.py
@@ -1,11 +1,7 @@
 import pytest
 
 from dbt.tests.util import run_dbt
-from tests.functional.test_selection.fixtures import (  # noqa: F401
-    models,
-    project_files,
-    tests,
-)
+from tests.functional.test_selection.fixtures import models, tests  # noqa: F401
 
 
 class TestSelectionExpansion: