Add dbt mesh to main page and improve tests (#52)

* Tests run dbt clean before tests

* Tests run dbt clean before tests

README.md

@@ -28,8 +28,13 @@ adapter and providing jinja context with further adapter/python methods.
   UI page.
 - Customized dbt Docs: Replace the default dbt documentation page with your own custom index.html file.
 - Run end to end ETL pipeline with dbt
-  using [dlt](https://dlthub.com/) [integration](https://github.com/memiiso/opendbt/issues/40)
-- Register dbt callbacks within a dbt project to trigger custom actions or alerts based on selected dbt events.
+  using [dlt extract and load](https://dlthub.com/) [integration](https://github.com/memiiso/opendbt/issues/40)
+- Register [dbt callbacks](https://docs.getdbt.com/reference/programmatic-invocations#registering-callbacks) within a
+  dbt project to trigger custom actions or alerts based on selected dbt events.
+- Use multi project dbt-mesh setup,
+  using [cross-project references](https://docs.getdbt.com/docs/collaborate/govern/project-dependencies#how-to-write-cross-project-ref).
+  ex: `{{ ref('jaffle_finance', 'monthly_revenue') }}`
+  - This feature was only available in "dbt Cloud Enterprise" so far.
 
 For detailed examples, see: [examples](docs/EXAMPLES.md).
 

tests/base_dbt_test.py

@@ -3,9 +3,18 @@
 
 from dbt.version import get_installed_version as get_dbt_version
 
+from opendbt import OpenDbtCli
+
 
 class BaseDbtTest(TestCase):
     RESOURCES_DIR = Path(__file__).parent.joinpath("resources")
     DBTCORE_DIR = RESOURCES_DIR.joinpath("dbtcore")
     DBTFINANCE_DIR = RESOURCES_DIR.joinpath("dbtfinance")
     DBT_VERSION = get_dbt_version()
+
+    @classmethod
+    def setUpClass(cls):
+        dpf = OpenDbtCli(project_dir=BaseDbtTest.DBTFINANCE_DIR, profiles_dir=BaseDbtTest.DBTFINANCE_DIR)
+        dpc = OpenDbtCli(project_dir=BaseDbtTest.DBTCORE_DIR, profiles_dir=BaseDbtTest.DBTCORE_DIR)
+        dpf.invoke(args=["clean"])
+        dpc.invoke(args=["clean"])

tests/resources/airflow/dags/dbt_mesh_workflow.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+
+from airflow import DAG
+from airflow.utils.dates import days_ago
+
+from opendbt.airflow import OpenDbtAirflowProject
+
+default_args = {
+    'owner': 'airflow',
+    'depends_on_past': False,
+    'email_on_failure': False,
+    'email_on_retry': False,
+    'retries': 1
+}
+
+with DAG(
+        dag_id='dbt_mesh_workflow',
+        default_args=default_args,
+        description='DAG To run multiple dbt projects',
+        schedule_interval=None,
+        start_date=days_ago(3),
+        catchup=False,
+        max_active_runs=1
+) as dag:
+    DBT_PROJ_DIR = Path("/opt/dbtfinance")
+
+    p = OpenDbtAirflowProject(project_dir=DBT_PROJ_DIR, profiles_dir=DBT_PROJ_DIR, target='dev')
+    p.load_dbt_tasks(dag=dag, include_singular_tests=True, include_dbt_seeds=True)

tests/resources/airflow/docker-compose.yaml

@@ -16,6 +16,7 @@ services:
       - ./dags:/opt/airflow/dags:rw
       - ./plugins:/opt/airflow/plugins:rw
       - ./../dbtcore:/opt/dbtcore:rw
+      - ./../dbtfinance:/opt/dbtfinance:rw
       - ./../../../opendbt/macros:/opt/dbtcore/macros:rw
     environment:
       - AIRFLOW__WEBSERVER__INSTANCE_NAME=LOCAL

tests/resources/dbtcore/dbt_project.yml

@@ -4,16 +4,15 @@ version: '1.0.0'
 profile: 'dbtcore'
 
 macro-paths: [ "macros", "../../../opendbt/macros/" ]
-snapshot-paths: [ "snapshots" ]
 docs-paths: [ "../../../opendbt/docs/" ]
 
-clean-targets: # directories to be removed by `dbt clean`
+clean-targets:
   - "target"
   - "dbt_packages"
+  - "logs"
 
 models:
   dbtcore:
-    # Config indicated by + and applies to all files under models/example/
     example:
       +materialized: view
 

tests/resources/dbtfinance/dbt_project.yml

@@ -6,4 +6,5 @@ profile: 'dbtfinance'
 # directories to be removed by `dbt clean`
 clean-targets:
   - "target"
-  - "dbt_packages"
+  - "dbt_packages"
+  - "logs"

tests/test_opendbt_cli.py

@@ -41,5 +41,5 @@ def test_cli_run_models(self):
     def test_cli_run_cross_project_ref_models(self):
         dpf = OpenDbtCli(project_dir=self.DBTFINANCE_DIR)
         dpc = OpenDbtCli(project_dir=self.DBTCORE_DIR)
-        dpc.invoke(args=['run', '--select', 'my_core_table1', "--profiles-dir", dpc.project_dir.as_posix()])
         dpf.invoke(args=['run', '--select', 'my_cross_project_ref_model', "--profiles-dir", dpf.project_dir.as_posix()])
+        dpc.invoke(args=['run', '--select', 'my_core_table1', "--profiles-dir", dpc.project_dir.as_posix()])