[Feature] Enable cross project dbt ref support, dbt mesh, multi porject dbt setup (#49)

* Enable multi project references and settings

Revert "Enable multi project references and settings"

This reverts commit 2686a19.

* Enable multi project references and settings

* Enable multi project references and settings

Author: ismailsimsek

opendbt/dbt/__init__.py

@@ -9,13 +9,17 @@ def patch_dbt():
     # ================================================================================================================
     dbt_version = Version(version.get_installed_version().to_version_string(skip_matcher=True))
     if Version("1.6.0") <= dbt_version < Version("1.8.0"):
+        from opendbt.dbt.v17.config.runtime import OpenDbtRuntimeConfig
+        dbt.config.RuntimeConfig = OpenDbtRuntimeConfig
         from opendbt.dbt.v17.task.docs.generate import OpenDbtGenerateTask
         dbt.task.generate.GenerateTask = OpenDbtGenerateTask
         from opendbt.dbt.v17.adapters.factory import OpenDbtAdapterContainer
         dbt.adapters.factory.FACTORY = OpenDbtAdapterContainer()
         from opendbt.dbt.v17.task.run import ModelRunner
         dbt.task.run.ModelRunner = ModelRunner
     elif Version("1.8.0") <= dbt_version < Version("1.10.0"):
+        from opendbt.dbt.v18.config.runtime import OpenDbtRuntimeConfig
+        dbt.config.RuntimeConfig = OpenDbtRuntimeConfig
         from opendbt.dbt.v18.task.docs.generate import OpenDbtGenerateTask
         dbt.task.docs.generate.GenerateTask = OpenDbtGenerateTask
         from opendbt.dbt.v18.adapters.factory import OpenDbtAdapterContainer

opendbt/dbt/v17/config/__init__.py

@@ -0,0 +1,58 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Mapping
+
+from dbt.config import RuntimeConfig
+from dbt.config.project import path_exists, _load_yaml
+from dbt.constants import DEPENDENCIES_FILE_NAME
+from dbt.exceptions import (
+    DbtProjectError, NonUniquePackageNameError,
+)
+from typing_extensions import override
+
+
+def load_yml_dict(file_path):
+    ret = {}
+    if path_exists(file_path):
+        ret = _load_yaml(file_path) or {}
+    return ret
+
+# pylint: disable=too-many-ancestors
+@dataclass
+class OpenDbtRuntimeConfig(RuntimeConfig):
+    def load_dependence_projects(self):
+        dependencies_yml_dict = load_yml_dict(f"{self.project_root}/{DEPENDENCIES_FILE_NAME}")
+
+        if "projects" not in dependencies_yml_dict:
+            return
+
+        projects = dependencies_yml_dict["projects"]
+        project_root_parent = Path(self.project_root).parent
+        for project in projects:
+            path = project_root_parent.joinpath(project['name'])
+            try:
+                project = self.new_project(str(path.as_posix()))
+            except DbtProjectError as e:
+                raise DbtProjectError(
+                    f"Failed to read depending project: {e} \n project path:{path.as_posix()}",
+                    result_type="invalid_project",
+                    path=path,
+                ) from e
+
+            yield project.project_name, project
+
+    @override
+    def load_dependencies(self, base_only=False) -> Mapping[str, "RuntimeConfig"]:
+        # if self.dependencies is None:
+
+        if self.dependencies is None:
+            # this sets self.dependencies variable!
+            self.dependencies = super().load_dependencies(base_only=base_only)
+
+            # additionally load `projects` defined in `dependencies.yml`
+            for project_name, project in self.load_dependence_projects():
+                if project_name in self.dependencies:
+                    raise NonUniquePackageNameError(project_name)
+                self.dependencies[project_name] = project
+
+        return self.dependencies

opendbt/dbt/v17/config/runtime.py

@@ -0,0 +1,52 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Mapping
+
+from dbt.config import RuntimeConfig
+from dbt.config.project import load_yml_dict
+from dbt.constants import DEPENDENCIES_FILE_NAME
+from dbt.exceptions import (
+    DbtProjectError, NonUniquePackageNameError,
+)
+from typing_extensions import override
+
+
+# pylint: disable=too-many-ancestors
+@dataclass
+class OpenDbtRuntimeConfig(RuntimeConfig):
+    def load_dependence_projects(self):
+        dependencies_yml_dict = load_yml_dict(f"{self.project_root}/{DEPENDENCIES_FILE_NAME}")
+
+        if "projects" not in dependencies_yml_dict:
+            return
+
+        projects = dependencies_yml_dict["projects"]
+        project_root_parent = Path(self.project_root).parent
+        for project in projects:
+            path = project_root_parent.joinpath(project['name'])
+            try:
+                project = self.new_project(str(path.as_posix()))
+            except DbtProjectError as e:
+                raise DbtProjectError(
+                    f"Failed to read depending project: {e} \n project path:{path.as_posix()}",
+                    result_type="invalid_project",
+                    path=path,
+                ) from e
+
+            yield project.project_name, project
+
+    @override
+    def load_dependencies(self, base_only=False) -> Mapping[str, "RuntimeConfig"]:
+        # if self.dependencies is None:
+
+        if self.dependencies is None:
+            # this sets self.dependencies variable!
+            self.dependencies = super().load_dependencies(base_only=base_only)
+
+            # additionally load `projects` defined in `dependencies.yml`
+            for project_name, project in self.load_dependence_projects():
+                if project_name in self.dependencies:
+                    raise NonUniquePackageNameError(project_name)
+                self.dependencies[project_name] = project
+
+        return self.dependencies

opendbt/dbt/v18/config/__init__.py

@@ -3,7 +3,7 @@ dbtcore:
     dev:
       type: duckdb
       adapter: my.dbt.custom.OpenAdapterXXX
-      path: dev.duckdb
+      path: ./../dev.duckdb
       threads: 1
 
     prod:

opendbt/dbt/v18/config/runtime.py

@@ -0,0 +1,4 @@
+
+target/
+dbt_packages/
+logs/

tests/resources/dbtcore/profiles.yml

@@ -0,0 +1,9 @@
+name: 'dbtfinance'
+version: '1.0.0'
+
+profile: 'dbtfinance'
+
+# directories to be removed by `dbt clean`
+clean-targets:
+  - "target"
+  - "dbt_packages"

tests/resources/dbtfinance/.gitignore

@@ -0,0 +1,7 @@
+#packages:
+#  - package: dbt-labs/dbt_utils
+#    version: 1.1.1
+
+# case-sensitive and matches the 'name' in the 'dbt_project.yml'
+projects:
+  - name: dbtcore

tests/resources/dbtfinance/dbt_project.yml

@@ -0,0 +1,2 @@
+
+select * from {{ ref('dbtcore', 'my_core_table1') }}