Merge pull request #8 from datacoves/DCV-2609-add-test-cases-for-dbt-…

…1-6-1-7-and-1-8-in-dbt-core-interface

DCV-2609 add test cases for dbt 1.6/7/8

.github/workflows/tests.yml

@@ -13,23 +13,33 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          # dbt-core now supports Python 3.12
+          - { python: "3.12", os: "ubuntu-latest", dbt: "1.7" }
+          - { python: "3.12", os: "ubuntu-latest", dbt: "1.8" }
           # Latest dbts' test matrix on 3.11 (well supported py version) on ubuntu
           - { python: "3.11", os: "ubuntu-latest", dbt: "1.4" }
           - { python: "3.11", os: "ubuntu-latest", dbt: "1.5" }
+          - { python: "3.11", os: "ubuntu-latest", dbt: "1.6" }
+          - { python: "3.11", os: "ubuntu-latest", dbt: "1.7" }
+          - { python: "3.11", os: "ubuntu-latest", dbt: "1.8" }
           # full test matrix on 3.10 (well supported py version) on ubuntu
-          - { python: "3.10", os: "ubuntu-latest", dbt: "1.0" }
           - { python: "3.10", os: "ubuntu-latest", dbt: "1.1" }
           - { python: "3.10", os: "ubuntu-latest", dbt: "1.2" }
           - { python: "3.10", os: "ubuntu-latest", dbt: "1.3" }
           - { python: "3.10", os: "ubuntu-latest", dbt: "1.4" }
           - { python: "3.10", os: "ubuntu-latest", dbt: "1.5" }
+          - { python: "3.10", os: "ubuntu-latest", dbt: "1.6" }
+          - { python: "3.10", os: "ubuntu-latest", dbt: "1.7" }
+          - { python: "3.10", os: "ubuntu-latest", dbt: "1.8" }
           # full test matrix on 3.9 (well supported py version) on ubuntu
-          - { python: "3.9", os: "ubuntu-latest", dbt: "1.0" }
           - { python: "3.9", os: "ubuntu-latest", dbt: "1.1" }
           - { python: "3.9", os: "ubuntu-latest", dbt: "1.2" }
           - { python: "3.9", os: "ubuntu-latest", dbt: "1.3" }
           - { python: "3.9", os: "ubuntu-latest", dbt: "1.4" }
           - { python: "3.9", os: "ubuntu-latest", dbt: "1.5" }
+          - { python: "3.9", os: "ubuntu-latest", dbt: "1.6" }
+          - { python: "3.9", os: "ubuntu-latest", dbt: "1.7" }
+          - { python: "3.9", os: "ubuntu-latest", dbt: "1.8" }
           # stability test on 3.8 (min python ver) with latest dbt
           - { python: "3.8", os: "ubuntu-latest", dbt: "1.5" }
           # stability test on 3.10 with latest dbt on macos (cross platform)

Makefile

@@ -1,7 +1,8 @@
 # Makes it easy to create virtual environments for different versions of dbt
 ADAPTERS = sqlite duckdb
 
-everything: .venv-dbt10/bin/python .venv-dbt11/bin/python .venv-dbt12/bin/python .venv-dbt13/bin/python .venv-dbt14/bin/python .venv-dbt15/bin/python
+everything: .venv-dbt10/bin/python .venv-dbt11/bin/python .venv-dbt12/bin/python .venv-dbt13/bin/python .venv-dbt14/bin/python \
+ .venv-dbt15/bin/python .venv-dbt16/bin/python .venv-dbt17/bin/python .venv-dbt18/bin/python
 .PHONY: everything
 
 .venv-dbt10/bin/python:
@@ -53,8 +54,39 @@ everything: .venv-dbt10/bin/python .venv-dbt11/bin/python .venv-dbt12/bin/python
 		.venv-dbt15/bin/pip install "dbt-$$adapter>=1.4.0,<1.6.0" "dbt-core>=1.5.0,<1.6.0"; \
 	done
 
+.venv-dbt16/bin/python:
+	python -m venv .venv-dbt16
+	.venv-dbt16/bin/pip install --upgrade wheel setuptools pip
+	.venv-dbt16/bin/pip install pytest WebTest .
+	for adapter in $(ADAPTERS); do \
+		.venv-dbt16/bin/pip install "dbt-$$adapter>=1.4.0,<1.7.0"; \
+		.venv-dbt16/bin/pip install "dbt-core>=1.6.0,<1.7.0"; \
+	done
+
+.venv-dbt17/bin/python:
+	python -m venv .venv-dbt17
+	.venv-dbt17/bin/pip install --upgrade wheel setuptools pip
+	.venv-dbt17/bin/pip install pytest WebTest .
+	for adapter in $(ADAPTERS); do \
+		.venv-dbt17/bin/pip install "dbt-$$adapter>=1.4.0,<1.8.0"; \
+		.venv-dbt17/bin/pip install "dbt-core>=1.7.0,<1.8.0"; \
+	done
+
+# SQLITE adapter is not supported in dbt 1.8+ yet: https://github.com/codeforkjeff/dbt-sqlite/issues
+ADAPTERS = duckdb
+
+.venv-dbt18/bin/python:
+	python -m venv .venv-dbt18
+	.venv-dbt18/bin/pip install --upgrade wheel setuptools pip
+	.venv-dbt18/bin/pip install pytest WebTest .
+	for adapter in $(ADAPTERS); do \
+		.venv-dbt18/bin/pip install "dbt-core>=1.8.0,<1.9.0"; \
+		.venv-dbt18/bin/pip install "dbt-$$adapter"; \
+	done
+	.venv-dbt18/bin/pip install --force-reinstall dbt-adapters dbt-common;
+
 clean:
-	rm -rf .venv-dbt10 .venv-dbt11 .venv-dbt12 .venv-dbt13 .venv-dbt14 .venv-dbt15
+	rm -rf .venv-dbt10 .venv-dbt11 .venv-dbt12 .venv-dbt13 .venv-dbt14 .venv-dbt15 .venv-dbt16 .venv-dbt17 .venv-dbt18
 .PHONY: clean
 
 test-dbt1.0: .venv-dbt10/bin/python
@@ -81,5 +113,17 @@ test-dbt1.5: .venv-dbt15/bin/python
 	.venv-dbt15/bin/python -m pytest tests/test_main.py
 .PHONY: test-dbt1.5
 
-test: test-dbt1.0 test-dbt1.1 test-dbt1.2 test-dbt1.3 test-dbt1.4 test-dbt1.5
+test-dbt1.6: .venv-dbt16/bin/python
+	.venv-dbt16/bin/python -m pytest tests/test_main.py
+.PHONY: test-dbt1.6
+
+test-dbt1.7: .venv-dbt17/bin/python
+	.venv-dbt17/bin/python -m pytest tests/test_main.py
+.PHONY: test-dbt1.7
+
+test-dbt1.8: .venv-dbt18/bin/python
+	.venv-dbt18/bin/python -m pytest tests/test_main.py
+.PHONY: test-dbt1.8
+
+test: test-dbt1.0 test-dbt1.1 test-dbt1.2 test-dbt1.3 test-dbt1.4 test-dbt1.5 test-dbt1.6 test-dbt1.7 test-dbt1.8
 .PHONY: test

pyproject.toml

@@ -14,7 +14,7 @@ classifiers = ["Development Status :: 3 - Alpha"]
 Changelog = "https://github.com/z3z1ma/dbt-core-interface/releases"
 
 [tool.poetry.dependencies]
-python = ">=3.8,<3.12"
+python = ">=3.8,<3.13"
 
 [tool.poetry.dev-dependencies]
 Pygments = ">=2.10.0"

src/dbt_core_interface/project.py

@@ -59,6 +59,7 @@
 from io import BytesIO
 from json import dumps as json_dumps
 from json import loads as json_lds
+from multiprocessing import get_context
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from traceback import format_exc, print_exc
@@ -88,7 +89,12 @@
 
 # We maintain the smallest possible surface area of dbt imports
 from dbt.adapters.factory import get_adapter_class_by_name
-from dbt.clients.system import make_directory
+try:
+    # dbt >= 1.8
+    from dbt_common.clients.system import make_directory
+except ImportError:
+    # dbt < 1.8
+    from dbt.clients.system import make_directory
 from dbt.config.runtime import RuntimeConfig
 from dbt.flags import set_from_args
 from dbt.node_types import NodeType
@@ -112,9 +118,16 @@
     # These imports are only used for type checking
     from agate import Table # type: ignore  # No stubs for agate
     from dbt.adapters.base import BaseAdapter, BaseRelation  # type: ignore
-    from dbt.contracts.connection import AdapterResponse
+    try:
+        # dbt >= 1.8
+        from dbt.adapters.contracts.connection import AdapterResponse
+        from dbt_common.semver import VersionSpecifier
+
+    except ImportError:
+        # dbt < 1.8
+        from dbt.contracts.connection import AdapterResponse
+        from dbt.semver import VersionSpecifier
     from dbt.contracts.results import ExecutionResult, RunExecutionResult
-    from dbt.semver import VersionSpecifier
     from dbt.task.runnable import ManifestTask
 
 try:
@@ -128,9 +141,15 @@
 from agate import Table, Number, Text, Column
 
 try:
-    from dbt.clients.agate_helper import Integer
+    # dbt >= 1.8
+    from dbt_common.clients.agate_helper import Integer
 except ImportError:
-    from dbt.clients.agate_helper import Number as Integer
+    try:
+        # dbt < 1.8 and older Agate version
+        from dbt.clients.agate_helper import Integer
+    except ImportError:
+        # dbt < 1.8 and newer Agate version
+        from dbt.clients.agate_helper import Number as Integer
 # dbt-core-interface is designed for non-standard use. There is no
 # reason to track usage of this package.
 disable_tracking()
@@ -266,8 +285,10 @@ class DbtConfiguration:
     use_experimental_parser: bool = False
     static_parser: bool = False
     partial_parse: bool = False
-    # A required attribute for dbt, not used by our interface
+    # required attributes for dbt, not used by our interface
     dependencies: List[str] = field(default_factory=list)
+    REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES: bool = field(default_factory=bool)
+    which: str = "blah"
 
     def __post_init__(self) -> None:
         """Post init hook to set single_threaded and remove target if not provided."""
@@ -474,7 +495,12 @@ def initialize_adapter(self) -> None:
                 LOGGER.debug(f"Failed to cleanup adapter connections: {e}")
         # The adapter.setter verifies connection, resets TTL, and updates adapter ref on config
         # this is thread safe by virtue of the adapter_mutex on the adapter.setter
-        self.adapter = self.get_adapter_cls()(self.config)
+        if (__dbt_major_version__, __dbt_minor_version__) < (1, 8):
+            self.adapter = self.get_adapter_cls()(self.config)
+        else:
+            # from dbt 1.8 adapter decoupling onwwards, 
+            # instantiating an Adapter requires a multiprocessing context.
+            self.adapter = self.get_adapter_cls()(self.config, get_context("spawn"))
 
     @property
     def adapter(self) -> "BaseAdapter":

tests/test_main.py

@@ -21,8 +21,8 @@ def test_list():
     from dbt_core_interface.project import DbtProject
 
     project = DbtProject(
-        project_dir="demo_sqlite",
-        profiles_dir="demo_sqlite",
+        project_dir="demo_duckdb",
+        profiles_dir="demo_duckdb",
         target="dev",
     )
     nodes = project.list("*")
@@ -58,23 +58,23 @@ def test_server():
     client = TestApp(app.default)
 
     SIMULATED_CLIENTS = 50  # noqa: N806
-    DUCKDB_PROJECTS = (  # noqa: N806
-        [
-            "j_shop_1_duckdb",
-            "j_shop_2_duckdb",
-            "h_niceserver_1_duckdb",
-            "h_niceserver_2_duckdb",
-        ]
-        if (__dbt_major_version__, __dbt_minor_version__) < (1, 4)
-        else []  # DuckDB adapter is not supported in dbt 1.4+ yet
-    )
-    SQLITE_PROJECTS = [  # noqa: N806
-        "j_shop_1_sqlite",
-        "j_shop_2_sqlite",
-        "j_shop_3_sqlite",
-        "j_shop_4_sqlite",
-        "h_niceserver_1_sqlite",
+    DUCKDB_PROJECTS = [  # noqa: N806
+        "j_shop_1_duckdb",
+        "j_shop_2_duckdb",
+        "h_niceserver_1_duckdb",
+        "h_niceserver_2_duckdb",
     ]
+    SQLITE_PROJECTS = (
+        [  # noqa: N806
+            "j_shop_1_sqlite",
+            "j_shop_2_sqlite",
+            "j_shop_3_sqlite",
+            "j_shop_4_sqlite",
+            "h_niceserver_1_sqlite",
+        ]
+        if (__dbt_major_version__, __dbt_minor_version__) < (1, 8)
+        else []
+    )  # SQLITE adapter is not supported in dbt 1.8+ yet: https://github.com/codeforkjeff/dbt-sqlite/issues
     PROJECTS = DUCKDB_PROJECTS + SQLITE_PROJECTS  # noqa: N806
 
     for proj in SQLITE_PROJECTS: