From d359a8ff5a345bee0f1d3f3d251a2cd669fac20a Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Jun 2023 10:24:42 -0700 Subject: [PATCH 01/28] Use PyHive pure-sasl import --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2cd78c3ac..b06825921 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ def _get_dbt_core_version(): odbc_extras = ["pyodbc~=4.0.30"] pyhive_extras = [ - "PyHive[hive]>=0.6.0,<0.7.0", + "PyHive[hive_pure_sasl]>=0.6.0,<0.7.0", "thrift>=0.11.0,<0.17.0", ] session_extras = ["pyspark>=3.0.0,<4.0.0"] From bb3c688e1186494d27e854d955ad0eacf21a112c Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Jun 2023 10:36:21 -0700 Subject: [PATCH 02/28] Test PyHive pure-sasl import --- .github/workflows/main.yml | 4 ++-- setup.py | 1 + tests/unit/utils.py | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6b3d93b6e..30126325e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -79,7 +79,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] env: TOXENV: "unit" @@ -177,7 +177,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - name: Set up Python ${{ matrix.python-version }} diff --git a/setup.py b/setup.py index b06825921..9d0102d8c 100644 --- a/setup.py +++ b/setup.py @@ -93,6 +93,7 @@ def _get_dbt_core_version(): "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], python_requires=">=3.8", ) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 722f40305..ac8c62244 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -75,7 +75,6 @@ def project_from_dict(project, profile, packages=None, selectors=None, cli_vars= project_root=project_root, project_dict=project, packages_dict=packages, - dependent_projects_dict={}, selectors_dict=selectors, ) return partial.render(renderer) From 44f1b14baa3edf31c1dec39d10a3897cdf2fd554 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Jun 2023 10:42:46 -0700 Subject: [PATCH 03/28] update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4132645ff..907958d89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -PyHive[hive]>=0.6.0,<0.7.0 +PyHive[hive_pure_sasl]>=0.6.0,<0.7.0 requests[python]>=2.28.1 pyodbc~=4.0.30 From 736035328fb1c568792611b122d8571d19016517 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Jun 2023 10:51:00 -0700 Subject: [PATCH 04/28] use github fix for >3.11 --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 907958d89..0f272116c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -PyHive[hive_pure_sasl]>=0.6.0,<0.7.0 +git+https://github.com/dropbox/PyHive.git@4367cc550252f9e6f85782bee7f8694325a742a6#egg=PyHive&subdirectory=pyhive; python_version >= "3.11" +PyHive[sasl]>=0.6.0,<0.7.0; python_version < "3.11" requests[python]>=2.28.1 pyodbc~=4.0.30 From 3779c2d3b7576af0006712da89ca194ca13cb5eb Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Jun 2023 10:58:29 -0700 Subject: [PATCH 05/28] use github fix for >3.11 --- requirements.txt | 2 +- setup.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0f272116c..215c891cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/dropbox/PyHive.git@4367cc550252f9e6f85782bee7f8694325a742a6#egg=PyHive&subdirectory=pyhive; python_version >= "3.11" +git+https://github.com/dropbox/PyHive.git@4367cc550252f9e6f85782bee7f8694325a742a6#egg=PyHive; python_version >= "3.11" PyHive[sasl]>=0.6.0,<0.7.0; python_version < "3.11" requests[python]>=2.28.1 diff --git a/setup.py b/setup.py index 9d0102d8c..f1b234ece 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,11 @@ print("Please upgrade to Python 3.8 or higher.") sys.exit(1) +# sasl does not support python 3.11 and higher +if sys.version_info < (3, 11): + PYHIVE_SASL_EXTRAS = ["sasl"] +else: + PYHIVE_SASL_EXTRAS = ["hive_pure_sasl"] # require version of setuptools that supports find_namespace_packages from setuptools import setup @@ -56,7 +61,7 @@ def _get_dbt_core_version(): odbc_extras = ["pyodbc~=4.0.30"] pyhive_extras = [ - "PyHive[hive_pure_sasl]>=0.6.0,<0.7.0", + f"PyHive[{PYHIVE_SASL_EXTRAS}]>=0.6.0,<0.7.0", "thrift>=0.11.0,<0.17.0", ] session_extras = ["pyspark>=3.0.0,<4.0.0"] From 49825898ccfd907c19a6fed09269a9f7107e3e4d Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Jun 2023 11:10:15 -0700 Subject: [PATCH 06/28] fix pyhive spec --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index f1b234ece..02c2862ba 100644 --- a/setup.py +++ b/setup.py @@ -11,9 +11,9 @@ # sasl does not support python 3.11 and higher if sys.version_info < (3, 11): - PYHIVE_SASL_EXTRAS = ["sasl"] + PYHIVE_SASL_EXTRA = "PyHive[sasl]>=0.6.0,<0.7.0" else: - PYHIVE_SASL_EXTRAS = ["hive_pure_sasl"] + PYHIVE_SASL_EXTRA = "PyHive[hive_pure_sasl]~=0.7.0" # require version of setuptools that supports find_namespace_packages from setuptools import setup @@ -61,7 +61,7 @@ def _get_dbt_core_version(): odbc_extras = ["pyodbc~=4.0.30"] pyhive_extras = [ - f"PyHive[{PYHIVE_SASL_EXTRAS}]>=0.6.0,<0.7.0", + PYHIVE_SASL_EXTRA, "thrift>=0.11.0,<0.17.0", ] session_extras = ["pyspark>=3.0.0,<4.0.0"] From 71030e398a09046b3237c7e5cffbba9b9642f259 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Jun 2023 11:48:09 -0700 Subject: [PATCH 07/28] remove unnecessary requests extra --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 215c891cc..11d7e3269 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -git+https://github.com/dropbox/PyHive.git@4367cc550252f9e6f85782bee7f8694325a742a6#egg=PyHive; python_version >= "3.11" +git+https://github.com/dropbox/PyHive.git@4367cc550252f9e6f85782bee7f8694325a742a6#egg=PyHive;python_version>="3.11" PyHive[sasl]>=0.6.0,<0.7.0; python_version < "3.11" -requests[python]>=2.28.1 +requests>=2.28.1 pyodbc~=4.0.30 sqlparams>=3.0.0 From ddd1f797ab443b2d609edf504129a1606d55137e Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Jun 2023 12:12:04 -0700 Subject: [PATCH 08/28] remove unnecessary sasl install --- dev-requirements.txt | 1 - setup.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index fd8c68131..ca10a2f38 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -30,5 +30,4 @@ wheel~=0.40 # Adapter specific dependencies mock~=5.0 -sasl~=0.3.1 thrift_sasl~=0.4.3 diff --git a/setup.py b/setup.py index 02c2862ba..e7a3aabb7 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,7 @@ if sys.version_info < (3, 11): PYHIVE_SASL_EXTRA = "PyHive[sasl]>=0.6.0,<0.7.0" else: + print("Using pure_sasl for python 3.11 and higher`") PYHIVE_SASL_EXTRA = "PyHive[hive_pure_sasl]~=0.7.0" # require version of setuptools that supports find_namespace_packages @@ -59,7 +60,7 @@ def _get_dbt_core_version(): dbt_core_version = _get_dbt_core_version() description = """The Apache Spark adapter plugin for dbt""" -odbc_extras = ["pyodbc~=4.0.30"] +odbc_extras = ["pyodbc~=4.0.39"] pyhive_extras = [ PYHIVE_SASL_EXTRA, "thrift>=0.11.0,<0.17.0", From bcb47f597398b95ad31ad6a15cc3310ad4ac7180 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Jun 2023 12:13:58 -0700 Subject: [PATCH 09/28] add changie --- .changes/unreleased/Dependencies-20230628-121341.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Dependencies-20230628-121341.yaml diff --git a/.changes/unreleased/Dependencies-20230628-121341.yaml b/.changes/unreleased/Dependencies-20230628-121341.yaml new file mode 100644 index 000000000..89e5f3096 --- /dev/null +++ b/.changes/unreleased/Dependencies-20230628-121341.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: Use PyHive pure-sasl import for >=3.11 +time: 2023-06-28T12:13:41.141588-07:00 +custom: + Author: colin-rogers-dbt + PR: "818" From b5b2b20260848e6177cec0f6eacfddfdda915dc6 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Jun 2023 13:23:21 -0700 Subject: [PATCH 10/28] specify no-binary for pyodbc install --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 11d7e3269..00ffc1339 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ git+https://github.com/dropbox/PyHive.git@4367cc550252f9e6f85782bee7f8694325a742 PyHive[sasl]>=0.6.0,<0.7.0; python_version < "3.11" requests>=2.28.1 -pyodbc~=4.0.30 +pyodbc~=4.0.30 --no-binary pyodbc sqlparams>=3.0.0 thrift>=0.13.0 sqlparse>=0.4.2 # not directly required, pinned by Snyk to avoid a vulnerability From 152a2eebba00ab369b55140ea8e65de9b0ea247c Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Jun 2023 15:39:54 -0700 Subject: [PATCH 11/28] conditionally import sasl vs puresasl --- dbt/adapters/spark/connections.py | 6 +++++- dev-requirements.txt | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/spark/connections.py b/dbt/adapters/spark/connections.py index 2a7f8188d..355441fc6 100644 --- a/dbt/adapters/spark/connections.py +++ b/dbt/adapters/spark/connections.py @@ -1,3 +1,4 @@ +import sys from contextlib import contextmanager import dbt.exceptions @@ -31,8 +32,11 @@ from thrift.transport.TSSLSocket import TSSLSocket import thrift import ssl - import sasl import thrift_sasl + if sys.version_info < (3, 11): + import sasl + else: + import puresasl as sasl except ImportError: pass # done deliberately: setting modules to None explicitly violates MyPy contracts by degrading type semantics diff --git a/dev-requirements.txt b/dev-requirements.txt index ca10a2f38..00cd3d5c2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -31,3 +31,5 @@ wheel~=0.40 # Adapter specific dependencies mock~=5.0 thrift_sasl~=0.4.3 +sasl~=0.3.1;python_version<"3.11" +puresasl;python_version>="3.11" From 0a51daa5cdd4016937e793816149ba262e65899b Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Jun 2023 15:45:30 -0700 Subject: [PATCH 12/28] conditionally import sasl vs pure-sasl --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 00cd3d5c2..ece1bbf83 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -32,4 +32,4 @@ wheel~=0.40 mock~=5.0 thrift_sasl~=0.4.3 sasl~=0.3.1;python_version<"3.11" -puresasl;python_version>="3.11" +pure-sasl;python_version>="3.11" From c80e599baa579dc8fdbd4367a783c65b44049cd4 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 29 Jun 2023 13:05:08 -0700 Subject: [PATCH 13/28] experiment with pure sasl SASLClient --- dbt/adapters/spark/connections.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt/adapters/spark/connections.py b/dbt/adapters/spark/connections.py index 355441fc6..06e39d51f 100644 --- a/dbt/adapters/spark/connections.py +++ b/dbt/adapters/spark/connections.py @@ -34,9 +34,9 @@ import ssl import thrift_sasl if sys.version_info < (3, 11): - import sasl + from sasl import Client as SASLClient else: - import puresasl as sasl + from puresasl.client import SASLClient except ImportError: pass # done deliberately: setting modules to None explicitly violates MyPy contracts by degrading type semantics @@ -552,8 +552,8 @@ def build_ssl_transport( # to be nonempty. password = "x" - def sasl_factory() -> sasl.Client: - sasl_client = sasl.Client() + def sasl_factory() -> SASLClient: + sasl_client = SASLClient() sasl_client.setAttr("host", host) if sasl_auth == "GSSAPI": sasl_client.setAttr("service", kerberos_service_name) From 50a5a54177b62605255cdd264d84257c290c8aae Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 29 Jun 2023 13:12:45 -0700 Subject: [PATCH 14/28] file formatting --- dbt/adapters/spark/connections.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/adapters/spark/connections.py b/dbt/adapters/spark/connections.py index 06e39d51f..f01c658b8 100644 --- a/dbt/adapters/spark/connections.py +++ b/dbt/adapters/spark/connections.py @@ -33,6 +33,7 @@ import thrift import ssl import thrift_sasl + if sys.version_info < (3, 11): from sasl import Client as SASLClient else: From 9e738b11f2f533361845ff4a8c50d55f0cedf63d Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Jun 2023 17:07:57 -0700 Subject: [PATCH 15/28] test using only puresasl --- dbt/adapters/spark/connections.py | 15 ++++----------- dev-requirements.txt | 4 ++-- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/dbt/adapters/spark/connections.py b/dbt/adapters/spark/connections.py index f01c658b8..ef0adf0dc 100644 --- a/dbt/adapters/spark/connections.py +++ b/dbt/adapters/spark/connections.py @@ -33,11 +33,7 @@ import thrift import ssl import thrift_sasl - - if sys.version_info < (3, 11): - from sasl import Client as SASLClient - else: - from puresasl.client import SASLClient + from puresasl.client import SASLClient except ImportError: pass # done deliberately: setting modules to None explicitly violates MyPy contracts by degrading type semantics @@ -554,16 +550,13 @@ def build_ssl_transport( password = "x" def sasl_factory() -> SASLClient: - sasl_client = SASLClient() - sasl_client.setAttr("host", host) if sasl_auth == "GSSAPI": - sasl_client.setAttr("service", kerberos_service_name) + sasl_client = SASLClient(host, kerberos_service_name, mechanism=sasl_auth) elif sasl_auth == "PLAIN": - sasl_client.setAttr("username", username) - sasl_client.setAttr("password", password) + sasl_client = SASLClient(host, mechanism=sasl_auth, + username=username, password=password) else: raise AssertionError - sasl_client.init() return sasl_client transport = thrift_sasl.TSaslClientTransport(sasl_factory, sasl_auth, socket) diff --git a/dev-requirements.txt b/dev-requirements.txt index ece1bbf83..68fd836f8 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -31,5 +31,5 @@ wheel~=0.40 # Adapter specific dependencies mock~=5.0 thrift_sasl~=0.4.3 -sasl~=0.3.1;python_version<"3.11" -pure-sasl;python_version>="3.11" +# sasl~=0.3.1;python_version<"3.11" +pure-sasl #;python_version>="3.11" From 52a2ba7863edba3d379b0f9c97af695f86f71d54 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Jun 2023 17:29:59 -0700 Subject: [PATCH 16/28] test using only puresasl --- setup.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index e7a3aabb7..eb9a871ae 100644 --- a/setup.py +++ b/setup.py @@ -9,12 +9,12 @@ print("Please upgrade to Python 3.8 or higher.") sys.exit(1) -# sasl does not support python 3.11 and higher -if sys.version_info < (3, 11): - PYHIVE_SASL_EXTRA = "PyHive[sasl]>=0.6.0,<0.7.0" -else: - print("Using pure_sasl for python 3.11 and higher`") - PYHIVE_SASL_EXTRA = "PyHive[hive_pure_sasl]~=0.7.0" +# # sasl does not support python 3.11 and higher +# if sys.version_info < (3, 11): +# PYHIVE_SASL_EXTRA = "PyHive[sasl]>=0.6.0,<0.7.0" +# else: +# print("Using pure_sasl for python 3.11 and higher`") +# PYHIVE_SASL_EXTRA = "PyHive[hive_pure_sasl]~=0.7.0" # require version of setuptools that supports find_namespace_packages from setuptools import setup @@ -62,7 +62,7 @@ def _get_dbt_core_version(): odbc_extras = ["pyodbc~=4.0.39"] pyhive_extras = [ - PYHIVE_SASL_EXTRA, + "PyHive[hive_pure_sasl]~=0.7.0", "thrift>=0.11.0,<0.17.0", ] session_extras = ["pyspark>=3.0.0,<4.0.0"] From a03e364854e501b9836886c74199d9b48d61bca5 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Jun 2023 17:30:49 -0700 Subject: [PATCH 17/28] update requirements.txt --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 00ffc1339..e60ca49e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -git+https://github.com/dropbox/PyHive.git@4367cc550252f9e6f85782bee7f8694325a742a6#egg=PyHive;python_version>="3.11" -PyHive[sasl]>=0.6.0,<0.7.0; python_version < "3.11" +git+https://github.com/dropbox/PyHive.git@4367cc550252f9e6f85782bee7f8694325a742a6#egg=PyHive +# PyHive[sasl]>=0.6.0,<0.7.0; python_version < "3.11" requests>=2.28.1 pyodbc~=4.0.30 --no-binary pyodbc From 7987673091efc90c76fcf12793b7d2e210bf5c01 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 5 Jul 2023 08:52:16 -0700 Subject: [PATCH 18/28] remove sasl from req --- dev-requirements.txt | 3 +-- requirements.txt | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 68fd836f8..b4725f4e1 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -31,5 +31,4 @@ wheel~=0.40 # Adapter specific dependencies mock~=5.0 thrift_sasl~=0.4.3 -# sasl~=0.3.1;python_version<"3.11" -pure-sasl #;python_version>="3.11" +pure-sasl diff --git a/requirements.txt b/requirements.txt index e60ca49e4..281e2da86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ git+https://github.com/dropbox/PyHive.git@4367cc550252f9e6f85782bee7f8694325a742a6#egg=PyHive -# PyHive[sasl]>=0.6.0,<0.7.0; python_version < "3.11" requests>=2.28.1 pyodbc~=4.0.30 --no-binary pyodbc From 394197982e38c06e53bfd6bc19614ee238f4bd16 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 5 Jul 2023 10:04:26 -0700 Subject: [PATCH 19/28] remove unused sys dependency --- dbt/adapters/spark/connections.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/spark/connections.py b/dbt/adapters/spark/connections.py index ef0adf0dc..b1d706505 100644 --- a/dbt/adapters/spark/connections.py +++ b/dbt/adapters/spark/connections.py @@ -1,4 +1,3 @@ -import sys from contextlib import contextmanager import dbt.exceptions @@ -553,8 +552,9 @@ def sasl_factory() -> SASLClient: if sasl_auth == "GSSAPI": sasl_client = SASLClient(host, kerberos_service_name, mechanism=sasl_auth) elif sasl_auth == "PLAIN": - sasl_client = SASLClient(host, mechanism=sasl_auth, - username=username, password=password) + sasl_client = SASLClient( + host, mechanism=sasl_auth, username=username, password=password + ) else: raise AssertionError return sasl_client From ef99d3e84d1e8d79cbda3302134624db1f33a39e Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 1 Aug 2023 16:46:14 -0700 Subject: [PATCH 20/28] test spark-session --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 71ca356cf..a12b27461 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -116,9 +116,9 @@ workflows: test-everything: jobs: - unit - # - integration-spark-session: - # requires: - # - unit + - integration-spark-session: + requires: + - unit - integration-spark-thrift: requires: - unit From 645c69558f2b0b7f0f2e7ec8ae353bfff5f65996 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 1 Aug 2023 16:43:28 -0700 Subject: [PATCH 21/28] fix pure-sasl --- dev-requirements.txt | 2 +- requirements.txt | 2 +- setup.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index b4725f4e1..befff907f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -31,4 +31,4 @@ wheel~=0.40 # Adapter specific dependencies mock~=5.0 thrift_sasl~=0.4.3 -pure-sasl +pure-sasl==0.7.1.dev0 --pre diff --git a/requirements.txt b/requirements.txt index 281e2da86..7decc330c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/dropbox/PyHive.git@4367cc550252f9e6f85782bee7f8694325a742a6#egg=PyHive +pure-sasl==0.7.1.dev0 --pre requests>=2.28.1 pyodbc~=4.0.30 --no-binary pyodbc diff --git a/setup.py b/setup.py index 92eb6e199..b6d681ba2 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ def _get_dbt_core_version(): install_requires=[ "dbt-core~={}".format(dbt_core_version), "sqlparams>=3.0.0", + "pure-sasl==0.7.1.dev0 --pre" ], extras_require={ "ODBC": odbc_extras, From 86d9767e1280298bd8d2d47802edc7f789ba28f0 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 3 Aug 2023 12:29:07 -0700 Subject: [PATCH 22/28] allow pre-release installs --- requirements.txt | 2 +- setup.py | 2 +- tox.ini | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7decc330c..e1315bdaf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pure-sasl==0.7.1.dev0 --pre +pyhive[hive_pure_sasl]~=0.7.1.dev0 --pre requests>=2.28.1 pyodbc~=4.0.30 --no-binary pyodbc diff --git a/setup.py b/setup.py index b6d681ba2..be58ab65a 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ def _get_dbt_core_version(): install_requires=[ "dbt-core~={}".format(dbt_core_version), "sqlparams>=3.0.0", - "pure-sasl==0.7.1.dev0 --pre" + "pyhive[hive_pure_sasl]~=0.7.1.dev0" ], extras_require={ "ODBC": odbc_extras, diff --git a/tox.ini b/tox.ini index 97017a926..bf0285c70 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,7 @@ [tox] skipsdist = True envlist = unit, flake8, integration-spark-thrift +pip_pre = true [testenv:{unit,py38,py39,py310,py}] allowlist_externals = From 5ea5c55330304d1355bc2ea677693f6b8d94e5da Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 3 Aug 2023 12:53:25 -0700 Subject: [PATCH 23/28] allow pre-release installs --- tox.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tox.ini b/tox.ini index bf0285c70..2835c34bb 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,7 @@ pip_pre = true allowlist_externals = /bin/bash commands = /bin/bash -c '{envpython} -m pytest -v {posargs} tests/unit' +pip_pre = true passenv = DBT_* PYTEST_ADDOPTS @@ -18,6 +19,7 @@ deps = allowlist_externals = /bin/bash basepython = python3.8 +pip_pre = true commands = /bin/bash -c '{envpython} -m pytest -v --profile databricks_http_cluster {posargs} -n4 tests/functional/adapter/*' passenv = DBT_* @@ -31,6 +33,7 @@ deps = allowlist_externals = /bin/bash basepython = python3.8 +pip_pre = true commands = /bin/bash -c '{envpython} -m pytest -v --profile databricks_cluster {posargs} -n4 tests/functional/adapter/*' passenv = DBT_* @@ -45,6 +48,7 @@ deps = allowlist_externals = /bin/bash basepython = python3.8 +pip_pre = true commands = /bin/bash -c '{envpython} -m pytest -v --profile databricks_sql_endpoint {posargs} -n4 tests/functional/adapter/*' passenv = DBT_* @@ -60,6 +64,7 @@ deps = allowlist_externals = /bin/bash basepython = python3.8 +pip_pre = true commands = /bin/bash -c '{envpython} -m pytest -v --profile apache_spark {posargs} -n4 tests/functional/adapter/*' passenv = DBT_* @@ -73,6 +78,7 @@ deps = allowlist_externals = /bin/bash basepython = python3.10 +pip_pre = true commands = /bin/bash -c '{envpython} -m pytest -v --profile spark_session {posargs} -n4 tests/functional/adapter/*' passenv = DBT_* From cede12fd10334198d1eccd054acea9bd0fc3b823 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 21 Aug 2023 09:26:01 -0700 Subject: [PATCH 24/28] use pyhive 0.7.0 --- dev-requirements.txt | 1 - requirements.txt | 1 - setup.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 88fe6b549..7b875a289 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -31,4 +31,3 @@ wheel~=0.40 # Adapter specific dependencies mock~=5.0 thrift_sasl~=0.4.3 -pure-sasl==0.7.1.dev0 --pre diff --git a/requirements.txt b/requirements.txt index e1315bdaf..66de5c2d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -pyhive[hive_pure_sasl]~=0.7.1.dev0 --pre requests>=2.28.1 pyodbc~=4.0.30 --no-binary pyodbc diff --git a/setup.py b/setup.py index be58ab65a..47cb0fbe8 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ def _get_dbt_core_version(): install_requires=[ "dbt-core~={}".format(dbt_core_version), "sqlparams>=3.0.0", - "pyhive[hive_pure_sasl]~=0.7.1.dev0" + "pyhive[hive_pure_sasl]~=0.7.0" ], extras_require={ "ODBC": odbc_extras, From c88567f2d93b9ff81a1da2b6d820472b01acc7bb Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 21 Aug 2023 09:37:17 -0700 Subject: [PATCH 25/28] add pyhive back to reqs --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index ef4f30cfd..ea5d1ad2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +pyhive[hive_pure_sasl]~=0.7.0 requests>=2.28.1 pyodbc~=4.0.39 From 2851b78f904048eb5798c526a8125b5cc73c5d67 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 21 Aug 2023 10:41:09 -0700 Subject: [PATCH 26/28] fix setup.py and remove sasl from dev-requirements.txt --- dev-requirements.txt | 1 - setup.py | 8 -------- 2 files changed, 9 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 8c50d55ac..cb5ea2cdc 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -30,5 +30,4 @@ wheel~=0.41 # Adapter specific dependencies mock~=5.1 -sasl~=0.3.1 thrift_sasl~=0.4.3 diff --git a/setup.py b/setup.py index 6d6968390..2ac9e63c1 100644 --- a/setup.py +++ b/setup.py @@ -9,13 +9,6 @@ print("Please upgrade to Python 3.8 or higher.") sys.exit(1) -# # sasl does not support python 3.11 and higher -# if sys.version_info < (3, 11): -# PYHIVE_SASL_EXTRA = "PyHive[sasl]>=0.6.0,<0.7.0" -# else: -# print("Using pure_sasl for python 3.11 and higher`") -# PYHIVE_SASL_EXTRA = "PyHive[hive_pure_sasl]~=0.7.0" - # require version of setuptools that supports find_namespace_packages from setuptools import setup @@ -82,7 +75,6 @@ def _get_dbt_core_version(): install_requires=[ "dbt-core~={}".format(dbt_core_version), "sqlparams>=3.0.0", - "pyhive[hive_pure_sasl]~=0.7.0" ], extras_require={ "ODBC": odbc_extras, From 9e0769e42a587cda376f431182125123b1fdb01c Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Mon, 21 Aug 2023 11:00:42 -0700 Subject: [PATCH 27/28] Update Dependencies-20230628-121341.yaml --- .changes/unreleased/Dependencies-20230628-121341.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Dependencies-20230628-121341.yaml b/.changes/unreleased/Dependencies-20230628-121341.yaml index 89e5f3096..5569c885d 100644 --- a/.changes/unreleased/Dependencies-20230628-121341.yaml +++ b/.changes/unreleased/Dependencies-20230628-121341.yaml @@ -1,5 +1,5 @@ kind: Dependencies -body: Use PyHive pure-sasl import for >=3.11 +body: Replace sasl with pure-sasl for PyHive time: 2023-06-28T12:13:41.141588-07:00 custom: Author: colin-rogers-dbt From fc2c610d81129a2daa49e98e490cab383e29e410 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 21 Aug 2023 11:02:17 -0700 Subject: [PATCH 28/28] remove unnecessary pip_pre arg from tox.ini --- tox.ini | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tox.ini b/tox.ini index 2835c34bb..97017a926 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,11 @@ [tox] skipsdist = True envlist = unit, flake8, integration-spark-thrift -pip_pre = true [testenv:{unit,py38,py39,py310,py}] allowlist_externals = /bin/bash commands = /bin/bash -c '{envpython} -m pytest -v {posargs} tests/unit' -pip_pre = true passenv = DBT_* PYTEST_ADDOPTS @@ -19,7 +17,6 @@ deps = allowlist_externals = /bin/bash basepython = python3.8 -pip_pre = true commands = /bin/bash -c '{envpython} -m pytest -v --profile databricks_http_cluster {posargs} -n4 tests/functional/adapter/*' passenv = DBT_* @@ -33,7 +30,6 @@ deps = allowlist_externals = /bin/bash basepython = python3.8 -pip_pre = true commands = /bin/bash -c '{envpython} -m pytest -v --profile databricks_cluster {posargs} -n4 tests/functional/adapter/*' passenv = DBT_* @@ -48,7 +44,6 @@ deps = allowlist_externals = /bin/bash basepython = python3.8 -pip_pre = true commands = /bin/bash -c '{envpython} -m pytest -v --profile databricks_sql_endpoint {posargs} -n4 tests/functional/adapter/*' passenv = DBT_* @@ -64,7 +59,6 @@ deps = allowlist_externals = /bin/bash basepython = python3.8 -pip_pre = true commands = /bin/bash -c '{envpython} -m pytest -v --profile apache_spark {posargs} -n4 tests/functional/adapter/*' passenv = DBT_* @@ -78,7 +72,6 @@ deps = allowlist_externals = /bin/bash basepython = python3.10 -pip_pre = true commands = /bin/bash -c '{envpython} -m pytest -v --profile spark_session {posargs} -n4 tests/functional/adapter/*' passenv = DBT_*