diff --git a/.changes/unreleased/Fixes-20240605-202614.yaml b/.changes/unreleased/Fixes-20240605-202614.yaml new file mode 100644 index 00000000..b7ab8eb0 --- /dev/null +++ b/.changes/unreleased/Fixes-20240605-202614.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Default to psycopg2-binary and allow overriding to psycopg2 via DBT_PSYCOPG2_NAME + (restores previous behavior) +time: 2024-06-05T20:26:14.801254-04:00 +custom: + Author: mikealfare + Issue: "96" diff --git a/.github/scripts/psycopg2-check.sh b/.github/scripts/psycopg2-check.sh new file mode 100755 index 00000000..faee902c --- /dev/null +++ b/.github/scripts/psycopg2-check.sh @@ -0,0 +1,20 @@ +python -m venv venv +source venv/bin/activate +python -m pip install . + +if [[ "$PSYCOPG2_WORKAROUND" == true ]]; then + if [[ $(pip show psycopg2-binary) ]]; then + PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) + pip uninstall -y psycopg2-binary + pip install psycopg2==$PSYCOPG2_VERSION + fi +fi + +PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2) +if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then + echo -e 'Expected: "$PSYCOPG2_EXPECTED_NAME" but found: "$PSYCOPG2_NAME"' + exit 1 +fi + +deactivate +rm -r ./venv diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 1aefb7f5..d7367190 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -45,7 +45,7 @@ defaults: jobs: integration: name: Integration Tests - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -102,24 +102,41 @@ jobs: psycopg2-check: name: "Test psycopg2 build version" - runs-on: ${{ matrix.scenario.platform }} + runs-on: ${{ matrix.platform }} strategy: fail-fast: false matrix: - scenario: - - {platform: ubuntu-latest, psycopg2-name: psycopg2} - - {platform: macos-12, psycopg2-name: psycopg2-binary} + platform: [ubuntu-22.04, macos-12] + python-version: ["3.8", "3.11"] steps: - name: "Check out repository" uses: actions/checkout@v4 - - name: "Test psycopg2 name" - run: | - python -m pip install . - PSYCOPG2_PIP_ENTRY=$(pip list | grep "psycopg2 " || pip list | grep psycopg2-binary) - echo $PSYCOPG2_PIP_ENTRY - PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" - echo $PSYCOPG2_NAME - if [[ "${PSYCOPG2_NAME}" != "${{ matrix.scenario.psycopg2-name }}" ]]; then - exit 1 - fi + - name: "Set up Python ${{ matrix.python-version }}" + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: "Test psycopg2 name - default" + run: .github/scripts/psycopg2-check.sh + env: + PSYCOPG2_EXPECTED_NAME: psycopg2-binary + + - name: "Test psycopg2 name - invalid override" + run: .github/scripts/psycopg2-check.sh + env: + DBT_PSYCOPG2_NAME: rubber-baby-buggy-bumpers + PSYCOPG2_EXPECTED_NAME: psycopg2-binary + + - name: "Test psycopg2 name - override" + run: .github/scripts/psycopg2-check.sh + env: + DBT_PSYCOPG2_NAME: psycopg2 + PSYCOPG2_EXPECTED_NAME: psycopg2-binary # we have not implemented the hook yet, so this doesn't work + + - name: "Test psycopg2 name - manual override" + # verify that the workaround documented in the `README.md` continues to work + run: .github/scripts/psycopg2-check.sh + env: + PSYCOPG2_WORKAROUND: true + PSYCOPG2_EXPECTED_NAME: psycopg2 diff --git a/.gitignore b/.gitignore index 094ee4a9..b8d4accc 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,9 @@ cython_debug/ # testing artifacts /logs + +# MacOS +.DS_Store + +# vscode +.vscode/ diff --git a/README.md b/README.md index d5b8900a..285f5144 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,20 @@ more information on using dbt with Postgres, consult [the docs](https://docs.get - [Install dbt](https://docs.getdbt.com/docs/installation) - Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) +### `psycopg2-binary` vs. `psycopg2` + +By default, `dbt-postgres` installs `psycopg2-binary`. This is great for development, and even testing, as it does not require any OS dependencies; it's a pre-built wheel. However, building `psycopg2` from source will grant performance improvements that are desired in a production environment. In order to install `psycopg2`, use the following steps: + +```bash +if [[ $(pip show psycopg2-binary) ]]; then + PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) + pip uninstall -y psycopg2-binary + pip install psycopg2==$PSYCOPG2_VERSION +fi +``` + +This ensures the version of `psycopg2` will match that of `psycopg2-binary`. + ## Join the dbt Community - Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/) diff --git a/pyproject.toml b/pyproject.toml index a99829d9..92fbf82a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,10 +23,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ - # install `psycopg2` on linux (assumed production) - 'psycopg2>=2.9,<3.0; platform_system == "Linux"', - # install `psycopg2-binary` on macos/windows (assumed development) - 'psycopg2-binary>=2.9,<3.0; platform_system != "Linux"', + "psycopg2-binary>=2.9,<3.0", "dbt-adapters>=0.1.0a1,<2.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0a1", diff --git a/tests/functional/test_selection/conftest.py b/tests/functional/test_selection/conftest.py deleted file mode 100644 index 2faa9e34..00000000 --- a/tests/functional/test_selection/conftest.py +++ /dev/null @@ -1,96 +0,0 @@ -from dbt.tests.fixtures.project import write_project_files -import pytest - - -tests__cf_a_b_sql = """ -select * from {{ ref('model_a') }} -cross join {{ ref('model_b') }} -where false -""" - -tests__cf_a_src_sql = """ -select * from {{ ref('model_a') }} -cross join {{ source('my_src', 'my_tbl') }} -where false -""" - -tests__just_a_sql = """ -{{ config(tags = ['data_test_tag']) }} - -select * from {{ ref('model_a') }} -where false -""" - -models__schema_yml = """ -version: 2 - -sources: - - name: my_src - schema: "{{ target.schema }}" - tables: - - name: my_tbl - identifier: model_b - columns: - - name: fun - data_tests: - - unique - -models: - - name: model_a - columns: - - name: fun - tags: [column_level_tag] - data_tests: - - unique - - relationships: - to: ref('model_b') - field: fun - tags: [test_level_tag] - - relationships: - to: source('my_src', 'my_tbl') - field: fun -""" - -models__model_b_sql = """ -{{ config( - tags = ['a_or_b'] -) }} - -select 1 as fun -""" - -models__model_a_sql = """ -{{ config( - tags = ['a_or_b'] -) }} - -select * FROM {{ref('model_b')}} -""" - - -@pytest.fixture(scope="class") -def tests(): - return { - "cf_a_b.sql": tests__cf_a_b_sql, - "cf_a_src.sql": tests__cf_a_src_sql, - "just_a.sql": tests__just_a_sql, - } - - -@pytest.fixture(scope="class") -def models(): - return { - "schema.yml": models__schema_yml, - "model_b.sql": models__model_b_sql, - "model_a.sql": models__model_a_sql, - } - - -@pytest.fixture(scope="class") -def project_files( - project_root, - tests, - models, -): - write_project_files(project_root, "tests", tests) - write_project_files(project_root, "models", models) diff --git a/tests/functional/test_selection/test_selection_expansion.py b/tests/functional/test_selection/test_selection_expansion.py deleted file mode 100644 index d17f27d7..00000000 --- a/tests/functional/test_selection/test_selection_expansion.py +++ /dev/null @@ -1,567 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - - -class TestSelectionExpansion: - @pytest.fixture(scope="class") - def project_config_update(self): - return {"config-version": 2, "test-paths": ["tests"]} - - def list_tests_and_assert( - self, - include, - exclude, - expected_tests, - indirect_selection="eager", - selector_name=None, - ): - list_args = ["ls", "--resource-type", "test"] - if include: - list_args.extend(("--select", include)) - if exclude: - list_args.extend(("--exclude", exclude)) - if indirect_selection: - list_args.extend(("--indirect-selection", indirect_selection)) - if selector_name: - list_args.extend(("--selector", selector_name)) - - listed = run_dbt(list_args) - assert len(listed) == len(expected_tests) - - test_names = [name.split(".")[-1] for name in listed] - assert sorted(test_names) == sorted(expected_tests) - - def run_tests_and_assert( - self, - include, - exclude, - expected_tests, - indirect_selection="eager", - selector_name=None, - ): - results = run_dbt(["run"]) - assert len(results) == 2 - - test_args = ["test"] - if include: - test_args.extend(("--models", include)) - if exclude: - test_args.extend(("--exclude", exclude)) - if indirect_selection: - test_args.extend(("--indirect-selection", indirect_selection)) - if selector_name: - test_args.extend(("--selector", selector_name)) - - results = run_dbt(test_args) - tests_run = [r.node.name for r in results] - assert len(tests_run) == len(expected_tests) - - assert sorted(tests_run) == sorted(expected_tests) - - def test_all_tests_no_specifiers( - self, - project, - ): - select = None - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "source_unique_my_src_my_tbl_fun", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_alone( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_model_b( - self, - project, - ): - select = "model_a model_b" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "unique_model_a_fun", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_sources( - self, - project, - ): - select = "model_a source:*" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "unique_model_a_fun", - "source_unique_my_src_my_tbl_fun", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_exclude_model_b( - self, - project, - ): - select = None - exclude = "model_b" - expected = [ - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "source_unique_my_src_my_tbl_fun", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_exclude_specific_test( - self, - project, - ): - select = "model_a" - exclude = "unique_model_a_fun" - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_exclude_specific_test_cautious( - self, - project, - ): - select = "model_a" - exclude = "unique_model_a_fun" - expected = ["just_a"] - indirect_selection = "cautious" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_model_a_exclude_specific_test_buildable( - self, - project, - ): - select = "model_a" - exclude = "unique_model_a_fun" - expected = [ - "just_a", - "cf_a_b", - "cf_a_src", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - indirect_selection = "buildable" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_only_generic( - self, - project, - ): - select = "test_type:generic" - exclude = None - expected = [ - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "source_unique_my_src_my_tbl_fun", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_only_singular_unset( - self, - project, - ): - select = "model_a,test_type:singular" - exclude = None - expected = ["cf_a_b", "cf_a_src", "just_a"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_only_singular_eager( - self, - project, - ): - select = "model_a,test_type:singular" - exclude = None - expected = ["cf_a_b", "cf_a_src", "just_a"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_only_singular_cautious( - self, - project, - ): - select = "model_a,test_type:singular" - exclude = None - expected = ["just_a"] - indirect_selection = "cautious" - - self.list_tests_and_assert( - select, exclude, expected, indirect_selection=indirect_selection - ) - self.run_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection) - - def test_only_singular( - self, - project, - ): - select = "test_type:singular" - exclude = None - expected = ["cf_a_b", "cf_a_src", "just_a"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_only_singular( - self, - project, - ): - select = "model_a,test_type:singular" - exclude = None - expected = ["cf_a_b", "cf_a_src", "just_a"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_test_name_intersection( - self, - project, - ): - select = "model_a,test_name:unique" - exclude = None - expected = ["unique_model_a_fun"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_tag_test_name_intersection( - self, - project, - ): - select = "tag:a_or_b,test_name:relationships" - exclude = None - expected = [ - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_select_column_level_tag( - self, - project, - ): - select = "tag:column_level_tag" - exclude = None - expected = [ - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_exclude_column_level_tag( - self, - project, - ): - select = None - exclude = "tag:column_level_tag" - expected = ["cf_a_b", "cf_a_src", "just_a", "source_unique_my_src_my_tbl_fun"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_test_level_tag( - self, - project, - ): - select = "tag:test_level_tag" - exclude = None - expected = ["relationships_model_a_fun__fun__ref_model_b_"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_exclude_data_test_tag( - self, - project, - ): - select = "model_a" - exclude = "tag:data_test_tag" - expected = [ - "cf_a_b", - "cf_a_src", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_indirect_selection( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_indirect_selection_eager( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - indirect_selection = "eager" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_model_a_indirect_selection_cautious( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "just_a", - "unique_model_a_fun", - ] - indirect_selection = "cautious" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_model_a_indirect_selection_buildable( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - indirect_selection = "buildable" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_model_a_indirect_selection_exclude_unique_tests( - self, - project, - ): - select = "model_a" - exclude = "test_name:unique" - indirect_selection = "eager" - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection) - - def test_model_a_indirect_selection_empty(self, project): - results = run_dbt(["ls", "--indirect-selection", "empty", "--select", "model_a"]) - assert len(results) == 1 - - -class TestExpansionWithSelectors(TestSelectionExpansion): - @pytest.fixture(scope="class") - def selectors(self): - return """ - selectors: - - name: model_a_unset_indirect_selection - definition: - method: fqn - value: model_a - - name: model_a_cautious_indirect_selection - definition: - method: fqn - value: model_a - indirect_selection: "cautious" - - name: model_a_eager_indirect_selection - definition: - method: fqn - value: model_a - indirect_selection: "eager" - - name: model_a_buildable_indirect_selection - definition: - method: fqn - value: model_a - indirect_selection: "buildable" - """ - - def test_selector_model_a_unset_indirect_selection( - self, - project, - ): - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_unset_indirect_selection", - ) - self.run_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_unset_indirect_selection", - ) - - def test_selector_model_a_cautious_indirect_selection( - self, - project, - ): - expected = ["just_a", "unique_model_a_fun"] - - self.list_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_cautious_indirect_selection", - ) - self.run_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_cautious_indirect_selection", - ) - - def test_selector_model_a_eager_indirect_selection( - self, - project, - ): - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_eager_indirect_selection", - ) - self.run_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_eager_indirect_selection", - ) - - def test_selector_model_a_buildable_indirect_selection( - self, - project, - ): - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_buildable_indirect_selection", - ) - self.run_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_buildable_indirect_selection", - )