From 3048bfcb274f6d83047e0039148e6470d4cf02c0 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 5 Apr 2024 18:39:41 -0400 Subject: [PATCH 01/37] add docker release to release pipeline --- .github/workflows/release.yml | 21 +++++++++++ docker/Dockerfile | 33 +++++++++++++++++ docker/README.md | 70 +++++++++++++++++++++++++++++++++++ docker/test.sh | 22 +++++++++++ 4 files changed, 146 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100755 docker/test.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 69542465..a10eed6c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,6 +27,9 @@ jobs: url: ${{ vars.PYPI_PROJECT_URL }} permissions: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + outputs: + version: ${{ steps.release-inputs.outputs.version }} + is-docker-release: ${{ steps.semver.outputs.is-pre-release == 0 }} steps: - name: Check out repository @@ -43,6 +46,13 @@ jobs: version=$(hatch version) archive_name=dbt-postgres-$version-${{ inputs.deploy-to }} echo "archive-name=$archive_name" >> $GITHUB_OUTPUT + echo "version=version" >> $GITHUB_OUTPUT + + - name: Audit version to determine if it is a pre-release + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ steps.release-inputs.outputs.version }} - name: Build `dbt-postgres` uses: dbt-labs/dbt-adapters/.github/actions/build-hatch@main @@ -54,3 +64,14 @@ jobs: with: pypi-repository-url: ${{ vars.PYPI_REPOSITORY_URL }} archive-name: ${{ steps.release-inputs.outputs.archive-name }} + + docker-release: + name: "Docker Release" + needs: [release] + if: ${{ needs.release.outputs.is-docker-release }} + permissions: + packages: write + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + with: + package: "dbt-postgres" + version_number: ${{ needs.release.outputs.version }} diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..a87beac3 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,33 @@ +ARG build_for=linux/amd64 + +FROM --platform=$build_for python:3.10.7-slim-bullseye as base + +# ref is updated automatically every final release via bumpversion +ARG dbt_postgres_ref=dbt-postgres@v1.7.10 + +RUN apt-get update \ + && apt-get dist-upgrade -y \ + && apt-get install -y --no-install-recommends \ + git \ + ssh-client \ + software-properties-common \ + make \ + build-essential \ + ca-certificates \ + libpq-dev \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +ENV PYTHONIOENCODING=utf-8 +ENV LANG=C.UTF-8 + +RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir + +WORKDIR /usr/app/dbt/ +ENTRYPOINT ["dbt"] + +FROM base as dbt-postgres +RUN python -m pip install --no-cache-dir "dbt-postgres @ git+https://github.com/dbt-labs/${dbt_postgres_ref}" diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..f9d01513 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,70 @@ +# Docker for dbt +This docker file is suitable for building dbt Docker images locally or using with CI/CD to automate populating a container registry. + + +## Building an image: +This Dockerfile can create images for the following target: `dbt-postgres` + +In order to build a new image, run the following docker command. +``` +docker build --tag --target dbt-postgres +``` +--- +> **Note:** Docker must be configured to use [BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/) in order for images to build properly! + +--- + +By default the images will be populated with the most recent release of `dbt-postgres`. If you need to use a different version you can specify it by git ref using the `--build-arg` flag: +``` +docker build --tag \ + --target dbt-postgres \ + --build-arg dbt_postgres_ref= \ + +``` + +### Examples: +To build an image named "my-dbt" that supports Snowflake using the latest releases: +``` +cd dbt-core/docker +docker build --tag my-dbt --target dbt-postgres . +``` + +To build an image named "my-other-dbt" that supports Snowflake using the adapter version 1.0.0b1: +``` +cd dbt-core/docker +docker build \ + --tag my-other-dbt \ + --target dbt-postgres \ + --build-arg dbt_postgres_ref=dbt-postgres@v1.0.0b1 \ + . +``` + +## Special cases +There are a few special cases worth noting: + +* If you need to build against another architecture (linux/arm64 in this example) you can override the `build_for` build arg: +``` +docker build --tag my_dbt \ + --target dbt-postgres \ + --build-arg build_for=linux/arm64 \ + +``` + +Supported architectures can be found in the python docker [dockerhub page](https://hub.docker.com/_/python). + +## Running an image in a container: +The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: +``` +docker run \ + --network=host \ + --mount type=bind,source=path/to/project,target=/usr/app \ + --mount type=bind,source=path/to/profiles.yml,target=/root/.dbt/profiles.yml \ + my-dbt \ + ls +``` +--- +**Notes:** +* Bind-mount sources _must_ be an absolute path +* You may need to make adjustments to the docker networking setting depending on the specifics of your data warehouse/database host. + +--- diff --git a/docker/test.sh b/docker/test.sh new file mode 100755 index 00000000..f7b86352 --- /dev/null +++ b/docker/test.sh @@ -0,0 +1,22 @@ +# - VERY rudimentary test script to run latest + specific branch image builds and test them all by running `--version` +# TODO: create a real test suite + +clear \ +&& echo "\n\n"\ +"########################################\n"\ +"##### Testing dbt-postgres latest #####\n"\ +"########################################\n"\ +&& docker build --tag dbt-postgres \ + --target dbt-postgres \ + docker \ +&& docker run dbt-postgres --version \ +\ +&& echo "\n\n"\ +"#########################################\n"\ +"##### Testing dbt-postgres-1.0.0b1 #####\n"\ +"#########################################\n"\ +&& docker build --tag dbt-postgres-1.0.0b1 \ + --target dbt-postgres \ + --build-arg dbt_postgres_ref=dbt-postgres@v1.0.0b1 \ + docker \ +&& docker run dbt-postgres-1.0.0b1 --version From 4d9292fa00f826bc1106be57bce13b66b6751506 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 5 Apr 2024 18:41:38 -0400 Subject: [PATCH 02/37] changelog --- .changes/unreleased/Under the Hood-20240405-184129.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20240405-184129.yaml diff --git a/.changes/unreleased/Under the Hood-20240405-184129.yaml b/.changes/unreleased/Under the Hood-20240405-184129.yaml new file mode 100644 index 00000000..19768603 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240405-184129.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Automate the Docker release as part of the PyPI release +time: 2024-04-05T18:41:29.303294-04:00 +custom: + Author: mikealfare + Issue: "51" From db3d70e7532fd8188bf3ab48476964099a3b2dab Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 5 Apr 2024 18:57:53 -0400 Subject: [PATCH 03/37] changelog --- docker/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/README.md b/docker/README.md index f9d01513..21deea2c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,7 +6,7 @@ This docker file is suitable for building dbt Docker images locally or using wit This Dockerfile can create images for the following target: `dbt-postgres` In order to build a new image, run the following docker command. -``` +```shell docker build --tag --target dbt-postgres ``` --- @@ -15,7 +15,7 @@ docker build --tag --target dbt-postgres --- By default the images will be populated with the most recent release of `dbt-postgres`. If you need to use a different version you can specify it by git ref using the `--build-arg` flag: -``` +```shell docker build --tag \ --target dbt-postgres \ --build-arg dbt_postgres_ref= \ @@ -24,13 +24,13 @@ docker build --tag \ ### Examples: To build an image named "my-dbt" that supports Snowflake using the latest releases: -``` +```shell cd dbt-core/docker docker build --tag my-dbt --target dbt-postgres . ``` To build an image named "my-other-dbt" that supports Snowflake using the adapter version 1.0.0b1: -``` +```shell cd dbt-core/docker docker build \ --tag my-other-dbt \ @@ -43,7 +43,7 @@ docker build \ There are a few special cases worth noting: * If you need to build against another architecture (linux/arm64 in this example) you can override the `build_for` build arg: -``` +```shell docker build --tag my_dbt \ --target dbt-postgres \ --build-arg build_for=linux/arm64 \ @@ -54,7 +54,7 @@ Supported architectures can be found in the python docker [dockerhub page](https ## Running an image in a container: The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: -``` +```shell docker run \ --network=host \ --mount type=bind,source=path/to/project,target=/usr/app \ From 7f240c0f42ccd4fed10304eccb40a33588c5785d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 10 Apr 2024 18:11:44 -0400 Subject: [PATCH 04/37] update dependabot to include Docker --- .github/dependabot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2a6f3449..28578550 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,3 +6,8 @@ updates: schedule: interval: "daily" rebase-strategy: "disabled" + - package-ecosystem: "docker" + directory: "/docker" + schedule: + interval: "weekly" + rebase-strategy: "disabled" From 938fc07fcced09614e2c7026c2f3a262c4127b30 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 12 Apr 2024 23:30:50 -0400 Subject: [PATCH 05/37] update to align with other docker release pipelines --- .github/workflows/release.yml | 14 ++--- dbt/adapters/postgres/relation.py | 30 ++++++----- docker/Dockerfile | 34 ++++++------ docker/README.md | 20 ++----- docker/dev.Dockerfile | 54 +++++++++++++++++++ docker/test.sh | 2 +- pyproject.toml | 6 +++ .../shared_tests/test_hooks/test_hooks.py | 1 + .../test_simple_seed/test_simple_seed.py | 1 + tests/unit/test_renamed_relations.py | 12 +++-- 10 files changed, 114 insertions(+), 60 deletions(-) create mode 100644 docker/dev.Dockerfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a10eed6c..89f4baa4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,7 +29,6 @@ jobs: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing outputs: version: ${{ steps.release-inputs.outputs.version }} - is-docker-release: ${{ steps.semver.outputs.is-pre-release == 0 }} steps: - name: Check out repository @@ -48,12 +47,6 @@ jobs: echo "archive-name=$archive_name" >> $GITHUB_OUTPUT echo "version=version" >> $GITHUB_OUTPUT - - name: Audit version to determine if it is a pre-release - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ steps.release-inputs.outputs.version }} - - name: Build `dbt-postgres` uses: dbt-labs/dbt-adapters/.github/actions/build-hatch@main with: @@ -68,10 +61,11 @@ jobs: docker-release: name: "Docker Release" needs: [release] - if: ${{ needs.release.outputs.is-docker-release }} + if: ${{ !failure() && !cancelled() }} permissions: packages: write - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release with: - package: "dbt-postgres" + package: ${{ github.event.repository.name }} version_number: ${{ needs.release.outputs.version }} + test_run: ${{ inputs.deploy-to == 'test' }} diff --git a/dbt/adapters/postgres/relation.py b/dbt/adapters/postgres/relation.py index 677b12ac..05d55237 100644 --- a/dbt/adapters/postgres/relation.py +++ b/dbt/adapters/postgres/relation.py @@ -20,19 +20,23 @@ @dataclass(frozen=True, eq=False, repr=False) class PostgresRelation(BaseRelation): - renameable_relations: FrozenSet[RelationType] = field(default_factory=lambda: frozenset( - { - RelationType.View, - RelationType.Table, - RelationType.MaterializedView, - } - )) - replaceable_relations: FrozenSet[RelationType] = field(default_factory=lambda: frozenset( - { - RelationType.View, - RelationType.Table, - } - )) + renameable_relations: FrozenSet[RelationType] = field( + default_factory=lambda: frozenset( + { + RelationType.View, + RelationType.Table, + RelationType.MaterializedView, + } + ) + ) + replaceable_relations: FrozenSet[RelationType] = field( + default_factory=lambda: frozenset( + { + RelationType.View, + RelationType.Table, + } + ) + ) def __post_init__(self): # Check for length of Postgres table/view names. diff --git a/docker/Dockerfile b/docker/Dockerfile index a87beac3..b29058be 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,20 +1,18 @@ -ARG build_for=linux/amd64 +# this image gets published to GHCR for production use +ARG py_version=3.10.7 -FROM --platform=$build_for python:3.10.7-slim-bullseye as base - -# ref is updated automatically every final release via bumpversion -ARG dbt_postgres_ref=dbt-postgres@v1.7.10 +FROM python:$py_version-slim-bullseye as base RUN apt-get update \ && apt-get dist-upgrade -y \ && apt-get install -y --no-install-recommends \ - git \ - ssh-client \ - software-properties-common \ - make \ - build-essential \ - ca-certificates \ - libpq-dev \ + build-essential=12.9 \ + ca-certificates=20210119 \ + git=1:2.30.2-1+deb11u2 \ + libpq-dev=13.14-0+deb11u1 \ + make=4.3-4.1 \ + openssh-client=1:8.4p1-5+deb11u3 \ + software-properties-common=0.96.20.2-2.1 \ && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ @@ -24,10 +22,16 @@ RUN apt-get update \ ENV PYTHONIOENCODING=utf-8 ENV LANG=C.UTF-8 -RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir +RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir + + +FROM base as dbt-postgres + +ARG commit_ref=main + +HEALTHCHECK CMD dbt --version || exit 1 WORKDIR /usr/app/dbt/ ENTRYPOINT ["dbt"] -FROM base as dbt-postgres -RUN python -m pip install --no-cache-dir "dbt-postgres @ git+https://github.com/dbt-labs/${dbt_postgres_ref}" +RUN python -m pip install --no-cache-dir "dbt-postgres @ git+https://github.com/dbt-labs/dbt-postgres@${commit_ref}" diff --git a/docker/README.md b/docker/README.md index 21deea2c..22af3fe9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -14,11 +14,12 @@ docker build --tag --target dbt-postgres --- -By default the images will be populated with the most recent release of `dbt-postgres`. If you need to use a different version you can specify it by git ref using the `--build-arg` flag: +By default the image will be populated with the latest version of `dbt-postgres` on `main`. +If you need to use a different version you can specify it by git ref using the `--build-arg` flag: ```shell docker build --tag \ --target dbt-postgres \ - --build-arg dbt_postgres_ref= \ + --build-arg commit_ref= \ ``` @@ -35,23 +36,10 @@ cd dbt-core/docker docker build \ --tag my-other-dbt \ --target dbt-postgres \ - --build-arg dbt_postgres_ref=dbt-postgres@v1.0.0b1 \ + --build-arg commit_ref=v1.0.0b1 \ . ``` -## Special cases -There are a few special cases worth noting: - -* If you need to build against another architecture (linux/arm64 in this example) you can override the `build_for` build arg: -```shell -docker build --tag my_dbt \ - --target dbt-postgres \ - --build-arg build_for=linux/arm64 \ - -``` - -Supported architectures can be found in the python docker [dockerhub page](https://hub.docker.com/_/python). - ## Running an image in a container: The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal: ```shell diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile new file mode 100644 index 00000000..a7d2eca3 --- /dev/null +++ b/docker/dev.Dockerfile @@ -0,0 +1,54 @@ +# this image does not get published, it is intended for local development only, see `Makefile` for usage +FROM ubuntu:22.04 as base + +# prevent python installation from asking for time zone region +ARG DEBIAN_FRONTEND=noninteractive + +# add python repository +RUN apt-get update \ + && apt-get install -y software-properties-common=0.99.22.9 \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# install python +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential=12.9ubuntu3 \ + git-all=1:2.34.1-1ubuntu1.10 \ + libpq-dev=14.11-0ubuntu0.22.04.1 \ + python3.8=3.8.19-1+jammy1 \ + python3.8-dev=3.8.19-1+jammy1 \ + python3.8-distutils=3.8.19-1+jammy1 \ + python3.8-venv=3.8.19-1+jammy1 \ + python3-pip=22.0.2+dfsg-1ubuntu0.4 \ + python3-wheel=0.37.1-2ubuntu0.22.04.1 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# update the default system interpreter to the newly installed version +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 + +# install python dependencies +RUN python3 -m pip install --upgrade --no-cache-dir "hatch==1.9.1" + + +FROM base as dbt-postgres-dev + +HEALTHCHECK CMD python3 --version || exit 1 + +# send stdout/stderr to terminal +ENV PYTHONUNBUFFERED=1 + +# setup mount for local code +WORKDIR /opt/code +VOLUME /opt/code + +# create a virtual environment +RUN python3 -m venv /opt/venv diff --git a/docker/test.sh b/docker/test.sh index f7b86352..32d25362 100755 --- a/docker/test.sh +++ b/docker/test.sh @@ -17,6 +17,6 @@ clear \ "#########################################\n"\ && docker build --tag dbt-postgres-1.0.0b1 \ --target dbt-postgres \ - --build-arg dbt_postgres_ref=dbt-postgres@v1.0.0b1 \ + --build-arg commit_ref=v1.0.0b1 \ docker \ && docker run dbt-postgres-1.0.0b1 --version diff --git a/pyproject.toml b/pyproject.toml index 996ea46a..0b212bcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,12 @@ dependencies = [ "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", ] +[tool.hatch.envs.default.scripts] +docker-dev = [ + "docker build -f docker/dev.Dockerfile -t dbt-postgres-dev .", + "docker run --rm -it --name dbt-postgres-dev -v $(pwd):/opt/code dbt-postgres-dev", +] +docker-prod = "docker build -f docker/Dockerfile -t dbt-postgres ." [tool.hatch.envs.lint] detached = true diff --git a/tests/functional/shared_tests/test_hooks/test_hooks.py b/tests/functional/shared_tests/test_hooks/test_hooks.py index 84381130..7e832038 100644 --- a/tests/functional/shared_tests/test_hooks/test_hooks.py +++ b/tests/functional/shared_tests/test_hooks/test_hooks.py @@ -2,6 +2,7 @@ This file needs to be in its own directory because it uses a `data` directory. Placing this file in its own directory avoids collisions. """ + from dbt.tests.adapter.hooks.test_model_hooks import ( BasePrePostModelHooks, BaseHookRefs, diff --git a/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py b/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py index cd849788..61664ca9 100644 --- a/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py +++ b/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py @@ -2,6 +2,7 @@ This file needs to be in its own directory because it creates a `data` directory at run time. Placing this file in its own directory avoids collisions. """ + from dbt.tests.adapter.simple_seed.test_seed import ( BaseBasicSeedTests, BaseSeedConfigFullRefreshOn, diff --git a/tests/unit/test_renamed_relations.py b/tests/unit/test_renamed_relations.py index 49900d8e..29bbabf2 100644 --- a/tests/unit/test_renamed_relations.py +++ b/tests/unit/test_renamed_relations.py @@ -9,8 +9,10 @@ def test_renameable_relation(): identifier="my_table", type=RelationType.Table, ) - assert relation.renameable_relations == frozenset({ - RelationType.View, - RelationType.Table, - RelationType.MaterializedView, - }) + assert relation.renameable_relations == frozenset( + { + RelationType.View, + RelationType.Table, + RelationType.MaterializedView, + } + ) From 8ae7fc0ce6c5bae9be98fb35b44997634528de2f Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 15 Apr 2024 10:26:22 -0400 Subject: [PATCH 06/37] removed defaulted input for docker package --- .github/workflows/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 89f4baa4..39f92841 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,6 +66,5 @@ jobs: packages: write uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release with: - package: ${{ github.event.repository.name }} version_number: ${{ needs.release.outputs.version }} test_run: ${{ inputs.deploy-to == 'test' }} From 9862ed670cacc4eff43d8a5c8e632d8d39a9b839 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 15 Apr 2024 17:29:06 -0400 Subject: [PATCH 07/37] point back to main --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39f92841..ade25138 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,7 +64,7 @@ jobs: if: ${{ !failure() && !cancelled() }} permissions: packages: write - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@docker-release + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main with: version_number: ${{ needs.release.outputs.version }} test_run: ${{ inputs.deploy-to == 'test' }} From 6b935b29e03cd8d82d40b2fd9b4b9aa5f3c38b10 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 15 Apr 2024 17:36:15 -0400 Subject: [PATCH 08/37] remove changie entry --- .changes/unreleased/Under the Hood-20240405-184129.yaml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .changes/unreleased/Under the Hood-20240405-184129.yaml diff --git a/.changes/unreleased/Under the Hood-20240405-184129.yaml b/.changes/unreleased/Under the Hood-20240405-184129.yaml deleted file mode 100644 index 19768603..00000000 --- a/.changes/unreleased/Under the Hood-20240405-184129.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Automate the Docker release as part of the PyPI release -time: 2024-04-05T18:41:29.303294-04:00 -custom: - Author: mikealfare - Issue: "51" From 619641ff7e7db748071ab71046091129ef05a9b9 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 16 Apr 2024 00:15:29 -0400 Subject: [PATCH 09/37] add github release --- .github/workflows/release.yml | 79 ++++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ade25138..69c98068 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,6 +10,10 @@ on: - prod - test default: prod + sha: + description: "The last commit sha in the release" + type: string + required: true permissions: read-all @@ -19,7 +23,7 @@ concurrency: cancel-in-progress: true jobs: - release: + release-inputs: name: PyPI - ${{ inputs.deploy-to }} runs-on: ubuntu-latest environment: @@ -28,7 +32,9 @@ jobs: permissions: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing outputs: - version: ${{ steps.release-inputs.outputs.version }} + version: ${{ steps.version.outputs.version }} + archive-name: ${{ steps.archive.outputs.name }} + changelog-path: ${{ steps.changelog.outputs.path }} steps: - name: Check out repository @@ -39,32 +45,85 @@ jobs: - name: Setup `hatch` uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - - name: Inputs - id: release-inputs + - name: Set version + id: version run: | version=$(hatch version) - archive_name=dbt-postgres-$version-${{ inputs.deploy-to }} - echo "archive-name=$archive_name" >> $GITHUB_OUTPUT echo "version=version" >> $GITHUB_OUTPUT + - name: Audit version and parse into parts + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ steps.version.outputs.version }} + + - name: Set archive name + id: archive + run: | + archive_name=dbt-postgres-$version-${{ inputs.deploy-to }} + echo "name=$archive_name" >> $GITHUB_OUTPUT + + - name: Set changelog path + id: changelog + shell: bash + run: | + path=".changes/" + if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] + then + path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" + else + path+="${{ steps.semver.outputs.base-version }}.md" + fi + echo "path=$path" >> $GITHUB_OUTPUT + + pypi-release: + name: PyPI - ${{ inputs.deploy-to }} + runs-on: ubuntu-latest + needs: [release-inputs] + environment: + name: ${{ inputs.deploy-to }} + url: ${{ vars.PYPI_PROJECT_URL }} + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setup `hatch` + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + - name: Build `dbt-postgres` uses: dbt-labs/dbt-adapters/.github/actions/build-hatch@main with: - archive-name: ${{ steps.release-inputs.outputs.archive-name }} + archive-name: ${{ needs.release-inputs.outputs.archive-name }} - name: Publish to PyPI uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main with: pypi-repository-url: ${{ vars.PYPI_REPOSITORY_URL }} - archive-name: ${{ steps.release-inputs.outputs.archive-name }} + archive-name: ${{ needs.release-inputs.outputs.archive-name }} + + github-release: + name: GitHub Release + if: ${{ !failure() && !cancelled() }} + needs: [release-inputs] + uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main + with: + sha: ${{ inputs.sha }} + version_number: ${{ needs.release-inputs.outputs.version }} + changelog_path: ${{ needs.release-inputs.outputs.changelog-path }} + test_run: ${{ inputs.deploy-to == 'test' }} docker-release: name: "Docker Release" - needs: [release] + needs: [release-inputs, github-release] # docker relies on the published tag from github-release if: ${{ !failure() && !cancelled() }} permissions: packages: write uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main with: - version_number: ${{ needs.release.outputs.version }} + version_number: ${{ needs.release-inputs.outputs.version }} test_run: ${{ inputs.deploy-to == 'test' }} From 858e994708335467fe73c388989c522d337b0580 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 16 Apr 2024 00:18:04 -0400 Subject: [PATCH 10/37] add github release --- .github/workflows/release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 69c98068..8943c99c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,8 @@ on: type: string required: true -permissions: read-all +permissions: + contents: write # this is the permission that allows creating a new release # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise concurrency: From 5323d3ffcc4c2334509def94da867d0f44d3ced7 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 16 Apr 2024 00:21:00 -0400 Subject: [PATCH 11/37] add github release --- .github/workflows/release.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8943c99c..8e610562 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,13 +25,8 @@ concurrency: jobs: release-inputs: - name: PyPI - ${{ inputs.deploy-to }} + name: Release inputs runs-on: ubuntu-latest - environment: - name: ${{ inputs.deploy-to }} - url: ${{ vars.PYPI_PROJECT_URL }} - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing outputs: version: ${{ steps.version.outputs.version }} archive-name: ${{ steps.archive.outputs.name }} @@ -78,7 +73,7 @@ jobs: echo "path=$path" >> $GITHUB_OUTPUT pypi-release: - name: PyPI - ${{ inputs.deploy-to }} + name: PyPI release runs-on: ubuntu-latest needs: [release-inputs] environment: @@ -108,7 +103,7 @@ jobs: archive-name: ${{ needs.release-inputs.outputs.archive-name }} github-release: - name: GitHub Release + name: GitHub release if: ${{ !failure() && !cancelled() }} needs: [release-inputs] uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main @@ -119,7 +114,7 @@ jobs: test_run: ${{ inputs.deploy-to == 'test' }} docker-release: - name: "Docker Release" + name: Docker release needs: [release-inputs, github-release] # docker relies on the published tag from github-release if: ${{ !failure() && !cancelled() }} permissions: From 41f665c806b3f67370eedd69076355c1b9cb0253 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 16 Apr 2024 00:22:45 -0400 Subject: [PATCH 12/37] add github release --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8e610562..7f48ccd6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,7 +45,7 @@ jobs: id: version run: | version=$(hatch version) - echo "version=version" >> $GITHUB_OUTPUT + echo "version=$version" >> $GITHUB_OUTPUT - name: Audit version and parse into parts id: semver From 07eb152e944b8a0730af7d6fd144cddff28e0087 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 16 Apr 2024 14:22:34 -0400 Subject: [PATCH 13/37] add release prep workflow --- .github/workflows/release_prep_hatch.yml | 430 +++++++++++++++++++++++ 1 file changed, 430 insertions(+) create mode 100644 .github/workflows/release_prep_hatch.yml diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml new file mode 100644 index 00000000..35b2aa03 --- /dev/null +++ b/.github/workflows/release_prep_hatch.yml @@ -0,0 +1,430 @@ +# **what?** +# Perform the version bump, generate the changelog and run tests. +# +# Inputs: +# branch: The branch that we will release from +# version: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) +# deploy-to: If we are deploying to prod or test, if test then release from branch +# is-nightly-release: Identifier that this is nightly release +# +# Outputs: +# release-sha: The sha that will actually be released. This can differ from the +# input sha if adding a version bump and/or changelog +# changelog-path: Path to the changelog file (ex .changes/1.2.3-rc1.md) +# +# Branching strategy: +# - During execution workflow execution the temp branch will be generated. +# - For normal runs the temp branch will be removed once changes were merged to target branch; +# - For test runs we will keep temp branch and will use it for release; +# Naming strategy: +# - For normal runs: prep-release/${{ inputs.deploy-to}}/${{ inputs.version }}_$GITHUB_RUN_ID +# - For nightly releases: prep-release/nightly-release/${{ inputs.version }}_$GITHUB_RUN_ID +# +# **why?** +# Reusable and consistent GitHub release process. +# +# **when?** +# Call when ready to kick off a build and release +# +# Validation Checks +# +# 1. Bump the version if it has not been bumped +# 2. Generate the changelog (via changie) if there is no markdown file for this version +name: "Release prep" +run-name: "Release prep: Generate changelog and bump ${{ inputs.package }} to ${{ inputs.version }} for release to ${{ inputs.deploy-to }}" +on: + workflow_call: + inputs: + branch: + description: "The branch to release from" + type: string + default: "main" + version: + required: true + type: string + deploy-to: + type: string + default: "prod" + is-nightly-release: + type: boolean + default: false + outputs: + release-sha: + description: "The SHA to be released" + value: ${{ jobs.release-sha.outputs.sha }} + changelog-path: + description: "The path to the changelog from the repo root for this version, e.g. .changes/1.8.0-b1.md" + value: ${{ jobs.release-inputs.outputs.changelog-path }} + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +defaults: + run: + shell: bash + +env: + PYTHON_TARGET_VERSION: 3.11 + NOTIFICATION_PREFIX: "[Release Prep]" + +jobs: + release-inputs: + runs-on: ubuntu-latest + outputs: + changelog-path: ${{ steps.changelog.outputs.path }} + changelog-exists: ${{ steps.changelog.outputs.exists }} + base-version: ${{ steps.semver.outputs.base-version }} + pre-release: ${{ steps.semver.outputs.pre-release }} + is-pre-release: ${{ steps.semver.outputs.is-pre-release }} + version-is-current: ${{ steps.version.outputs.is-current }} + + steps: + - name: "[DEBUG] Log inputs" + run: | + # WORKFLOW INPUTS + echo Branch: ${{ inputs.branch }} + echo Release version: ${{ inputs.version }} + echo Deploy to: ${{ inputs.deploy-to }} + echo Nightly release: ${{ inputs.is-nightly-release }} + # ENVIRONMENT VARIABLES + echo Python version: ${{ env.PYTHON_TARGET_VERSION }} + echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} + + - name: "Checkout ${{ github.repository }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup `hatch`" + uses: ./.github/actions/setup-hatch + + - name: "Parse input version" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ inputs.version }} + + - name: "Audit version" + id: version + run: | + is_current=false + current_version=$(hatch version) + if test "$current_version" = "${{ inputs.version }}" + then + is_current=true + fi + echo "is-current=$is_updated" >> $GITHUB_OUTPUT + + - name: "[INFO] Skip version bump" + if: steps.version.outputs.is-current == 'true' + run: | + title="Skip version bump" + message="The version matches the input version ${{ inputs.version }}, skipping version bump" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Audit changelog" + id: changelog + run: | + path=".changes/" + if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] + then + path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" + else + path+="${{ steps.semver.outputs.base-version }}.md" + fi + echo "path=$path" >> $GITHUB_OUTPUT + + does_exist=false + if test -f $path + then + does_exist=true + fi + echo "exists=$does_exist">> $GITHUB_OUTPUT + + - name: "[INFO] Skip changelog generation" + if: steps.changelog.outputs.exists == 'true' + run: | + title="Skip changelog generation" + message="A changelog already exists at ${{ steps.changelog.outputs.path }}, skipping generating changelog" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + release-branch: + runs-on: ubuntu-latest + needs: release-inputs + if: | + needs.release-inputs.outputs.changelog-exists == 'false' || + needs.release-inputs.outputs.version-is-current == 'false' + outputs: + name: ${{ steps.release-branch.outputs.name }} + + steps: + - name: "Checkout ${{ github.repository }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Set release branch" + id: release-branch + run: | + name="prep-release/" + if [[ ${{ inputs.is-nightly-release }} == true ]] + then + name+="nightly-release/" + else + name+="${{ inputs.deploy-to }}/" + fi + name+="${{ inputs.version }}_$GITHUB_RUN_ID" + echo "name=$name" >> $GITHUB_OUTPUT + + - name: "Create release branch ${{ steps.release-branch.outputs.name }}" + run: | + git checkout -b ${{ steps.release-branch.outputs.name }} + git push -u origin ${{ steps.release-branch.outputs.name }} + + - name: "[INFO] Create release branch" + run: | + title="Create release branch" + message="Create release branch: ${{ steps.release-branch.outputs.name }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + generate-changelog: + runs-on: ubuntu-latest + if: needs.release-inputs.outputs.changelog-exists == 'false' + # only runs if we need to make changes, determined by not skipping release-branch + needs: + - release-inputs + - release-branch + + steps: + - name: "Checkout ${{ github.repository }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v3 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Install `changie`" + run: | + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie + + - name: "Set Core team membership" + id: core-team + uses: dbt-labs/actions/.github/workflows/determine-team-membership.yml@main + with: + github_team: "core-group" + + - name: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + run: | + if [[ ${{ needs.release-inputs.outputs.is-pre-release }} -eq 1 ]] + then + changie batch ${{ needs.release-inputs.outputs.base-version }} \ + --move-dir '${{ needs.release-inputs.outputs.base-version }}' \ + --prerelease ${{ needs.release-inputs.outputs.pre-release }} + elif [[ -d ".changes/${{ needs.release-inputs.outputs.base-version }}" ]] + then + changie batch ${{ needs.release-inputs.outputs.base-version }} \ + --include '${{ needs.release-inputs.outputs.base-version }}' \ + --remove-prereleases + else # releasing a final patch with no pre-releases + changie batch ${{ needs.release-inputs.outputs.base-version }} + fi + changie merge + env: + CHANGIE_CORE_TEAM: ${{ steps.core-team.outputs.team_membership }} + + - name: "Remove trailing whitespace and missing new lines" + # this step will fail on whitespace errors but also correct them + continue-on-error: true + run: hatch run code-quality + + - name: "Commit & push changes" + run: | + git config user.name "$USER" + git config user.email "$EMAIL" + git pull + git add . + git commit -m "$COMMIT_MESSAGE" + git push + env: + USER: "GitHub Build Bot" + EMAIL: "buildbot@fishtownanalytics.com" + COMMIT_MESSAGE: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + + - name: "[INFO] Generated changelog at ${{ needs.release-inputs.outputs.changelog-path }}" + run: | + title="Changelog generation" + if [[ -f ${{ needs.release-inputs.outputs.changelog-path }} ]] + then + message="Generated changelog file successfully" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + else + message="Failed to generate changelog file" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + bump-version: + runs-on: ubuntu-latest + if: needs.release-inputs.outputs.version-is-current == 'false' + # only runs if we need to make changes, determined by not skipping release-branch + needs: + - release-inputs + - release-branch + - generate-changelog + + steps: + - name: "Checkout ${{ github.repository }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v3 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + + - name: "Bump version to ${{ inputs.version }}" + run: hatch version ${{ inputs.version }} + + - name: "Commit & push changes" + run: | + git config user.name "$USER" + git config user.email "$EMAIL" + git pull + git add . + git commit -m "$COMMIT_MESSAGE" + git push + env: + USER: "GitHub Build Bot" + EMAIL: "buildbot@fishtownanalytics.com" + COMMIT_MESSAGE: "Bump version to ${{ inputs.version }}" + + - name: "[INFO] Bumped version to ${{ inputs.version }}" + run: | + title="Version bump" + message="Bumped version to ${{ inputs.version }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + unit-tests: + runs-on: ubuntu-latest + # only run unit tests if we created a release branch and already bumped the version and generated the changelog + if: | + !failure() && !cancelled() && + needs.release-branch.outputs.name != '' + needs: + - release-branch + - generate-changelog + - bump-version + + steps: + - name: "Checkout ${{ github.repository }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v3 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Setup `hatch`" + uses: ./.github/actions/setup-hatch + + - name: "Run unit tests" + run: hatch run unit-tests:all + + integration-tests: + runs-on: ubuntu-latest + # only run integration tests if we created a release branch and already bumped the version and generated the changelog + if: | + !failure() && !cancelled() && + needs.release-branch.outputs.name != '' + needs: + - release-branch + - generate-changelog + - bump-version + + steps: + - name: "Checkout ${{ github.repository }}@${{ needs.release-branch.outputs.name }}" + uses: actions/checkout@v3 + with: + ref: ${{ needs.release-branch.outputs.name }} + + - name: "Set up `hatch`" + uses: ./.github/actions/setup-hatch + + - name: "Run integration tests" + run: hatch run integration-tests:all + + merge-release-branch: + runs-on: ubuntu-latest + needs: + - unit-tests + - integration-tests + - release-branch + - release-inputs + if: | + !failure() && !cancelled() && + needs.release-branch.outputs.name != '' && + inputs.deploy-to == 'prod' + + steps: + - name: "Checkout ${{ github.repository }}" + uses: actions/checkout@v3 + + - name: "Merge changes into ${{ inputs.branch }}" + uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ needs.release-branch.outputs.name }} + target_branch: ${{ inputs.branch }} + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" + + - name: "[INFO] Merge changes into ${{ inputs.branch }}" + run: | + title="Merge changes" + message="Merge ${{ needs.release-branch.outputs.name }} into ${{ inputs.branch }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + release-sha: + runs-on: ubuntu-latest + needs: + - release-branch + - merge-release-branch + if: ${{ !failure() && !cancelled() }} + + # Get the SHA that will be released. + # If the changelog already exists and the version was already current on the input branch, then release from there. + # Otherwise, we generated a changelog and/or did the version bump in this workflow and there is a + # new sha to use from the merge we just did. Grab that here instead. + outputs: + sha: ${{ steps.release-sha.outputs.sha }} + + steps: + - name: "Set release branch" + id: release-branch + run: | + branch="" + if [ ${{ inputs.deploy-to == 'test' }} ] || [ ${{ inputs.is-nightly-release == 'true' }} ] + then + branch=${{ needs.release-branch.outputs.name }} + else + branch="${{ inputs.branch }}" + fi + echo "name=$branch" >> $GITHUB_OUTPUT + + - name: "Checkout ${{ github.repository }}@${{ steps.release-branch.outputs.name }}" + uses: actions/checkout@v3 + with: + ref: ${{ steps.release-branch.outputs.name }} + + - name: "Set release SHA" + id: release-sha + run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + # if this is a real release and a release branch was created, delete it + - name: "Delete release branch: ${{ needs.release-branch.outputs.name }}" + if: ${{ inputs.deploy-to == 'prod' && inputs.is-nightly-release == 'false' && needs.release-branch.outputs.name != '' }} + run: git push origin -d ${{ needs.release-branch.outputs.name }} From 5af9942174a2d1d5be68f99bf87365f8023c6d79 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 16 Apr 2024 15:04:31 -0400 Subject: [PATCH 14/37] add release prep workflow to release workflow --- .github/workflows/release.yml | 112 ++++++++++------------- .github/workflows/release_prep_hatch.yml | 3 + 2 files changed, 50 insertions(+), 65 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7f48ccd6..946b7a44 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,9 +3,17 @@ name: Release on: workflow_dispatch: inputs: + branch: + description: "The branch to release from" + type: string + default: "main" + version: + description: "The version to release" + required: true + type: string deploy-to: + description: "Deploy to test or prod" type: choice - description: Choose where to publish (test/prod) options: - prod - test @@ -24,102 +32,76 @@ concurrency: cancel-in-progress: true jobs: - release-inputs: - name: Release inputs + release-prep: + name: "Release prep: generate changelog, bump version" + uses: ./github/workflow/release_prep_hatch.yml + with: + branch: ${{ inputs.branch }} + version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-to }} + + build-release: + name: "Build release" + needs: release-prep runs-on: ubuntu-latest outputs: - version: ${{ steps.version.outputs.version }} archive-name: ${{ steps.archive.outputs.name }} - changelog-path: ${{ steps.changelog.outputs.path }} - steps: - - name: Check out repository + - name: "Checkout ${{ github.repository }}@${{ inputs.branch }}" uses: actions/checkout@v4 with: + ref: ${{ inputs.branch }} persist-credentials: false - - name: Setup `hatch` + - name: "Setup `hatch`" uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - - name: Set version - id: version - run: | - version=$(hatch version) - echo "version=$version" >> $GITHUB_OUTPUT - - - name: Audit version and parse into parts - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ steps.version.outputs.version }} - - - name: Set archive name + - name: "Set archive name" id: archive run: | - archive_name=dbt-postgres-$version-${{ inputs.deploy-to }} + archive_name=${{ github.repository }}-${{ inputs.version }}-${{ inputs.deploy-to }} echo "name=$archive_name" >> $GITHUB_OUTPUT - - name: Set changelog path - id: changelog - shell: bash - run: | - path=".changes/" - if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] - then - path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" - else - path+="${{ steps.semver.outputs.base-version }}.md" - fi - echo "path=$path" >> $GITHUB_OUTPUT + - name: "Build ${{ github.repository }}" + uses: dbt-labs/dbt-adapters/.github/actions/build-hatch@main + with: + archive-name: ${{ steps.archive.outputs.name }} pypi-release: - name: PyPI release + name: "PyPI release" runs-on: ubuntu-latest - needs: [release-inputs] + needs: build-release environment: name: ${{ inputs.deploy-to }} url: ${{ vars.PYPI_PROJECT_URL }} permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - + id-token: write # this permission is required for trusted publishing steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Setup `hatch` - uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - - - name: Build `dbt-postgres` - uses: dbt-labs/dbt-adapters/.github/actions/build-hatch@main - with: - archive-name: ${{ needs.release-inputs.outputs.archive-name }} - - - name: Publish to PyPI + - name: "Publish to PyPI" uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main with: pypi-repository-url: ${{ vars.PYPI_REPOSITORY_URL }} - archive-name: ${{ needs.release-inputs.outputs.archive-name }} + archive-name: ${{ needs.build-release.outputs.archive-name }} github-release: - name: GitHub release - if: ${{ !failure() && !cancelled() }} - needs: [release-inputs] - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main + name: "GitHub release" + needs: + - build-release + - release-prep + uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main with: - sha: ${{ inputs.sha }} - version_number: ${{ needs.release-inputs.outputs.version }} - changelog_path: ${{ needs.release-inputs.outputs.changelog-path }} + sha: ${{ needs.release-prep.outputs.release-sha }} + version_number: ${{ inputs.version }} + changelog_path: ${{ needs.release-prep.outputs.changelog-path }} test_run: ${{ inputs.deploy-to == 'test' }} + archive_name: ${{ needs.build-release.outputs.archive-name }} docker-release: - name: Docker release - needs: [release-inputs, github-release] # docker relies on the published tag from github-release - if: ${{ !failure() && !cancelled() }} + name: "Docker release" + needs: github-release # docker relies on the published tag from github-release permissions: - packages: write + packages: write # this permission is required for publishing to GHCR uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main with: - version_number: ${{ needs.release-inputs.outputs.version }} + version_number: ${{ inputs.version }} test_run: ${{ inputs.deploy-to == 'test' }} diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index 35b2aa03..c6d48761 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -40,12 +40,15 @@ on: type: string default: "main" version: + description: "The version to release" required: true type: string deploy-to: + description: "Deploy to test or prod" type: string default: "prod" is-nightly-release: + description: "Identify if this is a nightly release" type: boolean default: false outputs: From 94b8d3cdfa1b6bcc14b03381881ae243d75595b9 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 16 Apr 2024 15:20:48 -0400 Subject: [PATCH 15/37] fix core team workflow ref --- .github/workflows/release_prep_hatch.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index c6d48761..cac5fa1c 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -197,6 +197,13 @@ jobs: message="Create release branch: ${{ steps.release-branch.outputs.name }}" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + core-team: + name: "Set Core team membership" + if: needs.release-inputs.outputs.changelog-exists == 'false' + uses: dbt-labs/actions/.github/workflows/determine-team-membership.yml@main + with: + github_team: "core-group" + generate-changelog: runs-on: ubuntu-latest if: needs.release-inputs.outputs.changelog-exists == 'false' @@ -204,6 +211,7 @@ jobs: needs: - release-inputs - release-branch + - core-team steps: - name: "Checkout ${{ github.repository }}@${{ needs.release-branch.outputs.name }}" @@ -219,12 +227,6 @@ jobs: brew tap miniscruff/changie https://github.com/miniscruff/changie brew install changie - - name: "Set Core team membership" - id: core-team - uses: dbt-labs/actions/.github/workflows/determine-team-membership.yml@main - with: - github_team: "core-group" - - name: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}" run: | if [[ ${{ needs.release-inputs.outputs.is-pre-release }} -eq 1 ]] @@ -242,7 +244,7 @@ jobs: fi changie merge env: - CHANGIE_CORE_TEAM: ${{ steps.core-team.outputs.team_membership }} + CHANGIE_CORE_TEAM: ${{ needs.core-team.outputs.team_membership }} - name: "Remove trailing whitespace and missing new lines" # this step will fail on whitespace errors but also correct them From eb8d083375f33277ca19515de67a584406a4ce48 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 16 Apr 2024 15:22:35 -0400 Subject: [PATCH 16/37] fix workflow ref --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 946b7a44..edc739c4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,7 +34,7 @@ concurrency: jobs: release-prep: name: "Release prep: generate changelog, bump version" - uses: ./github/workflow/release_prep_hatch.yml + uses: dbt-labs/dbt-postgres/.github/workflows/release_prep_hatch.yml@config/docker-release with: branch: ${{ inputs.branch }} version: ${{ inputs.version }} From 41f4864c2dfa967ec1a6f4419292c3fe79bf4069 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 16 Apr 2024 15:24:51 -0400 Subject: [PATCH 17/37] pass secrets to release prep workflow --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index edc739c4..5750af4d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,6 +39,7 @@ jobs: branch: ${{ inputs.branch }} version: ${{ inputs.version }} deploy-to: ${{ inputs.deploy-to }} + secrets: inherit build-release: name: "Build release" From f2c63dc121be61ce43c7c2e6026bf65ff607507b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 16 Apr 2024 16:48:44 -0400 Subject: [PATCH 18/37] pass secrets to release prep workflow --- .github/workflows/release_prep_hatch.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index cac5fa1c..7a0a6d95 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -198,7 +198,6 @@ jobs: echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" core-team: - name: "Set Core team membership" if: needs.release-inputs.outputs.changelog-exists == 'false' uses: dbt-labs/actions/.github/workflows/determine-team-membership.yml@main with: From ddb29b654341d54f6dede030c32c04b1d2ad6c51 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 17 Apr 2024 08:26:32 -0500 Subject: [PATCH 19/37] =?UTF-8?q?allow=20for=20only=20docker=20release,=20?= =?UTF-8?q?don=E2=80=99t=20release=20docker=20when=20deploying=20to=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/release.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5750af4d..9538504c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,6 +22,11 @@ on: description: "The last commit sha in the release" type: string required: true + only_docker: + description: "Only release Docker image, skip GitHub & PyPI" + type: boolean + default: false + required: false permissions: contents: write # this is the permission that allows creating a new release @@ -99,6 +104,10 @@ jobs: docker-release: name: "Docker release" + # We cannot release to docker on a test run because it uses the tag in GitHub as + # what we need to release but draft releases don't actually tag the commit so it + # finds nothing to release + if: ${{ !failure() && !cancelled() && (inputs.deploy-to == 'prod' || inputs.only_docker) }} needs: github-release # docker relies on the published tag from github-release permissions: packages: write # this permission is required for publishing to GHCR From 45952e74459eef66c51177ff293d66664a0148ac Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 26 Apr 2024 15:58:06 -0400 Subject: [PATCH 20/37] Update docker/Dockerfile Co-authored-by: Emily Rockman --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b29058be..7c8dc14e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ # this image gets published to GHCR for production use -ARG py_version=3.10.7 +ARG py_version=3.11.2 FROM python:$py_version-slim-bullseye as base From a7497e3b44d31ac47cdb85717f51d23a12363b0b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 09:57:25 -0400 Subject: [PATCH 21/37] sync with main --- .github/workflows/release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9f61e432..9178a9a5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,9 +57,8 @@ jobs: - name: "Checkout ${{ github.repository }}@${{ inputs.branch }}" uses: actions/checkout@v4 with: - ref: ${{ inputs.branch }} - persist-credentials: false ref: "${{ inputs.ref }}" + persist-credentials: false - name: "Setup `hatch`" uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main From 5be5dba55cb0296a671534913fa9973124a65ab2 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 10:27:14 -0400 Subject: [PATCH 22/37] clean up test script --- docker/test.sh | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/docker/test.sh b/docker/test.sh index 32d25362..ecd72b9f 100755 --- a/docker/test.sh +++ b/docker/test.sh @@ -1,22 +1,19 @@ # - VERY rudimentary test script to run latest + specific branch image builds and test them all by running `--version` # TODO: create a real test suite +set -e -clear \ -&& echo "\n\n"\ -"########################################\n"\ -"##### Testing dbt-postgres latest #####\n"\ -"########################################\n"\ -&& docker build --tag dbt-postgres \ - --target dbt-postgres \ - docker \ -&& docker run dbt-postgres --version \ -\ -&& echo "\n\n"\ -"#########################################\n"\ -"##### Testing dbt-postgres-1.0.0b1 #####\n"\ -"#########################################\n"\ -&& docker build --tag dbt-postgres-1.0.0b1 \ - --target dbt-postgres \ - --build-arg commit_ref=v1.0.0b1 \ - docker \ -&& docker run dbt-postgres-1.0.0b1 --version +echo "\n\n" +echo "#######################################" +echo "##### Testing dbt-postgres latest #####" +echo "#######################################" + +docker build --tag dbt-postgres --target dbt-postgres docker +docker run dbt-postgres --version + +echo "\n\n" +echo "########################################" +echo "##### Testing dbt-postgres-1.0.0b1 #####" +echo "########################################" + +docker build --tag dbt-postgres-1.0.0b1 --target dbt-postgres --build-arg commit_ref=v1.0.0b1 docker +docker run dbt-postgres-1.0.0b1 --version From e67d1401beab8ec59c2275463601141c4eeca837 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 10:38:45 -0400 Subject: [PATCH 23/37] pull branch through to release workflow --- .github/workflows/release.yml | 11 +++-------- .github/workflows/release_prep_hatch.yml | 18 +++++++++++------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9178a9a5..6c37e515 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,21 +13,16 @@ on: type: string deploy-to: description: "Deploy to test or prod" - type: choice - options: - - prod - - test + type: environment default: prod ref: description: "The ref (sha or branch name) to use" type: string default: "main" - required: true only_docker: description: "Only release Docker image, skip GitHub & PyPI" type: boolean default: false - required: false permissions: contents: write # this is the permission that allows creating a new release @@ -57,7 +52,7 @@ jobs: - name: "Checkout ${{ github.repository }}@${{ inputs.branch }}" uses: actions/checkout@v4 with: - ref: "${{ inputs.ref }}" + ref: ${{ needs.release-prep.outputs.release-branch }} persist-credentials: false - name: "Setup `hatch`" @@ -87,7 +82,7 @@ jobs: - name: "Publish to PyPI" uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main with: - pypi-repository-url: ${{ vars.PYPI_REPOSITORY_URL }} + repository-url: ${{ vars.PYPI_REPOSITORY_URL }} archive-name: ${{ needs.build-release.outputs.archive-name }} github-release: diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index 7a0a6d95..7f38d2c3 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -52,9 +52,12 @@ on: type: boolean default: false outputs: + release-branch: + description: "The branch to be released from" + value: ${{ jobs.release.outputs.branch }} release-sha: description: "The SHA to be released" - value: ${{ jobs.release-sha.outputs.sha }} + value: ${{ jobs.release.outputs.sha }} changelog-path: description: "The path to the changelog from the repo root for this version, e.g. .changes/1.8.0-b1.md" value: ${{ jobs.release-inputs.outputs.changelog-path }} @@ -392,7 +395,7 @@ jobs: message="Merge ${{ needs.release-branch.outputs.name }} into ${{ inputs.branch }}" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - release-sha: + release: runs-on: ubuntu-latest needs: - release-branch @@ -404,11 +407,12 @@ jobs: # Otherwise, we generated a changelog and/or did the version bump in this workflow and there is a # new sha to use from the merge we just did. Grab that here instead. outputs: + branch: ${{ steps.release-branch.outputs.name }} sha: ${{ steps.release-sha.outputs.sha }} steps: - name: "Set release branch" - id: release-branch + id: branch run: | branch="" if [ ${{ inputs.deploy-to == 'test' }} ] || [ ${{ inputs.is-nightly-release == 'true' }} ] @@ -419,16 +423,16 @@ jobs: fi echo "name=$branch" >> $GITHUB_OUTPUT - - name: "Checkout ${{ github.repository }}@${{ steps.release-branch.outputs.name }}" + - name: "Checkout ${{ github.repository }}@${{ steps.branch.outputs.name }}" uses: actions/checkout@v3 with: - ref: ${{ steps.release-branch.outputs.name }} + ref: ${{ steps.branch.outputs.name }} - name: "Set release SHA" id: release-sha run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT # if this is a real release and a release branch was created, delete it - - name: "Delete release branch: ${{ needs.release-branch.outputs.name }}" + - name: "Delete release branch: ${{ needs.branch.outputs.name }}" if: ${{ inputs.deploy-to == 'prod' && inputs.is-nightly-release == 'false' && needs.release-branch.outputs.name != '' }} - run: git push origin -d ${{ needs.release-branch.outputs.name }} + run: git push origin -d ${{ needs.branch.outputs.name }} From 1ece62d134482443532740b1e10d69ded5746033 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 10:56:32 -0400 Subject: [PATCH 24/37] remove sha from workflow --- .github/workflows/release.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6c37e515..07c926a8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,10 +15,6 @@ on: description: "Deploy to test or prod" type: environment default: prod - ref: - description: "The ref (sha or branch name) to use" - type: string - default: "main" only_docker: description: "Only release Docker image, skip GitHub & PyPI" type: boolean From 3a0fb25e1bb5cc94efa6a230c324714b954eab76 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 12:16:14 -0400 Subject: [PATCH 25/37] fix concurrency id --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 07c926a8..ea4ad7d6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,7 +25,7 @@ permissions: # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}-${{ inputs.deploy-to }} + group: "${{ github.workflow }}-${{ github.event_name }}-${{ inputs.version }}-${{ inputs.deploy-to }}" cancel-in-progress: true jobs: From 4ab3ecdecc66845525703baca51303fcd010991b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 13:41:19 -0400 Subject: [PATCH 26/37] fix action reference --- .github/workflows/release_prep_hatch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index 7f38d2c3..5563209d 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -109,7 +109,7 @@ jobs: ref: ${{ inputs.branch }} - name: "Setup `hatch`" - uses: ./.github/actions/setup-hatch + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - name: "Parse input version" id: semver From 35515ed46d81b048f5e6e38fcbb2da1d391d1639 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 13:56:53 -0400 Subject: [PATCH 27/37] fix action reference --- .github/workflows/release_prep_hatch.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index 5563209d..c556dda0 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -337,7 +337,7 @@ jobs: ref: ${{ needs.release-branch.outputs.name }} - name: "Setup `hatch`" - uses: ./.github/actions/setup-hatch + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - name: "Run unit tests" run: hatch run unit-tests:all @@ -360,7 +360,7 @@ jobs: ref: ${{ needs.release-branch.outputs.name }} - name: "Set up `hatch`" - uses: ./.github/actions/setup-hatch + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - name: "Run integration tests" run: hatch run integration-tests:all From de47a330e56f1b6a98530be3ddf10152b29650da Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 2 May 2024 15:31:55 -0400 Subject: [PATCH 28/37] add postgres setup to integration tests --- .github/workflows/release_prep_hatch.yml | 33 ++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index c556dda0..f4393e0a 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -332,7 +332,7 @@ jobs: steps: - name: "Checkout ${{ github.repository }}@${{ needs.release-branch.outputs.name }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ needs.release-branch.outputs.name }} @@ -353,17 +353,46 @@ jobs: - generate-changelog - bump-version + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - name: "Checkout ${{ github.repository }}@${{ needs.release-branch.outputs.name }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ needs.release-branch.outputs.name }} + - name: Setup postgres + run: psql -f ./scripts/setup_test_database.sql + env: + PGHOST: localhost + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: postgres + PGDATABASE: postgres + - name: "Set up `hatch`" uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - name: "Run integration tests" run: hatch run integration-tests:all + env: + POSTGRES_TEST_HOST: localhost + POSTGRES_TEST_PORT: 5432 + POSTGRES_TEST_USER: root + POSTGRES_TEST_PASS: password + POSTGRES_TEST_DATABASE: dbt + POSTGRES_TEST_THREADS: 4 merge-release-branch: runs-on: ubuntu-latest From e1fd3efe8529ecc82c21ae48951e0c59fc93f85a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 14 May 2024 09:36:00 -0400 Subject: [PATCH 29/37] remove owner portion from repo name --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ea4ad7d6..41919080 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,7 +45,7 @@ jobs: outputs: archive-name: ${{ steps.archive.outputs.name }} steps: - - name: "Checkout ${{ github.repository }}@${{ inputs.branch }}" + - name: "Checkout ${{ github.repository.name }}@${{ inputs.branch }}" uses: actions/checkout@v4 with: ref: ${{ needs.release-prep.outputs.release-branch }} @@ -57,10 +57,10 @@ jobs: - name: "Set archive name" id: archive run: | - archive_name=${{ github.repository }}-${{ inputs.version }}-${{ inputs.deploy-to }} + archive_name=${{ github.repository.name }}-${{ inputs.version }}-${{ inputs.deploy-to }} echo "name=$archive_name" >> $GITHUB_OUTPUT - - name: "Build ${{ github.repository }}" + - name: "Build ${{ github.repository.name }}" uses: dbt-labs/dbt-adapters/.github/actions/build-hatch@main with: archive-name: ${{ steps.archive.outputs.name }} From ebe2b5f1b6efb70796facdb39240f5b0364cc3bb Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 14 May 2024 12:39:20 -0400 Subject: [PATCH 30/37] skip pypi and github release when docker only --- .github/workflows/release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 41919080..696530f3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,6 +67,7 @@ jobs: pypi-release: name: "PyPI release" + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} runs-on: ubuntu-latest needs: build-release environment: @@ -83,6 +84,7 @@ jobs: github-release: name: "GitHub release" + if: ${{ !failure() && !cancelled() && !inputs.only_docker }} needs: - build-release - release-prep From 63c82e13a8377f2f2b637e84a99673d21b449e47 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 14 May 2024 17:36:57 -0400 Subject: [PATCH 31/37] point docker release to dev branch for now --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 696530f3..55c7ad53 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -105,7 +105,7 @@ jobs: needs: github-release # docker relies on the published tag from github-release permissions: packages: write # this permission is required for publishing to GHCR - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@latest-wrangler-new-docker with: version_number: ${{ inputs.version }} test_run: ${{ inputs.deploy-to == 'test' }} From 9cde4cf093621d8a6460e94fcea400d46832aa49 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 14 May 2024 19:05:45 -0400 Subject: [PATCH 32/37] fix release branch logic --- .github/workflows/release_prep_hatch.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index f4393e0a..ef5d38c3 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -403,7 +403,7 @@ jobs: - release-inputs if: | !failure() && !cancelled() && - needs.release-branch.outputs.name != '' && + needs.release-branch.result == 'success' && inputs.deploy-to == 'prod' steps: @@ -442,11 +442,11 @@ jobs: steps: - name: "Set release branch" id: branch + # If a release branch was created and not merged, use the release branch + # Otherwise, use the input branch because either nothing was done, or the changes were merged back in run: | - branch="" - if [ ${{ inputs.deploy-to == 'test' }} ] || [ ${{ inputs.is-nightly-release == 'true' }} ] - then - branch=${{ needs.release-branch.outputs.name }} + if [ ${{ needs.release-branch.result == 'success' }} && ${{ needs.merge-release-branch.result == 'skipped' }} ]; then + branch="${{ needs.release-branch.outputs.name }}" else branch="${{ inputs.branch }}" fi From 3291829cca1aa3ff2c8bc08ca4a63b7657671773 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 14 May 2024 19:43:55 -0400 Subject: [PATCH 33/37] fix repository name reference --- .github/workflows/release.yml | 6 +++--- .github/workflows/release_prep_hatch.yml | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 55c7ad53..283d81de 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,7 +45,7 @@ jobs: outputs: archive-name: ${{ steps.archive.outputs.name }} steps: - - name: "Checkout ${{ github.repository.name }}@${{ inputs.branch }}" + - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" uses: actions/checkout@v4 with: ref: ${{ needs.release-prep.outputs.release-branch }} @@ -57,10 +57,10 @@ jobs: - name: "Set archive name" id: archive run: | - archive_name=${{ github.repository.name }}-${{ inputs.version }}-${{ inputs.deploy-to }} + archive_name=${{ github.event.repository.name }}-${{ inputs.version }}-${{ inputs.deploy-to }} echo "name=$archive_name" >> $GITHUB_OUTPUT - - name: "Build ${{ github.repository.name }}" + - name: "Build ${{ github.event.repository.name }}" uses: dbt-labs/dbt-adapters/.github/actions/build-hatch@main with: archive-name: ${{ steps.archive.outputs.name }} diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index ef5d38c3..a2742ce7 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -103,7 +103,7 @@ jobs: echo Python version: ${{ env.PYTHON_TARGET_VERSION }} echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} - - name: "Checkout ${{ github.repository }}@${{ inputs.branch }}" + - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} @@ -171,7 +171,7 @@ jobs: name: ${{ steps.release-branch.outputs.name }} steps: - - name: "Checkout ${{ github.repository }}@${{ inputs.branch }}" + - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} @@ -216,7 +216,7 @@ jobs: - core-team steps: - - name: "Checkout ${{ github.repository }}@${{ needs.release-branch.outputs.name }}" + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" uses: actions/checkout@v3 with: ref: ${{ needs.release-branch.outputs.name }} @@ -289,7 +289,7 @@ jobs: - generate-changelog steps: - - name: "Checkout ${{ github.repository }}@${{ needs.release-branch.outputs.name }}" + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" uses: actions/checkout@v3 with: ref: ${{ needs.release-branch.outputs.name }} @@ -331,7 +331,7 @@ jobs: - bump-version steps: - - name: "Checkout ${{ github.repository }}@${{ needs.release-branch.outputs.name }}" + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" uses: actions/checkout@v4 with: ref: ${{ needs.release-branch.outputs.name }} @@ -367,7 +367,7 @@ jobs: - 5432:5432 steps: - - name: "Checkout ${{ github.repository }}@${{ needs.release-branch.outputs.name }}" + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}" uses: actions/checkout@v4 with: ref: ${{ needs.release-branch.outputs.name }} @@ -407,7 +407,7 @@ jobs: inputs.deploy-to == 'prod' steps: - - name: "Checkout ${{ github.repository }}" + - name: "Checkout ${{ github.event.repository.name }}" uses: actions/checkout@v3 - name: "Merge changes into ${{ inputs.branch }}" @@ -452,7 +452,7 @@ jobs: fi echo "name=$branch" >> $GITHUB_OUTPUT - - name: "Checkout ${{ github.repository }}@${{ steps.branch.outputs.name }}" + - name: "Checkout ${{ github.event.repository.name }}@${{ steps.branch.outputs.name }}" uses: actions/checkout@v3 with: ref: ${{ steps.branch.outputs.name }} From acc1df4bb085fb9cf3f716144cb3d22cad0d4446 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 14 May 2024 19:50:44 -0400 Subject: [PATCH 34/37] fix branch references --- .github/workflows/release.yml | 2 +- .github/workflows/release_prep_hatch.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 283d81de..54b648ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,7 +45,7 @@ jobs: outputs: archive-name: ${{ steps.archive.outputs.name }} steps: - - name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}" + - name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-prep.outputs.release-branch }}" uses: actions/checkout@v4 with: ref: ${{ needs.release-prep.outputs.release-branch }} diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index a2742ce7..37129c97 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -436,8 +436,8 @@ jobs: # Otherwise, we generated a changelog and/or did the version bump in this workflow and there is a # new sha to use from the merge we just did. Grab that here instead. outputs: - branch: ${{ steps.release-branch.outputs.name }} - sha: ${{ steps.release-sha.outputs.sha }} + branch: ${{ steps.branch.outputs.name }} + sha: ${{ steps.sha.outputs.sha }} steps: - name: "Set release branch" @@ -458,7 +458,7 @@ jobs: ref: ${{ steps.branch.outputs.name }} - name: "Set release SHA" - id: release-sha + id: sha run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT # if this is a real release and a release branch was created, delete it From 6a39cbdc556d564a9761011d12642dd0f8164806 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 15 May 2024 15:35:52 -0400 Subject: [PATCH 35/37] point everything back to main --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 54b648ff..b9bcca48 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,7 +31,7 @@ concurrency: jobs: release-prep: name: "Release prep: generate changelog, bump version" - uses: dbt-labs/dbt-postgres/.github/workflows/release_prep_hatch.yml@config/docker-release + uses: dbt-labs/dbt-postgres/.github/workflows/release_prep_hatch.yml@main with: branch: ${{ inputs.branch }} version: ${{ inputs.version }} @@ -105,7 +105,7 @@ jobs: needs: github-release # docker relies on the published tag from github-release permissions: packages: write # this permission is required for publishing to GHCR - uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@latest-wrangler-new-docker + uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main with: version_number: ${{ inputs.version }} test_run: ${{ inputs.deploy-to == 'test' }} From 1e7ad5b5555dd4cfb5762964c1ac190f3a2f9b50 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 20 May 2024 16:16:44 -0400 Subject: [PATCH 36/37] adding some requested docs --- .github/workflows/release.yml | 4 +++- pyproject.toml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b9bcca48..1139380a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,7 +74,9 @@ jobs: name: ${{ inputs.deploy-to }} url: ${{ vars.PYPI_PROJECT_URL }} permissions: - id-token: write # this permission is required for trusted publishing + # this permission is required for trusted publishing + # see https://github.com/marketplace/actions/pypi-publish + id-token: write steps: - name: "Publish to PyPI" uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main diff --git a/pyproject.toml b/pyproject.toml index 21e304d5..a99829d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ dependencies = [ dev = "pre-commit install" code-quality = "pre-commit run --all-files" docker-dev = [ + "echo Does not support integration testing, only development and unit testing. See issue https://github.com/dbt-labs/dbt-postgres/issues/99", "docker build -f docker/dev.Dockerfile -t dbt-postgres-dev .", "docker run --rm -it --name dbt-postgres-dev -v $(pwd):/opt/code dbt-postgres-dev", ] From 201f3e2ee2d38c9d3c49eb3c3725d03946ed9fd6 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 20 May 2024 17:05:24 -0400 Subject: [PATCH 37/37] remove unused script --- docker/test.sh | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100755 docker/test.sh diff --git a/docker/test.sh b/docker/test.sh deleted file mode 100755 index ecd72b9f..00000000 --- a/docker/test.sh +++ /dev/null @@ -1,19 +0,0 @@ -# - VERY rudimentary test script to run latest + specific branch image builds and test them all by running `--version` -# TODO: create a real test suite -set -e - -echo "\n\n" -echo "#######################################" -echo "##### Testing dbt-postgres latest #####" -echo "#######################################" - -docker build --tag dbt-postgres --target dbt-postgres docker -docker run dbt-postgres --version - -echo "\n\n" -echo "########################################" -echo "##### Testing dbt-postgres-1.0.0b1 #####" -echo "########################################" - -docker build --tag dbt-postgres-1.0.0b1 --target dbt-postgres --build-arg commit_ref=v1.0.0b1 docker -docker run dbt-postgres-1.0.0b1 --version