From 259319956dc0c823916bb2a6f61b6431e7fc6f93 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 2 Mar 2024 20:12:30 -0500 Subject: [PATCH 01/16] break workflows into actions and smaller workflows, reuse workflows in larger workflows --- .../actions/audit-github-commit/action.yml | 41 +++ .github/actions/audit-github-tag/action.yml | 74 +++++ .github/actions/build-archive-name/action.yml | 28 ++ .github/actions/build-hatch/action.yml | 37 --- .../actions/publish-github-draft/action.yml | 72 +++++ .../actions/publish-github-full/action.yml | 26 ++ .github/actions/publish-pypi/action.yml | 42 ++- .github/actions/publish-results/action.yml | 25 -- .../actions/publish-test-results/action.yml | 48 ++++ .github/actions/setup-hatch/action.yml | 16 +- .github/workflows/build.yml | 94 +++---- .github/workflows/changelog-existence.yml | 7 +- .github/workflows/code-quality.yml | 12 +- .github/workflows/github-release.yml | 259 ------------------ .github/workflows/publish-github.yml | 159 +++++++++++ .github/workflows/publish-pypi.yml | 75 +++++ .github/workflows/release.yml | 130 ++++----- .github/workflows/resubmit-for-triage.yml | 4 +- .github/workflows/unit-tests.yml | 22 +- 19 files changed, 686 insertions(+), 485 deletions(-) create mode 100644 .github/actions/audit-github-commit/action.yml create mode 100644 .github/actions/audit-github-tag/action.yml create mode 100644 .github/actions/build-archive-name/action.yml delete mode 100644 .github/actions/build-hatch/action.yml create mode 100644 .github/actions/publish-github-draft/action.yml create mode 100644 .github/actions/publish-github-full/action.yml delete mode 100644 .github/actions/publish-results/action.yml create mode 100644 .github/actions/publish-test-results/action.yml delete mode 100644 .github/workflows/github-release.yml create mode 100644 .github/workflows/publish-github.yml create mode 100644 .github/workflows/publish-pypi.yml diff --git a/.github/actions/audit-github-commit/action.yml b/.github/actions/audit-github-commit/action.yml new file mode 100644 index 00000000..10a87033 --- /dev/null +++ b/.github/actions/audit-github-commit/action.yml @@ -0,0 +1,41 @@ +name: "Audit GitHub commit" +description: "Get basic metadata associated with a commit" + +inputs: + sha: + description: "The commit to audit" + required: true +outputs: + tag: + description: "The associated release tag, if the release exists" + value: ${{ steps.commit.outputs.tag_name }} + release-id: + description: "The associated release id, if the release exists" + value: ${{ steps.commit.outputs.id }} + pre-release: + description: "If the associated release exists, is it a pre-release?" + value: ${{ steps.commit.outputs.prerelease }} + commitish: + description: "The associated commitish, if the release exists" + value: ${{ steps.commit.outputs.target_commitish }} + +runs: + using: composite + steps: + - name: "Check if a release exists for this commit" + id: commit + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: cardinalby/git-get-release-action@1.2.4 + with: + commitSha: ${{ inputs.sha }} + doNotFailIfNotFound: true # returns blank outputs when not found instead of error + searchLimit: 15 # Since we only care about recent releases, speed up the process + + - name: "[DEBUG] Print Release Details" + shell: bash + run: | + echo tag : ${{ steps.check_release_commit.outputs.tag_name }} + echo release-id : ${{ steps.check_release_commit.outputs.id }} + echo pre-release : ${{ steps.check_release_commit.outputs.prerelease }} + echo commitish : ${{ steps.check_release_commit.outputs.target_commitish }} diff --git a/.github/actions/audit-github-tag/action.yml b/.github/actions/audit-github-tag/action.yml new file mode 100644 index 00000000..6b8c747f --- /dev/null +++ b/.github/actions/audit-github-tag/action.yml @@ -0,0 +1,74 @@ +# When the GitHub CLI doesn't find a release for the given tag, it will exit 1 with a +# message of "release not found". In our case, it's not an actual error, just a +# confirmation that the release does not already exist, so we can create it. +# The `|| true` makes it so the step does not exit with a non-zero exit code. +# Also check if the release already exists is draft state. If it's a draft, +# and we are not testing, then we can publish that draft as-is. If it's in draft, +# and we are testing, skip the release. +name: "Audit GitHub tag" +description: "Get basic metadata associated with a release tag" + +inputs: + tag: + description: "The tag to audit (i.e. v1.0.0b1)" + required: true + repo-url: + description: "The URL to the repo (https://github.com/dbt-labs/dbt-adapters" + required: true +outputs: + exists: + description: "Does the tag exist?" + value: ${{ steps.tag.outputs.exists }} + sha: + description: "The commit associated with the tag if the tag exists" + value: ${{ steps.commit.outputs.sha }} + is-draft: + description: "If the tag exists, is it a draft?" + value: ${{ steps.draft.outputs.is-draft }} + +runs: + using: composite + steps: + - name: "Check if tag exists" + id: tag + shell: bash + run: | + output=$((gh release view ${{ inputs.tag }} --json isDraft,targetCommitish --repo ${{ inputs.repo-url }}) 2>&1) || true + echo "results=$output" >> $GITHUB_OUTPUT + if [[ "$output" == "release not found" ]] + then + echo "exists=false" >> $GITHUB_OUTPUT + else + echo "exists=true" >> $GITHUB_OUTPUT + fi + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: "Get commit associate with the tag" + id: commit + if: ${{ fromJSON(steps.tag.outputs.exists) == true }} + shell: bash + run: | + sha=$(jq -r '.targetCommitish' <<< "$output") + echo "sha=$sha" >> $GITHUB_OUTPUT + + - name: "Check if tag is a draft release" + id: draft + if: ${{ fromJSON(steps.tag.outputs.exists) == true }} + shell: bash + run: | + is-draft=$(jq -r '.isDraft' <<< "${{ steps.tag.outputs.results }}") + if [[ $is-draft == true ]] + then + echo "is-draft=true" >> $GITHUB_OUTPUT + else + echo "is-draft=false" >> $GITHUB_OUTPUT + fi + + - name: "[DEBUG] Tag metadata" + shell: bash + run: | + echo tag : ${{ inputs.tag }} + echo exists: : ${{ steps.release.outputs.exists }} + echo sha : ${{ steps.commit.outputs.sha }} + echo is-draft : ${{ steps.draft.outputs.is-draft }} diff --git a/.github/actions/build-archive-name/action.yml b/.github/actions/build-archive-name/action.yml new file mode 100644 index 00000000..d51f4fcb --- /dev/null +++ b/.github/actions/build-archive-name/action.yml @@ -0,0 +1,28 @@ +name: "Build" +description: "Provide a standard archive name for build artifacts" + +inputs: + package: + description: "Choose what to publish" + required: true + version: + description: "The release version number (i.e. 1.0.0b1)" + required: true + deploy-to: + description: "Choose where to publish" + default: prod +outputs: + archive-name: + description: "A unique archive name" + value: ${{ steps.archive.outputs.archive-name }} + +runs: + using: composite + + steps: + - name: "Archive name" + id: archive + shell: bash + run: | + archive_name=${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} + echo "archive-name=$archive_name" >> $GITHUB_OUTPUT diff --git a/.github/actions/build-hatch/action.yml b/.github/actions/build-hatch/action.yml deleted file mode 100644 index fe9825d4..00000000 --- a/.github/actions/build-hatch/action.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Build - `hatch` -description: Build artifacts using the `hatch` build backend - -inputs: - build-command: - description: The command to build distributable artifacts - default: "hatch build" - check-command: - description: The command to check built artifacts - default: "hatch run build:check-all" - working-dir: - description: Where to run commands from, supports namespace packaging - default: "./" - archive-name: - description: Where to upload the artifacts - required: true - -runs: - using: composite - steps: - - - name: Build artifacts - run: ${{ inputs.build-command }} - shell: bash - working-directory: ${{ inputs.working-dir }} - - - name: Check artifacts - run: ${{ inputs.check-command }} - shell: bash - working-directory: ${{ inputs.working-dir }} - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.archive-name }} - path: ${{ inputs.working-dir }}dist/ - retention-days: 3 diff --git a/.github/actions/publish-github-draft/action.yml b/.github/actions/publish-github-draft/action.yml new file mode 100644 index 00000000..0a1f77cf --- /dev/null +++ b/.github/actions/publish-github-draft/action.yml @@ -0,0 +1,72 @@ +name: "Publish to GitHub as draft release" +description: "Publish artifacts from an archive to GitHub as a draft release" + +inputs: + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local" + default: "" + tag: + description: "The release tag to publish under" + required: true + repo-url: + description: "The URL to the repo (https://github.com/dbt-labs/dbt-adapters" + required: true + sha: + description: "Commit SHA being released" + required: true + changelog-path: + description: "Path to the release notes" + required: true + +runs: + using: composite + steps: + + - name: "Download artifacts" + if: ${{ !(inputs.archive-name == "" }} + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.archive-name }} + path: dist/ + + - name: "[INFO] Downloaded artifacts" + if: ${{ !(inputs.archive-name == "" }} + shell: bash + run: | + title="Downloaded artifacts" + message="Downloaded artifacts from ${{ inputs.archive-name }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Found artifacts" + shell: bash + working-directory: ${{ inputs.working-dir }} + run: echo $(ls ./dist) + + - name: "Set: pre-release" + id: pre-release + shell: bash + run: | + if ${{ contains(inputs.tag, 'rc') || contains(inputs.tag, 'b') || contains(inputs.tag, 'a') }} + then + echo "pre-release=--prerelease" >> $GITHUB_OUTPUT + fi + + - name: "Publish artifacts to GitHub as draft release" + shell: bash + run: | + gh release create $TAG ./dist/* --title "$TITLE" --notes-file $RELEASE_NOTES --target $COMMIT $PRERELEASE --draft --repo ${{ inputs.repo-url }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ inputs.tag }} + TITLE: ${{ github.event.repository.name }} ${{ inputs.tag }} + RELEASE_NOTES: ${{ inputs.changelog-path }} + COMMIT: ${{ inputs.sha }} + PRERELEASE: ${{ steps.pre-release.outputs.pre-release }} + DRAFT: ${{ inputs.draft-flag }} + + - name: "[INFO] Published artifacts to GitHub as draft release" + shell: bash + run: | + title="Published artifacts to GitHub as draft release" + message="artifacts: $(ls ./dist) tag: ${{ inputs.tag }} pre-release: ${{ steps.prerelease.outputs.prerelease }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/publish-github-full/action.yml b/.github/actions/publish-github-full/action.yml new file mode 100644 index 00000000..4dc207d8 --- /dev/null +++ b/.github/actions/publish-github-full/action.yml @@ -0,0 +1,26 @@ +name: "Publish GitHub draft release as full release" +description: "Publish an existing draft release as a non-draft release" + +inputs: + tag: + description: "The tag to publish (i.e. v1.0.0b1)" + required: true + repo-url: + description: "The URL to the repo (https://github.com/dbt-labs/dbt-adapters" + required: true + +runs: + using: composite + steps: + - name: "Publish draft release" + shell: bash + run: gh release edit ${{ inputs.tag }} --draft=false --repo ${{ inputs.repo-url }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: "[INFO] Released draft as non-draft release on GitHub" + shell: bash + run: | + title="Released draft as non-draft release on GitHub" + message="tag: ${{ inputs.tag }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/publish-pypi/action.yml b/.github/actions/publish-pypi/action.yml index deffc6e3..44019426 100644 --- a/.github/actions/publish-pypi/action.yml +++ b/.github/actions/publish-pypi/action.yml @@ -1,29 +1,51 @@ -name: Publish - PyPI -description: Publish artifacts saved during build step to PyPI +name: "Publish to PyPI" +description: "Publish artifacts from an archive to PyPI" inputs: archive-name: - description: Where to download the artifacts from - required: true + description: "Name of the archive containing the artifacts, leave blank if local" + default: "" repository-url: - description: The PyPI index to publish to, test or prod + description: "The url for the PyPI index" required: true + working-dir: + description: "Where to run commands from, primarily supports namespace packaging" + default: "./" runs: using: composite steps: - - name: Download artifacts - uses: actions/download-artifact@v3 + - name: "Download artifacts" + if: ${{ !(inputs.archive-name == "" }} + uses: actions/download-artifact@v4 with: name: ${{ inputs.archive-name }} path: dist/ - - name: "[DEBUG]" - run : ls -R + - name: "[INFO] Downloaded artifacts" + if: ${{ !(inputs.archive-name == "" }} + shell: bash + run: | + title="Downloaded artifacts" + message="Downloaded artifacts from ${{ inputs.archive-name }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Found artifacts" shell: bash + working-directory: ${{ inputs.working-dir }} + run: echo $(ls ./dist) - - name: Publish artifacts to PyPI + - name: "Publish artifacts to PyPI" uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: ${{ inputs.repository-url }} + packages-dir: ${{ inputs.working-dir }}dist/ + + - name: "[INFO] Published artifacts to PyPI" + shell: bash + working-directory: ${{ inputs.working-dir }} + run: | + title="Published artifacts to PyPI" + message=$(ls ./dist) + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/publish-results/action.yml b/.github/actions/publish-results/action.yml deleted file mode 100644 index d863d659..00000000 --- a/.github/actions/publish-results/action.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Publish results - -inputs: - file-name: - description: File type for file name stub (e.g. "unit-tests") - required: true - python-version: - description: Python version for the file name stub (e.g. "3.8") - required: true - source-file: - description: File to be uploaded - required: true - -runs: - using: composite - steps: - - name: Get timestamp - id: timestamp - run: echo "ts=$(date +'%Y-%m-%dT%H-%M-%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - shell: bash - - - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.file-name }}_python-${{ inputs.python-version }}_${{ steps.timestamp.outputs.ts }}.csv - path: ${{ inputs.source-file }} diff --git a/.github/actions/publish-test-results/action.yml b/.github/actions/publish-test-results/action.yml new file mode 100644 index 00000000..0ec18bb4 --- /dev/null +++ b/.github/actions/publish-test-results/action.yml @@ -0,0 +1,48 @@ +name: "Publish test results" +description: "Upload test artifacts to a unique archive name" + +inputs: + archive-type: + description: "File type for archive name stub (e.g. unit-tests)" + required: true + python-version: + description: "Python version for the archive name stub (e.g. 3.8)" + required: true + os: + description: "OS for the archive name stub (e.g. ubuntu-latest)" + required: true + source-file: + description: "File/directory to be uploaded" + required: true +outputs: + archive-name: + description: "Name of the archive containing the test artifacts" + value: ${{ steps.archive.outputs.archive-name }} + +runs: + using: composite + steps: + - name: "Get timestamp for archive name" + id: timestamp + shell: bash + run: echo "ts=$(date +'%Y-%m-%dT%H-%M-%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts + + - name: "Build archive name" + id: archive + shell: bash + run: | + archive_name = ${{ inputs.archive-type }}_python-${{ inputs.python-version }}_${{ inputs.os }}_${{ steps.timestamp.outputs.ts }} + echo "archive-name=archive_name" >> $GITHUB_OUTPUT + + - name: "Upload test results" + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.archive.outputs.archive-name }} + path: ${{ inputs.source-file }} + + - name: "[INFO] Uploaded test artifacts" + shell: bash + run: | + title="Uploaded test artifacts" + message="Uploaded ${{ inputs.source-file }} to ${{ steps.archive.outputs.archive-name }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/setup-hatch/action.yml b/.github/actions/setup-hatch/action.yml index 7b7780ef..8c45ec98 100644 --- a/.github/actions/setup-hatch/action.yml +++ b/.github/actions/setup-hatch/action.yml @@ -1,22 +1,22 @@ -name: Setup - `hatch` -description: Setup a python environment with `hatch` installed +name: "Setup `hatch`" +description: "Setup a python environment with `hatch` installed" inputs: setup-command: - description: The command to setup development dependencies + description: "The command to setup development dependencies" default: "python -m pip install hatch" python-version: - description: The version of python to install + description: "The version of python to install" default: "3.11" runs: using: composite steps: - - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v4 + - name: "Set up python ${{ inputs.python-version }}" + uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} - - name: Install dev dependencies - run: ${{ inputs.setup-command }} + - name: "Install build dependencies" shell: bash + run: ${{ inputs.setup-command }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 33d94ff4..30c6e23f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,85 +1,87 @@ -# **what?** -# Verifies python build on all code commited to the repository. This workflow -# should not require any secrets since it runs for PRs from forked repos. By -# default, secrets are not passed to workflows running from a forked repos. - -# **why?** -# Ensure code for dbt meets a certain quality standard. - -# **when?** -# This will run for all PRs, when code is pushed to main, and when manually triggered. - name: "Build" +run-name: "Build ${{ inputs.package }}==${{ inputs.version}} for deployment to ${{ deploy-to }}" on: push: branches: - "main" - pull_request: merge_group: types: [checks_requested] workflow_dispatch: + inputs: + package: + description: "Choose what to publish" + type: choice + options: + - dbt-adapters + - dbt-tests-adapter + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + deploy-to: + description: "Choose where to publish" + type: choice + options: + - prod + - test workflow_call: inputs: - changelog_path: - description: "Path to changelog file" + package: + description: "Choose what to publish (dbt-adapters, dbt-tests-adapter)" + type: string + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string required: true + deploy-to: + description: "Choose where to publish" type: string + default: prod + outputs: + archive-name: ${{ jobs.build.outputs.archive-name }} permissions: read-all # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} cancel-in-progress: true defaults: run: shell: bash +env: + NOTIFICATION_PREFIX: "[Build]" + jobs: build: - name: Build, Test and publish to PyPi + name: "Build, test and publish to PyPi" runs-on: ubuntu-latest - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + outputs: + archive-name: ${{ steps.archive.outputs.archive-name }} + steps: - name: "Check out repository" uses: actions/checkout@v4 - - name: Setup `hatch` + - name: "Setup `hatch`" uses: ./.github/actions/setup-hatch - - name: Build `dbt-adapters` - if: ${{ inputs.package == 'dbt-adapters' }} - uses: ./.github/actions/build-hatch - - - name: Build `dbt-tests-adapter` - if: ${{ inputs.package == 'dbt-tests-adapter' }} - uses: ./.github/actions/build-hatch - with: - working-dir: "./dbt-tests-adapter/" + - name: "Archive name" + id: archive + uses: ./.github/actions/build-archive-name - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: Build `dbt-adapters` + - name: "Build `dbt-adapters`" if: ${{ inputs.package == 'dbt-adapters' }} - uses: ./.github/actions/build-hatch + uses: ./.github/actions/build + with: + archive-name: ${{ steps.archive.outputs.archive-name }} - - name: Build `dbt-tests-adapter` + - name: "Build `dbt-tests-adapter`" if: ${{ inputs.package == 'dbt-tests-adapter' }} - uses: ./.github/actions/build-hatch + uses: ./.github/actions/build with: + archive-name: ${{ steps.archive.outputs.archive-name }} working-dir: "./dbt-tests-adapter/" - - # this step is only needed for the release process - - name: "Upload Build Artifact" - if: ${{ github.event_name == 'workflow_call' }} - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.version.outputs.version_number }} - path: | - ${{ inputs.changelog_path }} - ./dist/ - retention-days: 3 diff --git a/.github/workflows/changelog-existence.yml b/.github/workflows/changelog-existence.yml index d778f565..b83bd11e 100644 --- a/.github/workflows/changelog-existence.yml +++ b/.github/workflows/changelog-existence.yml @@ -14,13 +14,12 @@ # and when new code is pushed to the branch. The action will get # skipped if the 'Skip Changelog' label is present is any of the labels. -name: Check Changelog Entry +name: "Check for changelog entry" on: - pull_request_target: + pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] paths-ignore: ['.changes/**', '.github/**', 'tests/**', 'third-party-stubs/**', '**.md', '**.yml'] - workflow_dispatch: defaults: @@ -37,4 +36,4 @@ jobs: with: changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry).' skip_label: 'Skip Changelog' - secrets: inherit + secrets: inherit # this is only acceptable because we own the action we're calling diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 4f5b392e..1c712eec 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -1,4 +1,4 @@ -name: Code Quality +name: "Code quality" on: push: @@ -21,20 +21,20 @@ concurrency: jobs: lint: - name: Code Quality + name: "Code quality" runs-on: ubuntu-latest steps: - - name: Check out repository + - name: "Check out repository" uses: actions/checkout@v4 with: persist-credentials: false - - name: Setup `hatch` + - name: "Setup `hatch`" uses: ./.github/actions/setup-hatch - - name: Run linters + - name: "Run linters" run: hatch run lint:all - - name: Run typechecks + - name: "Run typechecks" run: hatch run typecheck:all diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml deleted file mode 100644 index edd25172..00000000 --- a/.github/workflows/github-release.yml +++ /dev/null @@ -1,259 +0,0 @@ -# **what?** -# Create a new release on GitHub and include any artifacts in the `/dist` directory of the GitHub artifacts store. -# -# Inputs: -# sha: The commit to attach to this release -# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) -# changelog_path: Path to the changelog file for release notes -# test_run: Test run (Publish release as draft) -# -# **why?** -# Reusable and consistent GitHub release process. -# -# **when?** -# Call after a successful build. Build artifacts should be ready to release and live in a dist/ directory. -# -# This workflow expects the artifacts to already be built and living in the artifact store of the workflow. -# -# Validation Checks -# -# 1. If no release already exists for this commit and version, create the tag and release it to GitHub. -# 2. If a release already exists for this commit, skip creating the release but finish with a success. -# 3. If a release exists for this commit under a different tag, fail. -# 4. If the commit is already associated with a different release, fail. - -name: GitHub Release - -on: - workflow_call: - inputs: - sha: - description: The commit to attach to this release - required: true - type: string - version_number: - description: The release version number (i.e. 1.0.0b1) - required: true - type: string - changelog_path: - description: Path to the changelog file for release notes - required: true - type: string - test_run: - description: Test run (Publish release as draft) - required: true - type: boolean - archive_name: - description: artifact name to download - required: true - type: string - outputs: - tag: - description: The path to the changelog for this version - value: ${{ jobs.check-release-exists.outputs.tag }} - -permissions: - contents: write - -env: - REPO_LINK: ${{ github.server_url }}/${{ github.repository }} - NOTIFICATION_PREFIX: "[GitHub Release]" - -jobs: - log-inputs: - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Print Variables" - run: | - echo The last commit sha in the release: ${{ inputs.sha }} - echo The release version number: ${{ inputs.version_number }} - echo Expected Changelog path: ${{ inputs.changelog_path }} - echo Test run: ${{ inputs.test_run }} - echo Repo link: ${{ env.REPO_LINK }} - echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} - - check-release-exists: - runs-on: ubuntu-latest - outputs: - exists: ${{ steps.release_check.outputs.exists }} - draft_exists: ${{ steps.release_check.outputs.draft_exists }} - tag: ${{ steps.set_tag.outputs.tag }} - - steps: - - name: "Generate Release Tag" - id: set_tag - run: echo "tag=v${{ inputs.version_number }}" >> $GITHUB_OUTPUT - - # When the GitHub CLI doesn't find a release for the given tag, it will exit 1 with a - # message of "release not found". In our case, it's not an actual error, just a - # confirmation that the release does not already exists so we can go ahead and create it. - # The `|| true` makes it so the step does not exit with a non-zero exit code - # Also check if the release already exists is draft state. If it does, and we are not - # testing then we can publish that draft as is. If it's in draft and we are testing, skip the - # release. - - name: "Check If Release Exists For Tag ${{ steps.set_tag.outputs.tag }}" - id: release_check - run: | - output=$((gh release view ${{ steps.set_tag.outputs.tag }} --json isDraft,targetCommitish --repo ${{ env.REPO_LINK }}) 2>&1) || true - if [[ "$output" == "release not found" ]] - then - title="Release for tag ${{ steps.set_tag.outputs.tag }} does not exist." - message="Check passed." - echo "exists=false" >> $GITHUB_OUTPUT - echo "draft_exists=false" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 0 - fi - commit=$(jq -r '.targetCommitish' <<< "$output") - if [[ $commit != ${{ inputs.sha }} ]] - then - title="Release for tag ${{ steps.set_tag.outputs.tag }} already exists for commit $commit!" - message="Cannot create a new release for commit ${{ inputs.sha }}. Exiting." - echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 1 - fi - isDraft=$(jq -r '.isDraft' <<< "$output") - if [[ $isDraft == true ]] && [[ ${{ inputs.test_run }} == false ]] - then - title="Release tag ${{ steps.set_tag.outputs.tag }} already associated with the draft release." - message="Release workflow will publish the associated release." - echo "exists=false" >> $GITHUB_OUTPUT - echo "draft_exists=true" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 0 - fi - title="Release for tag ${{ steps.set_tag.outputs.tag }} already exists." - message="Skip GitHub Release Publishing." - echo "exists=true" >> $GITHUB_OUTPUT - echo "draft_exists=false" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ env.REPO_LINK }} - - - name: "[DEBUG] Log Job Outputs" - run: | - echo exists: ${{ steps.release_check.outputs.exists }} - echo draft_exists: ${{ steps.release_check.outputs.draft_exists }} - echo tag: ${{ steps.set_tag.outputs.tag }} - - skip-github-release: - runs-on: ubuntu-latest - needs: [check-release-exists] - if: needs.check-release-exists.outputs.exists == 'true' - - steps: - - name: "Tag Exists, Skip GitHub Release Job" - run: | - echo title="A tag already exists for ${{ needs.check-release-exists.outputs.tag }} and commit." - echo message="Skipping GitHub release." - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - audit-release-different-commit: - runs-on: ubuntu-latest - needs: [check-release-exists] - if: needs.check-release-exists.outputs.exists == 'false' - - steps: - - name: "Check If Release Already Exists For Commit" - uses: cardinalby/git-get-release-action@1.2.4 - id: check_release_commit - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - commitSha: ${{ inputs.sha }} - doNotFailIfNotFound: true # returns blank outputs when not found instead of error - searchLimit: 15 # Since we only care about recent releases, speed up the process - - - name: "[DEBUG] Print Release Details" - run: | - echo steps.check_release_commit.outputs.id: ${{ steps.check_release_commit.outputs.id }} - echo steps.check_release_commit.outputs.tag_name: ${{ steps.check_release_commit.outputs.tag_name }} - echo steps.check_release_commit.outputs.target_commitish: ${{ steps.check_release_commit.outputs.target_commitish }} - echo steps.check_release_commit.outputs.prerelease: ${{ steps.check_release_commit.outputs.prerelease }} - - # Since we already know a release for this tag does not exist, if we find anything it's for the wrong tag, exit - - name: "Check If The Tag Matches The Version Number" - if: steps.check_release_commit.outputs.id != '' - run: | - title="Tag ${{ steps.check_release_commit.outputs.tag_name }} already exists for this commit!" - message="Cannot create a new tag for ${{ needs.check-release-exists.outputs.tag }} for the same commit" - echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 1 - - publish-draft-release: - runs-on: ubuntu-latest - needs: [check-release-exists, audit-release-different-commit] - if: >- - needs.check-release-exists.outputs.draft_exists == 'true' && - inputs.test_run == false - - steps: - - name: "Publish Draft Release - ${{ needs.check-release-exists.outputs.tag }}" - run: | - gh release edit $TAG --draft=false --repo ${{ env.REPO_LINK }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ needs.check-release-exists.outputs.tag }} - - create-github-release: - runs-on: ubuntu-latest - needs: [check-release-exists, audit-release-different-commit] - if: needs.check-release-exists.outputs.draft_exists == 'false' - - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.sha }} - - - name: "Download Artifact ${{ inputs.archive_name }}" - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.archive_name }} - path: . - - - name: "[DEBUG] Display Structure Of Expected Files" - run: | - ls -R .changes - ls -l dist - - - name: "Set Release Type" - id: release_type - run: | - if ${{ contains(inputs.version_number, 'rc') || contains(inputs.version_number, 'b') }} - then - echo Release will be set as pre-release - echo "prerelease=--prerelease" >> $GITHUB_OUTPUT - else - echo This is not a prerelease - fi - - - name: "Set As Draft Release" - id: draft - run: | - if [[ ${{ inputs.test_run }} == true ]] - then - echo Release will be published as draft - echo "draft=--draft" >> $GITHUB_OUTPUT - else - echo This is not a draft release - fi - - - name: "GitHub Release Workflow Annotation" - run: | - title="Release ${{ needs.check-release-exists.outputs.tag }}" - message="Configuration: ${{ steps.release_type.outputs.prerelease }} ${{ steps.draft.outputs.draft }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - - name: "Create New GitHub Release - ${{ needs.check-release-exists.outputs.tag }}" - run: | - gh release create $TAG ./dist/* --title "$TITLE" --notes-file $RELEASE_NOTES --target $COMMIT $PRERELEASE $DRAFT --repo ${{ env.REPO_LINK }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ needs.check-release-exists.outputs.tag }} - TITLE: ${{ github.event.repository.name }} ${{ needs.check-release-exists.outputs.tag }} - RELEASE_NOTES: ${{ inputs.changelog_path }} - COMMIT: ${{ inputs.sha }} - PRERELEASE: ${{ steps.release_type.outputs.prerelease }} - DRAFT: ${{ steps.draft.outputs.draft }} \ No newline at end of file diff --git a/.github/workflows/publish-github.yml b/.github/workflows/publish-github.yml new file mode 100644 index 00000000..398ae450 --- /dev/null +++ b/.github/workflows/publish-github.yml @@ -0,0 +1,159 @@ +# **what?** +# 1. If a release exists for this commit under a different tag, fail. +# 2. If the commit is already associated with a different release, fail. +# 3. If no release/tag exists for this commit and version, create the tag and release it to GitHub as a draft. +# 4. If a release already exists for this commit, skip creating the release as a draft. +# 5. If this is deployed to prod, mark the draft release (created in this run or previously) as a full release. +# +# **when?** +# 1. Call after a successful build. Build artifacts should be ready to release.defaults: +# They need to either live in a dist/ directory locally, or an archive name needs to be provided. +# 2. Call to publish a draft release to become a full release. No archive or artifacts are needed in this case. +name: "Publish to GitHub" +run-name: "Publish `${{ version }}` to GitHub in ${{ deploy-to }}" + +on: + workflow_dispatch: + inputs: + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local or publishing draft to full" + type: string + default: "" + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + sha: + description: "Commit SHA being released" + type: string + required: true + changelog-path: + description: "Path to the release notes" + type: string + required: true + deploy-to: + description: "Choose where to publish" + type: choice + options: + - "prod" + - "test" + workflow_call: + inputs: + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local or publishing draft to full" + type: string + default: "" + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + sha: + description: "Commit SHA being released" + type: string + required: true + changelog-path: + description: "Path to the release notes" + type: string + required: true + deploy-to: + description: "Choose where to publish" + type: string + default: "prod" + +permissions: + contents: write + +# will cancel previous workflows triggered by the same event and for the same package and environment +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.archive-name }}-${{ inputs.version }}-${{ inputs.deploy-to }} + cancel-in-progress: true + +env: + REPO_LINK: ${{ github.server_url }}/${{ github.repository }} + NOTIFICATION_PREFIX: "[GitHub Release]" + +jobs: + log-inputs: + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo The last commit sha in the release: ${{ inputs.sha }} + echo The release version number: ${{ inputs.version_number }} + echo Expected Changelog path: ${{ inputs.changelog_path }} + echo Test run: ${{ inputs.test_run }} + echo Repo link: ${{ env.REPO_LINK }} + echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} + + publish: + runs-on: ubuntu-latest + outputs: + exists: ${{ steps.release_check.outputs.exists }} + draft_exists: ${{ steps.release_check.outputs.draft_exists }} + tag: ${{ steps.set_tag.outputs.tag }} + + steps: + - name: "Build tag" + id: tag + shell: bash + run: echo "tag=v${{ inputs.version }}" >> $GITHUB_OUTPUT + + - name: "Get tag metadata" + id: tag-metadata + uses: ./.github/actions/audit-github-tag + with: + tag: ${{ steps.tag.output.tag }} + repo-url: ${{ env.REPO_LINK }} + + - name: "[INFO] Release already exists" + if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == true }} + id: release_check + run: | + title="Tag already exists, skip publishing artifacts" + message="Tag ${{ steps.tag.outputs.tag }} already exists and is associated with the commit ${{ steps.tag-metadata.outputs.sha }}." + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[ERROR] Exit if the tag exists an is associated with a different commit than the one provided" + if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == true }} && ${{ steps.tag-metadata.outputs.sha }} != ${{ inputs.sha }} + shell: bash + run: | + title="Tag ${{ inputs.tag }} already exists with a different commit!" + message="Existing commit: {{ steps.tag-metadata.outputs.sha }} New commit: ${{ inputs.sha }}. Exiting." + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + + - name: "Get commit metadata" + id: commit-metadata + if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == false }} + uses: ./.github/actions/audit-github-commit + with: + tag: ${{ inputs.sha }} + + - name: "[ERROR] Exit if the commit is associated with a different tag than the version that was provided" + if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == false && steps.commit-metadata.outputs.tag != '' }} + run: | + title="Commit ${{ inputs.sha }} is already associated with a different tag!" + message="Existing tag: ${{ steps.commit-metadata.outputs.tag }} New tag: ${{ steps.tag.outputs.tag }}" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + + - name: "Publish to GitHub as draft release" + if: fromJSON(steps.tag-metadata.outputs.exists) == false + uses: ./.github/actions/publish-github-draft + with: + archive-name: ${{ inputs.archive-name }} + tag: ${{ needs.check-release-exists.outputs.tag }} + repo-url: ${{ env.REPO_LINK }} + sha: ${{ inputs.sha }} + changelog-path: ${{ inputs.changelog_path }} + + - name: "Publish draft release as full release" + if: >- + inputs.deploy-to == "prod" && ( + fromJSON(steps.tag-metadata.outputs.exists) == false || + fromJSON(steps.tag-metadata.outputs.is-draft) == true + ) + uses: ./.github/actions/publish-github-full + with: + tag: ${{ steps.tag.output.tag }} + repo-url: ${{ env.REPO_LINK }} diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml new file mode 100644 index 00000000..509a699c --- /dev/null +++ b/.github/workflows/publish-pypi.yml @@ -0,0 +1,75 @@ +# **what?** +# Releases artifacts to PyPI. +# +# **when?** +# Call after a successful build. Build artifacts should be ready to release.defaults: +# They need to either live in a dist/ directory locally, or an archive name needs to be provided. +name: "Publish to PyPI" +run-name: "Publish `${{ version }}` to PyPI in ${{ deploy-to }}" + +on: + workflow_dispatch: + inputs: + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local" + default: "" + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + deploy-to: + description: "Choose where to publish" + type: choice + options: + - "prod" + - "test" + workflow_call: + inputs: + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local" + type: string + default: "" + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + deploy-to: + description: "Choose where to publish" + type: string + default: "prod" + +permissions: + contents: write + +# will cancel previous workflows triggered by the same event and for the same package and environment +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.archive-name }}-${{ inputs.version }}-${{ inputs.deploy-to }} + cancel-in-progress: true + +jobs: + log-inputs: + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo archive-name : ${{ inputs.archive-name }} + echo version : ${{ inputs.version }} + echo deploy-to : ${{ inputs.deploy-to }} + echo PYPI_PROJECT_URL : ${{ vars.PYPI_PROJECT_URL }} + echo PYPI_REPOSITORY_URL : ${{ vars.PYPI_REPOSITORY_URL }} + + pypi-release: + name: "Publish to PyPI" + runs-on: ubuntu-latest + environment: + name: ${{ inputs.deploy-to }} + url: ${{ vars.PYPI_PROJECT_URL }} + steps: + - name: "Check out repository" + uses: actions/checkout@v4 + + - name: "Publish to PyPI" + uses: .github/actions/publish-pypi + with: + repository-url: ${{ vars.PYPI_REPOSITORY_URL }} + archive-name: ${{ inputs.archive-name }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1135adb8..d140a9ec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,70 +1,64 @@ -name: Release -run-name: Release ${{ inputs.package }}==${{ inputs.version_number }} to ${{ inputs.deploy-to }} +name: "Release" +run-name: "Release `${{ inputs.package }}==${{ inputs.version }}` to ${{ inputs.deploy-to }}" on: workflow_dispatch: inputs: package: + description: "Choose what to release" type: choice - description: Choose what to publish options: - - dbt-adapters - - dbt-tests-adapter - version_number: + - "dbt-adapters" + - "dbt-tests-adapter" + target-branch: + description: "Choose what branch to release from" + type: string + default: "main" + version: description: "The release version number (i.e. 1.0.0b1)" type: string required: true deploy-to: + description: "Choose where to publish" type: choice - description: Choose where to publish options: - - prod - - test - default: prod - nightly_release: - description: "Nightly release to dev environment" + - "prod" + - "test" + nightly-release: + description: "Is this the nightly release to the dev environment?" type: boolean default: false - required: false - target_branch: - description: "The branch to release from" - type: string - required: false - default: main - workflow_call: inputs: package: + description: "Choose what to release" type: string - description: Choose what to publish - required: true - version_number: + default: "dbt-adapters" + target-branch: + description: "Choose what branch to release from" + type: string + default: "main" + version: description: "The release version number (i.e. 1.0.0b1)" type: string required: true deploy-to: + description: "Choose where to publish" type: string - default: prod - required: false - nightly_release: - description: "Nightly release to dev environment" + default: "prod" + nightly-release: + description: "Is this the nightly release to the dev environment?" type: boolean default: false - required: false - target_branch: - description: "The branch to release from" - type: string - required: false - default: main -# this is the permission that allows creating a new release +# this is the permission that allows creating a new release to both GitHub and PyPI permissions: contents: write id-token: write -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise +# will cancel previous workflows triggered by the same event and for the same package and environment concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}-${{ inputs.package }}-${{ inputs.deploy-to }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} cancel-in-progress: true defaults: @@ -128,53 +122,33 @@ jobs: echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - build-and-test: - name: "Build and Test" + build-artifacts: + name: "Build ${{ inputs.package }} ${{ inputs.version }}" needs: [release-inputs, bump-version-generate-changelog] - runs-on: ubuntu-latest - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - with: - ref: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - - - name: "Setup `hatch`" - uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - - - name: "Build ${{ inputs.package }}" - uses: dbt-labs/dbt-adapters/.github/actions/build-hatch@main - with: - working-dir: ${{ needs.release-inputs.outputs.working-dir }} - archive-name: ${{ needs.release-inputs.outputs.archive-name }} + uses: dbt-labs/dbt-adapters/.github/workflows/build.yml@update-workflows + with: + package: ${{ inputs.package }} + version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-to }} - github-release: - name: "GitHub Release" - # ToDo: update GH release to handle adding dbt-tests-adapter and dbt-adapter assets to the same release + # TODO: update GH release to handle adding dbt-tests-adapter and dbt-adapter assets to the same release + publish-github: + name: "Publish to GitHub" if: ${{ !failure() && !cancelled() && inputs.package == 'dbt-adapters' }} - needs: [release-inputs, build-and-test, bump-version-generate-changelog] - uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main + needs: [build-artifacts, bump-version-generate-changelog] + uses: dbt-labs/dbt-adapters/.github/workflows/publish-github.yml@update-workflows with: + archive-name: ${{ needs.build-artifacts.outputs.archive-name }} + version: ${{ inputs.version }} sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - test_run: ${{ inputs.deploy-to == 'test' && true || false }} - archive_name: ${{ needs.release-inputs.outputs.archive-name }} + changelog-path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} + deploy-to: ${{ inputs.deploy-to }} - pypi-release: + publish-0pypi: name: "Publish to PyPI" - runs-on: ubuntu-latest - needs: [release-inputs, build-and-test] - environment: - name: ${{ inputs.deploy-to }} - url: ${{ vars.PYPI_PROJECT_URL }} - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - - - name: "Publish to PyPI" - uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main - with: - repository-url: ${{ vars.PYPI_REPOSITORY_URL }} - archive-name: ${{ needs.release-inputs.outputs.archive-name }} + needs: [build-artifacts] + uses: dbt-labs/dbt-adapters/.github/workflows/publish-github.yml@update-workflows + with: + archive-name: ${{ needs.build-artifacts.outputs.archive-name }} + version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-to }} diff --git a/.github/workflows/resubmit-for-triage.yml b/.github/workflows/resubmit-for-triage.yml index 385ef820..3020509c 100644 --- a/.github/workflows/resubmit-for-triage.yml +++ b/.github/workflows/resubmit-for-triage.yml @@ -10,7 +10,7 @@ # **when?** # This will run when a comment is added to an issue and that issue has an `awaiting_response` label. -name: Resubmit for Triage +name: "Resubmit for triage" on: issue_comment @@ -22,7 +22,7 @@ permissions: issues: write jobs: - triage_label: + triage-label: if: contains(github.event.issue.labels.*.name, 'awaiting_response') uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main with: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 26ff4aaa..5022a0c6 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,4 +1,4 @@ -name: Unit Tests +name: "Unit tests" on: push: @@ -17,33 +17,35 @@ concurrency: jobs: unit: - name: Unit Tests - runs-on: ubuntu-latest + name: "Unit tests" + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: python-version: ["3.8", "3.9", "3.10", "3.11"] + os: [ubuntu-latest] steps: - - name: Check out repository + - name: "Check out repository" uses: actions/checkout@v4 with: persist-credentials: false - - name: Setup `hatch` + - name: "Setup `hatch`" uses: ./.github/actions/setup-hatch with: python-version: ${{ matrix.python-version }} - - name: Run unit tests + - name: "Run unit tests" run: hatch run unit-tests:all shell: bash - - name: Publish results - uses: ./.github/actions/publish-results + - name: "Publish results" + uses: ./.github/actions/publish-test-results if: always() with: - source-file: "results.csv" - file-name: "unit_results" + archive-type: "unit_results" python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + source-file: "results.csv" From 8f7b47d881730f57b11cf1d17e3f0159d033b4da Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 5 Mar 2024 23:53:21 -0600 Subject: [PATCH 02/16] break workflows into actions and smaller workflows, reuse workflows in larger workflows --- .github/actions/audit-changelog/action.yml | 54 ++ .../actions/audit-github-commit/action.yml | 8 +- .github/actions/audit-github-tag/action.yml | 25 +- .github/actions/audit-github-team/action.yml | 46 ++ .github/actions/build-archive-name/action.yml | 28 - .github/actions/commit/action.yml | 34 ++ .github/actions/create-temp-branch/action.yml | 35 ++ .../actions/publish-github-draft/action.yml | 3 +- .../actions/publish-github-full/action.yml | 6 +- .github/actions/publish-pypi/action.yml | 3 +- .../actions/publish-test-results/action.yml | 11 +- .github/actions/setup-changie/action.yml | 16 + .github/workflows/build.yml | 85 +-- .github/workflows/bump-version.yml | 108 ++++ ...ence.yml => check-for-changelog-entry.yml} | 0 .github/workflows/code-quality.yml | 52 +- .github/workflows/generate-changelog.yml | 176 ++++++ .github/workflows/publish-github.yml | 47 +- .github/workflows/publish-pypi.yml | 10 +- .github/workflows/release-branch-create.yml | 67 +++ .github/workflows/release-branch-merge.yml | 85 +++ .github/workflows/release.yml | 123 ++-- .github/workflows/release_prep_hatch.yml | 542 ------------------ .github/workflows/unit-tests.yml | 32 +- pyproject.toml | 13 +- 25 files changed, 887 insertions(+), 722 deletions(-) create mode 100644 .github/actions/audit-changelog/action.yml create mode 100644 .github/actions/audit-github-team/action.yml delete mode 100644 .github/actions/build-archive-name/action.yml create mode 100644 .github/actions/commit/action.yml create mode 100644 .github/actions/create-temp-branch/action.yml create mode 100644 .github/actions/setup-changie/action.yml create mode 100644 .github/workflows/bump-version.yml rename .github/workflows/{changelog-existence.yml => check-for-changelog-entry.yml} (100%) create mode 100644 .github/workflows/generate-changelog.yml create mode 100644 .github/workflows/release-branch-create.yml create mode 100644 .github/workflows/release-branch-merge.yml delete mode 100644 .github/workflows/release_prep_hatch.yml diff --git a/.github/actions/audit-changelog/action.yml b/.github/actions/audit-changelog/action.yml new file mode 100644 index 00000000..cc878fae --- /dev/null +++ b/.github/actions/audit-changelog/action.yml @@ -0,0 +1,54 @@ +name: "Audit changelog" +description: "Get metadata about a changelog, including the expected file path, and if it exists" + +inputs: + version: + description: "The version whose changelog is being audited" + required: true + +outputs: + path: + description: "The expected file path to the change log, relative to the repo root" + value: ${{ steps.path.outputs.path }} + exists: + description: "Indicates if the changelog exists" + value: ${{ steps.exists.outputs.exists }} + +runs: + using: composite + steps: + - name: "Parse version: `${{ inputs.version }}`" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ inputs.version }} + + - name: "Set: path" + id: path + shell: bash + run: | + path=".changes/${{ steps.semver.outputs.base-version }}" + if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] + then + path+="-${{ steps.semver.outputs.pre-release }}" + fi + path+=".md" + echo "path=$path" >> $GITHUB_OUTPUT + + - name: "Set: exists" + id: exists + shell: bash + run: | + exists=false + if test -f ${{ steps.path.outputs.path }} + then + exists=true + fi + echo "exists=exists">> $GITHUB_OUTPUT + + - name: "[DEBUG] Changelog metadata" + shell: bash + run: | + echo version : ${{ inputs.version }} + echo path : ${{ steps.path.outputs.path }} + echo exists : ${{ steps.exists.outputs.exists }} diff --git a/.github/actions/audit-github-commit/action.yml b/.github/actions/audit-github-commit/action.yml index 10a87033..5d623884 100644 --- a/.github/actions/audit-github-commit/action.yml +++ b/.github/actions/audit-github-commit/action.yml @@ -1,10 +1,11 @@ name: "Audit GitHub commit" -description: "Get basic metadata associated with a commit" +description: "Get metadata about a commit" inputs: sha: description: "The commit to audit" required: true + outputs: tag: description: "The associated release tag, if the release exists" @@ -22,7 +23,7 @@ outputs: runs: using: composite steps: - - name: "Check if a release exists for this commit" + - name: "Check if a release exists for `${{ inputs.sha }}`" id: commit env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -32,9 +33,10 @@ runs: doNotFailIfNotFound: true # returns blank outputs when not found instead of error searchLimit: 15 # Since we only care about recent releases, speed up the process - - name: "[DEBUG] Print Release Details" + - name: "[DEBUG] Commit metadata" shell: bash run: | + echo sha : ${{ inputs.sha }} echo tag : ${{ steps.check_release_commit.outputs.tag_name }} echo release-id : ${{ steps.check_release_commit.outputs.id }} echo pre-release : ${{ steps.check_release_commit.outputs.prerelease }} diff --git a/.github/actions/audit-github-tag/action.yml b/.github/actions/audit-github-tag/action.yml index 6b8c747f..1118a505 100644 --- a/.github/actions/audit-github-tag/action.yml +++ b/.github/actions/audit-github-tag/action.yml @@ -1,35 +1,29 @@ -# When the GitHub CLI doesn't find a release for the given tag, it will exit 1 with a -# message of "release not found". In our case, it's not an actual error, just a -# confirmation that the release does not already exist, so we can create it. -# The `|| true` makes it so the step does not exit with a non-zero exit code. -# Also check if the release already exists is draft state. If it's a draft, -# and we are not testing, then we can publish that draft as-is. If it's in draft, -# and we are testing, skip the release. name: "Audit GitHub tag" -description: "Get basic metadata associated with a release tag" +description: "Get metadata about a release tag" inputs: tag: - description: "The tag to audit (i.e. v1.0.0b1)" + description: "The tag to audit (e.g. v1.0.0b1)" required: true repo-url: - description: "The URL to the repo (https://github.com/dbt-labs/dbt-adapters" + description: "The URL to the repo (e.g. https://github.com/dbt-labs/dbt-adapters)" required: true + outputs: exists: - description: "Does the tag exist?" + description: "Indicates if the tag exists" value: ${{ steps.tag.outputs.exists }} sha: description: "The commit associated with the tag if the tag exists" value: ${{ steps.commit.outputs.sha }} is-draft: - description: "If the tag exists, is it a draft?" + description: "If the tag exists, indicates if it is a draft" value: ${{ steps.draft.outputs.is-draft }} runs: using: composite steps: - - name: "Check if tag exists" + - name: "Check if `${{ inputs.tag }}` exists in `${{ inputs.repo-url }}`" id: tag shell: bash run: | @@ -44,7 +38,7 @@ runs: with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: "Get commit associate with the tag" + - name: "Get the commit associated with `${{ inputs.tag }}`" id: commit if: ${{ fromJSON(steps.tag.outputs.exists) == true }} shell: bash @@ -52,7 +46,7 @@ runs: sha=$(jq -r '.targetCommitish' <<< "$output") echo "sha=$sha" >> $GITHUB_OUTPUT - - name: "Check if tag is a draft release" + - name: "Check if `${{ inputs.tag }}` is a draft release" id: draft if: ${{ fromJSON(steps.tag.outputs.exists) == true }} shell: bash @@ -69,6 +63,7 @@ runs: shell: bash run: | echo tag : ${{ inputs.tag }} + echo repo-url : ${{ inputs.repo-url }} echo exists: : ${{ steps.release.outputs.exists }} echo sha : ${{ steps.commit.outputs.sha }} echo is-draft : ${{ steps.draft.outputs.is-draft }} diff --git a/.github/actions/audit-github-team/action.yml b/.github/actions/audit-github-team/action.yml new file mode 100644 index 00000000..6923cb8c --- /dev/null +++ b/.github/actions/audit-github-team/action.yml @@ -0,0 +1,46 @@ +name: "Audit GitHub team" +description: "Get metadata about a GitHub team, such as a list of team members" + +inputs: + organization: + description: "The GitHub organization that owns the team" + required: true + team: + description: "The name of the team" + required: true + +outputs: + members: + description: "A space delimited list of team members" + value: ${{ steps.members.outputs.membership }} + +runs: + using: composite + steps: + - name: "Set: output file name" + id: output-file + run: echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT + + - name: "Get team membership for `${{ inputs.organization }}/${{ inputs.team }}`" + run: | + url = orgs/${{ inputs.organization }}/teams/${{ inputs.team }}/members + header = Accept: application/vnd.github+json + gh api -H "$header" $url > ${{ steps.output-file.outputs.name }} + + - name: "Parse team membership" + id: members + shell: + run: | + team_list=$(jq -r '.[].login' ${{ steps.output-file.outputs.name }}) + team_list_single=$(echo $team_list | tr '\n' ' ') + echo "membership=$team_list_single" >> $GITHUB_OUTPUT + + - name: "[DEBUG] Parse team membership" + shell: bash + run: | + echo organization : ${{ inputs.organization }} + echo team : ${{ inputs.team }} + echo members : ${{ steps.members.outputs.membership }} + + - name: "Delete the output file" + run: rm ${{ steps.output-file.outputs.name }} diff --git a/.github/actions/build-archive-name/action.yml b/.github/actions/build-archive-name/action.yml deleted file mode 100644 index d51f4fcb..00000000 --- a/.github/actions/build-archive-name/action.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: "Build" -description: "Provide a standard archive name for build artifacts" - -inputs: - package: - description: "Choose what to publish" - required: true - version: - description: "The release version number (i.e. 1.0.0b1)" - required: true - deploy-to: - description: "Choose where to publish" - default: prod -outputs: - archive-name: - description: "A unique archive name" - value: ${{ steps.archive.outputs.archive-name }} - -runs: - using: composite - - steps: - - name: "Archive name" - id: archive - shell: bash - run: | - archive_name=${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} - echo "archive-name=$archive_name" >> $GITHUB_OUTPUT diff --git a/.github/actions/commit/action.yml b/.github/actions/commit/action.yml new file mode 100644 index 00000000..8da59ece --- /dev/null +++ b/.github/actions/commit/action.yml @@ -0,0 +1,34 @@ +name: "Commit changes" +description: "Commit changes and push back up to the remote" + +inputs: + message: + description: "The commit message" + required: true + user: + description: "The user for the commit" + default: "Github Build Bot" + email: + description: "The email for the commit" + default: "buildbot@fishtownanalytics.com" + +outputs: + path: + description: "The file path to the change log, relative to the repo root" + value: ${{ steps.path.outputs.path }} + exists: + description: "Does the changelog exist?" + value: ${{ steps.exists.outputs.exists }} + +runs: + using: composite + steps: + - name: "Commit and push changes" + shell: bash + run: | + git config user.name "${{ inputs.user }}" + git config user.email "${{ inputs.email }}" + git pull + git add . + git commit -m "${{ inputs.message }}" + git push diff --git a/.github/actions/create-temp-branch/action.yml b/.github/actions/create-temp-branch/action.yml new file mode 100644 index 00000000..cce80b4c --- /dev/null +++ b/.github/actions/create-temp-branch/action.yml @@ -0,0 +1,35 @@ +name: "Create temp branch" +description: "Create a unique temporary branch for running CI workflows, such as version bumps, changelog consolidation, etc." + +inputs: + branch-stub: + description: "Stub to use for naming the branch (e.g. prep-release/, nightly-release/, etc.)" + default: "temp/" + +outputs: + branch: + description: "Name of the newly created branch" + value: ${{ steps.branch.outputs.branch-name }} + +runs: + using: composite + steps: + - name: "Create a unique branch name" + id: branch + run: | + name="${{ inputs.branch-stub }}" + name+="$(date +'%Y-%m-%d')/$GITHUB_RUN_ID" + echo "branch-name=$name" >> $GITHUB_OUTPUT + + - name: "Create branch: `${{ steps.branch.outputs.branch-name }}`" + shell: bash + run: | + git checkout -b ${{ steps.branch.outputs.branch-name }} + git push -u origin ${{ steps.branch.outputs.branch-name }} + + - name: "[INFO] Created branch" + shell: bash + run: | + title="Created branch" + message="${{ steps.branch.outputs.branch-name }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/publish-github-draft/action.yml b/.github/actions/publish-github-draft/action.yml index 0a1f77cf..11d8dc23 100644 --- a/.github/actions/publish-github-draft/action.yml +++ b/.github/actions/publish-github-draft/action.yml @@ -21,8 +21,7 @@ inputs: runs: using: composite steps: - - - name: "Download artifacts" + - name: "Download artifacts from `${{ inputs.archive-name }}`" if: ${{ !(inputs.archive-name == "" }} uses: actions/download-artifact@v4 with: diff --git a/.github/actions/publish-github-full/action.yml b/.github/actions/publish-github-full/action.yml index 4dc207d8..8258c764 100644 --- a/.github/actions/publish-github-full/action.yml +++ b/.github/actions/publish-github-full/action.yml @@ -12,15 +12,15 @@ inputs: runs: using: composite steps: - - name: "Publish draft release" + - name: "Publish `${{ inputs.tag }}` as full release" shell: bash run: gh release edit ${{ inputs.tag }} --draft=false --repo ${{ inputs.repo-url }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: "[INFO] Released draft as non-draft release on GitHub" + - name: "[INFO] Published draft as full release" shell: bash run: | title="Released draft as non-draft release on GitHub" - message="tag: ${{ inputs.tag }}" + message="tag: ${{ inputs.tag }} repo: ${{ inputs.repo-url }}" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/publish-pypi/action.yml b/.github/actions/publish-pypi/action.yml index 44019426..ad92107b 100644 --- a/.github/actions/publish-pypi/action.yml +++ b/.github/actions/publish-pypi/action.yml @@ -15,8 +15,7 @@ inputs: runs: using: composite steps: - - - name: "Download artifacts" + - name: "Download artifacts from `${{ inputs.archive-name }}`" if: ${{ !(inputs.archive-name == "" }} uses: actions/download-artifact@v4 with: diff --git a/.github/actions/publish-test-results/action.yml b/.github/actions/publish-test-results/action.yml index 0ec18bb4..2b41b388 100644 --- a/.github/actions/publish-test-results/action.yml +++ b/.github/actions/publish-test-results/action.yml @@ -14,6 +14,7 @@ inputs: source-file: description: "File/directory to be uploaded" required: true + outputs: archive-name: description: "Name of the archive containing the test artifacts" @@ -22,16 +23,12 @@ outputs: runs: using: composite steps: - - name: "Get timestamp for archive name" - id: timestamp - shell: bash - run: echo "ts=$(date +'%Y-%m-%dT%H-%M-%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - name: "Build archive name" + - name: "Create unique test archive name" id: archive shell: bash run: | - archive_name = ${{ inputs.archive-type }}_python-${{ inputs.python-version }}_${{ inputs.os }}_${{ steps.timestamp.outputs.ts }} + timestamp=$(date +'%Y-%m-%dT%H-%M-%S') #no colons allowed for artifacts + archive_name = ${{ inputs.archive-type }}_python-${{ inputs.python-version }}_${{ inputs.os }}_$timestamp echo "archive-name=archive_name" >> $GITHUB_OUTPUT - name: "Upload test results" diff --git a/.github/actions/setup-changie/action.yml b/.github/actions/setup-changie/action.yml new file mode 100644 index 00000000..3a82dd25 --- /dev/null +++ b/.github/actions/setup-changie/action.yml @@ -0,0 +1,16 @@ +name: "Setup `changie`" +description: "Setup `changie` using `linuxbrew`" + +runs: + using: composite + steps: + - name: "Add `linuxbrew` to PATH" + shell: bash + run: echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH + + - name: "Install `linuxbrew` packages" + shell: bash + run: | + brew install pre-commit + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 30c6e23f..83d0cd39 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,51 +1,54 @@ name: "Build" -run-name: "Build ${{ inputs.package }}==${{ inputs.version}} for deployment to ${{ deploy-to }}" +run-name: "Build `${{ inputs.package }}==${{ inputs.version}}` from `${{ inputs.branch }}` for deployment to `${{ deploy-to }}`" on: - push: - branches: - - "main" - merge_group: - types: [checks_requested] - workflow_dispatch: + workflow_call: inputs: package: - description: "Choose what to publish" - type: choice - options: - - dbt-adapters - - dbt-tests-adapter + description: "Choose what to publish (dbt-adapters, dbt-tests-adapter)" + type: string + branch: + description: "Choose what branch to build from" + type: string + default: "main" version: description: "The release version number (i.e. 1.0.0b1)" type: string required: true deploy-to: description: "Choose where to publish" - type: choice - options: - - prod - - test - workflow_call: + type: string + default: prod + outputs: + archive-name: ${{ jobs.build.outputs.archive-name }} + workflow_dispatch: inputs: package: - description: "Choose what to publish (dbt-adapters, dbt-tests-adapter)" + description: "Choose what to build" + type: choice + options: + - dbt-adapters + - dbt-tests-adapter + branch: + description: "Choose what branch to build from" type: string + default: "main" version: description: "The release version number (i.e. 1.0.0b1)" type: string required: true deploy-to: description: "Choose where to publish" - type: string - default: prod - outputs: - archive-name: ${{ jobs.build.outputs.archive-name }} + type: choice + options: + - prod + - test permissions: read-all -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise +# don't try to build the same version of the same package, the archive name could collide concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} + group: ${{ github.workflow }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} cancel-in-progress: true defaults: @@ -56,32 +59,48 @@ env: NOTIFICATION_PREFIX: "[Build]" jobs: + debug-inputs: + name: "[DEBUG] Inputs" + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo package : ${{ inputs.package }} + echo branch : ${{ inputs.branch }} + echo version : ${{ inputs.version }} + echo deploy-to : ${{ inputs.deploy-to }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + build: - name: "Build, test and publish to PyPi" + name: "Build the artifacts" runs-on: ubuntu-latest outputs: archive-name: ${{ steps.archive.outputs.archive-name }} - steps: - - name: "Check out repository" + - name: "Check out `${{ inputs.branch }}`" uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Create the archive name" + id: archive + shell: bash + run: | + name=${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} + echo "name=$name" >> $GITHUB_OUTPUT - name: "Setup `hatch`" uses: ./.github/actions/setup-hatch - - name: "Archive name" - id: archive - uses: ./.github/actions/build-archive-name - - name: "Build `dbt-adapters`" if: ${{ inputs.package == 'dbt-adapters' }} uses: ./.github/actions/build with: - archive-name: ${{ steps.archive.outputs.archive-name }} + archive-name: ${{ steps.archive.outputs.name }} - name: "Build `dbt-tests-adapter`" if: ${{ inputs.package == 'dbt-tests-adapter' }} uses: ./.github/actions/build with: - archive-name: ${{ steps.archive.outputs.archive-name }} + archive-name: ${{ steps.archive.outputs.name }} working-dir: "./dbt-tests-adapter/" diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml new file mode 100644 index 00000000..194857da --- /dev/null +++ b/.github/workflows/bump-version.yml @@ -0,0 +1,108 @@ +name: "Bump version" +run-name: "Bump branch `${{ inputs.branch }}` to `${{ inputs.version }}`" + +on: + workflow_dispatch: + inputs: + branch: + description: "The branch to bump" + type: string + default: "main" + version: + description: "The version to bump to" + required: true + working-dir: + description: "The working directory, useful for mono-repos" + default: "./" + workflow_call: + inputs: + branch: + description: "The branch to bump" + type: string + default: "main" + version: + description: "The version to bump to" + type: string + required: true + working-dir: + description: "The working directory, useful for mono-repos" + type: string + default: "./" + outputs: + current-version: ${{ jobs.bump-version.outputs.version }} + bumped: ${{ jobs.bump-version.outputs.bumped }} + +permissions: write-all + +# only bump a branch/working-dir (package) to one version at a time +concurrency: + group: ${{ github.workflow }}-${{ inputs.branch }}-${{ inputs.working-dir }} + cancel-in-progress: true + +env: + NOTIFICATION_PREFIX: "[Bump version]" + +jobs: + debug-inputs: + name: "[DEBUG] Inputs" + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo branch : ${{ inputs.branch }} + echo version : ${{ inputs.version }} + echo working-dir : ${{ inputs.working-dir }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + + bump-version: + name: "Bump `${{ inputs.branch}}` to `${{ inputs.version }}`" + runs-on: ubuntu-latest + outputs: + current-version: ${{ steps.current-version.outputs.version }} + bumped: ${{ steps.current-version.outputs.bumped }} + steps: + - name: "Check out `${{ inputs.branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup `hatch`" + uses: ./.github/actions/setup-hatch + + - name: "Current version" + id: current-version + shell: bash + run: | + echo "version=$(hatch version)" >> $GITHUB_OUTPUT + if [[ $(hatch version) == "${{ inputs.version }} ]] + then + echo "bumped=false" >> $GITHUB_OUTPUT + else + echo "bumped=true" >> $GITHUB_OUTPUT + fi + + - name: "Bump version" + if: ${{ steps.current-version.outputs.version != inputs.version }} + shell: bash + run: hatch version ${{ inputs.version }} + working-directory: ${{ inputs.working-dir }} + + - name: "Commit and push changes" + if: ${{ steps.current-version.outputs.version != inputs.version }} + uses: ./.github/actions/commit + with: + message: "[automated] bump version to ${{ inputs.version }}" + + - name: "[INFO] Bumped version" + if: ${{ steps.current-version.outputs.version != inputs.version }} + run: | + title="Bumped version" + message="Bumped from ${{ steps.current-version.outputs.version }} to ${{ inputs.version }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[INFO] Skipped version bump" + if: ${{ steps.current-version.outputs.version == inputs.version }} + run: | + title="Skipped version bump" + message="${{ inputs.branch }} is already at version ${{ inputs.version }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/changelog-existence.yml b/.github/workflows/check-for-changelog-entry.yml similarity index 100% rename from .github/workflows/changelog-existence.yml rename to .github/workflows/check-for-changelog-entry.yml diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 1c712eec..7661f9aa 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -6,7 +6,31 @@ on: - "main" - "*.latest" pull_request: + workflow_call: + inputs: + branch: + description: "The branch to run code quality on" + type: string + default: "main" + lint-command: + description: "The lint command to run" + type: string + default: "hatch run lint:all" + typecheck-command: + description: "The typecheck command to run" + type: string + default: "hatch run typecheck:all" workflow_dispatch: + inputs: + branch: + description: "The branch to run code quality on" + default: "main" + lint-command: + description: "The lint command to run" + default: "hatch run lint:all" + typecheck-command: + description: "The typecheck command to run" + default: "hatch run typecheck:all" permissions: read-all @@ -14,27 +38,41 @@ defaults: run: shell: bash -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise +# cancel the previous code quality run if a new one starts with the same parameters concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'workflow') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} cancel-in-progress: true +env: + NOTIFICATION_PREFIX: "[Code quality]" + jobs: - lint: - name: "Code quality" + debug-inputs: + name: "[DEBUG] Inputs" runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo branch : ${{ inputs.branch }} + echo lint-command : ${{ inputs.lint-command }} + echo typecheck-command : ${{ inputs.typecheck-command }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + code-quality: + name: "Code quality" + runs-on: ubuntu-latest steps: - - name: "Check out repository" + - name: "Check out `${{ inputs.branch }}`" uses: actions/checkout@v4 with: + ref: ${{ contains(github.event_name, 'workflow') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} persist-credentials: false - name: "Setup `hatch`" uses: ./.github/actions/setup-hatch - name: "Run linters" - run: hatch run lint:all + run: ${{ inputs.lint-command }} - name: "Run typechecks" - run: hatch run typecheck:all + run: ${{ inputs.typecheck-command }} diff --git a/.github/workflows/generate-changelog.yml b/.github/workflows/generate-changelog.yml new file mode 100644 index 00000000..51a91ef8 --- /dev/null +++ b/.github/workflows/generate-changelog.yml @@ -0,0 +1,176 @@ +# **what?** +# Merge individual changelogs to generate the final changelog +# +# **when?** +# Call when ready to kick off a build and release, gets used automatically by `release.yml` +name: "Generate changelog" +run-name: "Merge changelogs into one changelog on ${{ inputs.branch }} for ${{ inputs.version }}" + +on: + workflow_call: + inputs: + branch: + description: "The branch where the changelogs should be merged" + type: string + default: "main" + version: + description: "The version whose changelogs should be merged" + type: string + required: true + outputs: + changelog-path: + description: "The path to the changelog for this version" + value: ${{ jobs.audit-changelog.outputs.path }} + created: + description: "Identifies if this workflows created the changelog" + value: ${{ !jobs.audit-changelog.outputs.path }} + workflow_dispatch: + inputs: + branch: + description: "The branch where the changelogs should be merged" + type: string + default: "main" + version: + description: "The version whose changelogs should be merged" + type: string + required: true + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +# cancel the previous run for this version if a new run is called on this branch +concurrency: + group: ${{ github.workflow }}-${{ inputs.branch }}-${{ inputs.version }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + NOTIFICATION_PREFIX: "[Generate changelog]" + +jobs: + debug-inputs: + name: "[DEBUG] Inputs" + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo branch : ${{ inputs.branch }} + echo version : ${{ inputs.version }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + + audit-changelog: + name: "Audit changelog" + runs-on: ubuntu-latest + outputs: + changelog-path: ${{ steps.audit-changelog.outputs.path }} + exists: ${{ steps.audit-changelog.outputs.exists }} + steps: + - name: "Check out `${{ inputs.branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup `changie`" + uses: ./.github/actions/setup-changie + + - name: "Audit changelog" + id: audit-changelog + uses: ./.github/actions/audit-changelog + with: + version: ${{ inputs.version }} + + skip-changelog: + name: "Skip changelog generation" + if: ${{ needs.audit-changelog.outputs.exists }} + runs-on: ubuntu-latest + needs: [audit-changelog] + steps: + - name: "[INFO] Skipped changelog generation" + run: | + title="Skipped changelog" + message="The changelog already exists for version ${{ inputs.version }}: ${{ needs.audit-changelog.outputs.path }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + generate-changelog: + name: "Generate changelog" + if: ${{ !needs.audit-changelog.outputs.exists }} + runs-on: ubuntu-latest + needs: [audit-changelog] + steps: + - name: "Check out `${{ inputs.branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup `hatch`" + uses: ./.github/actions/setup-hatch + + - name: "Parse `${{ inputs.version }}`" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ inputs.version }} + + - name: "Get Core team membership" + id: core_membership + uses: ./.github/actions/audit-github-team + with: + organization: "dbt-labs" + team: "core-group" + env: + GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} + + - name: "Set Core team membership for changie contributors exclusion" + id: set_team_membership + run: echo "CHANGIE_CORE_TEAM=${{ steps.core_membership.outputs.members }}" >> $GITHUB_ENV + + - name: "Generate changelog" + run: | + if [[ ${{ steps.semver.outputs.is_prerelease }} -eq 1 ]] + then + changie batch ${{ steps.semver.outputs.base_version }} --move-dir '${{ steps.semver.outputs.base_version }}' --prerelease ${{ steps.semver.outputs.prerelease }} + elif [[ -d ".changes/${{ steps.semver.outputs.base_version }}" ]] + then + changie batch ${{ steps.semver.outputs.base_version }} --include '${{ steps.semver.outputs.base_version }}' --remove-prereleases + else # releasing a final patch with no prereleases + changie batch ${{ needs.audit-changelog.outputs.base_version }} + fi + changie merge + + - name: "Check changelog generated successfully" + run: | + title="Generate changelog" + if [[ -f ${{ needs.audit-changelog.outputs.changelog_path }} ]] + then + message="Changelog file created successfully" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + else + message="Changelog failed to generate" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + - name: "Remove trailing whitespace and extra newlines" + continue-on-error: true + run: hatch run lint:pre-commit --files dbt/adapters/__about__.py CHANGELOG.md .changes/* + + - name: "Commit and push changes" + uses: ./.github/actions/commit + with: + message: "[automated] generate changelog for ${{ inputs.version }}" + + - name: "[INFO] Generated changelog" + run: | + title="Generated changelog" + message="Generated changelog for version ${{ inputs.version }}: ${{ needs.audit-changelog.outputs.path }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/publish-github.yml b/.github/workflows/publish-github.yml index 398ae450..6b1e1d11 100644 --- a/.github/workflows/publish-github.yml +++ b/.github/workflows/publish-github.yml @@ -65,35 +65,40 @@ permissions: # will cancel previous workflows triggered by the same event and for the same package and environment concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.archive-name }}-${{ inputs.version }}-${{ inputs.deploy-to }} + group: ${{ github.workflow }}-${{ inputs.archive-name }}-${{ inputs.version }}-${{ inputs.deploy-to }} cancel-in-progress: true env: REPO_LINK: ${{ github.server_url }}/${{ github.repository }} - NOTIFICATION_PREFIX: "[GitHub Release]" + NOTIFICATION_PREFIX: "[Publish - GitHub]" jobs: - log-inputs: + debug-inputs: + name: "[DEBUG] Inputs" runs-on: ubuntu-latest steps: - name: "[DEBUG] Inputs" run: | - echo The last commit sha in the release: ${{ inputs.sha }} - echo The release version number: ${{ inputs.version_number }} - echo Expected Changelog path: ${{ inputs.changelog_path }} - echo Test run: ${{ inputs.test_run }} - echo Repo link: ${{ env.REPO_LINK }} - echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} + echo archive-name : ${{ inputs.archive-name }} + echo version : ${{ inputs.version }} + echo sha : ${{ inputs.sha }} + echo changelog-path : ${{ inputs.changelog-path }} + echo deploy-to : ${{ inputs.deploy-to }} + echo REPO_LINK : ${{ env.REPO_LINK }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} publish: + name: "Publish to GitHub" runs-on: ubuntu-latest outputs: exists: ${{ steps.release_check.outputs.exists }} draft_exists: ${{ steps.release_check.outputs.draft_exists }} tag: ${{ steps.set_tag.outputs.tag }} - steps: - - name: "Build tag" + - name: "Check out `main`" + uses: actions/checkout@v4 + + - name: "Create tag" id: tag shell: bash run: echo "tag=v${{ inputs.version }}" >> $GITHUB_OUTPUT @@ -105,15 +110,22 @@ jobs: tag: ${{ steps.tag.output.tag }} repo-url: ${{ env.REPO_LINK }} + - name: "Get commit metadata" + id: commit-metadata + if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == false }} + uses: ./.github/actions/audit-github-commit + with: + tag: ${{ inputs.sha }} + - name: "[INFO] Release already exists" - if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == true }} + if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == true }} && ${{ steps.tag-metadata.outputs.sha }} == ${{ inputs.sha }} id: release_check run: | title="Tag already exists, skip publishing artifacts" message="Tag ${{ steps.tag.outputs.tag }} already exists and is associated with the commit ${{ steps.tag-metadata.outputs.sha }}." echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "[ERROR] Exit if the tag exists an is associated with a different commit than the one provided" + - name: "[ERROR] Exit if the tag exists and is associated with a different commit than the one provided" if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == true }} && ${{ steps.tag-metadata.outputs.sha }} != ${{ inputs.sha }} shell: bash run: | @@ -122,15 +134,8 @@ jobs: echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" exit 1 - - name: "Get commit metadata" - id: commit-metadata - if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == false }} - uses: ./.github/actions/audit-github-commit - with: - tag: ${{ inputs.sha }} - - name: "[ERROR] Exit if the commit is associated with a different tag than the version that was provided" - if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == false && steps.commit-metadata.outputs.tag != '' }} + if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == false }} && ${{ steps.commit-metadata.outputs.tag != '' }} run: | title="Commit ${{ inputs.sha }} is already associated with a different tag!" message="Existing tag: ${{ steps.commit-metadata.outputs.tag }} New tag: ${{ steps.tag.outputs.tag }}" diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 509a699c..efc97aa7 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -43,11 +43,15 @@ permissions: # will cancel previous workflows triggered by the same event and for the same package and environment concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.archive-name }}-${{ inputs.version }}-${{ inputs.deploy-to }} + group: ${{ github.workflow }}-${{ inputs.archive-name }}-${{ inputs.version }}-${{ inputs.deploy-to }} cancel-in-progress: true +env: + NOTIFICATION_PREFIX: "[Publish - PyPI]" + jobs: - log-inputs: + debug-inputs: + name: "[DEBUG] Inputs" runs-on: ubuntu-latest steps: - name: "[DEBUG] Inputs" @@ -65,7 +69,7 @@ jobs: name: ${{ inputs.deploy-to }} url: ${{ vars.PYPI_PROJECT_URL }} steps: - - name: "Check out repository" + - name: "Check out `main`" uses: actions/checkout@v4 - name: "Publish to PyPI" diff --git a/.github/workflows/release-branch-create.yml b/.github/workflows/release-branch-create.yml new file mode 100644 index 00000000..f99b5f1b --- /dev/null +++ b/.github/workflows/release-branch-create.yml @@ -0,0 +1,67 @@ +name: "Create release branch" +run-name: "Create a release branch for `${{ inputs.version }}` using `${{ inputs.branch }}`" + +on: + workflow_call: + inputs: + branch: + description: "The base branch to use when creating the release branch" + type: string + default: "main" + version: + description: "The version to use in the release branch name" + type: string + required: true + outputs: + branch: + description: "The name of the created release branch" + value: ${{ jobs.create-release-branch.outputs.branch }} + workflow_dispatch: + inputs: + branch: + description: "The base branch to use when creating the release branch" + default: "main" + version: + description: "The version to use in the release branch name" + required: true + +permissions: + contents: write + id-token: write + +defaults: + run: + shell: bash + +env: + NOTIFICATION_PREFIX: "[Create release branch]" + +jobs: + debug-inputs: + name: "[DEBUG] Inputs" + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo branch : ${{ inputs.branch }} + echo version : ${{ inputs.version }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + + create-release-branch: + name: "Create a release branch from `${{ inputs.branch }}`" + runs-on: ubuntu-latest + outputs: + branch: ${{ steps.create-branch.outputs.branch }} + steps: + - name: "Checkout `${{ inputs.branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Create release branch" + id: create-branch + uses: ./.github/actions/create-temp-branch + with: + branch-stub: "release/${{ inputs.version }}/" + env: + NOTIFICATION_PREFIX: "[create release branch]" diff --git a/.github/workflows/release-branch-merge.yml b/.github/workflows/release-branch-merge.yml new file mode 100644 index 00000000..e619ceef --- /dev/null +++ b/.github/workflows/release-branch-merge.yml @@ -0,0 +1,85 @@ +name: "Merge release branch" +run-name: "Merge `${{ release-branch }}` into `${{ inputs.branch }}`" + +on: + workflow_call: + inputs: + branch: + description: "The base branch receiving the changes" + type: string + default: "main" + release-branch: + description: "The branch containing the release changes" + type: string + default: "main" + outputs: + sha: + description: "The sha associated with this merge" + value: ${{ jobs.merge-release-branch.outputs.sha }} + +# this is the permission that allows creating a new release to both GitHub and PyPI +permissions: + contents: write + id-token: write + +# will cancel previous workflows triggered by the same event and for the same package and environment +concurrency: + group: ${{ github.workflow }}-${{ inputs.branch }}-${{ inputs.release-branch }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + NOTIFICATION_PREFIX: "[Merge release branch]" + +jobs: + debug-inputs: + name: "[DEBUG] Inputs" + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo branch : ${{ inputs.branch }} + echo release-branch : ${{ inputs.release-branch }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + + merge-release-branch: + name: "Merge `${{ inputs.release-branch }}` into `${{ inputs.branch }}`" + runs-on: ubuntu-latest + needs: [temp-branch] + outputs: + sha: ${{ steps.release-sha.outputs.sha }} + steps: + - name: "Checkout `${{ inputs.release-branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.temp-branch }} + + - name: "Merge `${{ inputs.release-branch }}` into `${{ inputs.branch }}`" + uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ inputs.release-branch }} + target_branch: ${{ inputs.branch }} + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_message_template: "[automated] merge {source_ref} into {target_branch} for a release" + + - name: "Checkout `${{ inputs.branch }}`" + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + + # TODO: see if everlytic/branch-merge has outputs that have this information + - name: "Get commit SHA for the merge" + id: release-sha + run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: "Delete `${{ inputs.release-branch }}`" + run: git push origin -d ${{ inputs.release-branch }} + + - name: "[INFO] Merged changes" + run: | + title="Merged changes" + message="The ${{ inputs.release-branch }} branch was merged into ${{ inputs.branch }} with sha ${{ steps.release-sha.outputs.sha }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d140a9ec..9ad5a51e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,5 @@ name: "Release" -run-name: "Release `${{ inputs.package }}==${{ inputs.version }}` to ${{ inputs.deploy-to }}" +run-name: "Release `${{ inputs.package }}==${{ inputs.version }}` to `${{ inputs.deploy-to }}`" on: workflow_dispatch: @@ -10,7 +10,7 @@ on: options: - "dbt-adapters" - "dbt-tests-adapter" - target-branch: + branch: description: "Choose what branch to release from" type: string default: "main" @@ -24,17 +24,13 @@ on: options: - "prod" - "test" - nightly-release: - description: "Is this the nightly release to the dev environment?" - type: boolean - default: false workflow_call: inputs: package: description: "Choose what to release" type: string default: "dbt-adapters" - target-branch: + branch: description: "Choose what branch to release from" type: string default: "main" @@ -46,19 +42,15 @@ on: description: "Choose where to publish" type: string default: "prod" - nightly-release: - description: "Is this the nightly release to the dev environment?" - type: boolean - default: false # this is the permission that allows creating a new release to both GitHub and PyPI permissions: contents: write id-token: write -# will cancel previous workflows triggered by the same event and for the same package and environment +# deploying the same version of a package to the same environment should override any previous deployment with those attributes concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} + group: ${{ github.workflow }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} cancel-in-progress: true defaults: @@ -70,16 +62,16 @@ jobs: name: "Release inputs" runs-on: ubuntu-latest outputs: - working-dir: ${{ steps.release-inputs.outputs.working-dir }} - run-unit-tests: ${{ steps.release-inputs.outputs.run-unit-tests }} - archive-name: ${{ steps.release-inputs.outputs.archive-name }} + archive-name: ${{ steps.computed-inputs.outputs.archive-name }} + working-dir: ${{ steps.computed-inputs.outputs.working-dir }} + run-unit-tests: ${{ steps.computed-inputs.outputs.run-unit-tests }} steps: - - name: "Inputs" - id: release-inputs + - name: "Computed inputs" + id: computed-inputs run: | + archive_name=${{ inputs.package }}-${{ inputs.version_number }}-${{ inputs.deploy-to }} working_dir="./" run_unit_tests=true - archive_name=${{ inputs.package }}-${{ inputs.version_number }}-${{ inputs.deploy-to }} if test "${{ inputs.package }}" = "dbt-tests-adapter" then @@ -87,65 +79,102 @@ jobs: run_unit_tests=false fi + echo "archive-name=$archive_name" >> $GITHUB_OUTPUT echo "working-dir=$working_dir" >> $GITHUB_OUTPUT echo "run-unit-tests=$run_unit_tests" >> $GITHUB_OUTPUT - echo "archive-name=$archive_name" >> $GITHUB_OUTPUT - - name: "[DEBUG]" + - name: "[DEBUG] Inputs" run: | echo package : ${{ inputs.package }} + echo branch : ${{ inputs.branch }} + echo version : ${{ inputs.version }} + echo deploy-to : ${{ inputs.deploy-to }} + echo archive-name : ${{ steps.release-inputs.outputs.archive-name }} echo working-dir : ${{ steps.release-inputs.outputs.working-dir }} echo run-unit-tests : ${{ steps.release-inputs.outputs.run-unit-tests }} - echo archive-name : ${{ steps.release-inputs.outputs.archive-name }} - bump-version-generate-changelog: - name: "Bump package version, Generate changelog" - uses: dbt-labs/dbt-adapters/.github/workflows/release_prep_hatch.yml@main + code-quality: + name: "Run code quality" + uses: dbt-labs/dbt-adapters/.github/workflows/code-quality.yml + with: + branch: ${{ inputs.branch }} + + unit-tests: + name: "Run unit tests" needs: [release-inputs] + if: ${{ fromJSON(needs.release-inputs.outputs.run-unit-tests) == true }} + uses: dbt-labs/dbt-adapters/.github/workflows/unit-tests.yml@update-workflows + with: + branch: ${{ inputs.branch }} + + release-branch: + name: "Create a release branch" + uses: dbt-labs/dbt-adapters/.github/workflows/release-branch-create.yml@update-workflows + with: + branch: ${{ inputs.branch }} + version: ${{ inputs.version }} + secrets: inherit + + bump-version: + name: "Bump the version" + uses: dbt-labs/dbt-adapters/.github/workflows/bump-version.yml@update-workflows + needs: [release-branch, release-inputs] with: - version_number: ${{ inputs.version_number }} - deploy_to: ${{ inputs.deploy-to }} - nightly_release: ${{ inputs.nightly_release }} - target_branch: ${{ inputs.target_branch }} + branch: ${{ needs.release-branch.outputs.branch }} + version: ${{ inputs.version }} working-dir: ${{ needs.release-inputs.outputs.working-dir }} - run-unit-tests: ${{ fromJSON(needs.release-inputs.outputs.run-unit-tests) }} secrets: inherit - log-outputs-bump-version-generate-changelog: - name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() }} - needs: [release-inputs, bump-version-generate-changelog] - runs-on: ubuntu-latest - steps: - - name: Print variables - run: | - echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} + generate-changelog: + name: "Generate a new changelog" + needs: [release-branch] + uses: dbt-labs/dbt-adapters/.github/workflows/generate-changelog.yml@update-workflows + with: + branch: ${{ needs.release-branch.outputs.branch }} + version: ${{ inputs.version }} + secrets: inherit + + merge-changes: + name: "Merge the version bump and changelog updates" + needs: [ + release-branch, + bump-version, + generate-changelog, + code-quality, + unit-tests, + ] + if: ${{ !failure() && !cancelled() && (needs.bump-version.outputs.bumped || needs.generate-changelog.outputs.created) }} + uses: dbt-labs/dbt-adapters/.github/workflows/release-branch-merge.yml@update-workflows + with: + branch: ${{ inputs.branch }} + temp-branch: ${{ needs.release-branch.outputs.branch }} + secrets: inherit build-artifacts: - name: "Build ${{ inputs.package }} ${{ inputs.version }}" - needs: [release-inputs, bump-version-generate-changelog] + name: "Build the artifacts" + if: ${{ !failure() && !cancelled() }} + needs: [merge-changes, release-inputs] uses: dbt-labs/dbt-adapters/.github/workflows/build.yml@update-workflows with: package: ${{ inputs.package }} version: ${{ inputs.version }} deploy-to: ${{ inputs.deploy-to }} - # TODO: update GH release to handle adding dbt-tests-adapter and dbt-adapter assets to the same release publish-github: - name: "Publish to GitHub" + name: "Publish the artifacts to GitHub" if: ${{ !failure() && !cancelled() && inputs.package == 'dbt-adapters' }} - needs: [build-artifacts, bump-version-generate-changelog] + needs: [build-artifacts, merge-changes] uses: dbt-labs/dbt-adapters/.github/workflows/publish-github.yml@update-workflows with: archive-name: ${{ needs.build-artifacts.outputs.archive-name }} version: ${{ inputs.version }} - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} + sha: ${{ needs.merge-changes.outputs.sha }} changelog-path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} deploy-to: ${{ inputs.deploy-to }} publish-0pypi: - name: "Publish to PyPI" + name: "Publish the artifacts to PyPI" + if: ${{ !failure() && !cancelled() }} needs: [build-artifacts] uses: dbt-labs/dbt-adapters/.github/workflows/publish-github.yml@update-workflows with: diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml deleted file mode 100644 index 32a267e0..00000000 --- a/.github/workflows/release_prep_hatch.yml +++ /dev/null @@ -1,542 +0,0 @@ -# **what?** -# Perform the version bump, generate the changelog and run tests. -# -# Inputs: -# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) -# target_branch: The branch that we will release from -# env_setup_script_path: Path to the environment setup script -# deploy_to: If we are deploying to prod or test, if test then release from branch -# nightly_release: Identifier that this is nightly release -# -# Outputs: -# final_sha: The sha that will actually be released. This can differ from the -# input sha if adding a version bump and/or changelog -# changelog_path: Path to the changelog file (ex .changes/1.2.3-rc1.md) -# -# Branching strategy: -# - During execution workflow execution the temp branch will be generated. -# - For normal runs the temp branch will be removed once changes were merged to target branch; -# - For test runs we will keep temp branch and will use it for release; -# Naming strategy: -# - For normal runs: prep-release/${{ inputs.deploy_to}}/${{ inputs.version_number }}_$GITHUB_RUN_ID -# - For nightly releases: prep-release/nightly-release/${{ inputs.version_number }}_$GITHUB_RUN_ID -# -# **why?** -# Reusable and consistent GitHub release process. -# -# **when?** -# Call when ready to kick off a build and release -# -# Validation Checks -# -# 1. Bump the version if it has not been bumped -# 2. Generate the changelog (via changie) if there is no markdown file for this version -# - -name: Version Bump and Changelog Generation -run-name: Bump ${{ inputs.package }}==${{ inputs.version_number }} for release to ${{ inputs.deploy_to }} and generate changelog -on: - workflow_call: - inputs: - version_number: - required: true - type: string - deploy_to: - type: string - default: prod - required: false - nightly_release: - type: boolean - default: false - required: false - env_setup_script_path: - type: string - required: false - default: '' - run-unit-tests: - type: boolean - default: false - run-integration-tests: - type: boolean - default: false - target_branch: - description: "The branch to release from" - type: string - required: false - default: main - working-dir: - description: "The working directory to use for run statements" - type: string - default: "./" - outputs: - changelog_path: - description: The path to the changelog for this version - value: ${{ jobs.audit-changelog.outputs.changelog_path }} - final_sha: - description: The sha that will actually be released - value: ${{ jobs.determine-release-branch.outputs.final_sha }} - secrets: - FISHTOWN_BOT_PAT: - description: "Token to commit/merge changes into branches" - required: true - IT_TEAM_MEMBERSHIP: - description: "Token that can view org level teams" - required: true - -permissions: - contents: write - -defaults: - run: - shell: bash - -env: - PYTHON_TARGET_VERSION: 3.11 - NOTIFICATION_PREFIX: "[Release Preparation]" - -jobs: - log-inputs: - runs-on: ubuntu-latest - - steps: - - name: "[DEBUG] Print Variables" - run: | - # WORKFLOW INPUTS - echo The release version number: ${{ inputs.version_number }} - echo Deploy to: ${{ inputs.deploy_to }} - echo Target branch: ${{ inputs.target_branch }} - echo Nightly release: ${{ inputs.nightly_release }} - echo Optional env setup script: ${{ inputs.env_setup_script_path }} - echo run-unit-tests: ${{ inputs.run-unit-tests }} - echo run-integration-tests: ${{ inputs.run-integration-tests }} - echo working-dir: ${{ inputs.working-dir }} - # ENVIRONMENT VARIABLES - echo Python target version: ${{ env.PYTHON_TARGET_VERSION }} - echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} - audit-changelog: - runs-on: ubuntu-latest - - outputs: - changelog_path: ${{ steps.set_path.outputs.changelog_path }} - exists: ${{ steps.set_existence.outputs.exists }} - base_version: ${{ steps.semver.outputs.base-version }} - prerelease: ${{ steps.semver.outputs.pre-release }} - is_prerelease: ${{ steps.semver.outputs.is-pre-release }} - - steps: - - name: "Checkout ${{ github.repository }}" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.target_branch }} - - - name: "Audit Version And Parse Into Parts" - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ inputs.version_number }} - - - name: "Set Changelog Path" - id: set_path - run: | - path=".changes/" - if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] - then - path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" - else - path+="${{ steps.semver.outputs.base-version }}.md" - fi - # Send notification - echo "changelog_path=$path" >> $GITHUB_OUTPUT - title="Changelog path" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$changelog_path" - - name: "Set Changelog Existence For Subsequent Jobs" - id: set_existence - run: | - does_exist=false - if test -f ${{ steps.set_path.outputs.changelog_path }} - then - does_exist=true - fi - echo "exists=$does_exist">> $GITHUB_OUTPUT - - name: "[Notification] Set Changelog Existence For Subsequent Jobs" - run: | - title="Changelog exists" - if [[ ${{ steps.set_existence.outputs.exists }} == true ]] - then - message="Changelog file ${{ steps.set_path.outputs.changelog_path }} already exists" - else - message="Changelog file ${{ steps.set_path.outputs.changelog_path }} doesn't exist" - fi - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "[DEBUG] Print Outputs" - run: | - echo changelog_path: ${{ steps.set_path.outputs.changelog_path }} - echo exists: ${{ steps.set_existence.outputs.exists }} - echo base_version: ${{ steps.semver.outputs.base-version }} - echo prerelease: ${{ steps.semver.outputs.pre-release }} - echo is_prerelease: ${{ steps.semver.outputs.is-pre-release }} - - audit-version-in-code: - runs-on: ubuntu-latest - - outputs: - up_to_date: ${{ steps.version-check.outputs.up_to_date }} - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ inputs.target_branch }}" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.target_branch }} - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: "Check Current Version In Code" - id: version-check - run: | - is_updated=false - current_version=$(hatch version) - if test "$current_version" = "${{ inputs.version_number }}" - then - is_updated=true - fi - echo "up_to_date=$is_updated" >> $GITHUB_OUTPUT - working-directory: ${{ inputs.working-dir }} - - - name: "[Notification] Check Current Version In Code" - run: | - title="Version check" - if [[ ${{ steps.version-check.outputs.up_to_date }} == true ]] - then - message="The version in the codebase is equal to the provided version" - else - message="The version in the codebase differs from the provided version" - fi - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "[DEBUG] Print Outputs" - run: | - echo up_to_date: ${{ steps.version-check.outputs.up_to_date }} - - skip-generate-changelog: - runs-on: ubuntu-latest - needs: [audit-changelog] - if: needs.audit-changelog.outputs.exists == 'true' - - steps: - - name: "Changelog Exists, Skip Generating New Changelog" - run: | - # Send notification - title="Skip changelog generation" - message="A changelog file already exists at ${{ needs.audit-changelog.outputs.changelog_path }}, skipping generating changelog" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - skip-version-bump: - runs-on: ubuntu-latest - needs: [audit-version-in-code] - if: needs.audit-version-in-code.outputs.up_to_date == 'true' - - steps: - - name: "Version Already Bumped" - run: | - # Send notification - title="Skip version bump" - message="The version has already been bumped to ${{ inputs.version_number }}, skipping version bump" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - create-temp-branch: - runs-on: ubuntu-latest - needs: [audit-changelog, audit-version-in-code] - if: needs.audit-changelog.outputs.exists == 'false' || needs.audit-version-in-code.outputs.up_to_date == 'false' - - outputs: - branch_name: ${{ steps.variables.outputs.branch_name }} - - steps: - - name: "Checkout ${{ github.repository }}" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.target_branch }} - - - name: "Generate Branch Name" - id: variables - run: | - name="prep-release/" - if [[ ${{ inputs.nightly_release }} == true ]] - then - name+="nightly-release/" - else - name+="${{ inputs.deploy_to }}/" - fi - name+="${{ inputs.version_number }}_$GITHUB_RUN_ID" - echo "branch_name=$name" >> $GITHUB_OUTPUT - - name: "Create Branch - ${{ steps.variables.outputs.branch_name }}" - run: | - git checkout -b ${{ steps.variables.outputs.branch_name }} - git push -u origin ${{ steps.variables.outputs.branch_name }} - - name: "[Notification] Temp branch created" - run: | - # Send notification - title="Temp branch generated" - message="The ${{ steps.variables.outputs.branch_name }} branch created" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "[DEBUG] Print Outputs" - run: | - echo branch_name ${{ steps.variables.outputs.branch_name }} - generate-changelog-bump-version: - runs-on: ubuntu-latest - needs: [audit-changelog, audit-version-in-code, create-temp-branch] - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 - with: - ref: ${{ needs.create-temp-branch.outputs.branch_name }} - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - name: "Add Homebrew To PATH" - run: | - echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH - - name: "Install Homebrew Packages" - run: | - brew install pre-commit - brew tap miniscruff/changie https://github.com/miniscruff/changie - brew install changie - - name: "Set json File Name" - id: json_file - run: | - echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT - - name: "Get Core Team Membership" - run: | - gh api -H "Accept: application/vnd.github+json" orgs/dbt-labs/teams/core-group/members > ${{ steps.json_file.outputs.name }} - env: - GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} - - name: "Set Core Team Membership for Changie Contributors exclusion" - id: set_team_membership - run: | - team_list=$(jq -r '.[].login' ${{ steps.json_file.outputs.name }}) - echo $team_list - team_list_single=$(echo $team_list | tr '\n' ' ') - echo "CHANGIE_CORE_TEAM=$team_list_single" >> $GITHUB_ENV - - name: "Delete the json File" - run: | - rm ${{ steps.json_file.outputs.name }} - - name: "Generate Release Changelog" - if: needs.audit-changelog.outputs.exists == 'false' - run: | - if [[ ${{ needs.audit-changelog.outputs.is_prerelease }} -eq 1 ]] - then - changie batch ${{ needs.audit-changelog.outputs.base_version }} --move-dir '${{ needs.audit-changelog.outputs.base_version }}' --prerelease ${{ needs.audit-changelog.outputs.prerelease }} - elif [[ -d ".changes/${{ needs.audit-changelog.outputs.base_version }}" ]] - then - changie batch ${{ needs.audit-changelog.outputs.base_version }} --include '${{ needs.audit-changelog.outputs.base_version }}' --remove-prereleases - else # releasing a final patch with no prereleases - changie batch ${{ needs.audit-changelog.outputs.base_version }} - fi - changie merge - git status - - name: "Check Changelog Created Successfully" - if: needs.audit-changelog.outputs.exists == 'false' - run: | - title="Changelog" - if [[ -f ${{ needs.audit-changelog.outputs.changelog_path }} ]] - then - message="Changelog file created successfully" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - else - message="Changelog failed to generate" - echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 1 - fi - - name: "Bump Version To ${{ inputs.version_number }}" - if: needs.audit-version-in-code.outputs.up_to_date == 'false' - run: | - hatch version ${{ inputs.version_number }} - working-directory: ${{ inputs.working-dir }} - - name: "[Notification] Bump Version To ${{ inputs.version_number }}" - if: needs.audit-version-in-code.outputs.up_to_date == 'false' - run: | - title="Version bump" - message="Version successfully bumped in codebase to ${{ inputs.version_number }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - # TODO: can these 2 steps be done via hatch? probably. - # this step will fail on whitespace errors but also correct them - - name: "Remove Trailing Whitespace Via Pre-commit" - continue-on-error: true - run: | - pre-commit run trailing-whitespace --files dbt/adapters/__about__.py CHANGELOG.md .changes/* - git status - # this step will fail on newline errors but also correct them - - name: "Removing Extra Newlines Via Pre-commit" - continue-on-error: true - run: | - pre-commit run end-of-file-fixer --files dbt/adapters/__about__.py CHANGELOG.md .changes/* - git status - - name: "Commit & Push Changes" - run: | - #Data for commit - user="Github Build Bot" - email="buildbot@fishtownanalytics.com" - commit_message="Bumping version to ${{ inputs.version_number }} and generate changelog" - #Commit changes to branch - git config user.name "$user" - git config user.email "$email" - git pull - git add . - git commit -m "$commit_message" - git push - - run-unit-tests: - if: inputs.run-unit-tests == true - runs-on: ubuntu-latest - needs: [create-temp-branch, generate-changelog-bump-version] - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 - with: - ref: ${{ needs.create-temp-branch.outputs.branch_name }} - - name: "Setup `hatch`" - uses: ./.github/actions/setup-hatch - - name: "Run Unit Tests" - run: hatch run unit-tests:all - - run-integration-tests: - runs-on: ubuntu-20.04 - needs: [create-temp-branch, generate-changelog-bump-version] - if: inputs.run-integration-tests == true - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 - with: - ref: ${{ needs.create-temp-branch.outputs.branch_name }} - - - name: "Setup Environment Variables - ./${{ inputs.env_setup_script_path }}" - if: inputs.env_setup_script_path != '' - run: source ./${{ inputs.env_setup_script_path }} - - - name: "Setup Environment Variables - Secrets Context" - if: inputs.env_setup_script_path != '' - uses: actions/github-script@v6 - id: check-env - with: - result-encoding: string - script: | - try { - const { SECRETS_CONTEXT, INTEGRATION_TESTS_SECRETS_PREFIX } = process.env - const secrets = JSON.parse(SECRETS_CONTEXT) - if (INTEGRATION_TESTS_SECRETS_PREFIX) { - for (const [key, value] of Object.entries(secrets)) { - if (key.startsWith(INTEGRATION_TESTS_SECRETS_PREFIX)) { - core.exportVariable(key, value) - } - } - } else { - core.info("The INTEGRATION_TESTS_SECRETS_PREFIX env variable is empty or not defined, skipping the secrets setup.") - } - } catch (err) { - core.error("Error while reading or parsing the JSON") - core.setFailed(err) - } - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: "Set up Python & Hatch - ${{ env.PYTHON_TARGET_VERSION }}" - uses: ./.github/actions/setup-python-env - with: - python-version: ${{ env.PYTHON_TARGET_VERSION }} - - - name: Run tests - run: hatch run integration-tests:all - - merge-changes-into-target-branch: - runs-on: ubuntu-latest - needs: [run-unit-tests, run-integration-tests, create-temp-branch, audit-version-in-code, audit-changelog] - if: | - !failure() && !cancelled() && - inputs.deploy_to == 'prod' && - ( - needs.audit-changelog.outputs.exists == 'false' || - needs.audit-version-in-code.outputs.up_to_date == 'false' - ) - steps: - - name: "[Debug] Print Variables" - run: | - echo branch_name: ${{ needs.create-temp-branch.outputs.branch_name }} - echo inputs.deploy_to: ${{ inputs.deploy_to }} - echo needs.audit-changelog.outputs.exists: ${{ needs.audit-changelog.outputs.exists }} - echo needs.audit-version-in-code.outputs.up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} - - name: "Checkout Repo ${{ github.repository }}" - uses: actions/checkout@v3 - - - name: "Merge Changes Into ${{ inputs.target_branch }}" - uses: everlytic/branch-merge@1.1.5 - with: - source_ref: ${{ needs.create-temp-branch.outputs.branch_name }} - target_branch: ${{ inputs.target_branch }} - github_token: ${{ secrets.FISHTOWN_BOT_PAT }} - commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" - - - name: "[Notification] Changes Merged into main" - run: | - title="Changelog and Version Bump Branch Merge" - message="The ${{ needs.create-temp-branch.outputs.branch_name }} branch was merged into mains" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - determine-release-branch: - runs-on: ubuntu-latest - needs: - [ - create-temp-branch, - merge-changes-into-target-branch, - audit-changelog, - audit-version-in-code, - ] - # always run this job, regardless of if the dependant jobs were skipped - if: ${{ !failure() && !cancelled() }} - - # Get the sha that will be released. If the changelog already exists on the input sha and the version has already been bumped, - # then it is what we will release. Otherwise we generated a changelog and did the version bump in this workflow and there is a - # new sha to use from the merge we just did. Grab that here instead. - outputs: - final_sha: ${{ steps.resolve_commit_sha.outputs.release_sha }} - - steps: - - name: "[Debug] Print Variables" - run: | - echo new_branch: ${{ needs.create-temp-branch.outputs.branch_name }} - echo changelog_exists: ${{ needs.audit-changelog.outputs.exists }} - echo up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} - - name: "Resolve Branch To Checkout" - id: resolve_branch - run: | - branch="" - if [ ${{ inputs.deploy_to == 'test' }}] || [ ${{ inputs.nightly_release == 'true' }} ] - then - branch=${{ needs.create-temp-branch.outputs.branch_name }} - else - branch="${{ inputs.target_branch }}" - fi - echo "target_branch=$branch" >> $GITHUB_OUTPUT - - name: "[Notification] Resolve Branch To Checkout" - run: | - title="Branch pick" - message="The ${{ steps.resolve_branch.outputs.target_branch }} branch will be used for release" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "Checkout Resolved Branch - ${{ steps.resolve_branch.outputs.target_branch }}" - uses: actions/checkout@v3 - with: - ref: ${{ steps.resolve_branch.outputs.target_branch }} - - - name: "[Debug] Log Branch" - run: git status - - - name: "Resolve Commit SHA For Release" - id: resolve_commit_sha - run: | - echo "release_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: "Remove Temp Branch - ${{ needs.create-temp-branch.outputs.branch_name }}" - if: ${{ inputs.deploy_to == 'prod' && inputs.nightly_release == 'false' && needs.create-temp-branch.outputs.branch_name != '' }} - run: | - git push origin -d ${{ needs.create-temp-branch.outputs.branch_name }} \ No newline at end of file diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5022a0c6..de729061 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -7,29 +7,51 @@ on: - "*.latest" pull_request: workflow_dispatch: + inputs: + branch: + description: "Choose what branch to test" + type: string + default: "main" + workflow_call: + inputs: + branch: + description: "Choose what branch to test" + type: string + default: "main" permissions: read-all # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'workflow_') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} cancel-in-progress: true +env: + NOTIFICATION_PREFIX: "[Unit tests]" + jobs: + debug-inputs: + name: "[DEBUG] Inputs" + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo branch : ${{ inputs.branch }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + unit: name: "Unit tests" runs-on: ${{ matrix.os }} - strategy: fail-fast: false matrix: python-version: ["3.8", "3.9", "3.10", "3.11"] os: [ubuntu-latest] - steps: - - name: "Check out repository" + - name: "Check out `${{ contains(github.event_name, 'workflow_') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}`" uses: actions/checkout@v4 with: + ref: ${{ contains(github.event_name, 'workflow_') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} persist-credentials: false - name: "Setup `hatch`" @@ -43,7 +65,7 @@ jobs: - name: "Publish results" uses: ./.github/actions/publish-test-results - if: always() + if: ${{ contains(github.event_name, 'workflow_call') }} with: archive-type: "unit_results" python-version: ${{ matrix.python-version }} diff --git a/pyproject.toml b/pyproject.toml index f9b8b0f7..9111b41d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,11 +65,16 @@ dependencies = [ ] [tool.hatch.envs.lint.scripts] all = [ - "- black-only", - "- flake8-only", + "- black", + "- flake8", + "- pre-commit" +] +black = "python -m black {args:.}" +flake8 = "python -m flake8 {args:.}" +pre-commit = [ + "pre-commit run trailing-whitespace {args}", + "pre-commit run end-of-file-fixer {args}", ] -black-only = "python -m black ." -flake8-only = "python -m flake8 ." [tool.hatch.envs.typecheck] dependencies = [ From 96fd1aa992d9e78a60552e4e96970010daf47b41 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 7 Mar 2024 00:06:03 -0500 Subject: [PATCH 03/16] push debug print jobs into main job to simplify PR checks --- .github/workflows/code-quality.yml | 8 ++------ .github/workflows/unit-tests.yml | 14 +++++--------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 7661f9aa..52fe0daf 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -47,8 +47,8 @@ env: NOTIFICATION_PREFIX: "[Code quality]" jobs: - debug-inputs: - name: "[DEBUG] Inputs" + code-quality: + name: "Code quality" runs-on: ubuntu-latest steps: - name: "[DEBUG] Inputs" @@ -58,10 +58,6 @@ jobs: echo typecheck-command : ${{ inputs.typecheck-command }} echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} - code-quality: - name: "Code quality" - runs-on: ubuntu-latest - steps: - name: "Check out `${{ inputs.branch }}`" uses: actions/checkout@v4 with: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index de729061..43915f40 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -30,15 +30,6 @@ env: NOTIFICATION_PREFIX: "[Unit tests]" jobs: - debug-inputs: - name: "[DEBUG] Inputs" - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Inputs" - run: | - echo branch : ${{ inputs.branch }} - echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} - unit: name: "Unit tests" runs-on: ${{ matrix.os }} @@ -48,6 +39,11 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11"] os: [ubuntu-latest] steps: + - name: "[DEBUG] Inputs" + run: | + echo branch : ${{ inputs.branch }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + - name: "Check out `${{ contains(github.event_name, 'workflow_') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}`" uses: actions/checkout@v4 with: From ddd88ab5319487a998b0bd950066cc0f8672b99e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 7 Mar 2024 00:08:59 -0500 Subject: [PATCH 04/16] push debug print jobs into main job to simplify PR checks --- .github/workflows/code-quality.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 52fe0daf..02b795a3 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -48,7 +48,6 @@ env: jobs: code-quality: - name: "Code quality" runs-on: ubuntu-latest steps: - name: "[DEBUG] Inputs" From 2d7416b1777fbfc78a86672f596cdb6a27127bf4 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 7 Mar 2024 00:09:55 -0500 Subject: [PATCH 05/16] push debug print jobs into main job to simplify PR checks --- .github/workflows/code-quality.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 02b795a3..b513e79c 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -1,4 +1,5 @@ name: "Code quality" +run-name: "Code quality" on: push: @@ -48,6 +49,7 @@ env: jobs: code-quality: + name: "Code quality" runs-on: ubuntu-latest steps: - name: "[DEBUG] Inputs" From b58b043f136bbc86347b370deaffa70ad9c0b3b1 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 7 Mar 2024 00:19:49 -0500 Subject: [PATCH 06/16] push debug print jobs into main job to simplify PR checks --- .github/workflows/code-quality.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index b513e79c..52fe0daf 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -1,5 +1,4 @@ name: "Code quality" -run-name: "Code quality" on: push: From 7d9af83bcb407f3bca9bebfc73100afa09883620 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 9 Mar 2024 16:02:09 -0500 Subject: [PATCH 07/16] update unit tests workflow to also publish artifacts on workflow dispatch --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 43915f40..27f950f2 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -61,7 +61,7 @@ jobs: - name: "Publish results" uses: ./.github/actions/publish-test-results - if: ${{ contains(github.event_name, 'workflow_call') }} + if: ${{ contains(github.event_name, 'workflow_') }} with: archive-type: "unit_results" python-version: ${{ matrix.python-version }} From ff2c8133ebb6d1e66e5c55ac0287dd8b15a66d02 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 9 Mar 2024 16:04:43 -0500 Subject: [PATCH 08/16] fix typo to get archive_name variable instead of a command --- .github/actions/publish-test-results/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/publish-test-results/action.yml b/.github/actions/publish-test-results/action.yml index 2b41b388..4a977b5c 100644 --- a/.github/actions/publish-test-results/action.yml +++ b/.github/actions/publish-test-results/action.yml @@ -29,7 +29,7 @@ runs: run: | timestamp=$(date +'%Y-%m-%dT%H-%M-%S') #no colons allowed for artifacts archive_name = ${{ inputs.archive-type }}_python-${{ inputs.python-version }}_${{ inputs.os }}_$timestamp - echo "archive-name=archive_name" >> $GITHUB_OUTPUT + echo "archive-name=$archive_name" >> $GITHUB_OUTPUT - name: "Upload test results" uses: actions/upload-artifact@v3 From 5ee56e78fafd3981d3df3d4a9ab0dc81e1e42f17 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 9 Mar 2024 16:29:42 -0500 Subject: [PATCH 09/16] fix quoting --- .github/actions/publish-test-results/action.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/publish-test-results/action.yml b/.github/actions/publish-test-results/action.yml index 4a977b5c..1ac7086d 100644 --- a/.github/actions/publish-test-results/action.yml +++ b/.github/actions/publish-test-results/action.yml @@ -27,8 +27,9 @@ runs: id: archive shell: bash run: | - timestamp=$(date +'%Y-%m-%dT%H-%M-%S') #no colons allowed for artifacts - archive_name = ${{ inputs.archive-type }}_python-${{ inputs.python-version }}_${{ inputs.os }}_$timestamp + echo "Create unique test archive name" + timestamp=$(date +'%Y-%m-%dT%H-%M-%S') + archive_name="${{ inputs.archive-type }}_python-${{ inputs.python-version }}_${{ inputs.os }}_$timestamp" echo "archive-name=$archive_name" >> $GITHUB_OUTPUT - name: "Upload test results" @@ -40,6 +41,7 @@ runs: - name: "[INFO] Uploaded test artifacts" shell: bash run: | + echo "[INFO] Uploaded test artifacts" title="Uploaded test artifacts" message="Uploaded ${{ inputs.source-file }} to ${{ steps.archive.outputs.archive-name }}" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" From 1818a670f6a48e0a783f6cef59cbdaa0b4cb3f99 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 9 Mar 2024 16:38:04 -0500 Subject: [PATCH 10/16] fix source file path --- .github/actions/publish-test-results/action.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/actions/publish-test-results/action.yml b/.github/actions/publish-test-results/action.yml index 1ac7086d..3b465293 100644 --- a/.github/actions/publish-test-results/action.yml +++ b/.github/actions/publish-test-results/action.yml @@ -27,7 +27,6 @@ runs: id: archive shell: bash run: | - echo "Create unique test archive name" timestamp=$(date +'%Y-%m-%dT%H-%M-%S') archive_name="${{ inputs.archive-type }}_python-${{ inputs.python-version }}_${{ inputs.os }}_$timestamp" echo "archive-name=$archive_name" >> $GITHUB_OUTPUT @@ -36,12 +35,11 @@ runs: uses: actions/upload-artifact@v3 with: name: ${{ steps.archive.outputs.archive-name }} - path: ${{ inputs.source-file }} + path: ./${{ inputs.source-file }} - name: "[INFO] Uploaded test artifacts" shell: bash run: | - echo "[INFO] Uploaded test artifacts" title="Uploaded test artifacts" message="Uploaded ${{ inputs.source-file }} to ${{ steps.archive.outputs.archive-name }}" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" From df8cdceca578739773f44ee5dfe5003477c61996 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 9 Mar 2024 16:56:39 -0500 Subject: [PATCH 11/16] update test archive publishing --- .../action.yml | 21 ++++++++----------- .github/workflows/unit-tests.yml | 7 +++---- 2 files changed, 12 insertions(+), 16 deletions(-) rename .github/actions/{publish-test-results => publish-test-logs}/action.yml (64%) diff --git a/.github/actions/publish-test-results/action.yml b/.github/actions/publish-test-logs/action.yml similarity index 64% rename from .github/actions/publish-test-results/action.yml rename to .github/actions/publish-test-logs/action.yml index 3b465293..7b0916d8 100644 --- a/.github/actions/publish-test-results/action.yml +++ b/.github/actions/publish-test-logs/action.yml @@ -1,5 +1,5 @@ -name: "Publish test results" -description: "Upload test artifacts to a unique archive name" +name: "Publish test logs" +description: "Upload test logs to a unique archive name" inputs: archive-type: @@ -11,9 +11,6 @@ inputs: os: description: "OS for the archive name stub (e.g. ubuntu-latest)" required: true - source-file: - description: "File/directory to be uploaded" - required: true outputs: archive-name: @@ -23,7 +20,7 @@ outputs: runs: using: composite steps: - - name: "Create unique test archive name" + - name: "Create unique archive name" id: archive shell: bash run: | @@ -31,15 +28,15 @@ runs: archive_name="${{ inputs.archive-type }}_python-${{ inputs.python-version }}_${{ inputs.os }}_$timestamp" echo "archive-name=$archive_name" >> $GITHUB_OUTPUT - - name: "Upload test results" - uses: actions/upload-artifact@v3 + - name: "Upload test logs" + uses: actions/upload-artifact@v4 with: name: ${{ steps.archive.outputs.archive-name }} - path: ./${{ inputs.source-file }} + path: ./logs - - name: "[INFO] Uploaded test artifacts" + - name: "[INFO] Uploaded test logs" shell: bash run: | - title="Uploaded test artifacts" - message="Uploaded ${{ inputs.source-file }} to ${{ steps.archive.outputs.archive-name }}" + title="Uploaded test logs" + message="Uploaded `./logs` to ${{ steps.archive.outputs.archive-name }}" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 27f950f2..6520a894 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -59,11 +59,10 @@ jobs: run: hatch run unit-tests:all shell: bash - - name: "Publish results" - uses: ./.github/actions/publish-test-results + - name: "Publish test logs" + uses: ./.github/actions/publish-test-logs if: ${{ contains(github.event_name, 'workflow_') }} with: - archive-type: "unit_results" + archive-type: "unit-test" python-version: ${{ matrix.python-version }} os: ${{ matrix.os }} - source-file: "results.csv" From e1509a4b638551d6a9d053b4b07a24722a72da0b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 9 Mar 2024 16:59:33 -0500 Subject: [PATCH 12/16] remove log publishing --- .github/actions/publish-test-logs/action.yml | 42 -------------------- .github/workflows/unit-tests.yml | 8 ---- 2 files changed, 50 deletions(-) delete mode 100644 .github/actions/publish-test-logs/action.yml diff --git a/.github/actions/publish-test-logs/action.yml b/.github/actions/publish-test-logs/action.yml deleted file mode 100644 index 7b0916d8..00000000 --- a/.github/actions/publish-test-logs/action.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: "Publish test logs" -description: "Upload test logs to a unique archive name" - -inputs: - archive-type: - description: "File type for archive name stub (e.g. unit-tests)" - required: true - python-version: - description: "Python version for the archive name stub (e.g. 3.8)" - required: true - os: - description: "OS for the archive name stub (e.g. ubuntu-latest)" - required: true - -outputs: - archive-name: - description: "Name of the archive containing the test artifacts" - value: ${{ steps.archive.outputs.archive-name }} - -runs: - using: composite - steps: - - name: "Create unique archive name" - id: archive - shell: bash - run: | - timestamp=$(date +'%Y-%m-%dT%H-%M-%S') - archive_name="${{ inputs.archive-type }}_python-${{ inputs.python-version }}_${{ inputs.os }}_$timestamp" - echo "archive-name=$archive_name" >> $GITHUB_OUTPUT - - - name: "Upload test logs" - uses: actions/upload-artifact@v4 - with: - name: ${{ steps.archive.outputs.archive-name }} - path: ./logs - - - name: "[INFO] Uploaded test logs" - shell: bash - run: | - title="Uploaded test logs" - message="Uploaded `./logs` to ${{ steps.archive.outputs.archive-name }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6520a894..6ff00368 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -58,11 +58,3 @@ jobs: - name: "Run unit tests" run: hatch run unit-tests:all shell: bash - - - name: "Publish test logs" - uses: ./.github/actions/publish-test-logs - if: ${{ contains(github.event_name, 'workflow_') }} - with: - archive-type: "unit-test" - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} From da1191c71e087eebab2544c0933e73153f49925e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 9 Mar 2024 17:08:44 -0500 Subject: [PATCH 13/16] update type for deploy-to to environment --- .github/workflows/publish-github.yml | 6 +----- .github/workflows/publish-pypi.yml | 5 +---- .github/workflows/release.yml | 5 +---- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish-github.yml b/.github/workflows/publish-github.yml index 6b1e1d11..a3350395 100644 --- a/.github/workflows/publish-github.yml +++ b/.github/workflows/publish-github.yml @@ -33,10 +33,7 @@ on: required: true deploy-to: description: "Choose where to publish" - type: choice - options: - - "prod" - - "test" + type: environment workflow_call: inputs: archive-name: @@ -119,7 +116,6 @@ jobs: - name: "[INFO] Release already exists" if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == true }} && ${{ steps.tag-metadata.outputs.sha }} == ${{ inputs.sha }} - id: release_check run: | title="Tag already exists, skip publishing artifacts" message="Tag ${{ steps.tag.outputs.tag }} already exists and is associated with the commit ${{ steps.tag-metadata.outputs.sha }}." diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index efc97aa7..4710b3d3 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -19,10 +19,7 @@ on: required: true deploy-to: description: "Choose where to publish" - type: choice - options: - - "prod" - - "test" + type: environment workflow_call: inputs: archive-name: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9ad5a51e..ff8baf70 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,10 +20,7 @@ on: required: true deploy-to: description: "Choose where to publish" - type: choice - options: - - "prod" - - "test" + type: environment workflow_call: inputs: package: From a8f48a260fb33c573afd3cf0e334c2b8d4375d27 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 19 Mar 2024 20:23:02 -0400 Subject: [PATCH 14/16] update workflows --- .github/actions/audit-changelog/action.yml | 7 +- .../actions/audit-github-commit/action.yml | 13 +- .github/actions/audit-github-tag/action.yml | 14 +- .github/actions/audit-github-team/action.yml | 25 ++- .github/actions/create-temp-branch/action.yml | 6 + .../{commit => github-commit}/action.yml | 24 ++- .github/actions/github-merge/action.yml | 59 ++++++ .github/actions/log-info/action.yml | 22 +++ .../actions/publish-github-draft/action.yml | 61 ++---- .../actions/publish-github-full/action.yml | 26 --- .github/actions/publish-github/action.yml | 91 +++++++++ .github/actions/publish-pypi/action.yml | 11 +- .github/actions/setup-changie/action.yml | 3 +- .github/actions/setup-environment/action.yml | 28 +++ .github/actions/setup-hatch/action.yml | 22 --- .github/workflows/build-artifacts.yml | 112 +++++++++++ ...rate-changelog.yml => build-changelog.yml} | 86 +++++---- ...ump-version.yml => build-version-bump.yml} | 93 ++++++---- .github/workflows/build.yml | 106 ----------- .github/workflows/check-changelog-entry.yml | 35 ++++ ...ode-quality.yml => check-code-quality.yml} | 26 +-- .../workflows/check-for-changelog-entry.yml | 39 ---- .../{unit-tests.yml => check-unit-tests.yml} | 15 +- ...age.yml => issues-resubmit-for-triage.yml} | 7 +- .github/workflows/issues-stale.yml | 24 +++ .github/workflows/publish-github.yml | 175 +++++++++++------- .github/workflows/publish-pypi.yml | 12 +- .github/workflows/release-branch-create.yml | 2 - .github/workflows/release-branch-merge.yml | 53 +++--- .github/workflows/release.yml | 121 ++++++------ .github/workflows/stale.yml | 30 --- 31 files changed, 759 insertions(+), 589 deletions(-) rename .github/actions/{commit => github-commit}/action.yml (53%) create mode 100644 .github/actions/github-merge/action.yml create mode 100644 .github/actions/log-info/action.yml delete mode 100644 .github/actions/publish-github-full/action.yml create mode 100644 .github/actions/publish-github/action.yml create mode 100644 .github/actions/setup-environment/action.yml delete mode 100644 .github/actions/setup-hatch/action.yml create mode 100644 .github/workflows/build-artifacts.yml rename .github/workflows/{generate-changelog.yml => build-changelog.yml} (81%) rename .github/workflows/{bump-version.yml => build-version-bump.yml} (58%) delete mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/check-changelog-entry.yml rename .github/workflows/{code-quality.yml => check-code-quality.yml} (62%) delete mode 100644 .github/workflows/check-for-changelog-entry.yml rename .github/workflows/{unit-tests.yml => check-unit-tests.yml} (89%) rename .github/workflows/{resubmit-for-triage.yml => issues-resubmit-for-triage.yml} (87%) create mode 100644 .github/workflows/issues-stale.yml delete mode 100644 .github/workflows/stale.yml diff --git a/.github/actions/audit-changelog/action.yml b/.github/actions/audit-changelog/action.yml index cc878fae..fb878fcf 100644 --- a/.github/actions/audit-changelog/action.yml +++ b/.github/actions/audit-changelog/action.yml @@ -1,5 +1,5 @@ name: "Audit changelog" -description: "Get metadata about a changelog, including the expected file path, and if it exists" +description: "Get metadata about a changelog" inputs: version: @@ -17,6 +17,11 @@ outputs: runs: using: composite steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo version : ${{ inputs.version }} + - name: "Parse version: `${{ inputs.version }}`" id: semver uses: dbt-labs/actions/parse-semver@v1.1.0 diff --git a/.github/actions/audit-github-commit/action.yml b/.github/actions/audit-github-commit/action.yml index 5d623884..852293b8 100644 --- a/.github/actions/audit-github-commit/action.yml +++ b/.github/actions/audit-github-commit/action.yml @@ -23,6 +23,11 @@ outputs: runs: using: composite steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo sha : ${{ inputs.sha }} + - name: "Check if a release exists for `${{ inputs.sha }}`" id: commit env: @@ -37,7 +42,7 @@ runs: shell: bash run: | echo sha : ${{ inputs.sha }} - echo tag : ${{ steps.check_release_commit.outputs.tag_name }} - echo release-id : ${{ steps.check_release_commit.outputs.id }} - echo pre-release : ${{ steps.check_release_commit.outputs.prerelease }} - echo commitish : ${{ steps.check_release_commit.outputs.target_commitish }} + echo tag : ${{ steps.commit.outputs.tag_name }} + echo release-id : ${{ steps.commit.outputs.id }} + echo pre-release : ${{ steps.commit.outputs.prerelease }} + echo commitish : ${{ steps.commit.outputs.target_commitish }} diff --git a/.github/actions/audit-github-tag/action.yml b/.github/actions/audit-github-tag/action.yml index 1118a505..77a8f608 100644 --- a/.github/actions/audit-github-tag/action.yml +++ b/.github/actions/audit-github-tag/action.yml @@ -23,7 +23,13 @@ outputs: runs: using: composite steps: - - name: "Check if `${{ inputs.tag }}` exists in `${{ inputs.repo-url }}`" + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo tag : ${{ inputs.tag }} + echo repo-url : ${{ inputs.repo-url }} + + - name: "Set: exists" id: tag shell: bash run: | @@ -38,15 +44,15 @@ runs: with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: "Get the commit associated with `${{ inputs.tag }}`" + - name: "Set: sha" id: commit if: ${{ fromJSON(steps.tag.outputs.exists) == true }} shell: bash run: | - sha=$(jq -r '.targetCommitish' <<< "$output") + sha=$(jq -r '.targetCommitish' <<< "${{ steps.tag.outputs.results }}") echo "sha=$sha" >> $GITHUB_OUTPUT - - name: "Check if `${{ inputs.tag }}` is a draft release" + - name: "Set: is-draft" id: draft if: ${{ fromJSON(steps.tag.outputs.exists) == true }} shell: bash diff --git a/.github/actions/audit-github-team/action.yml b/.github/actions/audit-github-team/action.yml index 6923cb8c..fdaca6ba 100644 --- a/.github/actions/audit-github-team/action.yml +++ b/.github/actions/audit-github-team/action.yml @@ -1,10 +1,10 @@ name: "Audit GitHub team" -description: "Get metadata about a GitHub team, such as a list of team members" +description: "Get metadata about a GitHub team" inputs: organization: description: "The GitHub organization that owns the team" - required: true + default: "dbt-labs" team: description: "The name of the team" required: true @@ -17,11 +17,19 @@ outputs: runs: using: composite steps: - - name: "Set: output file name" + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo organization : ${{ inputs.organization }} + echo team : ${{ inputs.team }} + + - name: "Create output file name" id: output-file + shell: bash run: echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT - - name: "Get team membership for `${{ inputs.organization }}/${{ inputs.team }}`" + - name: "Get team membership" + shell: bash run: | url = orgs/${{ inputs.organization }}/teams/${{ inputs.team }}/members header = Accept: application/vnd.github+json @@ -29,18 +37,19 @@ runs: - name: "Parse team membership" id: members - shell: + shell: bash run: | team_list=$(jq -r '.[].login' ${{ steps.output-file.outputs.name }}) team_list_single=$(echo $team_list | tr '\n' ' ') echo "membership=$team_list_single" >> $GITHUB_OUTPUT + - name: "Delete the output file" + shell: bash + run: rm ${{ steps.output-file.outputs.name }} + - name: "[DEBUG] Parse team membership" shell: bash run: | echo organization : ${{ inputs.organization }} echo team : ${{ inputs.team }} echo members : ${{ steps.members.outputs.membership }} - - - name: "Delete the output file" - run: rm ${{ steps.output-file.outputs.name }} diff --git a/.github/actions/create-temp-branch/action.yml b/.github/actions/create-temp-branch/action.yml index cce80b4c..6fe53ceb 100644 --- a/.github/actions/create-temp-branch/action.yml +++ b/.github/actions/create-temp-branch/action.yml @@ -14,8 +14,14 @@ outputs: runs: using: composite steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo branch-stub : ${{ inputs.branch-stub }} + - name: "Create a unique branch name" id: branch + shell: bash run: | name="${{ inputs.branch-stub }}" name+="$(date +'%Y-%m-%d')/$GITHUB_RUN_ID" diff --git a/.github/actions/commit/action.yml b/.github/actions/github-commit/action.yml similarity index 53% rename from .github/actions/commit/action.yml rename to .github/actions/github-commit/action.yml index 8da59ece..0ecab9fb 100644 --- a/.github/actions/commit/action.yml +++ b/.github/actions/github-commit/action.yml @@ -12,17 +12,16 @@ inputs: description: "The email for the commit" default: "buildbot@fishtownanalytics.com" -outputs: - path: - description: "The file path to the change log, relative to the repo root" - value: ${{ steps.path.outputs.path }} - exists: - description: "Does the changelog exist?" - value: ${{ steps.exists.outputs.exists }} - runs: using: composite steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo message : ${{ inputs.message }} + echo user : ${{ inputs.user }} + echo email : ${{ inputs.email }} + - name: "Commit and push changes" shell: bash run: | @@ -30,5 +29,12 @@ runs: git config user.email "${{ inputs.email }}" git pull git add . - git commit -m "${{ inputs.message }}" + git commit -m "[automated] ${{ inputs.message }}" git push + + - name: "[INFO] Committed and pushed changes" + shell: bash + run: | + title="Committed and pushed changes" + message="Committed and pushed changes back up to remote" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/github-merge/action.yml b/.github/actions/github-merge/action.yml new file mode 100644 index 00000000..efe7464c --- /dev/null +++ b/.github/actions/github-merge/action.yml @@ -0,0 +1,59 @@ +name: "Merge GitHub branch" +description: "Merge one branch into another on GitHub" + +inputs: + source-branch: + description: "The source branch to be merged" + required: true + target-branch: + description: "The target branch to receive the merge" + required: true + message: + description: "Commit message for the merge" + required: true + delete-source-branch: + description: "Will delete the source branch after merging" + default: "true" + +outputs: + sha: + description: "The commit SHA for the merge" + value: ${{ steps.merge-sha.outputs.sha }} + +runs: + using: composite + steps: + # TODO: are we comfortable passing our bot token to this third party action? + - name: "Merge `${{ inputs.source-branch }}` into `${{ inputs.target-branch }}`" + uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ inputs.source-branch }} + target_branch: ${{ inputs.target-branch }} + commit_message_template: "[automated] ${{ inputs.message }}" + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + + - name: "Checkout `${{ inputs.branch }}`" + uses: actions/checkout@v3 + with: + ref: ${{ inputs.target-branch }} + + # TODO: see if everlytic/branch-merge has outputs that have this information + - name: "Get commit SHA for the merge" + id: merge-sha + shell: bash + run: | + git pull + echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + # TODO: see if everlytic/branch-merge has options to do this + - name: "Delete `${{ inputs.source-branch }}`" + if: ${{ fromJSON(inputs.delete-source-branch) }} + shell: bash + run: git push origin -d ${{ inputs.source-branch }} + + - name: "[INFO] Merged changes" + shell: bash + run: | + title="Merged changes" + message="${{ inputs.source-branch }} was merged into ${{ inputs.target-branch }} with sha ${{ steps.merge-sha.outputs.sha }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/log-info/action.yml b/.github/actions/log-info/action.yml new file mode 100644 index 00000000..f6ea0cdc --- /dev/null +++ b/.github/actions/log-info/action.yml @@ -0,0 +1,22 @@ +name: "Raise a notification" +description: "Raises a notification to the workflow summary in a standardized pattern" + +inputs: + title: + description: "The notification title" + required: true + message: + description: "The detailed message" + required: true + prefix: + description: "The parent process raising the notification" + default: "INFO" + +runs: + using: composite + steps: + - shell: bash + run: 'echo "::notice title=$TITLE::$MESSAGE"' + env: + TITLE: "[${{ inputs.prefix }}]: ${{ inputs.title }}" + MESSAGE: ${{ inputs.message }} diff --git a/.github/actions/publish-github-draft/action.yml b/.github/actions/publish-github-draft/action.yml index 11d8dc23..8506d94b 100644 --- a/.github/actions/publish-github-draft/action.yml +++ b/.github/actions/publish-github-draft/action.yml @@ -1,71 +1,32 @@ -name: "Publish to GitHub as draft release" -description: "Publish artifacts from an archive to GitHub as a draft release" +name: "Publish GitHub draft release as full release" +description: "Publish an existing draft release as a non-draft release" inputs: - archive-name: - description: "Name of the archive containing the artifacts, leave blank if local" - default: "" tag: - description: "The release tag to publish under" + description: "The tag to publish (i.e. v1.0.0b1)" required: true repo-url: description: "The URL to the repo (https://github.com/dbt-labs/dbt-adapters" required: true - sha: - description: "Commit SHA being released" - required: true - changelog-path: - description: "Path to the release notes" - required: true runs: using: composite steps: - - name: "Download artifacts from `${{ inputs.archive-name }}`" - if: ${{ !(inputs.archive-name == "" }} - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.archive-name }} - path: dist/ - - - name: "[INFO] Downloaded artifacts" - if: ${{ !(inputs.archive-name == "" }} - shell: bash - run: | - title="Downloaded artifacts" - message="Downloaded artifacts from ${{ inputs.archive-name }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - - name: "[DEBUG] Found artifacts" - shell: bash - working-directory: ${{ inputs.working-dir }} - run: echo $(ls ./dist) - - - name: "Set: pre-release" - id: pre-release + - name: "[DEBUG] Inputs" shell: bash run: | - if ${{ contains(inputs.tag, 'rc') || contains(inputs.tag, 'b') || contains(inputs.tag, 'a') }} - then - echo "pre-release=--prerelease" >> $GITHUB_OUTPUT - fi + echo tag : ${{ inputs.tag }} + echo repo-url : ${{ inputs.repo-url }} - - name: "Publish artifacts to GitHub as draft release" + - name: "Publish `${{ inputs.tag }}` as full release" shell: bash - run: | - gh release create $TAG ./dist/* --title "$TITLE" --notes-file $RELEASE_NOTES --target $COMMIT $PRERELEASE --draft --repo ${{ inputs.repo-url }} + run: gh release edit ${{ inputs.tag }} --repo ${{ inputs.repo-url }} --draft=false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ inputs.tag }} - TITLE: ${{ github.event.repository.name }} ${{ inputs.tag }} - RELEASE_NOTES: ${{ inputs.changelog-path }} - COMMIT: ${{ inputs.sha }} - PRERELEASE: ${{ steps.pre-release.outputs.pre-release }} - DRAFT: ${{ inputs.draft-flag }} - - name: "[INFO] Published artifacts to GitHub as draft release" + - name: "[INFO] Published draft as full release" shell: bash run: | - title="Published artifacts to GitHub as draft release" - message="artifacts: $(ls ./dist) tag: ${{ inputs.tag }} pre-release: ${{ steps.prerelease.outputs.prerelease }}" + title="Released draft as non-draft release on GitHub" + message="tag: ${{ inputs.tag }} repo: ${{ inputs.repo-url }}" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/publish-github-full/action.yml b/.github/actions/publish-github-full/action.yml deleted file mode 100644 index 8258c764..00000000 --- a/.github/actions/publish-github-full/action.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: "Publish GitHub draft release as full release" -description: "Publish an existing draft release as a non-draft release" - -inputs: - tag: - description: "The tag to publish (i.e. v1.0.0b1)" - required: true - repo-url: - description: "The URL to the repo (https://github.com/dbt-labs/dbt-adapters" - required: true - -runs: - using: composite - steps: - - name: "Publish `${{ inputs.tag }}` as full release" - shell: bash - run: gh release edit ${{ inputs.tag }} --draft=false --repo ${{ inputs.repo-url }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: "[INFO] Published draft as full release" - shell: bash - run: | - title="Released draft as non-draft release on GitHub" - message="tag: ${{ inputs.tag }} repo: ${{ inputs.repo-url }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/publish-github/action.yml b/.github/actions/publish-github/action.yml new file mode 100644 index 00000000..6042357c --- /dev/null +++ b/.github/actions/publish-github/action.yml @@ -0,0 +1,91 @@ +name: "Publish to GitHub as draft release" +description: "Publish artifacts from an archive to GitHub as a draft release" + +inputs: + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local" + default: "" + tag: + description: "The release tag to publish under" + required: true + repo-url: + description: "The URL to the repo (e.g. https://github.com/dbt-labs/dbt-adapters)" + required: true + sha: + description: "Commit SHA being released" + required: true + changelog-path: + description: "Path to the release notes" + required: true + is-draft: + description: "Indicates that this is a draft release" + required: true + +runs: + using: composite + steps: + - name: "Set: pre-release" + id: pre-release + shell: bash + run: | + if ${{ contains(inputs.tag, 'rc') || contains(inputs.tag, 'b') || contains(inputs.tag, 'a' || contains(inputs.tag, 'dev') }} + then + echo "pre-release=--prerelease" >> $GITHUB_OUTPUT + fi + + - name: "Set: draft" + id: draft + shell: bash + run: | + if ${{ fromJSON(inputs.is-draft) }} + then + echo "draft=--draft" >> $GITHUB_OUTPUT + fi + + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo archive-name : ${{ inputs.archive-name }} + echo tag : ${{ inputs.tag }} + echo repo-url : ${{ inputs.repo-url }} + echo sha : ${{ inputs.sha }} + echo changelog-path : ${{ inputs.changelog-path }} + echo is-draft : ${{ inputs.is-draft }} + echo pre-release : ${{ steps.pre-release.outputs.pre-release }} + echo draft : ${{ steps.draft.outputs.draft }} + + - name: "Download artifacts from `${{ inputs.archive-name }}`" + if: ${{ !(inputs.archive-name == "" }} + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.archive-name }} + path: dist/ + + - name: "[INFO] Downloaded artifacts" + if: ${{ !(inputs.archive-name == "" }} + uses: ./.github/actions/log-info + with: + title: "Downloaded artifacts" + message: "${{ inputs.archive-name }} contained: $(ls ./dist)" + prefix: ${{ env.NOTIFICATION_PREFIX }} + + - name: "Publish artifacts to GitHub as draft release" + shell: bash + run: gh release create $TAG $ARTIFACTS --repo $REPO --title "$TITLE" --notes-file $RELEASE_NOTES --target $COMMIT $PRERELEASE $DRAFT + env: + TAG: ${{ inputs.tag }} + ARTIFACTS: "./dist/*" + REPO: ${{ inputs.repo-url }} + TITLE: ${{ github.event.repository.name }} ${{ inputs.tag }} + RELEASE_NOTES: ${{ inputs.changelog-path }} + COMMIT: ${{ inputs.sha }} + PRERELEASE: ${{ steps.pre-release.outputs.pre-release }} + DRAFT: ${{ steps.draft.outputs.draft }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: "[INFO] Published artifacts to GitHub as draft release" + uses: ./.github/actions/log-info + with: + title: "Published artifacts to GitHub as draft release" + message: "tag: ${{ inputs.tag }} pre-release: ${{ steps.pre-release.outputs.pre-release }} artifacts: $(ls ./dist)" + prefix: ${{ env.NOTIFICATION_PREFIX }} diff --git a/.github/actions/publish-pypi/action.yml b/.github/actions/publish-pypi/action.yml index ad92107b..472b42ca 100644 --- a/.github/actions/publish-pypi/action.yml +++ b/.github/actions/publish-pypi/action.yml @@ -5,7 +5,7 @@ inputs: archive-name: description: "Name of the archive containing the artifacts, leave blank if local" default: "" - repository-url: + index-url: description: "The url for the PyPI index" required: true working-dir: @@ -15,6 +15,13 @@ inputs: runs: using: composite steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo archive-name : ${{ inputs.archive-name }} + echo index-url : ${{ inputs.index-url }} + echo working-dir : ${{ inputs.working-dir }} + - name: "Download artifacts from `${{ inputs.archive-name }}`" if: ${{ !(inputs.archive-name == "" }} uses: actions/download-artifact@v4 @@ -38,7 +45,7 @@ runs: - name: "Publish artifacts to PyPI" uses: pypa/gh-action-pypi-publish@release/v1 with: - repository-url: ${{ inputs.repository-url }} + repository-url: ${{ inputs.index-url }} packages-dir: ${{ inputs.working-dir }}dist/ - name: "[INFO] Published artifacts to PyPI" diff --git a/.github/actions/setup-changie/action.yml b/.github/actions/setup-changie/action.yml index 3a82dd25..55e994e1 100644 --- a/.github/actions/setup-changie/action.yml +++ b/.github/actions/setup-changie/action.yml @@ -8,9 +8,8 @@ runs: shell: bash run: echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH - - name: "Install `linuxbrew` packages" + - name: "Install `changie`" shell: bash run: | - brew install pre-commit brew tap miniscruff/changie https://github.com/miniscruff/changie brew install changie diff --git a/.github/actions/setup-environment/action.yml b/.github/actions/setup-environment/action.yml new file mode 100644 index 00000000..80433cd3 --- /dev/null +++ b/.github/actions/setup-environment/action.yml @@ -0,0 +1,28 @@ +name: "Setup environment" +description: "Setup a python environment with `hatch` as the default build frontend" + +inputs: + python-version: + description: "The version of python to install" + default: "3.8" + setup-build-command: + description: "The command to setup build dependencies" + default: "python -m pip install hatch" + +runs: + using: composite + steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo python-version : ${{ inputs.python-version }} + echo setup-build-command : ${{ inputs.setup-build-command }} + + - name: "Set up python ${{ inputs.python-version }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + + - name: "Install build dependencies" + shell: bash + run: ${{ inputs.setup-build-command }} diff --git a/.github/actions/setup-hatch/action.yml b/.github/actions/setup-hatch/action.yml deleted file mode 100644 index 8c45ec98..00000000 --- a/.github/actions/setup-hatch/action.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "Setup `hatch`" -description: "Setup a python environment with `hatch` installed" - -inputs: - setup-command: - description: "The command to setup development dependencies" - default: "python -m pip install hatch" - python-version: - description: "The version of python to install" - default: "3.11" - -runs: - using: composite - steps: - - name: "Set up python ${{ inputs.python-version }}" - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version }} - - - name: "Install build dependencies" - shell: bash - run: ${{ inputs.setup-command }} diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 00000000..13e08bbb --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,112 @@ +name: "Build artifacts" +run-name: "Build `${{ inputs.package }}==${{ inputs.version}}` from `${{ inputs.branch }}` for deployment to `${{ deploy-environment }}`" + +on: + workflow_call: + inputs: + branch: + description: "The branch being used to build the package" + type: string + default: "main" + package: + description: "The name of the package being built" + type: string + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + deploy-environment: + description: "The environment where this package will be deployed" + type: string + default: "prod" + outputs: + archive-name: ${{ jobs.calculated-inputs.outputs.archive-name }} + workflow_dispatch: + inputs: + branch: + description: "The branch being used to build the package" + type: string + default: "main" + package: + description: "The name of the package being built" + type: choice + options: + - "dbt-adapters" + - "dbt-tests-adapter" + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + deploy-environment: + description: "The environment where this package will be deployed" + type: environment + +permissions: read-all + +# don't try to build the same version of the same package, the archive name could collide +concurrency: + group: ${{ github.workflow }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-environment }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + NOTIFICATION_PREFIX: "[Build]" + +jobs: + calculated-inputs: + name: "Calculated inputs" + runs-on: ubuntu-latest + outputs: + archive-name: ${{ steps.archive.outputs.name }} + working-dir: ${{ steps.working-dir.outputs.path }} + steps: + - name: "Set: archive-name" + id: archive + shell: bash + run: | + name=${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-environment }} + echo "name=$name" >> $GITHUB_OUTPUT + + - name: "Set: working-dir" + id: working-dir + shell: bash + run: | + if [[ ${{ inputs.package }} == "dbt-tests-adapter" ]] + then + path="./dbt-tests-adapter/" + else + path="./" + fi + echo "path=$path" >> $GITHUB_OUTPUT + + - name: "[DEBUG] Inputs" + run: | + echo package : ${{ inputs.package }} + echo branch : ${{ inputs.branch }} + echo version : ${{ inputs.version }} + echo deploy-environment : ${{ inputs.deploy-environment }} + echo archive-name : ${{ steps.archive.outputs.name }} + echo working-dir : ${{ steps.working-dir.outputs.path }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + + build: + name: "Build the artifacts" + needs: [calculated-inputs] + runs-on: ubuntu-latest + steps: + - name: "Check out `${{ inputs.branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup `hatch`" + uses: ./.github/actions/setup-environment + + - name: "Build `${{ inputs.package }}`" + uses: ./.github/actions/build + with: + archive-name: ${{ needs.calculated-inputs.outputs.archive-name }} + working-dir: ${{ needs.calculated-inputs.outputs.working-dir }} diff --git a/.github/workflows/generate-changelog.yml b/.github/workflows/build-changelog.yml similarity index 81% rename from .github/workflows/generate-changelog.yml rename to .github/workflows/build-changelog.yml index 51a91ef8..4e2dfa86 100644 --- a/.github/workflows/generate-changelog.yml +++ b/.github/workflows/build-changelog.yml @@ -1,9 +1,4 @@ -# **what?** -# Merge individual changelogs to generate the final changelog -# -# **when?** -# Call when ready to kick off a build and release, gets used automatically by `release.yml` -name: "Generate changelog" +name: "Build changelog" run-name: "Merge changelogs into one changelog on ${{ inputs.branch }} for ${{ inputs.version }}" on: @@ -55,25 +50,15 @@ defaults: shell: bash env: - NOTIFICATION_PREFIX: "[Generate changelog]" + NOTIFICATION_PREFIX: "[Build changelog]" jobs: - debug-inputs: - name: "[DEBUG] Inputs" - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Inputs" - run: | - echo branch : ${{ inputs.branch }} - echo version : ${{ inputs.version }} - echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} - - audit-changelog: - name: "Audit changelog" + calculated-inputs: + name: "Calculated inputs" runs-on: ubuntu-latest outputs: changelog-path: ${{ steps.audit-changelog.outputs.path }} - exists: ${{ steps.audit-changelog.outputs.exists }} + changelog-exists: ${{ steps.audit-changelog.outputs.exists }} steps: - name: "Check out `${{ inputs.branch }}`" uses: actions/checkout@v4 @@ -89,21 +74,17 @@ jobs: with: version: ${{ inputs.version }} - skip-changelog: - name: "Skip changelog generation" - if: ${{ needs.audit-changelog.outputs.exists }} - runs-on: ubuntu-latest - needs: [audit-changelog] - steps: - - name: "[INFO] Skipped changelog generation" + - name: "[DEBUG] Inputs" run: | - title="Skipped changelog" - message="The changelog already exists for version ${{ inputs.version }}: ${{ needs.audit-changelog.outputs.path }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + echo branch : ${{ inputs.branch }} + echo version : ${{ inputs.version }} + echo changelog-path : ${{ steps.audit-changelog.outputs.path }} + echo changelog-exists : ${{ steps.audit-changelog.outputs.exists }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} generate-changelog: name: "Generate changelog" - if: ${{ !needs.audit-changelog.outputs.exists }} + if: ${{ !needs.calculated-inputs.outputs.changelog-exists }} runs-on: ubuntu-latest needs: [audit-changelog] steps: @@ -122,35 +103,44 @@ jobs: version: ${{ inputs.version }} - name: "Get Core team membership" - id: core_membership + id: core-membership uses: ./.github/actions/audit-github-team with: - organization: "dbt-labs" team: "core-group" env: GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} - - name: "Set Core team membership for changie contributors exclusion" - id: set_team_membership + - name: "Set Core team membership for `changie` contributors exclusion" + id: set-team-membership run: echo "CHANGIE_CORE_TEAM=${{ steps.core_membership.outputs.members }}" >> $GITHUB_ENV - name: "Generate changelog" run: | + # this is a pre-release if [[ ${{ steps.semver.outputs.is_prerelease }} -eq 1 ]] then changie batch ${{ steps.semver.outputs.base_version }} --move-dir '${{ steps.semver.outputs.base_version }}' --prerelease ${{ steps.semver.outputs.prerelease }} + + # this is a final release with associated pre-releases elif [[ -d ".changes/${{ steps.semver.outputs.base_version }}" ]] then changie batch ${{ steps.semver.outputs.base_version }} --include '${{ steps.semver.outputs.base_version }}' --remove-prereleases - else # releasing a final patch with no prereleases + + # this is a final release with no associated pre-releases + else changie batch ${{ needs.audit-changelog.outputs.base_version }} fi + changie merge + - name: "Remove trailing whitespace and extra newlines" + continue-on-error: true + run: hatch run lint:pre-commit --files dbt/adapters/__about__.py CHANGELOG.md .changes/* + - name: "Check changelog generated successfully" run: | title="Generate changelog" - if [[ -f ${{ needs.audit-changelog.outputs.changelog_path }} ]] + if [[ -f ${{ needs.calculated-inputs.outputs.changelog-path }} ]] then message="Changelog file created successfully" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" @@ -160,17 +150,25 @@ jobs: exit 1 fi - - name: "Remove trailing whitespace and extra newlines" - continue-on-error: true - run: hatch run lint:pre-commit --files dbt/adapters/__about__.py CHANGELOG.md .changes/* - - name: "Commit and push changes" - uses: ./.github/actions/commit + uses: ./.github/actions/github-commit with: - message: "[automated] generate changelog for ${{ inputs.version }}" + message: "generate changelog for ${{ inputs.version }}" - name: "[INFO] Generated changelog" run: | title="Generated changelog" - message="Generated changelog for version ${{ inputs.version }}: ${{ needs.audit-changelog.outputs.path }}" + message="Generated changelog for version ${{ inputs.version }}: ${{ needs.calculated-inputs.outputs.changelog-path }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + skip-changelog: + name: "Skip changelog generation" + if: ${{ needs.calculated-inputs.outputs.changelog-exists }} + runs-on: ubuntu-latest + needs: [audit-changelog] + steps: + - name: "[INFO] Skipped changelog generation" + run: | + title="Skipped changelog" + message="The changelog already exists for version ${{ inputs.version }}: ${{ needs.calculated-inputs.outputs.changelog-path }}" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/bump-version.yml b/.github/workflows/build-version-bump.yml similarity index 58% rename from .github/workflows/bump-version.yml rename to .github/workflows/build-version-bump.yml index 194857da..37af14ba 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/build-version-bump.yml @@ -1,8 +1,8 @@ -name: "Bump version" +name: "Version bump" run-name: "Bump branch `${{ inputs.branch }}` to `${{ inputs.version }}`" on: - workflow_dispatch: + workflow_call: inputs: branch: description: "The branch to bump" @@ -10,11 +10,16 @@ on: default: "main" version: description: "The version to bump to" + type: string required: true working-dir: description: "The working directory, useful for mono-repos" + type: string default: "./" - workflow_call: + outputs: + current-version: ${{ jobs.bump-version.outputs.version }} + bumped: ${{ jobs.bump-version.outputs.bumped }} + workflow_dispatch: inputs: branch: description: "The branch to bump" @@ -22,15 +27,10 @@ on: default: "main" version: description: "The version to bump to" - type: string required: true working-dir: description: "The working directory, useful for mono-repos" - type: string default: "./" - outputs: - current-version: ${{ jobs.bump-version.outputs.version }} - bumped: ${{ jobs.bump-version.outputs.bumped }} permissions: write-all @@ -40,26 +40,15 @@ concurrency: cancel-in-progress: true env: - NOTIFICATION_PREFIX: "[Bump version]" + NOTIFICATION_PREFIX: "[Version bump]" jobs: - debug-inputs: - name: "[DEBUG] Inputs" - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Inputs" - run: | - echo branch : ${{ inputs.branch }} - echo version : ${{ inputs.version }} - echo working-dir : ${{ inputs.working-dir }} - echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} - - bump-version: - name: "Bump `${{ inputs.branch}}` to `${{ inputs.version }}`" + calculated-inputs: + name: "Calculated inputs" runs-on: ubuntu-latest outputs: current-version: ${{ steps.current-version.outputs.version }} - bumped: ${{ steps.current-version.outputs.bumped }} + needs-version-bump: ${{ steps.version-bump.outputs.needs-version-bump }} steps: - name: "Check out `${{ inputs.branch }}`" uses: actions/checkout@v4 @@ -67,41 +56,71 @@ jobs: ref: ${{ inputs.branch }} - name: "Setup `hatch`" - uses: ./.github/actions/setup-hatch + uses: ./.github/actions/setup-environment - - name: "Current version" + - name: "Set: current-version" id: current-version shell: bash + run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT + working-directory: ${{ inputs.working-dir }} + + - name: "Set: need-to-bump" + id: version-bump + shell: bash run: | - echo "version=$(hatch version)" >> $GITHUB_OUTPUT - if [[ $(hatch version) == "${{ inputs.version }} ]] + if [[ ${{ steps.current-version.outputs.version }} == "${{ inputs.version }} ]] then - echo "bumped=false" >> $GITHUB_OUTPUT + echo "needs-version-bump=false" >> $GITHUB_OUTPUT else - echo "bumped=true" >> $GITHUB_OUTPUT + echo "needs-version-bump=true" >> $GITHUB_OUTPUT fi - - name: "Bump version" - if: ${{ steps.current-version.outputs.version != inputs.version }} + - name: "[DEBUG] Parameters" + run: | + echo branch : ${{ inputs.branch }} + echo version : ${{ inputs.version }} + echo working-dir : ${{ inputs.working-dir }} + echo current-version : ${{ steps.current-version.outputs.version }} + echo need-to-bump : ${{ steps.version-bump.outputs.needs-version-bump }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + + bump-version: + name: "Bump version" + if: ${{ fromJSON(needs.calculated-inputs.outputs.needs-version-bump) }} + needs: [calculated-inputs] + runs-on: ubuntu-latest + steps: + - name: "Check out `${{ inputs.branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup `hatch`" + uses: ./.github/actions/setup-environment + + - name: "Bump version to `${{ inputs.version }}`" shell: bash run: hatch version ${{ inputs.version }} working-directory: ${{ inputs.working-dir }} - name: "Commit and push changes" - if: ${{ steps.current-version.outputs.version != inputs.version }} - uses: ./.github/actions/commit + uses: ./.github/actions/github-commit with: - message: "[automated] bump version to ${{ inputs.version }}" + message: "bump version to ${{ inputs.version }}" - name: "[INFO] Bumped version" - if: ${{ steps.current-version.outputs.version != inputs.version }} run: | title="Bumped version" - message="Bumped from ${{ steps.current-version.outputs.version }} to ${{ inputs.version }}" + message="Bumped version from ${{ needs.calculated-inputs.outputs.current-version }} to ${{ inputs.version }}" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + skip-version-bump: + name: "Skip version bump" + if: ${{ !fromJSON(needs.calculated-inputs.outputs.needs-version-bump) }} + needs: [calculated-inputs] + runs-on: ubuntu-latest + steps: - name: "[INFO] Skipped version bump" - if: ${{ steps.current-version.outputs.version == inputs.version }} run: | title="Skipped version bump" message="${{ inputs.branch }} is already at version ${{ inputs.version }}" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 83d0cd39..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: "Build" -run-name: "Build `${{ inputs.package }}==${{ inputs.version}}` from `${{ inputs.branch }}` for deployment to `${{ deploy-to }}`" - -on: - workflow_call: - inputs: - package: - description: "Choose what to publish (dbt-adapters, dbt-tests-adapter)" - type: string - branch: - description: "Choose what branch to build from" - type: string - default: "main" - version: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - deploy-to: - description: "Choose where to publish" - type: string - default: prod - outputs: - archive-name: ${{ jobs.build.outputs.archive-name }} - workflow_dispatch: - inputs: - package: - description: "Choose what to build" - type: choice - options: - - dbt-adapters - - dbt-tests-adapter - branch: - description: "Choose what branch to build from" - type: string - default: "main" - version: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - deploy-to: - description: "Choose where to publish" - type: choice - options: - - prod - - test - -permissions: read-all - -# don't try to build the same version of the same package, the archive name could collide -concurrency: - group: ${{ github.workflow }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -env: - NOTIFICATION_PREFIX: "[Build]" - -jobs: - debug-inputs: - name: "[DEBUG] Inputs" - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Inputs" - run: | - echo package : ${{ inputs.package }} - echo branch : ${{ inputs.branch }} - echo version : ${{ inputs.version }} - echo deploy-to : ${{ inputs.deploy-to }} - echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} - - build: - name: "Build the artifacts" - runs-on: ubuntu-latest - outputs: - archive-name: ${{ steps.archive.outputs.archive-name }} - steps: - - name: "Check out `${{ inputs.branch }}`" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - - - name: "Create the archive name" - id: archive - shell: bash - run: | - name=${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} - echo "name=$name" >> $GITHUB_OUTPUT - - - name: "Setup `hatch`" - uses: ./.github/actions/setup-hatch - - - name: "Build `dbt-adapters`" - if: ${{ inputs.package == 'dbt-adapters' }} - uses: ./.github/actions/build - with: - archive-name: ${{ steps.archive.outputs.name }} - - - name: "Build `dbt-tests-adapter`" - if: ${{ inputs.package == 'dbt-tests-adapter' }} - uses: ./.github/actions/build - with: - archive-name: ${{ steps.archive.outputs.name }} - working-dir: "./dbt-tests-adapter/" diff --git a/.github/workflows/check-changelog-entry.yml b/.github/workflows/check-changelog-entry.yml new file mode 100644 index 00000000..f7196bcb --- /dev/null +++ b/.github/workflows/check-changelog-entry.yml @@ -0,0 +1,35 @@ +name: "Changelog entry" + +on: + # run on pull_request_target because this workflow requires secrets to post comments + pull_request_target: + branches: + - "main" + - "*.latest" + # this is a cheap check, any change should trigger it + types: + - opened + - reopened + - labeled + - unlabeled + - synchronize + # we are only concerned about functional code changes, i.e. code changes that will appear in production + paths-ignore: + - ".changes/**" + - ".github/**" + - "tests/**" + - "third-party-stubs/**" + - "**.md" + - "**.yml" + +permissions: + contents: read + pull-requests: write + +jobs: + changelog: + uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main + with: + changelog_comment: "Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry)." + skip_label: "Skip Changelog" + secrets: inherit # this is only acceptable because we own the action we're calling diff --git a/.github/workflows/code-quality.yml b/.github/workflows/check-code-quality.yml similarity index 62% rename from .github/workflows/code-quality.yml rename to .github/workflows/check-code-quality.yml index 52fe0daf..80c9eaa3 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/check-code-quality.yml @@ -1,36 +1,22 @@ name: "Code quality" on: - push: + pull_request: branches: - "main" - "*.latest" - pull_request: workflow_call: inputs: branch: description: "The branch to run code quality on" type: string default: "main" - lint-command: - description: "The lint command to run" - type: string - default: "hatch run lint:all" - typecheck-command: - description: "The typecheck command to run" - type: string - default: "hatch run typecheck:all" workflow_dispatch: inputs: branch: description: "The branch to run code quality on" + type: string default: "main" - lint-command: - description: "The lint command to run" - default: "hatch run lint:all" - typecheck-command: - description: "The typecheck command to run" - default: "hatch run typecheck:all" permissions: read-all @@ -40,7 +26,7 @@ defaults: # cancel the previous code quality run if a new one starts with the same parameters concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'workflow') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + group: ${{ github.workflow }}-${{ contains(github.event_name, 'workflow') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref }} cancel-in-progress: true env: @@ -61,14 +47,14 @@ jobs: - name: "Check out `${{ inputs.branch }}`" uses: actions/checkout@v4 with: - ref: ${{ contains(github.event_name, 'workflow') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + ref: ${{ contains(github.event_name, 'workflow') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref }} persist-credentials: false - name: "Setup `hatch`" uses: ./.github/actions/setup-hatch - name: "Run linters" - run: ${{ inputs.lint-command }} + run: hatch run lint:all - name: "Run typechecks" - run: ${{ inputs.typecheck-command }} + run: hatch run typecheck:all diff --git a/.github/workflows/check-for-changelog-entry.yml b/.github/workflows/check-for-changelog-entry.yml deleted file mode 100644 index b83bd11e..00000000 --- a/.github/workflows/check-for-changelog-entry.yml +++ /dev/null @@ -1,39 +0,0 @@ -# **what?** -# Checks that a file has been committed under the /.changes directory -# as a new CHANGELOG entry. Cannot check for a specific filename as -# it is dynamically generated by change type and timestamp. -# This workflow runs on pull_request_target because it requires -# secrets to post comments. - -# **why?** -# Ensure code change gets reflected in the CHANGELOG. - -# **when?** -# This will run for all PRs going into main. It will -# run when they are opened, reopened, when any label is added or removed -# and when new code is pushed to the branch. The action will get -# skipped if the 'Skip Changelog' label is present is any of the labels. - -name: "Check for changelog entry" - -on: - pull_request: - types: [opened, reopened, labeled, unlabeled, synchronize] - paths-ignore: ['.changes/**', '.github/**', 'tests/**', 'third-party-stubs/**', '**.md', '**.yml'] - workflow_dispatch: - -defaults: - run: - shell: bash - -permissions: - contents: read - pull-requests: write - -jobs: - changelog: - uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main - with: - changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry).' - skip_label: 'Skip Changelog' - secrets: inherit # this is only acceptable because we own the action we're calling diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/check-unit-tests.yml similarity index 89% rename from .github/workflows/unit-tests.yml rename to .github/workflows/check-unit-tests.yml index 6ff00368..71db48a3 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/check-unit-tests.yml @@ -1,21 +1,20 @@ name: "Unit tests" on: - push: + pull_request: branches: - "main" - "*.latest" - pull_request: - workflow_dispatch: + workflow_call: inputs: branch: - description: "Choose what branch to test" + description: "The branch to run unit tests on" type: string default: "main" - workflow_call: + workflow_dispatch: inputs: branch: - description: "Choose what branch to test" + description: "The branch to run unit tests on" type: string default: "main" @@ -23,7 +22,7 @@ permissions: read-all # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'workflow_') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'workflow_') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref }} cancel-in-progress: true env: @@ -44,7 +43,7 @@ jobs: echo branch : ${{ inputs.branch }} echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} - - name: "Check out `${{ contains(github.event_name, 'workflow_') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}`" + - name: "Check out `${{ contains(github.event_name, 'workflow_') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref }}`" uses: actions/checkout@v4 with: ref: ${{ contains(github.event_name, 'workflow_') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} diff --git a/.github/workflows/resubmit-for-triage.yml b/.github/workflows/issues-resubmit-for-triage.yml similarity index 87% rename from .github/workflows/resubmit-for-triage.yml rename to .github/workflows/issues-resubmit-for-triage.yml index 3020509c..2b3237a5 100644 --- a/.github/workflows/resubmit-for-triage.yml +++ b/.github/workflows/issues-resubmit-for-triage.yml @@ -9,15 +9,10 @@ # **when?** # This will run when a comment is added to an issue and that issue has an `awaiting_response` label. - name: "Resubmit for triage" on: issue_comment -defaults: - run: - shell: bash - permissions: issues: write @@ -28,4 +23,4 @@ jobs: with: add_label: "triage" remove_label: "awaiting_response" - secrets: inherit # this is only acceptable because we own the action we're calling + secrets: inherit diff --git a/.github/workflows/issues-stale.yml b/.github/workflows/issues-stale.yml new file mode 100644 index 00000000..7360c9e1 --- /dev/null +++ b/.github/workflows/issues-stale.yml @@ -0,0 +1,24 @@ +# **what?** +# 1. Label issues and PRs which breach the threshold in our stale policy. +# This will notify participants that they will be closed soon. +# Any activity on the issue will remove the label. +# 2. Close issues or PRs which were previously marked as stale. +# +# **why?** +# Avoid triaging and working issues which are no longer relevant + +# **when?** +# Once a day +name: "Stale" + +on: + schedule: + - cron: "30 1 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main diff --git a/.github/workflows/publish-github.yml b/.github/workflows/publish-github.yml index a3350395..738eaaef 100644 --- a/.github/workflows/publish-github.yml +++ b/.github/workflows/publish-github.yml @@ -6,96 +6,83 @@ # 5. If this is deployed to prod, mark the draft release (created in this run or previously) as a full release. # # **when?** -# 1. Call after a successful build. Build artifacts should be ready to release.defaults: +# 1. Call after a successful build. Build artifacts should be ready to release. # They need to either live in a dist/ directory locally, or an archive name needs to be provided. -# 2. Call to publish a draft release to become a full release. No archive or artifacts are needed in this case. +# 2. Call to publish a draft release to become a full release. No archive, artifacts, or changelog path are needed in this case. name: "Publish to GitHub" -run-name: "Publish `${{ version }}` to GitHub in ${{ deploy-to }}" +run-name: "Publish `${{ version }}` to GitHub in ${{ deploy-environment }}" on: - workflow_dispatch: + workflow_call: inputs: - archive-name: - description: "Name of the archive containing the artifacts, leave blank if local or publishing draft to full" - type: string - default: "" version: description: "The release version number (i.e. 1.0.0b1)" type: string required: true sha: - description: "Commit SHA being released" - type: string - required: true - changelog-path: - description: "Path to the release notes" + description: "The SHA for the commit being released" type: string required: true - deploy-to: - description: "Choose where to publish" - type: environment - workflow_call: - inputs: archive-name: description: "Name of the archive containing the artifacts, leave blank if local or publishing draft to full" type: string default: "" + changelog-path: + description: "Path to the release notes, leave blank if publishing draft to full" + type: string + default: "" + deploy-environment: + description: "Choose where to publish" + type: string + default: "prod" + workflow_dispatch: + inputs: version: description: "The release version number (i.e. 1.0.0b1)" type: string required: true sha: - description: "Commit SHA being released" + description: "The SHA for the commit being released" type: string required: true + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local or publishing draft to full" + type: string + default: "" changelog-path: - description: "Path to the release notes" + description: "Path to the release notes, leave blank if publishing draft to full" type: string - required: true - deploy-to: + default: "" + deploy-environment: description: "Choose where to publish" - type: string - default: "prod" + type: environment permissions: contents: write # will cancel previous workflows triggered by the same event and for the same package and environment concurrency: - group: ${{ github.workflow }}-${{ inputs.archive-name }}-${{ inputs.version }}-${{ inputs.deploy-to }} + group: ${{ github.workflow }}-${{ inputs.archive-name }}-${{ inputs.version }}-${{ inputs.deploy-environment }} cancel-in-progress: true env: - REPO_LINK: ${{ github.server_url }}/${{ github.repository }} NOTIFICATION_PREFIX: "[Publish - GitHub]" jobs: - debug-inputs: - name: "[DEBUG] Inputs" - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Inputs" - run: | - echo archive-name : ${{ inputs.archive-name }} - echo version : ${{ inputs.version }} - echo sha : ${{ inputs.sha }} - echo changelog-path : ${{ inputs.changelog-path }} - echo deploy-to : ${{ inputs.deploy-to }} - echo REPO_LINK : ${{ env.REPO_LINK }} - echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} - - publish: - name: "Publish to GitHub" + calculated-inputs: + name: "Calculated inputs" runs-on: ubuntu-latest outputs: - exists: ${{ steps.release_check.outputs.exists }} - draft_exists: ${{ steps.release_check.outputs.draft_exists }} - tag: ${{ steps.set_tag.outputs.tag }} + tag: ${{ steps.tag.outputs.tag }} + tag-sha: ${{ steps.tag-metadata.outputs.sha }} + tag-exists: ${{ steps.tag-metadata.outputs.exists }} + tag-is-draft: ${{ steps.tag-metadata.outputs.is-draft }} + commit-tag: ${{ steps.commit-metadata.outputs.tag }} steps: - name: "Check out `main`" uses: actions/checkout@v4 - - name: "Create tag" + - name: "Set: tag" id: tag shell: bash run: echo "tag=v${{ inputs.version }}" >> $GITHUB_OUTPUT @@ -114,47 +101,93 @@ jobs: with: tag: ${{ inputs.sha }} - - name: "[INFO] Release already exists" - if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == true }} && ${{ steps.tag-metadata.outputs.sha }} == ${{ inputs.sha }} + - name: "[DEBUG] Inputs" + run: | + echo archive-name : ${{ inputs.archive-name }} + echo version : ${{ inputs.version }} + echo sha : ${{ inputs.sha }} + echo changelog-path : ${{ inputs.changelog-path }} + echo deploy-environment : ${{ inputs.deploy-environment }} + echo tag : ${{ steps.tag.outputs.tag }} + echo tag-sha : ${{ steps.tag-metadata.outputs.sha }} + echo tag-exists : ${{ steps.tag-metadata.outputs.exists }} + echo tag-is-draft : ${{ steps.tag-metadata.outputs.is-draft }} + echo commit-tag : ${{ steps.commit-metadata.outputs.tag }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + + validation: + name: "Validate input scenarios" + needs: [calculated-inputs] + runs-on: ubuntu-latest + steps: + - name: "[INFO] Exit gracefully if the tag exists and it's a full release" + if: ${{ fromJSON(needs.calculated-inputs.outputs.tag-exists) && !fromJSON(needs.calculated-inputs.outputs.tag-is-draft) }} run: | - title="Tag already exists, skip publishing artifacts" - message="Tag ${{ steps.tag.outputs.tag }} already exists and is associated with the commit ${{ steps.tag-metadata.outputs.sha }}." + title="Full release already exists" + message="`${{ needs.calculated-inputs.outputs.tag }}` already exists as a full release and is associated with the commit ${{ needs.calculated-inputs.outputs.tag-sha }}." echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 0 - - name: "[ERROR] Exit if the tag exists and is associated with a different commit than the one provided" - if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == true }} && ${{ steps.tag-metadata.outputs.sha }} != ${{ inputs.sha }} + - name: "[ERROR] Exit with error if the tag exists and is associated with a different commit than the one provided" + if: ${{ fromJSON(needs.calculated-inputs.outputs.tag-exists) }} && ${{ needs.calculated-inputs.outputs.tag-sha }} != ${{ inputs.sha }} shell: bash run: | - title="Tag ${{ inputs.tag }} already exists with a different commit!" + title="Tag `v${{ inputs.version }}` already exists with a different commit!" message="Existing commit: {{ steps.tag-metadata.outputs.sha }} New commit: ${{ inputs.sha }}. Exiting." echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" exit 1 - - name: "[ERROR] Exit if the commit is associated with a different tag than the version that was provided" - if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == false }} && ${{ steps.commit-metadata.outputs.tag != '' }} + - name: "[ERROR] Exit with error if the commit is associated with a different tag than the version that was provided" + if: ${{ needs.calculated-inputs.outputs.commit-tag != '' }} && ${{ needs.calculated-inputs.outputs.commit-tag != needs.calculated-inputs.outputs.tag }} run: | title="Commit ${{ inputs.sha }} is already associated with a different tag!" - message="Existing tag: ${{ steps.commit-metadata.outputs.tag }} New tag: ${{ steps.tag.outputs.tag }}" + message="Existing tag: ${{ needs.calculated-inputs.outputs.commit-tag }} New tag: ${{ needs.calculated-inputs.outputs.tag }}" echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" exit 1 - - name: "Publish to GitHub as draft release" - if: fromJSON(steps.tag-metadata.outputs.exists) == false + new-tag: + name: "Publish draft release to GitHub" + needs: [calculated-inputs, validation] + if: ${{ !fromJSON(needs.calculated-inputs.outputs.tag-exists) }} + runs-on: ubuntu-latest + steps: + - name: "Check out `main`" + uses: actions/checkout@v4 + + - name: "Publish to GitHub as a draft release" uses: ./.github/actions/publish-github-draft with: - archive-name: ${{ inputs.archive-name }} - tag: ${{ needs.check-release-exists.outputs.tag }} - repo-url: ${{ env.REPO_LINK }} + repo-url: ${{ github.server_url }}/${{ github.repository }} + tag: ${{ needs.calculated-inputs.outputs.tag }} sha: ${{ inputs.sha }} - changelog-path: ${{ inputs.changelog_path }} - - - name: "Publish draft release as full release" - if: >- - inputs.deploy-to == "prod" && ( - fromJSON(steps.tag-metadata.outputs.exists) == false || - fromJSON(steps.tag-metadata.outputs.is-draft) == true - ) + archive-name: ${{ inputs.archive-name }} + changelog-path: ${{ inputs.changelog-path }} + + - name: "[INFO] Published to GitHub as a draft release" + run: | + title="Published to GitHub as a draft release" + message="${{ needs.calculated-inputs.outputs.tag }} was created as a draft release with the commit ${{ inputs.sha }}." + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + existing-tag: + name: "Publish GitHub draft as full release" + # require publish-draft to cover the scenario where a production release is being created without an existing draft release + needs: [calculated-inputs, validation, publish-draft] + # allow for publish-draft to be skipped, e.g. if this is a production release where the draft already existed + if: ${{ !failure() && inputs.deploy-environment == "prod" }} + runs-on: ubuntu-latest + steps: + - name: "Check out `main`" + uses: actions/checkout@v4 + + - name: "Publish draft release as a full release" uses: ./.github/actions/publish-github-full with: - tag: ${{ steps.tag.output.tag }} - repo-url: ${{ env.REPO_LINK }} + repo-url: ${{ github.server_url }}/${{ github.repository }} + tag: ${{ needs.calculated-inputs.outputs.tag }} + + - name: "[INFO] Published draft release as a full release" + run: | + title="Published draft release as a full release" + message="`${{ needs.calculated-inputs.outputs.tag }}` was published as a full release." + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 4710b3d3..a34b6c29 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -5,7 +5,7 @@ # Call after a successful build. Build artifacts should be ready to release.defaults: # They need to either live in a dist/ directory locally, or an archive name needs to be provided. name: "Publish to PyPI" -run-name: "Publish `${{ version }}` to PyPI in ${{ deploy-to }}" +run-name: "Publish `${{ version }}` to PyPI in ${{ deploy-environment }}" on: workflow_dispatch: @@ -17,7 +17,7 @@ on: description: "The release version number (i.e. 1.0.0b1)" type: string required: true - deploy-to: + deploy-environment: description: "Choose where to publish" type: environment workflow_call: @@ -30,7 +30,7 @@ on: description: "The release version number (i.e. 1.0.0b1)" type: string required: true - deploy-to: + deploy-environment: description: "Choose where to publish" type: string default: "prod" @@ -40,7 +40,7 @@ permissions: # will cancel previous workflows triggered by the same event and for the same package and environment concurrency: - group: ${{ github.workflow }}-${{ inputs.archive-name }}-${{ inputs.version }}-${{ inputs.deploy-to }} + group: ${{ github.workflow }}-${{ inputs.archive-name }}-${{ inputs.version }}-${{ inputs.deploy-environment }} cancel-in-progress: true env: @@ -55,7 +55,7 @@ jobs: run: | echo archive-name : ${{ inputs.archive-name }} echo version : ${{ inputs.version }} - echo deploy-to : ${{ inputs.deploy-to }} + echo deploy-environment : ${{ inputs.deploy-environment }} echo PYPI_PROJECT_URL : ${{ vars.PYPI_PROJECT_URL }} echo PYPI_REPOSITORY_URL : ${{ vars.PYPI_REPOSITORY_URL }} @@ -63,7 +63,7 @@ jobs: name: "Publish to PyPI" runs-on: ubuntu-latest environment: - name: ${{ inputs.deploy-to }} + name: ${{ inputs.deploy-environment }} url: ${{ vars.PYPI_PROJECT_URL }} steps: - name: "Check out `main`" diff --git a/.github/workflows/release-branch-create.yml b/.github/workflows/release-branch-create.yml index f99b5f1b..5fda905e 100644 --- a/.github/workflows/release-branch-create.yml +++ b/.github/workflows/release-branch-create.yml @@ -63,5 +63,3 @@ jobs: uses: ./.github/actions/create-temp-branch with: branch-stub: "release/${{ inputs.version }}/" - env: - NOTIFICATION_PREFIX: "[create release branch]" diff --git a/.github/workflows/release-branch-merge.yml b/.github/workflows/release-branch-merge.yml index e619ceef..7d8f362d 100644 --- a/.github/workflows/release-branch-merge.yml +++ b/.github/workflows/release-branch-merge.yml @@ -7,15 +7,19 @@ on: branch: description: "The base branch receiving the changes" type: string - default: "main" + required: true release-branch: description: "The branch containing the release changes" type: string - default: "main" + required: true + deploy-environment: + description: "Choose where to publish" + type: string + required: true outputs: sha: description: "The sha associated with this merge" - value: ${{ jobs.merge-release-branch.outputs.sha }} + value: ${{ jobs.merge.outputs.sha }} # this is the permission that allows creating a new release to both GitHub and PyPI permissions: @@ -27,10 +31,6 @@ concurrency: group: ${{ github.workflow }}-${{ inputs.branch }}-${{ inputs.release-branch }} cancel-in-progress: true -defaults: - run: - shell: bash - env: NOTIFICATION_PREFIX: "[Merge release branch]" @@ -45,41 +45,30 @@ jobs: echo release-branch : ${{ inputs.release-branch }} echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} - merge-release-branch: + merge: name: "Merge `${{ inputs.release-branch }}` into `${{ inputs.branch }}`" runs-on: ubuntu-latest needs: [temp-branch] outputs: - sha: ${{ steps.release-sha.outputs.sha }} + sha: ${{ steps.merge.outputs.sha }} || ${{ steps.test-merge.outputs.sha }} steps: - name: "Checkout `${{ inputs.release-branch }}`" uses: actions/checkout@v4 with: - ref: ${{ inputs.temp-branch }} + ref: ${{ inputs.release-branch }} - name: "Merge `${{ inputs.release-branch }}` into `${{ inputs.branch }}`" - uses: everlytic/branch-merge@1.1.5 - with: - source_ref: ${{ inputs.release-branch }} - target_branch: ${{ inputs.branch }} - github_token: ${{ secrets.FISHTOWN_BOT_PAT }} - commit_message_template: "[automated] merge {source_ref} into {target_branch} for a release" - - - name: "Checkout `${{ inputs.branch }}`" - uses: actions/checkout@v3 + if: ${{ inputs.deploy-environment == "prod" }} + id: merge + uses: ./.github/actions/github-merge with: - ref: ${{ inputs.branch }} - - # TODO: see if everlytic/branch-merge has outputs that have this information - - name: "Get commit SHA for the merge" - id: release-sha - run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: "Delete `${{ inputs.release-branch }}`" - run: git push origin -d ${{ inputs.release-branch }} + source-branch: ${{ inputs.release-branch }} + target-branch: ${{ inputs.branch }} + message: "merge {source_ref} into {target_branch} for a release" - - name: "[INFO] Merged changes" + - name: "Get HEAD sha for test release" + id: test-merge + shell: bash run: | - title="Merged changes" - message="The ${{ inputs.release-branch }} branch was merged into ${{ inputs.branch }} with sha ${{ steps.release-sha.outputs.sha }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + git pull + echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff8baf70..01382506 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,43 +2,43 @@ name: "Release" run-name: "Release `${{ inputs.package }}==${{ inputs.version }}` to `${{ inputs.deploy-to }}`" on: - workflow_dispatch: + workflow_call: inputs: - package: - description: "Choose what to release" - type: choice - options: - - "dbt-adapters" - - "dbt-tests-adapter" branch: description: "Choose what branch to release from" type: string default: "main" + package: + description: "Choose what to release" + type: string + default: "dbt-adapters" version: description: "The release version number (i.e. 1.0.0b1)" type: string required: true - deploy-to: + deploy-environment: description: "Choose where to publish" - type: environment - workflow_call: - inputs: - package: - description: "Choose what to release" type: string - default: "dbt-adapters" + default: "prod" + workflow_dispatch: + inputs: branch: description: "Choose what branch to release from" type: string default: "main" + package: + description: "Choose what to release" + type: choice + options: + - "dbt-adapters" + - "dbt-tests-adapter" version: description: "The release version number (i.e. 1.0.0b1)" type: string required: true - deploy-to: + deploy-environment: description: "Choose where to publish" - type: string - default: "prod" + type: environment # this is the permission that allows creating a new release to both GitHub and PyPI permissions: @@ -47,7 +47,7 @@ permissions: # deploying the same version of a package to the same environment should override any previous deployment with those attributes concurrency: - group: ${{ github.workflow }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-to }} + group: ${{ github.workflow }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-environment }} cancel-in-progress: true defaults: @@ -55,7 +55,7 @@ defaults: shell: bash jobs: - release-inputs: + calculated-inputs: name: "Release inputs" runs-on: ubuntu-latest outputs: @@ -66,7 +66,7 @@ jobs: - name: "Computed inputs" id: computed-inputs run: | - archive_name=${{ inputs.package }}-${{ inputs.version_number }}-${{ inputs.deploy-to }} + archive_name=${{ inputs.package }}-${{ inputs.version_number }}-${{ inputs.deploy-environment }} working_dir="./" run_unit_tests=true @@ -82,25 +82,25 @@ jobs: - name: "[DEBUG] Inputs" run: | - echo package : ${{ inputs.package }} - echo branch : ${{ inputs.branch }} - echo version : ${{ inputs.version }} - echo deploy-to : ${{ inputs.deploy-to }} - echo archive-name : ${{ steps.release-inputs.outputs.archive-name }} - echo working-dir : ${{ steps.release-inputs.outputs.working-dir }} - echo run-unit-tests : ${{ steps.release-inputs.outputs.run-unit-tests }} + echo branch : ${{ inputs.branch }} + echo package : ${{ inputs.package }} + echo version : ${{ inputs.version }} + echo deploy-environment : ${{ inputs.deploy-environment }} + echo archive-name : ${{ steps.release-inputs.outputs.archive-name }} + echo working-dir : ${{ steps.release-inputs.outputs.working-dir }} + echo run-unit-tests : ${{ steps.release-inputs.outputs.run-unit-tests }} code-quality: name: "Run code quality" - uses: dbt-labs/dbt-adapters/.github/workflows/code-quality.yml + uses: dbt-labs/dbt-adapters/.github/workflows/check-code-quality.yml@update-workflows with: branch: ${{ inputs.branch }} unit-tests: name: "Run unit tests" - needs: [release-inputs] - if: ${{ fromJSON(needs.release-inputs.outputs.run-unit-tests) == true }} - uses: dbt-labs/dbt-adapters/.github/workflows/unit-tests.yml@update-workflows + needs: [calculated-inputs] + if: ${{ fromJSON(needs.calculated-inputs.outputs.run-unit-tests) }} + uses: dbt-labs/dbt-adapters/.github/workflows/check-unit-tests.yml@update-workflows with: branch: ${{ inputs.branch }} @@ -112,69 +112,70 @@ jobs: version: ${{ inputs.version }} secrets: inherit - bump-version: + version-bump: name: "Bump the version" - uses: dbt-labs/dbt-adapters/.github/workflows/bump-version.yml@update-workflows - needs: [release-branch, release-inputs] + uses: dbt-labs/dbt-adapters/.github/workflows/build-version-bump.yml@update-workflows + needs: [release-branch, calculated-inputs] with: branch: ${{ needs.release-branch.outputs.branch }} version: ${{ inputs.version }} working-dir: ${{ needs.release-inputs.outputs.working-dir }} secrets: inherit - generate-changelog: + changelog: name: "Generate a new changelog" needs: [release-branch] - uses: dbt-labs/dbt-adapters/.github/workflows/generate-changelog.yml@update-workflows + uses: dbt-labs/dbt-adapters/.github/workflows/build-changelog.yml@update-workflows with: branch: ${{ needs.release-branch.outputs.branch }} version: ${{ inputs.version }} secrets: inherit - merge-changes: - name: "Merge the version bump and changelog updates" + build-artifacts: + name: "Build the artifacts" + if: ${{ !failure() && !cancelled() }} needs: [ - release-branch, - bump-version, - generate-changelog, code-quality, unit-tests, + release-branch, + version-bump, + changelog, ] - if: ${{ !failure() && !cancelled() && (needs.bump-version.outputs.bumped || needs.generate-changelog.outputs.created) }} + uses: dbt-labs/dbt-adapters/.github/workflows/build-artifacts.yml@update-workflows + with: + branch: ${{ needs.release-branch.outputs.branch }} + package: ${{ inputs.package }} + version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-environment }} + + merge-changes: + name: "Merge the version bump and changelog updates" + needs: [build-artifacts, version-bump, changelog] + if: ${{ !failure() && !cancelled() && inputs.deploy-environment == "prod" && (needs.version-bump.outputs.bumped || needs.changelog.outputs.created) }} uses: dbt-labs/dbt-adapters/.github/workflows/release-branch-merge.yml@update-workflows with: branch: ${{ inputs.branch }} - temp-branch: ${{ needs.release-branch.outputs.branch }} + release-branch: ${{ needs.release-branch.outputs.branch }} secrets: inherit - build-artifacts: - name: "Build the artifacts" - if: ${{ !failure() && !cancelled() }} - needs: [merge-changes, release-inputs] - uses: dbt-labs/dbt-adapters/.github/workflows/build.yml@update-workflows - with: - package: ${{ inputs.package }} - version: ${{ inputs.version }} - deploy-to: ${{ inputs.deploy-to }} - publish-github: name: "Publish the artifacts to GitHub" if: ${{ !failure() && !cancelled() && inputs.package == 'dbt-adapters' }} - needs: [build-artifacts, merge-changes] + needs: [build-artifacts, changelog, merge-changes] uses: dbt-labs/dbt-adapters/.github/workflows/publish-github.yml@update-workflows with: archive-name: ${{ needs.build-artifacts.outputs.archive-name }} version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-environment }} sha: ${{ needs.merge-changes.outputs.sha }} - changelog-path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - deploy-to: ${{ inputs.deploy-to }} + changelog-path: ${{ needs.changelog.outputs.changelog-path }} - publish-0pypi: + publish-pypi: name: "Publish the artifacts to PyPI" if: ${{ !failure() && !cancelled() }} - needs: [build-artifacts] - uses: dbt-labs/dbt-adapters/.github/workflows/publish-github.yml@update-workflows + needs: [build-artifacts, merge-changes] + uses: dbt-labs/dbt-adapters/.github/workflows/publish-pypi.yml@update-workflows with: archive-name: ${{ needs.build-artifacts.outputs.archive-name }} version: ${{ inputs.version }} - deploy-to: ${{ inputs.deploy-to }} + deploy-to: ${{ inputs.deploy-environment }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index 75a14dd4..00000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,30 +0,0 @@ -# **what?** -# For issues that have been open for awhile without activity, label -# them as stale with a warning that they will be closed out. If -# anyone comments to keep the issue open, it will automatically -# remove the stale label and keep it open. - -# Stale label rules: -# awaiting_response, more_information_needed -> 90 days -# good_first_issue, help_wanted -> 360 days (a year) -# tech_debt -> 720 (2 years) -# all else defaults -> 180 days (6 months) - -# **why?** -# To keep the repo in a clean state from issues that aren't relevant anymore - -# **when?** -# Once a day - -name: "Close stale issues and PRs" -on: - schedule: - - cron: "30 1 * * *" - -permissions: - issues: write - pull-requests: write - -jobs: - stale: - uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main From 415e2d81bdfa85b5a1db6d8972c78445e7d5075a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 20 Apr 2024 11:20:34 -0400 Subject: [PATCH 15/16] make updates for dbt-redshift draft --- .github/actions/audit-changelog/action.yml | 92 +++++++------ .../actions/audit-github-commit/action.yml | 76 +++++------ .github/actions/audit-github-tag/action.yml | 124 +++++++++--------- .github/actions/audit-github-team/action.yml | 94 ++++++------- .github/actions/build-artifacts/action.yml | 65 +++++++++ .github/actions/setup-changie/action.yml | 15 --- .github/actions/setup-environment/action.yml | 50 ++++--- 7 files changed, 286 insertions(+), 230 deletions(-) create mode 100644 .github/actions/build-artifacts/action.yml delete mode 100644 .github/actions/setup-changie/action.yml diff --git a/.github/actions/audit-changelog/action.yml b/.github/actions/audit-changelog/action.yml index fb878fcf..8599f112 100644 --- a/.github/actions/audit-changelog/action.yml +++ b/.github/actions/audit-changelog/action.yml @@ -2,58 +2,56 @@ name: "Audit changelog" description: "Get metadata about a changelog" inputs: - version: - description: "The version whose changelog is being audited" - required: true + version: + description: "The version whose changelog is being audited" + required: true outputs: - path: - description: "The expected file path to the change log, relative to the repo root" - value: ${{ steps.path.outputs.path }} - exists: - description: "Indicates if the changelog exists" - value: ${{ steps.exists.outputs.exists }} + path: + description: "The expected file path to the change log, relative to the repo root" + value: ${{ steps.path.outputs.path }} + exists: + description: "Indicates if the changelog exists" + value: ${{ steps.exists.outputs.exists }} runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo version : ${{ inputs.version }} + using: composite + steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo version : ${{ inputs.version }} - - name: "Parse version: `${{ inputs.version }}`" - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ inputs.version }} + - name: "Parse version: `${{ inputs.version }}`" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ inputs.version }} - - name: "Set: path" - id: path - shell: bash - run: | - path=".changes/${{ steps.semver.outputs.base-version }}" - if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] - then - path+="-${{ steps.semver.outputs.pre-release }}" - fi - path+=".md" - echo "path=$path" >> $GITHUB_OUTPUT + - name: "Set: path" + id: path + shell: bash + run: | + path=".changes/${{ steps.semver.outputs.base-version }}" + if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]]; then + path+="-${{ steps.semver.outputs.pre-release }}" + fi + path+=".md" + echo "path=$path" >> $GITHUB_OUTPUT - - name: "Set: exists" - id: exists - shell: bash - run: | - exists=false - if test -f ${{ steps.path.outputs.path }} - then - exists=true - fi - echo "exists=exists">> $GITHUB_OUTPUT + - name: "Set: exists" + id: exists + shell: bash + run: | + exists=false + if test -f ${{ steps.path.outputs.path }}; then + exists=true + fi + echo "exists=exists">> $GITHUB_OUTPUT - - name: "[DEBUG] Changelog metadata" - shell: bash - run: | - echo version : ${{ inputs.version }} - echo path : ${{ steps.path.outputs.path }} - echo exists : ${{ steps.exists.outputs.exists }} + - name: "[DEBUG] Changelog metadata" + shell: bash + run: | + echo version : ${{ inputs.version }} + echo path : ${{ steps.path.outputs.path }} + echo exists : ${{ steps.exists.outputs.exists }} diff --git a/.github/actions/audit-github-commit/action.yml b/.github/actions/audit-github-commit/action.yml index 852293b8..8a0ab35d 100644 --- a/.github/actions/audit-github-commit/action.yml +++ b/.github/actions/audit-github-commit/action.yml @@ -2,47 +2,47 @@ name: "Audit GitHub commit" description: "Get metadata about a commit" inputs: - sha: - description: "The commit to audit" - required: true + sha: + description: "The commit to audit" + required: true outputs: - tag: - description: "The associated release tag, if the release exists" - value: ${{ steps.commit.outputs.tag_name }} - release-id: - description: "The associated release id, if the release exists" - value: ${{ steps.commit.outputs.id }} - pre-release: - description: "If the associated release exists, is it a pre-release?" - value: ${{ steps.commit.outputs.prerelease }} - commitish: - description: "The associated commitish, if the release exists" - value: ${{ steps.commit.outputs.target_commitish }} + tag: + description: "The associated release tag, if the release exists" + value: ${{ steps.commit.outputs.tag_name }} + release-id: + description: "The associated release id, if the release exists" + value: ${{ steps.commit.outputs.id }} + pre-release: + description: "If the associated release exists, is it a pre-release?" + value: ${{ steps.commit.outputs.prerelease }} + commitish: + description: "The associated commitish, if the release exists" + value: ${{ steps.commit.outputs.target_commitish }} runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo sha : ${{ inputs.sha }} + using: composite + steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo sha : ${{ inputs.sha }} - - name: "Check if a release exists for `${{ inputs.sha }}`" - id: commit - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: cardinalby/git-get-release-action@1.2.4 - with: - commitSha: ${{ inputs.sha }} - doNotFailIfNotFound: true # returns blank outputs when not found instead of error - searchLimit: 15 # Since we only care about recent releases, speed up the process + - name: "Check if a release exists for `${{ inputs.sha }}`" + id: commit + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: cardinalby/git-get-release-action@1.2.4 + with: + commitSha: ${{ inputs.sha }} + doNotFailIfNotFound: true # returns blank outputs when not found instead of error + searchLimit: 15 # Since we only care about recent releases, speed up the process - - name: "[DEBUG] Commit metadata" - shell: bash - run: | - echo sha : ${{ inputs.sha }} - echo tag : ${{ steps.commit.outputs.tag_name }} - echo release-id : ${{ steps.commit.outputs.id }} - echo pre-release : ${{ steps.commit.outputs.prerelease }} - echo commitish : ${{ steps.commit.outputs.target_commitish }} + - name: "[DEBUG] Commit metadata" + shell: bash + run: | + echo sha : ${{ inputs.sha }} + echo tag : ${{ steps.commit.outputs.tag_name }} + echo release-id : ${{ steps.commit.outputs.id }} + echo pre-release : ${{ steps.commit.outputs.prerelease }} + echo commitish : ${{ steps.commit.outputs.target_commitish }} diff --git a/.github/actions/audit-github-tag/action.yml b/.github/actions/audit-github-tag/action.yml index 77a8f608..deeaaeca 100644 --- a/.github/actions/audit-github-tag/action.yml +++ b/.github/actions/audit-github-tag/action.yml @@ -2,74 +2,72 @@ name: "Audit GitHub tag" description: "Get metadata about a release tag" inputs: - tag: - description: "The tag to audit (e.g. v1.0.0b1)" - required: true - repo-url: - description: "The URL to the repo (e.g. https://github.com/dbt-labs/dbt-adapters)" - required: true + tag: + description: "The tag to audit (e.g. v1.0.0b1)" + required: true + repo-url: + description: "The URL to the repo (e.g. https://github.com/dbt-labs/dbt-adapters)" + required: true outputs: - exists: - description: "Indicates if the tag exists" - value: ${{ steps.tag.outputs.exists }} - sha: - description: "The commit associated with the tag if the tag exists" - value: ${{ steps.commit.outputs.sha }} - is-draft: - description: "If the tag exists, indicates if it is a draft" - value: ${{ steps.draft.outputs.is-draft }} + exists: + description: "Indicates if the tag exists" + value: ${{ steps.tag.outputs.exists }} + sha: + description: "The commit associated with the tag if the tag exists" + value: ${{ steps.commit.outputs.sha }} + is-draft: + description: "If the tag exists, indicates if it is a draft" + value: ${{ steps.draft.outputs.is-draft }} runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo tag : ${{ inputs.tag }} - echo repo-url : ${{ inputs.repo-url }} + using: composite + steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo tag : ${{ inputs.tag }} + echo repo-url : ${{ inputs.repo-url }} - - name: "Set: exists" - id: tag - shell: bash - run: | - output=$((gh release view ${{ inputs.tag }} --json isDraft,targetCommitish --repo ${{ inputs.repo-url }}) 2>&1) || true - echo "results=$output" >> $GITHUB_OUTPUT - if [[ "$output" == "release not found" ]] - then - echo "exists=false" >> $GITHUB_OUTPUT - else - echo "exists=true" >> $GITHUB_OUTPUT - fi - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: "Set: exists" + id: tag + shell: bash + run: | + output=$((gh release view ${{ inputs.tag }} --json isDraft,targetCommitish --repo ${{ inputs.repo-url }}) 2>&1) || true + echo "results=$output" >> $GITHUB_OUTPUT + if [[ "$output" == "release not found" ]]; then + echo "exists=false" >> $GITHUB_OUTPUT + else + echo "exists=true" >> $GITHUB_OUTPUT + fi + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: "Set: sha" - id: commit - if: ${{ fromJSON(steps.tag.outputs.exists) == true }} - shell: bash - run: | - sha=$(jq -r '.targetCommitish' <<< "${{ steps.tag.outputs.results }}") - echo "sha=$sha" >> $GITHUB_OUTPUT + - name: "Set: sha" + id: commit + if: ${{ fromJSON(steps.tag.outputs.exists) == true }} + shell: bash + run: | + sha=$(jq -r '.targetCommitish' <<< "${{ steps.tag.outputs.results }}") + echo "sha=$sha" >> $GITHUB_OUTPUT - - name: "Set: is-draft" - id: draft - if: ${{ fromJSON(steps.tag.outputs.exists) == true }} - shell: bash - run: | - is-draft=$(jq -r '.isDraft' <<< "${{ steps.tag.outputs.results }}") - if [[ $is-draft == true ]] - then - echo "is-draft=true" >> $GITHUB_OUTPUT - else - echo "is-draft=false" >> $GITHUB_OUTPUT - fi + - name: "Set: is-draft" + id: draft + if: ${{ fromJSON(steps.tag.outputs.exists) == true }} + shell: bash + run: | + is-draft=$(jq -r '.isDraft' <<< "${{ steps.tag.outputs.results }}") + if [[ $is-draft == true ]]; then + echo "is-draft=true" >> $GITHUB_OUTPUT + else + echo "is-draft=false" >> $GITHUB_OUTPUT + fi - - name: "[DEBUG] Tag metadata" - shell: bash - run: | - echo tag : ${{ inputs.tag }} - echo repo-url : ${{ inputs.repo-url }} - echo exists: : ${{ steps.release.outputs.exists }} - echo sha : ${{ steps.commit.outputs.sha }} - echo is-draft : ${{ steps.draft.outputs.is-draft }} + - name: "[DEBUG] Tag metadata" + shell: bash + run: | + echo tag : ${{ inputs.tag }} + echo repo-url : ${{ inputs.repo-url }} + echo exists: : ${{ steps.release.outputs.exists }} + echo sha : ${{ steps.commit.outputs.sha }} + echo is-draft : ${{ steps.draft.outputs.is-draft }} diff --git a/.github/actions/audit-github-team/action.yml b/.github/actions/audit-github-team/action.yml index fdaca6ba..f04d09bf 100644 --- a/.github/actions/audit-github-team/action.yml +++ b/.github/actions/audit-github-team/action.yml @@ -2,54 +2,54 @@ name: "Audit GitHub team" description: "Get metadata about a GitHub team" inputs: - organization: - description: "The GitHub organization that owns the team" - default: "dbt-labs" - team: - description: "The name of the team" - required: true + organization: + description: "The GitHub organization that owns the team" + default: "dbt-labs" + team: + description: "The name of the team" + required: true outputs: - members: - description: "A space delimited list of team members" - value: ${{ steps.members.outputs.membership }} + members: + description: "A space delimited list of team members" + value: ${{ steps.members.outputs.membership }} runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo organization : ${{ inputs.organization }} - echo team : ${{ inputs.team }} - - - name: "Create output file name" - id: output-file - shell: bash - run: echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT - - - name: "Get team membership" - shell: bash - run: | - url = orgs/${{ inputs.organization }}/teams/${{ inputs.team }}/members - header = Accept: application/vnd.github+json - gh api -H "$header" $url > ${{ steps.output-file.outputs.name }} - - - name: "Parse team membership" - id: members - shell: bash - run: | - team_list=$(jq -r '.[].login' ${{ steps.output-file.outputs.name }}) - team_list_single=$(echo $team_list | tr '\n' ' ') - echo "membership=$team_list_single" >> $GITHUB_OUTPUT - - - name: "Delete the output file" - shell: bash - run: rm ${{ steps.output-file.outputs.name }} - - - name: "[DEBUG] Parse team membership" - shell: bash - run: | - echo organization : ${{ inputs.organization }} - echo team : ${{ inputs.team }} - echo members : ${{ steps.members.outputs.membership }} + using: composite + steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo organization : ${{ inputs.organization }} + echo team : ${{ inputs.team }} + + - name: "Create output file name" + id: output-file + shell: bash + run: echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT + + - name: "Get team membership" + shell: bash + run: | + url = orgs/${{ inputs.organization }}/teams/${{ inputs.team }}/members + header = Accept: application/vnd.github+json + gh api -H "$header" $url > ${{ steps.output-file.outputs.name }} + + - name: "Parse team membership" + id: members + shell: bash + run: | + team_list=$(jq -r '.[].login' ${{ steps.output-file.outputs.name }}) + team_list_single=$(echo $team_list | tr '\n' ' ') + echo "membership=$team_list_single" >> $GITHUB_OUTPUT + + - name: "Delete the output file" + shell: bash + run: rm ${{ steps.output-file.outputs.name }} + + - name: "[DEBUG] Parse team membership" + shell: bash + run: | + echo organization : ${{ inputs.organization }} + echo team : ${{ inputs.team }} + echo members : ${{ steps.members.outputs.membership }} diff --git a/.github/actions/build-artifacts/action.yml b/.github/actions/build-artifacts/action.yml new file mode 100644 index 00000000..8b7acb90 --- /dev/null +++ b/.github/actions/build-artifacts/action.yml @@ -0,0 +1,65 @@ +name: "Build artifacts" +description: "Build artifacts using `hatch` as the default build frontend" + +inputs: + build-command: + description: "The command to build distributable artifacts" + default: "hatch build" + check-command: + description: "The command to check built artifacts" + default: "hatch run build:check-all" + archive-name: + description: "Name for the GitHub archive for uploading built artifacts, leave blank for no upload" + default: "" + archive-retention: + description: "Duration in days to keep the GitHub archive, requires `archive-name`" + default: "3" + working-dir: + description: "Where to run commands from, primarily supports namespace packaging" + default: "./" + +runs: + using: composite + steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo build-command : ${{ inputs.build-command }} + echo check-command : ${{ inputs.check-command }} + echo archive-name : ${{ inputs.archive-name }} + echo archive-retention : ${{ inputs.archive-retention }} + echo working-dir : ${{ inputs.working-dir }} + + - name: "Build artifacts" + shell: bash + working-directory: ${{ inputs.working-dir }} + run: ${{ inputs.build-command }} + + - name: "[INFO] Built artifacts" + shell: bash + working-directory: ${{ inputs.working-dir }} + run: | + title="Built artifacts" + message=$(ls ./dist) + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Check artifacts" + shell: bash + working-directory: ${{ inputs.working-dir }} + run: ${{ inputs.check-command }} + + - name: "Upload artifacts" + if: ${{ !inputs.archive-name == '' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.archive-name }} + path: ${{ inputs.working-dir }}dist/ + retention-days: ${{ inputs.archive-retention }} + + - name: "[INFO] Uploaded artifacts" + if: ${{ !inputs.archive-name == '' }} + shell: bash + run: | + title="Uploaded artifacts" + message="Uploaded artifacts to ${{ inputs.archive-name }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/setup-changie/action.yml b/.github/actions/setup-changie/action.yml deleted file mode 100644 index 55e994e1..00000000 --- a/.github/actions/setup-changie/action.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: "Setup `changie`" -description: "Setup `changie` using `linuxbrew`" - -runs: - using: composite - steps: - - name: "Add `linuxbrew` to PATH" - shell: bash - run: echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH - - - name: "Install `changie`" - shell: bash - run: | - brew tap miniscruff/changie https://github.com/miniscruff/changie - brew install changie diff --git a/.github/actions/setup-environment/action.yml b/.github/actions/setup-environment/action.yml index 80433cd3..d75e4788 100644 --- a/.github/actions/setup-environment/action.yml +++ b/.github/actions/setup-environment/action.yml @@ -2,27 +2,37 @@ name: "Setup environment" description: "Setup a python environment with `hatch` as the default build frontend" inputs: - python-version: - description: "The version of python to install" - default: "3.8" - setup-build-command: - description: "The command to setup build dependencies" - default: "python -m pip install hatch" + python-version: + description: "The version of python to install" + default: "3.8" + setup-build-command: + description: "The command to setup build dependencies" + default: "python -m pip install hatch" runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo python-version : ${{ inputs.python-version }} - echo setup-build-command : ${{ inputs.setup-build-command }} + using: composite + steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo python-version : ${{ inputs.python-version }} + echo setup-build-command : ${{ inputs.setup-build-command }} - - name: "Set up python ${{ inputs.python-version }}" - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version }} + - name: "Set up python ${{ inputs.python-version }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} - - name: "Install build dependencies" - shell: bash - run: ${{ inputs.setup-build-command }} + - name: "Install build dependencies" + shell: bash + run: ${{ inputs.setup-build-command }} + + - name: "Add `linuxbrew` to PATH" + shell: bash + run: echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH + + - name: "Install `changie`" + shell: bash + run: | + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie From 4afbda6bec62efd810f4269c1e1b2f3c1cc170ad Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 24 Apr 2024 22:33:24 -0400 Subject: [PATCH 16/16] move actions to dbt-labs/actions repo --- .github/actions/audit-changelog/action.yml | 57 ------- .../actions/audit-github-commit/action.yml | 48 ------ .github/actions/audit-github-tag/action.yml | 73 --------- .github/actions/audit-github-team/action.yml | 55 ------- .github/actions/build-artifacts/action.yml | 65 -------- .github/actions/create-temp-branch/action.yml | 41 ----- .github/actions/github-commit/action.yml | 40 ----- .github/actions/github-merge/action.yml | 59 ------- .github/actions/log-info/action.yml | 22 --- .../actions/publish-github-draft/action.yml | 32 ---- .github/actions/publish-github/action.yml | 91 ----------- .github/actions/publish-pypi/action.yml | 57 ------- .github/actions/setup-environment/action.yml | 38 ----- .github/workflows/build-artifacts.yml | 146 ++++++------------ 14 files changed, 49 insertions(+), 775 deletions(-) delete mode 100644 .github/actions/audit-changelog/action.yml delete mode 100644 .github/actions/audit-github-commit/action.yml delete mode 100644 .github/actions/audit-github-tag/action.yml delete mode 100644 .github/actions/audit-github-team/action.yml delete mode 100644 .github/actions/build-artifacts/action.yml delete mode 100644 .github/actions/create-temp-branch/action.yml delete mode 100644 .github/actions/github-commit/action.yml delete mode 100644 .github/actions/github-merge/action.yml delete mode 100644 .github/actions/log-info/action.yml delete mode 100644 .github/actions/publish-github-draft/action.yml delete mode 100644 .github/actions/publish-github/action.yml delete mode 100644 .github/actions/publish-pypi/action.yml delete mode 100644 .github/actions/setup-environment/action.yml diff --git a/.github/actions/audit-changelog/action.yml b/.github/actions/audit-changelog/action.yml deleted file mode 100644 index 8599f112..00000000 --- a/.github/actions/audit-changelog/action.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: "Audit changelog" -description: "Get metadata about a changelog" - -inputs: - version: - description: "The version whose changelog is being audited" - required: true - -outputs: - path: - description: "The expected file path to the change log, relative to the repo root" - value: ${{ steps.path.outputs.path }} - exists: - description: "Indicates if the changelog exists" - value: ${{ steps.exists.outputs.exists }} - -runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo version : ${{ inputs.version }} - - - name: "Parse version: `${{ inputs.version }}`" - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ inputs.version }} - - - name: "Set: path" - id: path - shell: bash - run: | - path=".changes/${{ steps.semver.outputs.base-version }}" - if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]]; then - path+="-${{ steps.semver.outputs.pre-release }}" - fi - path+=".md" - echo "path=$path" >> $GITHUB_OUTPUT - - - name: "Set: exists" - id: exists - shell: bash - run: | - exists=false - if test -f ${{ steps.path.outputs.path }}; then - exists=true - fi - echo "exists=exists">> $GITHUB_OUTPUT - - - name: "[DEBUG] Changelog metadata" - shell: bash - run: | - echo version : ${{ inputs.version }} - echo path : ${{ steps.path.outputs.path }} - echo exists : ${{ steps.exists.outputs.exists }} diff --git a/.github/actions/audit-github-commit/action.yml b/.github/actions/audit-github-commit/action.yml deleted file mode 100644 index 8a0ab35d..00000000 --- a/.github/actions/audit-github-commit/action.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: "Audit GitHub commit" -description: "Get metadata about a commit" - -inputs: - sha: - description: "The commit to audit" - required: true - -outputs: - tag: - description: "The associated release tag, if the release exists" - value: ${{ steps.commit.outputs.tag_name }} - release-id: - description: "The associated release id, if the release exists" - value: ${{ steps.commit.outputs.id }} - pre-release: - description: "If the associated release exists, is it a pre-release?" - value: ${{ steps.commit.outputs.prerelease }} - commitish: - description: "The associated commitish, if the release exists" - value: ${{ steps.commit.outputs.target_commitish }} - -runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo sha : ${{ inputs.sha }} - - - name: "Check if a release exists for `${{ inputs.sha }}`" - id: commit - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: cardinalby/git-get-release-action@1.2.4 - with: - commitSha: ${{ inputs.sha }} - doNotFailIfNotFound: true # returns blank outputs when not found instead of error - searchLimit: 15 # Since we only care about recent releases, speed up the process - - - name: "[DEBUG] Commit metadata" - shell: bash - run: | - echo sha : ${{ inputs.sha }} - echo tag : ${{ steps.commit.outputs.tag_name }} - echo release-id : ${{ steps.commit.outputs.id }} - echo pre-release : ${{ steps.commit.outputs.prerelease }} - echo commitish : ${{ steps.commit.outputs.target_commitish }} diff --git a/.github/actions/audit-github-tag/action.yml b/.github/actions/audit-github-tag/action.yml deleted file mode 100644 index deeaaeca..00000000 --- a/.github/actions/audit-github-tag/action.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: "Audit GitHub tag" -description: "Get metadata about a release tag" - -inputs: - tag: - description: "The tag to audit (e.g. v1.0.0b1)" - required: true - repo-url: - description: "The URL to the repo (e.g. https://github.com/dbt-labs/dbt-adapters)" - required: true - -outputs: - exists: - description: "Indicates if the tag exists" - value: ${{ steps.tag.outputs.exists }} - sha: - description: "The commit associated with the tag if the tag exists" - value: ${{ steps.commit.outputs.sha }} - is-draft: - description: "If the tag exists, indicates if it is a draft" - value: ${{ steps.draft.outputs.is-draft }} - -runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo tag : ${{ inputs.tag }} - echo repo-url : ${{ inputs.repo-url }} - - - name: "Set: exists" - id: tag - shell: bash - run: | - output=$((gh release view ${{ inputs.tag }} --json isDraft,targetCommitish --repo ${{ inputs.repo-url }}) 2>&1) || true - echo "results=$output" >> $GITHUB_OUTPUT - if [[ "$output" == "release not found" ]]; then - echo "exists=false" >> $GITHUB_OUTPUT - else - echo "exists=true" >> $GITHUB_OUTPUT - fi - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: "Set: sha" - id: commit - if: ${{ fromJSON(steps.tag.outputs.exists) == true }} - shell: bash - run: | - sha=$(jq -r '.targetCommitish' <<< "${{ steps.tag.outputs.results }}") - echo "sha=$sha" >> $GITHUB_OUTPUT - - - name: "Set: is-draft" - id: draft - if: ${{ fromJSON(steps.tag.outputs.exists) == true }} - shell: bash - run: | - is-draft=$(jq -r '.isDraft' <<< "${{ steps.tag.outputs.results }}") - if [[ $is-draft == true ]]; then - echo "is-draft=true" >> $GITHUB_OUTPUT - else - echo "is-draft=false" >> $GITHUB_OUTPUT - fi - - - name: "[DEBUG] Tag metadata" - shell: bash - run: | - echo tag : ${{ inputs.tag }} - echo repo-url : ${{ inputs.repo-url }} - echo exists: : ${{ steps.release.outputs.exists }} - echo sha : ${{ steps.commit.outputs.sha }} - echo is-draft : ${{ steps.draft.outputs.is-draft }} diff --git a/.github/actions/audit-github-team/action.yml b/.github/actions/audit-github-team/action.yml deleted file mode 100644 index f04d09bf..00000000 --- a/.github/actions/audit-github-team/action.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: "Audit GitHub team" -description: "Get metadata about a GitHub team" - -inputs: - organization: - description: "The GitHub organization that owns the team" - default: "dbt-labs" - team: - description: "The name of the team" - required: true - -outputs: - members: - description: "A space delimited list of team members" - value: ${{ steps.members.outputs.membership }} - -runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo organization : ${{ inputs.organization }} - echo team : ${{ inputs.team }} - - - name: "Create output file name" - id: output-file - shell: bash - run: echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT - - - name: "Get team membership" - shell: bash - run: | - url = orgs/${{ inputs.organization }}/teams/${{ inputs.team }}/members - header = Accept: application/vnd.github+json - gh api -H "$header" $url > ${{ steps.output-file.outputs.name }} - - - name: "Parse team membership" - id: members - shell: bash - run: | - team_list=$(jq -r '.[].login' ${{ steps.output-file.outputs.name }}) - team_list_single=$(echo $team_list | tr '\n' ' ') - echo "membership=$team_list_single" >> $GITHUB_OUTPUT - - - name: "Delete the output file" - shell: bash - run: rm ${{ steps.output-file.outputs.name }} - - - name: "[DEBUG] Parse team membership" - shell: bash - run: | - echo organization : ${{ inputs.organization }} - echo team : ${{ inputs.team }} - echo members : ${{ steps.members.outputs.membership }} diff --git a/.github/actions/build-artifacts/action.yml b/.github/actions/build-artifacts/action.yml deleted file mode 100644 index 8b7acb90..00000000 --- a/.github/actions/build-artifacts/action.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: "Build artifacts" -description: "Build artifacts using `hatch` as the default build frontend" - -inputs: - build-command: - description: "The command to build distributable artifacts" - default: "hatch build" - check-command: - description: "The command to check built artifacts" - default: "hatch run build:check-all" - archive-name: - description: "Name for the GitHub archive for uploading built artifacts, leave blank for no upload" - default: "" - archive-retention: - description: "Duration in days to keep the GitHub archive, requires `archive-name`" - default: "3" - working-dir: - description: "Where to run commands from, primarily supports namespace packaging" - default: "./" - -runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo build-command : ${{ inputs.build-command }} - echo check-command : ${{ inputs.check-command }} - echo archive-name : ${{ inputs.archive-name }} - echo archive-retention : ${{ inputs.archive-retention }} - echo working-dir : ${{ inputs.working-dir }} - - - name: "Build artifacts" - shell: bash - working-directory: ${{ inputs.working-dir }} - run: ${{ inputs.build-command }} - - - name: "[INFO] Built artifacts" - shell: bash - working-directory: ${{ inputs.working-dir }} - run: | - title="Built artifacts" - message=$(ls ./dist) - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - - name: "Check artifacts" - shell: bash - working-directory: ${{ inputs.working-dir }} - run: ${{ inputs.check-command }} - - - name: "Upload artifacts" - if: ${{ !inputs.archive-name == '' }} - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.archive-name }} - path: ${{ inputs.working-dir }}dist/ - retention-days: ${{ inputs.archive-retention }} - - - name: "[INFO] Uploaded artifacts" - if: ${{ !inputs.archive-name == '' }} - shell: bash - run: | - title="Uploaded artifacts" - message="Uploaded artifacts to ${{ inputs.archive-name }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/create-temp-branch/action.yml b/.github/actions/create-temp-branch/action.yml deleted file mode 100644 index 6fe53ceb..00000000 --- a/.github/actions/create-temp-branch/action.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: "Create temp branch" -description: "Create a unique temporary branch for running CI workflows, such as version bumps, changelog consolidation, etc." - -inputs: - branch-stub: - description: "Stub to use for naming the branch (e.g. prep-release/, nightly-release/, etc.)" - default: "temp/" - -outputs: - branch: - description: "Name of the newly created branch" - value: ${{ steps.branch.outputs.branch-name }} - -runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo branch-stub : ${{ inputs.branch-stub }} - - - name: "Create a unique branch name" - id: branch - shell: bash - run: | - name="${{ inputs.branch-stub }}" - name+="$(date +'%Y-%m-%d')/$GITHUB_RUN_ID" - echo "branch-name=$name" >> $GITHUB_OUTPUT - - - name: "Create branch: `${{ steps.branch.outputs.branch-name }}`" - shell: bash - run: | - git checkout -b ${{ steps.branch.outputs.branch-name }} - git push -u origin ${{ steps.branch.outputs.branch-name }} - - - name: "[INFO] Created branch" - shell: bash - run: | - title="Created branch" - message="${{ steps.branch.outputs.branch-name }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/github-commit/action.yml b/.github/actions/github-commit/action.yml deleted file mode 100644 index 0ecab9fb..00000000 --- a/.github/actions/github-commit/action.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: "Commit changes" -description: "Commit changes and push back up to the remote" - -inputs: - message: - description: "The commit message" - required: true - user: - description: "The user for the commit" - default: "Github Build Bot" - email: - description: "The email for the commit" - default: "buildbot@fishtownanalytics.com" - -runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo message : ${{ inputs.message }} - echo user : ${{ inputs.user }} - echo email : ${{ inputs.email }} - - - name: "Commit and push changes" - shell: bash - run: | - git config user.name "${{ inputs.user }}" - git config user.email "${{ inputs.email }}" - git pull - git add . - git commit -m "[automated] ${{ inputs.message }}" - git push - - - name: "[INFO] Committed and pushed changes" - shell: bash - run: | - title="Committed and pushed changes" - message="Committed and pushed changes back up to remote" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/github-merge/action.yml b/.github/actions/github-merge/action.yml deleted file mode 100644 index efe7464c..00000000 --- a/.github/actions/github-merge/action.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: "Merge GitHub branch" -description: "Merge one branch into another on GitHub" - -inputs: - source-branch: - description: "The source branch to be merged" - required: true - target-branch: - description: "The target branch to receive the merge" - required: true - message: - description: "Commit message for the merge" - required: true - delete-source-branch: - description: "Will delete the source branch after merging" - default: "true" - -outputs: - sha: - description: "The commit SHA for the merge" - value: ${{ steps.merge-sha.outputs.sha }} - -runs: - using: composite - steps: - # TODO: are we comfortable passing our bot token to this third party action? - - name: "Merge `${{ inputs.source-branch }}` into `${{ inputs.target-branch }}`" - uses: everlytic/branch-merge@1.1.5 - with: - source_ref: ${{ inputs.source-branch }} - target_branch: ${{ inputs.target-branch }} - commit_message_template: "[automated] ${{ inputs.message }}" - github_token: ${{ secrets.FISHTOWN_BOT_PAT }} - - - name: "Checkout `${{ inputs.branch }}`" - uses: actions/checkout@v3 - with: - ref: ${{ inputs.target-branch }} - - # TODO: see if everlytic/branch-merge has outputs that have this information - - name: "Get commit SHA for the merge" - id: merge-sha - shell: bash - run: | - git pull - echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - # TODO: see if everlytic/branch-merge has options to do this - - name: "Delete `${{ inputs.source-branch }}`" - if: ${{ fromJSON(inputs.delete-source-branch) }} - shell: bash - run: git push origin -d ${{ inputs.source-branch }} - - - name: "[INFO] Merged changes" - shell: bash - run: | - title="Merged changes" - message="${{ inputs.source-branch }} was merged into ${{ inputs.target-branch }} with sha ${{ steps.merge-sha.outputs.sha }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/log-info/action.yml b/.github/actions/log-info/action.yml deleted file mode 100644 index f6ea0cdc..00000000 --- a/.github/actions/log-info/action.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "Raise a notification" -description: "Raises a notification to the workflow summary in a standardized pattern" - -inputs: - title: - description: "The notification title" - required: true - message: - description: "The detailed message" - required: true - prefix: - description: "The parent process raising the notification" - default: "INFO" - -runs: - using: composite - steps: - - shell: bash - run: 'echo "::notice title=$TITLE::$MESSAGE"' - env: - TITLE: "[${{ inputs.prefix }}]: ${{ inputs.title }}" - MESSAGE: ${{ inputs.message }} diff --git a/.github/actions/publish-github-draft/action.yml b/.github/actions/publish-github-draft/action.yml deleted file mode 100644 index 8506d94b..00000000 --- a/.github/actions/publish-github-draft/action.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: "Publish GitHub draft release as full release" -description: "Publish an existing draft release as a non-draft release" - -inputs: - tag: - description: "The tag to publish (i.e. v1.0.0b1)" - required: true - repo-url: - description: "The URL to the repo (https://github.com/dbt-labs/dbt-adapters" - required: true - -runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo tag : ${{ inputs.tag }} - echo repo-url : ${{ inputs.repo-url }} - - - name: "Publish `${{ inputs.tag }}` as full release" - shell: bash - run: gh release edit ${{ inputs.tag }} --repo ${{ inputs.repo-url }} --draft=false - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: "[INFO] Published draft as full release" - shell: bash - run: | - title="Released draft as non-draft release on GitHub" - message="tag: ${{ inputs.tag }} repo: ${{ inputs.repo-url }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/publish-github/action.yml b/.github/actions/publish-github/action.yml deleted file mode 100644 index 6042357c..00000000 --- a/.github/actions/publish-github/action.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: "Publish to GitHub as draft release" -description: "Publish artifacts from an archive to GitHub as a draft release" - -inputs: - archive-name: - description: "Name of the archive containing the artifacts, leave blank if local" - default: "" - tag: - description: "The release tag to publish under" - required: true - repo-url: - description: "The URL to the repo (e.g. https://github.com/dbt-labs/dbt-adapters)" - required: true - sha: - description: "Commit SHA being released" - required: true - changelog-path: - description: "Path to the release notes" - required: true - is-draft: - description: "Indicates that this is a draft release" - required: true - -runs: - using: composite - steps: - - name: "Set: pre-release" - id: pre-release - shell: bash - run: | - if ${{ contains(inputs.tag, 'rc') || contains(inputs.tag, 'b') || contains(inputs.tag, 'a' || contains(inputs.tag, 'dev') }} - then - echo "pre-release=--prerelease" >> $GITHUB_OUTPUT - fi - - - name: "Set: draft" - id: draft - shell: bash - run: | - if ${{ fromJSON(inputs.is-draft) }} - then - echo "draft=--draft" >> $GITHUB_OUTPUT - fi - - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo archive-name : ${{ inputs.archive-name }} - echo tag : ${{ inputs.tag }} - echo repo-url : ${{ inputs.repo-url }} - echo sha : ${{ inputs.sha }} - echo changelog-path : ${{ inputs.changelog-path }} - echo is-draft : ${{ inputs.is-draft }} - echo pre-release : ${{ steps.pre-release.outputs.pre-release }} - echo draft : ${{ steps.draft.outputs.draft }} - - - name: "Download artifacts from `${{ inputs.archive-name }}`" - if: ${{ !(inputs.archive-name == "" }} - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.archive-name }} - path: dist/ - - - name: "[INFO] Downloaded artifacts" - if: ${{ !(inputs.archive-name == "" }} - uses: ./.github/actions/log-info - with: - title: "Downloaded artifacts" - message: "${{ inputs.archive-name }} contained: $(ls ./dist)" - prefix: ${{ env.NOTIFICATION_PREFIX }} - - - name: "Publish artifacts to GitHub as draft release" - shell: bash - run: gh release create $TAG $ARTIFACTS --repo $REPO --title "$TITLE" --notes-file $RELEASE_NOTES --target $COMMIT $PRERELEASE $DRAFT - env: - TAG: ${{ inputs.tag }} - ARTIFACTS: "./dist/*" - REPO: ${{ inputs.repo-url }} - TITLE: ${{ github.event.repository.name }} ${{ inputs.tag }} - RELEASE_NOTES: ${{ inputs.changelog-path }} - COMMIT: ${{ inputs.sha }} - PRERELEASE: ${{ steps.pre-release.outputs.pre-release }} - DRAFT: ${{ steps.draft.outputs.draft }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: "[INFO] Published artifacts to GitHub as draft release" - uses: ./.github/actions/log-info - with: - title: "Published artifacts to GitHub as draft release" - message: "tag: ${{ inputs.tag }} pre-release: ${{ steps.pre-release.outputs.pre-release }} artifacts: $(ls ./dist)" - prefix: ${{ env.NOTIFICATION_PREFIX }} diff --git a/.github/actions/publish-pypi/action.yml b/.github/actions/publish-pypi/action.yml deleted file mode 100644 index 472b42ca..00000000 --- a/.github/actions/publish-pypi/action.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: "Publish to PyPI" -description: "Publish artifacts from an archive to PyPI" - -inputs: - archive-name: - description: "Name of the archive containing the artifacts, leave blank if local" - default: "" - index-url: - description: "The url for the PyPI index" - required: true - working-dir: - description: "Where to run commands from, primarily supports namespace packaging" - default: "./" - -runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo archive-name : ${{ inputs.archive-name }} - echo index-url : ${{ inputs.index-url }} - echo working-dir : ${{ inputs.working-dir }} - - - name: "Download artifacts from `${{ inputs.archive-name }}`" - if: ${{ !(inputs.archive-name == "" }} - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.archive-name }} - path: dist/ - - - name: "[INFO] Downloaded artifacts" - if: ${{ !(inputs.archive-name == "" }} - shell: bash - run: | - title="Downloaded artifacts" - message="Downloaded artifacts from ${{ inputs.archive-name }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - - name: "[DEBUG] Found artifacts" - shell: bash - working-directory: ${{ inputs.working-dir }} - run: echo $(ls ./dist) - - - name: "Publish artifacts to PyPI" - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: ${{ inputs.index-url }} - packages-dir: ${{ inputs.working-dir }}dist/ - - - name: "[INFO] Published artifacts to PyPI" - shell: bash - working-directory: ${{ inputs.working-dir }} - run: | - title="Published artifacts to PyPI" - message=$(ls ./dist) - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/actions/setup-environment/action.yml b/.github/actions/setup-environment/action.yml deleted file mode 100644 index d75e4788..00000000 --- a/.github/actions/setup-environment/action.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: "Setup environment" -description: "Setup a python environment with `hatch` as the default build frontend" - -inputs: - python-version: - description: "The version of python to install" - default: "3.8" - setup-build-command: - description: "The command to setup build dependencies" - default: "python -m pip install hatch" - -runs: - using: composite - steps: - - name: "[DEBUG] Inputs" - shell: bash - run: | - echo python-version : ${{ inputs.python-version }} - echo setup-build-command : ${{ inputs.setup-build-command }} - - - name: "Set up python ${{ inputs.python-version }}" - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version }} - - - name: "Install build dependencies" - shell: bash - run: ${{ inputs.setup-build-command }} - - - name: "Add `linuxbrew` to PATH" - shell: bash - run: echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH - - - name: "Install `changie`" - shell: bash - run: | - brew tap miniscruff/changie https://github.com/miniscruff/changie - brew install changie diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index 13e08bbb..0da8c151 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -2,111 +2,63 @@ name: "Build artifacts" run-name: "Build `${{ inputs.package }}==${{ inputs.version}}` from `${{ inputs.branch }}` for deployment to `${{ deploy-environment }}`" on: - workflow_call: - inputs: - branch: - description: "The branch being used to build the package" - type: string - default: "main" - package: - description: "The name of the package being built" - type: string - version: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - deploy-environment: - description: "The environment where this package will be deployed" - type: string - default: "prod" - outputs: - archive-name: ${{ jobs.calculated-inputs.outputs.archive-name }} - workflow_dispatch: - inputs: - branch: - description: "The branch being used to build the package" - type: string - default: "main" - package: - description: "The name of the package being built" - type: choice - options: - - "dbt-adapters" - - "dbt-tests-adapter" - version: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - deploy-environment: - description: "The environment where this package will be deployed" - type: environment + workflow_call: + inputs: + package: + description: "The package being built" + type: string + branch: + description: "The branch to use to build the artifacts" + type: string + default: "main" + outputs: + archive-name: + value: ${{ jobs.build-artifacts.outputs.archive-name }} + description: "The name used for the upload archive" + type: string + working-directory: + value: ${{ jobs.build-artifacts.outputs.working-directory }} + description: "The working directory for the package" + type: string permissions: read-all # don't try to build the same version of the same package, the archive name could collide concurrency: - group: ${{ github.workflow }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-environment }} - cancel-in-progress: true - -defaults: - run: - shell: bash + group: "${{ github.workflow }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-environment }}" + cancel-in-progress: true env: - NOTIFICATION_PREFIX: "[Build]" + NOTIFICATION_PREFIX: "Build artifacts" jobs: - calculated-inputs: - name: "Calculated inputs" - runs-on: ubuntu-latest - outputs: - archive-name: ${{ steps.archive.outputs.name }} - working-dir: ${{ steps.working-dir.outputs.path }} - steps: - - name: "Set: archive-name" - id: archive - shell: bash - run: | - name=${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-environment }} - echo "name=$name" >> $GITHUB_OUTPUT - - - name: "Set: working-dir" - id: working-dir - shell: bash - run: | - if [[ ${{ inputs.package }} == "dbt-tests-adapter" ]] - then - path="./dbt-tests-adapter/" - else - path="./" - fi - echo "path=$path" >> $GITHUB_OUTPUT - - - name: "[DEBUG] Inputs" - run: | - echo package : ${{ inputs.package }} - echo branch : ${{ inputs.branch }} - echo version : ${{ inputs.version }} - echo deploy-environment : ${{ inputs.deploy-environment }} - echo archive-name : ${{ steps.archive.outputs.name }} - echo working-dir : ${{ steps.working-dir.outputs.path }} - echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} - - build: - name: "Build the artifacts" - needs: [calculated-inputs] - runs-on: ubuntu-latest - steps: - - name: "Check out `${{ inputs.branch }}`" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} + build-artifacts: + name: "Build artifacts" + runs-on: ubuntu-latest + outputs: + archive-name: ${{ steps.archive.outputs.name }} + working-directory: ${{ steps.working-directory.outputs.path }} + steps: + - name: "Set: archive-name" + id: archive + shell: bash + run: | + name=${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-environment }} + echo "name=$name" >> $GITHUB_OUTPUT - - name: "Setup `hatch`" - uses: ./.github/actions/setup-environment + - name: "Set: working-directory" + id: working-directory + shell: bash + run: | + if [[ ${{ inputs.package }} == "dbt-tests-adapter" ]]; then + path="./dbt-tests-adapter/" + else + path="./" + fi + echo "path=$path" >> $GITHUB_OUTPUT - - name: "Build `${{ inputs.package }}`" - uses: ./.github/actions/build - with: - archive-name: ${{ needs.calculated-inputs.outputs.archive-name }} - working-dir: ${{ needs.calculated-inputs.outputs.working-dir }} + - name: "Build artifacts" + uses: dbt-labs/actions/hatch/artifacts/create.yml@add-hatch-actions + with: + archive-name: ${{ steps.archive.outputs.name }} + working-directory: ${{ needs.working-directory.outputs.path }}