diff --git a/.github/actions/build-hatch/action.yml b/.github/actions/build-hatch/action.yml deleted file mode 100644 index fe9825d4..00000000 --- a/.github/actions/build-hatch/action.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Build - `hatch` -description: Build artifacts using the `hatch` build backend - -inputs: - build-command: - description: The command to build distributable artifacts - default: "hatch build" - check-command: - description: The command to check built artifacts - default: "hatch run build:check-all" - working-dir: - description: Where to run commands from, supports namespace packaging - default: "./" - archive-name: - description: Where to upload the artifacts - required: true - -runs: - using: composite - steps: - - - name: Build artifacts - run: ${{ inputs.build-command }} - shell: bash - working-directory: ${{ inputs.working-dir }} - - - name: Check artifacts - run: ${{ inputs.check-command }} - shell: bash - working-directory: ${{ inputs.working-dir }} - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.archive-name }} - path: ${{ inputs.working-dir }}dist/ - retention-days: 3 diff --git a/.github/actions/publish-pypi/action.yml b/.github/actions/publish-pypi/action.yml deleted file mode 100644 index deffc6e3..00000000 --- a/.github/actions/publish-pypi/action.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Publish - PyPI -description: Publish artifacts saved during build step to PyPI - -inputs: - archive-name: - description: Where to download the artifacts from - required: true - repository-url: - description: The PyPI index to publish to, test or prod - required: true - -runs: - using: composite - steps: - - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.archive-name }} - path: dist/ - - - name: "[DEBUG]" - run : ls -R - shell: bash - - - name: Publish artifacts to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: ${{ inputs.repository-url }} diff --git a/.github/actions/publish-results/action.yml b/.github/actions/publish-results/action.yml deleted file mode 100644 index d863d659..00000000 --- a/.github/actions/publish-results/action.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Publish results - -inputs: - file-name: - description: File type for file name stub (e.g. "unit-tests") - required: true - python-version: - description: Python version for the file name stub (e.g. "3.8") - required: true - source-file: - description: File to be uploaded - required: true - -runs: - using: composite - steps: - - name: Get timestamp - id: timestamp - run: echo "ts=$(date +'%Y-%m-%dT%H-%M-%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - shell: bash - - - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.file-name }}_python-${{ inputs.python-version }}_${{ steps.timestamp.outputs.ts }}.csv - path: ${{ inputs.source-file }} diff --git a/.github/actions/setup-hatch/action.yml b/.github/actions/setup-hatch/action.yml deleted file mode 100644 index 7b7780ef..00000000 --- a/.github/actions/setup-hatch/action.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Setup - `hatch` -description: Setup a python environment with `hatch` installed - -inputs: - setup-command: - description: The command to setup development dependencies - default: "python -m pip install hatch" - python-version: - description: The version of python to install - default: "3.11" - -runs: - using: composite - steps: - - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python-version }} - - - name: Install dev dependencies - run: ${{ inputs.setup-command }} - shell: bash diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 00000000..0da8c151 --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,64 @@ +name: "Build artifacts" +run-name: "Build `${{ inputs.package }}==${{ inputs.version}}` from `${{ inputs.branch }}` for deployment to `${{ deploy-environment }}`" + +on: + workflow_call: + inputs: + package: + description: "The package being built" + type: string + branch: + description: "The branch to use to build the artifacts" + type: string + default: "main" + outputs: + archive-name: + value: ${{ jobs.build-artifacts.outputs.archive-name }} + description: "The name used for the upload archive" + type: string + working-directory: + value: ${{ jobs.build-artifacts.outputs.working-directory }} + description: "The working directory for the package" + type: string + +permissions: read-all + +# don't try to build the same version of the same package, the archive name could collide +concurrency: + group: "${{ github.workflow }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-environment }}" + cancel-in-progress: true + +env: + NOTIFICATION_PREFIX: "Build artifacts" + +jobs: + build-artifacts: + name: "Build artifacts" + runs-on: ubuntu-latest + outputs: + archive-name: ${{ steps.archive.outputs.name }} + working-directory: ${{ steps.working-directory.outputs.path }} + steps: + - name: "Set: archive-name" + id: archive + shell: bash + run: | + name=${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-environment }} + echo "name=$name" >> $GITHUB_OUTPUT + + - name: "Set: working-directory" + id: working-directory + shell: bash + run: | + if [[ ${{ inputs.package }} == "dbt-tests-adapter" ]]; then + path="./dbt-tests-adapter/" + else + path="./" + fi + echo "path=$path" >> $GITHUB_OUTPUT + + - name: "Build artifacts" + uses: dbt-labs/actions/hatch/artifacts/create.yml@add-hatch-actions + with: + archive-name: ${{ steps.archive.outputs.name }} + working-directory: ${{ needs.working-directory.outputs.path }} diff --git a/.github/workflows/build-changelog.yml b/.github/workflows/build-changelog.yml new file mode 100644 index 00000000..4e2dfa86 --- /dev/null +++ b/.github/workflows/build-changelog.yml @@ -0,0 +1,174 @@ +name: "Build changelog" +run-name: "Merge changelogs into one changelog on ${{ inputs.branch }} for ${{ inputs.version }}" + +on: + workflow_call: + inputs: + branch: + description: "The branch where the changelogs should be merged" + type: string + default: "main" + version: + description: "The version whose changelogs should be merged" + type: string + required: true + outputs: + changelog-path: + description: "The path to the changelog for this version" + value: ${{ jobs.audit-changelog.outputs.path }} + created: + description: "Identifies if this workflows created the changelog" + value: ${{ !jobs.audit-changelog.outputs.path }} + workflow_dispatch: + inputs: + branch: + description: "The branch where the changelogs should be merged" + type: string + default: "main" + version: + description: "The version whose changelogs should be merged" + type: string + required: true + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +# cancel the previous run for this version if a new run is called on this branch +concurrency: + group: ${{ github.workflow }}-${{ inputs.branch }}-${{ inputs.version }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + NOTIFICATION_PREFIX: "[Build changelog]" + +jobs: + calculated-inputs: + name: "Calculated inputs" + runs-on: ubuntu-latest + outputs: + changelog-path: ${{ steps.audit-changelog.outputs.path }} + changelog-exists: ${{ steps.audit-changelog.outputs.exists }} + steps: + - name: "Check out `${{ inputs.branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup `changie`" + uses: ./.github/actions/setup-changie + + - name: "Audit changelog" + id: audit-changelog + uses: ./.github/actions/audit-changelog + with: + version: ${{ inputs.version }} + + - name: "[DEBUG] Inputs" + run: | + echo branch : ${{ inputs.branch }} + echo version : ${{ inputs.version }} + echo changelog-path : ${{ steps.audit-changelog.outputs.path }} + echo changelog-exists : ${{ steps.audit-changelog.outputs.exists }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + + generate-changelog: + name: "Generate changelog" + if: ${{ !needs.calculated-inputs.outputs.changelog-exists }} + runs-on: ubuntu-latest + needs: [audit-changelog] + steps: + - name: "Check out `${{ inputs.branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup `hatch`" + uses: ./.github/actions/setup-hatch + + - name: "Parse `${{ inputs.version }}`" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ inputs.version }} + + - name: "Get Core team membership" + id: core-membership + uses: ./.github/actions/audit-github-team + with: + team: "core-group" + env: + GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} + + - name: "Set Core team membership for `changie` contributors exclusion" + id: set-team-membership + run: echo "CHANGIE_CORE_TEAM=${{ steps.core_membership.outputs.members }}" >> $GITHUB_ENV + + - name: "Generate changelog" + run: | + # this is a pre-release + if [[ ${{ steps.semver.outputs.is_prerelease }} -eq 1 ]] + then + changie batch ${{ steps.semver.outputs.base_version }} --move-dir '${{ steps.semver.outputs.base_version }}' --prerelease ${{ steps.semver.outputs.prerelease }} + + # this is a final release with associated pre-releases + elif [[ -d ".changes/${{ steps.semver.outputs.base_version }}" ]] + then + changie batch ${{ steps.semver.outputs.base_version }} --include '${{ steps.semver.outputs.base_version }}' --remove-prereleases + + # this is a final release with no associated pre-releases + else + changie batch ${{ needs.audit-changelog.outputs.base_version }} + fi + + changie merge + + - name: "Remove trailing whitespace and extra newlines" + continue-on-error: true + run: hatch run lint:pre-commit --files dbt/adapters/__about__.py CHANGELOG.md .changes/* + + - name: "Check changelog generated successfully" + run: | + title="Generate changelog" + if [[ -f ${{ needs.calculated-inputs.outputs.changelog-path }} ]] + then + message="Changelog file created successfully" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + else + message="Changelog failed to generate" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + - name: "Commit and push changes" + uses: ./.github/actions/github-commit + with: + message: "generate changelog for ${{ inputs.version }}" + + - name: "[INFO] Generated changelog" + run: | + title="Generated changelog" + message="Generated changelog for version ${{ inputs.version }}: ${{ needs.calculated-inputs.outputs.changelog-path }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + skip-changelog: + name: "Skip changelog generation" + if: ${{ needs.calculated-inputs.outputs.changelog-exists }} + runs-on: ubuntu-latest + needs: [audit-changelog] + steps: + - name: "[INFO] Skipped changelog generation" + run: | + title="Skipped changelog" + message="The changelog already exists for version ${{ inputs.version }}: ${{ needs.calculated-inputs.outputs.changelog-path }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/build-version-bump.yml b/.github/workflows/build-version-bump.yml new file mode 100644 index 00000000..37af14ba --- /dev/null +++ b/.github/workflows/build-version-bump.yml @@ -0,0 +1,127 @@ +name: "Version bump" +run-name: "Bump branch `${{ inputs.branch }}` to `${{ inputs.version }}`" + +on: + workflow_call: + inputs: + branch: + description: "The branch to bump" + type: string + default: "main" + version: + description: "The version to bump to" + type: string + required: true + working-dir: + description: "The working directory, useful for mono-repos" + type: string + default: "./" + outputs: + current-version: ${{ jobs.bump-version.outputs.version }} + bumped: ${{ jobs.bump-version.outputs.bumped }} + workflow_dispatch: + inputs: + branch: + description: "The branch to bump" + type: string + default: "main" + version: + description: "The version to bump to" + required: true + working-dir: + description: "The working directory, useful for mono-repos" + default: "./" + +permissions: write-all + +# only bump a branch/working-dir (package) to one version at a time +concurrency: + group: ${{ github.workflow }}-${{ inputs.branch }}-${{ inputs.working-dir }} + cancel-in-progress: true + +env: + NOTIFICATION_PREFIX: "[Version bump]" + +jobs: + calculated-inputs: + name: "Calculated inputs" + runs-on: ubuntu-latest + outputs: + current-version: ${{ steps.current-version.outputs.version }} + needs-version-bump: ${{ steps.version-bump.outputs.needs-version-bump }} + steps: + - name: "Check out `${{ inputs.branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup `hatch`" + uses: ./.github/actions/setup-environment + + - name: "Set: current-version" + id: current-version + shell: bash + run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT + working-directory: ${{ inputs.working-dir }} + + - name: "Set: need-to-bump" + id: version-bump + shell: bash + run: | + if [[ ${{ steps.current-version.outputs.version }} == "${{ inputs.version }} ]] + then + echo "needs-version-bump=false" >> $GITHUB_OUTPUT + else + echo "needs-version-bump=true" >> $GITHUB_OUTPUT + fi + + - name: "[DEBUG] Parameters" + run: | + echo branch : ${{ inputs.branch }} + echo version : ${{ inputs.version }} + echo working-dir : ${{ inputs.working-dir }} + echo current-version : ${{ steps.current-version.outputs.version }} + echo need-to-bump : ${{ steps.version-bump.outputs.needs-version-bump }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + + bump-version: + name: "Bump version" + if: ${{ fromJSON(needs.calculated-inputs.outputs.needs-version-bump) }} + needs: [calculated-inputs] + runs-on: ubuntu-latest + steps: + - name: "Check out `${{ inputs.branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup `hatch`" + uses: ./.github/actions/setup-environment + + - name: "Bump version to `${{ inputs.version }}`" + shell: bash + run: hatch version ${{ inputs.version }} + working-directory: ${{ inputs.working-dir }} + + - name: "Commit and push changes" + uses: ./.github/actions/github-commit + with: + message: "bump version to ${{ inputs.version }}" + + - name: "[INFO] Bumped version" + run: | + title="Bumped version" + message="Bumped version from ${{ needs.calculated-inputs.outputs.current-version }} to ${{ inputs.version }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + skip-version-bump: + name: "Skip version bump" + if: ${{ !fromJSON(needs.calculated-inputs.outputs.needs-version-bump) }} + needs: [calculated-inputs] + runs-on: ubuntu-latest + steps: + - name: "[INFO] Skipped version bump" + run: | + title="Skipped version bump" + message="${{ inputs.branch }} is already at version ${{ inputs.version }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 33d94ff4..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,85 +0,0 @@ -# **what?** -# Verifies python build on all code commited to the repository. This workflow -# should not require any secrets since it runs for PRs from forked repos. By -# default, secrets are not passed to workflows running from a forked repos. - -# **why?** -# Ensure code for dbt meets a certain quality standard. - -# **when?** -# This will run for all PRs, when code is pushed to main, and when manually triggered. - -name: "Build" - -on: - push: - branches: - - "main" - pull_request: - merge_group: - types: [checks_requested] - workflow_dispatch: - workflow_call: - inputs: - changelog_path: - description: "Path to changelog file" - required: true - type: string - -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - build: - name: Build, Test and publish to PyPi - runs-on: ubuntu-latest - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: Build `dbt-adapters` - if: ${{ inputs.package == 'dbt-adapters' }} - uses: ./.github/actions/build-hatch - - - name: Build `dbt-tests-adapter` - if: ${{ inputs.package == 'dbt-tests-adapter' }} - uses: ./.github/actions/build-hatch - with: - working-dir: "./dbt-tests-adapter/" - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: Build `dbt-adapters` - if: ${{ inputs.package == 'dbt-adapters' }} - uses: ./.github/actions/build-hatch - - - name: Build `dbt-tests-adapter` - if: ${{ inputs.package == 'dbt-tests-adapter' }} - uses: ./.github/actions/build-hatch - with: - working-dir: "./dbt-tests-adapter/" - - # this step is only needed for the release process - - name: "Upload Build Artifact" - if: ${{ github.event_name == 'workflow_call' }} - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.version.outputs.version_number }} - path: | - ${{ inputs.changelog_path }} - ./dist/ - retention-days: 3 diff --git a/.github/workflows/changelog-existence.yml b/.github/workflows/changelog-existence.yml deleted file mode 100644 index d778f565..00000000 --- a/.github/workflows/changelog-existence.yml +++ /dev/null @@ -1,40 +0,0 @@ -# **what?** -# Checks that a file has been committed under the /.changes directory -# as a new CHANGELOG entry. Cannot check for a specific filename as -# it is dynamically generated by change type and timestamp. -# This workflow runs on pull_request_target because it requires -# secrets to post comments. - -# **why?** -# Ensure code change gets reflected in the CHANGELOG. - -# **when?** -# This will run for all PRs going into main. It will -# run when they are opened, reopened, when any label is added or removed -# and when new code is pushed to the branch. The action will get -# skipped if the 'Skip Changelog' label is present is any of the labels. - -name: Check Changelog Entry - -on: - pull_request_target: - types: [opened, reopened, labeled, unlabeled, synchronize] - paths-ignore: ['.changes/**', '.github/**', 'tests/**', 'third-party-stubs/**', '**.md', '**.yml'] - - workflow_dispatch: - -defaults: - run: - shell: bash - -permissions: - contents: read - pull-requests: write - -jobs: - changelog: - uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main - with: - changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry).' - skip_label: 'Skip Changelog' - secrets: inherit diff --git a/.github/workflows/check-changelog-entry.yml b/.github/workflows/check-changelog-entry.yml new file mode 100644 index 00000000..f7196bcb --- /dev/null +++ b/.github/workflows/check-changelog-entry.yml @@ -0,0 +1,35 @@ +name: "Changelog entry" + +on: + # run on pull_request_target because this workflow requires secrets to post comments + pull_request_target: + branches: + - "main" + - "*.latest" + # this is a cheap check, any change should trigger it + types: + - opened + - reopened + - labeled + - unlabeled + - synchronize + # we are only concerned about functional code changes, i.e. code changes that will appear in production + paths-ignore: + - ".changes/**" + - ".github/**" + - "tests/**" + - "third-party-stubs/**" + - "**.md" + - "**.yml" + +permissions: + contents: read + pull-requests: write + +jobs: + changelog: + uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main + with: + changelog_comment: "Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry)." + skip_label: "Skip Changelog" + secrets: inherit # this is only acceptable because we own the action we're calling diff --git a/.github/workflows/check-code-quality.yml b/.github/workflows/check-code-quality.yml new file mode 100644 index 00000000..80c9eaa3 --- /dev/null +++ b/.github/workflows/check-code-quality.yml @@ -0,0 +1,60 @@ +name: "Code quality" + +on: + pull_request: + branches: + - "main" + - "*.latest" + workflow_call: + inputs: + branch: + description: "The branch to run code quality on" + type: string + default: "main" + workflow_dispatch: + inputs: + branch: + description: "The branch to run code quality on" + type: string + default: "main" + +permissions: read-all + +defaults: + run: + shell: bash + +# cancel the previous code quality run if a new one starts with the same parameters +concurrency: + group: ${{ github.workflow }}-${{ contains(github.event_name, 'workflow') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref }} + cancel-in-progress: true + +env: + NOTIFICATION_PREFIX: "[Code quality]" + +jobs: + code-quality: + name: "Code quality" + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo branch : ${{ inputs.branch }} + echo lint-command : ${{ inputs.lint-command }} + echo typecheck-command : ${{ inputs.typecheck-command }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + + - name: "Check out `${{ inputs.branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ contains(github.event_name, 'workflow') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref }} + persist-credentials: false + + - name: "Setup `hatch`" + uses: ./.github/actions/setup-hatch + + - name: "Run linters" + run: hatch run lint:all + + - name: "Run typechecks" + run: hatch run typecheck:all diff --git a/.github/workflows/check-unit-tests.yml b/.github/workflows/check-unit-tests.yml new file mode 100644 index 00000000..71db48a3 --- /dev/null +++ b/.github/workflows/check-unit-tests.yml @@ -0,0 +1,59 @@ +name: "Unit tests" + +on: + pull_request: + branches: + - "main" + - "*.latest" + workflow_call: + inputs: + branch: + description: "The branch to run unit tests on" + type: string + default: "main" + workflow_dispatch: + inputs: + branch: + description: "The branch to run unit tests on" + type: string + default: "main" + +permissions: read-all + +# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'workflow_') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref }} + cancel-in-progress: true + +env: + NOTIFICATION_PREFIX: "[Unit tests]" + +jobs: + unit: + name: "Unit tests" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + os: [ubuntu-latest] + steps: + - name: "[DEBUG] Inputs" + run: | + echo branch : ${{ inputs.branch }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + + - name: "Check out `${{ contains(github.event_name, 'workflow_') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref }}`" + uses: actions/checkout@v4 + with: + ref: ${{ contains(github.event_name, 'workflow_') && inputs.branch || contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + persist-credentials: false + + - name: "Setup `hatch`" + uses: ./.github/actions/setup-hatch + with: + python-version: ${{ matrix.python-version }} + + - name: "Run unit tests" + run: hatch run unit-tests:all + shell: bash diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml deleted file mode 100644 index 4f5b392e..00000000 --- a/.github/workflows/code-quality.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Code Quality - -on: - push: - branches: - - "main" - - "*.latest" - pull_request: - workflow_dispatch: - -permissions: read-all - -defaults: - run: - shell: bash - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -jobs: - lint: - name: Code Quality - runs-on: ubuntu-latest - - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: Run linters - run: hatch run lint:all - - - name: Run typechecks - run: hatch run typecheck:all diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml deleted file mode 100644 index edd25172..00000000 --- a/.github/workflows/github-release.yml +++ /dev/null @@ -1,259 +0,0 @@ -# **what?** -# Create a new release on GitHub and include any artifacts in the `/dist` directory of the GitHub artifacts store. -# -# Inputs: -# sha: The commit to attach to this release -# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) -# changelog_path: Path to the changelog file for release notes -# test_run: Test run (Publish release as draft) -# -# **why?** -# Reusable and consistent GitHub release process. -# -# **when?** -# Call after a successful build. Build artifacts should be ready to release and live in a dist/ directory. -# -# This workflow expects the artifacts to already be built and living in the artifact store of the workflow. -# -# Validation Checks -# -# 1. If no release already exists for this commit and version, create the tag and release it to GitHub. -# 2. If a release already exists for this commit, skip creating the release but finish with a success. -# 3. If a release exists for this commit under a different tag, fail. -# 4. If the commit is already associated with a different release, fail. - -name: GitHub Release - -on: - workflow_call: - inputs: - sha: - description: The commit to attach to this release - required: true - type: string - version_number: - description: The release version number (i.e. 1.0.0b1) - required: true - type: string - changelog_path: - description: Path to the changelog file for release notes - required: true - type: string - test_run: - description: Test run (Publish release as draft) - required: true - type: boolean - archive_name: - description: artifact name to download - required: true - type: string - outputs: - tag: - description: The path to the changelog for this version - value: ${{ jobs.check-release-exists.outputs.tag }} - -permissions: - contents: write - -env: - REPO_LINK: ${{ github.server_url }}/${{ github.repository }} - NOTIFICATION_PREFIX: "[GitHub Release]" - -jobs: - log-inputs: - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Print Variables" - run: | - echo The last commit sha in the release: ${{ inputs.sha }} - echo The release version number: ${{ inputs.version_number }} - echo Expected Changelog path: ${{ inputs.changelog_path }} - echo Test run: ${{ inputs.test_run }} - echo Repo link: ${{ env.REPO_LINK }} - echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} - - check-release-exists: - runs-on: ubuntu-latest - outputs: - exists: ${{ steps.release_check.outputs.exists }} - draft_exists: ${{ steps.release_check.outputs.draft_exists }} - tag: ${{ steps.set_tag.outputs.tag }} - - steps: - - name: "Generate Release Tag" - id: set_tag - run: echo "tag=v${{ inputs.version_number }}" >> $GITHUB_OUTPUT - - # When the GitHub CLI doesn't find a release for the given tag, it will exit 1 with a - # message of "release not found". In our case, it's not an actual error, just a - # confirmation that the release does not already exists so we can go ahead and create it. - # The `|| true` makes it so the step does not exit with a non-zero exit code - # Also check if the release already exists is draft state. If it does, and we are not - # testing then we can publish that draft as is. If it's in draft and we are testing, skip the - # release. - - name: "Check If Release Exists For Tag ${{ steps.set_tag.outputs.tag }}" - id: release_check - run: | - output=$((gh release view ${{ steps.set_tag.outputs.tag }} --json isDraft,targetCommitish --repo ${{ env.REPO_LINK }}) 2>&1) || true - if [[ "$output" == "release not found" ]] - then - title="Release for tag ${{ steps.set_tag.outputs.tag }} does not exist." - message="Check passed." - echo "exists=false" >> $GITHUB_OUTPUT - echo "draft_exists=false" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 0 - fi - commit=$(jq -r '.targetCommitish' <<< "$output") - if [[ $commit != ${{ inputs.sha }} ]] - then - title="Release for tag ${{ steps.set_tag.outputs.tag }} already exists for commit $commit!" - message="Cannot create a new release for commit ${{ inputs.sha }}. Exiting." - echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 1 - fi - isDraft=$(jq -r '.isDraft' <<< "$output") - if [[ $isDraft == true ]] && [[ ${{ inputs.test_run }} == false ]] - then - title="Release tag ${{ steps.set_tag.outputs.tag }} already associated with the draft release." - message="Release workflow will publish the associated release." - echo "exists=false" >> $GITHUB_OUTPUT - echo "draft_exists=true" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 0 - fi - title="Release for tag ${{ steps.set_tag.outputs.tag }} already exists." - message="Skip GitHub Release Publishing." - echo "exists=true" >> $GITHUB_OUTPUT - echo "draft_exists=false" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ env.REPO_LINK }} - - - name: "[DEBUG] Log Job Outputs" - run: | - echo exists: ${{ steps.release_check.outputs.exists }} - echo draft_exists: ${{ steps.release_check.outputs.draft_exists }} - echo tag: ${{ steps.set_tag.outputs.tag }} - - skip-github-release: - runs-on: ubuntu-latest - needs: [check-release-exists] - if: needs.check-release-exists.outputs.exists == 'true' - - steps: - - name: "Tag Exists, Skip GitHub Release Job" - run: | - echo title="A tag already exists for ${{ needs.check-release-exists.outputs.tag }} and commit." - echo message="Skipping GitHub release." - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - audit-release-different-commit: - runs-on: ubuntu-latest - needs: [check-release-exists] - if: needs.check-release-exists.outputs.exists == 'false' - - steps: - - name: "Check If Release Already Exists For Commit" - uses: cardinalby/git-get-release-action@1.2.4 - id: check_release_commit - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - commitSha: ${{ inputs.sha }} - doNotFailIfNotFound: true # returns blank outputs when not found instead of error - searchLimit: 15 # Since we only care about recent releases, speed up the process - - - name: "[DEBUG] Print Release Details" - run: | - echo steps.check_release_commit.outputs.id: ${{ steps.check_release_commit.outputs.id }} - echo steps.check_release_commit.outputs.tag_name: ${{ steps.check_release_commit.outputs.tag_name }} - echo steps.check_release_commit.outputs.target_commitish: ${{ steps.check_release_commit.outputs.target_commitish }} - echo steps.check_release_commit.outputs.prerelease: ${{ steps.check_release_commit.outputs.prerelease }} - - # Since we already know a release for this tag does not exist, if we find anything it's for the wrong tag, exit - - name: "Check If The Tag Matches The Version Number" - if: steps.check_release_commit.outputs.id != '' - run: | - title="Tag ${{ steps.check_release_commit.outputs.tag_name }} already exists for this commit!" - message="Cannot create a new tag for ${{ needs.check-release-exists.outputs.tag }} for the same commit" - echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 1 - - publish-draft-release: - runs-on: ubuntu-latest - needs: [check-release-exists, audit-release-different-commit] - if: >- - needs.check-release-exists.outputs.draft_exists == 'true' && - inputs.test_run == false - - steps: - - name: "Publish Draft Release - ${{ needs.check-release-exists.outputs.tag }}" - run: | - gh release edit $TAG --draft=false --repo ${{ env.REPO_LINK }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ needs.check-release-exists.outputs.tag }} - - create-github-release: - runs-on: ubuntu-latest - needs: [check-release-exists, audit-release-different-commit] - if: needs.check-release-exists.outputs.draft_exists == 'false' - - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.sha }} - - - name: "Download Artifact ${{ inputs.archive_name }}" - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.archive_name }} - path: . - - - name: "[DEBUG] Display Structure Of Expected Files" - run: | - ls -R .changes - ls -l dist - - - name: "Set Release Type" - id: release_type - run: | - if ${{ contains(inputs.version_number, 'rc') || contains(inputs.version_number, 'b') }} - then - echo Release will be set as pre-release - echo "prerelease=--prerelease" >> $GITHUB_OUTPUT - else - echo This is not a prerelease - fi - - - name: "Set As Draft Release" - id: draft - run: | - if [[ ${{ inputs.test_run }} == true ]] - then - echo Release will be published as draft - echo "draft=--draft" >> $GITHUB_OUTPUT - else - echo This is not a draft release - fi - - - name: "GitHub Release Workflow Annotation" - run: | - title="Release ${{ needs.check-release-exists.outputs.tag }}" - message="Configuration: ${{ steps.release_type.outputs.prerelease }} ${{ steps.draft.outputs.draft }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - - name: "Create New GitHub Release - ${{ needs.check-release-exists.outputs.tag }}" - run: | - gh release create $TAG ./dist/* --title "$TITLE" --notes-file $RELEASE_NOTES --target $COMMIT $PRERELEASE $DRAFT --repo ${{ env.REPO_LINK }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ needs.check-release-exists.outputs.tag }} - TITLE: ${{ github.event.repository.name }} ${{ needs.check-release-exists.outputs.tag }} - RELEASE_NOTES: ${{ inputs.changelog_path }} - COMMIT: ${{ inputs.sha }} - PRERELEASE: ${{ steps.release_type.outputs.prerelease }} - DRAFT: ${{ steps.draft.outputs.draft }} \ No newline at end of file diff --git a/.github/workflows/resubmit-for-triage.yml b/.github/workflows/issues-resubmit-for-triage.yml similarity index 83% rename from .github/workflows/resubmit-for-triage.yml rename to .github/workflows/issues-resubmit-for-triage.yml index 385ef820..2b3237a5 100644 --- a/.github/workflows/resubmit-for-triage.yml +++ b/.github/workflows/issues-resubmit-for-triage.yml @@ -9,23 +9,18 @@ # **when?** # This will run when a comment is added to an issue and that issue has an `awaiting_response` label. - -name: Resubmit for Triage +name: "Resubmit for triage" on: issue_comment -defaults: - run: - shell: bash - permissions: issues: write jobs: - triage_label: + triage-label: if: contains(github.event.issue.labels.*.name, 'awaiting_response') uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main with: add_label: "triage" remove_label: "awaiting_response" - secrets: inherit # this is only acceptable because we own the action we're calling + secrets: inherit diff --git a/.github/workflows/issues-stale.yml b/.github/workflows/issues-stale.yml new file mode 100644 index 00000000..7360c9e1 --- /dev/null +++ b/.github/workflows/issues-stale.yml @@ -0,0 +1,24 @@ +# **what?** +# 1. Label issues and PRs which breach the threshold in our stale policy. +# This will notify participants that they will be closed soon. +# Any activity on the issue will remove the label. +# 2. Close issues or PRs which were previously marked as stale. +# +# **why?** +# Avoid triaging and working issues which are no longer relevant + +# **when?** +# Once a day +name: "Stale" + +on: + schedule: + - cron: "30 1 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main diff --git a/.github/workflows/publish-github.yml b/.github/workflows/publish-github.yml new file mode 100644 index 00000000..738eaaef --- /dev/null +++ b/.github/workflows/publish-github.yml @@ -0,0 +1,193 @@ +# **what?** +# 1. If a release exists for this commit under a different tag, fail. +# 2. If the commit is already associated with a different release, fail. +# 3. If no release/tag exists for this commit and version, create the tag and release it to GitHub as a draft. +# 4. If a release already exists for this commit, skip creating the release as a draft. +# 5. If this is deployed to prod, mark the draft release (created in this run or previously) as a full release. +# +# **when?** +# 1. Call after a successful build. Build artifacts should be ready to release. +# They need to either live in a dist/ directory locally, or an archive name needs to be provided. +# 2. Call to publish a draft release to become a full release. No archive, artifacts, or changelog path are needed in this case. +name: "Publish to GitHub" +run-name: "Publish `${{ version }}` to GitHub in ${{ deploy-environment }}" + +on: + workflow_call: + inputs: + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + sha: + description: "The SHA for the commit being released" + type: string + required: true + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local or publishing draft to full" + type: string + default: "" + changelog-path: + description: "Path to the release notes, leave blank if publishing draft to full" + type: string + default: "" + deploy-environment: + description: "Choose where to publish" + type: string + default: "prod" + workflow_dispatch: + inputs: + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + sha: + description: "The SHA for the commit being released" + type: string + required: true + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local or publishing draft to full" + type: string + default: "" + changelog-path: + description: "Path to the release notes, leave blank if publishing draft to full" + type: string + default: "" + deploy-environment: + description: "Choose where to publish" + type: environment + +permissions: + contents: write + +# will cancel previous workflows triggered by the same event and for the same package and environment +concurrency: + group: ${{ github.workflow }}-${{ inputs.archive-name }}-${{ inputs.version }}-${{ inputs.deploy-environment }} + cancel-in-progress: true + +env: + NOTIFICATION_PREFIX: "[Publish - GitHub]" + +jobs: + calculated-inputs: + name: "Calculated inputs" + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.tag.outputs.tag }} + tag-sha: ${{ steps.tag-metadata.outputs.sha }} + tag-exists: ${{ steps.tag-metadata.outputs.exists }} + tag-is-draft: ${{ steps.tag-metadata.outputs.is-draft }} + commit-tag: ${{ steps.commit-metadata.outputs.tag }} + steps: + - name: "Check out `main`" + uses: actions/checkout@v4 + + - name: "Set: tag" + id: tag + shell: bash + run: echo "tag=v${{ inputs.version }}" >> $GITHUB_OUTPUT + + - name: "Get tag metadata" + id: tag-metadata + uses: ./.github/actions/audit-github-tag + with: + tag: ${{ steps.tag.output.tag }} + repo-url: ${{ env.REPO_LINK }} + + - name: "Get commit metadata" + id: commit-metadata + if: ${{ fromJSON(steps.tag-metadata.outputs.exists) == false }} + uses: ./.github/actions/audit-github-commit + with: + tag: ${{ inputs.sha }} + + - name: "[DEBUG] Inputs" + run: | + echo archive-name : ${{ inputs.archive-name }} + echo version : ${{ inputs.version }} + echo sha : ${{ inputs.sha }} + echo changelog-path : ${{ inputs.changelog-path }} + echo deploy-environment : ${{ inputs.deploy-environment }} + echo tag : ${{ steps.tag.outputs.tag }} + echo tag-sha : ${{ steps.tag-metadata.outputs.sha }} + echo tag-exists : ${{ steps.tag-metadata.outputs.exists }} + echo tag-is-draft : ${{ steps.tag-metadata.outputs.is-draft }} + echo commit-tag : ${{ steps.commit-metadata.outputs.tag }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + + validation: + name: "Validate input scenarios" + needs: [calculated-inputs] + runs-on: ubuntu-latest + steps: + - name: "[INFO] Exit gracefully if the tag exists and it's a full release" + if: ${{ fromJSON(needs.calculated-inputs.outputs.tag-exists) && !fromJSON(needs.calculated-inputs.outputs.tag-is-draft) }} + run: | + title="Full release already exists" + message="`${{ needs.calculated-inputs.outputs.tag }}` already exists as a full release and is associated with the commit ${{ needs.calculated-inputs.outputs.tag-sha }}." + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 0 + + - name: "[ERROR] Exit with error if the tag exists and is associated with a different commit than the one provided" + if: ${{ fromJSON(needs.calculated-inputs.outputs.tag-exists) }} && ${{ needs.calculated-inputs.outputs.tag-sha }} != ${{ inputs.sha }} + shell: bash + run: | + title="Tag `v${{ inputs.version }}` already exists with a different commit!" + message="Existing commit: {{ steps.tag-metadata.outputs.sha }} New commit: ${{ inputs.sha }}. Exiting." + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + + - name: "[ERROR] Exit with error if the commit is associated with a different tag than the version that was provided" + if: ${{ needs.calculated-inputs.outputs.commit-tag != '' }} && ${{ needs.calculated-inputs.outputs.commit-tag != needs.calculated-inputs.outputs.tag }} + run: | + title="Commit ${{ inputs.sha }} is already associated with a different tag!" + message="Existing tag: ${{ needs.calculated-inputs.outputs.commit-tag }} New tag: ${{ needs.calculated-inputs.outputs.tag }}" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + + new-tag: + name: "Publish draft release to GitHub" + needs: [calculated-inputs, validation] + if: ${{ !fromJSON(needs.calculated-inputs.outputs.tag-exists) }} + runs-on: ubuntu-latest + steps: + - name: "Check out `main`" + uses: actions/checkout@v4 + + - name: "Publish to GitHub as a draft release" + uses: ./.github/actions/publish-github-draft + with: + repo-url: ${{ github.server_url }}/${{ github.repository }} + tag: ${{ needs.calculated-inputs.outputs.tag }} + sha: ${{ inputs.sha }} + archive-name: ${{ inputs.archive-name }} + changelog-path: ${{ inputs.changelog-path }} + + - name: "[INFO] Published to GitHub as a draft release" + run: | + title="Published to GitHub as a draft release" + message="${{ needs.calculated-inputs.outputs.tag }} was created as a draft release with the commit ${{ inputs.sha }}." + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + existing-tag: + name: "Publish GitHub draft as full release" + # require publish-draft to cover the scenario where a production release is being created without an existing draft release + needs: [calculated-inputs, validation, publish-draft] + # allow for publish-draft to be skipped, e.g. if this is a production release where the draft already existed + if: ${{ !failure() && inputs.deploy-environment == "prod" }} + runs-on: ubuntu-latest + steps: + - name: "Check out `main`" + uses: actions/checkout@v4 + + - name: "Publish draft release as a full release" + uses: ./.github/actions/publish-github-full + with: + repo-url: ${{ github.server_url }}/${{ github.repository }} + tag: ${{ needs.calculated-inputs.outputs.tag }} + + - name: "[INFO] Published draft release as a full release" + run: | + title="Published draft release as a full release" + message="`${{ needs.calculated-inputs.outputs.tag }}` was published as a full release." + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml new file mode 100644 index 00000000..a34b6c29 --- /dev/null +++ b/.github/workflows/publish-pypi.yml @@ -0,0 +1,76 @@ +# **what?** +# Releases artifacts to PyPI. +# +# **when?** +# Call after a successful build. Build artifacts should be ready to release.defaults: +# They need to either live in a dist/ directory locally, or an archive name needs to be provided. +name: "Publish to PyPI" +run-name: "Publish `${{ version }}` to PyPI in ${{ deploy-environment }}" + +on: + workflow_dispatch: + inputs: + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local" + default: "" + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + deploy-environment: + description: "Choose where to publish" + type: environment + workflow_call: + inputs: + archive-name: + description: "Name of the archive containing the artifacts, leave blank if local" + type: string + default: "" + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + deploy-environment: + description: "Choose where to publish" + type: string + default: "prod" + +permissions: + contents: write + +# will cancel previous workflows triggered by the same event and for the same package and environment +concurrency: + group: ${{ github.workflow }}-${{ inputs.archive-name }}-${{ inputs.version }}-${{ inputs.deploy-environment }} + cancel-in-progress: true + +env: + NOTIFICATION_PREFIX: "[Publish - PyPI]" + +jobs: + debug-inputs: + name: "[DEBUG] Inputs" + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo archive-name : ${{ inputs.archive-name }} + echo version : ${{ inputs.version }} + echo deploy-environment : ${{ inputs.deploy-environment }} + echo PYPI_PROJECT_URL : ${{ vars.PYPI_PROJECT_URL }} + echo PYPI_REPOSITORY_URL : ${{ vars.PYPI_REPOSITORY_URL }} + + pypi-release: + name: "Publish to PyPI" + runs-on: ubuntu-latest + environment: + name: ${{ inputs.deploy-environment }} + url: ${{ vars.PYPI_PROJECT_URL }} + steps: + - name: "Check out `main`" + uses: actions/checkout@v4 + + - name: "Publish to PyPI" + uses: .github/actions/publish-pypi + with: + repository-url: ${{ vars.PYPI_REPOSITORY_URL }} + archive-name: ${{ inputs.archive-name }} diff --git a/.github/workflows/release-branch-create.yml b/.github/workflows/release-branch-create.yml new file mode 100644 index 00000000..5fda905e --- /dev/null +++ b/.github/workflows/release-branch-create.yml @@ -0,0 +1,65 @@ +name: "Create release branch" +run-name: "Create a release branch for `${{ inputs.version }}` using `${{ inputs.branch }}`" + +on: + workflow_call: + inputs: + branch: + description: "The base branch to use when creating the release branch" + type: string + default: "main" + version: + description: "The version to use in the release branch name" + type: string + required: true + outputs: + branch: + description: "The name of the created release branch" + value: ${{ jobs.create-release-branch.outputs.branch }} + workflow_dispatch: + inputs: + branch: + description: "The base branch to use when creating the release branch" + default: "main" + version: + description: "The version to use in the release branch name" + required: true + +permissions: + contents: write + id-token: write + +defaults: + run: + shell: bash + +env: + NOTIFICATION_PREFIX: "[Create release branch]" + +jobs: + debug-inputs: + name: "[DEBUG] Inputs" + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo branch : ${{ inputs.branch }} + echo version : ${{ inputs.version }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + + create-release-branch: + name: "Create a release branch from `${{ inputs.branch }}`" + runs-on: ubuntu-latest + outputs: + branch: ${{ steps.create-branch.outputs.branch }} + steps: + - name: "Checkout `${{ inputs.branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Create release branch" + id: create-branch + uses: ./.github/actions/create-temp-branch + with: + branch-stub: "release/${{ inputs.version }}/" diff --git a/.github/workflows/release-branch-merge.yml b/.github/workflows/release-branch-merge.yml new file mode 100644 index 00000000..7d8f362d --- /dev/null +++ b/.github/workflows/release-branch-merge.yml @@ -0,0 +1,74 @@ +name: "Merge release branch" +run-name: "Merge `${{ release-branch }}` into `${{ inputs.branch }}`" + +on: + workflow_call: + inputs: + branch: + description: "The base branch receiving the changes" + type: string + required: true + release-branch: + description: "The branch containing the release changes" + type: string + required: true + deploy-environment: + description: "Choose where to publish" + type: string + required: true + outputs: + sha: + description: "The sha associated with this merge" + value: ${{ jobs.merge.outputs.sha }} + +# this is the permission that allows creating a new release to both GitHub and PyPI +permissions: + contents: write + id-token: write + +# will cancel previous workflows triggered by the same event and for the same package and environment +concurrency: + group: ${{ github.workflow }}-${{ inputs.branch }}-${{ inputs.release-branch }} + cancel-in-progress: true + +env: + NOTIFICATION_PREFIX: "[Merge release branch]" + +jobs: + debug-inputs: + name: "[DEBUG] Inputs" + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Inputs" + run: | + echo branch : ${{ inputs.branch }} + echo release-branch : ${{ inputs.release-branch }} + echo NOTIFICATION_PREFIX : ${{ env.NOTIFICATION_PREFIX }} + + merge: + name: "Merge `${{ inputs.release-branch }}` into `${{ inputs.branch }}`" + runs-on: ubuntu-latest + needs: [temp-branch] + outputs: + sha: ${{ steps.merge.outputs.sha }} || ${{ steps.test-merge.outputs.sha }} + steps: + - name: "Checkout `${{ inputs.release-branch }}`" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.release-branch }} + + - name: "Merge `${{ inputs.release-branch }}` into `${{ inputs.branch }}`" + if: ${{ inputs.deploy-environment == "prod" }} + id: merge + uses: ./.github/actions/github-merge + with: + source-branch: ${{ inputs.release-branch }} + target-branch: ${{ inputs.branch }} + message: "merge {source_ref} into {target_branch} for a release" + + - name: "Get HEAD sha for test release" + id: test-merge + shell: bash + run: | + git pull + echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1135adb8..01382506 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,70 +1,53 @@ -name: Release -run-name: Release ${{ inputs.package }}==${{ inputs.version_number }} to ${{ inputs.deploy-to }} +name: "Release" +run-name: "Release `${{ inputs.package }}==${{ inputs.version }}` to `${{ inputs.deploy-to }}`" on: - workflow_dispatch: + workflow_call: inputs: + branch: + description: "Choose what branch to release from" + type: string + default: "main" package: - type: choice - description: Choose what to publish - options: - - dbt-adapters - - dbt-tests-adapter - version_number: + description: "Choose what to release" + type: string + default: "dbt-adapters" + version: description: "The release version number (i.e. 1.0.0b1)" type: string required: true - deploy-to: - type: choice - description: Choose where to publish - options: - - prod - - test - default: prod - nightly_release: - description: "Nightly release to dev environment" - type: boolean - default: false - required: false - target_branch: - description: "The branch to release from" + deploy-environment: + description: "Choose where to publish" type: string - required: false - default: main - - workflow_call: + default: "prod" + workflow_dispatch: inputs: - package: + branch: + description: "Choose what branch to release from" type: string - description: Choose what to publish - required: true - version_number: + default: "main" + package: + description: "Choose what to release" + type: choice + options: + - "dbt-adapters" + - "dbt-tests-adapter" + version: description: "The release version number (i.e. 1.0.0b1)" type: string required: true - deploy-to: - type: string - default: prod - required: false - nightly_release: - description: "Nightly release to dev environment" - type: boolean - default: false - required: false - target_branch: - description: "The branch to release from" - type: string - required: false - default: main + deploy-environment: + description: "Choose where to publish" + type: environment -# this is the permission that allows creating a new release +# this is the permission that allows creating a new release to both GitHub and PyPI permissions: contents: write id-token: write -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise +# deploying the same version of a package to the same environment should override any previous deployment with those attributes concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}-${{ inputs.package }}-${{ inputs.deploy-to }} + group: ${{ github.workflow }}-${{ inputs.package }}-${{ inputs.version }}-${{ inputs.deploy-environment }} cancel-in-progress: true defaults: @@ -72,20 +55,20 @@ defaults: shell: bash jobs: - release-inputs: + calculated-inputs: name: "Release inputs" runs-on: ubuntu-latest outputs: - working-dir: ${{ steps.release-inputs.outputs.working-dir }} - run-unit-tests: ${{ steps.release-inputs.outputs.run-unit-tests }} - archive-name: ${{ steps.release-inputs.outputs.archive-name }} + archive-name: ${{ steps.computed-inputs.outputs.archive-name }} + working-dir: ${{ steps.computed-inputs.outputs.working-dir }} + run-unit-tests: ${{ steps.computed-inputs.outputs.run-unit-tests }} steps: - - name: "Inputs" - id: release-inputs + - name: "Computed inputs" + id: computed-inputs run: | + archive_name=${{ inputs.package }}-${{ inputs.version_number }}-${{ inputs.deploy-environment }} working_dir="./" run_unit_tests=true - archive_name=${{ inputs.package }}-${{ inputs.version_number }}-${{ inputs.deploy-to }} if test "${{ inputs.package }}" = "dbt-tests-adapter" then @@ -93,88 +76,106 @@ jobs: run_unit_tests=false fi + echo "archive-name=$archive_name" >> $GITHUB_OUTPUT echo "working-dir=$working_dir" >> $GITHUB_OUTPUT echo "run-unit-tests=$run_unit_tests" >> $GITHUB_OUTPUT - echo "archive-name=$archive_name" >> $GITHUB_OUTPUT - - name: "[DEBUG]" + - name: "[DEBUG] Inputs" run: | - echo package : ${{ inputs.package }} - echo working-dir : ${{ steps.release-inputs.outputs.working-dir }} - echo run-unit-tests : ${{ steps.release-inputs.outputs.run-unit-tests }} - echo archive-name : ${{ steps.release-inputs.outputs.archive-name }} - - bump-version-generate-changelog: - name: "Bump package version, Generate changelog" - uses: dbt-labs/dbt-adapters/.github/workflows/release_prep_hatch.yml@main - needs: [release-inputs] + echo branch : ${{ inputs.branch }} + echo package : ${{ inputs.package }} + echo version : ${{ inputs.version }} + echo deploy-environment : ${{ inputs.deploy-environment }} + echo archive-name : ${{ steps.release-inputs.outputs.archive-name }} + echo working-dir : ${{ steps.release-inputs.outputs.working-dir }} + echo run-unit-tests : ${{ steps.release-inputs.outputs.run-unit-tests }} + + code-quality: + name: "Run code quality" + uses: dbt-labs/dbt-adapters/.github/workflows/check-code-quality.yml@update-workflows + with: + branch: ${{ inputs.branch }} + + unit-tests: + name: "Run unit tests" + needs: [calculated-inputs] + if: ${{ fromJSON(needs.calculated-inputs.outputs.run-unit-tests) }} + uses: dbt-labs/dbt-adapters/.github/workflows/check-unit-tests.yml@update-workflows + with: + branch: ${{ inputs.branch }} + + release-branch: + name: "Create a release branch" + uses: dbt-labs/dbt-adapters/.github/workflows/release-branch-create.yml@update-workflows + with: + branch: ${{ inputs.branch }} + version: ${{ inputs.version }} + secrets: inherit + + version-bump: + name: "Bump the version" + uses: dbt-labs/dbt-adapters/.github/workflows/build-version-bump.yml@update-workflows + needs: [release-branch, calculated-inputs] with: - version_number: ${{ inputs.version_number }} - deploy_to: ${{ inputs.deploy-to }} - nightly_release: ${{ inputs.nightly_release }} - target_branch: ${{ inputs.target_branch }} + branch: ${{ needs.release-branch.outputs.branch }} + version: ${{ inputs.version }} working-dir: ${{ needs.release-inputs.outputs.working-dir }} - run-unit-tests: ${{ fromJSON(needs.release-inputs.outputs.run-unit-tests) }} secrets: inherit - log-outputs-bump-version-generate-changelog: - name: "[Log output] Bump package version, Generate changelog" + changelog: + name: "Generate a new changelog" + needs: [release-branch] + uses: dbt-labs/dbt-adapters/.github/workflows/build-changelog.yml@update-workflows + with: + branch: ${{ needs.release-branch.outputs.branch }} + version: ${{ inputs.version }} + secrets: inherit + + build-artifacts: + name: "Build the artifacts" if: ${{ !failure() && !cancelled() }} - needs: [release-inputs, bump-version-generate-changelog] - runs-on: ubuntu-latest - steps: - - name: Print variables - run: | - echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} + needs: [ + code-quality, + unit-tests, + release-branch, + version-bump, + changelog, + ] + uses: dbt-labs/dbt-adapters/.github/workflows/build-artifacts.yml@update-workflows + with: + branch: ${{ needs.release-branch.outputs.branch }} + package: ${{ inputs.package }} + version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-environment }} + + merge-changes: + name: "Merge the version bump and changelog updates" + needs: [build-artifacts, version-bump, changelog] + if: ${{ !failure() && !cancelled() && inputs.deploy-environment == "prod" && (needs.version-bump.outputs.bumped || needs.changelog.outputs.created) }} + uses: dbt-labs/dbt-adapters/.github/workflows/release-branch-merge.yml@update-workflows + with: + branch: ${{ inputs.branch }} + release-branch: ${{ needs.release-branch.outputs.branch }} + secrets: inherit - build-and-test: - name: "Build and Test" - needs: [release-inputs, bump-version-generate-changelog] - runs-on: ubuntu-latest - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - with: - ref: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - - - name: "Setup `hatch`" - uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - - - name: "Build ${{ inputs.package }}" - uses: dbt-labs/dbt-adapters/.github/actions/build-hatch@main - with: - working-dir: ${{ needs.release-inputs.outputs.working-dir }} - archive-name: ${{ needs.release-inputs.outputs.archive-name }} - - github-release: - name: "GitHub Release" - # ToDo: update GH release to handle adding dbt-tests-adapter and dbt-adapter assets to the same release + publish-github: + name: "Publish the artifacts to GitHub" if: ${{ !failure() && !cancelled() && inputs.package == 'dbt-adapters' }} - needs: [release-inputs, build-and-test, bump-version-generate-changelog] - uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main + needs: [build-artifacts, changelog, merge-changes] + uses: dbt-labs/dbt-adapters/.github/workflows/publish-github.yml@update-workflows with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - test_run: ${{ inputs.deploy-to == 'test' && true || false }} - archive_name: ${{ needs.release-inputs.outputs.archive-name }} - - pypi-release: - name: "Publish to PyPI" - runs-on: ubuntu-latest - needs: [release-inputs, build-and-test] - environment: - name: ${{ inputs.deploy-to }} - url: ${{ vars.PYPI_PROJECT_URL }} - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - - - name: "Publish to PyPI" - uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main - with: - repository-url: ${{ vars.PYPI_REPOSITORY_URL }} - archive-name: ${{ needs.release-inputs.outputs.archive-name }} + archive-name: ${{ needs.build-artifacts.outputs.archive-name }} + version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-environment }} + sha: ${{ needs.merge-changes.outputs.sha }} + changelog-path: ${{ needs.changelog.outputs.changelog-path }} + + publish-pypi: + name: "Publish the artifacts to PyPI" + if: ${{ !failure() && !cancelled() }} + needs: [build-artifacts, merge-changes] + uses: dbt-labs/dbt-adapters/.github/workflows/publish-pypi.yml@update-workflows + with: + archive-name: ${{ needs.build-artifacts.outputs.archive-name }} + version: ${{ inputs.version }} + deploy-to: ${{ inputs.deploy-environment }} diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml deleted file mode 100644 index 32a267e0..00000000 --- a/.github/workflows/release_prep_hatch.yml +++ /dev/null @@ -1,542 +0,0 @@ -# **what?** -# Perform the version bump, generate the changelog and run tests. -# -# Inputs: -# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) -# target_branch: The branch that we will release from -# env_setup_script_path: Path to the environment setup script -# deploy_to: If we are deploying to prod or test, if test then release from branch -# nightly_release: Identifier that this is nightly release -# -# Outputs: -# final_sha: The sha that will actually be released. This can differ from the -# input sha if adding a version bump and/or changelog -# changelog_path: Path to the changelog file (ex .changes/1.2.3-rc1.md) -# -# Branching strategy: -# - During execution workflow execution the temp branch will be generated. -# - For normal runs the temp branch will be removed once changes were merged to target branch; -# - For test runs we will keep temp branch and will use it for release; -# Naming strategy: -# - For normal runs: prep-release/${{ inputs.deploy_to}}/${{ inputs.version_number }}_$GITHUB_RUN_ID -# - For nightly releases: prep-release/nightly-release/${{ inputs.version_number }}_$GITHUB_RUN_ID -# -# **why?** -# Reusable and consistent GitHub release process. -# -# **when?** -# Call when ready to kick off a build and release -# -# Validation Checks -# -# 1. Bump the version if it has not been bumped -# 2. Generate the changelog (via changie) if there is no markdown file for this version -# - -name: Version Bump and Changelog Generation -run-name: Bump ${{ inputs.package }}==${{ inputs.version_number }} for release to ${{ inputs.deploy_to }} and generate changelog -on: - workflow_call: - inputs: - version_number: - required: true - type: string - deploy_to: - type: string - default: prod - required: false - nightly_release: - type: boolean - default: false - required: false - env_setup_script_path: - type: string - required: false - default: '' - run-unit-tests: - type: boolean - default: false - run-integration-tests: - type: boolean - default: false - target_branch: - description: "The branch to release from" - type: string - required: false - default: main - working-dir: - description: "The working directory to use for run statements" - type: string - default: "./" - outputs: - changelog_path: - description: The path to the changelog for this version - value: ${{ jobs.audit-changelog.outputs.changelog_path }} - final_sha: - description: The sha that will actually be released - value: ${{ jobs.determine-release-branch.outputs.final_sha }} - secrets: - FISHTOWN_BOT_PAT: - description: "Token to commit/merge changes into branches" - required: true - IT_TEAM_MEMBERSHIP: - description: "Token that can view org level teams" - required: true - -permissions: - contents: write - -defaults: - run: - shell: bash - -env: - PYTHON_TARGET_VERSION: 3.11 - NOTIFICATION_PREFIX: "[Release Preparation]" - -jobs: - log-inputs: - runs-on: ubuntu-latest - - steps: - - name: "[DEBUG] Print Variables" - run: | - # WORKFLOW INPUTS - echo The release version number: ${{ inputs.version_number }} - echo Deploy to: ${{ inputs.deploy_to }} - echo Target branch: ${{ inputs.target_branch }} - echo Nightly release: ${{ inputs.nightly_release }} - echo Optional env setup script: ${{ inputs.env_setup_script_path }} - echo run-unit-tests: ${{ inputs.run-unit-tests }} - echo run-integration-tests: ${{ inputs.run-integration-tests }} - echo working-dir: ${{ inputs.working-dir }} - # ENVIRONMENT VARIABLES - echo Python target version: ${{ env.PYTHON_TARGET_VERSION }} - echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} - audit-changelog: - runs-on: ubuntu-latest - - outputs: - changelog_path: ${{ steps.set_path.outputs.changelog_path }} - exists: ${{ steps.set_existence.outputs.exists }} - base_version: ${{ steps.semver.outputs.base-version }} - prerelease: ${{ steps.semver.outputs.pre-release }} - is_prerelease: ${{ steps.semver.outputs.is-pre-release }} - - steps: - - name: "Checkout ${{ github.repository }}" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.target_branch }} - - - name: "Audit Version And Parse Into Parts" - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ inputs.version_number }} - - - name: "Set Changelog Path" - id: set_path - run: | - path=".changes/" - if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] - then - path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" - else - path+="${{ steps.semver.outputs.base-version }}.md" - fi - # Send notification - echo "changelog_path=$path" >> $GITHUB_OUTPUT - title="Changelog path" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$changelog_path" - - name: "Set Changelog Existence For Subsequent Jobs" - id: set_existence - run: | - does_exist=false - if test -f ${{ steps.set_path.outputs.changelog_path }} - then - does_exist=true - fi - echo "exists=$does_exist">> $GITHUB_OUTPUT - - name: "[Notification] Set Changelog Existence For Subsequent Jobs" - run: | - title="Changelog exists" - if [[ ${{ steps.set_existence.outputs.exists }} == true ]] - then - message="Changelog file ${{ steps.set_path.outputs.changelog_path }} already exists" - else - message="Changelog file ${{ steps.set_path.outputs.changelog_path }} doesn't exist" - fi - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "[DEBUG] Print Outputs" - run: | - echo changelog_path: ${{ steps.set_path.outputs.changelog_path }} - echo exists: ${{ steps.set_existence.outputs.exists }} - echo base_version: ${{ steps.semver.outputs.base-version }} - echo prerelease: ${{ steps.semver.outputs.pre-release }} - echo is_prerelease: ${{ steps.semver.outputs.is-pre-release }} - - audit-version-in-code: - runs-on: ubuntu-latest - - outputs: - up_to_date: ${{ steps.version-check.outputs.up_to_date }} - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ inputs.target_branch }}" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.target_branch }} - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: "Check Current Version In Code" - id: version-check - run: | - is_updated=false - current_version=$(hatch version) - if test "$current_version" = "${{ inputs.version_number }}" - then - is_updated=true - fi - echo "up_to_date=$is_updated" >> $GITHUB_OUTPUT - working-directory: ${{ inputs.working-dir }} - - - name: "[Notification] Check Current Version In Code" - run: | - title="Version check" - if [[ ${{ steps.version-check.outputs.up_to_date }} == true ]] - then - message="The version in the codebase is equal to the provided version" - else - message="The version in the codebase differs from the provided version" - fi - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "[DEBUG] Print Outputs" - run: | - echo up_to_date: ${{ steps.version-check.outputs.up_to_date }} - - skip-generate-changelog: - runs-on: ubuntu-latest - needs: [audit-changelog] - if: needs.audit-changelog.outputs.exists == 'true' - - steps: - - name: "Changelog Exists, Skip Generating New Changelog" - run: | - # Send notification - title="Skip changelog generation" - message="A changelog file already exists at ${{ needs.audit-changelog.outputs.changelog_path }}, skipping generating changelog" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - skip-version-bump: - runs-on: ubuntu-latest - needs: [audit-version-in-code] - if: needs.audit-version-in-code.outputs.up_to_date == 'true' - - steps: - - name: "Version Already Bumped" - run: | - # Send notification - title="Skip version bump" - message="The version has already been bumped to ${{ inputs.version_number }}, skipping version bump" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - create-temp-branch: - runs-on: ubuntu-latest - needs: [audit-changelog, audit-version-in-code] - if: needs.audit-changelog.outputs.exists == 'false' || needs.audit-version-in-code.outputs.up_to_date == 'false' - - outputs: - branch_name: ${{ steps.variables.outputs.branch_name }} - - steps: - - name: "Checkout ${{ github.repository }}" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.target_branch }} - - - name: "Generate Branch Name" - id: variables - run: | - name="prep-release/" - if [[ ${{ inputs.nightly_release }} == true ]] - then - name+="nightly-release/" - else - name+="${{ inputs.deploy_to }}/" - fi - name+="${{ inputs.version_number }}_$GITHUB_RUN_ID" - echo "branch_name=$name" >> $GITHUB_OUTPUT - - name: "Create Branch - ${{ steps.variables.outputs.branch_name }}" - run: | - git checkout -b ${{ steps.variables.outputs.branch_name }} - git push -u origin ${{ steps.variables.outputs.branch_name }} - - name: "[Notification] Temp branch created" - run: | - # Send notification - title="Temp branch generated" - message="The ${{ steps.variables.outputs.branch_name }} branch created" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "[DEBUG] Print Outputs" - run: | - echo branch_name ${{ steps.variables.outputs.branch_name }} - generate-changelog-bump-version: - runs-on: ubuntu-latest - needs: [audit-changelog, audit-version-in-code, create-temp-branch] - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 - with: - ref: ${{ needs.create-temp-branch.outputs.branch_name }} - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - name: "Add Homebrew To PATH" - run: | - echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH - - name: "Install Homebrew Packages" - run: | - brew install pre-commit - brew tap miniscruff/changie https://github.com/miniscruff/changie - brew install changie - - name: "Set json File Name" - id: json_file - run: | - echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT - - name: "Get Core Team Membership" - run: | - gh api -H "Accept: application/vnd.github+json" orgs/dbt-labs/teams/core-group/members > ${{ steps.json_file.outputs.name }} - env: - GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} - - name: "Set Core Team Membership for Changie Contributors exclusion" - id: set_team_membership - run: | - team_list=$(jq -r '.[].login' ${{ steps.json_file.outputs.name }}) - echo $team_list - team_list_single=$(echo $team_list | tr '\n' ' ') - echo "CHANGIE_CORE_TEAM=$team_list_single" >> $GITHUB_ENV - - name: "Delete the json File" - run: | - rm ${{ steps.json_file.outputs.name }} - - name: "Generate Release Changelog" - if: needs.audit-changelog.outputs.exists == 'false' - run: | - if [[ ${{ needs.audit-changelog.outputs.is_prerelease }} -eq 1 ]] - then - changie batch ${{ needs.audit-changelog.outputs.base_version }} --move-dir '${{ needs.audit-changelog.outputs.base_version }}' --prerelease ${{ needs.audit-changelog.outputs.prerelease }} - elif [[ -d ".changes/${{ needs.audit-changelog.outputs.base_version }}" ]] - then - changie batch ${{ needs.audit-changelog.outputs.base_version }} --include '${{ needs.audit-changelog.outputs.base_version }}' --remove-prereleases - else # releasing a final patch with no prereleases - changie batch ${{ needs.audit-changelog.outputs.base_version }} - fi - changie merge - git status - - name: "Check Changelog Created Successfully" - if: needs.audit-changelog.outputs.exists == 'false' - run: | - title="Changelog" - if [[ -f ${{ needs.audit-changelog.outputs.changelog_path }} ]] - then - message="Changelog file created successfully" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - else - message="Changelog failed to generate" - echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 1 - fi - - name: "Bump Version To ${{ inputs.version_number }}" - if: needs.audit-version-in-code.outputs.up_to_date == 'false' - run: | - hatch version ${{ inputs.version_number }} - working-directory: ${{ inputs.working-dir }} - - name: "[Notification] Bump Version To ${{ inputs.version_number }}" - if: needs.audit-version-in-code.outputs.up_to_date == 'false' - run: | - title="Version bump" - message="Version successfully bumped in codebase to ${{ inputs.version_number }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - # TODO: can these 2 steps be done via hatch? probably. - # this step will fail on whitespace errors but also correct them - - name: "Remove Trailing Whitespace Via Pre-commit" - continue-on-error: true - run: | - pre-commit run trailing-whitespace --files dbt/adapters/__about__.py CHANGELOG.md .changes/* - git status - # this step will fail on newline errors but also correct them - - name: "Removing Extra Newlines Via Pre-commit" - continue-on-error: true - run: | - pre-commit run end-of-file-fixer --files dbt/adapters/__about__.py CHANGELOG.md .changes/* - git status - - name: "Commit & Push Changes" - run: | - #Data for commit - user="Github Build Bot" - email="buildbot@fishtownanalytics.com" - commit_message="Bumping version to ${{ inputs.version_number }} and generate changelog" - #Commit changes to branch - git config user.name "$user" - git config user.email "$email" - git pull - git add . - git commit -m "$commit_message" - git push - - run-unit-tests: - if: inputs.run-unit-tests == true - runs-on: ubuntu-latest - needs: [create-temp-branch, generate-changelog-bump-version] - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 - with: - ref: ${{ needs.create-temp-branch.outputs.branch_name }} - - name: "Setup `hatch`" - uses: ./.github/actions/setup-hatch - - name: "Run Unit Tests" - run: hatch run unit-tests:all - - run-integration-tests: - runs-on: ubuntu-20.04 - needs: [create-temp-branch, generate-changelog-bump-version] - if: inputs.run-integration-tests == true - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 - with: - ref: ${{ needs.create-temp-branch.outputs.branch_name }} - - - name: "Setup Environment Variables - ./${{ inputs.env_setup_script_path }}" - if: inputs.env_setup_script_path != '' - run: source ./${{ inputs.env_setup_script_path }} - - - name: "Setup Environment Variables - Secrets Context" - if: inputs.env_setup_script_path != '' - uses: actions/github-script@v6 - id: check-env - with: - result-encoding: string - script: | - try { - const { SECRETS_CONTEXT, INTEGRATION_TESTS_SECRETS_PREFIX } = process.env - const secrets = JSON.parse(SECRETS_CONTEXT) - if (INTEGRATION_TESTS_SECRETS_PREFIX) { - for (const [key, value] of Object.entries(secrets)) { - if (key.startsWith(INTEGRATION_TESTS_SECRETS_PREFIX)) { - core.exportVariable(key, value) - } - } - } else { - core.info("The INTEGRATION_TESTS_SECRETS_PREFIX env variable is empty or not defined, skipping the secrets setup.") - } - } catch (err) { - core.error("Error while reading or parsing the JSON") - core.setFailed(err) - } - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: "Set up Python & Hatch - ${{ env.PYTHON_TARGET_VERSION }}" - uses: ./.github/actions/setup-python-env - with: - python-version: ${{ env.PYTHON_TARGET_VERSION }} - - - name: Run tests - run: hatch run integration-tests:all - - merge-changes-into-target-branch: - runs-on: ubuntu-latest - needs: [run-unit-tests, run-integration-tests, create-temp-branch, audit-version-in-code, audit-changelog] - if: | - !failure() && !cancelled() && - inputs.deploy_to == 'prod' && - ( - needs.audit-changelog.outputs.exists == 'false' || - needs.audit-version-in-code.outputs.up_to_date == 'false' - ) - steps: - - name: "[Debug] Print Variables" - run: | - echo branch_name: ${{ needs.create-temp-branch.outputs.branch_name }} - echo inputs.deploy_to: ${{ inputs.deploy_to }} - echo needs.audit-changelog.outputs.exists: ${{ needs.audit-changelog.outputs.exists }} - echo needs.audit-version-in-code.outputs.up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} - - name: "Checkout Repo ${{ github.repository }}" - uses: actions/checkout@v3 - - - name: "Merge Changes Into ${{ inputs.target_branch }}" - uses: everlytic/branch-merge@1.1.5 - with: - source_ref: ${{ needs.create-temp-branch.outputs.branch_name }} - target_branch: ${{ inputs.target_branch }} - github_token: ${{ secrets.FISHTOWN_BOT_PAT }} - commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" - - - name: "[Notification] Changes Merged into main" - run: | - title="Changelog and Version Bump Branch Merge" - message="The ${{ needs.create-temp-branch.outputs.branch_name }} branch was merged into mains" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - determine-release-branch: - runs-on: ubuntu-latest - needs: - [ - create-temp-branch, - merge-changes-into-target-branch, - audit-changelog, - audit-version-in-code, - ] - # always run this job, regardless of if the dependant jobs were skipped - if: ${{ !failure() && !cancelled() }} - - # Get the sha that will be released. If the changelog already exists on the input sha and the version has already been bumped, - # then it is what we will release. Otherwise we generated a changelog and did the version bump in this workflow and there is a - # new sha to use from the merge we just did. Grab that here instead. - outputs: - final_sha: ${{ steps.resolve_commit_sha.outputs.release_sha }} - - steps: - - name: "[Debug] Print Variables" - run: | - echo new_branch: ${{ needs.create-temp-branch.outputs.branch_name }} - echo changelog_exists: ${{ needs.audit-changelog.outputs.exists }} - echo up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} - - name: "Resolve Branch To Checkout" - id: resolve_branch - run: | - branch="" - if [ ${{ inputs.deploy_to == 'test' }}] || [ ${{ inputs.nightly_release == 'true' }} ] - then - branch=${{ needs.create-temp-branch.outputs.branch_name }} - else - branch="${{ inputs.target_branch }}" - fi - echo "target_branch=$branch" >> $GITHUB_OUTPUT - - name: "[Notification] Resolve Branch To Checkout" - run: | - title="Branch pick" - message="The ${{ steps.resolve_branch.outputs.target_branch }} branch will be used for release" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "Checkout Resolved Branch - ${{ steps.resolve_branch.outputs.target_branch }}" - uses: actions/checkout@v3 - with: - ref: ${{ steps.resolve_branch.outputs.target_branch }} - - - name: "[Debug] Log Branch" - run: git status - - - name: "Resolve Commit SHA For Release" - id: resolve_commit_sha - run: | - echo "release_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: "Remove Temp Branch - ${{ needs.create-temp-branch.outputs.branch_name }}" - if: ${{ inputs.deploy_to == 'prod' && inputs.nightly_release == 'false' && needs.create-temp-branch.outputs.branch_name != '' }} - run: | - git push origin -d ${{ needs.create-temp-branch.outputs.branch_name }} \ No newline at end of file diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index 75a14dd4..00000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,30 +0,0 @@ -# **what?** -# For issues that have been open for awhile without activity, label -# them as stale with a warning that they will be closed out. If -# anyone comments to keep the issue open, it will automatically -# remove the stale label and keep it open. - -# Stale label rules: -# awaiting_response, more_information_needed -> 90 days -# good_first_issue, help_wanted -> 360 days (a year) -# tech_debt -> 720 (2 years) -# all else defaults -> 180 days (6 months) - -# **why?** -# To keep the repo in a clean state from issues that aren't relevant anymore - -# **when?** -# Once a day - -name: "Close stale issues and PRs" -on: - schedule: - - cron: "30 1 * * *" - -permissions: - issues: write - pull-requests: write - -jobs: - stale: - uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index 26ff4aaa..00000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Unit Tests - -on: - push: - branches: - - "main" - - "*.latest" - pull_request: - workflow_dispatch: - -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -jobs: - unit: - name: Unit Tests - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - with: - python-version: ${{ matrix.python-version }} - - - name: Run unit tests - run: hatch run unit-tests:all - shell: bash - - - name: Publish results - uses: ./.github/actions/publish-results - if: always() - with: - source-file: "results.csv" - file-name: "unit_results" - python-version: ${{ matrix.python-version }} diff --git a/pyproject.toml b/pyproject.toml index f9b8b0f7..9111b41d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,11 +65,16 @@ dependencies = [ ] [tool.hatch.envs.lint.scripts] all = [ - "- black-only", - "- flake8-only", + "- black", + "- flake8", + "- pre-commit" +] +black = "python -m black {args:.}" +flake8 = "python -m flake8 {args:.}" +pre-commit = [ + "pre-commit run trailing-whitespace {args}", + "pre-commit run end-of-file-fixer {args}", ] -black-only = "python -m black ." -flake8-only = "python -m flake8 ." [tool.hatch.envs.typecheck] dependencies = [