diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 86520f238ce..317a7d23af8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,12 +21,9 @@ permissions: contents: write jobs: - create_release_branch: + release: name: Create Release - runs-on: ubuntu-latest - outputs: - release_branch_name: ${{ steps.variables.outputs.release_branch_name }} - tag_name: ${{ steps.variables.outputs.tag_name }} + runs-on: [self-hosted, release] steps: - name: Get releaser identity run: | @@ -35,8 +32,8 @@ jobs: - name: Declare release branch name and tag name id: variables run: | - echo "release_branch_name=release_${CANDIDATE_NAME,,}" >> $GITHUB_OUTPUT - echo "tag_name=${CANDIDATE_NAME^^}" >> $GITHUB_OUTPUT + echo "releaseBranchName=release_${CANDIDATE_NAME,,}" >> $GITHUB_OUTPUT + echo "tagName=${CANDIDATE_NAME^^}" >> $GITHUB_OUTPUT env: CANDIDATE_NAME: ${{ inputs.candidateName }} - name: Checkout code @@ -47,7 +44,7 @@ jobs: - name: Create release branch run: git checkout -b $RELEASE_BRANCH_NAME $BRANCH_COMMIT env: - RELEASE_BRANCH_NAME: ${{ steps.variables.outputs.release_branch_name }} + RELEASE_BRANCH_NAME: ${{ steps.variables.outputs.releaseBranchName }} BRANCH_COMMIT: ${{ inputs.branchCommit }} - name: Cherry pick commits run: | @@ -59,157 +56,62 @@ jobs: done env: CHERRYPICK_COMMITS: ${{ inputs.cherrypickCommits }} - - name: Push release branch + - name: Add tag to most recent commit run: | - git push origin --delete $RELEASE_BRANCH_NAME || true - git push -u origin $RELEASE_BRANCH_NAME + DATE=$(date -d"next-monday - 1week" +'%Y-%m-%d') + T_COMMIT=$(git log -n 1 $RELEASE_BRANCH_NAME --pretty=format:'%H') + git tag -a $TAG_NAME -m "Release week of $DATE" $T_COMMIT env: - RELEASE_BRANCH_NAME: ${{ steps.variables.outputs.release_branch_name }} - java_unit_tests: - name: Unit Tests - needs: create_release_branch - timeout-minutes: 60 - runs-on: [ self-hosted, release ] - steps: - - name: Checkout code - uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0 - with: - ref: ${{ needs.create_release_branch.outputs.release_branch_name }} - fetch-depth: 0 - token: ${{ secrets.RELEASE_TOKEN }} - - name: Setup Environment - id: setup-env - uses: ./.github/actions/setup-env - - name: Run Unit Tests - run: ./cicd/run-unit-tests - - name: Upload Unit Tests Site Report - uses: ./.github/actions/publish-site-report - with: - output-zip-file: unit-test-report - if: always() - - name: Cleanup Java Environment - uses: ./.github/actions/cleanup-java-env - if: always() - java_integration_smoke_tests_templates: - name: Dataflow Templates Integration Smoke Tests - needs: create_release_branch - timeout-minutes: 60 - # Run on any runner that matches all the specified runs-on values. - runs-on: [ self-hosted, release ] - steps: - - name: Checkout code - uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0 - with: - ref: ${{ needs.create_release_branch.outputs.release_branch_name }} - fetch-depth: 0 - token: ${{ secrets.RELEASE_TOKEN }} - - name: Setup Environment - id: setup-env - uses: ./.github/actions/setup-env - - name: Run Integration Smoke Tests - run: | - ./cicd/run-it-smoke-tests \ - --changed-files="pom.xml" \ - --it-region="us-central1" \ - --it-project="cloud-teleport-testing" \ - --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ - --it-private-connectivity="datastream-private-connect-us-central1" \ - --it-release=true \ - --it-retry-failures=2 - - name: Publish Smoke Test Site Report - uses: ./.github/actions/publish-site-report - with: - output-zip-file: smoke-test-report - if: always() - - name: Cleanup Java Environment - uses: ./.github/actions/cleanup-java-env - if: always() - java_integration_tests_templates: - name: Dataflow Templates Integration Tests - needs: create_release_branch - timeout-minutes: 240 - # Run on any runner that matches all the specified runs-on values. - runs-on: [ self-hosted, release ] - steps: - - name: Checkout code - uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0 - with: - ref: ${{ needs.create_release_branch.outputs.release_branch_name }} - fetch-depth: 0 - token: ${{ secrets.RELEASE_TOKEN }} - - name: Setup Environment - id: setup-env - uses: ./.github/actions/setup-env - - name: Run Integration Tests - shell: bash - run: | - ./cicd/run-it-tests \ - --changed-files="pom.xml" \ - --it-region="us-central1" \ - --it-project="cloud-teleport-testing" \ - --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ - --it-private-connectivity="datastream-private-connect-us-central1" \ - --it-release=true \ - --it-retry-failures=2 - - name: Publish Integration Test Site Report - uses: ./.github/actions/publish-site-report - with: - output-zip-file: integration-test-report - if: always() - - name: Cleanup Java Environment - uses: ./.github/actions/cleanup-java-env - if: always() - upload_artifacts: - name: Upload Template Artifacts - needs: [ create_release_branch, java_unit_tests, java_integration_smoke_tests_templates, java_integration_tests_templates ] - runs-on: [ self-hosted, release ] - steps: - - name: Checkout code - uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0 - with: - ref: ${{ needs.create_release_branch.outputs.release_branch_name }} - fetch-depth: 0 - token: ${{ secrets.RELEASE_TOKEN }} - - name: Create artifacts and push - run: | - ./cicd/run-release \ - --it-project="dataflow-templates" \ - --release-bucket-name="dataflow-templates-staging" \ - --release-libraries-bucket-name="dataflow-templates-libraries" \ - --release-stage-prefix="${CANDIDATE_NAME}" - env: - CANDIDATE_NAME: ${{ inputs.candidateName }} - - name: Cleanup Java Environment - uses: ./.github/actions/cleanup-java-env - if: always() - release_github: - name: Release on GitHub - needs: [ create_release_branch, upload_artifacts ] - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0 - with: - ref: ${{ needs.create_release_branch.outputs.release_branch_name }} - fetch-depth: 0 - token: ${{ secrets.RELEASE_TOKEN }} - - name: Add tag to most recent commit - run: | - DATE=$(date -d"next-monday - 1week" +'%Y-%m-%d') - T_COMMIT=$(git log -n 1 $RELEASE_BRANCH_NAME --pretty=format:'%H') - git tag -a $TAG_NAME -m "Release week of $DATE" $T_COMMIT - env: - RELEASE_BRANCH_NAME: ${{ needs.create_release_branch.outputs.release_branch_name }} - TAG_NAME: ${{ needs.create_release_branch.outputs.tag_name }} - - name: Push tags - run: | - git push --delete origin $TAG_NAME || true - git push -u origin --tags - env: - TAG_NAME: ${{ needs.create_release_branch.outputs.tag_name }} - - name: Release - run: | - gh release create $TAG_NAME --title "Dataflow Templates $TAG_NAME" --notes "" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG_NAME: ${{ needs.create_release_branch.outputs.tag_name }} + RELEASE_BRANCH_NAME: ${{ steps.variables.outputs.releaseBranchName }} + TAG_NAME: ${{ steps.variables.outputs.tagName }} + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Build + run: ./cicd/run-build + - name: Run Unit Tests + run: ./cicd/run-unit-tests + - name: Run Integration Smoke Tests + run: | + ./cicd/run-it-smoke-tests \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" \ + --it-spanner-host="https://staging-wrenchworks.sandbox.googleapis.com/" \ + --it-release=true \ + --it-retry-failures=2 + - name: Run Integration Tests + run: | + ./cicd/run-it-tests \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" \ + --it-spanner-host="https://staging-wrenchworks.sandbox.googleapis.com/" \ + --it-release=true \ + --it-retry-failures=2 + - name: Upload Unit Tests Site Report + uses: ./.github/actions/publish-site-report + with: + output-zip-file: test-report + if: always() + - name: Create artifacts and push + run: | + mvn verify -PtemplatesRelease,splunkDeps,missing-artifact-repos \ + -DprojectId="dataflow-templates" \ + -DbucketName="dataflow-templates-staging" \ + -DlibrariesBucketName="dataflow-templates-libraries" \ + -DstagePrefix="${CANDIDATE_NAME}" \ + -Dmaven.test.skip -T8 -e + env: + CANDIDATE_NAME: ${{ inputs.candidateName }} + - name: Push tags + run: | + git push -u origin --tags + - name: Release + run: | + gh release create $TAG_NAME --title "Dataflow Templates $TAG_NAME" --notes "" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG_NAME: ${{ steps.variables.outputs.tagName }} diff --git a/cicd/cmd/run-release/main.go b/cicd/cmd/run-release/main.go deleted file mode 100644 index c64b368ffc6..00000000000 --- a/cicd/cmd/run-release/main.go +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package main - -import ( - "flag" - "log" - - "github.com/GoogleCloudPlatform/DataflowTemplates/cicd/internal/flags" - "github.com/GoogleCloudPlatform/DataflowTemplates/cicd/internal/workflows" -) - -func main() { - flags.RegisterCommonFlags() - flags.RegisterItFlags() - flags.RegisterReleaseFlags() - flag.Parse() - - // Run mvn install before running release - mvnFlags := workflows.NewMavenFlags() - err := workflows.MvnCleanInstall().Run( - mvnFlags.IncludeDependencies(), - mvnFlags.IncludeDependents(), - mvnFlags.SkipDependencyAnalysis(), - mvnFlags.SkipCheckstyle(), - mvnFlags.SkipJib(), - mvnFlags.SkipTests(), - mvnFlags.SkipJacoco(), - mvnFlags.SkipShade(), - mvnFlags.ThreadCount(8)) - if err != nil { - log.Fatalf("%v\n", err) - } - - // Run release - mvnFlags = workflows.NewMavenFlags() - err = workflows.MvnVerify().Run( - mvnFlags.RunRelease(), - mvnFlags.SkipTests(), - mvnFlags.ThreadCount(8), - mvnFlags.ProduceErrors(), - flags.Project(), - flags.BucketName(), - flags.LibrariesBucketName(), - flags.StagePrefix()) - if err != nil { - log.Fatalf("%v\n", err) - } - log.Println("Release Successful!") -} diff --git a/cicd/internal/flags/release-flags.go b/cicd/internal/flags/release-flags.go deleted file mode 100644 index 3045c7cd39a..00000000000 --- a/cicd/internal/flags/release-flags.go +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package flags - -import ( - "flag" -) - -// Avoid making these vars public. -var ( - dBucketName string - dLibrariesBucketName string - dStagePrefix string -) - -// Registers all release flags. Must be called before flag.Parse(). -func RegisterReleaseFlags() { - flag.StringVar(&dBucketName, "release-bucket-name", "", "The GCP bucket to stage the released templates.") - flag.StringVar(&dLibrariesBucketName, "release-libraries-bucket-name", "", "The GCP bucket to stage the released template libraries.") - flag.StringVar(&dStagePrefix, "release-stage-prefix", "", "Prefix to use as parent folder in GCS for released templates.") -} - -func BucketName() string { - return "-DbucketName=" + dBucketName -} - -func LibrariesBucketName() string { - return "-DlibrariesBucketName=" + dLibrariesBucketName -} - -func StagePrefix() string { - return "-DstagePrefix=" + dStagePrefix -}