chore: Refactoring release yamls to be more modular and remove redundancies

.github/workflows/build_wheels.yml

@@ -5,8 +5,8 @@ name: build_wheels
 # Devs should check out their fork, add a tag to the last master commit on their fork, and run the release off of their fork on the added tag to ensure wheels will be built correctly.
 on:
   workflow_dispatch:
-      tags:
-        - 'v*.*.*'
+    tags:
+      - 'v*.*.*'
   workflow_call:
     inputs:
       release_version:
@@ -20,58 +20,10 @@ on:
 
 jobs:
   get-version:
-    runs-on:  ubuntu-latest
-    outputs:
-      release_version: ${{ steps.get_release_version.outputs.release_version }}
-      version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
-      highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - name: Get release version
-        id: get_release_version
-        run: |
-          if [[ -n "${{ inputs.release_version }}" ]]; then
-            echo "Using provided release version: ${{ inputs.release_version }}"
-            echo "::set-output name=release_version::${{ inputs.release_version }}"
-          else
-            echo "No release version provided. Falling back to GITHUB_REF."
-            echo "::set-output name=release_version::${GITHUB_REF#refs/tags/}"
-          fi
-      - name: Get release version without prefix
-        id: get_release_version_without_prefix
-        env:
-          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
-        run: |
-          echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1}
-      - name: Get highest semver
-        id: get_highest_semver
-        env:
-          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
-        run: |
-          if [[ -n "${{ inputs.highest_semver_tag }}" ]]; then
-            echo "Using provided highest semver version: ${{ inputs.highest_semver_tag }}"
-            echo "::set-output name=highest_semver_tag::${{ inputs.highest_semver_tag }}"
-          else
-            echo "No release version provided. Falling back to infra/scripts/setup-common-functions.sh."
-            source infra/scripts/setup-common-functions.sh
-            SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
-            if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
-              echo ::set-output name=highest_semver_tag::$(get_tag_release -m)
-            fi
-          fi
-      - name: Check output
-        id: check_output
-        env:
-          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
-          VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
-          HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
-        run: |
-          echo $RELEASE_VERSION
-          echo $VERSION_WITHOUT_PREFIX
-          echo $HIGHEST_SEMVER_TAG
+    uses: ./.github/workflows/get_semantic_release_version.yaml
+    with:
+      custom_version: ${{ github.event.inputs.custom_version }}
+      token: ${{ github.event.inputs.token }}
 
   build-python-wheel:
     name: Build wheels
@@ -141,7 +93,7 @@ jobs:
     needs: get-version
     strategy:
       matrix:
-        component: [feature-server, feature-server-java, feature-transformation-server]
+        component: [ feature-server, feature-server-java, feature-transformation-server ]
     env:
       REGISTRY: feastdev
     steps:
@@ -158,11 +110,11 @@ jobs:
 
   verify-python-wheels:
     runs-on: ${{ matrix.os }}
-    needs: [build-python-wheel, build-source-distribution, get-version]
+    needs: [ build-python-wheel, build-source-distribution, get-version ]
     strategy:
       matrix:
-        os: [ubuntu-latest,  macos-13 ]
-        python-version: ["3.9", "3.10", "3.11"]
+        os: [ ubuntu-latest,  macos-13 ]
+        python-version: [ "3.9", "3.10", "3.11" ]
         from-source: [ True, False ]
     env:
       # this script is for testing servers

.github/workflows/get_semantic_release_version.yml

@@ -0,0 +1,84 @@
+name: Get semantic release version
+
+on:
+  workflow_dispatch: # Allows manual trigger of the workflow
+    inputs:
+      custom_version: # Optional input for a custom version
+        description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing'
+        required: false
+      token:
+        description: 'Personal Access Token'
+        required: true
+        default: ""
+        type: string
+
+jobs:
+  get-version:
+    if: github.repository == 'feast-dev/feast'
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ github.event.inputs.token }}
+      GIT_AUTHOR_NAME: feast-ci-bot
+      GIT_AUTHOR_EMAIL: [email protected]
+      GIT_COMMITTER_NAME: feast-ci-bot
+      GIT_COMMITTER_EMAIL: [email protected]
+    outputs:
+      release_version: ${{ steps.get_release_version.outputs.release_version }}
+      version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
+      highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get release version
+        id: get_release_version
+        run: |
+          if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then
+            VERSION_REGEX="^v[0-9]+\.[0-9]+\.[0-9]+$"
+            echo "Using custom version: ${{ github.event.inputs.custom_version }}"
+            if [[ ! "${{ github.event.inputs.custom_version }}" =~ $VERSION_REGEX ]]; then
+              echo "Error: custom_version must match semantic versioning (e.g., v1.2.3)."
+              exit 1
+            fi
+            echo "::set-output name=release_version::${{ github.event.inputs.custom_version }}"
+          elif [[ "${GITHUB_REF}" == refs/tags/* ]]; then
+            echo "Using tag reference: ${GITHUB_REF#refs/tags/}"
+            echo "::set-output name=release_version::${GITHUB_REF#refs/tags/}"
+          else
+            echo "Defaulting to branch name: ${GITHUB_REF#refs/heads/}"
+            echo "::set-output name=release_version::${GITHUB_REF#refs/heads/}"
+          fi
+      - name: Get release version without prefix
+        id: get_release_version_without_prefix
+        env:
+          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
+        run: |
+          if [[ "${RELEASE_VERSION}" == v* ]]; then
+            echo "::set-output name=version_without_prefix::${RELEASE_VERSION:1}"
+          else
+            echo "::set-output name=version_without_prefix::${RELEASE_VERSION}"
+          fi
+      - name: Get highest semver
+        id: get_highest_semver
+        env:
+          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
+        run: |
+          if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then
+            HIGHEST_SEMVER_TAG="${{ github.event.inputs.custom_version }}"
+            echo "::set-output name=highest_semver_tag::$HIGHEST_SEMVER_TAG"
+            echo "Using custom version as highest semantic version: $HIGHEST_SEMVER_TAG"
+          else 
+            source infra/scripts/setup-common-functions.sh
+            SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
+            if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
+              echo ::set-output name=highest_semver_tag::$(get_tag_release -m)
+              echo "Using infra/scripts/setup-common-functions.sh to generate highest semantic version: $HIGHEST_SEMVER_TAG"
+            fi
+          fi
+      - name: Check output
+        env:
+          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
+          VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
+          HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
+        run: |
+          echo $RELEASE_VERSION
+          echo $VERSION_WITHOUT_PREFIX
+          echo $HIGHEST_SEMVER_TAG

.github/workflows/publish.yml

@@ -17,81 +17,23 @@ on:
 
 jobs:
   get-version:
-    if: github.repository == 'feast-dev/feast'
-    runs-on:  ubuntu-latest
-    env:
-      GITHUB_TOKEN: ${{ github.event.inputs.token }}
-      GIT_AUTHOR_NAME: feast-ci-bot
-      GIT_AUTHOR_EMAIL: [email protected]
-      GIT_COMMITTER_NAME: feast-ci-bot
-      GIT_COMMITTER_EMAIL: [email protected]
-    outputs:
-      release_version: ${{ steps.get_release_version.outputs.release_version }}
-      version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
-      highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Get release version
-        id: get_release_version
-        run: |
-          if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then
-            VERSION_REGEX="^v[0-9]+\.[0-9]+\.[0-9]+$"
-            echo "Using custom version: ${{ github.event.inputs.custom_version }}"
-            if [[ ! "${{ github.event.inputs.custom_version }}" =~ $VERSION_REGEX ]]; then
-              echo "Error: custom_version must match semantic versioning (e.g., v1.2.3)."
-              exit 1
-            fi
-            echo "::set-output name=release_version::${{ github.event.inputs.custom_version }}"
-          elif [[ "${GITHUB_REF}" == refs/tags/* ]]; then
-            echo "Using tag reference: ${GITHUB_REF#refs/tags/}"
-            echo "::set-output name=release_version::${GITHUB_REF#refs/tags/}"
-          else
-            echo "Defaulting to branch name: ${GITHUB_REF#refs/heads/}"
-            echo "::set-output name=release_version::${GITHUB_REF#refs/heads/}"
-          fi
-      - name: Get release version without prefix
-        id: get_release_version_without_prefix
-        env:
-          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
-        run: |
-          if [[ "${RELEASE_VERSION}" == v* ]]; then
-            echo "::set-output name=version_without_prefix::${RELEASE_VERSION:1}"
-          else
-            echo "::set-output name=version_without_prefix::${RELEASE_VERSION}"
-          fi
-      - name: Get highest semver
-        id: get_highest_semver
-        env:
-          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
-        run: |
-          if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then
-            HIGHEST_SEMVER_TAG="${{ github.event.inputs.custom_version }}"
-            echo "::set-output name=highest_semver_tag::$HIGHEST_SEMVER_TAG"
-            echo "Using custom version as highest semantic version: $HIGHEST_SEMVER_TAG"
-          else 
-            source infra/scripts/setup-common-functions.sh
-            SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
-            if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
-              echo ::set-output name=highest_semver_tag::$(get_tag_release -m)
-              echo "Using infra/scripts/setup-common-functions.sh to generate highest semantic version: $HIGHEST_SEMVER_TAG"
-            fi
-          fi
-      - name: Check output
-        env:
-          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
-          VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
-          HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
-        run: |
-          echo $RELEASE_VERSION
-          echo $VERSION_WITHOUT_PREFIX
-          echo $HIGHEST_SEMVER_TAG
+    uses: ./.github/workflows/get_semantic_release_version.yaml
+    with:
+      custom_version: ${{ github.event.inputs.custom_version }}
+      token: ${{ github.event.inputs.token }}
+
+  publish-python-sdk:
+    uses: ./.github/workflows/publish_python_sdk.yaml
+    with:
+      custom_version: ${{ github.event.inputs.custom_version }}
+      token: ${{ github.event.inputs.token }}
 
   build-publish-docker-images:
     runs-on: ubuntu-latest
-    needs: [get-version, publish-python-sdk]
+    needs: [ get-version, publish-python-sdk ]
     strategy:
       matrix:
-        component: [feature-server, feature-server-java, feature-transformation-server, feast-helm-operator, feast-operator]
+        component: [ feature-server, feature-server-java, feature-transformation-server, feast-helm-operator, feast-operator ]
     env:
       MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
       REGISTRY: feastdev
@@ -140,60 +82,17 @@ jobs:
           fi
 
   publish-helm-charts:
-    if: github.repository == 'feast-dev/feast'
-    runs-on:  ubuntu-latest
-    needs: get-version
-    env:
-      HELM_VERSION: v3.8.0
-      VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Authenticate to Google Cloud
-        uses: 'google-github-actions/auth@v1'
-        with:
-          credentials_json: '${{ secrets.GCP_SA_KEY }}'
-      - name: Set up gcloud SDK
-        uses: google-github-actions/setup-gcloud@v1
-        with:
-          project_id: ${{ secrets.GCP_PROJECT_ID }}
-      - run: gcloud auth configure-docker --quiet
-      - name: Remove previous Helm
-        run: sudo rm -rf $(which helm)
-      - name: Install Helm
-        run: ./infra/scripts/helm/install-helm.sh
-      - name: Validate Helm chart prior to publishing
-        run: ./infra/scripts/helm/validate-helm-chart-publish.sh
-      - name: Validate all version consistency
-        run: ./infra/scripts/helm/validate-helm-chart-versions.sh $VERSION_WITHOUT_PREFIX
-      - name: Publish Helm charts
-        run: ./infra/scripts/helm/push-helm-charts.sh $VERSION_WITHOUT_PREFIX
-
-  build_wheels:
-    uses: ./.github/workflows/build_wheels.yml
-    needs: get-version
+    uses: ./.github/workflows/publish_helm_charts.yml
+    needs: [ get-version, publish-python-sdk ]
     with:
-        release_version: ${{ needs.get-version.outputs.release_version }}
-        highest_semver_tag: ${{ needs.get-version.outputs.highest_semver_tag }}
-
-  publish-python-sdk:
-    if: github.repository == 'feast-dev/feast'
-    runs-on: ubuntu-latest
-    needs: [build_wheels]
-    steps:
-      - uses: actions/[email protected]
-        with:
-          name: python-wheels
-          path: dist
-      - uses: pypa/[email protected]
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI_PASSWORD }}
+      custom_version: ${{ github.event.inputs.custom_version }}
+      token: ${{ github.event.inputs.token }}
 
   publish-java-sdk:
     if: github.repository == 'feast-dev/feast'
     container: maven:3.6-jdk-11
     runs-on: ubuntu-latest
-    needs: get-version
+    needs: [ get-version, publish-python-sdk ]
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/publish_helm_charts.yml

@@ -0,0 +1,50 @@
+name: publish images
+
+on:
+  workflow_dispatch: # Allows manual trigger of the workflow
+    inputs:
+      custom_version: # Optional input for a custom version
+        description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing'
+        required: false
+      token:
+        description: 'Personal Access Token'
+        required: true
+        default: ""
+        type: string
+
+jobs:
+  get-version:
+    uses: ./.github/workflows/get_semantic_release_version.yaml
+    with:
+      custom_version: ${{ github.event.inputs.custom_version }}
+      token: ${{ github.event.inputs.token }}
+
+  publish-helm-charts:
+    if: github.repository == 'feast-dev/feast'
+    runs-on: ubuntu-latest
+    needs: get-version
+    env:
+      HELM_VERSION: v3.8.0
+      VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Authenticate to Google Cloud
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: '${{ secrets.GCP_SA_KEY }}'
+      - name: Set up gcloud SDK
+        uses: google-github-actions/setup-gcloud@v1
+        with:
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+      - run: gcloud auth configure-docker --quiet
+      - name: Remove previous Helm
+        run: sudo rm -rf $(which helm)
+      - name: Install Helm
+        run: ./infra/scripts/helm/install-helm.sh
+      - name: Validate Helm chart prior to publishing
+        run: ./infra/scripts/helm/validate-helm-chart-publish.sh
+      - name: Validate all version consistency
+        run: ./infra/scripts/helm/validate-helm-chart-versions.sh $VERSION_WITHOUT_PREFIX
+      - name: Publish Helm charts
+        run: ./infra/scripts/helm/push-helm-charts.sh $VERSION_WITHOUT_PREFIX
+