e2e-perf.yaml

name: Test - E2E - Performance # e2e tests running against a k8s cluster

on:
  pull_request:
    branches:
      - main
    types:
      - opened
      - reopened
      - synchronize
      - closed

env:
  N_SIMUL_PERF_DATA_FILE: n-simul-perf-data.json
  E2E_UTILS: ./hack/e2e/perf/scripts/utils.sh

permissions:
  contents: write
  pull-requests: write
  id-token: write

jobs:
  build-operator:
    uses: ./.github/workflows/build-operator.yaml
    name: Build Operator Image
    secrets: inherit

  perf-test-minikube:
    needs:
    - build-operator
    name: Performance Test - Minikube
    runs-on: ubuntu-latest
    strategy:
      matrix:
        git_branch:
          - main
          - PR
    steps:
      - name: Checkout repo
        uses: actions/checkout@v3
        with:
          ref: ${{ matrix.git_branch == 'main' && 'main' || github.head_ref }}

      - name: Checkout hack from PR
        uses: actions/checkout@v3
        with:
          ref: ${{ github.head_ref }}
          path: pr-temp

      - name: Copy hack outside of pr-temp
        run: cp -r pr-temp/hack/* hack

#      - name: List contents
#        run: |
#          rm -rf pr-temp
#          ls -R ./

      # Identify comment to be updated
      - name: Find comment for Performance Overview
        uses: peter-evans/find-comment@v2
        id: find-comment
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: "github-actions[bot]"
          body-includes: Performance Overview
          direction: last

      # Create/Update comment with action deployment status
      - name: Create or Update Comment with Performance Overview
        id: notification
        uses: peter-evans/create-or-update-comment@v2
        with:
          comment-id: ${{ steps.find-comment.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body: |
            ## Operator Performance Overview

            :gear: Running performance tests on Minikube

          edit-mode: replace

      - name: install yq
        run: sudo snap install yq

      - name: Download Helm Values Artifact
        uses: actions/download-artifact@v3
        with:
          name: helm-values

      - name: Start Minikube
        run: minikube start --addons default-storageclass,storage-provisioner --driver=docker

      - name: Install Uffizzi Cluster Operator
        id: prev
        run: |
          helm dep update ./chart
          helm upgrade --install --wait pr-${{ github.event.pull_request.number }} \
          ./chart -f helm-values.yaml

      - name: Create PERF_DATA_FILE
        run: |
          # Ensure the file exists and initialize it as an empty array if not
          if [ ! -f "$PERF_DATA_FILE" ]; then
            echo '[]' > "$PERF_DATA_FILE"
          fi

      - name: Time taken to create UffizziClusters with different numbers of workers
        run: |
          n_simultaneous_clusters=3
          for n_workers in $(seq 1 5 31); do
            # update concurrent workers
            yq -i '.concurrent = "$n_workers"' helm-values.yaml
          
            # upgrade helm chart 
            helm upgrade --install --wait pr-${{ github.event.pull_request.number }} \
            ./chart -f helm-values.yaml

            # time taken to create n clusters simultaneously
            time=$(bash hack/e2e/perf/01-multicluster.sh $n_simultaneous_clusters)
            bash $E2E_UTILS update_json_with_workers_and_time $n_workers $time $PERF_DATA_FILE
          done
          
          cat $PERF_DATA_FILE

#      - name: dummy data
#        shell: bash
#        run: |
#          if [[ "${{ github.ref_name }}" == "main" ]]; then
#            factor=2
#          else
#            factor=1
#          fi
#          echo '[
#            {"workers": 5, "time": '$((50 * factor))'},
#            {"workers": 10, "time": '$((100 * factor))'},
#            {"workers": 15, "time": '$((150 * factor))'},
#            {"workers": 20, "time": '$((200 * factor))'},
#            {"workers": 25, "time": '$((250 * factor))'},
#            {"workers": 30, "time": '$((300 * factor))'}
#           ]' > $PERF_DATA_FILE
          
      - name: Rename and update PERF_DATA_FILE
        run: |
          NEW_NAME="${PERF_DATA_FILE%.json}-${{ matrix.git_branch }}.json"
          mv $PERF_DATA_FILE $NEW_NAME
          echo "PERF_DATA_FILE=$NEW_NAME" >> $GITHUB_ENV

      - name: Upload PERF_DATA_FILE artifact
        uses: actions/upload-artifact@v2
        with:
          name: perf-data-${{ matrix.git_branch }}
          path: ${{ env.PERF_DATA_FILE }}

  performance-overview:
    name: Collate performance data and post overview
    needs:
      - perf-test-minikube
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      # Identify comment to be updated
      - name: Find comment for Performance Overview
        uses: peter-evans/find-comment@v2
        id: find-comment
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: "github-actions[bot]"
          body-includes: Performance Overview
          direction: last

      - name: Download perf data - PR
        uses: actions/download-artifact@v2
        with:
          name: perf-data-PR

      - name: Download perf data - main
        uses: actions/download-artifact@v2
        with:
          name: perf-data-main

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.x'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install matplotlib

      - name: Create Graph
        run: python hack/e2e/perf/viz/generate_n_simul_graph.py

      - name: Upload the simultaneous cluster creation test image to Imgur
        id: upload_image
        uses: devicons/public-upload-to-imgur@v2.2.2
        with:
          path: ./simul_graph.png
          client_id: ${{ secrets.IMGUR_CLIENT_ID }}

      - name: Update Comment with Performance Overview
        uses: peter-evans/create-or-update-comment@v2
        with:
          comment-id: ${{ steps.find-comment.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body: |
            ## Operator Performance Overview
            
            ### Cold Start Performance Test
            
            This test would assess the duration and efficiency of initializing the first cluster, 
            which typically takes longer due to initial setup tasks such as pulling images and configuring the 
            environment. The test aims to capture the performance impact of these one-time operations to understand 
            the startup behavior and identify potential areas for optimization.
            
            ### Simultaneous cluster creation
            
            Creating simultaneously to check if operator is able to manage the load of creating multiple clusters
            at once. Creating multiple clusters simultaneously, in operator deployments; of varying number of concurrent
            workers help us understand how the operator manages high load in restricted environments as well.
            
            ![Graph](${{ fromJSON(steps.upload_image.outputs.imgur_urls)[0] }})
          edit-mode: replace