release-tests.yml

name: Release
on:
  pull_request:
    paths:
      - 'auto/**'
  push:
    paths:
      - 'auto/**'
    tags:
      - 'v*'
jobs:
  test-controller-api:
    runs-on: ubuntu-latest
    outputs:
      envfiles: ${{ steps.params.outputs.envfiles }}
      pump: ${{ steps.params.outputs.pump }}
      sink: ${{ steps.params.outputs.sink }}
    steps:
      - name: set params
        id: params
        shell: bash
        env:
          # Cover pull_request_target too
          BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref}}
        run: |
          set -eo pipefail
          endpoint="http://tui.internal.dev.tyk.technology/api/tyk-ci/$BASE_REF/${{ github.event_name}}/api"
          curl="curl -s --retry 5 --retry-delay 10 --fail-with-body --retry-all-errors"
          echo "pump<<EOF
          $($curl ${endpoint}/Pump)
          EOF
          sink<<EOF
          $($curl ${endpoint}/Sink)
          EOF
          envfiles<<EOF
          $($curl ${endpoint}/EnvFiles)
          EOF" | tee -a "$GITHUB_OUTPUT"
  api-tests:
    needs:
      - test-controller-api
    runs-on: ubuntu-latest
    env:
      METADATA_REPORT_PATH: /tmp/metadata.toml
      XUNIT_REPORT_PATH: ${{ github.workspace }}/reports/pytest-report.xml
    permissions:
      id-token: write # This is required for requesting the Github JWT
      contents: read # This is required for actions/checkout
    strategy:
      fail-fast: false
      matrix:
        envfiles: ${{ fromJson(needs.test-controller-api.outputs.envfiles) }}
        pump: ${{ fromJson(needs.test-controller-api.outputs.pump) }}
        sink: ${{ fromJson(needs.test-controller-api.outputs.sink) }}
        include:
          - db: postgres15
            markers: "and not sql"
        exclude:
          - pump: tykio/tyk-pump-docker-pub:v1.8
            sink: $ECR/tyk-sink:master
          - pump: $ECR/tyk-pump:master
            sink: tykio/tyk-mdcb-docker:v2.4
    steps:
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
          role-session-name: cipush
          aws-region: eu-central-1
      - id: ecr
        uses: aws-actions/amazon-ecr-login@v2
        with:
          mask-password: 'true'
          # Only ${{ github.actor }} has access
          # See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys
      - name: Setup tmate session only in debug mode
        uses: mxschmitt/action-tmate@v3
        if: runner.debug == '1'
        with:
          detached: true
          limit-access-to-actor: true
      - name: fetch env from tyk-ci
        env:
          GH_TOKEN: ${{ github.token }}
        uses: actions/checkout@v4
        with:
          sparse-checkout: |
            auto
            reports
      - name: env up
        shell: bash
        working-directory: auto
        id: env_up
        env:
          pull_policy: 'if_not_present'
          GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
          TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
          TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
          BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref}}
        run: |
          tags=(${{steps.ecr.outputs.registry}}/$REPO:$BASE_REF)
          docker run -q --rm -v ~/.docker/config.json:/root/.docker/config.json tykio/gromit policy match ${tags[0]} 2>versions.env
          echo '# alfa and beta have to come after the override
          tyk_alfa_image=$tyk_image
          tyk_beta_image=$tyk_image
          ECR=${{steps.ecr.outputs.registry}}
          tyk_pump_image=${{matrix.pump}}
          tyk_sink_image=${{matrix.sink}}
          confs_dir=./pro-ha
          env_file=local-${{ matrix.envfiles.db }}.env' >> versions.env
          echo "::group::versions"
          cat versions.env
          echo "::endgroup::"
          # Add Tyk component config variations to $env_file
          cat confs/${{ matrix.envfiles.config }}.env >> local-${{ matrix.envfiles.db }}.env
          # bring up env, the project name is important
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml  --env-file versions.env --profile master-datacenter up --quiet-pull -d
          ./dash-bootstrap.sh http://localhost:3000
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml  --env-file versions.env --profile slave-datacenter up --quiet-pull -d
      - name: Run tests
        working-directory: auto
        id: test_execution
        env:
          # Cover pull_request_target too
          BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref}}
        run: |
          # Generate report id
          echo "id=$(date +%s%N)" >> $GITHUB_OUTPUT
          # Run tests
          set -o pipefail
          echo "### API tests ${{ matrix.envfiles.db }} ${{ matrix.envfiles.conf }}" >> $GITHUB_STEP_SUMMARY
          if docker run --rm --network auto_default --env-file pytest.env -v ${{ github.workspace }}/reports:/app/reports \
            ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:$BASE_REF \
            pytest -c pytest_ci.ini --junitxml=./${XUNIT_REPORT_PATH#"${{ github.workspace }}"} --ci -m "not local and not dind ${{ matrix.markers }}" | tee tests.out; then
              echo "All tests passed!" >> $GITHUB_STEP_SUMMARY
          else
            echo "::error title=API tests ${{ matrix.envfiles.db }} ${{ matrix.envfiles.conf }}::Test execution failed"
            cat tests.out >> $GITHUB_STEP_SUMMARY
            exit 1
          fi
      - name: Generate metadata
        if: always() && steps.test_execution.outcome != 'skipped'
        id: metadata_report
        env:
          BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref}}
          REPORT_NAME: ${{ github.repository }}_${{ github.run_id }}_${{ github.run_attempt }}-${{steps.test_execution.outputs.id}}
        run: |
          # Generate metadata report
          echo "[metadata]
          repo = ${{ github.repository }}
          branch = ${{ github.ref }}
          commit = ${{ github.sha }}
          test_suite_version = $BASE_REF
          test_suite_name = ${{ github.job }}
          test_suite_run = ${{ github.run_id }}-${{ github.run_attempt }}
          db = ${{ matrix.envfiles.db }}
          conf = ${{ matrix.envfiles.config }}
          cache = ${{ matrix.envfiles.cache }}
          pump_compatibility = ${{ matrix.pump }}
          sink_compatibility = ${{ matrix.sink }}
          " > ${METADATA_REPORT_PATH}
          # Print metadata report
          if [[ "${{ runner.debug }}" == "1" ]]; then
            echo "::group::metadata report"
            cat ${METADATA_REPORT_PATH}
            echo "::endgroup::"
          fi
          #Upload xunit report
          aws s3 cp ${XUNIT_REPORT_PATH}  s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.xml
          #Upload metadata report
          aws s3 cp ${METADATA_REPORT_PATH} s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.metadata.toml
      - name: Docker logs for all components
        if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
        working-directory: auto
        env:
          pull_policy: 'if_not_present'
          GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
          TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
          TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
          ECR: ${{ steps.ecr.outputs.registry }}
        run: |
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml  --env-file versions.env --profile all logs | sort > ${{ github.workspace }}/docker-compose.log
          echo "::group::DockerLogs"
          cat ${{ github.workspace }}/docker-compose.log
          echo "::endgroup::"
      - name: Upload Artifact
        uses: actions/upload-artifact@v4
        if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
        with:
          name: docker-compose-logs-${{ github.job }}-${{ matrix.envfiles.db }}-${{ matrix.envfiles.conf }}-${{ github.run_id }}
          path: ${{ github.workspace }}/docker-compose.log
          retention-days: 3
          overwrite: true
      - name: Archive Integration tests report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: api-test-report-${{ matrix.envfiles.db }}-${{ matrix.envfiles.conf }}-${{ github.run_id }}
          retention-days: 3
          path: ${{ github.workspace }}/reports
          overwrite: true
  test-controller-ui:
    runs-on: ubuntu-latest
    outputs:
      envfiles: ${{ steps.params.outputs.envfiles }}
      pump: ${{ steps.params.outputs.pump }}
      sink: ${{ steps.params.outputs.sink }}
    steps:
      - name: set params
        id: params
        shell: bash
        env:
          # Cover pull_request_target too
          BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref}}
        run: |
          set -eo pipefail
          endpoint="http://tui.internal.dev.tyk.technology/api/tyk-ci/$BASE_REF/${{ github.event_name}}/ui"
          curl="curl -s --retry 5 --retry-delay 10 --fail-with-body --retry-all-errors"
          echo "pump<<EOF
          $($curl ${endpoint}/Pump)
          EOF
          sink<<EOF
          $($curl ${endpoint}/Sink)
          EOF
          envfiles<<EOF
          $($curl ${endpoint}/EnvFiles)
          EOF" | tee -a "$GITHUB_OUTPUT"
  ui-tests:
    needs:
      - test-controller-ui
    runs-on: ubuntu-latest
    env:
      METADATA_REPORT_PATH: /tmp/metadata.toml
      XUNIT_REPORT_PATH: ${{ github.workspace }}/tyk-analytics/tests/ui/playwright-report/results.xml
    permissions:
      id-token: write # This is required for requesting the Github JWT
      contents: read # This is required for actions/checkout
    strategy:
      fail-fast: false
      matrix:
        envfiles: ${{ fromJson(needs.test-controller-ui.outputs.envfiles) }}
        pump: ${{ fromJson(needs.test-controller-ui.outputs.pump) }}
        sink: ${{ fromJson(needs.test-controller-ui.outputs.sink) }}
        include:
          - db: postgres15
            markers: "and not sql"
        exclude:
          - pump: tykio/tyk-pump-docker-pub:v1.8
            sink: $ECR/tyk-sink:master
          - pump: $ECR/tyk-pump:master
            sink: tykio/tyk-mdcb-docker:v2.4
    steps:
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
          role-session-name: cipush
          aws-region: eu-central-1
      - id: ecr
        uses: aws-actions/amazon-ecr-login@v2
        with:
          mask-password: 'true'
          # Only ${{ github.actor }} has access
          # See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys
      - name: Setup tmate session only in debug mode
        uses: mxschmitt/action-tmate@v3
        if: runner.debug == '1'
        with:
          detached: true
          limit-access-to-actor: true
      - name: fetch env from tyk-ci
        env:
          GH_TOKEN: ${{ github.token }}
        uses: actions/checkout@v4
        with:
          sparse-checkout: |
            auto
            reports
      - name: Checkout of Dashboard
        uses: actions/checkout@v4
        with:
          repository: TykTechnologies/tyk-analytics
          path: tyk-analytics
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 1
          token: ${{ secrets.ORG_GH_TOKEN }}
          sparse-checkout: |
            tests/ui
            bin
          submodules: false
      - name: env up
        shell: bash
        working-directory: auto
        id: env_up
        env:
          pull_policy: 'if_not_present'
          GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
          TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
          TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
          BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref}}
        run: |
          tags=(${{steps.ecr.outputs.registry}}/$REPO:$BASE_REF)
          docker run -q --rm -v ~/.docker/config.json:/root/.docker/config.json tykio/gromit policy match ${tags[0]} 2>versions.env
          echo '# alfa and beta have to come after the override
          tyk_alfa_image=$tyk_image
          tyk_beta_image=$tyk_image
          ECR=${{steps.ecr.outputs.registry}}
          tyk_pump_image=${{matrix.pump}}
          tyk_sink_image=${{matrix.sink}}
          confs_dir=./pro-ha
          env_file=local-${{ matrix.envfiles.db }}.env' >> versions.env
          echo "::group::versions"
          cat versions.env
          echo "::endgroup::"
          # Add Tyk component config variations to $env_file
          cat confs/${{ matrix.envfiles.config }}.env >> local-${{ matrix.envfiles.db }}.env
          # bring up env, the project name is important
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml  --env-file versions.env --profile master-datacenter up --quiet-pull -d
          ./dash-bootstrap.sh http://localhost:3000
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml  --env-file versions.env --profile slave-datacenter up --quiet-pull -d
      - name: Install Node.js 18.16
        uses: actions/setup-node@v2.4.1
        with:
          node-version: "18.16"
      - name: Fix private module deps
        env:
          TOKEN: '${{ secrets.ORG_GH_TOKEN }}'
        run: "git config --global url.\"https://${TOKEN}@github.com\".insteadOf \"https://github.com\"                    \n"
      - name: Install test dependecies
        run: |
          npm ci
        working-directory: tyk-analytics/tests/ui
      - name: Install Playwright Browsers
        run: npx playwright install --with-deps chromium
        working-directory: tyk-analytics/tests/ui
      - name: Execute UI tests
        id: test_execution
        run: |
          # Generate report id
          echo "id=$(date +%s%N)" >> $GITHUB_OUTPUT
          npm run test -- --workers=3
        working-directory: tyk-analytics/tests/ui
        env:
          GW_URL: 'https://localhost:8080/'
          NODE_TLS_REJECT_UNAUTHORIZED: 0
          SLACK_AUTH_TOKEN: ${{ secrets.UI_SLACK_AUTH_TOKEN }}
          EVENT_TRIGGER: ${{ github.event_name }} (${{ github.actor }})
          JOB_RUN_ID: ${{ github.run_id }}
          JOB_NAME: tyk-analytics - ${{ github.ref }} ${{ github.event.head_commit.message }}
          RUN_ID: 'tyk-analytics/${{ github.run_id }}'
      - name: Upload Playwright Test Report to S3
        if: failure() && steps.test_execution.outcome != 'success' && steps.env_up.outcome == 'success'
        run: npm run upload_report_to_s3
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.UI_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.UI_AWS_SECRET_ACCESS_KEY }}
          RUN_ID: 'tyk-analytics/${{ github.run_id }}'
        working-directory: tyk-analytics/tests/ui
      - name: Post message to automation team
        if: failure() && steps.test_execution.outcome != 'success' && !github.event.pull_request.draft
        id: slack
        uses: slackapi/slack-github-action@v1.24.0
        with:
          channel-id: '@konrad'
          slack-message: "GitHub build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}"
        env:
          SLACK_BOT_TOKEN: ${{ secrets.UI_SLACK_AUTH_TOKEN }}
      - name: Share S3 report link into summary
        if: always() && !github.event.pull_request.draft
        run: |
          echo "# :clipboard: S3 ui Test REPORT: ${{ matrix.envfiles.db }}-${{ matrix.envfiles.conf }}" >> $GITHUB_STEP_SUMMARY
          echo "- Status: ${{ steps.test_execution.outcome == 'success' && ':white_check_mark:' || ':no_entry_sign:' }}" >> $GITHUB_STEP_SUMMARY
          echo "- [Link to report](https://tyk-qa-reports.s3.eu-central-1.amazonaws.com/tyk-analytics/${{ github.run_id }}/index.html)" >> $GITHUB_STEP_SUMMARY
      - name: Xray update
        working-directory: tyk-analytics/bin
        if: ${{ always() }} && github.event_name != 'pull_request'
        run: |
          ./update_xray.sh
        env:
          TEST: "QA-890"
          STATUS: "${{ steps.test_execution.outcome }}"
          CLIENT_ID: ${{secrets.XRAY_CLIENT_ID}}
          CLIENT_SECRET: ${{secrets.XRAY_CLIENT_SECRET}}
          BRANCH: ${{ github.ref }}
      - name: Generate metadata
        if: always() && steps.test_execution.outcome != 'skipped'
        id: metadata_report
        env:
          BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref}}
          REPORT_NAME: ${{ github.repository }}_${{ github.run_id }}_${{ github.run_attempt }}-${{steps.test_execution.outputs.id}}
        run: |
          # Generate metadata report
          echo "[metadata]
          repo = ${{ github.repository }}
          branch = ${{ github.ref }}
          commit = ${{ github.sha }}
          test_suite_version = $BASE_REF
          test_suite_name = ${{ github.job }}
          test_suite_run = ${{ github.run_id }}-${{ github.run_attempt }}
          db = ${{ matrix.envfiles.db }}
          conf = ${{ matrix.envfiles.config }}
          cache = ${{ matrix.envfiles.cache }}
          pump_compatibility = ${{ matrix.pump }}
          sink_compatibility = ${{ matrix.sink }}
          " > ${METADATA_REPORT_PATH}
          # Print metadata report
          if [[ "${{ runner.debug }}" == "1" ]]; then
            echo "::group::metadata report"
            cat ${METADATA_REPORT_PATH}
            echo "::endgroup::"
          fi
          #Upload xunit report
          aws s3 cp ${XUNIT_REPORT_PATH}  s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.xml
          #Upload metadata report
          aws s3 cp ${METADATA_REPORT_PATH} s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.metadata.toml
      - name: Docker logs for all components
        if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
        working-directory: auto
        env:
          pull_policy: 'if_not_present'
          GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
          TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
          TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
          ECR: ${{ steps.ecr.outputs.registry }}
        run: |
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml  --env-file versions.env --profile all logs | sort > ${{ github.workspace }}/docker-compose.log
          echo "::group::DockerLogs"
          cat ${{ github.workspace }}/docker-compose.log
          echo "::endgroup::"
      - name: Upload Artifact
        uses: actions/upload-artifact@v4
        if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
        with:
          name: docker-compose-logs-${{ github.job }}-${{ matrix.envfiles.db }}-${{ matrix.envfiles.conf }}-${{ github.run_id }}
          path: ${{ github.workspace }}/docker-compose.log
          retention-days: 3
          overwrite: true
      - name: Archive Integration tests report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: api-test-report-${{ matrix.envfiles.db }}-${{ matrix.envfiles.conf }}-${{ github.run_id }}
          retention-days: 3
          path: ${{ github.workspace }}/reports
          overwrite: true
  release:
    if: github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    needs: [api-tests, ui-tests]
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v3
      - name: CI env
        run: tar czf ci-env.tgz auto/
      - uses: softprops/action-gh-release@v1
        with:
          token: ${{ secrets.ORG_GH_TOKEN }}
          name: ${{ github.ref_name }}
          tag_name: ${{ github.ref_name }}
          body_path: auto/release.md
          files: ci-env.tgz