E2E Performance Tests #62
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Performance Tests | |
| on: | |
| workflow_call: | |
| inputs: | |
| PR_NUMBER: | |
| description: A PR number to run performance tests against. If the PR is already merged, the merge commit will be used. If not, the PR will be merged locally before running the performance tests. | |
| type: string | |
| required: true | |
| workflow_dispatch: | |
| inputs: | |
| PR_NUMBER: | |
| description: A PR number to run performance tests against. If the PR is already merged, the merge commit will be used. If not, the PR will be merged locally before running the performance tests. | |
| type: string | |
| required: true | |
| concurrency: | |
| group: ${{ github.ref == 'refs/heads/main' && format('{0}-{1}', github.ref, github.sha) || github.ref }}-e2e | |
| cancel-in-progress: true | |
| jobs: | |
| prep: | |
| runs-on: ubuntu-latest | |
| name: Find the baseline and delta refs, and check for an existing build artifact for that commit | |
| outputs: | |
| BASELINE_REF: ${{ steps.getBaselineRef.outputs.BASELINE_REF }} | |
| DELTA_REF: ${{ steps.getDeltaRef.outputs.DELTA_REF }} | |
| IS_PR_MERGED: ${{ steps.getPullRequestDetails.outputs.IS_MERGED }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Fetches the entire history | |
| - name: Determine "baseline ref" (prev merge commit) | |
| id: getBaselineRef | |
| run: | | |
| # Get the name of the current branch | |
| current_branch=$(git rev-parse --abbrev-ref HEAD) | |
| echo "$current_branch" | |
| if [ "$current_branch" = "main" ]; then | |
| # On the main branch, find the previous merge commit | |
| previous_merge=$(git rev-list --merges HEAD~1 | head -n 1) | |
| else | |
| # On a feature branch, find the common ancestor of the current branch and main | |
| previous_merge=$(git merge-base HEAD main) | |
| fi | |
| echo "$previous_merge" | |
| echo "BASELINE_REF=$previous_merge" >> "$GITHUB_OUTPUT" | |
| - name: Get pull request details | |
| id: getPullRequestDetails | |
| uses: ./.github/actions/javascript/getPullRequestDetails | |
| with: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| PULL_REQUEST_NUMBER: ${{ inputs.PR_NUMBER }} | |
| USER: ${{ github.actor }} | |
| - name: Determine "delta ref" | |
| id: getDeltaRef | |
| run: | | |
| if [ '${{ steps.getPullRequestDetails.outputs.IS_MERGED }}' == 'true' ]; then | |
| echo "DELTA_REF=${{ steps.getPullRequestDetails.outputs.MERGE_COMMIT_SHA }}" >> "$GITHUB_OUTPUT" | |
| else | |
| # Set dummy git credentials | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "Test" | |
| # Fetch head_ref of unmerged PR | |
| git fetch origin ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1 | |
| # Merge pull request locally and get merge commit sha | |
| git merge --allow-unrelated-histories -X ours --no-commit ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} | |
| git checkout ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} | |
| # Create and push a branch so it can be checked out in another runner | |
| git checkout -b e2eDelta-${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} | |
| git push origin e2eDelta-${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} | |
| echo "DELTA_REF=e2eDelta-${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}" >> "$GITHUB_OUTPUT" | |
| fi | |
| buildBaseline: | |
| name: Build apk from baseline | |
| uses: ./.github/workflows/buildAndroid.yml | |
| needs: prep | |
| secrets: inherit | |
| with: | |
| type: e2e | |
| ref: ${{ needs.prep.outputs.BASELINE_REF }} | |
| artifact-prefix: baseline-${{ needs.prep.outputs.BASELINE_REF }} | |
| buildDelta: | |
| name: Build apk from delta ref | |
| uses: ./.github/workflows/buildAndroid.yml | |
| needs: prep | |
| secrets: inherit | |
| with: | |
| type: e2eDelta | |
| ref: ${{ needs.prep.outputs.DELTA_REF }} | |
| artifact-prefix: delta-${{ needs.prep.outputs.DELTA_REF }} | |
| runTestsInAWS: | |
| runs-on: ubuntu-latest | |
| needs: [prep, buildBaseline, buildDelta] | |
| if: ${{ always() }} | |
| name: Run E2E tests in AWS device farm | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # The OS_BOTIFY_COMMIT_TOKEN is a personal access token tied to osbotify (we need a PAT to access the artifact API) | |
| token: ${{ secrets.OS_BOTIFY_COMMIT_TOKEN }} | |
| - name: Setup Node | |
| uses: ./.github/actions/composite/setupNode | |
| - name: Make zip directory for everything to send to AWS Device Farm | |
| run: mkdir zip | |
| - name: Download baseline APK | |
| uses: actions/download-artifact@v4 | |
| id: downloadBaselineAPK | |
| with: | |
| name: ${{ needs.buildBaseline.outputs.APK_ARTIFACT_NAME }} | |
| path: zip | |
| # The downloaded artifact will be a file named "app-e2e-release.apk" so we have to rename it | |
| - name: Rename baseline APK | |
| run: mv "${{ steps.downloadBaselineAPK.outputs.download-path }}/app-e2e-release.apk" "${{ steps.downloadBaselineAPK.outputs.download-path }}/app-e2eRelease.apk" | |
| - name: Download delta APK | |
| uses: actions/download-artifact@v4 | |
| id: downloadDeltaAPK | |
| with: | |
| name: ${{ needs.buildDelta.outputs.APK_ARTIFACT_NAME }} | |
| path: zip | |
| - name: Rename delta APK | |
| run: mv "${{ steps.downloadDeltaAPK.outputs.download-path }}/app-e2edelta-release.apk" "${{ steps.downloadDeltaAPK.outputs.download-path }}/app-e2edeltaRelease.apk" | |
| - name: Compile test runner to be executable in a nodeJS environment | |
| run: npm run e2e-test-runner-build | |
| - name: Copy e2e code into zip folder | |
| run: cp tests/e2e/dist/index.js zip/testRunner.ts | |
| - name: Copy profiler binaries into zip folder | |
| run: cp -r node_modules/@perf-profiler/android/cpp-profiler/bin zip/bin | |
| - name: Zip everything in the zip directory up | |
| run: zip -qr App.zip ./zip | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Schedule AWS Device Farm test run on main branch | |
| uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b | |
| id: schedule-awsdf-main | |
| with: | |
| name: App E2E Performance Regression Tests | |
| project_arn: ${{ secrets.AWS_PROJECT_ARN }} | |
| device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }} | |
| app_file: zip/app-e2eRelease.apk | |
| app_type: ANDROID_APP | |
| test_type: APPIUM_NODE | |
| test_package_file: App.zip | |
| test_package_type: APPIUM_NODE_TEST_PACKAGE | |
| test_spec_file: tests/e2e/TestSpec.yml | |
| test_spec_type: APPIUM_NODE_TEST_SPEC | |
| remote_src: false | |
| file_artifacts: | | |
| Customer Artifacts.zip | |
| Test spec output.txt | |
| log_artifacts: debug.log | |
| cleanup: true | |
| timeout: 7200 | |
| - name: Print logs if run failed | |
| if: failure() | |
| run: | | |
| echo ${{ steps.schedule-awsdf-main.outputs.data }} | |
| unzip "Customer Artifacts.zip" -d mainResults | |
| cat "./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/logcat.txt" || true | |
| cat ./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/debug.log || true | |
| cat "./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/Test spec output.txt" || true | |
| - name: Announce failed workflow in Slack | |
| if: failure() | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: custom | |
| custom_payload: | | |
| { | |
| channel: '#e2e-announce', | |
| attachments: [{ | |
| color: 'danger', | |
| text: `💥 ${process.env.AS_REPO} E2E Test run failed on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}> workflow 💥`, | |
| }] | |
| } | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} | |
| - name: Unzip AWS Device Farm results | |
| if: always() | |
| run: unzip "Customer Artifacts.zip" | |
| - name: Print AWS Device Farm run results | |
| if: always() | |
| run: cat "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md" | |
| - name: Check if test failed, if so post the results and add the DeployBlocker label | |
| id: checkIfRegressionDetected | |
| run: | | |
| if grep -q '🔴' "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md"; then | |
| # Create an output to the GH action that the test failed: | |
| echo "performanceRegressionDetected=true" >> "$GITHUB_OUTPUT" | |
| gh pr edit ${{ inputs.PR_NUMBER }} --add-label DeployBlockerCash | |
| gh pr comment ${{ inputs.PR_NUMBER }} -F "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md" | |
| gh pr comment ${{ inputs.PR_NUMBER }} -b "@Expensify/mobile-deployers 📣 Please look into this performance regression as it's a deploy blocker." | |
| else | |
| echo "performanceRegressionDetected=false" >> "$GITHUB_OUTPUT" | |
| echo '✅ no performance regression detected' | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| - name: Check if test has skipped tests | |
| id: checkIfSkippedTestsDetected | |
| run: | | |
| if grep -q '⚠️' "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md"; then | |
| # Create an output to the GH action that the tests were skipped: | |
| echo "skippedTestsDetected=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "skippedTestsDetected=false" >> "$GITHUB_OUTPUT" | |
| echo '✅ no skipped tests detected' | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| - name: 'Announce skipped tests in Slack' | |
| if: ${{ steps.checkIfSkippedTestsDetected.outputs.skippedTestsDetected == 'true' }} | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: custom | |
| custom_payload: | | |
| { | |
| channel: '#e2e-announce', | |
| attachments: [{ | |
| color: 'danger', | |
| text: `⚠️ ${process.env.AS_REPO} Some of E2E tests were skipped on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}> workflow ⚠️`, | |
| }] | |
| } | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} | |
| - name: 'Announce regression in Slack' | |
| if: ${{ steps.checkIfRegressionDetected.outputs.performanceRegressionDetected == 'true' }} | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: custom | |
| custom_payload: | | |
| { | |
| channel: '#quality', | |
| attachments: [{ | |
| color: 'danger', | |
| text: `🔴 Performance regression detected in PR ${{ inputs.PR_NUMBER }}\nDetected in <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}> workflow.`, | |
| }] | |
| } | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} | |
| cleanupDeltaRef: | |
| needs: [prep, runTestsInAWS] | |
| if: ${{ always() && needs.prep.outputs.IS_PR_MERGED != 'true' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Delete temporary merge branch created for delta ref | |
| run: git push -d origin ${{ needs.prep.outputs.DELTA_REF }} |