Skip to content

Commit

Permalink
chore: update metric reporting canaries
Browse files Browse the repository at this point in the history
  • Loading branch information
kyle committed Aug 11, 2023
1 parent ba4377c commit 38fdb15
Show file tree
Hide file tree
Showing 12 changed files with 437 additions and 81 deletions.
118 changes: 107 additions & 11 deletions .github/composite_actions/log_metric/action.yaml
Original file line number Diff line number Diff line change
@@ -1,37 +1,133 @@
name: Log Metric
description: Log data point to a metric with the provided value. If the metric is not there, it will create one
description: Log data point to a metric with the provided value. If the metric is not there, it will create one.
# To avoid 'Credentials could not be loaded' calling workflows must include:
# permissions:
# id-token: write
# contents: read
inputs:
aws-region:
required: true
description: The AWS region
description: The AWS region.
role-to-assume:
required: true
description: The role to assume in the STS session
metric-name:
description: Name of the metric to track in Cloudwatch.
description: The role to assume in the STS session.
github-token:
required: true
value:
description: Github token for requesting failing steps

#value:
# Why we publish value 0 on success: https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#publishingZero
description: Value of the metric to track in Cloudwatch.
# description: Value of the metric to track in Cloudwatch.
# required: true

job-status:
description: Job status
required: true
job-identifier:
desciption: Identifiers for the job
required: true



# Global Metric Dimensions
testType:
description: canary, integration, unit testType.
required: true
category:
description: analytics, api, authenticator, etc.
required: true
dimensions:
description: Dimensions of metric to track in Cloudwatch, in format dimensionName1=value,dimensionName2=value,...
workflowName:
description: The Github Action workflow.yaml file name. ie "AmplifyCanaries".
required: true
# failingStep: Look into getting via code


# FlutterDart Workflows Metric Dimensions
framework:
description: flutter, dart.
required: false
flutterDartChannel:
description: beta, stable.
required: false
dartVersion:
description: 3, 2.19, 2.18, etc.
required: false
flutterVersion:
description: 3.10.6, 3.10.5, etc.
required: false
dartCompiler:
description: dart2js, ddc, dart, dart2wasm.
required: false

# Platform Workflows Metric Dimensions
platform:
description: android, ios, web, linux, windows.
required: false
platformVersion:
description: ios-14.5, ios-16, android-25-x86, etc.
required: false

runs:
using: 'composite'
steps:
#- name: Exit if not scheduled
# shell: bash
# run: |
# if [ "${{ github.event_name }}" != "schedule" ]; then
# echo "This was not triggered by a schedule, exiting."
# exit 1
# fi

- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 # 1.7.0
with:
role-to-assume: ${{ inputs.role-to-assume }}
aws-region: ${{ inputs.aws-region }}
role-duration-seconds: 900

- name: Get Job ID
run: |
substring="${{ inputs.job-identifier }}"
jobs_json=$(curl -s -H "Authorization: token ${{ inputs.github-token }}" \
-H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/fjnoyp/amplify-flutter/actions/runs/${{ github.run_id }}/jobs")
job_id=$(echo "$jobs_json" | jq --arg substring "$substring" '.jobs[] | select(.name | contains($substring)) | .id')
echo "substring is $substring"
echo "job json is $jobs_json"
echo "github run id is ${{ github.run_id}}"
echo "Job ID for job containing $substring is $job_id"
echo "job_id=$job_id" >> $GITHUB_ENV
shell: bash

- name: Get Failing Step
if: ${{ job.status == 'failure' }}
shell: bash
run: |
response=$(curl -s -H "Authorization: token ${{ inputs.github-token }}" \
-H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/fjnoyp/amplify-flutter/actions/jobs/${{ env.job_id }}")
failing_step_name=$(echo "$response" | jq -r '.steps[] | select(.conclusion == "failure") | .name')
echo "FAILING_STEP=$failing_step_name" >> $GITHUB_ENV
- name: Run Dart script
# Run a Dart script to put metric data.
run: dart ./tool/send_metric_data.dart ${{ inputs.metric-name }} ${{ inputs.value }} ${{ inputs.dimensions }}
shell: bash
run: dart ./tool/send_metric_data.dart \
github_metric_1.0 \
${{ job.status == 'failure' }} \
${{ inputs.testType }} \
${{ inputs.category }} \
${{ inputs.workflowName }} \
${{ inputs.framework }} \
${{ inputs.flutterDartChannel }} \
${{ inputs.dartVersion }} \
${{ inputs.flutterVersion }} \
${{ inputs.dartCompiler }} \
${{ inputs.platform }} \
${{ inputs.platformVersion }} \
${{ env.FAILING_STEP }}
shell: bash
89 changes: 43 additions & 46 deletions .github/workflows/amplify_canaries.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ jobs:
- channel: "stable"
flutter-version: "3.10.1"
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # 3.5.3
- name: Git Checkout
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # 3.5.3
with:
persist-credentials: false

Expand All @@ -53,24 +54,21 @@ jobs:
- name: Build Canary (Android)
run: build-support/build_canary.sh apk

- name: Log failing builds
if: ${{ failure() }}
uses: ./.github/composite_actions/log_metric
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: ${{ secrets.AWS_REGION }}
metric-name: BuildCanaryTestFailure
value: 1
dimensions: channel=${{ matrix.channel }}
- name: Log succeeding builds
if: ${{ success() }}
- name: Log success/failure
if: always()
uses: ./.github/composite_actions/log_metric
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: ${{ secrets.AWS_REGION }}
metric-name: BuildCanaryTestFailure
value: 0
dimensions: channel=${{ matrix.channel }}
aws-region: ${{ secrets.AWS_REGION }}
github-token: ${{ secrets.GH_TOKEN }}
job-status: ${{ job.status }}
job-identifier: ${{ matrix.channel }}, ${{ matrix.flutter-version}}
testType: canary
category: all
workflowName: amplify_canaries/build
framework: flutter
flutterDartChannel: ${{ matrix.channel }}
flutterVersion: ${{ matrix.flutter-version }}

e2e-android:
runs-on: macos-latest
Expand Down Expand Up @@ -127,24 +125,22 @@ jobs:
# Perform a build to reduce startup time of `flutter test` and prevent timeout
script: cd canaries && flutter build apk --debug && flutter test -d emulator-5554 integration_test/main_test.dart

- name: Log failing android runs
if: ${{ failure() }}
uses: ./.github/composite_actions/log_metric
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: ${{ secrets.AWS_REGION }}
metric-name: E2ECanaryTestFailure
value: 1
dimensions: channel=${{ matrix.channel }},platform=android
- name: Log succeeding android runs
if: ${{ success() }}
- name: Log success/failure
if: always()
uses: ./.github/composite_actions/log_metric
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: ${{ secrets.AWS_REGION }}
metric-name: E2ECanaryTestFailure
value: 0
dimensions: channel=${{ matrix.channel }},platform=android
aws-region: ${{ secrets.AWS_REGION }}
github-token: ${{ secrets.GH_TOKEN }}
job-status: ${{ job.status }}
job-identifier: ${{ matrix.channel }}, ${{ matrix.flutter-version}}
testType: canary
category: all
workflowName: amplify_canaries/e2e-android
framework: flutter
platform: android
flutterDartChannel: ${{ matrix.channel }}
flutterVersion: ${{ matrix.flutter-version }}

e2e-ios:
runs-on: macos-13
Expand Down Expand Up @@ -205,21 +201,22 @@ jobs:
flutter build ios --simulator --target=integration_test/main_test.dart
flutter test -d test integration_test/main_test.dart --verbose
- name: Log failing ios runs
if: ${{ failure() }}
uses: ./.github/composite_actions/log_metric
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: ${{ secrets.AWS_REGION }}
metric-name: E2ECanaryTestFailure
value: 1
dimensions: channel=${{ matrix.channel }},platform=ios
- name: Log succeeding ios runs
if: ${{ success() }}
- name: Log success/failure
if: always()
uses: ./.github/composite_actions/log_metric
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: ${{ secrets.AWS_REGION }}
metric-name: E2ECanaryTestFailure
value: 0
dimensions: channel=${{ matrix.channel }},platform=ios
aws-region: ${{ secrets.AWS_REGION }}
github-token: ${{ secrets.GH_TOKEN }}
job-status: ${{ job.status }}
job-identifier: ${{ matrix.channel }}, ${{ matrix.flutter-version}}
testType: canary
category: all
workflowName: amplify_canaries/e2e-ios
framework: flutter
platform: ios
platformVersion: ${{ matrix.ios-version }}
flutterDartChannel: ${{ matrix.channel }}
flutterVersion: ${{ matrix.flutter-version }}


84 changes: 83 additions & 1 deletion .github/workflows/amplify_integration_tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,23 @@ jobs:
arch: ${{ matrix.arch }}
script: aft exec --include=${{ matrix.scope }} -- small=true "<AFT_ROOT>/build-support/integ_test.sh" -d emulator-5554 --retries 1

- name: Log success/failure
if: always()
uses: ./.github/composite_actions/log_metric
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: ${{ secrets.AWS_REGION }}
github-token: ${{ secrets.GH_TOKEN }}
job-status: ${{ job.status }}
job-identifier: ${{ matrix.scope }}, ${{ matrix.api-level}}, ${{ matrix.target }}
testType: integration
category: ${{ matrix.scope }}
workflowName: ${{ matrix.scope }}/amplify_integration_tests
framework: flutter
flutterDartChannel: stable
platform: android
platformVersion: ${{ matrix.api-level }} ${{ matrix.arch }}

ios:
runs-on: macos-latest
# These permissions are needed to interact with GitHub's OIDC Token endpoint.
Expand Down Expand Up @@ -124,7 +141,23 @@ jobs:

- name: Run integration tests
timeout-minutes: 60
run: aft exec --include=${{ matrix.scope }} -- small=true "<AFT_ROOT>/build-support/integ_test_ios.sh" -d test
run: aft exec --include=${{ matrix.scope }} -- small=true "<AFT_ROOT>/build-support/integ_test_ios.sh" -d test

- name: Log success/failure
if: always()
uses: ./.github/composite_actions/log_metric
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: ${{ secrets.AWS_REGION }}
github-token: ${{ secrets.GH_TOKEN }}
job-status: ${{ job.status }}
job-identifier: ${{ matrix.scope }}
testType: integration
category: ${{ matrix.scope }}
workflowName: ${{ matrix.scope }}/amplify_integration_tests
framework: flutter
flutterDartChannel: stable
platform: ios

web:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -171,6 +204,23 @@ jobs:
chromedriver --port=4444 &
aft exec --include=${{ matrix.scope }} -- "<AFT_ROOT>/build-support/integ_test.sh" -d web-server
- name: Log success/failure
if: always()
uses: ./.github/composite_actions/log_metric
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: ${{ secrets.AWS_REGION }}
github-token: ${{ secrets.GH_TOKEN }}
job-status: ${{ job.status }}
job-identifier: ${{ matrix.scope }}
testType: integration
category: ${{ matrix.scope }}
workflowName: ${{ matrix.scope }}/amplify_integration_tests
framework: flutter
flutterDartChannel: stable
platform: web
platformVersion: chrome

# TODO(dnys1): Re-enable with self-hosted runners
# macos:
# runs-on: macos-latest
Expand Down Expand Up @@ -257,6 +307,22 @@ jobs:
flutter config --enable-linux-desktop
aft exec --include=${{ matrix.scope }} -- "<AFT_ROOT>/build-support/integ_test.sh" -d linux
- name: Log success/failure
if: always()
uses: ./.github/composite_actions/log_metric
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: ${{ secrets.AWS_REGION }}
github-token: ${{ secrets.GH_TOKEN }}
job-status: ${{ job.status }}
job-identifier: ${{ matrix.scope }}
testType: integration
category: ${{ matrix.scope }}
workflowName: ${{ matrix.scope }}/amplify_integration_tests
framework: flutter
flutterDartChannel: stable
platform: linux

windows:
runs-on: windows-latest
# These permissions are needed to interact with GitHub's OIDC Token endpoint.
Expand Down Expand Up @@ -301,3 +367,19 @@ jobs:
run: |
flutter config --enable-windows-desktop
dart pub global run aft exec --include=${{ matrix.scope }} -- flutter test integration_test/main_test.dart -d windows
- name: Log success/failure
if: always()
uses: ./.github/composite_actions/log_metric
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: ${{ secrets.AWS_REGION }}
github-token: ${{ secrets.GH_TOKEN }}
job-status: ${{ job.status }}
job-identifier: ${{ matrix.scope }}
testType: integration
category: ${{ matrix.scope }}
workflowName: ${{ matrix.scope }}/amplify_integration_tests
framework: flutter
flutterDartChannel: stable
platform: windows
Loading

0 comments on commit 38fdb15

Please sign in to comment.