Skip to content

Commit 38fdb15

Browse files
author
kyle
committed
chore: update metric reporting canaries
1 parent ba4377c commit 38fdb15

12 files changed

+437
-81
lines changed

‎.github/composite_actions/log_metric/action.yaml

Lines changed: 107 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,133 @@
11
name: Log Metric
2-
description: Log data point to a metric with the provided value. If the metric is not there, it will create one
2+
description: Log data point to a metric with the provided value. If the metric is not there, it will create one.
33
# To avoid 'Credentials could not be loaded' calling workflows must include:
44
# permissions:
55
# id-token: write
66
# contents: read
77
inputs:
88
aws-region:
99
required: true
10-
description: The AWS region
10+
description: The AWS region.
1111
role-to-assume:
1212
required: true
13-
description: The role to assume in the STS session
14-
metric-name:
15-
description: Name of the metric to track in Cloudwatch.
13+
description: The role to assume in the STS session.
14+
github-token:
1615
required: true
17-
value:
16+
description: Github token for requesting failing steps
17+
18+
#value:
1819
# Why we publish value 0 on success: https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#publishingZero
19-
description: Value of the metric to track in Cloudwatch.
20+
# description: Value of the metric to track in Cloudwatch.
21+
# required: true
22+
23+
job-status:
24+
description: Job status
25+
required: true
26+
job-identifier:
27+
desciption: Identifiers for the job
28+
required: true
29+
30+
31+
32+
# Global Metric Dimensions
33+
testType:
34+
description: canary, integration, unit testType.
35+
required: true
36+
category:
37+
description: analytics, api, authenticator, etc.
2038
required: true
21-
dimensions:
22-
description: Dimensions of metric to track in Cloudwatch, in format dimensionName1=value,dimensionName2=value,...
39+
workflowName:
40+
description: The Github Action workflow.yaml file name. ie "AmplifyCanaries".
41+
required: true
42+
# failingStep: Look into getting via code
43+
44+
45+
# FlutterDart Workflows Metric Dimensions
46+
framework:
47+
description: flutter, dart.
48+
required: false
49+
flutterDartChannel:
50+
description: beta, stable.
51+
required: false
52+
dartVersion:
53+
description: 3, 2.19, 2.18, etc.
54+
required: false
55+
flutterVersion:
56+
description: 3.10.6, 3.10.5, etc.
57+
required: false
58+
dartCompiler:
59+
description: dart2js, ddc, dart, dart2wasm.
2360
required: false
61+
62+
# Platform Workflows Metric Dimensions
63+
platform:
64+
description: android, ios, web, linux, windows.
65+
required: false
66+
platformVersion:
67+
description: ios-14.5, ios-16, android-25-x86, etc.
68+
required: false
69+
2470
runs:
2571
using: 'composite'
2672
steps:
73+
#- name: Exit if not scheduled
74+
# shell: bash
75+
# run: |
76+
# if [ "${{ github.event_name }}" != "schedule" ]; then
77+
# echo "This was not triggered by a schedule, exiting."
78+
# exit 1
79+
# fi
80+
2781
- name: Configure AWS credentials
2882
uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 # 1.7.0
2983
with:
3084
role-to-assume: ${{ inputs.role-to-assume }}
3185
aws-region: ${{ inputs.aws-region }}
3286
role-duration-seconds: 900
3387

88+
- name: Get Job ID
89+
run: |
90+
substring="${{ inputs.job-identifier }}"
91+
92+
jobs_json=$(curl -s -H "Authorization: token ${{ inputs.github-token }}" \
93+
-H "Accept: application/vnd.github.v3+json" \
94+
"https://api.github.com/repos/fjnoyp/amplify-flutter/actions/runs/${{ github.run_id }}/jobs")
95+
96+
job_id=$(echo "$jobs_json" | jq --arg substring "$substring" '.jobs[] | select(.name | contains($substring)) | .id')
97+
98+
echo "substring is $substring"
99+
echo "job json is $jobs_json"
100+
echo "github run id is ${{ github.run_id}}"
101+
102+
echo "Job ID for job containing $substring is $job_id"
103+
echo "job_id=$job_id" >> $GITHUB_ENV
104+
shell: bash
105+
106+
- name: Get Failing Step
107+
if: ${{ job.status == 'failure' }}
108+
shell: bash
109+
run: |
110+
response=$(curl -s -H "Authorization: token ${{ inputs.github-token }}" \
111+
-H "Accept: application/vnd.github.v3+json" \
112+
"https://api.github.com/repos/fjnoyp/amplify-flutter/actions/jobs/${{ env.job_id }}")
113+
failing_step_name=$(echo "$response" | jq -r '.steps[] | select(.conclusion == "failure") | .name')
114+
115+
echo "FAILING_STEP=$failing_step_name" >> $GITHUB_ENV
116+
34117
- name: Run Dart script
35118
# Run a Dart script to put metric data.
36-
run: dart ./tool/send_metric_data.dart ${{ inputs.metric-name }} ${{ inputs.value }} ${{ inputs.dimensions }}
37-
shell: bash
119+
run: dart ./tool/send_metric_data.dart \
120+
github_metric_1.0 \
121+
${{ job.status == 'failure' }} \
122+
${{ inputs.testType }} \
123+
${{ inputs.category }} \
124+
${{ inputs.workflowName }} \
125+
${{ inputs.framework }} \
126+
${{ inputs.flutterDartChannel }} \
127+
${{ inputs.dartVersion }} \
128+
${{ inputs.flutterVersion }} \
129+
${{ inputs.dartCompiler }} \
130+
${{ inputs.platform }} \
131+
${{ inputs.platformVersion }} \
132+
${{ env.FAILING_STEP }}
133+
shell: bash

‎.github/workflows/amplify_canaries.yaml

Lines changed: 43 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ jobs:
3131
- channel: "stable"
3232
flutter-version: "3.10.1"
3333
steps:
34-
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # 3.5.3
34+
- name: Git Checkout
35+
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # 3.5.3
3536
with:
3637
persist-credentials: false
3738

@@ -53,24 +54,21 @@ jobs:
5354
- name: Build Canary (Android)
5455
run: build-support/build_canary.sh apk
5556

56-
- name: Log failing builds
57-
if: ${{ failure() }}
58-
uses: ./.github/composite_actions/log_metric
59-
with:
60-
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
61-
aws-region: ${{ secrets.AWS_REGION }}
62-
metric-name: BuildCanaryTestFailure
63-
value: 1
64-
dimensions: channel=${{ matrix.channel }}
65-
- name: Log succeeding builds
66-
if: ${{ success() }}
57+
- name: Log success/failure
58+
if: always()
6759
uses: ./.github/composite_actions/log_metric
6860
with:
6961
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
70-
aws-region: ${{ secrets.AWS_REGION }}
71-
metric-name: BuildCanaryTestFailure
72-
value: 0
73-
dimensions: channel=${{ matrix.channel }}
62+
aws-region: ${{ secrets.AWS_REGION }}
63+
github-token: ${{ secrets.GH_TOKEN }}
64+
job-status: ${{ job.status }}
65+
job-identifier: ${{ matrix.channel }}, ${{ matrix.flutter-version}}
66+
testType: canary
67+
category: all
68+
workflowName: amplify_canaries/build
69+
framework: flutter
70+
flutterDartChannel: ${{ matrix.channel }}
71+
flutterVersion: ${{ matrix.flutter-version }}
7472

7573
e2e-android:
7674
runs-on: macos-latest
@@ -127,24 +125,22 @@ jobs:
127125
# Perform a build to reduce startup time of `flutter test` and prevent timeout
128126
script: cd canaries && flutter build apk --debug && flutter test -d emulator-5554 integration_test/main_test.dart
129127

130-
- name: Log failing android runs
131-
if: ${{ failure() }}
132-
uses: ./.github/composite_actions/log_metric
133-
with:
134-
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
135-
aws-region: ${{ secrets.AWS_REGION }}
136-
metric-name: E2ECanaryTestFailure
137-
value: 1
138-
dimensions: channel=${{ matrix.channel }},platform=android
139-
- name: Log succeeding android runs
140-
if: ${{ success() }}
128+
- name: Log success/failure
129+
if: always()
141130
uses: ./.github/composite_actions/log_metric
142131
with:
143132
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
144-
aws-region: ${{ secrets.AWS_REGION }}
145-
metric-name: E2ECanaryTestFailure
146-
value: 0
147-
dimensions: channel=${{ matrix.channel }},platform=android
133+
aws-region: ${{ secrets.AWS_REGION }}
134+
github-token: ${{ secrets.GH_TOKEN }}
135+
job-status: ${{ job.status }}
136+
job-identifier: ${{ matrix.channel }}, ${{ matrix.flutter-version}}
137+
testType: canary
138+
category: all
139+
workflowName: amplify_canaries/e2e-android
140+
framework: flutter
141+
platform: android
142+
flutterDartChannel: ${{ matrix.channel }}
143+
flutterVersion: ${{ matrix.flutter-version }}
148144

149145
e2e-ios:
150146
runs-on: macos-13
@@ -205,21 +201,22 @@ jobs:
205201
flutter build ios --simulator --target=integration_test/main_test.dart
206202
flutter test -d test integration_test/main_test.dart --verbose
207203
208-
- name: Log failing ios runs
209-
if: ${{ failure() }}
210-
uses: ./.github/composite_actions/log_metric
211-
with:
212-
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
213-
aws-region: ${{ secrets.AWS_REGION }}
214-
metric-name: E2ECanaryTestFailure
215-
value: 1
216-
dimensions: channel=${{ matrix.channel }},platform=ios
217-
- name: Log succeeding ios runs
218-
if: ${{ success() }}
204+
- name: Log success/failure
205+
if: always()
219206
uses: ./.github/composite_actions/log_metric
220207
with:
221208
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
222-
aws-region: ${{ secrets.AWS_REGION }}
223-
metric-name: E2ECanaryTestFailure
224-
value: 0
225-
dimensions: channel=${{ matrix.channel }},platform=ios
209+
aws-region: ${{ secrets.AWS_REGION }}
210+
github-token: ${{ secrets.GH_TOKEN }}
211+
job-status: ${{ job.status }}
212+
job-identifier: ${{ matrix.channel }}, ${{ matrix.flutter-version}}
213+
testType: canary
214+
category: all
215+
workflowName: amplify_canaries/e2e-ios
216+
framework: flutter
217+
platform: ios
218+
platformVersion: ${{ matrix.ios-version }}
219+
flutterDartChannel: ${{ matrix.channel }}
220+
flutterVersion: ${{ matrix.flutter-version }}
221+
222+

‎.github/workflows/amplify_integration_tests.yaml

Lines changed: 83 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,23 @@ jobs:
8282
arch: ${{ matrix.arch }}
8383
script: aft exec --include=${{ matrix.scope }} -- small=true "<AFT_ROOT>/build-support/integ_test.sh" -d emulator-5554 --retries 1
8484

85+
- name: Log success/failure
86+
if: always()
87+
uses: ./.github/composite_actions/log_metric
88+
with:
89+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
90+
aws-region: ${{ secrets.AWS_REGION }}
91+
github-token: ${{ secrets.GH_TOKEN }}
92+
job-status: ${{ job.status }}
93+
job-identifier: ${{ matrix.scope }}, ${{ matrix.api-level}}, ${{ matrix.target }}
94+
testType: integration
95+
category: ${{ matrix.scope }}
96+
workflowName: ${{ matrix.scope }}/amplify_integration_tests
97+
framework: flutter
98+
flutterDartChannel: stable
99+
platform: android
100+
platformVersion: ${{ matrix.api-level }} ${{ matrix.arch }}
101+
85102
ios:
86103
runs-on: macos-latest
87104
# These permissions are needed to interact with GitHub's OIDC Token endpoint.
@@ -124,7 +141,23 @@ jobs:
124141

125142
- name: Run integration tests
126143
timeout-minutes: 60
127-
run: aft exec --include=${{ matrix.scope }} -- small=true "<AFT_ROOT>/build-support/integ_test_ios.sh" -d test
144+
run: aft exec --include=${{ matrix.scope }} -- small=true "<AFT_ROOT>/build-support/integ_test_ios.sh" -d test
145+
146+
- name: Log success/failure
147+
if: always()
148+
uses: ./.github/composite_actions/log_metric
149+
with:
150+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
151+
aws-region: ${{ secrets.AWS_REGION }}
152+
github-token: ${{ secrets.GH_TOKEN }}
153+
job-status: ${{ job.status }}
154+
job-identifier: ${{ matrix.scope }}
155+
testType: integration
156+
category: ${{ matrix.scope }}
157+
workflowName: ${{ matrix.scope }}/amplify_integration_tests
158+
framework: flutter
159+
flutterDartChannel: stable
160+
platform: ios
128161

129162
web:
130163
runs-on: ubuntu-latest
@@ -171,6 +204,23 @@ jobs:
171204
chromedriver --port=4444 &
172205
aft exec --include=${{ matrix.scope }} -- "<AFT_ROOT>/build-support/integ_test.sh" -d web-server
173206
207+
- name: Log success/failure
208+
if: always()
209+
uses: ./.github/composite_actions/log_metric
210+
with:
211+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
212+
aws-region: ${{ secrets.AWS_REGION }}
213+
github-token: ${{ secrets.GH_TOKEN }}
214+
job-status: ${{ job.status }}
215+
job-identifier: ${{ matrix.scope }}
216+
testType: integration
217+
category: ${{ matrix.scope }}
218+
workflowName: ${{ matrix.scope }}/amplify_integration_tests
219+
framework: flutter
220+
flutterDartChannel: stable
221+
platform: web
222+
platformVersion: chrome
223+
174224
# TODO(dnys1): Re-enable with self-hosted runners
175225
# macos:
176226
# runs-on: macos-latest
@@ -257,6 +307,22 @@ jobs:
257307
flutter config --enable-linux-desktop
258308
aft exec --include=${{ matrix.scope }} -- "<AFT_ROOT>/build-support/integ_test.sh" -d linux
259309
310+
- name: Log success/failure
311+
if: always()
312+
uses: ./.github/composite_actions/log_metric
313+
with:
314+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
315+
aws-region: ${{ secrets.AWS_REGION }}
316+
github-token: ${{ secrets.GH_TOKEN }}
317+
job-status: ${{ job.status }}
318+
job-identifier: ${{ matrix.scope }}
319+
testType: integration
320+
category: ${{ matrix.scope }}
321+
workflowName: ${{ matrix.scope }}/amplify_integration_tests
322+
framework: flutter
323+
flutterDartChannel: stable
324+
platform: linux
325+
260326
windows:
261327
runs-on: windows-latest
262328
# These permissions are needed to interact with GitHub's OIDC Token endpoint.
@@ -301,3 +367,19 @@ jobs:
301367
run: |
302368
flutter config --enable-windows-desktop
303369
dart pub global run aft exec --include=${{ matrix.scope }} -- flutter test integration_test/main_test.dart -d windows
370+
371+
- name: Log success/failure
372+
if: always()
373+
uses: ./.github/composite_actions/log_metric
374+
with:
375+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
376+
aws-region: ${{ secrets.AWS_REGION }}
377+
github-token: ${{ secrets.GH_TOKEN }}
378+
job-status: ${{ job.status }}
379+
job-identifier: ${{ matrix.scope }}
380+
testType: integration
381+
category: ${{ matrix.scope }}
382+
workflowName: ${{ matrix.scope }}/amplify_integration_tests
383+
framework: flutter
384+
flutterDartChannel: stable
385+
platform: windows

0 commit comments

Comments
 (0)