From c7d3d1c094d681691b352174dc1f57616b7b49f0 Mon Sep 17 00:00:00 2001 From: kyle Date: Tue, 8 Aug 2023 11:26:11 -0700 Subject: [PATCH] First attempt rewrite global metrics tracking. --- .../composite_actions/log_metric/action.yaml | 66 +++++++++++++--- .github/workflows/amplify_canaries.yaml | 70 +++++++---------- tool/send_metric_data.dart | 75 ++++++++++++++++--- 3 files changed, 147 insertions(+), 64 deletions(-) diff --git a/.github/composite_actions/log_metric/action.yaml b/.github/composite_actions/log_metric/action.yaml index ecc94fd3d4..bad32fec17 100644 --- a/.github/composite_actions/log_metric/action.yaml +++ b/.github/composite_actions/log_metric/action.yaml @@ -1,5 +1,5 @@ name: Log Metric -description: Log data point to a metric with the provided value. If the metric is not there, it will create one +description: Log data point to a metric with the provided value. If the metric is not there, it will create one. # To avoid 'Credentials could not be loaded' calling workflows must include: # permissions: # id-token: write @@ -7,20 +7,53 @@ description: Log data point to a metric with the provided value. If the metric i inputs: aws-region: required: true - description: The AWS region + description: The AWS region. role-to-assume: required: true - description: The role to assume in the STS session - metric-name: - description: Name of the metric to track in Cloudwatch. - required: true + description: The role to assume in the STS session. value: # Why we publish value 0 on success: https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#publishingZero description: Value of the metric to track in Cloudwatch. required: true - dimensions: - description: Dimensions of metric to track in Cloudwatch, in format dimensionName1=value,dimensionName2=value,... + + # Global Metric Dimensions + testType: + description: canary, integration, unit testType. + required: true + category: + description: analytics, api, authenticator, etc. + required: true + workflowName: + description: The Github Action workflow.yaml file name. ie "AmplifyCanaries". + required: true + # failingStep: Look into getting via code + + + # FlutterDart Workflows Metric Dimensions + framework: + description: flutter, dart. + required: false + flutterDartChannel: + description: beta, stable. required: false + dartVersion: + description: 3, 2.19, 2.18, etc. + required: false + flutterVersion: + description: 3.10.6, 3.10.5, etc. + required: false + dartCompiler: + description: dart2js, ddc, dart, dart2wasm. + required: false + + # Platform Workflows Metric Dimensions + platform: + description: android, ios, web, linux, windows. + required: false + platformVersion: + description: ios-14.5, ios-16, android-25-x86, etc. + required: false + runs: using: 'composite' steps: @@ -33,5 +66,18 @@ runs: - name: Run Dart script # Run a Dart script to put metric data. - run: dart ./tool/send_metric_data.dart ${{ inputs.metric-name }} ${{ inputs.value }} ${{ inputs.dimensions }} - shell: bash \ No newline at end of file + run: dart ./tool/send_metric_data.dart \ + "GithubMetric v1.0" \ + ${{ inputs.value }} \ + ${{ inputs.testType }} \ + ${{ inputs.category }} \ + ${{ inputs.workflowName }} \ + ${{ inputs.framework }} \ + ${{ inputs.flutterDartChannel }} \ + ${{ inputs.dartVersion }} \ + ${{ inputs.flutterVersion }} \ + ${{ inputs.dartCompiler }} \ + ${{ inputs.platform }} \ + ${{ inputs.platformVersion }} \ + $GITHUB_RUN_ID + shell: bash diff --git a/.github/workflows/amplify_canaries.yaml b/.github/workflows/amplify_canaries.yaml index 472fd06466..3ccd371396 100644 --- a/.github/workflows/amplify_canaries.yaml +++ b/.github/workflows/amplify_canaries.yaml @@ -53,24 +53,18 @@ jobs: - name: Build Canary (Android) run: build-support/build_canary.sh apk - - name: Log failing builds - if: ${{ failure() }} + - name: Log build runs uses: ./.github/composite_actions/log_metric with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: ${{ secrets.AWS_REGION }} - metric-name: BuildCanaryTestFailure - value: 1 - dimensions: channel=${{ matrix.channel }} - - name: Log succeeding builds - if: ${{ success() }} - uses: ./.github/composite_actions/log_metric - with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - metric-name: BuildCanaryTestFailure - value: 0 - dimensions: channel=${{ matrix.channel }} + value: ${{ failure() && '1' || '0' }} + testType: canary + category: all + workflowName: amplify_canaries/build + framework: flutter + flutterDartChannel: ${{ matrix.channel }} + flutterVersion: $${{ matrix.flutter-version }} e2e-android: runs-on: macos-latest @@ -127,24 +121,20 @@ jobs: # Perform a build to reduce startup time of `flutter test` and prevent timeout script: cd canaries && flutter build apk --debug && flutter test -d emulator-5554 integration_test/main_test.dart - - name: Log failing android runs - if: ${{ failure() }} + - name: Log android runs uses: ./.github/composite_actions/log_metric with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: ${{ secrets.AWS_REGION }} - metric-name: E2ECanaryTestFailure - value: 1 - dimensions: channel=${{ matrix.channel }},platform=android - - name: Log succeeding android runs - if: ${{ success() }} - uses: ./.github/composite_actions/log_metric - with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - metric-name: E2ECanaryTestFailure - value: 0 - dimensions: channel=${{ matrix.channel }},platform=android + value: ${{ failure() && '1' || '0' }} + testType: canary + category: all + workflowName: amplify_canaries/e2e-android + framework: flutter + flutterDartChannel: ${{ matrix.channel }} + flutterVersion: $${{ matrix.flutter-version }} + platform: android + e2e-ios: runs-on: macos-13 @@ -216,21 +206,17 @@ jobs: flutter build ios --simulator --target=integration_test/main_test.dart flutter test -d test integration_test/main_test.dart --verbose - - name: Log failing ios runs - if: ${{ failure() }} - uses: ./.github/composite_actions/log_metric - with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - metric-name: E2ECanaryTestFailure - value: 1 - dimensions: channel=${{ matrix.channel }},platform=ios - - name: Log succeeding ios runs - if: ${{ success() }} + - name: Log ios runs uses: ./.github/composite_actions/log_metric with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: ${{ secrets.AWS_REGION }} - metric-name: E2ECanaryTestFailure - value: 0 - dimensions: channel=${{ matrix.channel }},platform=ios + value: ${{ failure() && '1' || '0' }} + testType: canary + category: all + workflowName: amplify_canaries/e2e-ios + framework: flutter + flutterDartChannel: ${{ matrix.channel }} + flutterVersion: $${{ matrix.flutter-version }} + platform: ios + platformVersion: $${{ matrix.ios-version }} diff --git a/tool/send_metric_data.dart b/tool/send_metric_data.dart index a74c81887a..8917585304 100755 --- a/tool/send_metric_data.dart +++ b/tool/send_metric_data.dart @@ -1,21 +1,29 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import 'dart:convert'; import 'dart:io'; -/// Parse and send metric data using AWS CLI void main(List args) { final metricName = args[0]; final value = args[1]; - final dimensions = args.length > 2 ? args[2] : ''; + final testType = args[2]; + final category = args[3]; + final workflowName = args[4]; + final framework = args[5]; + final flutterDartChannel = args[6]; + final dartVersion = args[7]; + final flutterVersion = args[8]; + final dartCompiler = args[9]; + final platform = args[10]; + final platformVersion = args[11]; + final githubRunId = args[12]; final metricNameTrimmed = metricName.trim(); final valueTrimmed = value.trim(); - final dimensionsTrimmed = dimensions.trim(); final metricNameRegex = RegExp(r'^[a-zA-Z0-9\ \_\-]+$'); final valueRegex = RegExp(r'^[-+]?[0-9]+\.?[0-9]*$'); - final dimensionsRegex = RegExp(r'^([^=,]+=[^=,]+(?:,[^=,]+=[^=,]+)*)?$'); if (!metricNameRegex.hasMatch(metricNameTrimmed)) { print( @@ -26,12 +34,39 @@ void main(List args) { print('Metric value must be a valid number'); exit(1); } - if (!dimensionsRegex.hasMatch(dimensionsTrimmed)) { - print( - 'Dimensions must be empty or be in format string=string,string=string,...'); + + if (testType.isEmpty) { + print('Must provide testType dimension'); + exit(1); + } + + if (category.isEmpty) { + print('Must provide category dimension'); + exit(1); + } + + if (workflowName.isEmpty) { + print('Must provide workflowName dimension'); exit(1); } + final dimensions = { + 'testType': testType, + 'category': category, + 'workflowName': workflowName, + if (framework.isNotEmpty) 'framework': framework, + if (flutterDartChannel.isNotEmpty) 'flutterDartChannel': flutterDartChannel, + if (dartVersion.isNotEmpty) 'dartVersion': dartVersion, + if (flutterVersion.isNotEmpty) 'flutterVersion': flutterVersion, + if (dartCompiler.isNotEmpty) 'dartCompiler': dartCompiler, + if (platform.isNotEmpty) 'platform': platform, + if (platformVersion.isNotEmpty) 'platformVersion': platformVersion, + 'failingStep': getFailingStep(githubRunId) + }; + + final dimensionString = + dimensions.entries.map((e) => '${e.key}=${e.value}').join(','); + final cloudArgs = [ 'cloudwatch', 'put-metric-data', @@ -41,12 +76,28 @@ void main(List args) { 'GithubCanaryApps', '--value', value, + '--dimension', + dimensionString, ]; - if (!dimensionsTrimmed.isEmpty) { - cloudArgs.add('--dimensions'); - cloudArgs.add(dimensionsTrimmed); - } - Process.runSync('aws', cloudArgs); } + +Future getFailingStep(String githubRunId) async { + // Fetch the workflow run details + var result = + await Process.run('gh', ['run', 'view', githubRunId, '--json', 'jobs']); + var runData = jsonDecode(result.stdout); + + // Parse the data and get the name of the failed step + var failedStep = 'none'; + for (var job in runData['jobs']) { + for (var step in job['steps']) { + if (step['status'] == 'completed' && step['conclusion'] == 'failure') { + failedStep = step['name']; + break; + } + } + } + return failedStep; +}