From 38fdb15afdb6a34c33d40944c47bd1e8fbf82fc3 Mon Sep 17 00:00:00 2001 From: kyle Date: Fri, 11 Aug 2023 11:59:10 -0700 Subject: [PATCH] chore: update metric reporting canaries --- .../composite_actions/log_metric/action.yaml | 118 ++++++++++++++++-- .github/workflows/amplify_canaries.yaml | 89 +++++++------ .../workflows/amplify_integration_tests.yaml | 84 ++++++++++++- .github/workflows/dart_dart2js.yaml | 19 +++ .github/workflows/dart_ddc.yaml | 18 +++ .github/workflows/dart_native.yaml | 15 +++ .github/workflows/dart_vm.yaml | 16 +++ .github/workflows/flutter_android.build.yaml | 15 +++ .github/workflows/flutter_android.test.yaml | 15 +++ .github/workflows/flutter_ios.yaml | 15 +++ .github/workflows/flutter_vm.yaml | 14 +++ tool/send_metric_data.dart | 100 +++++++++++---- 12 files changed, 437 insertions(+), 81 deletions(-) diff --git a/.github/composite_actions/log_metric/action.yaml b/.github/composite_actions/log_metric/action.yaml index ecc94fd3d4..7e4a19acb2 100644 --- a/.github/composite_actions/log_metric/action.yaml +++ b/.github/composite_actions/log_metric/action.yaml @@ -1,5 +1,5 @@ name: Log Metric -description: Log data point to a metric with the provided value. If the metric is not there, it will create one +description: Log data point to a metric with the provided value. If the metric is not there, it will create one. # To avoid 'Credentials could not be loaded' calling workflows must include: # permissions: # id-token: write @@ -7,23 +7,77 @@ description: Log data point to a metric with the provided value. If the metric i inputs: aws-region: required: true - description: The AWS region + description: The AWS region. role-to-assume: required: true - description: The role to assume in the STS session - metric-name: - description: Name of the metric to track in Cloudwatch. + description: The role to assume in the STS session. + github-token: required: true - value: + description: Github token for requesting failing steps + + #value: # Why we publish value 0 on success: https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#publishingZero - description: Value of the metric to track in Cloudwatch. + # description: Value of the metric to track in Cloudwatch. +# required: true + + job-status: + description: Job status + required: true + job-identifier: + desciption: Identifiers for the job + required: true + + + + # Global Metric Dimensions + testType: + description: canary, integration, unit testType. + required: true + category: + description: analytics, api, authenticator, etc. required: true - dimensions: - description: Dimensions of metric to track in Cloudwatch, in format dimensionName1=value,dimensionName2=value,... + workflowName: + description: The Github Action workflow.yaml file name. ie "AmplifyCanaries". + required: true + # failingStep: Look into getting via code + + + # FlutterDart Workflows Metric Dimensions + framework: + description: flutter, dart. + required: false + flutterDartChannel: + description: beta, stable. + required: false + dartVersion: + description: 3, 2.19, 2.18, etc. + required: false + flutterVersion: + description: 3.10.6, 3.10.5, etc. + required: false + dartCompiler: + description: dart2js, ddc, dart, dart2wasm. required: false + + # Platform Workflows Metric Dimensions + platform: + description: android, ios, web, linux, windows. + required: false + platformVersion: + description: ios-14.5, ios-16, android-25-x86, etc. + required: false + runs: using: 'composite' steps: + #- name: Exit if not scheduled + # shell: bash + # run: | + # if [ "${{ github.event_name }}" != "schedule" ]; then + # echo "This was not triggered by a schedule, exiting." + # exit 1 + # fi + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 # 1.7.0 with: @@ -31,7 +85,49 @@ runs: aws-region: ${{ inputs.aws-region }} role-duration-seconds: 900 + - name: Get Job ID + run: | + substring="${{ inputs.job-identifier }}" + + jobs_json=$(curl -s -H "Authorization: token ${{ inputs.github-token }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/fjnoyp/amplify-flutter/actions/runs/${{ github.run_id }}/jobs") + + job_id=$(echo "$jobs_json" | jq --arg substring "$substring" '.jobs[] | select(.name | contains($substring)) | .id') + + echo "substring is $substring" + echo "job json is $jobs_json" + echo "github run id is ${{ github.run_id}}" + + echo "Job ID for job containing $substring is $job_id" + echo "job_id=$job_id" >> $GITHUB_ENV + shell: bash + + - name: Get Failing Step + if: ${{ job.status == 'failure' }} + shell: bash + run: | + response=$(curl -s -H "Authorization: token ${{ inputs.github-token }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/fjnoyp/amplify-flutter/actions/jobs/${{ env.job_id }}") + failing_step_name=$(echo "$response" | jq -r '.steps[] | select(.conclusion == "failure") | .name') + + echo "FAILING_STEP=$failing_step_name" >> $GITHUB_ENV + - name: Run Dart script # Run a Dart script to put metric data. - run: dart ./tool/send_metric_data.dart ${{ inputs.metric-name }} ${{ inputs.value }} ${{ inputs.dimensions }} - shell: bash \ No newline at end of file + run: dart ./tool/send_metric_data.dart \ + github_metric_1.0 \ + ${{ job.status == 'failure' }} \ + ${{ inputs.testType }} \ + ${{ inputs.category }} \ + ${{ inputs.workflowName }} \ + ${{ inputs.framework }} \ + ${{ inputs.flutterDartChannel }} \ + ${{ inputs.dartVersion }} \ + ${{ inputs.flutterVersion }} \ + ${{ inputs.dartCompiler }} \ + ${{ inputs.platform }} \ + ${{ inputs.platformVersion }} \ + ${{ env.FAILING_STEP }} + shell: bash diff --git a/.github/workflows/amplify_canaries.yaml b/.github/workflows/amplify_canaries.yaml index 9cb3e4e9b9..c0ea6a1244 100644 --- a/.github/workflows/amplify_canaries.yaml +++ b/.github/workflows/amplify_canaries.yaml @@ -31,7 +31,8 @@ jobs: - channel: "stable" flutter-version: "3.10.1" steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # 3.5.3 + - name: Git Checkout + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # 3.5.3 with: persist-credentials: false @@ -53,24 +54,21 @@ jobs: - name: Build Canary (Android) run: build-support/build_canary.sh apk - - name: Log failing builds - if: ${{ failure() }} - uses: ./.github/composite_actions/log_metric - with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - metric-name: BuildCanaryTestFailure - value: 1 - dimensions: channel=${{ matrix.channel }} - - name: Log succeeding builds - if: ${{ success() }} + - name: Log success/failure + if: always() uses: ./.github/composite_actions/log_metric with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - metric-name: BuildCanaryTestFailure - value: 0 - dimensions: channel=${{ matrix.channel }} + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.channel }}, ${{ matrix.flutter-version}} + testType: canary + category: all + workflowName: amplify_canaries/build + framework: flutter + flutterDartChannel: ${{ matrix.channel }} + flutterVersion: ${{ matrix.flutter-version }} e2e-android: runs-on: macos-latest @@ -127,24 +125,22 @@ jobs: # Perform a build to reduce startup time of `flutter test` and prevent timeout script: cd canaries && flutter build apk --debug && flutter test -d emulator-5554 integration_test/main_test.dart - - name: Log failing android runs - if: ${{ failure() }} - uses: ./.github/composite_actions/log_metric - with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - metric-name: E2ECanaryTestFailure - value: 1 - dimensions: channel=${{ matrix.channel }},platform=android - - name: Log succeeding android runs - if: ${{ success() }} + - name: Log success/failure + if: always() uses: ./.github/composite_actions/log_metric with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - metric-name: E2ECanaryTestFailure - value: 0 - dimensions: channel=${{ matrix.channel }},platform=android + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.channel }}, ${{ matrix.flutter-version}} + testType: canary + category: all + workflowName: amplify_canaries/e2e-android + framework: flutter + platform: android + flutterDartChannel: ${{ matrix.channel }} + flutterVersion: ${{ matrix.flutter-version }} e2e-ios: runs-on: macos-13 @@ -205,21 +201,22 @@ jobs: flutter build ios --simulator --target=integration_test/main_test.dart flutter test -d test integration_test/main_test.dart --verbose - - name: Log failing ios runs - if: ${{ failure() }} - uses: ./.github/composite_actions/log_metric - with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - metric-name: E2ECanaryTestFailure - value: 1 - dimensions: channel=${{ matrix.channel }},platform=ios - - name: Log succeeding ios runs - if: ${{ success() }} + - name: Log success/failure + if: always() uses: ./.github/composite_actions/log_metric with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - metric-name: E2ECanaryTestFailure - value: 0 - dimensions: channel=${{ matrix.channel }},platform=ios + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.channel }}, ${{ matrix.flutter-version}} + testType: canary + category: all + workflowName: amplify_canaries/e2e-ios + framework: flutter + platform: ios + platformVersion: ${{ matrix.ios-version }} + flutterDartChannel: ${{ matrix.channel }} + flutterVersion: ${{ matrix.flutter-version }} + + \ No newline at end of file diff --git a/.github/workflows/amplify_integration_tests.yaml b/.github/workflows/amplify_integration_tests.yaml index 0f8b2df949..08a1a7a6c0 100644 --- a/.github/workflows/amplify_integration_tests.yaml +++ b/.github/workflows/amplify_integration_tests.yaml @@ -82,6 +82,23 @@ jobs: arch: ${{ matrix.arch }} script: aft exec --include=${{ matrix.scope }} -- small=true "/build-support/integ_test.sh" -d emulator-5554 --retries 1 + - name: Log success/failure + if: always() + uses: ./.github/composite_actions/log_metric + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.scope }}, ${{ matrix.api-level}}, ${{ matrix.target }} + testType: integration + category: ${{ matrix.scope }} + workflowName: ${{ matrix.scope }}/amplify_integration_tests + framework: flutter + flutterDartChannel: stable + platform: android + platformVersion: ${{ matrix.api-level }} ${{ matrix.arch }} + ios: runs-on: macos-latest # These permissions are needed to interact with GitHub's OIDC Token endpoint. @@ -124,7 +141,23 @@ jobs: - name: Run integration tests timeout-minutes: 60 - run: aft exec --include=${{ matrix.scope }} -- small=true "/build-support/integ_test_ios.sh" -d test + run: aft exec --include=${{ matrix.scope }} -- small=true "/build-support/integ_test_ios.sh" -d test + + - name: Log success/failure + if: always() + uses: ./.github/composite_actions/log_metric + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.scope }} + testType: integration + category: ${{ matrix.scope }} + workflowName: ${{ matrix.scope }}/amplify_integration_tests + framework: flutter + flutterDartChannel: stable + platform: ios web: runs-on: ubuntu-latest @@ -171,6 +204,23 @@ jobs: chromedriver --port=4444 & aft exec --include=${{ matrix.scope }} -- "/build-support/integ_test.sh" -d web-server + - name: Log success/failure + if: always() + uses: ./.github/composite_actions/log_metric + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.scope }} + testType: integration + category: ${{ matrix.scope }} + workflowName: ${{ matrix.scope }}/amplify_integration_tests + framework: flutter + flutterDartChannel: stable + platform: web + platformVersion: chrome + # TODO(dnys1): Re-enable with self-hosted runners # macos: # runs-on: macos-latest @@ -257,6 +307,22 @@ jobs: flutter config --enable-linux-desktop aft exec --include=${{ matrix.scope }} -- "/build-support/integ_test.sh" -d linux + - name: Log success/failure + if: always() + uses: ./.github/composite_actions/log_metric + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.scope }} + testType: integration + category: ${{ matrix.scope }} + workflowName: ${{ matrix.scope }}/amplify_integration_tests + framework: flutter + flutterDartChannel: stable + platform: linux + windows: runs-on: windows-latest # These permissions are needed to interact with GitHub's OIDC Token endpoint. @@ -301,3 +367,19 @@ jobs: run: | flutter config --enable-windows-desktop dart pub global run aft exec --include=${{ matrix.scope }} -- flutter test integration_test/main_test.dart -d windows + + - name: Log success/failure + if: always() + uses: ./.github/composite_actions/log_metric + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.scope }} + testType: integration + category: ${{ matrix.scope }} + workflowName: ${{ matrix.scope }}/amplify_integration_tests + framework: flutter + flutterDartChannel: stable + platform: windows diff --git a/.github/workflows/dart_dart2js.yaml b/.github/workflows/dart_dart2js.yaml index 5028d40a72..15a15c3186 100644 --- a/.github/workflows/dart_dart2js.yaml +++ b/.github/workflows/dart_dart2js.yaml @@ -84,3 +84,22 @@ jobs: if: "always() && steps.bootstrap.conclusion == 'success'" run: dart run build_runner test --release --delete-conflicting-outputs -- -p ${{ matrix.browser }} working-directory: ${{ inputs.working-directory }} + + - name: Log success/failure + uses: ./.github/composite_actions/log_metric + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.sdk }}, ${{ matrix.browser }} + testType: unit test + category: ${{ working-directory }} + workflowName: ${{ package-name }}/dart_dart2js + framework: dart + flutterDartChannel: ${{ matrix.sdk }} + dartCompiler: dart2js + platform: web + # 'chrome/firefox' + platformVersion: $${{ matrix.browser }} + diff --git a/.github/workflows/dart_ddc.yaml b/.github/workflows/dart_ddc.yaml index 8fdfdf296b..1fd354ac03 100644 --- a/.github/workflows/dart_ddc.yaml +++ b/.github/workflows/dart_ddc.yaml @@ -84,3 +84,21 @@ jobs: if: "always() && steps.bootstrap.conclusion == 'success'" run: dart run build_runner test --delete-conflicting-outputs -- -p ${{ matrix.browser }} working-directory: ${{ inputs.working-directory }} + + - name: Log success/failure + uses: ./.github/composite_actions/log_metric + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.sdk }}, ${{ matrix.browser }} + testType: unit test + category: ${{ working-directory }} + workflowName: ${{ package-name }}/dart_ddc + framework: dart + flutterDartChannel: ${{ matrix.sdk }} + dartCompiler: dart2js + platform: web + # 'chrome/firefox' + platformVersion: $${{ matrix.browser }} diff --git a/.github/workflows/dart_native.yaml b/.github/workflows/dart_native.yaml index 5adf018ed1..8fb9bc0868 100644 --- a/.github/workflows/dart_native.yaml +++ b/.github/workflows/dart_native.yaml @@ -92,3 +92,18 @@ jobs: if: "always() && steps.bootstrap.conclusion == 'success'" run: dart test --exclude-tags=build working-directory: ${{ inputs.working-directory }} + + - name: Log success/failure + uses: ./.github/composite_actions/log_metric + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.os }} + testType: unit test + category: ${{ working-directory }} + workflowName: ${{ package-name }}/dart_native + framework: dart + flutterDartChannel: stable + dartCompiler: dart diff --git a/.github/workflows/dart_vm.yaml b/.github/workflows/dart_vm.yaml index 41647169f3..b3af432be7 100644 --- a/.github/workflows/dart_vm.yaml +++ b/.github/workflows/dart_vm.yaml @@ -118,3 +118,19 @@ jobs: if: "always() && steps.bootstrap.conclusion == 'success' && steps.testCheck.outputs.hasTests == 'true' && matrix.sdk != 'stable'" run: dart test --exclude-tags=build working-directory: ${{ inputs.working-directory }} + + - name: Log success/failure + uses: ./.github/composite_actions/log_metric + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.sdk }} + testType: unit test + category: ${{ working-directory }} + workflowName: ${{ package-name }}/dart_vm + framework: dart + flutterDartChannel: ${{ matrix.sdk }} + dartCompiler: dart + diff --git a/.github/workflows/flutter_android.build.yaml b/.github/workflows/flutter_android.build.yaml index ae01db0dd9..296f2f6866 100644 --- a/.github/workflows/flutter_android.build.yaml +++ b/.github/workflows/flutter_android.build.yaml @@ -47,3 +47,18 @@ jobs: - name: Build for Android run: flutter build apk --debug --verbose working-directory: ${{ inputs.working-directory }} + + - name: Log success/failure + uses: ./.github/composite_actions/log_metric + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.channel }} + testType: unit test + category: ${{ working-directory }} + workflowName: ${{ package-name }}/flutter_android.build + framework: flutter + flutterDartChannel: ${{ matrix.channel }} + platform: android diff --git a/.github/workflows/flutter_android.test.yaml b/.github/workflows/flutter_android.test.yaml index ac758f6ccf..4cd8b36ebb 100644 --- a/.github/workflows/flutter_android.test.yaml +++ b/.github/workflows/flutter_android.test.yaml @@ -49,3 +49,18 @@ jobs: flutter build apk --debug --verbose ./gradlew :"${{ inputs.package-name }}":testDebugUnitTest --stacktrace working-directory: ${{ inputs.working-directory }} + + - name: Log success/failure + uses: ./.github/composite_actions/log_metric + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.channel }} + testType: unit test + category: ${{ working-directory }} + workflowName: ${{ package-name }}/flutter_android.test + framework: flutter + flutterDartChannel: ${{ matrix.channel }} + platform: android diff --git a/.github/workflows/flutter_ios.yaml b/.github/workflows/flutter_ios.yaml index 1c5ce05a1e..9eab67f16d 100644 --- a/.github/workflows/flutter_ios.yaml +++ b/.github/workflows/flutter_ios.yaml @@ -59,3 +59,18 @@ jobs: -scheme Runner \ -destination "$XCODEBUILD_DESTINATION" | xcpretty working-directory: ${{ inputs.working-directory }} + + - name: Log success/failure + uses: ./.github/composite_actions/log_metric + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.channel }} + testType: unit test + category: ${{ working-directory }} + workflowName: ${{ package-name }}/flutter_ios + framework: flutter + flutterDartChannel: ${{ matrix.channel }} + platform: ios diff --git a/.github/workflows/flutter_vm.yaml b/.github/workflows/flutter_vm.yaml index ed74f1d2da..e943cd2bc9 100644 --- a/.github/workflows/flutter_vm.yaml +++ b/.github/workflows/flutter_vm.yaml @@ -85,3 +85,17 @@ jobs: if: "always() && steps.bootstrap.conclusion == 'success' && steps.testCheck.outputs.hasTests == 'true'" run: flutter test working-directory: ${{ inputs.working-directory }} + + - name: Log success/failure + uses: ./.github/composite_actions/log_metric + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + github-token: ${{ secrets.GH_TOKEN }} + job-status: ${{ job.status }} + job-identifier: ${{ matrix.channel }}, ${{ matrix.flutter-version }} + testType: unit test + category: ${{ working-directory }} + workflowName: ${{ package-name }}/flutter_vm + framework: flutter + flutterDartChannel: ${{ matrix.channel }} diff --git a/tool/send_metric_data.dart b/tool/send_metric_data.dart index a74c81887a..0946d65dbd 100755 --- a/tool/send_metric_data.dart +++ b/tool/send_metric_data.dart @@ -3,35 +3,92 @@ import 'dart:io'; -/// Parse and send metric data using AWS CLI +final testTypes = [ + 'canary', + 'integration', + 'unit test', +]; + +final frameworkTypes = [ + 'flutter', + 'dart', +]; + +final platformTypes = ['web', 'android', 'ios', 'linux', 'windows']; + void main(List args) { - final metricName = args[0]; - final value = args[1]; - final dimensions = args.length > 2 ? args[2] : ''; + final metricName = args[0].trim(); + final isFailed = args[1] == 'failed'; + final testType = args[2].trim(); + var category = args[3].trim(); + final workflowName = args[4].trim(); + final framework = args[5].trim(); + final flutterDartChannel = args[6].trim(); + final dartVersion = args[7].trim(); + final flutterVersion = args[8].trim(); + final dartCompiler = args[9].trim(); + final platform = args[10].trim(); + final platformVersion = args[11].trim(); + final failingStep = args[12].trim(); - final metricNameTrimmed = metricName.trim(); - final valueTrimmed = value.trim(); - final dimensionsTrimmed = dimensions.trim(); + if (metricName.isEmpty) { + print('Must provide metricName'); + exit(1); + } + + final value = isFailed ? '1' : '0'; + + if (testType.isEmpty) { + print('Must provide testType dimension'); + exit(1); + } else if (!testTypes.contains(testType)) { + print('TestType is not valid: ${testType}'); + exit(1); + } - final metricNameRegex = RegExp(r'^[a-zA-Z0-9\ \_\-]+$'); - final valueRegex = RegExp(r'^[-+]?[0-9]+\.?[0-9]*$'); - final dimensionsRegex = RegExp(r'^([^=,]+=[^=,]+(?:,[^=,]+=[^=,]+)*)?$'); + if (category.isEmpty) { + print('Must provide category dimension'); + exit(1); + } else if (category.contains('/')) { + // For working directory "packages/analytics/amplify_analytics_pinpoint" + category = category.split('/')[1]; + } else if (category.contains('_')) { + // For integration test scope "amplify_analytics_pinpoint_example" + category = category.split('_')[1]; + } - if (!metricNameRegex.hasMatch(metricNameTrimmed)) { - print( - 'Metric name can only contain alphanumeric characters, space character, -, and _.'); + if (workflowName.isEmpty) { + print('Must provide workflowName dimension'); exit(1); } - if (!valueRegex.hasMatch(valueTrimmed)) { - print('Metric value must be a valid number'); + + if (framework.isNotEmpty && !frameworkTypes.contains(framework)) { + print('Framework is not valid: ${framework}'); exit(1); } - if (!dimensionsRegex.hasMatch(dimensionsTrimmed)) { - print( - 'Dimensions must be empty or be in format string=string,string=string,...'); + + if (platform.isNotEmpty && !platformTypes.contains(platform)) { + print('Platform is not valid: ${platform}'); exit(1); } + final dimensions = { + 'testType': testType, + 'category': category, + 'workflowName': workflowName, + if (framework.isNotEmpty) 'framework': framework, + if (flutterDartChannel.isNotEmpty) 'flutterDartChannel': flutterDartChannel, + if (dartVersion.isNotEmpty) 'dartVersion': dartVersion, + if (flutterVersion.isNotEmpty) 'flutterVersion': flutterVersion, + if (dartCompiler.isNotEmpty) 'dartCompiler': dartCompiler, + if (platform.isNotEmpty) 'platform': platform, + if (platformVersion.isNotEmpty) 'platformVersion': platformVersion, + if (failingStep.isNotEmpty) 'failingStep': failingStep, + }; + + final dimensionString = + dimensions.entries.map((e) => '${e.key}=${e.value}').join(','); + final cloudArgs = [ 'cloudwatch', 'put-metric-data', @@ -41,12 +98,9 @@ void main(List args) { 'GithubCanaryApps', '--value', value, + '--dimension', + dimensionString, ]; - if (!dimensionsTrimmed.isEmpty) { - cloudArgs.add('--dimensions'); - cloudArgs.add(dimensionsTrimmed); - } - Process.runSync('aws', cloudArgs); }