Test: Cloudwatch Global Metrics

.github/composite_actions/log_metric/action.yaml

@@ -1,26 +1,59 @@
 name: Log Metric
-description: Log data point to a metric with the provided value. If the metric is not there, it will create one
+description: Log data point to a metric with the provided value. If the metric is not there, it will create one.
 # To avoid 'Credentials could not be loaded' calling workflows must include:
 # permissions:
 #  id-token: write
 #  contents: read
 inputs:
   aws-region:
     required: true
-    description: The AWS region
+    description: The AWS region.
   role-to-assume:
     required: true
-    description: The role to assume in the STS session
-  metric-name:
-    description: Name of the metric to track in Cloudwatch.
-    required: true
+    description: The role to assume in the STS session.
   value:
     # Why we publish value 0 on success: https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#publishingZero
     description: Value of the metric to track in Cloudwatch.
     required: true
-  dimensions:
-    description: Dimensions of metric to track in Cloudwatch, in format dimensionName1=value,dimensionName2=value,...
+
+  # Global Metric Dimensions 
+  testType:
+    description: canary, integration, unit testType.
+    required: true
+  category:
+    description: analytics, api, authenticator, etc. 
+    required: true
+  workflowName: 
+    description: The Github Action workflow.yaml file name.  ie "AmplifyCanaries".
+    required: true 
+  # failingStep: Look into getting via code 
+
+
+  # FlutterDart Workflows Metric Dimensions 
+  framework:
+    description: flutter, dart. 
+    required: false
+  flutterDartChannel:
+    description: beta, stable. 
     required: false
+  dartVersion: 
+    description: 3, 2.19, 2.18, etc. 
+    required: false
+  flutterVersion:
+    description: 3.10.6, 3.10.5, etc. 
+    required: false
+  dartCompiler:
+    description: dart2js, ddc, dart, dart2wasm.
+    required: false
+
+  # Platform Workflows Metric Dimensions
+  platform:
+    description: android, ios, web, linux, windows.
+    required: false
+  platformVersion: 
+    description: ios-14.5, ios-16, android-25-x86, etc. 
+    required: false 
+
 runs:
   using: 'composite'
   steps:
@@ -33,5 +66,18 @@ runs:
 
     - name: Run Dart script
       # Run a Dart script to put metric data.
-      run: dart ./tool/send_metric_data.dart ${{ inputs.metric-name }} ${{ inputs.value }} ${{ inputs.dimensions }}
-      shell: bash
+      run: dart ./tool/send_metric_data.dart \
+        "GithubMetric v1.0" \
+        ${{ inputs.value }} \
+        ${{ inputs.testType }} \
+        ${{ inputs.category }} \
+        ${{ inputs.workflowName }} \
+        ${{ inputs.framework }} \
+        ${{ inputs.flutterDartChannel }} \
+        ${{ inputs.dartVersion }} \
+        ${{ inputs.flutterVersion }} \
+        ${{ inputs.dartCompiler }} \
+        ${{ inputs.platform }} \
+        ${{ inputs.platformVersion }} \
+        $GITHUB_RUN_ID 
+      shell: bash

.github/workflows/amplify_canaries.yaml

@@ -53,24 +53,18 @@ jobs:
       - name: Build Canary (Android)
         run: build-support/build_canary.sh apk
 
-      - name: Log failing builds
-        if: ${{ failure() }}
+      - name: Log build runs
         uses: ./.github/composite_actions/log_metric
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
           aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: BuildCanaryTestFailure
-          value: 1
-          dimensions: channel=${{ matrix.channel }}
-      - name: Log succeeding builds
-        if: ${{ success() }}
-        uses: ./.github/composite_actions/log_metric
-        with:
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: BuildCanaryTestFailure
-          value: 0
-          dimensions: channel=${{ matrix.channel }}
+          value: ${{ failure() && '1' || '0' }}
+          testType: canary
+          category: all
+          workflowName: amplify_canaries/build
+          framework: flutter
+          flutterDartChannel: ${{ matrix.channel }}
+          flutterVersion: $${{ matrix.flutter-version }}
 
   e2e-android:
     runs-on: macos-latest
@@ -127,24 +121,20 @@ jobs:
           # Perform a build to reduce startup time of `flutter test` and prevent timeout
           script: cd canaries && flutter build apk --debug && flutter test -d emulator-5554 integration_test/main_test.dart
 
-      - name: Log failing android runs
-        if: ${{ failure() }}
+      - name: Log android runs
         uses: ./.github/composite_actions/log_metric
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
           aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: E2ECanaryTestFailure
-          value: 1
-          dimensions: channel=${{ matrix.channel }},platform=android
-      - name: Log succeeding android runs
-        if: ${{ success() }}
-        uses: ./.github/composite_actions/log_metric
-        with:
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: E2ECanaryTestFailure
-          value: 0
-          dimensions: channel=${{ matrix.channel }},platform=android
+          value: ${{ failure() && '1' || '0' }}
+          testType: canary
+          category: all
+          workflowName: amplify_canaries/e2e-android
+          framework: flutter
+          flutterDartChannel: ${{ matrix.channel }}
+          flutterVersion: $${{ matrix.flutter-version }}
+          platform: android
+
 
   e2e-ios:
     runs-on: macos-13
@@ -216,21 +206,17 @@ jobs:
           flutter build ios --simulator --target=integration_test/main_test.dart
           flutter test -d test integration_test/main_test.dart --verbose
 
-      - name: Log failing ios runs
-        if: ${{ failure() }}
-        uses: ./.github/composite_actions/log_metric
-        with:
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: E2ECanaryTestFailure
-          value: 1
-          dimensions: channel=${{ matrix.channel }},platform=ios
-      - name: Log succeeding ios runs
-        if: ${{ success() }}
+      - name: Log ios runs
         uses: ./.github/composite_actions/log_metric
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
           aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: E2ECanaryTestFailure
-          value: 0
-          dimensions: channel=${{ matrix.channel }},platform=ios
+          value: ${{ failure() && '1' || '0' }}
+          testType: canary
+          category: all
+          workflowName: amplify_canaries/e2e-ios
+          framework: flutter
+          flutterDartChannel: ${{ matrix.channel }}
+          flutterVersion: $${{ matrix.flutter-version }}
+          platform: ios
+          platformVersion: $${{ matrix.ios-version }}

tool/send_metric_data.dart

@@ -1,21 +1,29 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import 'dart:convert';
 import 'dart:io';
 
-/// Parse and send metric data using AWS CLI
 void main(List<String> args) {
   final metricName = args[0];
   final value = args[1];
-  final dimensions = args.length > 2 ? args[2] : '';
+  final testType = args[2];
+  final category = args[3];
+  final workflowName = args[4];
+  final framework = args[5];
+  final flutterDartChannel = args[6];
+  final dartVersion = args[7];
+  final flutterVersion = args[8];
+  final dartCompiler = args[9];
+  final platform = args[10];
+  final platformVersion = args[11];
+  final githubRunId = args[12];
 
   final metricNameTrimmed = metricName.trim();
   final valueTrimmed = value.trim();
-  final dimensionsTrimmed = dimensions.trim();
 
   final metricNameRegex = RegExp(r'^[a-zA-Z0-9\ \_\-]+$');
   final valueRegex = RegExp(r'^[-+]?[0-9]+\.?[0-9]*$');
-  final dimensionsRegex = RegExp(r'^([^=,]+=[^=,]+(?:,[^=,]+=[^=,]+)*)?$');
 
   if (!metricNameRegex.hasMatch(metricNameTrimmed)) {
     print(
@@ -26,12 +34,39 @@ void main(List<String> args) {
     print('Metric value must be a valid number');
     exit(1);
   }
-  if (!dimensionsRegex.hasMatch(dimensionsTrimmed)) {
-    print(
-        'Dimensions must be empty or be in format string=string,string=string,...');
+
+  if (testType.isEmpty) {
+    print('Must provide testType dimension');
+    exit(1);
+  }
+
+  if (category.isEmpty) {
+    print('Must provide category dimension');
+    exit(1);
+  }
+
+  if (workflowName.isEmpty) {
+    print('Must provide workflowName dimension');
     exit(1);
   }
 
+  final dimensions = {
+    'testType': testType,
+    'category': category,
+    'workflowName': workflowName,
+    if (framework.isNotEmpty) 'framework': framework,
+    if (flutterDartChannel.isNotEmpty) 'flutterDartChannel': flutterDartChannel,
+    if (dartVersion.isNotEmpty) 'dartVersion': dartVersion,
+    if (flutterVersion.isNotEmpty) 'flutterVersion': flutterVersion,
+    if (dartCompiler.isNotEmpty) 'dartCompiler': dartCompiler,
+    if (platform.isNotEmpty) 'platform': platform,
+    if (platformVersion.isNotEmpty) 'platformVersion': platformVersion,
+    'failingStep': getFailingStep(githubRunId)
+  };
+
+  final dimensionString =
+      dimensions.entries.map((e) => '${e.key}=${e.value}').join(',');
+
   final cloudArgs = [
     'cloudwatch',
     'put-metric-data',
@@ -41,12 +76,28 @@ void main(List<String> args) {
     'GithubCanaryApps',
     '--value',
     value,
+    '--dimension',
+    dimensionString,
   ];
 
-  if (!dimensionsTrimmed.isEmpty) {
-    cloudArgs.add('--dimensions');
-    cloudArgs.add(dimensionsTrimmed);
-  }
-
   Process.runSync('aws', cloudArgs);
 }
+
+Future<String> getFailingStep(String githubRunId) async {
+  // Fetch the workflow run details
+  var result =
+      await Process.run('gh', ['run', 'view', githubRunId, '--json', 'jobs']);
+  var runData = jsonDecode(result.stdout);
+
+  // Parse the data and get the name of the failed step
+  var failedStep = 'none';
+  for (var job in runData['jobs']) {
+    for (var step in job['steps']) {
+      if (step['status'] == 'completed' && step['conclusion'] == 'failure') {
+        failedStep = step['name'];
+        break;
+      }
+    }
+  }
+  return failedStep;
+}