new combined test 2

.github/composite_actions/log_metric/action.yaml

@@ -1,38 +1,122 @@
 name: Log Metric
-description: Log data point to a metric with the provided value. If the metric is not there, it will create one
+description: Log data point to a metric with the provided value. If the metric is not there, it will create one.
 # To avoid 'Credentials could not be loaded' calling workflows must include:
 # permissions:
 #  id-token: write
 #  contents: read
 inputs:
   aws-region:
     required: true
-    description: The AWS region
+    description: The AWS region.
   role-to-assume:
     required: true
-    description: The role to assume in the STS session
-  metric-name:
-    description: Name of the metric to track in Cloudwatch.
+    description: The role to assume in the STS session.
+  github-token:
     required: true
-  value:
-    # Why we publish value 0 on success: https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#publishingZero
-    description: Value of the metric to track in Cloudwatch.
+    description: Github token for requesting failing steps.
+  job-status:
+    description: Used to determine if we track success or failure.
     required: true
-  dimensions:
-    description: Dimensions of metric to track in Cloudwatch, in format dimensionName1=value,dimensionName2=value,...
+  job-identifier:
+    description: For differentiating jobs of a run.
+    required: true
+
+
+  # Global Metric Dimensions
+  testType:
+    description: canary, integration, unit testType.
+    required: true
+  category:
+    description: analytics, api, authenticator, etc.
+    required: true
+  workflowName:
+    description: The Github Action workflow.yaml file name.  ie "AmplifyCanaries".
+    required: true
+
+  # FlutterDart Workflows Metric Dimensions
+  framework:
+    description: flutter, dart.
+    required: false
+  flutterDartChannel:
+    description: beta, stable.
+    required: false
+  dartVersion:
+    description: 3, 2.19, 2.18, etc.
+    required: false
+  flutterVersion:
+    description: 3.10.6, 3.10.5, etc.
+    required: false
+  dartCompiler:
+    description: dart2js, ddc, dart, dart2wasm.
     required: false
+
+  # Platform Workflows Metric Dimensions
+  platform:
+    description: android, ios, web, linux, windows.
+    required: false
+  platformVersion:
+    description: ios-14.5, ios-16, android-25-x86, etc.
+    required: false
+
 runs:
   using: "composite"
   steps:
+    #- name: Exit if not scheduled
+    #  shell: bash
+    #  run: |
+    #    if [ "${{ github.event_name }}" != "schedule" ]; then
+    #      echo "This was not triggered by a schedule, skipping."
+    #      echo "SKIP=true" >> $GITHUB_ENV
+    #    fi
+
     - name: Configure AWS credentials
+      if: env.SKIP != 'true'
       uses: aws-actions/configure-aws-credentials@04b98b3f9e85f563fb061be8751a0352327246b0 # 3.0.1
       with:
         unset-current-credentials: true
         role-to-assume: ${{ inputs.role-to-assume }}
         aws-region: ${{ inputs.aws-region }}
         role-duration-seconds: 900
 
+    - name: Change to Dart script directory
+      if: env.SKIP != 'true'
+      run: cd ./tool
+      shell: bash
+
+    - name: Install Dart dependencies (args)
+      if: env.SKIP != 'true'
+      run: dart pub get
+      shell: bash
+
+    - name: Get Failing Step      
+      if: env.SKIP != 'true' && inputs.job-status == 'failure'
+      run: |          
+        dart ./tool/get_failing_step.dart \
+          --job-status "failure" \
+          --substring "e2e-android (stable, any)" \
+          --github-token ${{ secrets.GITHUB_TOKEN }} \
+          --repo "fjnoyp/amplify-flutter" \
+          --run-id "6038852014" > dart_output.txt
+        echo FAILING_STEP=$(cat dart_output.txt) >> $GITHUB_ENV
+      shell: bash
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
     - name: Run Dart script
-      # Run a Dart script to put metric data.
-      run: dart ./tool/send_metric_data.dart ${{ inputs.metric-name }} ${{ inputs.value }} ${{ inputs.dimensions }}
+      if: env.SKIP != 'true'
+      run: |
+        dart ./tool/send_metric_data.dart \
+          --metric-name="github_metric_1.0" \
+          --is-failed="${{ inputs.job-status == 'failure' }}" \
+          --test-type="${{ inputs.testType }}" \
+          --category="${{ inputs.category }}" \
+          --workflow-name="${{ inputs.workflowName }}" \
+          --framework="${{ inputs.framework }}" \
+          --flutter-dart-channel="${{ inputs.flutterDartChannel }}" \
+          --dart-version="${{ inputs.dartVersion }}" \
+          --flutter-version="${{ inputs.flutterVersion }}" \
+          --dart-compiler="${{ inputs.dartCompiler }}" \
+          --platform="${{ inputs.platform }}" \
+          --platform-version="${{ inputs.platformVersion }}" \
+          --failing-step="${{ env.FAILING_STEP }}"
       shell: bash

.github/workflows/amplify_canaries.yaml

@@ -1,5 +1,13 @@
 name: Amplify Canaries
 on:
+# TEMP ENSURE PR TRIGGER
+  push:
+    branches:
+      - main
+      - stable
+      - feat/**
+      - fix/**
+      - test/**
   pull_request:
     paths:
       - ".github/workflows/amplify_canaries.yaml"
@@ -55,24 +63,21 @@ jobs:
       - name: Build Canary (Android)
         run: build-support/build_canary.sh apk
 
-      - name: Log failing builds
-        if: ${{ failure() }}
+      - name: Log success/failure
+        if: always()
         uses: ./.github/composite_actions/log_metric
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
           aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: BuildCanaryTestFailure
-          value: 1
-          dimensions: channel=${{ matrix.channel }}
-      - name: Log succeeding builds
-        if: ${{ success() }}
-        uses: ./.github/composite_actions/log_metric
-        with:
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: BuildCanaryTestFailure
-          value: 0
-          dimensions: channel=${{ matrix.channel }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          job-status: ${{ job.status }}
+          job-identifier: build (${{ matrix.channel }}, ${{ matrix.flutter-version}})
+          testType: canary
+          category: all
+          workflowName: amplify_canaries/build
+          framework: flutter
+          flutterDartChannel: ${{ matrix.channel }}
+          flutterVersion: ${{ matrix.flutter-version }}
 
   e2e-android:
     runs-on:
@@ -130,24 +135,22 @@ jobs:
           # Perform a build to reduce startup time of `flutter test` and prevent timeout
           script: cd canaries && flutter build apk --debug && flutter test -d emulator-5554 integration_test/main_test.dart
 
-      - name: Log failing android runs
-        if: ${{ failure() }}
+      - name: Log success/failure
+        if: always()
         uses: ./.github/composite_actions/log_metric
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
           aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: E2ECanaryTestFailure
-          value: 1
-          dimensions: channel=${{ matrix.channel }},platform=android
-      - name: Log succeeding android runs
-        if: ${{ success() }}
-        uses: ./.github/composite_actions/log_metric
-        with:
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: E2ECanaryTestFailure
-          value: 0
-          dimensions: channel=${{ matrix.channel }},platform=android
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          job-status: ${{ job.status }}
+          job-identifier: e2e-android (${{ matrix.channel }}, ${{ matrix.flutter-version}})
+          testType: canary
+          category: all
+          workflowName: amplify_canaries/e2e-android
+          framework: flutter
+          platform: android
+          flutterDartChannel: ${{ matrix.channel }}
+          flutterVersion: ${{ matrix.flutter-version }}
 
   e2e-ios:
     runs-on: macos-latest-xl
@@ -208,21 +211,21 @@ jobs:
           flutter build ios --simulator --target=integration_test/main_test.dart
           flutter test -d test integration_test/main_test.dart --verbose
 
-      - name: Log failing ios runs
-        if: ${{ failure() }}
-        uses: ./.github/composite_actions/log_metric
-        with:
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: E2ECanaryTestFailure
-          value: 1
-          dimensions: channel=${{ matrix.channel }},platform=ios
-      - name: Log succeeding ios runs
-        if: ${{ success() }}
+      - name: Log success/failure
+        if: always()
         uses: ./.github/composite_actions/log_metric
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
           aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: E2ECanaryTestFailure
-          value: 0
-          dimensions: channel=${{ matrix.channel }},platform=ios
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          job-status: ${{ job.status }}
+          job-identifier: e2e-ios (${{ matrix.channel }}, ${{ matrix.flutter-version}})
+          testType: canary
+          category: all
+          workflowName: amplify_canaries/e2e-ios
+          framework: flutter
+          platform: ios
+          platformVersion: ${{ matrix.ios-version }}
+          flutterDartChannel: ${{ matrix.channel }}
+          flutterVersion: ${{ matrix.flutter-version }}
+

.github/workflows/dart_dart2js.yaml

@@ -82,3 +82,23 @@ jobs:
         if: "always() && steps.bootstrap.conclusion == 'success'"
         run: dart run build_runner test --release --delete-conflicting-outputs -- -p ${{ matrix.browser }}
         working-directory: ${{ inputs.working-directory }}
+
+      - name: Log success/failure
+        if: always()
+        uses: ./.github/composite_actions/log_metric
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.AWS_REGION }}     
+          github-token: ${{ secrets.GITHUB_TOKEN }}    
+          job-status: ${{ job.status }}
+          job-identifier: ${{ matrix.sdk }}, ${{ matrix.browser }}
+          testType: unit test
+          category: ${{ inputs.working-directory }}
+          workflowName: ${{ inputs.package-name }}/dart_dart2js
+          framework: dart
+          flutterDartChannel: ${{ matrix.sdk }}
+          dartCompiler: dart2js
+          platform: web
+          # 'chrome/firefox'
+          platformVersion: $${{ matrix.browser }}
+

.github/workflows/dart_ddc.yaml

@@ -82,3 +82,22 @@ jobs:
         if: "always() && steps.bootstrap.conclusion == 'success'"
         run: dart run build_runner test --delete-conflicting-outputs -- -p ${{ matrix.browser }}
         working-directory: ${{ inputs.working-directory }}
+
+      - name: Log success/failure
+        if: always()
+        uses: ./.github/composite_actions/log_metric
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.AWS_REGION }}     
+          github-token: ${{ secrets.GITHUB_TOKEN }}    
+          job-status: ${{ job.status }}
+          job-identifier: ${{ matrix.sdk }}, ${{ matrix.browser }}
+          testType: unit test
+          category: ${{ inputs.working-directory }}
+          workflowName: ${{ inputs.package-name }}/dart_ddc
+          framework: dart
+          flutterDartChannel: ${{ matrix.sdk }}
+          dartCompiler: dart2js
+          platform: web
+          # 'chrome/firefox'
+          platformVersion: $${{ matrix.browser }}

.github/workflows/dart_native.yaml

@@ -88,3 +88,19 @@ jobs:
         if: "always() && steps.bootstrap.conclusion == 'success'"
         run: dart test --exclude-tags=build
         working-directory: ${{ inputs.working-directory }}
+
+      - name: Log success/failure
+        if: always()
+        uses: ./.github/composite_actions/log_metric
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.AWS_REGION }}     
+          github-token: ${{ secrets.GITHUB_TOKEN }}    
+          job-status: ${{ job.status }}
+          job-identifier: ${{ matrix.os }}
+          testType: unit test
+          category: ${{ inputs.working-directory }}
+          workflowName: ${{ inputs.package-name }}/dart_native
+          framework: dart
+          flutterDartChannel: stable
+          dartCompiler: dart

.github/workflows/dart_vm.yaml

@@ -114,3 +114,20 @@ jobs:
         if: "always() && steps.bootstrap.conclusion == 'success' && steps.testCheck.outputs.hasTests == 'true' && matrix.sdk != 'stable'"
         run: dart test --exclude-tags=build
         working-directory: ${{ inputs.working-directory }}
+
+      - name: Log success/failure
+        if: always()
+        uses: ./.github/composite_actions/log_metric
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.AWS_REGION }}     
+          github-token: ${{ secrets.GITHUB_TOKEN }}    
+          job-status: ${{ job.status }}
+          job-identifier: ${{ matrix.sdk }}
+          testType: unit test
+          category: ${{ inputs.working-directory }}
+          workflowName: ${{ inputs.package-name }}/dart_vm
+          framework: dart
+          flutterDartChannel: ${{ matrix.sdk }}
+          dartCompiler: dart
+

.github/workflows/flutter_android.yaml

@@ -58,3 +58,19 @@ jobs:
         if: inputs.has-native-tests
         run: ./gradlew :"${{ inputs.package-name }}":testDebugUnitTest --stacktrace
         working-directory: ${{ inputs.example-directory }}/android
+
+      - name: Log success/failure
+        if: always()
+        uses: ./.github/composite_actions/log_metric
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          job-status: ${{ job.status }}
+          job-identifier: ${{ matrix.channel }}
+          testType: unit test
+          category: ${{ inputs.working-directory }}
+          workflowName: ${{ inputs.package-name }}/flutter_android.test
+          framework: flutter
+          flutterDartChannel: ${{ matrix.channel }}
+          platform: android

.github/workflows/flutter_ios.yaml

@@ -63,3 +63,19 @@ jobs:
               -scheme Runner \
               -destination "$XCODEBUILD_DESTINATION" | xcpretty
         working-directory: ${{ inputs.example-directory }}/ios
+
+      - name: Log success/failure
+        if: always()
+        uses: ./.github/composite_actions/log_metric
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          job-status: ${{ job.status }}
+          job-identifier: ${{ matrix.channel }}
+          testType: unit test
+          category: ${{ inputs.working-directory }}
+          workflowName: ${{ inputs.package-name }}/flutter_ios
+          framework: flutter
+          flutterDartChannel: ${{ matrix.channel }}
+          platform: ios