Feat/mine global metrics

.github/composite_actions/log_metric/action.yaml

@@ -1,37 +1,137 @@
 name: Log Metric
-description: Log data point to a metric with the provided value. If the metric is not there, it will create one
+description: Log data point to a metric with the provided value. If the metric is not there, it will create one.
 # To avoid 'Credentials could not be loaded' calling workflows must include:
 # permissions:
 #  id-token: write
 #  contents: read
 inputs:
   aws-region:
     required: true
-    description: The AWS region
+    description: The AWS region.
   role-to-assume:
     required: true
-    description: The role to assume in the STS session
-  metric-name:
-    description: Name of the metric to track in Cloudwatch.
+    description: The role to assume in the STS session.
+  github-token:
     required: true
-  value:
-    # Why we publish value 0 on success: https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#publishingZero
-    description: Value of the metric to track in Cloudwatch.
+    description: Github token for requesting failing steps.
+  job-status:
+    description: Used to determine if we track success or failure.
+    required: true 
+  job-identifier: 
+    description: For differentiating jobs of a run.
+    required: true 
+
+
+  # Global Metric Dimensions 
+  testType:
+    description: canary, integration, unit testType.
+    required: true
+  category:
+    description: analytics, api, authenticator, etc. 
     required: true
-  dimensions:
-    description: Dimensions of metric to track in Cloudwatch, in format dimensionName1=value,dimensionName2=value,...
+  workflowName: 
+    description: The Github Action workflow.yaml file name.  ie "AmplifyCanaries".
+    required: true 
+
+  # FlutterDart Workflows Metric Dimensions 
+  framework:
+    description: flutter, dart. 
+    required: false
+  flutterDartChannel:
+    description: beta, stable. 
+    required: false
+  dartVersion: 
+    description: 3, 2.19, 2.18, etc. 
     required: false
+  flutterVersion:
+    description: 3.10.6, 3.10.5, etc. 
+    required: false
+  dartCompiler:
+    description: dart2js, ddc, dart, dart2wasm.
+    required: false
+
+  # Platform Workflows Metric Dimensions
+  platform:
+    description: android, ios, web, linux, windows.
+    required: false
+  platformVersion: 
+    description: ios-14.5, ios-16, android-25-x86, etc. 
+    required: false 
+
 runs:
   using: 'composite'
   steps:
+    #- name: Exit if not scheduled
+    #  shell: bash
+    #  run: |
+    #    if [ "${{ github.event_name }}" != "schedule" ]; then
+    #      echo "This was not triggered by a schedule, exiting."
+    #      echo "SKIP=true" >> $GITHUB_ENV
+    #    fi
+
     - name: Configure AWS credentials
+      if: env.SKIP != 'true'
       uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 # 1.7.0
       with:
         role-to-assume: ${{ inputs.role-to-assume }}
         aws-region: ${{ inputs.aws-region }}
         role-duration-seconds: 900
 
+    - name: Get Job ID
+      if: env.SKIP != 'true'
+      run: |
+        substring="${{ inputs.job-identifier }}"
+
+        jobs_json=$(curl -s -H "Authorization: token ${{ inputs.github-token }}" \
+          -H "Accept: application/vnd.github.v3+json" \
+          "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs")
+
+        job_id=$(echo "$jobs_json" | jq --arg substring "$substring" '.jobs[] | select(.name | contains($substring)) | .id')
+
+        echo "substring is $substring"
+        echo "job json is $jobs_json"
+        echo "github run id is ${{ github.run_id}}"
+
+        echo "Job ID for job containing $substring is $job_id"
+        echo "JOB_ID=$job_id" >> $GITHUB_OUTPUT
+      shell: bash
+
+    - name: Get Failing Step
+      if: env.SKIP != 'true' && ${{ job.status == 'failure' }}
+      shell: bash
+      run: |
+        response=$(curl -s -H "Authorization: token ${{ inputs.github-token }}" \
+          -H "Accept: application/vnd.github.v3+json" \
+          "${{ github.server_url }}/${{ github.repository }}/actions/jobs/${{ env.JOB_ID }}")
+        failing_step_name=$(echo "$response" | jq -r '.steps[] | select(.conclusion == "failure") | .name')
+
+        echo "FAILING_STEP=$failing_step_name" >> $GITHUB_OUTPUT
+
+    - name: Change to Dart script directory
+      if: env.SKIP != 'true'
+      run: cd ./tool
+      shell: bash
+
+    - name: Install Dart dependencies (args)
+      if: env.SKIP != 'true'
+      run: dart pub get
+      shell: bash
+
     - name: Run Dart script
-      # Run a Dart script to put metric data.
-      run: dart ./tool/send_metric_data.dart ${{ inputs.metric-name }} ${{ inputs.value }} ${{ inputs.dimensions }}
-      shell: bash
+      if: env.SKIP != 'true'
+      run: |
+        dart ./tool/send_metric_data.dart \
+          --metric-name="github_metric_1.0" \
+          --job-status="${{ job.status == 'failure' }}" \
+          --test-type="${{ inputs.testType }}" \
+          --category="${{ inputs.category }}" \
+          --workflow-name="${{ inputs.workflowName }}" \
+          --framework="${{ inputs.framework }}" \
+          --flutter-dart-channel="${{ inputs.flutterDartChannel }}" \
+          --dart-version="${{ inputs.dartVersion }}" \
+          --flutter-version="${{ inputs.flutterVersion }}" \
+          --dart-compiler="${{ inputs.dartCompiler }}" \
+          --platform="${{ inputs.platform }}" \
+          --platform-version="${{ inputs.platformVersion }}" \
+          --failing-step="${{ env.FAILING_STEP }}"
+      shell: bash

.github/workflows/amplify_canaries.yaml

@@ -1,5 +1,6 @@
 name: Amplify Canaries
 on:
+# TEMP ENSURE PR TRIGGER 
   pull_request:
     paths:
       - ".github/workflows/amplify_canaries.yaml"
@@ -31,7 +32,8 @@ jobs:
           - channel: "stable"
             flutter-version: "3.10.1"
     steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # 3.5.3
+      - name: Git Checkout
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # 3.5.3
         with:
           persist-credentials: false
 
@@ -53,24 +55,21 @@ jobs:
       - name: Build Canary (Android)
         run: build-support/build_canary.sh apk
 
-      - name: Log failing builds
-        if: ${{ failure() }}
-        uses: ./.github/composite_actions/log_metric
-        with:
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: BuildCanaryTestFailure
-          value: 1
-          dimensions: channel=${{ matrix.channel }}
-      - name: Log succeeding builds
-        if: ${{ success() }}
+      - name: Log success/failure
+        if: always()
         uses: ./.github/composite_actions/log_metric
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: BuildCanaryTestFailure
-          value: 0
-          dimensions: channel=${{ matrix.channel }}
+          aws-region: ${{ secrets.AWS_REGION }}     
+          github-token: ${{ secrets.GITHUB_TOKEN }}    
+          job-status: ${{ job.status }}
+          job-identifier: ${{ matrix.channel }}, ${{ matrix.flutter-version}}
+          testType: canary
+          category: all
+          workflowName: amplify_canaries/build
+          framework: flutter          
+          flutterDartChannel: ${{ matrix.channel }}
+          flutterVersion: ${{ matrix.flutter-version }}
 
   e2e-android:
     runs-on: macos-latest
@@ -127,24 +126,22 @@ jobs:
           # Perform a build to reduce startup time of `flutter test` and prevent timeout
           script: cd canaries && flutter build apk --debug && flutter test -d emulator-5554 integration_test/main_test.dart
 
-      - name: Log failing android runs
-        if: ${{ failure() }}
-        uses: ./.github/composite_actions/log_metric
-        with:
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: E2ECanaryTestFailure
-          value: 1
-          dimensions: channel=${{ matrix.channel }},platform=android
-      - name: Log succeeding android runs
-        if: ${{ success() }}
+      - name: Log success/failure
+        if: always()
         uses: ./.github/composite_actions/log_metric
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: E2ECanaryTestFailure
-          value: 0
-          dimensions: channel=${{ matrix.channel }},platform=android
+          aws-region: ${{ secrets.AWS_REGION }}     
+          github-token: ${{ secrets.GITHUB_TOKEN }}    
+          job-status: ${{ job.status }}
+          job-identifier: ${{ matrix.channel }}, ${{ matrix.flutter-version}}
+          testType: canary
+          category: all
+          workflowName: amplify_canaries/e2e-android
+          framework: flutter
+          platform: android
+          flutterDartChannel: ${{ matrix.channel }}
+          flutterVersion: ${{ matrix.flutter-version }}
 
   e2e-ios:
     runs-on: macos-13
@@ -205,21 +202,22 @@ jobs:
           flutter build ios --simulator --target=integration_test/main_test.dart
           flutter test -d test integration_test/main_test.dart --verbose
 
-      - name: Log failing ios runs
-        if: ${{ failure() }}
-        uses: ./.github/composite_actions/log_metric
-        with:
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: E2ECanaryTestFailure
-          value: 1
-          dimensions: channel=${{ matrix.channel }},platform=ios
-      - name: Log succeeding ios runs
-        if: ${{ success() }}
+      - name: Log success/failure
+        if: always()
         uses: ./.github/composite_actions/log_metric
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: ${{ secrets.AWS_REGION }}
-          metric-name: E2ECanaryTestFailure
-          value: 0
-          dimensions: channel=${{ matrix.channel }},platform=ios
+          aws-region: ${{ secrets.AWS_REGION }}     
+          github-token: ${{ secrets.GITHUB_TOKEN }}    
+          job-status: ${{ job.status }}
+          job-identifier: ${{ matrix.channel }}, ${{ matrix.flutter-version}}
+          testType: canary
+          category: all
+          workflowName: amplify_canaries/e2e-ios
+          framework: flutter
+          platform: ios
+          platformVersion: ${{ matrix.ios-version }}
+          flutterDartChannel: ${{ matrix.channel }}
+          flutterVersion: ${{ matrix.flutter-version }}
+
+

.github/workflows/amplify_integration_tests.yaml

@@ -82,6 +82,23 @@ jobs:
           arch: ${{ matrix.arch }}
           script: aft exec --include=${{ matrix.scope }} -- small=true "<AFT_ROOT>/build-support/integ_test.sh" -d emulator-5554 --retries 1
 
+      - name: Log success/failure
+        if: always()
+        uses: ./.github/composite_actions/log_metric
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.AWS_REGION }}     
+          github-token: ${{ secrets.GITHUB_TOKEN }}    
+          job-status: ${{ job.status }}
+          job-identifier: ${{ matrix.scope }}, ${{ matrix.api-level}}, ${{ matrix.target }}
+          testType: integration
+          category: ${{ matrix.scope }}
+          workflowName: ${{ matrix.scope }}/amplify_integration_tests
+          framework: flutter
+          flutterDartChannel: stable 
+          platform: android
+          platformVersion: ${{ matrix.api-level }} ${{ matrix.arch }}     
+
   ios:
     runs-on: macos-latest
     # These permissions are needed to interact with GitHub's OIDC Token endpoint.
@@ -124,7 +141,23 @@ jobs:
 
       - name: Run integration tests
         timeout-minutes: 60
-        run: aft exec --include=${{ matrix.scope }} -- small=true "<AFT_ROOT>/build-support/integ_test_ios.sh" -d test
+        run: aft exec --include=${{ matrix.scope }} -- small=true "<AFT_ROOT>/build-support/integ_test_ios.sh" -d test    
+
+      - name: Log success/failure
+        if: always()
+        uses: ./.github/composite_actions/log_metric
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.AWS_REGION }}     
+          github-token: ${{ secrets.GITHUB_TOKEN }}    
+          job-status: ${{ job.status }}
+          job-identifier: ${{ matrix.scope }}
+          testType: integration
+          category: ${{ matrix.scope }}
+          workflowName: ${{ matrix.scope }}/amplify_integration_tests
+          framework: flutter
+          flutterDartChannel: stable 
+          platform: ios     
 
   web:
     runs-on: ubuntu-latest
@@ -171,6 +204,23 @@ jobs:
           chromedriver --port=4444 &
           aft exec --include=${{ matrix.scope }} -- "<AFT_ROOT>/build-support/integ_test.sh" -d web-server
 
+      - name: Log success/failure
+        if: always()
+        uses: ./.github/composite_actions/log_metric
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.AWS_REGION }}     
+          github-token: ${{ secrets.GITHUB_TOKEN }}    
+          job-status: ${{ job.status }}
+          job-identifier: ${{ matrix.scope }}
+          testType: integration
+          category: ${{ matrix.scope }}
+          workflowName: ${{ matrix.scope }}/amplify_integration_tests
+          framework: flutter          
+          flutterDartChannel: stable 
+          platform: web      
+          platformVersion: chrome
+
   # TODO(dnys1): Re-enable with self-hosted runners
   # macos:
   #   runs-on: macos-latest
@@ -257,6 +307,22 @@ jobs:
           flutter config --enable-linux-desktop
           aft exec --include=${{ matrix.scope }} -- "<AFT_ROOT>/build-support/integ_test.sh" -d linux
 
+      - name: Log success/failure
+        if: always()
+        uses: ./.github/composite_actions/log_metric
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.AWS_REGION }}     
+          github-token: ${{ secrets.GITHUB_TOKEN }}    
+          job-status: ${{ job.status }}
+          job-identifier: ${{ matrix.scope }}
+          testType: integration
+          category: ${{ matrix.scope }}
+          workflowName: ${{ matrix.scope }}/amplify_integration_tests
+          framework: flutter          
+          flutterDartChannel: stable 
+          platform: linux     
+
   windows:
     runs-on: windows-latest
     # These permissions are needed to interact with GitHub's OIDC Token endpoint.
@@ -301,3 +367,19 @@ jobs:
         run: |
           flutter config --enable-windows-desktop
           dart pub global run aft exec --include=${{ matrix.scope }} -- flutter test integration_test/main_test.dart -d windows
+
+      - name: Log success/failure
+        if: always()
+        uses: ./.github/composite_actions/log_metric
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.AWS_REGION }}     
+          github-token: ${{ secrets.GITHUB_TOKEN }}    
+          job-status: ${{ job.status }}
+          job-identifier: ${{ matrix.scope }}
+          testType: integration
+          category: ${{ matrix.scope }}
+          workflowName: ${{ matrix.scope }}/amplify_integration_tests
+          framework: flutter          
+          flutterDartChannel: stable 
+          platform: windows