.github/workflows/actions/test-monitor-process-results/action.yml

@@ -0,0 +1,55 @@
+name: Test Monitor - Process Results
+
+description: Custom action that's used in multiple Flaky Test Monitor jobs to process test results and upload them to BigQuery
+
+inputs:
+  service_account:
+    description: 'The GCP Service Account'
+    required: true
+  workload_identity_provider:
+    description: 'The GCP Workload Identity Provider'
+    required: true
+
+runs:
+  using : "composite"
+  steps:
+    - name: Get commit date
+      id: commit_date
+      run: echo "::set-output name=date::$(git show --no-patch --no-notes --pretty='%cI' $COMMIT_SHA)"
+      shell: bash
+
+    - name: Get job run date
+      id: job_run_date
+      run: echo "::set-output name=date::$(TZ=":UTC" date -Iseconds)"
+      shell: bash
+
+    - name: Process test results
+      run: cat test-output | go run tools/test_monitor/level1/process_summary1_results.go
+      env:
+        JOB_STARTED: ${{ steps.job_run_date.outputs.date }}
+        COMMIT_DATE: ${{ steps.commit_date.outputs.date }}
+      shell: bash
+
+    - name: Google auth
+      id: auth
+      uses: google-github-actions/auth@v2
+      with:
+        service_account: ${{ inputs.service_account }}
+        token_format: 'access_token'
+        workload_identity_provider: ${{ inputs.workload_identity_provider }}
+
+    - name: 'Set up Cloud SDK'
+      uses: 'google-github-actions/setup-gcloud@v2'
+
+    - name: Upload results to BigQuery (skipped tests)
+      uses: nick-fields/retry@v3
+      with:
+        timeout_minutes: 1
+        max_attempts: 3
+        command: bq load --source_format=NEWLINE_DELIMITED_JSON $BIGQUERY_DATASET.$BIGQUERY_TABLE $SKIPPED_TESTS_FILE tools/test_monitor/schemas/skipped_tests_schema.json
+    - name: Upload results to BigQuery (test run)
+      uses: nick-fields/retry@v3
+      with:
+        timeout_minutes: 2
+        max_attempts: 3
+        command: bq load --source_format=NEWLINE_DELIMITED_JSON $BIGQUERY_DATASET.$BIGQUERY_TABLE2 $RESULTS_FILE tools/test_monitor/schemas/test_results_schema.json

.github/workflows/builds.yml

@@ -0,0 +1,154 @@
+name: Build Node Docker Images
+# This workflow is used to build and push one-off images for specific node types. This is useful
+# when deploying hotfixes or any time a change is not needed for all node roles.
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        type: string
+        description: 'Git tag/commit'
+        required: true
+      docker_tag:
+        type: string
+        description: 'Docker tag'
+        required: true
+      # GHA doesn't support multi-selects, so simulating it with one boolean for each option
+      build_access:
+        type: boolean
+        description: 'Access'
+        required: false
+      build_collection:
+        type: boolean
+        description: 'Collection'
+        required: false
+      build_consensus:
+        type: boolean
+        description: 'Consensus'
+        required: false
+      build_execution:
+        type: boolean
+        description: 'Execution'
+        required: false
+      build_verification:
+        type: boolean
+        description: 'Verification'
+        required: false
+      build_observer:
+        type: boolean
+        description: 'Observer'
+        required: false
+      # GHA allows only up to 10 inputs - regroup two entries in one
+      include_alternative_builds:
+        type: boolean
+        description: 'Build amd64 `without_adx` and `without_netgo_without_adx` images, and arm64 images'
+        required: false
+      private_build:
+        type: boolean
+        description: 'Build private images'
+        required: false
+env:
+  GO_VERSION: "1.23"
+
+jobs:
+  # matrix_builder generates a matrix that includes the roles selected in the input
+  matrix_builder:
+    name: Setup build jobs
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.generate.outputs.matrix }}
+    steps:
+    - name: Print all input variables
+      run: echo '${{ toJson(inputs) }}' | jq
+
+    - id: generate
+      run: |
+        roles=()
+        if [[ "${{ inputs.build_access }}" = "true" ]]; then
+          roles+=( "access" )
+        fi
+        if [[ "${{ inputs.build_collection }}" = "true" ]]; then
+          roles+=( "collection" )
+        fi
+        if [[ "${{ inputs.build_consensus }}" = "true" ]]; then
+          roles+=( "consensus" )
+        fi
+        if [[ "${{ inputs.build_execution }}" = "true" ]]; then
+          roles+=( "execution" )
+        fi
+        if [[ "${{ inputs.build_verification }}" = "true" ]]; then
+          roles+=( "verification" )
+        fi
+        if [[ "${{ inputs.build_observer }}" = "true" ]]; then
+          roles+=( "observer" )
+        fi
+        rolesJSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${roles[@]}")
+        echo "matrix={\"role\":$(echo $rolesJSON)}" >> $GITHUB_OUTPUT
+
+  docker-push:
+    name: ${{ matrix.role }} images
+    runs-on: ubuntu-latest
+    environment: Production Docker Registry
+    needs: matrix_builder
+
+    # setup jobs for each role
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.matrix_builder.outputs.matrix) }}
+
+    steps:
+    - name: Setup Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: ${{ env.GO_VERSION }}
+
+    - name: Checkout repo
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.tag }}
+
+    # Provide Google Service Account credentials to Github Action, allowing interaction with the Google Container Registry
+    # Logging in as github-actions@dl-flow.iam.gserviceaccount.com
+    - id: auth
+      uses: google-github-actions/auth@v1
+      with:
+        credentials_json: ${{ secrets.GCR_SERVICE_KEY_SECRET }}
+    - name: Set up Google Cloud SDK
+      uses: google-github-actions/setup-gcloud@v1
+
+    - name: Authenticate Docker with gcloud
+      run: |
+        if [[ "${{ github.event.inputs.private_build }}" == "true" ]]; then
+          gcloud auth configure-docker us-central1-docker.pkg.dev
+        else
+          gcloud auth configure-docker
+        fi
+
+    - name: Set CONTAINER_REGISTRY
+      id: set-registry
+      run: |
+        if [[ "${{ github.event.inputs.private_build }}" == "true" ]]; then
+          echo "CONTAINER_REGISTRY=${{ vars.PRIVATE_REGISTRY }}" >> $GITHUB_ENV
+        else
+          echo "CONTAINER_REGISTRY=${{ vars.PUBLIC_REGISTRY }}" >> $GITHUB_ENV
+        fi
+
+    - name: Build/Push ${{ matrix.role }} amd64 images with adx (default)
+      env:
+        IMAGE_TAG: ${{ inputs.docker_tag }}
+        CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
+      run: |
+        make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx CONTAINER_REGISTRY=$CONTAINER_REGISTRY
+
+    - name: Build/Push ${{ matrix.role }} amd64 images without netgo and without adx, arm64 images
+      if: ${{ inputs.include_alternative_builds }}
+      env:
+        IMAGE_TAG: ${{ inputs.docker_tag }}
+        CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
+      run: |
+        make docker-build-${{ matrix.role }}-without-adx docker-push-${{ matrix.role }}-without-adx \
+          docker-build-${{ matrix.role }}-without-netgo-without-adx docker-push-${{ matrix.role }}-without-netgo-without-adx \
+          docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm CONTAINER_REGISTRY=$CONTAINER_REGISTRY
+
+
+        

.github/workflows/cd.yml

@@ -1,33 +1,59 @@
 name: CD
 
 on:
-  push:
-    tags: 
-      - '*'
-      - "!daily-*"
+# Workflow dispatch for now, while we're testing environments
+  # push:
+  #   tags:
+  #     - '*'
+  #     - "!daily-*"
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Tag/commit'
+        required: true
+        type: string
+
+
+env:
+  GO_VERSION: "1.23"
 
 jobs:
   docker-push:
     name: Push to container registry
     runs-on: ubuntu-latest
+    environment: Production Docker Registry
     steps:
     - name: Setup Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
+      timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time
       with:
-        go-version: '1.19'
+        go-version: ${{ env.GO_VERSION }}
     - name: Checkout repo
-      uses: actions/checkout@v2
-    - name: Build relic
-      run: make crypto_setup_gopath
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.tag }}
     # Provide Google Service Account credentials to Github Action, allowing interaction with the Google Container Registry
     # Logging in as github-actions@dl-flow.iam.gserviceaccount.com
-    - name: Docker login
-      uses: docker/login-action@v1
+    - id: auth
+      uses: google-github-actions/auth@v1
       with:
-        registry: gcr.io
-        username: _json_key
-        password: ${{ secrets.GCR_SERVICE_KEY }}
+        credentials_json: ${{ secrets.GCR_SERVICE_KEY_SECRET }}
+    - name: Set up Google Cloud SDK
+      uses: google-github-actions/setup-gcloud@v1
+    - name: Authenticate docker with gcloud
+      run: |
+        gcloud auth configure-docker
     - name: Docker build
-      run: make docker-build-flow
+      env:
+        CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
+      run: |
+        make docker-build-flow-with-adx
+        make docker-build-flow-without-adx
+        make docker-build-flow-without-netgo-without-adx
+        make docker-cross-build-flow-arm
     - name: Docker push
-      run: make docker-push-flow
+      run: |
+        make docker-push-flow-with-adx
+        make docker-push-flow-without-adx
+        make docker-push-flow-without-netgo-without-adx
+        make docker-push-flow-arm