test-monitor-flaky.yml

# This workflow runs ALL tests, including all tests that are skipped because they are flaky, as well as all the normal (non-skipped) tests.
# This workflow is run less frequently because running flaky tests is problematic and causes errors.

name: Test Monitor - Flaky

on:
  schedule:
    - cron: '0 */12 * * *' # every 12 hours
  push:
    paths:
      - 'tools/test_monitor/**'

env:
  BIGQUERY_DATASET: production_src_flow_test_metrics
  BIGQUERY_TABLE: test_results
  GO_VERSION: 1.19

concurrency: 
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

jobs:
  flaky-test-run:
    name: Test Monitor - Flaky Tests Run
    strategy:
      fail-fast: false
      matrix:
        test-category:
          - unit
          - unit-crypto
          - unit-insecure
          - unit-integration
          - integration-bft
          - integration-mvp
          - integration-ghost
          - integration-network
          - integration-epochs
          - integration-access
          - integration-collection
          - integration-consensus
          - integration-execution
          - integration-verification
    env:
      TEST_CATEGORY: ${{ matrix.test-category }}
      COMMIT_SHA: ${{ github.sha }}
      RUN_ID: ${{ github.run_id }}
      RESULTS_FILE: test-results
    runs-on: ubuntu-latest
    steps:
    - name: Get job run date
      id: job_run_date
      run: echo "::set-output name=date::$(TZ=":America/Los_Angeles" date -Iseconds)"
    - name: Set up Cloud SDK
      uses: google-github-actions/setup-gcloud@v0
      with:
        service_account_key: ${{ secrets.GCP_SA_KEY }}
    - name: Setup Go
      uses: actions/setup-go@v2
      with:
        go-version: ${{ env.GO_VERSION }}
    - name: Checkout repo
      uses: actions/checkout@v2
      with:
        ref: ${{ env.COMMIT_SHA }}
    - name: Get commit date
      id: commit_date
      run: echo "::set-output name=date::$(git show --no-patch --no-notes --pretty='%cI' $COMMIT_SHA)"
    - name: Run tests
      continue-on-error: true
      run: ./tools/test_monitor/run-tests.sh
      env:
        TEST_FLAKY: true
        JSON_OUTPUT: true
        RACE_DETECTOR: 1
    - name: Print test results
      run: cat test-output
    - name: Process test results
      run: cat test-output | go run tools/test_monitor/level1/process_summary1_results.go
      env:
        JOB_STARTED: ${{ steps.job_run_date.outputs.date }}
        COMMIT_DATE: ${{ steps.commit_date.outputs.date }}
    - name: Upload results to BigQuery
      uses: nick-fields/retry@v2
      with:
        timeout_minutes: 2
        max_attempts: 3
        command: bq load --source_format=NEWLINE_DELIMITED_JSON $BIGQUERY_DATASET.$BIGQUERY_TABLE $RESULTS_FILE tools/test_monitor/schemas/test_results_schema.json