ci-core.yml

name: CI Core
run-name: CI Core ${{ inputs.distinct_run_name && inputs.distinct_run_name || '' }}

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.distinct_run_name }}
  cancel-in-progress: true

# Run on key branches to make sure integration is good, otherwise run on all PR's
on:
  push:
    branches:
      - develop
      - "release/*"
  merge_group:
  pull_request:
  schedule:
    - cron: "0 0,6,12,18 * * *"

jobs:
  filter:
    name: Detect Changes
    permissions:
      pull-requests: read
    outputs:
      affected-modules: ${{ steps.resolved-modules.outputs.module_names }}
      deployment-changes: ${{ steps.match-some.outputs.deployment == 'true' }}
      scripts-changes: ${{ steps.match-some.outputs.scripts == 'true' }}
      should-run-ci-core: >-
        ${{
          steps.match-some.outputs.core-ci == 'true' ||
          steps.match-every.outputs.non-ignored == 'true' ||
          github.event_name == 'schedule' ||
          github.event_name == 'workflow_dispatch'
        }}
      should-run-golangci: >-
        ${{
          steps.match-some.outputs.golang-ci == 'true' ||
          steps.match-every.outputs.non-ignored == 'true'
        }}
      should-run-scripts-test: >-
        ${{
          steps.match-some.outputs.scripts == 'true' ||
          github.event_name == 'schedule' ||
          github.event_name == 'workflow_dispatch'
        }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout the repo
        uses: actions/checkout@v4.2.1
        with:
          persist-credentials: false
          repository: smartcontractkit/chainlink
      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        id: match-some
        with:
          # "if any changed file matches one or more of the conditions" (https://github.com/dorny/paths-filter/issues/225)
          predicate-quantifier: some
          # deployment - any changes to files in the `deployments/`
          # scripts - any changes to files in the `core/scripts/`
          # core-ci - any changes that could affect this workflow definition
          # golang-ci - any changes that could affect the linting result
          filters: |
            deployment:
              - 'deployment/**'
            core-ci:
              - '.github/workflows/ci-core.yml'
              - '.github/actions/**'
            golang-ci:
              - '.golangci.yml'
              - '.github/workflows/ci-core.yml'
              - '.github/actions/**'
            scripts:
              - 'core/scripts/**'
      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        id: match-every
        with:
          # "if any changed file match all of the conditions" (https://github.com/dorny/paths-filter/issues/225)
          predicate-quantifier: every
          # non-integration-tests - only changes made outside of the `integration-tests` directory
          # non-ignored - only changes except for the negated ones
          # all - changes in any directory
          #  - This is opt-in on purpose. To be safe, new files are assumed to have an affect on CI Core unless listed here specifically.
          # Enable listing of files matching each filter.
          # Paths to files will be available in `${FILTER_NAME}_files` output variable.
          # Paths will be formatted as JSON array
          list-files: json
          filters: |
            non-integration-tests:
              - '**'
              - '!integration-tests/**'
            non-ignored:
              - '**'
              - '!integration-tests/**'
              - '!tools/secrets/**'
              - '!tools/goreleaser-config/**'
              - '!tools/docker/**'
              - '!tools/benchmark/**'
              - '!**/README.md'
              - '!**/CHANGELOG.md'
              - '!.goreleaser.develop.yaml'
              - '!.goreleaser.devspace.yaml'
              - '!.goreleaser.production.yaml'
              - '!*.nix'
              - '!sonar-project.properties'
              - '!nix.conf'
              - '!nix-darwin-shell-hook.sh'
              - '!LICENSE'
              - '!.github/**'
            all:
              - '**'

      - name: Resolve affected files to affected modules
        id: resolved-modules
        shell: bash
        env:
          GH_EVENT_NAME: ${{ github.event_name }}
        run: |
          # if scheduled, run for all modules. Otherwise, run for only affected modules.
          if [[ "$GH_EVENT_NAME" == "schedule" ]]; then
            json_array=$(find . -name 'go.mod' -exec dirname {} \; | sed 's|^./||' | uniq |  jq -R -s -c 'split("\n") | map(select(length > 0))')
            echo "module_names=$json_array" >> "$GITHUB_OUTPUT"
          else
            # Ensure the step uses `with.list-files: json` to get the list of files in JSON format
            bash ./.github/scripts/map-affected-files-to-modules.sh '${{ steps.match-every.outputs.all_files }}'
          fi

  golangci:
    name: GolangCI Lint
    needs: [filter, run-frequency]
    # We don't directly merge dependabot PRs to not waste the resources.
    if: ${{ (github.event_name == 'pull_request' ||  github.event_name == 'schedule') && github.actor != 'dependabot[bot]' }}
    permissions:
      # To annotate code in the PR.
      checks: write
      contents: read
      # For golangci-lint-action's `only-new-issues` option.
      pull-requests: read
    runs-on: ubuntu-24.04-8cores-32GB-ARM
    strategy:
      fail-fast: false
      matrix:
        modules: ${{ fromJson(needs.filter.outputs.affected-modules) }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4.2.1
        with:
          persist-credentials: false
      - name: Golang Lint (${{ matrix.modules }})
        id: golang-lint
        uses: ./.github/actions/golangci-lint
        with:
          go-directory: ${{ matrix.modules }}
      - name: Notify Slack
        if: ${{ failure() && needs.run-frequency.outputs.one-per-day-frequency == 'true' }}
        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
        with:
          method: chat.postMessage
          token: ${{ secrets.QA_SLACK_API_KEY }}
          payload: |
            channel: ${{ secrets.SLACK_TEAM_CORE_CHANNEL_ID}}
            text: "golangci-lint failed (${{ matrix.modules }}): <${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}|Run> - <${{ steps.golang-lint.outputs.golang-report-artifact-url }}|Report>"

  # Fails if any golangci-lint matrix jobs fails and silently succeeds otherwise
  # Consolidates golangci-lint matrix job results under one required `lint` check
  # Inclusive check: all (new) modules are analyzed, but no need to enable "required" checks for each one
  golangci-matrix-results-validation:
    name: lint
    if: ${{ always() && needs.golangci.result != 'skipped' }}
    needs: [golangci]
    runs-on: ubuntu-latest
    steps:
      - name: Check Golangci-lint Matrix Results
        if: ${{ needs.golangci.result != 'success' }}
        run: |
          echo "At least one 'GolangCI Lint' matrix job failed. Check the failed lint jobs."
          exit 1

  core:
    env:
      # We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests
      # when they should not be using it, while still allowing us to DRY up the setup
      DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable
    strategy:
      fail-fast: false
      matrix:
        type:
          - cmd: go_core_tests
            os: ubuntu22.04-32cores-128GB
            printResults: true
          - cmd: go_core_tests_integration
            os: ubuntu22.04-32cores-128GB
            printResults: true
          - cmd: go_core_ccip_deployment_tests
            os: ubuntu22.04-32cores-128GB
            printResults: true
          - cmd: go_core_fuzz
            os: ubuntu22.04-8cores-32GB
          - cmd: go_core_race_tests
            # use 64cores for certain scheduled runs only
            os: ${{ needs.run-frequency.outputs.two-per-day-frequency == 'true' && 'ubuntu-latest-64cores-256GB' || 'ubuntu-latest-32cores-128GB' }}
    name: Core Tests (${{ matrix.type.cmd }})
    # We don't directly merge dependabot PRs, so let's not waste the resources
    if: ${{ github.actor != 'dependabot[bot]' }}
    needs: [filter, run-frequency]
    runs-on: ${{ matrix.type.os }}
    permissions:
      id-token: write
      contents: read
      actions: read
    steps:
      - name: Checkout the repo
        uses: actions/checkout@v4.2.1
        with:
          persist-credentials: false

      - name: Change Modtime of Files (cache optimization)
        shell: bash
        run: |
          find . -type f,d -exec touch -r {} -d '1970-01-01T00:00:01' {} \; || true

      - name: Setup NodeJS
        if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
        uses: ./.github/actions/setup-nodejs
        with:
          prod: "true"

      - name: Install Foundry
        uses: ./.github/actions/install-solidity-foundry

      - name: Setup Go
        if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
        uses: ./.github/actions/setup-go
        with:
          # race/fuzz tests don't benefit repeated caching, so restore from develop's build cache
          restore-build-cache-only: ${{ matrix.type.cmd == 'go_core_fuzz' }}
          build-cache-version: ${{ matrix.type.cmd }}

      - name: Setup Solana
        if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
        uses: ./.github/actions/setup-solana

      - name: Build Solana artifacts
        if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
        uses: ./.github/actions/setup-solana/build-contracts

      - name: Setup wasmd
        if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
        uses: ./.github/actions/setup-wasmd

      - name: Setup Postgres
        if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
        uses: ./.github/actions/setup-postgres

      - name: Touching core/web/assets/index.html
        if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
        run: mkdir -p core/web/assets && touch core/web/assets/index.html

      - name: Download Go vendor packages
        if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
        run: go mod download

      - name: Build binary
        if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
        run: go build -o chainlink.test .

      - name: Setup DB
        if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
        run: ./chainlink.test local db preparetest
        env:
          CL_DATABASE_URL: ${{ env.DB_URL }}

      - name: Install LOOP Plugins
        if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
        run: make install-plugins

      - name: Increase Timeouts for Fuzz/Race
        # Increase timeouts for scheduled runs only
        if: ${{ github.event.schedule != '' && needs.filter.outputs.should-run-ci-core == 'true' }}
        run: |
          echo "TIMEOUT=10m" >> $GITHUB_ENV
          echo "COUNT=50" >> $GITHUB_ENV
          echo "FUZZ_TIMEOUT_MINUTES=10">> $GITHUB_ENV

      - name: Run tests
        if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}
        id: run-tests
        env:
          OUTPUT_FILE: ./output.txt
          CL_DATABASE_URL: ${{ env.DB_URL }}
        run: ./tools/bin/${{ matrix.type.cmd }} ./...

      - name: Print Races
        id: print-races
        if: ${{ failure() && matrix.type.cmd == 'go_core_race_tests' && needs.filter.outputs.should-run-ci-core == 'true' }}
        env:
          GH_REPO: ${{ github.repository }}
          GH_RUN_ID: ${{ github.run_id }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          run_url=$(gh run --repo "$GH_REPO" view "$GH_RUN_ID" --json jobs --jq '.jobs[] | select(.name | contains("${{ matrix.type.cmd }}")) | .url')
          echo "run_url=$run_url" >> $GITHUB_OUTPUT

          find race.* | xargs cat > race.txt
          if [[ -s race.txt ]]; then
            cat race.txt
            echo "post_to_slack=true" >> $GITHUB_OUTPUT
          else
            echo "post_to_slack=false" >> $GITHUB_OUTPUT
          fi
          echo "github.event_name: ${{ github.event_name }}"
          echo "github.ref: ${{ github.ref }}"

      - name: Print postgres logs
        if: ${{ always() &&  needs.filter.outputs.should-run-ci-core == 'true' }}
        run: docker compose logs postgres | tee ../../../postgres_logs.txt
        working-directory: ./.github/actions/setup-postgres

      - name: Store logs artifacts
        if: ${{ always() && needs.filter.outputs.should-run-ci-core == 'true' }}
        uses: actions/upload-artifact@v4.4.3
        with:
          name: ${{ matrix.type.cmd }}_logs
          path: |
            ./output.txt
            ./output-short.txt
            ./race.*
            ./coverage.txt
            ./postgres_logs.txt
          retention-days: 7

      - name: Notify Slack on Race Test Failure
        if: |
          failure() &&
          matrix.type.cmd == 'go_core_race_tests' &&
          steps.print-races.outputs.post_to_slack == 'true'  &&
          (github.event_name == 'merge_group' || github.ref == 'refs/heads/develop') &&
          needs.filter.outputs.should-run-ci-core == 'true'
        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
        with:
          method: chat.postMessage
          token: ${{ secrets.QA_SLACK_API_KEY }}
          payload: |
            channel: ${{ secrets.SLACK_TOPIC_DATA_RACES_CHANNEL_ID}}
            text: "Race Tests Failed: <${{ steps.print-races.outputs.run_url }}|Run>"

  core-scripts-tests:
    name: test-scripts
    needs: [filter]
    runs-on: ubuntu-latest
    if: ${{  needs.filter.outputs.should-run-scripts-test == 'true' }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4.2.1

      - name: Setup Go
        uses: ./.github/actions/setup-go
        with:
          go-version-file: core/scripts/go.mod
          go-module-file: core/scripts/go.sum

      - name: Run Tests
        env:
          OUTPUT_FILE: ./output.txt
        run: ./tools/bin/go_core_scripts_tests ./...

      - name: Store test report artifacts
        if: ${{ always() }}
        uses: actions/upload-artifact@v4.4.3
        with:
          name: go_core_scripts_tests_logs
          path: |
            ./output.txt
            ./coverage.txt
          retention-days: 7

  scan:
    name: SonarQube Scan
    needs: [golangci, core, core-scripts-tests]
    if: ${{ always() && github.actor != 'dependabot[bot]' }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout the repo
        uses: actions/checkout@v4.2.1
        with:
          persist-credentials: false
          fetch-depth: 0 # fetches all history for all tags and branches to provide more metadata for sonar reports

      - name: Download all workflow artifacts
        uses: actions/download-artifact@v4.1.8

      - name: Check and Set SonarQube Report Paths
        shell: bash
        run: |
          # Check and assign paths for coverage/test reports in go_core_tests_logs
          core_artifact="go_core_tests_logs"
          if [ -d "$core_artifact" ]; then
            echo "Found $core_artifact"
            sonarqube_coverage_report_paths=$(find "$core_artifact" -name coverage.txt | paste -sd "," -)
            sonarqube_tests_report_paths=$(find "$core_artifact" -name output.txt | paste -sd "," -)
            echo "Coverage report paths: $sonarqube_coverage_report_paths"
            echo "Tests report paths: $sonarqube_tests_report_paths"
          else
            echo "Did not find $core_artifact"
            sonarqube_coverage_report_paths=""
            sonarqube_tests_report_paths=""
          fi

          # Check and assign paths for coverage/test reports in go_core_tests_integration_logs
          integration_tests_artifact="go_core_tests_integration_logs"
          if [ -d "$integration_tests_artifact" ]; then
            echo "Found $integration_tests_artifact"
            integration_coverage_paths=$(find "$integration_tests_artifact" -name coverage.txt | paste -sd "," -)
            integration_tests_paths=$(find "$integration_tests_artifact" -name output.txt | paste -sd "," -)

            # Append to existing paths if they are set, otherwise assign directly
            sonarqube_coverage_report_paths="${sonarqube_coverage_report_paths:+$sonarqube_coverage_report_paths,}$integration_coverage_paths"
            sonarqube_tests_report_paths="${sonarqube_tests_report_paths:+$sonarqube_tests_report_paths,}$integration_tests_paths"
          fi

          # Check and assign paths for coverage/test reports in go_core_scripts_tests_logs
          scripts_tests_artifact="go_core_scripts_tests_logs"
          if [ -d "$scripts_tests_artifact" ]; then
              echo "Found $scripts_tests_artifact"
              scripts_coverage_paths=$(find "$scripts_tests_artifact" -name coverage.txt | paste -sd "," -)
              scripts_tests_paths=$(find "$scripts_tests_artifact" -name output.txt | paste -sd "," -)

              # Append to existing paths if they are set, otherwise assign directly
              sonarqube_coverage_report_paths="${sonarqube_coverage_report_paths:+$sonarqube_coverage_report_paths,}$scripts_coverage_paths"
              sonarqube_tests_report_paths="${sonarqube_tests_report_paths:+$sonarqube_tests_report_paths,}$scripts_tests_paths"
          fi

          # Check and assign paths for lint reports
          # To find reports in the folders named differently (because of the matrix strategy),
          # We need to loop through the artifacts. It allows usage of RegExp folders (skipped if not found).
          for golang_lint_artifact in golangci-lint-report*
          do
              echo "Found golangci-lint-report artifacts"
              sonarqube_lint_report_paths=$(find -type f -name 'golangci-lint-report.xml' -printf "%p,")
              echo "Lint report paths: $sonarqube_lint_report_paths"
              break
          done

          ARGS=""
          if [[ -z "$sonarqube_tests_report_paths" ]]; then
            echo "::warning::No test report paths found, will not pass to sonarqube"
          else
            echo "Found test report paths: $sonarqube_tests_report_paths"
            ARGS="$ARGS -Dsonar.go.tests.reportPaths=$sonarqube_tests_report_paths"
          fi

          if [[ -z "$sonarqube_coverage_report_paths" ]]; then
            echo "::warning::No coverage report paths found, will not pass to sonarqube"
          else
            echo "Found coverage report paths: $sonarqube_coverage_report_paths"
            ARGS="$ARGS -Dsonar.go.coverage.reportPaths=$sonarqube_coverage_report_paths"
          fi

          if [[ -z "$sonarqube_lint_report_paths" ]]; then
            echo "::warning::No lint report paths found, will not pass to sonarqube"
          else
            echo "Found lint report paths: $sonarqube_lint_report_paths"
            ARGS="$ARGS -Dsonar.go.golangci-lint.reportPaths=$sonarqube_lint_report_paths"
          fi

          echo "Final SONARQUBE_ARGS: $ARGS"
          echo "SONARQUBE_ARGS=$ARGS" >> $GITHUB_ENV

      - name: SonarQube Scan
        if: ${{ env.SONARQUBE_ARGS != '' }}
        uses: sonarsource/sonarqube-scan-action@aecaf43ae57e412bd97d70ef9ce6076e672fe0a9 # v2.3.0
        with:
          args: ${{ env.SONARQUBE_ARGS }}
        env:
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
          SONAR_SCANNER_OPTS: "-Xms6g -Xmx8g"

  clean:
    name: Clean Go Tidy & Generate
    if: ${{ github.actor != 'dependabot[bot]' }}
    runs-on: ubuntu22.04-8cores-32GB
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@v4.2.1
        with:
          persist-credentials: false
          fetch-depth: 0
      - name: Setup Go
        uses: ./.github/actions/setup-go
        with:
          only-modules: "true"
      - name: Install protoc-gen-go-wsrpc
        run: curl https://github.com/smartcontractkit/wsrpc/raw/main/cmd/protoc-gen-go-wsrpc/protoc-gen-go-wsrpc --output $HOME/go/bin/protoc-gen-go-wsrpc && chmod +x $HOME/go/bin/protoc-gen-go-wsrpc
      - name: Setup NodeJS
        uses: ./.github/actions/setup-nodejs
      - name: Install Foundry
        uses: ./.github/actions/install-solidity-foundry
      - name: make generate
        run: |
          make rm-mocked
          make generate
      - name: Ensure clean after generate
        run: |
          git add --all
          git diff --stat --cached --exit-code
      - run: make gomodtidy
      - name: Ensure clean after tidy
        run: |
          git add --all
          git diff --minimal --cached --exit-code

  run-frequency:
    name: Scheduled Run Frequency
    outputs:
      one-per-day-frequency: ${{ steps.check-time.outputs.one-per-day-frequency || 'false' }}
      two-per-day-frequency: ${{ steps.check-time.outputs.two-per-day-frequency || 'false' }}
      four-per-day-frequency: ${{ steps.check-time.outputs.four-per-day-frequency || 'false' }}
      six-per-day-frequency: ${{ steps.check-time.outputs.six-per-day-frequency || 'false' }}
    runs-on: ubuntu-latest
    steps:
      - name: Check time and set frequencies
        id: check-time
        shell: bash
        run: |
          if [ "$GITHUB_EVENT_NAME" != "schedule" ]; then
            # Not a scheduled event, set all frequencies to false
            echo "one-per-day-frequency=false" >> $GITHUB_OUTPUT
            echo "two-per-day-frequency=false" >> $GITHUB_OUTPUT
            echo "four-per-day-frequency=false" >> $GITHUB_OUTPUT
            echo "six-per-day-frequency=false" >> $GITHUB_OUTPUT
          else
            # Scheduled event, check current time for frequencies
            current_hour=$(date +"%H")

            # Check if the current hour is 00 (one per day)
            if [ "$current_hour" -eq "00" ]; then
              echo "one-per-day-frequency=true" >> $GITHUB_OUTPUT
            fi

            # Check if the current hour is 00 or 12 (twice per day)
            if [ "$current_hour" -eq "00" ] || [ "$current_hour" -eq "12" ]; then
              echo "two-per-day-frequency=true" >> $GITHUB_OUTPUT
            fi

            # Check if the current hour is 00, 06, 12, or 18 (four times per day)
            if [ "$current_hour" -eq "00" ] || [ "$current_hour" -eq "06" ] || [ "$current_hour" -eq "12" ] || [ "$current_hour" -eq "18" ]; then
              echo "four-per-day-frequency=true" >> $GITHUB_OUTPUT
            fi

            # Check if the current hour is one of 00, 04, 08, 12, 16, or 20 (six times per day)
            if [ "$current_hour" -eq "00" ] || [ "$current_hour" -eq "04" ] || [ "$current_hour" -eq "08" ] || [ "$current_hour" -eq "12" ] || [ "$current_hour" -eq "16" ] || [ "$current_hour" -eq "20" ]; then
              echo "six-per-day-frequency=true" >> $GITHUB_OUTPUT
            fi
          fi