selfdrive_tests.yaml

name: selfdrive

on:
  push:
    branches:
      - master
  pull_request:
  workflow_dispatch:

concurrency:
  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
  cancel-in-progress: true

env:
  PYTHONWARNINGS: error
  BASE_IMAGE: openpilot-base
  AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}

  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
  BUILD: selfdrive/test/docker_build.sh base

  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PRE_COMMIT_HOME=/tmp/pre-commit -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/pre-commit:/tmp/pre-commit -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c

  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical

jobs:
  build_release:
    name: build release
    runs-on: ubuntu-20.04
    env:
      STRIPPED_DIR: /tmp/releasepilot
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - run: git lfs pull
    - name: Build devel
      timeout-minutes: 1
      run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
    - uses: ./.github/workflows/setup-pre-commit
    - uses: ./.github/workflows/setup-with-retry
    - name: Check submodules
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      timeout-minutes: 1
      run: release/check-submodules.sh
    - name: Build openpilot and run checks
      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
      run: |
        cd $STRIPPED_DIR
        ${{ env.RUN }} "python selfdrive/manager/build.py"
    - name: Run tests
      timeout-minutes: 3
      run: |
        cd $STRIPPED_DIR
        ${{ env.RUN }} "release/check-dirty.sh && \
                        MAX_EXAMPLES=5 $PYTEST selfdrive/car"
    - name: pre-commit
      timeout-minutes: 3
      run: |
        cd $GITHUB_WORKSPACE
        cp .pre-commit-config.yaml $STRIPPED_DIR
        cp pyproject.toml $STRIPPED_DIR
        cp poetry.lock $STRIPPED_DIR
        cd $STRIPPED_DIR
        ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files pre-commit run --all && chmod -R 777 /tmp/pre-commit"

  build:
    strategy:
      matrix:
        arch: ${{ fromJson(
           ((github.repository == 'commaai/openpilot') &&
              ((github.event_name != 'pull_request') || 
               (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
    runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-20.04' }}
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
      with:
        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
    - uses: ./.github/workflows/compile-openpilot
      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache

  docker_push:
    name: docker push
    strategy:
      matrix:
        arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
    runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-20.04' }}
    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - name: Setup to push to repo
      run: |
        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
        echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
        $DOCKER_LOGIN
    - uses: ./.github/workflows/setup-with-retry
      with:
        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}

  docker_push_multiarch:
    name: docker push multiarch tag
    runs-on: ubuntu-20.04
    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
    needs: [docker_push]
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: false
    - name: Setup docker
      run: |
        $DOCKER_LOGIN
    - name: Merge x64 and arm64 tags
      run: |
        export PUSH_IMAGE=true
        scripts/retry.sh selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64

  static_analysis:
    name: static analysis
    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                   ((github.event_name != 'pull_request') || 
                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }}
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-pre-commit
    - uses: ./.github/workflows/setup-with-retry
    - name: pre-commit
      timeout-minutes: 4
      run: ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all && chmod -R 777 /tmp/pre-commit"

  valgrind:
    name: valgrind
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Build openpilot
      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run valgrind
      timeout-minutes: 1
      run: |
        ${{ env.RUN }} "python selfdrive/test/test_valgrind_replay.py"
    - name: Print logs
      if: always()
      run: cat selfdrive/test/valgrind_logs.txt

  unit_tests:
    name: unit tests
    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                   ((github.event_name != 'pull_request') || 
                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }}
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
      with:
        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
    - name: Build openpilot
      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run unit tests
      timeout-minutes: 15
      run: |
        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
                        export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \
                        $PYTEST --timeout 60 -m 'not slow' && \
                        ./selfdrive/ui/tests/create_test_translations.sh && \
                        QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
                        ./selfdrive/ui/tests/test_translations.py"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v3
      with:
        name: ${{ github.job }}
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  process_replay:
    name: process replay
    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                   ((github.event_name != 'pull_request') || 
                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }}
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
      with:
        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
    - name: Cache test routes
      id: dependency-cache
      uses: actions/cache@v3
      with:
        path: .ci_cache/comma_download_cache
        key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit') }}
    - name: Build openpilot
      run: |
        ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run replay
      timeout-minutes: 30
      run: |
        ${{ env.RUN }} "coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
                        chmod -R 777 /tmp/comma_download_cache && \
                        coverage combine && \
                        coverage xml"
    - name: Print diff
      id: print-diff
      if: always()
      run: cat selfdrive/test/process_replay/diff.txt
    - uses: actions/upload-artifact@v3
      if: always()
      continue-on-error: true
      with:
        name: process_replay_diff.txt
        path: selfdrive/test/process_replay/diff.txt
    - name: Upload reference logs
      if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
      run: |
        ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v3
      with:
        name: ${{ github.job }}
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  regen:
    name: regen
    runs-on: 'ubuntu-20.04'
    steps: 
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Cache test routes
      id: dependency-cache
      uses: actions/cache@v3
      with:
        path: .ci_cache/comma_download_cache
        key: regen-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_regen.py') }}
    - name: Build base Docker image
      run: eval "$BUILD"
    - name: Build openpilot
      run: |
        ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run regen
      timeout-minutes: 30
      run: |
        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
                           chmod -R 777 /tmp/comma_download_cache"

  test_modeld:
    name: model tests
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Build base Docker image
      run: eval "$BUILD"
    - name: Build openpilot
      run: |
        ${{ env.RUN }} "scons -j$(nproc)"
    # PYTHONWARNINGS triggers a SyntaxError in onnxruntime
    - name: Run model replay with ONNX
      timeout-minutes: 4
      run: |
        ${{ env.RUN }} "unset PYTHONWARNINGS && \
                           ONNXCPU=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \
                           coverage combine && \
                           coverage xml"
    - name: Run unit tests
      timeout-minutes: 4
      run: |
        ${{ env.RUN }} "unset PYTHONWARNINGS && \
                           $PYTEST selfdrive/modeld"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v3
      with:
        name: ${{ github.job }}
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  test_cars:
    name: cars
    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                   ((github.event_name != 'pull_request') || 
                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }}
    strategy:
      fail-fast: false
      matrix:
        job: [0, 1, 2, 3, 4]
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Cache test routes
      id: dependency-cache
      uses: ./.github/workflows/auto-cache
      with:
        path: .ci_cache/comma_download_cache
        key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
    - name: Build openpilot
      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: Test car models
      timeout-minutes: 10
      run: |
        ${{ env.RUN }} "$PYTEST selfdrive/car/tests/test_models.py && \
                        chmod -R 777 /tmp/comma_download_cache"
      env:
        NUM_JOBS: 5
        JOB_ID: ${{ matrix.job }}
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v3
      with:
        name: ${{ github.job }}-${{ matrix.job }}
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  car_docs_diff:
    name: PR comments
    runs-on: ubuntu-20.04
    if: github.event_name == 'pull_request'
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
          ref: ${{ github.event.pull_request.base.ref }}
      - run: git lfs pull
      - uses: ./.github/workflows/setup-with-retry
      - name: Get base car info
        run: |
          ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_info.py --path /tmp/openpilot_cache/base_car_info"
          sudo chown -R $USER:$USER ${{ github.workspace }}
      - uses: actions/checkout@v4
        with:
          submodules: true
          path: current
      - run: cd current && git lfs pull
      - name: Save car docs diff
        id: save_diff
        run: |
          cd current
          ${{ env.RUN }} "scons -j$(nproc)"
          output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_info")
          output="${output//$'\n'/'%0A'}"
          echo "::set-output name=diff::$output"
      - name: Find comment
        if: ${{ env.AZURE_TOKEN != '' }}
        uses: peter-evans/find-comment@1769778a0c5bd330272d749d12c036d65e70d39d
        id: fc
        with:
          issue-number: ${{ github.event.pull_request.number }}
          body-includes: This PR makes changes to
      - name: Update comment
        if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }}
        uses: peter-evans/create-or-update-comment@b95e16d2859ad843a14218d1028da5b2c4cbc4b4
        with:
          comment-id: ${{ steps.fc.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body: "${{ steps.save_diff.outputs.diff }}"
          edit-mode: replace
      - name: Delete comment
        if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }}
        uses: actions/github-script@v6
        with:
          script: |
            github.rest.issues.deleteComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: ${{ steps.fc.outputs.comment-id }}
            })

  create_ui_report:
    name: Create UI Report
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - uses: ./.github/workflows/setup-with-retry
      - name: Build openpilot
        run: ${{ env.RUN }} "scons -j$(nproc)"
      - name: Create Test Report
        run: >
            ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
                            source selfdrive/test/setup_xvfb.sh &&
                            export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && 
                            python selfdrive/ui/tests/test_ui/run.py"
      - name: Upload Test Report
        uses: actions/upload-artifact@v2
        with:
          name: report
          path: selfdrive/ui/tests/test_ui/report