WIP: do openpilot ci build in a cached ubuntu chroot instead of docker.

.github/workflows/setup/action.yaml

@@ -37,27 +37,24 @@ runs:
       run: |
         docker login -u adeebshihadeh -p ${{ inputs.docker_hub_pat }}
 
-    # build cache
-    - id: date
-      shell: bash
-      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
-    - shell: bash
-      run: echo "$CACHE_COMMIT_DATE"
-    - id: scons-cache
-      uses: ./.github/workflows/auto-cache
-      with:
-        path: .ci_cache/scons_cache
-        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
-        restore-keys: |
-          scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
-          scons-${{ runner.arch }}
-    # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
-    - id: normalize-file-permissions
-      shell: bash
-      name: Normalize file permissions to ensure a consistent docker build cache
-      run: |
-        find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
-        find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
+    ## build cache
+    #- id: date
+    #  shell: bash
+    #  run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
+    #- shell: bash
+    #  run: echo "$CACHE_COMMIT_DATE"
+    #- id: scons-cache
+    #  uses: ./.github/workflows/auto-cache
+    #  with:
+    #    path: .ci_cache/scons_cache
+    #    key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+    #    restore-keys: |
+    #      scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
+    #      scons-${{ runner.arch }}
+
+    - id: debug
+      uses: ./.github/workflows/ubuntu-chroot
+
     # build our docker image
-    - shell: bash
-      run: eval ${{ env.BUILD }}
+    #- shell: bash
+    #  run: eval ${{ env.BUILD }}

.github/workflows/tools_tests.yaml

@@ -1,28 +1,14 @@
 name: tools
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-  workflow_call:
-    inputs:
-      run_number:
-        default: '1'
-        required: true
-        type: string
+on: workflow_dispatch
 concurrency:
-  group: tools-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  group: wip
   cancel-in-progress: true
 
 env:
   BASE_IMAGE: openpilot-base
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-
-  BUILD: selfdrive/test/docker_build.sh base
-
-  RUN: docker run --shm-size 2G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
-
+  RUN: sudo chroot /tmp/chroot /run_ci.sh
 
 jobs:
   simulator_driving:
@@ -37,36 +23,9 @@ jobs:
     - name: Build openpilot
       run: |
         ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Run bridge test
-      run: |
-        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
-                        source selfdrive/test/setup_vsound.sh && \
-                        CI=1 pytest tools/sim/tests/test_metadrive_bridge.py"
+    #- name: Run bridge test
+    #  run: |
+    #    ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
+    #                    source selfdrive/test/setup_vsound.sh && \
+    #                    CI=1 pytest tools/sim/tests/test_metadrive_bridge.py"
 
-  devcontainer:
-    name: devcontainer
-    runs-on: ubuntu-latest
-    if: false  # we can re-enable once this is faster
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Use local image for testing devcontainer with latest base image
-      run: |
-        echo "USE_LOCAL_IMAGE=true" >> "$GITHUB_ENV"
-    - name: Setup Dev Container CLI
-      run: npm install -g @devcontainers/cli
-    - name: Build dev container image
-      run: ./scripts/retry.sh devcontainer build --workspace-folder .
-    - name: Run dev container
-      run: |
-        mkdir -p /tmp/devcontainer_scons_cache/
-        cp -r $GITHUB_WORKSPACE/.ci_cache/scons_cache/. /tmp/devcontainer_scons_cache/
-        devcontainer up --workspace-folder .
-    - name: Test environment
-      run: |
-        devcontainer exec --workspace-folder . scons -j$(nproc) cereal/ common/
-        devcontainer exec --workspace-folder . pip3 install pip-install-test
-        devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
-        devcontainer exec --workspace-folder . sudo touch /root/test.txt

.github/workflows/ubuntu-chroot/action.yaml

@@ -0,0 +1,41 @@
+name: 'build and mount an ubuntu chroot for openpilot ci.'
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Calculate Cache Key
+      id: cache-key
+      shell: bash
+      run: echo "CACHE_KEY=chroot-${{ hashFiles('tools/build_ci_chroot.sh') }}"  >> $GITHUB_ENV
+
+    - name: Restore cache
+      id: restore-cache
+      uses: 'actions/cache/restore@v4'
+      with:
+        path: /tmp/chroot.squashfs
+        key: ${{ env.CACHE_KEY }}
+
+    - name: Rebuild chroot
+      if: steps.restore-cache.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        sudo bash -c "GITHUB_WORKSPACE=$GITHUB_WORKSPACE ./tools/build_ci_chroot.sh"
+
+    - name: Cache chroot
+      if: steps.restore-cache.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: /tmp/chroot.squashfs
+        key: ${{ env.CACHE_KEY }}
+
+    - name: Mount chroot
+      shell: bash
+      run: |
+        mkdir /tmp/chroot
+        mkdir /tmp/.chroot_lower
+        mkdir /tmp/.chroot_upper
+        mkdir /tmp/.chroot_work
+        sudo mount -t squashfs /tmp/chroot.squashfs /tmp/.chroot_lower
+        sudo mount -t overlay overlay -o lowerdir=/tmp/.chroot_lower,upperdir=/tmp/.chroot_upper,workdir=/tmp/.chroot_work /tmp/chroot
+        sudo mount --bind "$GITHUB_WORKSPACE" /tmp/chroot/tmp/openpilot

tools/build_ci_chroot.sh

@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+set -eux
+
+apt-get update
+apt-get install -y debootstrap squashfs-tools
+
+mkdir /tmp/chroot
+
+# Install ubuntu into a chroot for us to later install our dependencies into.
+debootstrap --variant=minbase noble /tmp/chroot http://archive.ubuntu.com/ubuntu
+# The devices cause untar issues and we don't need them.
+rm -rf /tmp/chroot/dev/*
+
+cat <<EOF > /tmp/chroot/etc/apt/sources.list
+deb http://archive.ubuntu.com/ubuntu noble main
+deb http://archive.ubuntu.com/ubuntu noble universe
+deb http://archive.ubuntu.com/ubuntu noble multiverse
+deb http://archive.ubuntu.com/ubuntu noble restricted
+EOF
+
+cat <<EOF >> /tmp/chroot/etc/profile
+DEBIAN_FRONTEND=noninteractive 
+NVIDIA_VISIBLE_DEVICES=all
+NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
+QTWEBENGINE_DISABLE_SANDBOX=1
+LANG=en_US.UTF-8
+LANGUAGE=en_US:en
+LC_ALL=en_US.UTF-8
+PYTHONUNBUFFERED=1
+LD_LIBRARY_PATH=/usr/lib:/usr/lib/x86_64-linux-gnu
+EOF
+
+cat <<EOF > /tmp/chroot/tmp/install-deps.sh
+#!/usr/bin/env bash
+set -eux
+
+apt-get update
+apt-get install -y --no-install-recommends \
+  sudo \
+  tzdata \
+  locales \
+  ssh \
+  pulseaudio \
+  xvfb \
+  x11-xserver-utils \
+  gnome-screenshot \
+  apt-utils \
+  alien \
+  unzip \
+  tar \
+  curl \
+  xz-utils \
+  dbus \
+  gcc-arm-none-eabi \
+  tmux \
+  vim \
+  libx11-6 \
+  wget
+
+dbus-uuidgen > /etc/machine-id
+
+sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
+locale-gen
+
+cd /tmp/openpilot
+./tools/ubuntu_setup.sh
+
+mkdir -p /tmp/opencl-driver-intel
+cd /tmp/opencl-driver-intel
+wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz
+wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz
+mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel
+cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel
+tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz
+mkdir -p /etc/OpenCL/vendors
+echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd
+cd /opt/intel
+tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz
+ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64
+ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64
+ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64
+ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64
+mkdir -p /etc/ld.so.conf.d 
+echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf
+ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf
+cd /
+rm -rf /tmp/opencl-driver-intel
+
+# Remove arm architecture toolchains that we don't want.
+cd /usr/lib/gcc/arm-none-eabi/*
+rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
+
+git config --global --add safe.directory /tmp/openpilot
+
+# Remove all tmp files except 'openpilot' directory and its contents
+find /tmp -mindepth 1 -path '/tmp/openpilot' -prune -o -exec rm -rf {} +
+
+# Remove cached apt archives.
+apt clean
+EOF
+chmod +x /tmp/chroot/tmp/install-deps.sh
+
+cat <<EOF > /tmp/chroot/run_ci.sh
+#!/usr/bin/env bash
+set -eux
+env
+cd /tmp/openpilot
+ldd ./.venv/bin/python
+. ./.venv/bin/activate
+bash -c "\$1"
+EOF
+chmod +x /tmp/chroot/run_ci.sh
+
+cp /etc/resolv.conf /tmp/chroot/resolv.conf
+mount --bind /proc /tmp/chroot/proc
+mount --bind /sys /tmp/chroot/sys
+mount --bind /dev /tmp/chroot/dev
+mkdir /tmp/chroot/tmp/openpilot
+mount --bind "$GITHUB_WORKSPACE" /tmp/chroot/tmp/openpilot
+chroot /tmp/chroot bash /tmp/install-deps.sh
+umount /tmp/chroot/tmp/openpilot
+umount /tmp/chroot/proc
+umount /tmp/chroot/sys
+umount /tmp/chroot/dev
+cd /tmp/chroot
+# A squashfs is faster than docker and we don't need to decompress unused files.
+mksquashfs . /tmp/chroot.squashfs -b 256k -comp zstd -Xcompression-level 1 
+cd /
+rm -rf /tmp/chroot