diff --git a/.github/workflows/push-and-comment.yaml b/.github/workflows/push-and-comment.yaml index 817fa965..78382d21 100644 --- a/.github/workflows/push-and-comment.yaml +++ b/.github/workflows/push-and-comment.yaml @@ -49,7 +49,7 @@ jobs: - name: Push boot, system and agnos.json working-directory: ${{ github.workspace }}/ci-artifacts run: | - mv ota.json agnos.json && rm ota-staging.json system-skip-chunks-*.img.xz + mv ota.json agnos.json && rm ota-staging.json git checkout -b agnos-builder/pr-${{ env.PR_NUMBER }} git config user.name "GitHub Actions Bot" git config user.email "<>" diff --git a/Dockerfile.builder b/Dockerfile.builder index d2de8cd8..2a23ae04 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -17,8 +17,9 @@ RUN apt-get update && \ python-is-python2 \ openssl \ ccache \ - android-sdk-libsparse-utils \ libcap2-bin \ + android-sdk-libsparse-utils \ + e2fsprogs \ && rm -rf /var/lib/apt/lists/* RUN if [ ${UID:-0} -ne 0 ] && [ ${GID:-0} -ne 0 ]; then \ diff --git a/build_system.sh b/build_system.sh index 433d26e0..fb3095f0 100755 --- a/build_system.sh +++ b/build_system.sh @@ -14,11 +14,14 @@ BUILD_DIR="$DIR/build" OUTPUT_DIR="$DIR/output" ROOTFS_DIR="$BUILD_DIR/agnos-rootfs" -ROOTFS_IMAGE="$BUILD_DIR/system.img.raw" -ROOTFS_IMAGE_SIZE=10G -SPARSE_IMAGE="$OUTPUT_DIR/system.img" +ROOTFS_IMAGE="$BUILD_DIR/system.img" SKIP_CHUNKS_IMAGE="$OUTPUT_DIR/system-skip-chunks.img" +# the partition is 10G, but openpilot's updater didn't always handle the full size +# - the size will also get shrunk with "resize2fs -M" +# - openpilot fix, shipped in 0.9.8 (8/18/24): https://github.com/commaai/openpilot/pull/33320 +ROOTFS_IMAGE_SIZE=5G + # Create temp dir if non-existent mkdir -p $BUILD_DIR $OUTPUT_DIR @@ -129,18 +132,20 @@ exec_as_root bash -c "set -e; export ROOTFS_DIR=$ROOTFS_DIR GIT_HASH=$GIT_HASH; echo "Unmount filesystem" exec_as_root umount -l $ROOTFS_DIR -# Sparsify -echo "Sparsify image $(basename $SPARSE_IMAGE)" -exec_as_user bash -c "\ -TMP_SPARSE=\$(mktemp); \ -img2simg $ROOTFS_IMAGE \$TMP_SPARSE; \ -mv \$TMP_SPARSE $SPARSE_IMAGE" - # Make image with skipped chunks echo "Sparsify image $(basename $SKIP_CHUNKS_IMAGE)" exec_as_user bash -c "\ +TMP_SPARSE=\$(mktemp); \ +img2simg $ROOTFS_IMAGE \$TMP_SPARSE; \ TMP_SKIP=\$(mktemp); \ -$DIR/tools/simg2dontcare.py $SPARSE_IMAGE \$TMP_SKIP; \ +$DIR/tools/simg2dontcare.py $TMP_SPARSE \$TMP_SKIP; \ mv \$TMP_SKIP $SKIP_CHUNKS_IMAGE" +# Reduce system image to the minimum size +exec_as_user e2fsck -fy $ROOTFS_IMAGE +exec_as_user resize2fs -M $ROOTFS_IMAGE + +# Copy system image to output +cp $ROOTFS_IMAGE $OUTPUT_DIR + echo "Done!" diff --git a/load_kernel.sh b/load_kernel.sh index 2b64aa5b..b5693b70 100755 --- a/load_kernel.sh +++ b/load_kernel.sh @@ -9,6 +9,7 @@ sudo dd if=/data/tmp/boot.img of=/dev/disk/by-partlabel/boot_a sudo dd if=/data/tmp/boot.img of=/dev/disk/by-partlabel/boot_b sudo mount -o rw,remount / +sudo resize2fs $(findmnt -n -o SOURCE /) sudo mv /data/tmp/wlan.ko /usr/comma/wlan.ko rm -rf /data/tmp/* sudo mount -o ro,remount / || true diff --git a/scripts/check-space.sh b/scripts/check-space.sh index d67f3b14..d1956a71 100755 --- a/scripts/check-space.sh +++ b/scripts/check-space.sh @@ -2,6 +2,6 @@ # sudo apt install ncdu -sudo mount build/system.img.raw build/agnos-rootfs +sudo mount build/system.img build/agnos-rootfs sudo ncdu build/agnos-rootfs/ || true sudo umount build/agnos-rootfs diff --git a/scripts/package_ota.py b/scripts/package_ota.py index d10a3d7f..eb45d27c 100755 --- a/scripts/package_ota.py +++ b/scripts/package_ota.py @@ -25,23 +25,12 @@ def compress(fin, fout) -> None: subprocess.check_call(f"xz -T4 -vc {fin} > {fout}", shell=True) -def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None): +def process_file(fn, name, full_check=True, has_ab=True, alt=None): print(name) hash_raw = hash = checksum(fn) size = fn.stat().st_size print(f" {size} bytes, hash {hash}") - if sparse: - raw_img = BUILD_DIR / "system.img.raw" - if raw_img.exists(): - print(" using existing raw image") - hash_raw = checksum(raw_img) - size = raw_img.stat().st_size - else: - print("Error: existing raw image not found") - exit(1) - print(f" {size} bytes, hash {hash_raw} (raw)") - print(" compressing") xz_fn = OTA_OUTPUT_DIR / f"{fn.stem}-{hash_raw}.img.xz" compress(fn, xz_fn) @@ -52,7 +41,7 @@ def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None) "hash": hash, "hash_raw": hash_raw, "size": size, - "sparse": sparse, + "sparse": False, "full_check": full_check, "has_ab": has_ab, } @@ -81,7 +70,7 @@ def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None) files = [ process_file(OUTPUT_DIR / "boot.img", "boot"), - process_file(OUTPUT_DIR / "system.img", "system", sparse=True, full_check=False, alt=OUTPUT_DIR / "system-skip-chunks.img"), + process_file(OUTPUT_DIR / "system.img", "system", full_check=False, alt=OUTPUT_DIR / "system-skip-chunks.img"), ] configs = [ (AGNOS_UPDATE_URL, "ota.json"), diff --git a/scripts/setup-on-device.sh b/scripts/setup-on-device.sh index ba9e21d1..e5d1bd06 100755 --- a/scripts/setup-on-device.sh +++ b/scripts/setup-on-device.sh @@ -10,6 +10,7 @@ if [ ! -f /AGNOS ]; then fi sudo mount -o rw,remount / +sudo resize2fs $(findmnt -n -o SOURCE /) echo "symlink /usr/comma" sudo rm -rf /usr/comma diff --git a/userspace/usr/comma/apt_setup.sh b/userspace/usr/comma/apt_setup.sh index d84e5b81..75e3b604 100755 --- a/userspace/usr/comma/apt_setup.sh +++ b/userspace/usr/comma/apt_setup.sh @@ -1,6 +1,7 @@ #!/bin/bash -e sudo mount -o rw,remount / +sudo resize2fs $(findmnt -n -o SOURCE /) &>/dev/null || sudo resize2fs $(findmnt -n -o SOURCE /) sudo mount -o remount,size=1500M /var sudo cp -r /usr/default/var/lib/dpkg /var/lib/ sudo sed -i '/bionic/s/^/#/' /etc/apt/sources.list diff --git a/userspace/usr/comma/shims/pip b/userspace/usr/comma/shims/pip index b97b2142..1eeac50f 100755 --- a/userspace/usr/comma/shims/pip +++ b/userspace/usr/comma/shims/pip @@ -7,6 +7,7 @@ export TMPDIR=/tmp/pip-tmp mkdir -p $TMPDIR sudo mount -o remount,size=2G /tmp sudo mount -o rw,remount / +sudo resize2fs $(findmnt -n -o SOURCE /) &>/dev/null || sudo resize2fs $(findmnt -n -o SOURCE /) # run command sudo TMPDIR=$TMPDIR PIP_NO_CACHE_DIR=1 $PIP_PATH "$@"