diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index edafadd2..49fbdbf9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -43,13 +43,6 @@ jobs: id: kernel-submodule run: echo "ref=$(git ls-tree HEAD | awk '$4 == "agnos-kernel-sdm845"' | awk '{print $3}')" | tee -a $GITHUB_OUTPUT - - name: Checkout agnos-kernel-sdm845 - uses: actions/checkout@v4 - with: - repository: commaai/agnos-kernel-sdm845 - ref: ${{ steps.kernel-submodule.outputs.ref }} - path: agnos-kernel-sdm845 - - name: ccache uses: hendrikmuhs/ccache-action@c92f40bee50034e84c763e33b317c77adaa81c92 with: diff --git a/.gitmodules b/.gitmodules index d91f9a6a..c8520a81 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,3 @@ -[submodule "edk2_tici"] - path = edk2_tici - url = ../../commaai/edk2_tici.git -[submodule "agnos-firmware"] - path = agnos-firmware - url = ../../commaai/agnos-firmware.git [submodule "agnos-kernel-sdm845"] path = agnos-kernel-sdm845 url = ../../commaai/agnos-kernel-sdm845.git -[submodule "agnos-binaries"] - path = agnos-binaries - url = ../../commaai/agnos-binaries.git diff --git a/README.md b/README.md index f2450fba..5bab7416 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ # agnos-builder -This is the buidler repo for AGNOS, our Ubuntu based OS. AGNOS runs on the comma three devkit and [comma 3X](https://comma.ai/shop/comma-3x). +This is the builder repo for AGNOS, our Ubuntu based OS. AGNOS runs on the comma three devkit and [comma 3X](https://comma.ai/shop/comma-3x). -NOTE: the `edk2_tici` and `agnos-firmware` submodules are internal, private repos. +> [!NOTE] +> Use https://flash.comma.ai if you're looking to reflash your device to a clean factory state. + +AGNOS includes all the software to boot and run a comma 3/3X. +This repo builds the two main parts of that (system and kernel) and packages it all up into a "release". ## Development @@ -32,7 +36,7 @@ Flashing to a comma 3/3X: Validating changes: * Running openpilot is a good smoketest for general AGNOS functionality * [CI](https://github.com/commaai/agnos-builder/blob/master/.github/workflows/build.yaml) ensures the kernel and system builds work (and pushes the images for you to download) -* [this](https://github.com/commaai/agnos-builder/blob/master/internal/README.md) is the checklist we go through before shipping new AGNOS releases to openpilot +* [this](https://github.com/commaai/agnos-builder/blob/master/TESTING.md) is the checklist we go through before shipping new AGNOS releases to openpilot ### macOS @@ -71,6 +75,6 @@ Some concrete things on the roadmap: - [ ] make the image tiny, for fast updating and flashing https://github.com/commaai/agnos-builder/issues/225 - [ ] update to Ubuntu 24.04 https://github.com/commaai/openpilot/issues/32386 - [ ] mainline Linux kernel https://github.com/commaai/openpilot/issues/32386 -- [ ] fully open source - - [ ] anything from `agnos-firmware`: XBL, ABL, etc. - - [ ] open source Weston https://github.com/commaai/agnos-builder/issues/16 +- [ ] fully open source + - [ ] anything from [firmware.json](firmware.json): XBL, ABL, etc. + - [ ] Weston https://github.com/commaai/agnos-builder/issues/16 diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 00000000..4fbc83ff --- /dev/null +++ b/TESTING.md @@ -0,0 +1,50 @@ +# testing + +## Release Checklist + +- [ ] [`test_onroad`](https://github.com/commaai/openpilot/blob/master/selfdrive/test/test_onroad.py) passes +- [ ] Wi-Fi: lists networks and connects +- [ ] Modem: connects to cell network +- [ ] Image sizes haven't increased +- [ ] Sounds work +- [ ] `python` is our python, not system version +- [ ] Clean openpilot build: `scons -c && scons -j8` +- [ ] Factory reset + - [ ] from openpilot menu + - [ ] tapping on boot + - [ ] corrupt userdata +- [ ] Color calibration + - [ ] from /persist/comma/ + - [ ] directly from panel over sysfs +- [ ] Clean setup: factory reset -> install openpilot -> openpilot works +- [ ] AGNOS update works on warm boot + - [ ] previous -> new + - [ ] new -> previous + +### ABL + +- [ ] Fastboot USB enumeration +- [ ] Boot time hasn't regressed (3.8s) + +### XBL + +- [ ] Display init works in cold and hot temperatures +- [ ] Boot time hasn't regressed (2.4s) + +### Setup + +#### Networking + +- [ ] Continue button disabled when no connection +- [ ] Forget/connect to wifi + +#### Custom URL +- (a) Not a real URL (e.g. `comma`, `abc123`, `...`) + - [ ] "Ensure the entered URL is valid" + - [ ] Start over + - [ ] Reboot device +- (b) Website but not an installer URL (e.g. `github.com`, `comma.ai`, `installer.comma.ai`) + - [ ] "No custom software found at this URL." +- (c) Valid installer URL (e.g. `openpilot.comma.ai`) + - [ ] Download successful (comma logo or installer appears) + - [ ] `/tmp/installer_url` should contain the installer URL diff --git a/agnos-binaries b/agnos-binaries deleted file mode 160000 index 72a4c036..00000000 --- a/agnos-binaries +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 72a4c0362cc25424d001c05bcab01cca20c231ef diff --git a/agnos-firmware b/agnos-firmware deleted file mode 160000 index 8dbaed43..00000000 --- a/agnos-firmware +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8dbaed43475157aa39736194ecaec62d998ac52c diff --git a/build_bootloader.sh b/build_bootloader.sh deleted file mode 100755 index 542f11f7..00000000 --- a/build_bootloader.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -e - -# Get directories and make sure we're in the correct spot to start the build -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -OUTPUT_DIR=$DIR/output -cd $DIR - -# Clone bootloader if not done already -if [ ! -d edk2_tici ]; then - git submodule init edk2_tici -fi -cd edk2_tici - -# Create output directory -mkdir -p $OUTPUT_DIR - -# Run build -./build.sh - -# Copy output -cp out/* $OUTPUT_DIR/ diff --git a/edk2_tici b/edk2_tici deleted file mode 160000 index 67600416..00000000 --- a/edk2_tici +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 67600416eec240ad754a22707180628b562565d8 diff --git a/firmware.json b/firmware.json new file mode 100644 index 00000000..8f143eb5 --- /dev/null +++ b/firmware.json @@ -0,0 +1,32 @@ +[ + { + "name": "abl", + "url": "https://commadist.azureedge.net/agnosupdate/abl-eeb89a74c968a5a2ffce96f23158b72e03e2814adf72ef59d1200ba8ea5d2f39.img.xz", + "hash": "eeb89a74c968a5a2ffce96f23158b72e03e2814adf72ef59d1200ba8ea5d2f39", + "size": 274432 + }, + { + "name": "xbl", + "url": "https://commadist.azureedge.net/agnosupdate/xbl-bcef195b00a1ab685da601f4072722569773ab161e91c8753ad99ca4217a28f5.img.xz", + "hash": "bcef195b00a1ab685da601f4072722569773ab161e91c8753ad99ca4217a28f5", + "size": 3282672 + }, + { + "name": "xbl_config", + "url": "https://commadist.azureedge.net/agnosupdate/xbl_config-19791056558c16f8dae787531b5e30b3b3db2ded9d666688df45ce1b91a72bac.img.xz", + "hash": "19791056558c16f8dae787531b5e30b3b3db2ded9d666688df45ce1b91a72bac", + "size": 98124 + }, + { + "name": "devcfg", + "url": "https://commadist.azureedge.net/agnosupdate/devcfg-be44b73dda5be840b09d5347d536459e31098da3fea97596956c0bdad19bdf27.img.xz", + "hash": "be44b73dda5be840b09d5347d536459e31098da3fea97596956c0bdad19bdf27", + "size": 40336 + }, + { + "name": "aop", + "url": "https://commadist.azureedge.net/agnosupdate/aop-5d764611a683d6a738cf06a1dcf8a926d0f47b5117ad40d3054167de6dd8bd0f.img.xz", + "hash": "5d764611a683d6a738cf06a1dcf8a926d0f47b5117ad40d3054167de6dd8bd0f", + "size": 184364 + } +] diff --git a/flash_all.sh b/flash_all.sh index cd3e53e9..66a39094 100755 --- a/flash_all.sh +++ b/flash_all.sh @@ -4,11 +4,11 @@ set -e DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" cd $DIR -for part in aop xbl xbl_config devcfg; do - tools/edl w ${part}_a $DIR/agnos-firmware/$part.bin - tools/edl w ${part}_b $DIR/agnos-firmware/$part.bin -done +# TODO: download firmware from firmware.json +#for part in aop abl xbl xbl_config devcfg; do +# tools/edl w ${part}_a $DIR/agnos-firmware/$part.bin +# tools/edl w ${part}_b $DIR/agnos-firmware/$part.bin +#done -./flash_bootloader.sh ./flash_kernel.sh ./flash_system.sh diff --git a/flash_bootloader.sh b/flash_bootloader.sh deleted file mode 100755 index 45eb2d2e..00000000 --- a/flash_bootloader.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -e - -GREEN="\033[0;32m" -NO_COLOR='\033[0m' - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -cd $DIR - -echo "Flashing bootloader..." -tools/edl w abl_a output/abl.elf -tools/edl w abl_b output/abl.elf - -echo -e "${GREEN}Flashed abl_a and abl_b!${NO_COLOR}" diff --git a/internal/README.md b/internal/README.md deleted file mode 100644 index eece9897..00000000 --- a/internal/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# internal - -## release checklist - -- [ ] `test_onroad` passes -- [ ] wifi -- [ ] modem -- [ ] image size -- [ ] sounds -- [ ] `python` is our python, not system version -- [ ] pyenv shims works -- [ ] clean openpilot build -- [ ] factory reset - - [ ] from openpilot menu - - [ ] tapping on boot - - [ ] corrupt userdata -- [ ] color calibration - - [ ] from /persist/comma/ - - [ ] directly from panel over sysfs -- [ ] clean setup -- [ ] update works on warm boot - - [ ] previous -> new - - [ ] new -> previous - -### ABL - -- [ ] fastboot USB enumeration -- [ ] boot time hasn't regressed (3.8s) - -### XBL - -- [ ] display init works in cold and hot temperatures -- [ ] boot time hasn't regressed (2.4s) - -### setup - -#### networking - -- [ ] continue button disabled when no connection -- [ ] forget/connect to wifi - -#### custom URL -- (a) Not a real URL (e.g. `comma`, `abc123`, `...`) - - [ ] "Ensure the entered URL is valid" - - [ ] Start over - - [ ] Reboot device -- (b) Website but not an installer URL (e.g. `github.com`, `comma.ai`, `installer.comma.ai`) - - [ ] "No custom software found at this URL." -- (c) Valid installer URL (e.g. `openpilot.comma.ai`) - - [ ] Download successful (comma logo or installer appears) - - [ ] `/tmp/installer_url` should contain the installer URL diff --git a/internal/build_and_package.sh b/internal/build_and_package.sh deleted file mode 100755 index 2ec5bf3c..00000000 --- a/internal/build_and_package.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -e - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -cd $DIR/.. - -read -p "Is the kernel repo up to date? " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "Update it and run again!" - [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 -fi - -read -p "Is the firmware repo up to date? Copied in the new abl if needed? " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "Update it and run again!" - [[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1 -fi - -./build_kernel.sh -./build_system.sh -internal/package_ota.py -internal/ota_push.sh staging diff --git a/scripts/build_and_package.sh b/scripts/build_and_package.sh new file mode 100755 index 00000000..370343af --- /dev/null +++ b/scripts/build_and_package.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd $DIR/.. + +./build_kernel.sh +./build_system.sh +scripts/package_ota.py + +# push to azure on an internal machine +if [ "$USER" == "batman" ]; then + scripts/ota_push.sh staging +fi diff --git a/internal/ota_push.sh b/scripts/ota_push.sh similarity index 100% rename from internal/ota_push.sh rename to scripts/ota_push.sh diff --git a/internal/package_ota.py b/scripts/package_ota.py similarity index 80% rename from internal/package_ota.py rename to scripts/package_ota.py index 0c1bc24c..b0ed8b8e 100755 --- a/internal/package_ota.py +++ b/scripts/package_ota.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import json import os +import hashlib import subprocess from copy import deepcopy from pathlib import Path @@ -9,15 +10,16 @@ ROOT = Path(__file__).parent.parent OUTPUT_DIR = ROOT / "output" OTA_OUTPUT_DIR = OUTPUT_DIR / "ota" -FIRMWARE_DIR = ROOT / "agnos-firmware" AGNOS_UPDATE_URL = os.getenv("AGNOS_UPDATE_URL", "https://commadist.azureedge.net/agnosupdate") AGNOS_STAGING_UPDATE_URL = os.getenv("AGNOS_STAGING_UPDATE_URL", "https://commadist.azureedge.net/agnosupdate-staging") - def checksum(fn): - return subprocess.check_output(["sha256sum", fn]).decode().split()[0] - + sha256 = hashlib.sha256() + with open(fn, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b""): + sha256.update(chunk) + return sha256.hexdigest() def compress(fin, fout) -> None: subprocess.check_call(f"xz -T4 -vc {fin} > {fout}", shell=True) @@ -32,7 +34,7 @@ def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None) if sparse: with NamedTemporaryFile() as tmp_f: print(" converting sparse image to raw") - subprocess.check_call(["simg2img", fn, tmp_f.name]) + subprocess.check_call(["simg2img", fn, tmp_f.name], shell=True) hash_raw = checksum(tmp_f.name) size = Path(tmp_f.name).stat().st_size print(f" {size} bytes, hash {hash} (raw)") @@ -76,11 +78,6 @@ def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None) files = [ process_file(OUTPUT_DIR / "boot.img", "boot"), - process_file(FIRMWARE_DIR / "abl.bin", "abl"), - process_file(FIRMWARE_DIR / "xbl.bin", "xbl"), - process_file(FIRMWARE_DIR / "xbl_config.bin", "xbl_config"), - process_file(FIRMWARE_DIR / "devcfg.bin", "devcfg"), - process_file(FIRMWARE_DIR / "aop.bin", "aop"), process_file(OUTPUT_DIR / "system.img", "system", sparse=True, full_check=False, alt=OUTPUT_DIR / "system-skip-chunks.img"), ] configs = [ @@ -88,6 +85,21 @@ def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None) (AGNOS_STAGING_UPDATE_URL, "ota-staging.json"), ] + # pull in firmware not built in this repo + with open(ROOT/"firmware.json") as f: + fws = json.loads(f.read()) + for fw in fws: + files.append({ + "name": fw["name"], + "url": fw["url"], + "hash": fw["hash"], + "hash_raw": fw["hash"], + "size": fw["size"], + "sparse": False, + "full_check": True, + "has_ab": True, + }) + for remote_url, output_fn in configs: processed_files = [] for f in deepcopy(files): diff --git a/internal/pull_ota.sh b/scripts/pull_ota.sh similarity index 100% rename from internal/pull_ota.sh rename to scripts/pull_ota.sh diff --git a/userspace/usr/comma/comma.sh b/userspace/usr/comma/comma.sh index 8f7414d5..031b4406 100755 --- a/userspace/usr/comma/comma.sh +++ b/userspace/usr/comma/comma.sh @@ -63,7 +63,7 @@ fi # symlink vscode to userdata mkdir -p /data/tmp/vscode-server ln -s /data/tmp/vscode-server ~/.vscode-server - +ln -s /data/tmp/vscode-server ~/.cursor-server while true; do pkill -f "$SETUP"