Merge pull request adafruit#387 from adafruit/rework-ci

Rework ci

.github/PULL_REQUEST_TEMPLATE.md

@@ -6,7 +6,6 @@
 - [ ] Please provide related links (*eg. Issue which will be closed by this Pull Request*)
 - [ ] If you are adding an new boards, please make sure
   - [ ] Provide link to your allocated VID/PID if applicable
-  - [ ] Add your board to [action ci](/.github/workflows) in correct workflow and alphabet order for release binary
   - [ ] `UF2_BOARD_ID` in your board.h follow correct format from [uf2 specs](https://github.com/microsoft/uf2#files-exposed-by-bootloaders)
 
 *This checklist items that are not applicable to your PR can be deleted.*

.github/actions/setup_toolchain/action.yml

@@ -0,0 +1,41 @@
+name: Setup Toolchain
+
+inputs:
+  toolchain:
+    description: 'Toolchain name'
+    required: true
+  toolchain_url:
+    description: 'Toolchain URL or version'
+    required: false
+
+outputs:
+  build_option:
+    description: 'Build option for the toolchain e.g --toolchain clang'
+    value: ${{ steps.set-toolchain-option.outputs.build_option }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install ARM GCC
+      if: inputs.toolchain == 'arm-gcc'
+      uses: carlosperate/arm-none-eabi-gcc-action@v1
+      with:
+#        release: '12.3.Rel1'
+        release: '11.2-2022.02'
+
+    - name: Pull ESP-IDF docker
+      if: inputs.toolchain == 'esp-idf'
+      uses: ./.github/actions/setup_toolchain/espressif
+      with:
+        toolchain: ${{ inputs.toolchain }}
+        toolchain_url: ${{ inputs.toolchain_url }}
+
+    - name: Download Toolchain
+      if: >-
+        inputs.toolchain != 'arm-gcc' &&
+        inputs.toolchain != 'arm-iar' &&
+        inputs.toolchain != 'esp-idf'
+      uses: ./.github/actions/setup_toolchain/download
+      with:
+        toolchain: ${{ inputs.toolchain }}
+        toolchain_url: ${{ inputs.toolchain_url }}

.github/actions/setup_toolchain/download/action.yml

@@ -0,0 +1,33 @@
+name: Download Toolchain
+
+inputs:
+  toolchain:
+    description: 'Toolchain name'
+    required: true
+  toolchain_url:
+    description: 'Toolchain URL'
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Cache Toolchain
+      if: ${{ !startsWith(inputs.toolchain_url, 'https://github.com') }}
+      uses: actions/cache@v4
+      id: cache-toolchain-download
+      with:
+        path: ~/cache/${{ inputs.toolchain }}
+        key: ${{ runner.os }}-${{ inputs.toolchain }}-${{ inputs.toolchain_url }}
+
+    - name: Install Toolchain
+      if: steps.cache-toolchain-download.outputs.cache-hit != 'true'
+      run: |
+        mkdir -p ~/cache/${{ inputs.toolchain }}
+        wget --progress=dot:giga ${{ inputs.toolchain_url }} -O toolchain.tar.gz
+        tar -C ~/cache/${{ inputs.toolchain }} -xaf toolchain.tar.gz
+      shell: bash
+
+    - name: Set Toolchain Path
+      run: |
+        echo >> $GITHUB_PATH `echo ~/cache/${{ inputs.toolchain }}/*/bin`
+      shell: bash

.github/actions/setup_toolchain/espressif/action.yml

@@ -0,0 +1,41 @@
+name: Setup ESP-IDF Toolchain
+
+inputs:
+  toolchain:
+    description: 'Toolchain name'
+    required: true
+  toolchain_url:
+    description: 'Toolchain URL or version'
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set DOCKER_ESP_IDF
+      run: |
+        DOCKER_ESP_IDF=$HOME/cache/${{ inputs.toolchain }}/docker_image.tar
+        echo "DOCKER_ESP_IDF=$DOCKER_ESP_IDF" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Cache Docker Image
+      uses: actions/cache@v4
+      id: cache-toolchain-espressif
+      with:
+        path: ${{ env.DOCKER_ESP_IDF }}
+        key: ${{ inputs.toolchain }}-${{ inputs.toolchain_url }}
+
+    - name: Pull and Save Docker Image
+      if: steps.cache-toolchain-espressif.outputs.cache-hit != 'true'
+      run: |
+        docker pull espressif/idf:${{ inputs.toolchain_url }}
+        mkdir -p $(dirname $DOCKER_ESP_IDF)
+        docker save -o $DOCKER_ESP_IDF espressif/idf:${{ inputs.toolchain_url }}
+        du -sh $DOCKER_ESP_IDF
+      shell: bash
+
+    - name: Load Docker Image
+      if: steps.cache-toolchain-espressif.outputs.cache-hit == 'true'
+      run: |
+        du -sh $DOCKER_ESP_IDF
+        docker load --input $DOCKER_ESP_IDF
+      shell: bash

.github/workflows/build.yml

@@ -0,0 +1,132 @@
+name: Build
+
+on:
+  push:
+    paths:
+      - 'apps/**'
+      - 'lib/**'
+      - 'ports/**'
+      - 'src/**'
+      - '.github/actions/**'
+      - '.github/workflows/build.yml'
+  pull_request:
+    branches: [ master ]
+    paths:
+      - 'apps/**'
+      - 'lib/**'
+      - 'ports/**'
+      - 'src/**'
+      - '.github/actions/**'
+      - '.github/workflows/build.yml'
+  repository_dispatch:
+  release:
+    types:
+      - created
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  set-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      json: ${{ steps.set-matrix-json.outputs.matrix }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run pre-commit
+        uses: pre-commit/[email protected]
+
+      - name: Generate matrix json
+        id: set-matrix-json
+        run: |
+          MATRIX_JSON=$(python .github/workflows/ci_set_matrix.py)
+          echo "matrix=$MATRIX_JSON"
+          echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT
+
+  # ---------------------------------------
+  # Build ARM
+  # ---------------------------------------
+  arm:
+    needs: set-matrix
+    uses: ./.github/workflows/build_util.yml
+    strategy:
+      fail-fast: false
+      matrix:
+        port:
+        # Alphabetical order
+        #- 'kinetis_k32l2' skip since self-update is not implemented
+        - 'lpc55'
+        - 'mimxrt10xx'
+        - 'stm32f3'
+        - 'stm32f4'
+        - 'stm32l4'
+    with:
+      port: ${{ matrix.port }}
+      boards: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.port].board) }}
+      build-system: 'make'
+      toolchain: 'arm-gcc'
+
+  # ---------------------------------------
+  # Build ESP
+  # ---------------------------------------
+  esp:
+    needs: set-matrix
+    uses: ./.github/workflows/build_util.yml
+    with:
+      port: 'espressif'
+      boards: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)['espressif'].board) }}
+      build-system: 'make'
+      toolchain: 'esp-idf'
+      toolchain_url: 'v5.1.1'
+
+  # ---------------------------------------
+  # Unit testing with ghostfat
+  # ---------------------------------------
+  ghostfat:
+    needs: set-matrix
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        board: ${{ fromJSON(needs.set-matrix.outputs.json)['test_ghostfat'].board }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Build
+        run: |
+          make -C ports/test_ghostfat/ BOARD=${{ matrix.board }} all
+
+      - name: Decompress known good filesystem image
+        run: |
+          # copy
+          cp ./boards/${{ matrix.board }}/knowngood.img.gz.gz ./_build/${{ matrix.board }}/
+
+          # NOTE: test_huge's knowngood.img file starts as 1.5 GiB
+          #       Compressing once with GZip results gives 85 MiB
+          #       Compressing it a second time gives 10 MiB ...
+          #       Therefore, store known good images double-compressed...
+          gzip --decompress ./_build/${{ matrix.board }}/knowngood.img.gz.gz
+          gzip --decompress ./_build/${{ matrix.board }}/knowngood.img.gz
+        working-directory: ports/test_ghostfat
+
+      - name: Execute native self-test
+        run: |
+          chmod +x ./tinyuf2-${{ matrix.board }}.elf
+          ./tinyuf2-${{ matrix.board }}.elf
+
+          # Compress (double) newly generated self-test images
+          mv ghostfat.img ghostfat_${{ matrix.board }}.img
+          gzip --keep ghostfat_${{ matrix.board }}.img
+          gzip --keep --force --best ghostfat_${{ matrix.board }}.img.gz
+        working-directory: ports/test_ghostfat/_build/${{ matrix.board }}
+
+      - name: Save newly generated self-test images as CI artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ghostfat_selftest_images
+          path: ./ports/test_ghostfat/_build/${{ matrix.board }}/ghostfat_${{ matrix.board }}.img.gz.gz

.github/workflows/build_cmake.yml

@@ -41,13 +41,11 @@ jobs:
         - 'stm32f411ve_discovery'
 
     steps:
-    - name: Checkout code
+    - name: Checkout
       uses: actions/checkout@v4
       with:
-        fetch-depth: 0
-
-    - name: Checkout common submodules in lib
-      run: git submodule update --init lib/tinyusb lib/uf2
+        submodules: true
+        fetch-tags: true
 
     - name: Install ARM GCC
       uses: carlosperate/arm-none-eabi-gcc-action@v1