Merge branch 'JonathonHall-Purism-module_checkout_submodules-flake_mo…

…reutils_for_timestamps_CircleCI_intermediary_arch_musl-cross-make_step' into ivy_coreboot_2402-defconfig_unification_to_oldconfig Signed-off-by: Thierry Laurion <[email protected]>
linuxboot · Jul 26, 2024 · 3fe554d · 3fe554d
2 parents 7b0d995 + f0c951f
commit 3fe554d
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 70 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -11,41 +11,41 @@ commands:
         type: string
     steps:
       - run:
-          name: Make Board (console logs not contiguous because multiple CPUS used to build under CircleCI, see logs in artifacts or Output build failing logs below in case of failed step)
+          name: Make Board (console logs not contiguous because multiple CPUs used to build under CircleCI, see logs in artifacts or Output build failing logs below in case of failed step)
           command: |
             echo "Sourcing /devenv.sh since docker entrypoint doesn't do it as expected"
             source /devenv.sh
-            rm -rf build/<<parameters.arch>>/<<parameters.target>>/* build/<<parameters.arch>>/log/*
-            make V=1 BOARD=<<parameters.target>> <<parameters.subcommand>> || touch ./tmpDir/failed_build
+            rm -rf build/<< parameters.arch >>/<< parameters.target >>/* build/<< parameters.arch >>/log/*
+            make V=1 BOARD=<< parameters.target >> << parameters.subcommand >>  | ts || touch ./tmpDir/failed_build
           no_output_timeout: 3h
       - run:
           name: Output hashes
           command: |
-            cat build/<<parameters.arch>>/<<parameters.target>>/hashes.txt || echo "No hashes.txt for this build step..."
+            cat build/<< parameters.arch >>/<< parameters.target >>/hashes.txt || echo "No hashes.txt for this build step..."
       - run:
           name: Output sizes
           command: |
-            cat build/<<parameters.arch>>/<<parameters.target>>/sizes.txt || echo "No sizes.txt for this build step..."
+            cat build/<< parameters.arch >>/<< parameters.target >>/sizes.txt || echo "No sizes.txt for this build step..."
       - run:
           name: Archiving build logs.
           command: |
-            tar zcvf build/<<parameters.arch>>/<<parameters.target>>/logs.tar.gz $(find build/ -name "*.log")
+            tar zcvf build/<< parameters.arch >>/<< parameters.target >>/logs.tar.gz $(find build/ -name "*.log")
       - run:
           name: Output build failing logs
           command: |
             if [[ -f ./tmpDir/failed_build ]]; then
-            	find "./build/<<parameters.arch>>/" -name "*.log" -type f -mmin -1 -exec tail -n +1 '{}' +
-            	exit 1
+              find "./build/<< parameters.arch >>/" -name "*.log" -type f -mmin -1 -exec tail -n +1 '{}' +
+              exit 1
             else
-            	echo "Step hasn't failed. Continuing with next step..."
+              echo "Step hasn't failed. Continuing with next step..."
             fi
       - store_artifacts:
-          path: build/<<parameters.arch>>/<<parameters.target>>
+          path: build/<< parameters.arch >>/<< parameters.target >>
 
 jobs:
   prep_env:
     docker:
-      - image: tlaurion/heads-dev-env:v0.2.0
+      - image: tlaurion/heads-dev-env:v0.2.1
     resource_class: large
     working_directory: ~/heads
     steps:
@@ -73,7 +73,7 @@ jobs:
       - restore_cache:
           # First matched/found key wins and following keys are not tried
           keys:
-            # Cache for matching modules digest, validated to be exactly the same as in github current commit.
+            # Cache for matching modules digest, validated to be exactly the same as in GitHub current commit.
             # This cache was made on top of below caches, if previously existing.
             # If no module definition changed, we reuse this one
             - nix-docker-heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }}
@@ -111,7 +111,7 @@ jobs:
 
   build_and_persist:
     docker:
-      - image: tlaurion/heads-dev-env:v0.2.0
+      - image: tlaurion/heads-dev-env:v0.2.1
     resource_class: large
     working_directory: ~/heads
     parameters:
@@ -126,20 +126,20 @@ jobs:
       - attach_workspace:
           at: ~/heads
       - build_board:
-          arch: <<parameters.arch>>
-          target: <<parameters.target>>
-          subcommand: <<parameters.subcommand>>
+          arch: << parameters.arch >>
+          target: << parameters.target >>
+          subcommand: << parameters.subcommand >>
       - persist_to_workspace:
           root: ~/heads
           paths:
-            - packages/<<parameters.arch>>
-            - build/<<parameters.arch>>
-            - crossgcc/<<parameters.arch>>
-            - install/<<parameters.arch>>
+            - packages/<< parameters.arch >>
+            - build/<< parameters.arch >>
+            - crossgcc/<< parameters.arch >>
+            - install/<< parameters.arch >>
 
   build:
     docker:
-      - image: tlaurion/heads-dev-env:v0.2.0
+      - image: tlaurion/heads-dev-env:v0.2.1
     resource_class: large
     working_directory: ~/heads
     parameters:
@@ -160,7 +160,7 @@ jobs:
 
   save_cache:
     docker:
-      - image: tlaurion/heads-dev-env:v0.2.0
+      - image: tlaurion/heads-dev-env:v0.2.1
     resource_class: large
     working_directory: ~/heads
     steps:
@@ -176,25 +176,21 @@ jobs:
             - crossgcc
             - packages
       - save_cache:
-          # Generate cache for the same coreboot mnd musl-cross-make modules definition if hash is not previously existing
+          # Generate cache for the same coreboot and musl-cross-make modules definition if hash is not previously existing
           # CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names
           key: nix-docker-heads-coreboot-musl-cross-{{ checksum "./tmpDir/coreboot_musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }}
           paths:
             - build/ppc64/coreboot-talos_2
             - build/ppc64/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1
             - build/x86/coreboot-4.11
-            - build/x86/coreboot-4.13
-            - build/x86/coreboot-4.14
-            - build/x86/coreboot-4.15
-            - build/x86/coreboot-4.17
             - build/x86/coreboot-24.02.01
-            - build/x86/coreboot-nitrokey
+            - build/x86/coreboot-dasharo
             - build/x86/coreboot-purism
             - build/x86/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1
             - crossgcc
             - packages
       - save_cache:
-          #Generate cache for the exact same modules definitions if hash is not previously existing
+          # Generate cache for the exact same modules definitions if hash is not previously existing
           key: nix-docker-heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }}
           paths:
             - build
@@ -208,6 +204,23 @@ workflows:
     jobs:
       - prep_env
 
+      # This step builds musl-cross for x86 architecture, which will be used by subsequent x86 board builds
+      - build_and_persist:
+          name: x86-musl-cross
+          target: x230-hotp-maximized
+          subcommand: "musl-cross"
+          requires:
+            - prep_env
+
+      # This step builds musl-cross for ppc64 architecture, which will be used by subsequent ppc64 board builds
+      - build_and_persist:
+          name: ppc64-musl-cross
+          arch: ppc64
+          target: talos-2
+          subcommand: "musl-cross"
+          requires:
+            - prep_env
+
       # Below, sequentially build one board for each coreboot version.
       # The last board in the sequence is the dependency for the parallel boards built at the end, and also save_cache.
 
@@ -217,7 +230,7 @@ workflows:
           target: x230-hotp-maximized
           subcommand: ""
           requires:
-            - prep_env
+            - x86-musl-cross
 
       # coreboot purism
       - build_and_persist:
@@ -237,7 +250,7 @@ workflows:
           target: nitropad-nv41
           subcommand: ""
           requires:
-            - prep_env
+            - x86-musl-cross
 
       # coreboot talos_2
       - build_and_persist:
@@ -246,15 +259,15 @@ workflows:
           target: talos-2
           subcommand: ""
           requires:
-            - prep_env
+            - ppc64-musl-cross
 
       # coreboot 4.11
       - build_and_persist:
           name: librem_l1um
           target: librem_l1um
           subcommand: ""
           requires:
-            - prep_env
+            - x86-musl-cross
 
       # Cache one workspace per architecture
       # Make sure workspace caches are chainloaded and the last in chain for an arch is saved
@@ -263,11 +276,8 @@ workflows:
             - talos-2
             - librem_14
 
-      #
       # Those onboarding new boards should add their entries below.
-      #
-
-      # coreboot 24.02.01 boards
+      # coreboot 4.22.01 boards
       - build:
           name: x220-hotp-maximized
           target: x220-hotp-maximized
@@ -520,31 +530,3 @@ workflows:
           subcommand: ""
           requires:
             - librem_l1um
-
-#      - build:
-#          name: UNMAINTAINED_kgpe-d16_workstation-usb_keyboard
-#          target: UNMAINTAINED_kgpe-d16_workstation-usb_keyboard
-#          subcommand: ""
-#          requires:
-#            - UNMAINTAINED_kgpe-d16_workstation
-
-#      - build:
-#          name: UNMAINTAINED_kgpe-d16_server
-#          target: UNMAINTAINED_kgpe-d16_server
-#          subcommand: ""
-#          requires:
-#            - UNMAINTAINED_kgpe-d16_workstation
-
-#      - build:
-#          name: UNMAINTAINED_kgpe-d16_server-whiptail
-#          target: UNMAINTAINED_kgpe-d16_server-whiptail
-#          subcommand: ""
-#          requires:
-#            - UNMAINTAINED_kgpe-d16_workstation
-
-#      - build:
-#          name: librem_l1um
-#          target: librem_l1um
-#          subcommand: ""
-#          requires:
-#            - librem_14
diff --git a/Makefile b/Makefile
@@ -378,7 +378,9 @@ define define_module =
     # First time:
     #   Checkout the tree instead and create the canary file with repo and
     #   revision so that we know that the files are all present and their
-    #   version.
+    #   version.  Submodules are _not_ checked out, because coreboot has
+    #   many submodules that won't be used, let coreboot check out its own
+    #   submodules during build
     #
     # Other times:
     #   If .canary contains the same repo and revision combination, do nothing.
@@ -392,7 +394,7 @@ define define_module =
     $(build)/$($1_base_dir)/.canary: FORCE
 	if [ ! -e "$$@" ]; then \
 		git clone $($1_repo) "$(build)/$($1_base_dir)"; \
-		git -C "$(build)/$($1_base_dir)" reset --hard $($1_commit_hash) && git submodule update --init --checkout; \
+		git -C "$(build)/$($1_base_dir)" reset --hard $($1_commit_hash); \
 		echo -n '$($1_repo)|$($1_commit_hash)' > "$$@"; \
 	elif [ "$$$$(cat "$$@")" != '$($1_repo)|$($1_commit_hash)' ]; then \
 		echo "Switching $1 to $($1_repo) at $($1_commit_hash)" && \

diff --git a/flake.nix b/flake.nix
@@ -79,6 +79,7 @@
         diffoscopeMinimal # Not sure exactly what is packed here, let's try. Might need diffoscope if something is missing
         gnupg #to inject public key inside of qemu create rom through inject_gpg target of targets/qemu.mk TODO: remove when pflash supported by flashrom + modify code
         less # so 'git log' is usable
+        moreutils # so that 'make 2>&1 | ts' can give timestamps
       ] ++ [
         # Tools for handling binary blobs in their compressed state. (blobs/xx30/vbios_[tw]530.sh)
         bundler

diff --git a/unmaintained_boards/README.md b/unmaintained_boards/README.md
@@ -1,4 +1,13 @@
-Boards under this directory were moved from UNTESTED_ to UNMAINTAINED_ automatically since untested for a while.
-Boards not having UNMAINTAINED in their names are not compliant with current build system (linuxboot) and need reupstreaming.
+Boards listed under this directory are not made available from CircleCI. No .rom file is available for endusers. This is protecting as good as possible regular endusers from having to open up their devices and having to own a SPI-clip and a external flasher.
+After core changes in coreboot or heads, known testers with external flasher are asked to test a new release and report if their system is starting up fine. If those known testers do not respond, a .rom id made available from CircleCI to the public with addition in filename UNTESTED_ . This invite people with external flasher that could recover a not booting system to report if the release is working fine.
+Warning: Do not try to use UNTESTED_ images if you do not have a external flasher.
 
-To test those boards, move them to boards directory and follow normal build operations, test builds on boards and open an issue stating you are willing to test those when builds are made available from CircleCI in the future.
+After about a month passes by and there is still no report if the system is starting up fine with the new release, the device is moved to this directory to stop building UNTESTED_ images. Reason for this is because there are always users ignoring all warnings and then asking questions like how to recover a not starting system without external programmer.
+
+To get a device out of this directory and make it at available from CircleCI again, open up a issue here https://github.com/linuxboot/heads/issues and ask for a build for the specific device you like to test or build it youself. When building it yourself, please dont forget to report a working state.
+
+The additional name UNMAINTAINED_ is added to the device name, when maintanance is known needed.
+When device is UNTESTED_ and someone test and report a not booting image, the device get as soon as possibe moved to this directory and changed from UNTESTED_ to UNMAINTAINED_. Its then tested and not working.
+
+When a device have just the addition UNMAINTAINED_ but is not in this directory and there are CircleCI .rom files available to the endusers, then its tested starting up the system but have some problems like for example not working Network card and there is no maintainer to fix the problem.
+If a UNMAINTAINED_ device dont get tested on a new release, it follow up the same UNTESTED_ procedure like described above and have UNMAINTAINED_UNTESTED_ in the name.