Merge remote-tracking branch 'osresearch/master' into pr/tlaurion/1541

.envrc

@@ -0,0 +1 @@
+has nix && use flake

.github/ISSUE_TEMPLATE/bug-report-for-everything-except-build-errors.md

@@ -11,90 +11,92 @@ assignees: ''
 
 ### A. Provide Hardware Details
 
-**1. What board are you using (see list of boards [here](https://github.com/eganonoa/heads/tree/master/boards))?**
-
-**2. Does your computer have a dGPU or is it iGPU-only?**
-- [ ] dGPU
-- [ ] iGPU-only
-
-**3. Who installed Heads on this computer?**
-- [ ] Insurgo
-- [ ] Nitrokey
-- [ ] Purism
-- [ ] Other provider
-- [ ] Self-installed
-
-**4. What PGP key is being used?**
-- [ ] Librem Key
-- [ ] Nitrokey Pro 2
-- [ ] Nitrokey Storage
-- [ ] Yubikey
-- [ ] Other
-
-**5. Are you using the PGP key to provide HOTP verification?**
-- [ ] Yes
-- [ ] No
-- [ ] I don't know
+1. What board are you using? (Choose from the list of boards [here](https://github.com/eganonoa/heads/tree/master/boards))
+
+2. Does your computer have a dGPU or is it iGPU-only?
+    - [ ] dGPU (Distinct GPU other then internal GPU)
+    - [ ] iGPU-only (Internal GPU, normally Intel GPU)
+
+3. Who installed Heads on this computer?
+    - [ ] Insurgo (Issues to be reported at https://github.com/linuxboot/heads/issues)
+    - [ ] Nitrokey (Issues to be reported at https://github.com/Nitrokey/heads/issues)
+    - [ ] Purism (Issues to be reported at https://source.puri.sm/firmware/pureboot/-/issues)
+    - [ ] Novacustom (Issues to be reported at https://github.com/Dasharo/dasharo-issues)
+    - [ ] HardnenedVault (Issues to be reported at https://github.com/hardenedvault/vaultboot/issues)
+    - [ ] Other provider
+    - [ ] Self-installed
+
+4. What PGP key is being used?
+    - [ ] Librem Key (Nitrokey Pro 2 rebranded)
+    - [ ] Nitrokey Pro
+    - [ ] Nitrokey Pro 2
+    - [ ] Nitrokey 3 NFC
+    - [ ] Nitrokey 3 NFC Mini
+    - [ ] Nitrokey Storage
+    - [ ] Nitrokey Storage 2
+    - [ ] Yubikey
+    - [ ] Other
+
+5. Are you using the PGP key to provide HOTP verification?
+    - [ ] Yes
+    - [ ] No
+    - [ ] I don't know
 
 ### B. Identify how the board was flashed
 
-**1. Is this problem related to updating heads or flashing it for the first time?**
-- [ ] First-time flash
-- [ ] Updating heads 
+1. Is this problem related to updating heads or flashing it for the first time?
+    - [ ] First-time flash
+    - [ ] Updating heads 
 
-**2. If the problem is related to an update, how did you attempt to apply the update?**
-- [ ] Using the Heads GUI
-- [ ] Flashrom via the Recovery Shell
-- [ ] External flashing
+2. If the problem is related to an update, how did you attempt to apply the update?
+    - [ ] Using the Heads menus
+    - [ ] Flashrom via the Recovery Shell
+    - [ ] External flashing
 
-**3. How was Heads initially flashed**
-- [ ] External flashing
-- [ ] Internal-only / 1vyrain
-- [ ] Don't know
+3. How was Heads initially flashed?
+    - [ ] External flashing
+    - [ ] Internal-only / 1vyprep+1vyrain / skulls
+    - [ ] Don't know
 
-**4. Was the board flashed with a maximized or non-maximized/legacy rom?**
-- [ ] Maximized
-- [ ] Non-maximized / legacy
-- [ ] I don't know
+4. Was the board flashed with a maximized or non-maximized/legacy rom?
+    - [ ] Maximized
+    - [ ] Non-maximized / legacy
+    - [ ] I don't know
 
-**5. If Heads was externally flashed, was IFD unlocked?**
-- [ ] Yes
-- [ ] No
-- [ ] Don't know
+5. If Heads was externally flashed, was IFD unlocked?
+    - [ ] Yes
+    - [ ] No
+    - [ ] Don't know
 
 ### C. Identify the rom related to this bug report
 
-**1. Did you download or build the rom at issue in this bug report?**
-- [ ] I downloaded it
-- [ ] I built it
-
-**2. If you downloaded your rom, where did you get it from?**
-- [ ] Heads CircleCi
-- [ ] Purism
-- [ ] Nitrokey
-- [ ] Somewhere else (please identify)
-
-*Please provide the release number or otherwise identify the rom downloaded*
-
-**3. If you built your rom, which repository:branch did you use?**
-- [ ] Heads:Master
-- [ ] Other (please identify)
-
-**4. What version of coreboot did you use in building?**
-- [ ] 4.8.1 (current default in heads:master)
-- [ ] 4.13
-- [ ] 4.14
-- [ ] 4.15
-- [ ] Other (please specify)
-- [ ] I don't know
-
-**5. In building the rom where did you get the blobs?**
-- [ ] No blobs required
-- [ ] Provided by the company that installed Heads on the device
-- [ ] Extracted from a backup rom taken from this device
-- [ ] Extracted from another backup rom taken from another device (please identify the board model)
-- [ ] Extracted from the online bios using the automated tools provided in Heads
-- [ ] I don't know
+1. Did you download or build the rom at issue in this bug report?
+    - [ ] I downloaded it
+    - [ ] I built it
+
+2. If you downloaded your rom, where did you get it from?
+    - [ ] Heads CircleCi
+    - [ ] Purism
+    - [ ] Nitrokey
+    - [ ] Dasharo DTS (Novacustom)
+    - [ ] Somewhere else (please identify)
+
+    *Please provide the release number or otherwise identify the rom downloaded*
+
+3. If you built your rom, which repository:branch did you use?
+    - [ ] Heads:Master
+    - [ ] Other (please identify)
+
+4. What version of coreboot did you use in building?
+{ You can find this information from github commit ID or once flashed, by giving the complete version from Sytem Information under Options --> menu}
+
+5. In building the rom, where did you get the blobs?
+    - [ ] No blobs required
+    - [ ] Provided by the company that installed Heads on the device
+    - [ ] Extracted from a backup rom taken from this device
+    - [ ] Extracted from another backup rom taken from another device (please identify the board model)
+    - [ ] Extracted from the online bios using the automated tools provided in Heads
+    - [ ] I don't know
 
 ## Please describe the problem
 

.github/ISSUE_TEMPLATE/bug-report-for-heads-build-errors.md

@@ -16,11 +16,7 @@ assignees: ''
 - [ ] Other (please specify)
 
 **3. What version of coreboot are you trying to build**
-- [ ] 4.13
-- [ ] 4.14
-- [ ] 4.15
-- [ ] 4.17
-- [ ] Other (please specify)
+<!-- Please provide GitHub commit ID, or if firmware flashed, the information found under System Information for version -->
 
 **4. In building the rom where did you get the blobs?**
 - [ ] No blobs required

.gitignore

@@ -1,25 +1,25 @@
-.*.sw*
-*.xz
+*.bad
 *.bz2
+*.cpio
+*.dep
+*.ffs
+*.fv
 *.gz
-*.sign
-*.rom
-*.o
-*.gz
-*.tgz
 *.img
-*.rom
-*.cpio
-typescript*
-config/*.old
 *.log
-*~
-crossgcc
-clean
+*.lz
+*.o
+*.rom
 *.sec
-*.dep
-*.ffs
+*.sign
+*.tgz
 *.vol
-*.lz
-*.fv
-*.bad
+*.xz
+*~
+.*.sw*
+/.direnv
+clean
+config/*.old
+crossgcc
+typescript*
+result

Makefile

@@ -25,11 +25,48 @@ INSTALL		= $(pwd)/install/$(CONFIG_TARGET_ARCH)
 log_dir		= $(build)/log
 board_build	= $(build)/$(BOARD)
 
+
+# Estimated memory required per job in GB (e.g., 1GB for gcc)
+MEM_PER_JOB_GB ?= 1
+
 # Controls how many parallel jobs are invoked in subshells
-CPUS		?= $(shell nproc)
-MAKE_JOBS	?= -j$(CPUS) --max-load 16
+CPUS            ?= $(shell nproc)
+AVAILABLE_MEM_GB   ?= $(shell cat /proc/meminfo | grep MemAvailable | awk '{print int($$2 / 1024)}')
+
+# Calculate the maximum number of jobs based on available memory
+MAX_JOBS_MEM := $(shell echo $$(( $(AVAILABLE_MEM_GB) / $(MEM_PER_JOB_GB) )))
+
+# Use the minimum of the system's CPUs and the calculated max jobs based on memory
+CPUS            := $(shell echo $$(($(CPUS) < $(MAX_JOBS_MEM) ? $(CPUS) : $(MAX_JOBS_MEM))))
+
+# Load average can be adjusted to be higher than CPUS to allow for some CPU overcommit
+# Multiply by 3 and then divide by 2 to achieve the effect of multiplying by 1.5 using integer arithmetic
+LOADAVG         ?= $(shell echo $$(( ($(CPUS) * 3) / 2 )))
+
+# Construct MAKE_JOBS with dynamic CPU count and load average
+MAKE_JOBS       := -j$(CPUS) --load-average=$(LOADAVG) # Add other flags as needed to be more adaptive to CIs
+
+# Print out the settings and compare system values with actual ones used
+$(info ----------------------------------------------------------------------)
+$(info !!!!!! BUILD SYSTEM INFO !!!!!!)
+$(info System CPUS: $(shell nproc))
+$(info System Available Memory: $(AVAILABLE_MEM_GB) GB)
+$(info System Load Average: $(shell uptime | awk '{print $$10}'))
+$(info ----------------------------------------------------------------------)
+$(info Used **CPUS**: $(CPUS))
+$(info Used **LOADAVG**: $(LOADAVG))
+$(info Used **AVAILABLE_MEM_GB**: $(AVAILABLE_MEM_GB) GB)
+$(info ----------------------------------------------------------------------)
+$(info **MAKE_JOBS**: $(MAKE_JOBS))
+$(info )
+$(info Variables available for override (use 'make VAR_NAME=value'):)
+$(info **CPUS** (default: number of processors, e.g., 'make CPUS=4'))
+$(info **LOADAVG** (default: 1.5 times CPUS, e.g., 'make LOADAVG=54'))
+$(info **AVAILABLE_MEM_GB** (default: memory available on the system in GB, e.g., 'make AVAILABLE_MEM_GB=4'))
+$(info **MEM_PER_JOB_GB** (default: 1GB per job, e.g., 'make MEM_PER_JOB_GB=2'))
+$(info ----------------------------------------------------------------------)
+$(info !!!!!! Build starts !!!!!!)
 
-WGET ?= wget
 
 # Timestamps should be in ISO format
 DATE=`date --rfc-3339=seconds`
@@ -162,6 +199,7 @@ heads_cc	:= $(CROSS)gcc \
 	-fdebug-prefix-map=$(pwd)=heads \
 	-gno-record-gcc-switches \
 	-D__MUSL__ \
+	--sysroot  $(INSTALL) \
 	-isystem $(INSTALL)/include \
 	-L$(INSTALL)/lib \
 
@@ -230,12 +268,10 @@ all payload:
 FORCE:
 
 # Copies config while replacing predefined placeholders with actual values
+# This is used in a command like 'this && $(call install_config ...) && that'
+# so it needs to evaluate to a shell command.
 define install_config =
-	sed -e 's!@BOARD_BUILD_DIR@!$(board_build)!g' \
-	    -e 's!@BLOB_DIR@!$(pwd)/blobs!g' \
-	    -e 's!@BRAND_DIR@!$(pwd)/branding/$(BRAND_NAME)!g' \
-	    -e 's!@BRAND_NAME@!$(BRAND_NAME)!g' \
-	    "$1" > "$2"
+	$(pwd)/bin/prepare_module_config.sh "$1" "$2" "$(board_build)" "$(BRAND_NAME)"
 endef
 
 # Make helpers to operate on lists of things
@@ -783,6 +819,8 @@ $(board_build)/$(CB_OUTPUT_BASENAME)-gpg-injected.rom: $(board_build)/$(CB_OUTPU
 	./bin/inject_gpg_key.sh --cbfstool "$(build)/$(coreboot_dir)/cbfstool" \
 		"$(board_build)/$(CB_OUTPUT_FILE_GPG_INJ)" "$(PUBKEY_ASC)"
 
+
+#Dev cycles helpers:
 real.clean:
 	for dir in \
 		$(module_dirs) \
@@ -794,6 +832,28 @@ real.clean:
 	done
 	cd install && rm -rf -- *
 real.gitclean:
+	@echo "Cleaning the repository using Git ignore file as a base..."
+	@echo "This will wipe everything not in the Git tree, but keep downloaded coreboot forks (detected as Git repos)."
 	git clean -fxd
+
 real.gitclean_keep_packages:
+	@echo "Cleaning the repository using Git ignore file as a base..."
+	@echo "This will wipe everything not in the Git tree, but keep the 'packages' directory."
 	git clean -fxd -e "packages"
+
+real.remove_canary_files-extract_patch_rebuild_what_changed:
+	@echo "Removing 'canary' files to force Heads to restart building board configurations..."
+	@echo "This will check package integrity, extract them, redo patching on files, and rebuild what needs to be rebuilt."
+	@echo "It will also reinstall the necessary files under './install'."
+	@echo "Limitations: If a patch creates a file in an extracted package directory, this approach may fail without further manual actions."
+	@echo "In such cases, Git will inform you about the file that couldn't be created as expected. Simply delete those files and relaunch the build."
+	@echo "This approach economizes time since most build artifacts do not need to be rebuilt, as the file dates should be the same as when you originally built them."
+	@echo "Only a minimal time is needed for rebuilding, which is also good for your SSD."
+	@echo "*** USE THIS APPROACH FIRST ***"
+	find ./build/ -type f -name ".canary" -print -delete
+	find ./install/*/* -print -exec rm -rf {} +
+
+real.gitclean_keep_packages_and_build:
+	@echo "Cleaning the repository using Git ignore file as a base..."
+	@echo "This will wipe everything not in the Git tree, but keep the 'packages' and 'build' directories."
+	git clean -fxd -e "packages" -e "build"