From 4f3d210f89cbe9e3fa30736a3f329a888c8a5665 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Sun, 28 Jan 2024 23:12:48 -0500 Subject: [PATCH 01/51] Sort .gitignore Noticed a couple of dupes (gz and rom) so I figured I should sort it now to hopefully avoid another one in the future. Signed-off-by: Manuel Mendez Signed-off-by: Thierry Laurion --- .gitignore | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index b41afbf1c..e69113fbb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,25 +1,23 @@ -.*.sw* -*.xz +*.bad *.bz2 +*.cpio +*.dep +*.ffs +*.fv *.gz -*.sign -*.rom -*.o -*.gz -*.tgz *.img -*.rom -*.cpio -typescript* -config/*.old *.log -*~ -crossgcc -clean +*.lz +*.o +*.rom *.sec -*.dep -*.ffs +*.sign +*.tgz *.vol -*.lz -*.fv -*.bad +*.xz +*~ +.*.sw* +clean +config/*.old +crossgcc +typescript* From d396236a833cd08b18e30069c227ec71d3803b78 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Wed, 7 Jun 2023 15:49:13 -0400 Subject: [PATCH 02/51] Remove hard coded paths in shebang lines Remove hard coded paths from shebangs and other references because they do not play well in nix-land. Either use /usr/bin/env to do runtime PATH based lookup or avoid absolute paths so PATH look up happens instead. Signed-off-by: Thierry Laurion Signed-off-by: Manuel Mendez Signed-off-by: Thierry Laurion --- bin/cpio-clean | 2 +- bin/verity-sign | 2 +- blobs/haswell/obtain-mrc | 2 +- blobs/p8z77-m_pro/download_BIOS_clean.sh | 2 +- blobs/t420/extract.sh | 2 +- blobs/t440p/download-clean-me | 2 +- blobs/t440p/extract | 2 +- blobs/w541/download-clean-me | 2 +- blobs/w541/extract | 2 +- blobs/x220/extract.sh | 2 +- blobs/xx20/download_parse_me.sh | 2 +- blobs/xx20/me7_update_parser.py | 2 +- blobs/xx30/download_clean_me.sh | 2 +- blobs/xx30/download_clean_me_manually.sh | 2 +- blobs/xx30/extract.sh | 2 +- blobs/xx30/me_cleaner.py | 2 +- blobs/xx30/vbios_t530.sh | 2 +- blobs/xx30/vbios_w530.sh | 2 +- blobs/z220/download_BIOS_clean.sh | 5 +- modules/slang | 2 +- patches/linux-5.10.5/shebangs.patch | 100 +++++++++ patches/linux-5.5-openpower/shebangs.patch | 190 ++++++++++++++++++ .../unhardcode-ln-in-Makefile.patch | 13 ++ 23 files changed, 325 insertions(+), 21 deletions(-) create mode 100644 patches/linux-5.10.5/shebangs.patch create mode 100644 patches/linux-5.5-openpower/shebangs.patch create mode 100644 patches/slang-2.3.1a/unhardcode-ln-in-Makefile.patch diff --git a/bin/cpio-clean b/bin/cpio-clean index c7e7ea51b..8eaa81f14 100755 --- a/bin/cpio-clean +++ b/bin/cpio-clean @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # Clean all non-deterministric fields in a newc cpio file # # Items fixed: diff --git a/bin/verity-sign b/bin/verity-sign index d616a0d37..2074555ee 100755 --- a/bin/verity-sign +++ b/bin/verity-sign @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # Generate dm-verity hashes and sign the root hash # # Output looks like diff --git a/blobs/haswell/obtain-mrc b/blobs/haswell/obtain-mrc index 42a6daabd..3e87cab32 100755 --- a/blobs/haswell/obtain-mrc +++ b/blobs/haswell/obtain-mrc @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e diff --git a/blobs/p8z77-m_pro/download_BIOS_clean.sh b/blobs/p8z77-m_pro/download_BIOS_clean.sh index 673a617e4..6f604106e 100755 --- a/blobs/p8z77-m_pro/download_BIOS_clean.sh +++ b/blobs/p8z77-m_pro/download_BIOS_clean.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # P7 ASUS function printusage { diff --git a/blobs/t420/extract.sh b/blobs/t420/extract.sh index 422924741..3c8920430 100755 --- a/blobs/t420/extract.sh +++ b/blobs/t420/extract.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash function printusage { echo "Usage: $0 -f -m (optional) -i (optional)" diff --git a/blobs/t440p/download-clean-me b/blobs/t440p/download-clean-me index 26297d89e..37411a40f 100755 --- a/blobs/t440p/download-clean-me +++ b/blobs/t440p/download-clean-me @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e diff --git a/blobs/t440p/extract b/blobs/t440p/extract index 0307b93b6..88a66bbc7 100755 --- a/blobs/t440p/extract +++ b/blobs/t440p/extract @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e diff --git a/blobs/w541/download-clean-me b/blobs/w541/download-clean-me index 26297d89e..37411a40f 100755 --- a/blobs/w541/download-clean-me +++ b/blobs/w541/download-clean-me @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e diff --git a/blobs/w541/extract b/blobs/w541/extract index 0307b93b6..88a66bbc7 100755 --- a/blobs/w541/extract +++ b/blobs/w541/extract @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e diff --git a/blobs/x220/extract.sh b/blobs/x220/extract.sh index 422924741..3c8920430 100755 --- a/blobs/x220/extract.sh +++ b/blobs/x220/extract.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash function printusage { echo "Usage: $0 -f -m (optional) -i (optional)" diff --git a/blobs/xx20/download_parse_me.sh b/blobs/xx20/download_parse_me.sh index 894a2e069..1289c66aa 100755 --- a/blobs/xx20/download_parse_me.sh +++ b/blobs/xx20/download_parse_me.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash BLOBDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" diff --git a/blobs/xx20/me7_update_parser.py b/blobs/xx20/me7_update_parser.py index 8f201c26d..f862140d9 100644 --- a/blobs/xx20/me7_update_parser.py +++ b/blobs/xx20/me7_update_parser.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """ME7 Update binary parser.""" diff --git a/blobs/xx30/download_clean_me.sh b/blobs/xx30/download_clean_me.sh index a259a6477..994f1cdbc 100755 --- a/blobs/xx30/download_clean_me.sh +++ b/blobs/xx30/download_clean_me.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash function printusage { echo "Usage: $0 -m (optional)" diff --git a/blobs/xx30/download_clean_me_manually.sh b/blobs/xx30/download_clean_me_manually.sh index c2cfc9d36..d7e44d010 100755 --- a/blobs/xx30/download_clean_me_manually.sh +++ b/blobs/xx30/download_clean_me_manually.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash function printusage { echo "Usage: $0 -m (optional)" diff --git a/blobs/xx30/extract.sh b/blobs/xx30/extract.sh index f63c4d31e..3a5ee504a 100755 --- a/blobs/xx30/extract.sh +++ b/blobs/xx30/extract.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash function printusage { echo "Usage: $0 -f -m (optional) -i (optional)" diff --git a/blobs/xx30/me_cleaner.py b/blobs/xx30/me_cleaner.py index c2adf0e80..46f883e30 100755 --- a/blobs/xx30/me_cleaner.py +++ b/blobs/xx30/me_cleaner.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # me_cleaner - Tool for partial deblobbing of Intel ME/TXE firmware images # Copyright (C) 2016-2018 Nicola Corna diff --git a/blobs/xx30/vbios_t530.sh b/blobs/xx30/vbios_t530.sh index 0669859d1..dae75f3f0 100755 --- a/blobs/xx30/vbios_t530.sh +++ b/blobs/xx30/vbios_t530.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash BLOBDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" ROMPARSER="94a615302f89b94e70446270197e0f5138d678f3" diff --git a/blobs/xx30/vbios_w530.sh b/blobs/xx30/vbios_w530.sh index b5230e0a6..bbff9c8b5 100755 --- a/blobs/xx30/vbios_w530.sh +++ b/blobs/xx30/vbios_w530.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash BLOBDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" ROMPARSER="94a615302f89b94e70446270197e0f5138d678f3" diff --git a/blobs/z220/download_BIOS_clean.sh b/blobs/z220/download_BIOS_clean.sh index 870574c78..b21ae3b2e 100755 --- a/blobs/z220/download_BIOS_clean.sh +++ b/blobs/z220/download_BIOS_clean.sh @@ -1,4 +1,5 @@ -#!/bin/bash +#!/usr/bin/env bash + # Z220 CMT HP function printusage { @@ -65,4 +66,4 @@ echo "$FINAL_ME_SHA256SUM" | sha256sum --check || { echo "Failed sha256sum verif echo "###Cleaning up..." cd - -rm -r "$extractdir" \ No newline at end of file +rm -r "$extractdir" diff --git a/modules/slang b/modules/slang index 91c57402d..779d6db64 100644 --- a/modules/slang +++ b/modules/slang @@ -8,7 +8,7 @@ slang_hash := 54f0c3007fde918039c058965dffdfd6c5aec0bad0f4227192cc486021f08c36 slang_configure := ./configure \ $(CROSS_TOOLS) \ - ac_cv_path_nc5config=no \ + ac_cv_path_nc5config=ncurses-config \ --prefix "/" \ --host $(MUSL_ARCH)-elf-linux \ --with-z=no \ diff --git a/patches/linux-5.10.5/shebangs.patch b/patches/linux-5.10.5/shebangs.patch new file mode 100644 index 000000000..76a73b5d8 --- /dev/null +++ b/patches/linux-5.10.5/shebangs.patch @@ -0,0 +1,100 @@ +diff --git a/scripts/bloat-o-meter b/scripts/bloat-o-meter +index d7ca46c612b3..652e9542043f 100755 +--- a/scripts/bloat-o-meter ++++ b/scripts/bloat-o-meter +@@ -1,4 +1,4 @@ +-#!/usr/bin/python ++#!/usr/bin/env python + # + # Copyright 2004 Matt Mackall + # +diff --git a/scripts/check-sysctl-docs b/scripts/check-sysctl-docs +index 8bcb9e26c7bc..90137319c50a 100755 +--- a/scripts/check-sysctl-docs ++++ b/scripts/check-sysctl-docs +@@ -1,4 +1,4 @@ +-#!/usr/bin/gawk -f ++#!/usr/bin/env -S gawk -f + # SPDX-License-Identifier: GPL-2.0 + + # Script to check sysctl documentation against source files +diff --git a/scripts/diffconfig b/scripts/diffconfig +index 89abf777f197..627eba5849b5 100755 +--- a/scripts/diffconfig ++++ b/scripts/diffconfig +@@ -1,4 +1,4 @@ +-#!/usr/bin/python ++#!/usr/bin/env python + # SPDX-License-Identifier: GPL-2.0 + # + # diffconfig - a tool to compare .config files. +diff --git a/scripts/get_abi.pl b/scripts/get_abi.pl +index 68dab828a722..92d9aa6cc4f5 100755 +--- a/scripts/get_abi.pl ++++ b/scripts/get_abi.pl +@@ -1,4 +1,4 @@ +-#!/usr/bin/perl ++#!/usr/bin/env perl + # SPDX-License-Identifier: GPL-2.0 + + use strict; +diff --git a/scripts/ld-version.sh b/scripts/ld-version.sh +index f2be0ff9a738..7a5b546ece16 100755 +--- a/scripts/ld-version.sh ++++ b/scripts/ld-version.sh +@@ -1,4 +1,4 @@ +-#!/usr/bin/awk -f ++#!/usr/bin/env -S awk -f + # SPDX-License-Identifier: GPL-2.0 + # extract linker version number from stdin and turn into single number + { +diff --git a/scripts/parse-maintainers.pl b/scripts/parse-maintainers.pl +index 2ca4eb3f190d..9515765158fa 100755 +--- a/scripts/parse-maintainers.pl ++++ b/scripts/parse-maintainers.pl +@@ -1,4 +1,4 @@ +-#!/usr/bin/perl -w ++#!/usr/bin/env -S perl -w + # SPDX-License-Identifier: GPL-2.0 + + use strict; +diff --git a/scripts/show_delta b/scripts/show_delta +index 264399307c4f..28e67e178194 100755 +--- a/scripts/show_delta ++++ b/scripts/show_delta +@@ -1,4 +1,4 @@ +-#!/usr/bin/python ++#!/usr/bin/env python + # SPDX-License-Identifier: GPL-2.0-only + # + # show_deltas: Read list of printk messages instrumented with +diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install +index 40fa6923e80a..828a8615a918 100755 +--- a/scripts/sphinx-pre-install ++++ b/scripts/sphinx-pre-install +@@ -1,4 +1,4 @@ +-#!/usr/bin/perl ++#!/usr/bin/env perl + # SPDX-License-Identifier: GPL-2.0-or-later + use strict; + +diff --git a/scripts/split-man.pl b/scripts/split-man.pl +index c3db607ee9ec..96bd99dc977a 100755 +--- a/scripts/split-man.pl ++++ b/scripts/split-man.pl +@@ -1,4 +1,4 @@ +-#!/usr/bin/perl ++#!/usr/bin/env perl + # SPDX-License-Identifier: GPL-2.0 + # + # Author: Mauro Carvalho Chehab +diff --git a/scripts/ver_linux b/scripts/ver_linux +index 0968a3070eff..345b92f71d2d 100755 +--- a/scripts/ver_linux ++++ b/scripts/ver_linux +@@ -1,4 +1,4 @@ +-#!/usr/bin/awk -f ++#!/usr/bin/env -S awk -f + # SPDX-License-Identifier: GPL-2.0 + # Before running this script please ensure that your PATH is + # typical as you use for compilation/installation. I use diff --git a/patches/linux-5.5-openpower/shebangs.patch b/patches/linux-5.5-openpower/shebangs.patch new file mode 100644 index 000000000..38f5cb533 --- /dev/null +++ b/patches/linux-5.5-openpower/shebangs.patch @@ -0,0 +1,190 @@ +diff --git a/scripts/bloat-o-meter b/scripts/bloat-o-meter +index 8c965f6a9881..bcd66a014453 100755 +--- a/scripts/bloat-o-meter ++++ b/scripts/bloat-o-meter +@@ -1,4 +1,4 @@ +-#!/usr/bin/python ++#!/usr/bin/env python + # + # Copyright 2004 Matt Mackall + # +diff --git a/scripts/coccicheck b/scripts/coccicheck +index e04d328210ac..851c3307ca74 100755 +--- a/scripts/coccicheck ++++ b/scripts/coccicheck +@@ -1,4 +1,4 @@ +-#!/bin/bash ++#!/usr/bin/env bash + # SPDX-License-Identifier: GPL-2.0 + # Linux kernel coccicheck + # +diff --git a/scripts/config b/scripts/config +index e0e39826dae9..ba11e350e8af 100755 +--- a/scripts/config ++++ b/scripts/config +@@ -1,4 +1,4 @@ +-#!/bin/bash ++#!/usr/bin/env bash + # SPDX-License-Identifier: GPL-2.0 + # Manipulate options in a .config file from the command line + +diff --git a/scripts/decode_stacktrace.sh b/scripts/decode_stacktrace.sh +index 13e5fbafdf2f..8c22ac90123b 100755 +--- a/scripts/decode_stacktrace.sh ++++ b/scripts/decode_stacktrace.sh +@@ -1,4 +1,4 @@ +-#!/bin/bash ++#!/usr/bin/env bash + # SPDX-License-Identifier: GPL-2.0 + # (c) 2014, Sasha Levin + #set -x +diff --git a/scripts/diffconfig b/scripts/diffconfig +index 89abf777f197..627eba5849b5 100755 +--- a/scripts/diffconfig ++++ b/scripts/diffconfig +@@ -1,4 +1,4 @@ +-#!/usr/bin/python ++#!/usr/bin/env python + # SPDX-License-Identifier: GPL-2.0 + # + # diffconfig - a tool to compare .config files. +diff --git a/scripts/faddr2line b/scripts/faddr2line +index 6c6439f69a72..5079957d4993 100755 +--- a/scripts/faddr2line ++++ b/scripts/faddr2line +@@ -1,4 +1,4 @@ +-#!/bin/bash ++#!/usr/bin/env bash + # SPDX-License-Identifier: GPL-2.0 + # + # Translate stack dump function offsets. +diff --git a/scripts/find-unused-docs.sh b/scripts/find-unused-docs.sh +index 3f46f8977dc4..363e7ecab554 100755 +--- a/scripts/find-unused-docs.sh ++++ b/scripts/find-unused-docs.sh +@@ -1,4 +1,4 @@ +-#!/bin/bash ++#!/usr/bin/env bash + # (c) 2017, Jonathan Corbet + # sayli karnik + # +diff --git a/scripts/get_abi.pl b/scripts/get_abi.pl +index c738cb795514..ba87b230fe0a 100755 +--- a/scripts/get_abi.pl ++++ b/scripts/get_abi.pl +@@ -1,4 +1,4 @@ +-#!/usr/bin/perl ++#!/usr/bin/env perl + # SPDX-License-Identifier: GPL-2.0 + + use strict; +diff --git a/scripts/gfp-translate b/scripts/gfp-translate +index b2ce416d944b..2fcea3e9afb3 100755 +--- a/scripts/gfp-translate ++++ b/scripts/gfp-translate +@@ -1,4 +1,4 @@ +-#!/bin/bash ++#!/usr/bin/env bash + # SPDX-License-Identifier: GPL-2.0-only + # Translate the bits making up a GFP mask + # (c) 2009, Mel Gorman +diff --git a/scripts/ld-version.sh b/scripts/ld-version.sh +index f2be0ff9a738..081efc679705 100755 +--- a/scripts/ld-version.sh ++++ b/scripts/ld-version.sh +@@ -1,4 +1,4 @@ +-#!/usr/bin/awk -f ++#!/usr/bin/env -S awk -f + # SPDX-License-Identifier: GPL-2.0 + # extract linker version number from stdin and turn into single number + { +diff --git a/scripts/mkuboot.sh b/scripts/mkuboot.sh +index 4b1fe09e9042..0b6df121a3b5 100755 +--- a/scripts/mkuboot.sh ++++ b/scripts/mkuboot.sh +@@ -1,4 +1,4 @@ +-#!/bin/bash ++#!/usr/bin/env bash + # SPDX-License-Identifier: GPL-2.0 + + # +diff --git a/scripts/objdiff b/scripts/objdiff +index 72b0b63c3fe1..95f087300637 100755 +--- a/scripts/objdiff ++++ b/scripts/objdiff +@@ -1,4 +1,4 @@ +-#!/bin/bash ++#!/usr/bin/env bash + # SPDX-License-Identifier: GPL-2.0-only + + # objdiff - a small script for validating that a commit or series of commits +diff --git a/scripts/parse-maintainers.pl b/scripts/parse-maintainers.pl +index 255cef1b098d..c2d11c87fdb4 100644 +--- a/scripts/parse-maintainers.pl ++++ b/scripts/parse-maintainers.pl +@@ -1,4 +1,4 @@ +-#!/usr/bin/perl -w ++#!/usr/bin/env -S perl -w + # SPDX-License-Identifier: GPL-2.0 + + use strict; +diff --git a/scripts/prune-kernel b/scripts/prune-kernel +index e8aa940bc0a9..620230f677bc 100755 +--- a/scripts/prune-kernel ++++ b/scripts/prune-kernel +@@ -1,4 +1,4 @@ +-#!/bin/bash ++#!/usr/bin/env bash + # SPDX-License-Identifier: GPL-2.0 + + # because I use CONFIG_LOCALVERSION_AUTO, not the same version again and +diff --git a/scripts/show_delta b/scripts/show_delta +index 264399307c4f..28e67e178194 100755 +--- a/scripts/show_delta ++++ b/scripts/show_delta +@@ -1,4 +1,4 @@ +-#!/usr/bin/python ++#!/usr/bin/env python + # SPDX-License-Identifier: GPL-2.0-only + # + # show_deltas: Read list of printk messages instrumented with +diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install +index 470ccfe678aa..214ed660c0d0 100755 +--- a/scripts/sphinx-pre-install ++++ b/scripts/sphinx-pre-install +@@ -1,4 +1,4 @@ +-#!/usr/bin/perl ++#!/usr/bin/env perl + # SPDX-License-Identifier: GPL-2.0-or-later + use strict; + +diff --git a/scripts/split-man.pl b/scripts/split-man.pl +index c3db607ee9ec..96bd99dc977a 100755 +--- a/scripts/split-man.pl ++++ b/scripts/split-man.pl +@@ -1,4 +1,4 @@ +-#!/usr/bin/perl ++#!/usr/bin/env perl + # SPDX-License-Identifier: GPL-2.0 + # + # Author: Mauro Carvalho Chehab +diff --git a/scripts/tags.sh b/scripts/tags.sh +index 4e18ae5282a6..02c08d460997 100755 +--- a/scripts/tags.sh ++++ b/scripts/tags.sh +@@ -1,4 +1,4 @@ +-#!/bin/bash ++#!/usr/bin/env bash + # SPDX-License-Identifier: GPL-2.0-only + # Generate tags or cscope files + # Usage tags.sh +diff --git a/scripts/ver_linux b/scripts/ver_linux +index 85005d6b7f10..34e4530fd5b6 100755 +--- a/scripts/ver_linux ++++ b/scripts/ver_linux +@@ -1,4 +1,4 @@ +-#!/usr/bin/awk -f ++#!/usr/bin/env -S awk -f + # SPDX-License-Identifier: GPL-2.0 + # Before running this script please ensure that your PATH is + # typical as you use for compilation/installation. I use diff --git a/patches/slang-2.3.1a/unhardcode-ln-in-Makefile.patch b/patches/slang-2.3.1a/unhardcode-ln-in-Makefile.patch new file mode 100644 index 000000000..590e06915 --- /dev/null +++ b/patches/slang-2.3.1a/unhardcode-ln-in-Makefile.patch @@ -0,0 +1,13 @@ +diff --git a/src/Makefile.in b/src/Makefile.in +index 7fe4dc2..38fe174 100644 +--- a/src/Makefile.in ++++ b/src/Makefile.in +@@ -75,7 +75,7 @@ RM = rm -f + RM_R = rm -rf + AR_CR = ar cr + RMDIR = rmdir +-LN = /bin/ln -sf ++LN = ln -sf + CP = cp + @SET_MAKE@ + #--------------------------------------------------------------------------- From a4ba76fd90f04c1ddb95a4cc045b56b9b6df22c6 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Sun, 28 Jan 2024 22:55:11 -0500 Subject: [PATCH 03/51] modules: minor refactor/tweaks Just some minor clean ups like fixing whitespace and sorting things. I added (bash)/removed (libusb) white space in order to look like the other modules. I sorted the --enable/--disable/--with blocks so that common stuff looked similar which should aid in comparing modules. I also removed a couple of duplicate config options (--disable-fallback-curses & --disable-regex). Signed-off-by: Manuel Mendez Signed-off-by: Thierry Laurion --- modules/bash | 12 ++++++------ modules/cairo | 12 +++++++++--- modules/cryptsetup2 | 2 +- modules/dropbear | 2 +- modules/gpg | 16 ++++++++-------- modules/gpg2 | 41 ++++++++++++++++++++--------------------- modules/ioport | 2 +- modules/kbd | 4 ++-- modules/libassuan | 2 +- modules/libgcrypt | 2 +- modules/libgpg-error | 8 ++++---- modules/libnitrokey | 10 +++++----- modules/libusb | 10 +++++----- modules/lvm2 | 18 +++++++++--------- modules/newt | 2 +- modules/openssl | 5 ++++- modules/pinentry | 13 ++++++------- modules/popt | 3 +-- modules/powerpc-utils | 2 +- modules/qrencode | 2 +- modules/slang | 8 ++++---- modules/tpm2-tss | 2 +- modules/util-linux | 12 ++++++------ 23 files changed, 98 insertions(+), 92 deletions(-) diff --git a/modules/bash b/modules/bash index e29fed726..62f063f93 100644 --- a/modules/bash +++ b/modules/bash @@ -14,16 +14,16 @@ bash_configure := CFLAGS="-g0 -Os" LDFLAGS="-s" ./configure \ $(CROSS_TOOLS) \ --host $(target) \ --prefix="/usr" \ - --enable-largefile \ - --infodir=/usr/share/info \ - --mandir=/usr/share/man \ - --without-bash-malloc \ + --infodir=/usr/share/info \ + --mandir=/usr/share/man \ --disable-coprocesses \ - --enable-debugger \ --disable-net-redirections \ - --enable-single-help-strings \ --disable-nls \ + --enable-debugger \ + --enable-largefile \ --enable-readline \ + --enable-single-help-strings \ + --without-bash-malloc bash_target := $(MAKE_JOBS) \ && $(MAKE) -C $(build)/$(bash_dir) \ diff --git a/modules/cairo b/modules/cairo index b4970652a..81d71e9a4 100644 --- a/modules/cairo +++ b/modules/cairo @@ -12,10 +12,16 @@ cairo_configure := \ ./configure \ --host $(MUSL_ARCH)-elf-linux \ --prefix="/" \ - --disable-xlib --disable-xcb --disable-pdf \ - --disable-ps --disable-svg --disable-script \ - --disable-ft --disable-fc --disable-pthread \ + --disable-fc \ + --disable-ft \ --disable-gobject \ + --disable-pdf \ + --disable-ps \ + --disable-pthread \ + --disable-script \ + --disable-svg \ + --disable-xcb \ + --disable-xlib \ && sed \ -e 's/^hardcode_libdir_flag_spec.*/hardcode_libdir_flag_spec=" -D__LIBTOOL_RPATH_DISABLE__ "/' \ < libtool \ diff --git a/modules/cryptsetup2 b/modules/cryptsetup2 index b7ffd8d43..6becf6db8 100644 --- a/modules/cryptsetup2 +++ b/modules/cryptsetup2 @@ -16,8 +16,8 @@ cryptsetup2_configure := \ ./configure \ --host $(MUSL_ARCH)-elf-linux \ --prefix "/" \ - --disable-rpath \ --disable-gcrypt-pbkdf2 \ + --disable-rpath \ --enable-cryptsetup-reencrypt \ --with-crypto_backend=kernel \ --with-tmpfilesdir=$(INSTALL)/lib/tmpfiles.d diff --git a/modules/dropbear b/modules/dropbear index 285f97bb7..ac32a59ca 100644 --- a/modules/dropbear +++ b/modules/dropbear @@ -11,8 +11,8 @@ dropbear_configure := ./configure \ $(CROSS_TOOLS) \ --host $(MUSL_ARCH)-elf-linux \ --prefix "/" \ - --disable-syslog \ --disable-lastlog \ + --disable-syslog \ --disable-utmp \ --disable-utmpx \ --disable-wtmp \ diff --git a/modules/gpg b/modules/gpg index e6b8f2c52..7bab72e71 100644 --- a/modules/gpg +++ b/modules/gpg @@ -23,20 +23,20 @@ gpg_configure := \ ./configure \ --build $(MUSL_ARCH)-elf-linux \ --host $(MUSL_ARCH)-linux-musl \ - --with-libusb="$(INSTALL)" \ --prefix "/" \ - --enable-card-support \ - --disable-bzip2 \ --disable-asm \ + --disable-bzip2 \ + --disable-dns-cert \ + --disable-dns-srv \ --disable-exec \ - --disable-photo-viewers \ + --disable-finger \ + --disable-hkp \ --disable-keyserver-helpers \ --disable-ldap \ - --disable-hkp \ - --disable-finger \ - --disable-dns-srv \ - --disable-dns-cert \ + --disable-photo-viewers \ --disable-regex \ + --enable-card-support \ + --with-libusb="$(INSTALL)" \ # Run one build to generate the executables with the pre-defined # exec_prefix and datarootdir, then a second make to install the binaries diff --git a/modules/gpg2 b/modules/gpg2 index 141dbeb3b..6190ec5c2 100644 --- a/modules/gpg2 +++ b/modules/gpg2 @@ -18,35 +18,34 @@ gpg2_configure := \ ./configure \ CPPFLAGS="-I$(INSTALL)/include/libusb-1.0" \ --host $(MUSL_ARCH)-linux-musl \ - --with-libusb="$(INSTALL)" \ - --with-gpg-error-prefix="$(INSTALL)" \ - --with-libgcrypt-prefix="$(INSTALL)" \ - --with-libassuan-prefix="$(INSTALL)" \ - --with-ksba-prefix="$(INSTALL)" \ - --with-npth-prefix="$(INSTALL)" \ --prefix "/" \ --libexecdir "/bin" \ - --enable-scdaemon \ - --enable-ccid-driver \ - --disable-tofu \ - --disable-rpath \ - --disable-regex \ - --disable-doc \ + --disable-all-tests \ --disable-bzip2 \ + --disable-dirmngr \ + --disable-doc \ --disable-exec \ - --disable-photo-viewers \ + --disable-gnutls \ + --disable-gpgsm \ --disable-ldap \ - --disable-regex \ + --disable-libdns \ --disable-nls \ - --disable-all-tests \ - --disable-wks-tools \ - --disable-gnutls \ - --disable-dirmngr \ --disable-ntbtls \ - --disable-libdns \ - --disable-zip \ + --disable-photo-viewers \ + --disable-regex \ + --disable-rpath \ --disable-sqlite \ - --disable-gpgsm \ + --disable-tofu \ + --disable-wks-tools \ + --disable-zip \ + --enable-ccid-driver \ + --enable-scdaemon \ + --with-gpg-error-prefix="$(INSTALL)" \ + --with-ksba-prefix="$(INSTALL)" \ + --with-libassuan-prefix="$(INSTALL)" \ + --with-libgcrypt-prefix="$(INSTALL)" \ + --with-libusb="$(INSTALL)" \ + --with-npth-prefix="$(INSTALL)" \ # Run one build to generate the executables with the pre-defined # exec_prefix and datarootdir, then a second make to install the binaries diff --git a/modules/ioport b/modules/ioport index 2d1c6b352..ab25c9299 100644 --- a/modules/ioport +++ b/modules/ioport @@ -8,8 +8,8 @@ ioport_hash := 7fac1c4b61eb9411275de0e1e7d7a8c3f34166f64f16413f50741e8fce2b8dc0 ioport_configure := CFLAGS=-Os ./configure \ $(CROSS_TOOLS) \ + --host i386-elf-linux \ --prefix "/" \ - --host i386-elf-linux ioport_target := \ $(MAKE_JOBS) $(CROSS_TOOLS) inb && \ diff --git a/modules/kbd b/modules/kbd index f6d3f405a..2286e5cc7 100644 --- a/modules/kbd +++ b/modules/kbd @@ -16,11 +16,11 @@ kbd_hash := aaed530a1490d63d041448372e2ad4f38c3179042903251000b71d527c46e945 kbd_configure := CFLAGS=-Os ./configure \ $(CROSS_TOOLS) \ - --prefix "" \ --host i386-elf-linux \ - --disable-optional-progs \ + --prefix "" \ --disable-libkeymap \ --disable-libkfont \ + --disable-optional-progs \ --disable-vlock \ kbd_target := \ diff --git a/modules/libassuan b/modules/libassuan index a2536be7c..e641854f5 100644 --- a/modules/libassuan +++ b/modules/libassuan @@ -12,8 +12,8 @@ libassuan_configure := \ ./configure \ --host $(MUSL_ARCH)-linux-musl \ --prefix "/" \ - --disable-static \ --disable-doc \ + --disable-static \ --with-gpg-error-prefix="$(INSTALL)" \ libassuan_target := $(MAKE_JOBS) \ diff --git a/modules/libgcrypt b/modules/libgcrypt index c46fe6085..a3ece058b 100644 --- a/modules/libgcrypt +++ b/modules/libgcrypt @@ -12,9 +12,9 @@ libgcrypt_configure := \ ./configure \ --host=$(MUSL_ARCH)-linux-musl \ --prefix "/" \ + --disable-doc \ --disable-static \ --with-gpg-error-prefix="$(INSTALL)" \ - --disable-doc \ libgcrypt_target := $(MAKE_JOBS) \ DESTDIR="$(INSTALL)" \ diff --git a/modules/libgpg-error b/modules/libgpg-error index 28b01f32b..0c8553560 100644 --- a/modules/libgpg-error +++ b/modules/libgpg-error @@ -10,12 +10,12 @@ libgpg-error_configure := \ $(CROSS_TOOLS) \ CFLAGS="-Os" \ ./configure \ - --prefix "/" \ --host=$(MUSL_ARCH)-linux-musl \ - --disable-static \ - --disable-nls \ - --disable-languages \ + --prefix "/" \ --disable-doc \ + --disable-languages \ + --disable-nls \ + --disable-static \ --disable-tests \ --enable-install-gpg-error-config \ diff --git a/modules/libnitrokey b/modules/libnitrokey index 455680f48..8bdb7116b 100644 --- a/modules/libnitrokey +++ b/modules/libnitrokey @@ -14,20 +14,20 @@ libnitrokey_hash := 4f3382b6193afe69c2001321038fce9490bc28803ed687152a397ccd8914 cmake_cross := "-DCMAKE_AR=$(CROSS)ar" \ -DCMAKE_CXX_COMPILER="$(CROSS)g++" \ - -DCMAKE_C_COMPILER="$(CROSS)gcc" \ + -DCMAKE_CXX_COMPILER_AR="$(CROSS)ar" \ -DCMAKE_CXX_FLAGS="-Os -fdata-sections -ffunction-sections -ffile-prefix-map=$(pwd)=heads -gno-record-gcc-switches -D__MUSL__ -I$(INSTALL)/include -L$(INSTALL)/lib " \ + -DCMAKE_C_COMPILER="$(CROSS)gcc" \ -DCMAKE_LINKER="$(CROSS)ld" - -DCMAKE_SHARED_LINKER_FLAGS="-gc-sections" - -DCMAKE_CXX_COMPILER_AR="$(CROSS)ar" \ -DCMAKE_NM="$(CROSS)nm" \ - -DCMAKE_OBJDUMP="$(CROSS)objdump" \ -DCMAKE_OBCOPY="$(CROSS)obcopy" \ + -DCMAKE_OBJDUMP="$(CROSS)objdump" \ + -DCMAKE_SHARED_LINKER_FLAGS="-gc-sections" -DCMAKE_STRIP="$(CROSS)strip" libnitrokey_configure := \ mkdir build -p && \ cd build && \ - $(CROSS_TOOLS) cmake .. -DNO_LOG=ON -DBUILD_SHARED_LIBS=OFF -DCOMPILE_TESTS=OFF -DCMAKE_INSTALL_PREFIX=/ -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_BUILD_TYPE=MinSizeRel $(cmake_cross) + $(CROSS_TOOLS) cmake .. -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=MinSizeRel -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_PREFIX=/ -DCOMPILE_TESTS=OFF -DNO_LOG=ON $(cmake_cross) # install "by-hand" as INSTALL_PREFIX is not working as expected libnitrokey_target := $(CROSS_TOOLS) $(MAKE_JOBS) -C build DESTDIR="$(INSTALL)" && \ diff --git a/modules/libusb b/modules/libusb index a3d40c563..2e9569f43 100644 --- a/modules/libusb +++ b/modules/libusb @@ -12,11 +12,11 @@ libusb_hash := 7dce9cce9a81194b7065ee912bcd55eeffebab694ea403ffb91b67db66b1824b libusb_configure := \ CFLAGS="-Os" \ ./configure \ - $(CROSS_TOOLS)\ - --host $(MUSL_ARCH)-elf-linux\ - --prefix "/"\ - --disable-udev\ - --disable-tests\ + $(CROSS_TOOLS) \ + --host $(MUSL_ARCH)-elf-linux \ + --prefix "/" \ + --disable-tests \ + --disable-udev \ # Run one build to generate the executables with the pre-defined # exec_prefix and datarootdir, then a second make to install the binaries diff --git a/modules/lvm2 b/modules/lvm2 index be1435a36..e51292d6d 100644 --- a/modules/lvm2 +++ b/modules/lvm2 @@ -18,20 +18,20 @@ lvm2_configure := \ ./configure \ --host $(MUSL_ARCH)-elf-linux \ --prefix "/" \ - --enable-devmapper \ - --disable-selinux \ - --disable-udev-systemd-background-jobs \ - --disable-realtime \ + --disable-blkid_wiping \ + --disable-cache_check_needs_check \ + --disable-cmirrord \ --disable-dmeventd \ --disable-lvmetad \ --disable-lvmpolld \ - --disable-use-lvmlockd \ + --disable-realtime \ + --disable-selinux \ + --disable-thin_check_needs_check \ + --disable-udev-systemd-background-jobs \ --disable-use-lvmetad \ + --disable-use-lvmlockd \ --disable-use-lvmpolld \ - --disable-blkid_wiping \ - --disable-cmirrord \ - --disable-cache_check_needs_check \ - --disable-thin_check_needs_check \ + --enable-devmapper \ --with-cluster=none \ --with-thin-check= \ diff --git a/modules/newt b/modules/newt index 3cd37658e..d9fb2c2d2 100644 --- a/modules/newt +++ b/modules/newt @@ -23,6 +23,6 @@ newt_libraries := \ newt_configure := \ ./autogen.sh && CFLAGS="-Os" ./configure \ $(CROSS_TOOLS) \ - --prefix "/" \ --host $(MUSL_ARCH)-elf-linux \ + --prefix "/" \ --without-tcl diff --git a/modules/openssl b/modules/openssl index a9cbdcf5e..47072442b 100644 --- a/modules/openssl +++ b/modules/openssl @@ -7,6 +7,9 @@ openssl_tar := openssl-$(openssl_version).tar.gz openssl_url := https://www.openssl.org/source/$(openssl_tar) openssl_hash := 6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d0df4a41253e4b3e +# hack to provide path to libgcc +LIBGCC_DIR := $(dir $(shell $(heads_cc) -print-libgcc-file-name)) + # The only optional algorithm that's enabled is SM3. tpm2-tss uses SHA, AES, # and SM3. openssl_configure := \ @@ -14,7 +17,6 @@ openssl_configure := \ CFLAGS="-Os" \ ./Configure \ --prefix="/" \ - no-tests \ linux-$(strip $(arch)) \ no-aria \ no-bf \ @@ -42,6 +44,7 @@ openssl_configure := \ no-siv \ no-sm2 \ no-sm4 \ + no-tests \ no-whirlpool \ openssl_target := $(MAKE_JOBS) \ diff --git a/modules/pinentry b/modules/pinentry index 7fa92cdfe..239c6b755 100644 --- a/modules/pinentry +++ b/modules/pinentry @@ -19,17 +19,16 @@ pinentry_configure := \ ./configure \ --host $(MUSL_ARCH)-linux-musl \ --prefix "/" \ - --enable-pinentry-tty \ - --disable-libsecret \ --disable-fallback-curses \ + --disable-libsecret \ --disable-pinentry-curses \ - --disable-pinentry-qt \ - --disable-pinentry-gtk2 \ - --disable-pinentry-gnome3 \ - --disable-pinentry-fltk \ --disable-pinentry-emacs \ - --disable-fallback-curses \ + --disable-pinentry-fltk \ + --disable-pinentry-gnome3 \ + --disable-pinentry-gtk2 \ + --disable-pinentry-qt \ --disable-pinentry-qt5 \ + --enable-pinentry-tty \ --with-gpg-error-prefix="$(INSTALL)" \ --with-libassuan-prefix="$(INSTALL)" \ diff --git a/modules/popt b/modules/popt index 9a0ec319b..6e24422d6 100644 --- a/modules/popt +++ b/modules/popt @@ -8,9 +8,8 @@ popt_hash := c25a4838fc8e4c1c8aacb8bd620edb3084a3d63bf8987fdad3ca2758c63240f9 popt_configure := ./configure \ $(CROSS_TOOLS) \ - --prefix "/" \ --host $(MUSL_ARCH)-elf-linux \ - + --prefix "/" \ popt_target := \ $(MAKE_JOBS) \ diff --git a/modules/powerpc-utils b/modules/powerpc-utils index dc19d13f6..5526afd8b 100644 --- a/modules/powerpc-utils +++ b/modules/powerpc-utils @@ -13,8 +13,8 @@ powerpc-utils_configure := \ ./configure \ --host $(MUSL_ARCH)-elf-linux \ --prefix "/" \ + --without-librtas \ --without-systemd \ - --without-librtas powerpc-utils_target := \ $(MAKE_JOBS) \ diff --git a/modules/qrencode b/modules/qrencode index 565b98f64..fbfbb870c 100644 --- a/modules/qrencode +++ b/modules/qrencode @@ -12,9 +12,9 @@ qrencode_configure := \ CFLAGS="-Os" \ ./configure \ $(CROSS_TOOLS) \ + --host $(MUSL_ARCH)-elf-linux \ --prefix "/" \ --without-tools \ - --host $(MUSL_ARCH)-elf-linux \ qrencode_target := \ $(MAKE_JOBS) \ diff --git a/modules/slang b/modules/slang index 779d6db64..c99c64497 100644 --- a/modules/slang +++ b/modules/slang @@ -9,12 +9,12 @@ slang_hash := 54f0c3007fde918039c058965dffdfd6c5aec0bad0f4227192cc486021f08c36 slang_configure := ./configure \ $(CROSS_TOOLS) \ ac_cv_path_nc5config=ncurses-config \ - --prefix "/" \ --host $(MUSL_ARCH)-elf-linux \ - --with-z=no \ - --with-png=no \ - --with-pcre=no \ + --prefix "/" \ --with-onig=no \ + --with-pcre=no \ + --with-png=no \ + --with-z=no \ && mkdir -p src/elfobjs # Disable parallel make for the install target diff --git a/modules/tpm2-tss b/modules/tpm2-tss index 3795e21f5..c8c1c84e8 100644 --- a/modules/tpm2-tss +++ b/modules/tpm2-tss @@ -22,9 +22,9 @@ tpm2-tss_configure := aclocal && automake --add-missing && autoreconf -fi \ --host $(MUSL_ARCH)-elf-linux \ --prefix "/" \ --disable-doxygen-doc \ + --disable-doxygen-html \ --disable-doxygen-man \ --disable-doxygen-rtf \ - --disable-doxygen-html \ --disable-fapi \ --disable-static \ diff --git a/modules/util-linux b/modules/util-linux index 804ae56ea..9ab8dae55 100644 --- a/modules/util-linux +++ b/modules/util-linux @@ -13,16 +13,16 @@ util-linux_configure := \ --host $(MUSL_ARCH)-elf-linux \ --prefix "/" \ --oldincludedir "$(INSTALL)/include" \ + --disable-all-programs \ + --disable-bash-completion \ + --enable-agetty \ + --enable-libblkid \ + --enable-libuuid \ --without-ncurses \ --without-ncursesw \ + --without-python \ --without-tinfo \ --without-udev \ - --without-python \ - --disable-bash-completion \ - --disable-all-programs \ - --enable-libuuid \ - --enable-libblkid \ - --enable-agetty \ util-linux_target := \ From 67e5973b5db5e0e54480368665c3253d6d42db95 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 24 Jan 2024 12:01:01 -0500 Subject: [PATCH 04/51] modules: Remove unrecognized configure options Signed-off-by: Thierry Laurion Signed-off-by: Manuel Mendez Signed-off-by: Thierry Laurion --- modules/gpg2 | 2 -- modules/libusb | 1 - modules/npth | 1 - 3 files changed, 4 deletions(-) diff --git a/modules/gpg2 b/modules/gpg2 index 6190ec5c2..b121febc8 100644 --- a/modules/gpg2 +++ b/modules/gpg2 @@ -32,7 +32,6 @@ gpg2_configure := \ --disable-nls \ --disable-ntbtls \ --disable-photo-viewers \ - --disable-regex \ --disable-rpath \ --disable-sqlite \ --disable-tofu \ @@ -44,7 +43,6 @@ gpg2_configure := \ --with-ksba-prefix="$(INSTALL)" \ --with-libassuan-prefix="$(INSTALL)" \ --with-libgcrypt-prefix="$(INSTALL)" \ - --with-libusb="$(INSTALL)" \ --with-npth-prefix="$(INSTALL)" \ # Run one build to generate the executables with the pre-defined diff --git a/modules/libusb b/modules/libusb index 2e9569f43..56dfc044e 100644 --- a/modules/libusb +++ b/modules/libusb @@ -15,7 +15,6 @@ libusb_configure := \ $(CROSS_TOOLS) \ --host $(MUSL_ARCH)-elf-linux \ --prefix "/" \ - --disable-tests \ --disable-udev \ # Run one build to generate the executables with the pre-defined diff --git a/modules/npth b/modules/npth index b6bdb79db..1bbec1c03 100644 --- a/modules/npth +++ b/modules/npth @@ -11,7 +11,6 @@ npth_configure := ./configure \ --host $(MUSL_ARCH)-linux-musl \ --prefix "/" \ --disable-static \ - --with-gpg-error-prefix="$(INSTALL)" \ npth_target := $(MAKE_JOBS) \ DESTDIR="$(INSTALL)" \ From 236f9b117cf3fcc7424bbe2d6f38dbc06d7663ca Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Fri, 30 Jun 2023 14:20:23 -0400 Subject: [PATCH 05/51] modules/gpg2: Bump version 2.4.0 -> 2.4.2 More pending work needed to fix fragility of buildsystem and fix nix build issues as well like: https://app.circleci.com/pipelines/github/mmlb/osresearch-heads/11/workflows/32cc883c-5074-4f28-94b8-a83a2ec44414/jobs/252 https://app.circleci.com/pipelines/github/mmlb/osresearch-heads/11/workflows/32cc883c-5074-4f28-94b8-a83a2ec44414/jobs/221 https://app.circleci.com/pipelines/github/tlaurion/heads/1781/workflows/ee402ead-6739-4549-88ae-105b695fb3cd https://app.circleci.com/pipelines/github/tlaurion/heads/1783/workflows/2b35826c-aff4-4f48-8809-4e66259f9aa4/jobs/25877/parallel-runs/0/steps/0-103 Signed-off-by: Thierry Laurion Signed-off-by: Manuel Mendez Signed-off-by: Thierry Laurion --- Makefile | 1 + modules/flashrom | 8 ++++++-- modules/gpg2 | 4 ++-- modules/libassuan | 6 +++--- modules/libgcrypt | 6 +++--- modules/libgpg-error | 4 ++-- modules/libksba | 6 +++--- modules/pinentry | 2 +- patches/{gpg2-2.4.0.patch => gpg2-2.4.2.patch} | 0 patches/{libassuan-2.5.5.patch => libassuan-2.5.6.patch} | 0 patches/{libgcrypt-1.8.3.patch => libgcrypt-1.10.2.patch} | 0 .../{libgpg-error-1.46.patch => libgpg-error-1.47.patch} | 0 patches/{libksba-1.6.3.patch => libksba-1.6.4.patch} | 0 13 files changed, 21 insertions(+), 16 deletions(-) rename patches/{gpg2-2.4.0.patch => gpg2-2.4.2.patch} (100%) rename patches/{libassuan-2.5.5.patch => libassuan-2.5.6.patch} (100%) rename patches/{libgcrypt-1.8.3.patch => libgcrypt-1.10.2.patch} (100%) rename patches/{libgpg-error-1.46.patch => libgpg-error-1.47.patch} (100%) rename patches/{libksba-1.6.3.patch => libksba-1.6.4.patch} (100%) diff --git a/Makefile b/Makefile index 4f113d9c9..84f5af112 100644 --- a/Makefile +++ b/Makefile @@ -162,6 +162,7 @@ heads_cc := $(CROSS)gcc \ -fdebug-prefix-map=$(pwd)=heads \ -gno-record-gcc-switches \ -D__MUSL__ \ + --sysroot $(INSTALL) \ -isystem $(INSTALL)/include \ -L$(INSTALL)/lib \ diff --git a/modules/flashrom b/modules/flashrom index c7ce3e949..52db5ec6e 100644 --- a/modules/flashrom +++ b/modules/flashrom @@ -27,10 +27,14 @@ ifeq "$(CONFIG_FLASHROM_AST1100)" "y" flashrom_cfg += CONFIG_AST1100=yes endif - - flashrom_target := \ $(MAKE_JOBS) \ + CFLAGS="-Os -I$(INSTALL)/include/pci" \ + DESTDIR="$(INSTALL)" \ + INSTALL="$(INSTALL)" \ + LDFLAGS="-L$(INSTALL)/lib" \ + LIBS_BASE="$(INSTALL)" \ + PREFIX="$(INSTALL)" \ $(CROSS_TOOLS) \ $(flashrom_cfg) \ flashrom diff --git a/modules/gpg2 b/modules/gpg2 index b121febc8..fe340c4ff 100644 --- a/modules/gpg2 +++ b/modules/gpg2 @@ -1,10 +1,10 @@ modules-$(CONFIG_GPG2) += gpg2 -gpg2_version := 2.4.0 +gpg2_version := 2.4.2 gpg2_dir := gnupg-$(gpg2_version) gpg2_tar := gnupg-$(gpg2_version).tar.bz2 gpg2_url := https://www.gnupg.org/ftp/gcrypt/gnupg/$(gpg2_tar) -gpg2_hash := 1d79158dd01d992431dd2e3facb89fdac97127f89784ea2cb610c600fb0c1483 +gpg2_hash := 97eb47df8ae5a3ff744f868005a090da5ab45cb48ee9836dbf5ee739a4e5cf49 gpg2_depends := libgpg-error libgcrypt libksba libassuan npth libusb $(musl_dep) # For reproducibility reasons we have to override the exec_prefix diff --git a/modules/libassuan b/modules/libassuan index e641854f5..7143534d1 100644 --- a/modules/libassuan +++ b/modules/libassuan @@ -1,10 +1,10 @@ modules-$(CONFIG_GPG2) += libassuan -libassuan_version := 2.5.5 +libassuan_version := 2.5.6 libassuan_dir := libassuan-$(libassuan_version) libassuan_tar := libassuan-$(libassuan_version).tar.bz2 libassuan_url := https://gnupg.org/ftp/gcrypt/libassuan/$(libassuan_tar) -libassuan_hash := 8e8c2fcc982f9ca67dcbb1d95e2dc746b1739a4668bc20b3a3c5be632edb34e4 +libassuan_hash := e9fd27218d5394904e4e39788f9b1742711c3e6b41689a31aa3380bd5aa4f426 libassuan_configure := \ CFLAGS="-Os" \ @@ -14,7 +14,7 @@ libassuan_configure := \ --prefix "/" \ --disable-doc \ --disable-static \ - --with-gpg-error-prefix="$(INSTALL)" \ + --with-libgpg-error-prefix="$(INSTALL)" \ libassuan_target := $(MAKE_JOBS) \ DESTDIR="$(INSTALL)" \ diff --git a/modules/libgcrypt b/modules/libgcrypt index a3ece058b..2b630c02d 100644 --- a/modules/libgcrypt +++ b/modules/libgcrypt @@ -1,10 +1,10 @@ modules-$(CONFIG_GPG2) += libgcrypt -libgcrypt_version := 1.10.1 +libgcrypt_version := 1.10.2 libgcrypt_dir := libgcrypt-$(libgcrypt_version) libgcrypt_tar := libgcrypt-$(libgcrypt_version).tar.bz2 libgcrypt_url := https://gnupg.org/ftp/gcrypt/libgcrypt/$(libgcrypt_tar) -libgcrypt_hash := ef14ae546b0084cd84259f61a55e07a38c3b53afc0f546bffcef2f01baffe9de +libgcrypt_hash := 3b9c02a004b68c256add99701de00b383accccf37177e0d6c58289664cce0c03 libgcrypt_configure := \ $(CROSS_TOOLS) \ @@ -14,7 +14,7 @@ libgcrypt_configure := \ --prefix "/" \ --disable-doc \ --disable-static \ - --with-gpg-error-prefix="$(INSTALL)" \ + --with-libgpg-error-prefix="$(INSTALL)" \ libgcrypt_target := $(MAKE_JOBS) \ DESTDIR="$(INSTALL)" \ diff --git a/modules/libgpg-error b/modules/libgpg-error index 0c8553560..b24c9f74f 100644 --- a/modules/libgpg-error +++ b/modules/libgpg-error @@ -1,10 +1,10 @@ modules-$(CONFIG_GPG2) += libgpg-error -libgpg-error_version := 1.46 +libgpg-error_version := 1.47 libgpg-error_dir := libgpg-error-$(libgpg-error_version) libgpg-error_tar := libgpg-error-$(libgpg-error_version).tar.bz2 libgpg-error_url := https://gnupg.org/ftp/gcrypt/libgpg-error/$(libgpg-error_tar) -libgpg-error_hash := b7e11a64246bbe5ef37748de43b245abd72cfcd53c9ae5e7fc5ca59f1c81268d +libgpg-error_hash := 9e3c670966b96ecc746c28c2c419541e3bcb787d1a73930f5e5f5e1bcbbb9bdb libgpg-error_configure := \ $(CROSS_TOOLS) \ diff --git a/modules/libksba b/modules/libksba index 7230237a7..99c226788 100644 --- a/modules/libksba +++ b/modules/libksba @@ -1,10 +1,10 @@ modules-$(CONFIG_GPG2) += libksba -libksba_version := 1.6.3 +libksba_version := 1.6.4 libksba_dir := libksba-$(libksba_version) libksba_tar := libksba-$(libksba_version).tar.bz2 libksba_url := https://gnupg.org/ftp/gcrypt/libksba/$(libksba_tar) -libksba_hash := 3f72c68db30971ebbf14367527719423f0a4d5f8103fc9f4a1c01a9fa440de5c +libksba_hash := bbb43f032b9164d86c781ffe42213a83bf4f2fee91455edfa4654521b8b03b6b libksba_configure := \ $(CROSS_TOOLS) \ @@ -13,7 +13,7 @@ libksba_configure := \ --host $(MUSL_ARCH)-linux-musl \ --prefix "/" \ --disable-static \ - --with-gpg-error-prefix="$(INSTALL)" \ + --with-libgpg-error-prefix="$(INSTALL)" \ libksba_target := $(MAKE_JOBS) \ DESTDIR="$(INSTALL)" \ diff --git a/modules/pinentry b/modules/pinentry index 239c6b755..86ab96b6a 100644 --- a/modules/pinentry +++ b/modules/pinentry @@ -29,8 +29,8 @@ pinentry_configure := \ --disable-pinentry-qt \ --disable-pinentry-qt5 \ --enable-pinentry-tty \ - --with-gpg-error-prefix="$(INSTALL)" \ --with-libassuan-prefix="$(INSTALL)" \ + --with-libgpg-error-prefix="$(INSTALL)" \ # Run one build to generate the executables with the pre-defined # exec_prefix and datarootdir, then a second make to install the binaries diff --git a/patches/gpg2-2.4.0.patch b/patches/gpg2-2.4.2.patch similarity index 100% rename from patches/gpg2-2.4.0.patch rename to patches/gpg2-2.4.2.patch diff --git a/patches/libassuan-2.5.5.patch b/patches/libassuan-2.5.6.patch similarity index 100% rename from patches/libassuan-2.5.5.patch rename to patches/libassuan-2.5.6.patch diff --git a/patches/libgcrypt-1.8.3.patch b/patches/libgcrypt-1.10.2.patch similarity index 100% rename from patches/libgcrypt-1.8.3.patch rename to patches/libgcrypt-1.10.2.patch diff --git a/patches/libgpg-error-1.46.patch b/patches/libgpg-error-1.47.patch similarity index 100% rename from patches/libgpg-error-1.46.patch rename to patches/libgpg-error-1.47.patch diff --git a/patches/libksba-1.6.3.patch b/patches/libksba-1.6.4.patch similarity index 100% rename from patches/libksba-1.6.3.patch rename to patches/libksba-1.6.4.patch From fcb9596f7e6edececc5ed385ecaf4554c46d1bd5 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Thu, 25 Jan 2024 21:47:37 -0500 Subject: [PATCH 06/51] modules/tpm2-tss: Add with-sysroot to configure args Thanks to @JonathonHall-Purism, that pointed to me that sysroot was neglected in tpm2-tools configure step. Signed-off-by: Thierry Laurion Signed-off-by: Manuel Mendez Signed-off-by: Thierry Laurion --- modules/tpm2-tss | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/tpm2-tss b/modules/tpm2-tss index c8c1c84e8..91e1a26cd 100644 --- a/modules/tpm2-tss +++ b/modules/tpm2-tss @@ -21,6 +21,7 @@ tpm2-tss_configure := aclocal && automake --add-missing && autoreconf -fi \ $(CROSS_TOOLS) \ --host $(MUSL_ARCH)-elf-linux \ --prefix "/" \ + --with-sysroot=$(INSTALL) \ --disable-doxygen-doc \ --disable-doxygen-html \ --disable-doxygen-man \ From 43d806f20528101e251d9d446eac694fe38c202f Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Sun, 2 Jul 2023 14:21:11 -0400 Subject: [PATCH 07/51] modules/tpm2-tools: add with-sysroot, TSS2_ESYS_3_0_LIBS to configure args Thanks to @JonathonHall-Purism, that pointed to me that sysroot was neglected in tpm2-tools configure step. I wonder why this is not respected if not forced with --with-sysroot and TSS2_ESYS_3_0_LIBS="-ltss2-esys -L$(INSTALL)/lib"? Signed-off-by: Thierry Laurion Signed-off-by: Manuel Mendez Signed-off-by: Thierry Laurion --- modules/tpm2-tools | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/tpm2-tools b/modules/tpm2-tools index bba25bc5d..9d4f1247c 100644 --- a/modules/tpm2-tools +++ b/modules/tpm2-tools @@ -32,6 +32,9 @@ tpm2-tools_configure := \ --prefix "/" \ --disable-fapi \ CFLAGS="-fdebug-prefix-map=$(INSTALL)=." \ + --with-sysroot=$(INSTALL) \ + TSS2_ESYS_3_0_CFLAGS="-I$(INSTALL)/include" \ + TSS2_ESYS_3_0_LIBS="-ltss2-esys -L$(INSTALL)/lib" \ tpm2-tools_target := $(MAKE_JOBS) \ DESTDIR="$(INSTALL)" \ From 87ec2ca043845445b1977a702ea2262f5913dbe4 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Wed, 28 Feb 2024 19:03:30 -0500 Subject: [PATCH 08/51] blobs: Ignore build generated binaries These are generated during the build process so should be ignored. Signed-off-by: Manuel Mendez Signed-off-by: Thierry Laurion --- blobs/haswell/.gitignore | 1 + blobs/t440p/.gitignore | 1 + blobs/w541/.gitignore | 1 + blobs/xx20/.gitignore | 1 + blobs/xx30/.gitignore | 1 + blobs/z220/.gitignore | 2 ++ 6 files changed, 7 insertions(+) create mode 100644 blobs/haswell/.gitignore create mode 100644 blobs/t440p/.gitignore create mode 100644 blobs/w541/.gitignore create mode 100644 blobs/xx20/.gitignore create mode 100644 blobs/xx30/.gitignore create mode 100644 blobs/z220/.gitignore diff --git a/blobs/haswell/.gitignore b/blobs/haswell/.gitignore new file mode 100644 index 000000000..b3810c18c --- /dev/null +++ b/blobs/haswell/.gitignore @@ -0,0 +1 @@ +mrc.bin diff --git a/blobs/t440p/.gitignore b/blobs/t440p/.gitignore new file mode 100644 index 000000000..24d49395b --- /dev/null +++ b/blobs/t440p/.gitignore @@ -0,0 +1 @@ +me.bin diff --git a/blobs/w541/.gitignore b/blobs/w541/.gitignore new file mode 100644 index 000000000..24d49395b --- /dev/null +++ b/blobs/w541/.gitignore @@ -0,0 +1 @@ +me.bin diff --git a/blobs/xx20/.gitignore b/blobs/xx20/.gitignore new file mode 100644 index 000000000..24d49395b --- /dev/null +++ b/blobs/xx20/.gitignore @@ -0,0 +1 @@ +me.bin diff --git a/blobs/xx30/.gitignore b/blobs/xx30/.gitignore new file mode 100644 index 000000000..24d49395b --- /dev/null +++ b/blobs/xx30/.gitignore @@ -0,0 +1 @@ +me.bin diff --git a/blobs/z220/.gitignore b/blobs/z220/.gitignore new file mode 100644 index 000000000..0aa573338 --- /dev/null +++ b/blobs/z220/.gitignore @@ -0,0 +1,2 @@ +ifd.bin +me.bin From 713eadc1293b76d74b9f3037a163e28a6d1dab0b Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Wed, 27 Mar 2024 15:19:37 -0400 Subject: [PATCH 09/51] ci: Simple/mechanical tweaks to config file Got rid of long lines in favor of more lines for readability. Cleaned up some comments/typos and unnecessary cruft*. Finally ran prettier on the file for its automatic formatting, including whitespace clean ups. cruft: - && when already set -e - run commands with trailing \ - deleted commented out "OLD STUFF" - sorted listy looking things because unsorted stuff bothers me :) (I held back on sorting the board build definitions though, thats probably too much). Signed-off-by: Manuel Mendez Signed-off-by: Thierry Laurion --- .circleci/config.yml | 247 +++++++++++++++++++++++++++---------------- 1 file changed, 158 insertions(+), 89 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 303e11219..50c8d4e4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,11 +15,63 @@ commands: command: | ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime apt update - apt install -y build-essential zlib1g-dev uuid-dev libdigest-sha-perl libelf-dev bc bzip2 bison flex git gnupg gawk iasl m4 nasm patch python python2 python3 wget gnat cpio ccache pkg-config cmake libusb-1.0-0-dev autoconf texinfo ncurses-dev doxygen graphviz udev libudev1 libudev-dev automake libtool rsync innoextract sudo libssl-dev device-tree-compiler u-boot-tools sharutils e2fsprogs parted curl unzip imagemagick libncurses5-dev zip + apt install -y \ + autoconf \ + automake \ + bc \ + bison \ + build-essential \ + bzip2 \ + ccache \ + cmake \ + cpio \ + curl \ + device-tree-compiler \ + doxygen \ + e2fsprogs \ + flex \ + gawk \ + git \ + gnat \ + gnupg \ + graphviz \ + iasl \ + imagemagick \ + innoextract \ + libdigest-sha-perl \ + libelf-dev \ + libncurses5-dev \ + libssl-dev \ + libtool \ + libudev-dev \ + libudev1 \ + libusb-1.0-0-dev \ + m4 \ + nasm \ + ncurses-dev \ + parted \ + patch \ + pkg-config \ + python \ + python2 \ + python3 \ + rsync \ + sharutils \ + sudo \ + texinfo \ + u-boot-tools \ + udev \ + unzip \ + uuid-dev \ + wget \ + zip \ + zlib1g-dev \ + ; - run: name: Make Board (FULL ORDERED BUILD LOGS HERE UNTIL JOB FAILED) command: | - rm -rf build/<>/<>/* build/<>/log/* && make V=1 BOARD=<> <> || touch ./tmpDir/failed_build + rm -rf build/<>/<>/* build/<>/log/* + make V=1 BOARD=<> <> || touch ./tmpDir/failed_build no_output_timeout: 3h - run: name: Output hashes @@ -32,11 +84,22 @@ commands: - run: name: Archiving build logs. command: | - tar zcvf build/<>/<>/logs.tar.gz $(find build/ -name "*.log") + tar zcvf build/<>/<>/logs.tar.gz $(find build/ -name "*.log") - run: name: Output build failing logs command: | - if [[ -f ./tmpDir/failed_build ]]; then find ./build/<>/ -name "*.log" -type f -mmin -1|while read log; do echo ""; echo '==>' "$log" '<=='; echo ""; cat $log;done; exit 1;else echo "Step hasn't failed. Continuing with next step..."; fi \ + if [[ -f ./tmpDir/failed_build ]]; then + find "./build/<>/" -name "*.log" -type f -mmin -1 | + while read log; do + echo "" + echo '==>' "$log" '<==' + echo "" + cat "$log" + done + exit 1 + else + echo "Step hasn't failed. Continuing with next step..." + fi - store_artifacts: path: build/<>/<> @@ -52,65 +115,100 @@ jobs: command: | ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime apt update - apt install -y build-essential zlib1g-dev uuid-dev libdigest-sha-perl libelf-dev bc bzip2 bison flex git gnupg gawk iasl m4 nasm patch python python2 python3 wget gnat cpio ccache pkg-config cmake libusb-1.0-0-dev autoconf texinfo ncurses-dev doxygen graphviz udev libudev1 libudev-dev automake libtool rsync innoextract sudo imagemagick libncurses5-dev + apt install -y \ + autoconf \ + automake \ + bc \ + bison \ + build-essential \ + bzip2 \ + ccache \ + cmake \ + cpio \ + doxygen \ + flex \ + gawk \ + git \ + gnat \ + gnupg \ + graphviz \ + iasl \ + imagemagick \ + innoextract \ + libdigest-sha-perl \ + libelf-dev \ + libncurses5-dev \ + libtool \ + libudev-dev \ + libudev1 \ + libusb-1.0-0-dev \ + m4 \ + nasm \ + ncurses-dev \ + patch \ + pkg-config \ + python \ + python2 \ + python3 \ + rsync \ + sudo \ + texinfo \ + udev \ + uuid-dev \ + wget \ + zlib1g-dev \ + ; - checkout - run: name: git reset command: | - git reset --hard "$CIRCLE_SHA1" \ - + git reset --hard "$CIRCLE_SHA1" - run: name: Make tmp dir command: | - mkdir ./tmpDir \ - + mkdir ./tmpDir - run: name: Creating all modules and patches digest (All modules cache digest) command: | - find ./Makefile ./patches/ ./modules/ -type f | sort -h |xargs sha256sum > ./tmpDir/all_modules_and_patches.sha256sums \ - + find ./Makefile ./patches/ ./modules/ -type f | sort -h |xargs sha256sum > ./tmpDir/all_modules_and_patches.sha256sums - run: name: Creating coreboot (and associated patches) and musl-cross-make modules digest (musl-cross-make and coreboot cache digest) command: | - find ./Makefile ./modules/coreboot ./modules/musl-cross* ./patches/coreboot* -type f | sort -h | xargs sha256sum > ./tmpDir/coreboot_musl-cross.sha256sums \ - + find ./Makefile ./modules/coreboot ./modules/musl-cross* ./patches/coreboot* -type f | sort -h | xargs sha256sum > ./tmpDir/coreboot_musl-cross.sha256sums - run: name: Creating musl-cross-make and musl-cross-make patches digest (musl-cross-make cache digest) command: | - find ./Makefile modules/musl-cross* -type f | sort -h | xargs sha256sum > ./tmpDir/musl-cross.sha256sums \ - + find ./Makefile modules/musl-cross* -type f | sort -h | xargs sha256sum > ./tmpDir/musl-cross.sha256sums - restore_cache: + # First matched/found key wins and following keys are not tried keys: - #Restore existing cache for matching modules digest, validated to be exactly the same as in github current commit. - #This cache was made on top of below caches, if previously existing. If no module definition changed, we reuse this one. Otherwise... + # Cache for matching modules digest, validated to be exactly the same as in github current commit. + # This cache was made on top of below caches, if previously existing. + # If no module definition changed, we reuse this one - heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }} - #If precedent cache not found, restore cache for coreboot module (and patches) and musl-cross-make digests (coreboot: triannual release) - #Otehrwise.... + + # Cache for coreboot module (and patches) and musl-cross-make digests (coreboot: triannual release) - heads-coreboot-musl-cross-{{ checksum "./tmpDir/coreboot_musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} - #If precedent cache not found. Restore cache for musl-cross-make module digest (rarely modified). - #Otherwise, we build cleanly. + + # Cache for musl-cross-make module digest (rarely modified). - heads-musl-cross-{{ checksum "./tmpDir/musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} - run: name: Download and neuter xx20 ME (keep generated GBE and extracted IFD in tree) command: | ./blobs/xx20/download_parse_me.sh - - run: - name: Download and neuter xx30 ME (keep generated GBE and extracted IFD in tree) # me_cleaner.py present under heads xx30 blobs dir comes from https://github.com/corna/me_cleaner/blob/43612a630c79f3bc6f2653bfe90dfe0b7b137e08/me_cleaner.py + name: Download and neuter xx30 ME (keep generated GBE and extracted IFD in tree) command: | ./blobs/xx30/download_clean_me_manually.sh -m $(readlink -f ./blobs/xx30/me_cleaner.py) - - run: name: Download and extract t530 vbios roms for dgpu boards command: | ./blobs/xx30/vbios_t530.sh - - run: name: Download and extract w530 vbios roms for dgpu boards command: | ./blobs/xx30/vbios_w530.sh - - persist_to_workspace: root: ~/ paths: @@ -162,7 +260,7 @@ jobs: at: ~/ - build_board: arch: <> - target: <> + target: <> subcommand: <> save_cache: @@ -174,40 +272,40 @@ jobs: - attach_workspace: at: ~/ - save_cache: - #Generate cache for the same musl-cross module definition if hash is not previously existing - #CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names + # Generate cache for the same musl-cross module definition if hash is not previously existing + # CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names key: heads-musl-cross-{{ checksum "./tmpDir/musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} paths: - - crossgcc - - build/x86/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1 - build/ppc64/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1 + - build/x86/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1 + - crossgcc - packages - save_cache: - #Generate cache for the same coreboot mnd musl-cross-make modules definition if hash is not previously existing - #CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names + # Generate cache for the same coreboot mnd musl-cross-make modules definition if hash is not previously existing + # CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names key: heads-coreboot-musl-cross-{{ checksum "./tmpDir/coreboot_musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} paths: - - crossgcc - - build/x86/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1 + - build/ppc64/coreboot-talos_2 - build/ppc64/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1 - - packages - build/x86/coreboot-4.11 - build/x86/coreboot-4.13 - build/x86/coreboot-4.14 - build/x86/coreboot-4.15 - build/x86/coreboot-4.17 - build/x86/coreboot-4.22.01 - - build/x86/coreboot-purism - build/x86/coreboot-nitrokey - - build/ppc64/coreboot-talos_2 + - build/x86/coreboot-purism + - build/x86/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1 + - crossgcc + - packages - save_cache: #Generate cache for the exact same modules definitions if hash is not previously existing key: heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }} paths: - - crossgcc - build - - packages + - crossgcc - install + - packages workflows: version: 2 @@ -215,9 +313,8 @@ workflows: jobs: - prep_env -# Below, sequentially build one board for each coreboot -# version. The last board in the sequence is the dependency -# for the parallel boards built at the end, and also save_cache. + # Below, sequentially build one board for each coreboot version. + # The last board in the sequence is the dependency for the parallel boards built at the end, and also save_cache. # coreboot 4.22.01 - build_and_persist: @@ -227,7 +324,7 @@ workflows: requires: - prep_env - # coreboot-git librems + # coreboot purism - build_and_persist: name: librem_14 target: librem_14 @@ -235,18 +332,19 @@ workflows: requires: - x230-hotp-maximized - # coreboot-git Nitropads depending on x230-hotp-maximized cache - # since kernel is 6.x and coreboot is git is unshared + # coreboot nitropad + # Nitropads depending on x230-hotp-maximized cache since kernel is 6.x and coreboot is git is unshared # We use nitropad's coreboot's fork crossgcc - # No need to wait further for other board's cache. + # No need to wait further for other board's cache + # We reuse built modules from x230-hotp-maximized cache only - build_and_persist: name: nitropad-nv41 target: nitropad-nv41 subcommand: "" requires: - - prep_env + - x230-hotp-maximized - # coreboot-git Talos II (PPC) + # coreboot talos_2 - build_and_persist: name: talos-2 arch: ppc64 @@ -255,18 +353,18 @@ workflows: requires: - prep_env - #Cache one workspace per architecture. Make sure workspace caches are chainloaded and the last in chain for an arch is saved. + # Cache one workspace per architecture + # Make sure workspace caches are chainloaded and the last in chain for an arch is saved - save_cache: requires: - talos-2 - librem_14 -# -# Those onboarding new boards should add their entries below. -# -# + # + # Those onboarding new boards should add their entries below. + # -#Coreboot 4.22.01 boards + # coreboot 4.22.01 boards - build: name: x220-hotp-maximized target: x220-hotp-maximized @@ -377,7 +475,7 @@ workflows: target: t440p-maximized subcommand: "" requires: - - x230-hotp-maximized + - x230-hotp-maximized - build: name: t440p-hotp-maximized @@ -421,7 +519,8 @@ workflows: requires: - x230-hotp-maximized -#coreboot-git librem boards + # coreboot purism + # librem boards - build: name: librem_13v2 target: librem_13v2 @@ -464,7 +563,8 @@ workflows: requires: - librem_14 -#coreboot-git dasharo clevo_release + staging IASL patch + # coreboot purism + # dasharo clevo_release + staging IASL patch - build: name: nitropad-ns50 target: nitropad-ns50 @@ -499,34 +599,3 @@ workflows: # subcommand: "" # requires: # - librem_14 - -######################## -######################## -### OLD STUFF ### -######################## -######################## -# linuxboot steps need something to pass in the kernel header path -# skipping for now -# - run: -# name: UNMAINTAINED_qemu-linuxboot-edk2 -# command: | -# ./build/make-4.2.1/make \ -# CROSS=/cross/bin/x86_64-linux-musl- \ -# BOARD=UNMAINTAINED_qemu-linuxboot \ -# `/bin/pwd`/build/linuxboot-git/build/qemu/.configured \ -# # Run first to avoid too many processes -# -# - run: -# name: UNMAINTAINED_qemu-linuxboot -# command: | -# ./build/make-4.2.1/make \ -# CROSS=/cross/bin/x86_64-linux-musl- \ -# CPUS=16 \ -# V=1 \ -# BOARD=UNMAINTAINED_qemu-linuxboot \ -# -# - store-artifacts: -# path: build/UNMAINTAINED_qemu-linuxboot/linuxboot.rom -# - store-artifacts: -# path: build/UNMAINTAINED_qemu-linuxboot/hashes.txt - From de3f4ec2a329a1506f6ce491eb433a68b7084145 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Wed, 27 Mar 2024 15:29:09 -0400 Subject: [PATCH 10/51] ci: Replace while loop with tail of multiple files Gives the exact same output: ``` docker run --rm -ti debian:11 bash -c ' mkdir -p build/subdir1/ build/subdir2 echo "subdir1 error" >build/subdir1/fail.log echo "subdir2 error" >build/subdir2/fail.log find build -type f -name "*.log" -exec tail -n +1 "{}" + ' ==> build/subdir1/fail.log <== subdir1 error ==> build/subdir2/fail.log <== subdir2 error ``` Signed-off-by: Manuel Mendez Signed-off-by: Thierry Laurion --- .circleci/config.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 50c8d4e4e..88925a3da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,13 +89,7 @@ commands: name: Output build failing logs command: | if [[ -f ./tmpDir/failed_build ]]; then - find "./build/<>/" -name "*.log" -type f -mmin -1 | - while read log; do - echo "" - echo '==>' "$log" '<==' - echo "" - cat "$log" - done + find "./build/<>/" -name "*.log" -type f -mmin -1 -exec tail -n +1 '{}' + exit 1 else echo "Step hasn't failed. Continuing with next step..." @@ -571,7 +565,6 @@ workflows: subcommand: "" requires: - nitropad-nv41 - # - build: # name: UNMAINTAINED_kgpe-d16_workstation-usb_keyboard # target: UNMAINTAINED_kgpe-d16_workstation-usb_keyboard From 2824f48f9e99a94d230e2cd76cc784d8ba6a0d28 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 20 Oct 2022 22:32:02 -0400 Subject: [PATCH 11/51] Add nix files Signed-off-by: Manuel Mendez Signed-off-by: Thierry Laurion --- .envrc | 1 + .gitignore | 2 + flake.lock | 61 ++++++++++++++++++++++++++ flake.nix | 126 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 190 insertions(+) create mode 100644 .envrc create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..b85b26d14 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +has nix && use flake diff --git a/.gitignore b/.gitignore index e69113fbb..720f911aa 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,9 @@ *.xz *~ .*.sw* +/.direnv clean config/*.old crossgcc typescript* +result diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..491e53f71 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1711703276, + "narHash": "sha256-iMUFArF0WCatKK6RzfUJknjem0H9m4KgorO/p3Dopkk=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "d8fe5e6c92d0d190646fb9f1056741a229980089", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..8796f9290 --- /dev/null +++ b/flake.nix @@ -0,0 +1,126 @@ +{ + description = "heads flake, mostly for devshell for now"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { + self, + flake-utils, + nixpkgs, + ... + }: + flake-utils.lib.eachDefaultSystem (system: let + pkgs = nixpkgs.legacyPackages.${system}; + lib = pkgs.lib; + deps = with pkgs; + [ + autoconf + automake + bashInteractive + coreutils + bc + bison # Generate flashmap descriptor parser + bzip2 + cacert + ccache + cmake + cpio + curl + diffutils + dtc + e2fsprogs + elfutils + findutils + flex + gawk + git + gnat + gnugrep + gnumake + gnused + gnutar + gzip + imagemagick + innoextract + libtool + m4 + ncurses5 # make menuconfig and slang + parted + patch + perl + pkg-config + python3 + rsync + sharutils + texinfo + unzip + wget + which + xz + zip + zlib + zlib.dev + ] + ++ [ + # blobs/xx30/vbios_[tw]530.sh + bundler + p7zip + ruby + sudo # ( °-° ) + upx + ] + ++ [ + # debugging/fixing/testing + qemu + vim + ]; + in { + devShell = pkgs.mkShellNoCC { + buildInputs = deps; + }; + packages.myDevShell = + pkgs.runCommand "my-dev-shell" {} + #bash + '' + grep \ + -e CMAKE_PREFIX_PATH \ + -e NIX_CC_WRAPPER_TARGET_TARGET \ + -e NIX_CFLAGS_COMPILE_FOR_TARGET \ + -e NIX_LDFLAGS_FOR_TARGET \ + -e PKG_CONFIG_PATH_FOR_TARGET \ + ${self.devShell.${system}} >$out + ''; + packages.dockerImage = pkgs.dockerTools.buildLayeredImage { + name = "linuxboot/heads"; + tag = "dev-env"; + config.Entrypoint = ["bash" "-c" ''source /devenv.sh; if (( $# == 0 )); then exec bash; else exec "$0" "$@"; fi'']; + contents = + deps + ++ [ + pkgs.dockerTools.binSh + pkgs.dockerTools.caCertificates + pkgs.dockerTools.usrBinEnv + ]; + enableFakechroot = true; + fakeRootCommands = + #bash + '' + set -e + + grep \ + -e NIX_CC_WRAPPER_TARGET_TARGET \ + -e NIX_CFLAGS_COMPILE_FOR_TARGET \ + -e NIX_LDFLAGS_FOR_TARGET \ + -e NIX_PKG_CONFIG_WRAPPER_TARGET \ + -e PKG_CONFIG_PATH_FOR_TARGET \ + ${self.devShell.${system}} >/devenv.sh + + printf '[safe]\n\tdirectory = *\n' >/.gitconfig + mkdir /tmp; + ''; + }; + }); +} From 7169fab81bfd43b4235dd5cff30128a70ae9261d Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Fri, 6 Jan 2023 12:04:50 -0500 Subject: [PATCH 12/51] ci: Switch image from debian to nix Signed-off-by: Manuel Mendez Signed-off-by: Thierry Laurion --- .circleci/config.yml | 133 +++++++------------------------------------ 1 file changed, 19 insertions(+), 114 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 88925a3da..69356c113 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,68 +10,11 @@ commands: subcommand: type: string steps: - - run: - name: Install dependencies - command: | - ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime - apt update - apt install -y \ - autoconf \ - automake \ - bc \ - bison \ - build-essential \ - bzip2 \ - ccache \ - cmake \ - cpio \ - curl \ - device-tree-compiler \ - doxygen \ - e2fsprogs \ - flex \ - gawk \ - git \ - gnat \ - gnupg \ - graphviz \ - iasl \ - imagemagick \ - innoextract \ - libdigest-sha-perl \ - libelf-dev \ - libncurses5-dev \ - libssl-dev \ - libtool \ - libudev-dev \ - libudev1 \ - libusb-1.0-0-dev \ - m4 \ - nasm \ - ncurses-dev \ - parted \ - patch \ - pkg-config \ - python \ - python2 \ - python3 \ - rsync \ - sharutils \ - sudo \ - texinfo \ - u-boot-tools \ - udev \ - unzip \ - uuid-dev \ - wget \ - zip \ - zlib1g-dev \ - ; - run: name: Make Board (FULL ORDERED BUILD LOGS HERE UNTIL JOB FAILED) command: | rm -rf build/<>/<>/* build/<>/log/* - make V=1 BOARD=<> <> || touch ./tmpDir/failed_build + nix --print-build-logs --verbose develop --ignore-environment --command make V=1 BOARD=<> <> || touch ./tmpDir/failed_build no_output_timeout: 3h - run: name: Output hashes @@ -100,63 +43,21 @@ commands: jobs: prep_env: docker: - - image: debian:11 + - image: nixos/nix:2.16.1 resource_class: large working_directory: ~/heads steps: - - run: - name: Install dependencies - command: | - ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime - apt update - apt install -y \ - autoconf \ - automake \ - bc \ - bison \ - build-essential \ - bzip2 \ - ccache \ - cmake \ - cpio \ - doxygen \ - flex \ - gawk \ - git \ - gnat \ - gnupg \ - graphviz \ - iasl \ - imagemagick \ - innoextract \ - libdigest-sha-perl \ - libelf-dev \ - libncurses5-dev \ - libtool \ - libudev-dev \ - libudev1 \ - libusb-1.0-0-dev \ - m4 \ - nasm \ - ncurses-dev \ - patch \ - pkg-config \ - python \ - python2 \ - python3 \ - rsync \ - sudo \ - texinfo \ - udev \ - uuid-dev \ - wget \ - zlib1g-dev \ - ; - checkout - run: name: git reset command: | git reset --hard "$CIRCLE_SHA1" + - run: + name: Fetch nix dependencies + command: | + mkdir -p ~/.config/nix + echo 'experimental-features = nix-command flakes' >~/.config/nix/nix.conf + nix --print-build-logs --verbose develop --ignore-environment --command true - run: name: Make tmp dir command: | @@ -189,20 +90,24 @@ jobs: - run: name: Download and neuter xx20 ME (keep generated GBE and extracted IFD in tree) command: | - ./blobs/xx20/download_parse_me.sh + nix --print-build-logs --verbose develop --ignore-environment --command ./blobs/xx20/download_parse_me.sh - run: # me_cleaner.py present under heads xx30 blobs dir comes from https://github.com/corna/me_cleaner/blob/43612a630c79f3bc6f2653bfe90dfe0b7b137e08/me_cleaner.py name: Download and neuter xx30 ME (keep generated GBE and extracted IFD in tree) command: | - ./blobs/xx30/download_clean_me_manually.sh -m $(readlink -f ./blobs/xx30/me_cleaner.py) + nix --print-build-logs --verbose develop --ignore-environment --command ./blobs/xx30/download_clean_me_manually.sh -m $(readlink -f ./blobs/xx30/me_cleaner.py) - run: name: Download and extract t530 vbios roms for dgpu boards command: | - ./blobs/xx30/vbios_t530.sh + echo skipping for now + exit 0 + nix --print-build-logs --verbose develop --ignore-environment --command ./blobs/xx30/vbios_t530.sh - run: name: Download and extract w530 vbios roms for dgpu boards command: | - ./blobs/xx30/vbios_w530.sh + echo skipping for now + exit 0 + nix --print-build-logs --verbose develop --ignore-environment --command ./blobs/xx30/vbios_w530.sh - persist_to_workspace: root: ~/ paths: @@ -210,7 +115,7 @@ jobs: build_and_persist: docker: - - image: debian:11 + - image: nixos/nix:2.16.1 resource_class: large working_directory: ~/heads parameters: @@ -238,7 +143,7 @@ jobs: build: docker: - - image: debian:11 + - image: nixos/nix:2.16.1 resource_class: large working_directory: ~/heads parameters: @@ -259,7 +164,7 @@ jobs: save_cache: docker: - - image: debian:11 + - image: nixos/nix:2.16.1 resource_class: large working_directory: ~/heads steps: From 1174282bc46688c495d6aa8fb57d12b1121594ab Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Sun, 2 Jul 2023 12:23:49 -0400 Subject: [PATCH 13/51] ci: Prepend nix- to save and restore cache statements Until nix PR is merged to not interfere with master/other pr caches Signed-off-by: Manuel Mendez Signed-off-by: Thierry Laurion --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 69356c113..8044a330e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,13 +80,13 @@ jobs: # Cache for matching modules digest, validated to be exactly the same as in github current commit. # This cache was made on top of below caches, if previously existing. # If no module definition changed, we reuse this one - - heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }} + - nix-heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }} # Cache for coreboot module (and patches) and musl-cross-make digests (coreboot: triannual release) - - heads-coreboot-musl-cross-{{ checksum "./tmpDir/coreboot_musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} + - nix-heads-coreboot-musl-cross-{{ checksum "./tmpDir/coreboot_musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} # Cache for musl-cross-make module digest (rarely modified). - - heads-musl-cross-{{ checksum "./tmpDir/musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} + - nix-heads-musl-cross-{{ checksum "./tmpDir/musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} - run: name: Download and neuter xx20 ME (keep generated GBE and extracted IFD in tree) command: | @@ -173,7 +173,7 @@ jobs: - save_cache: # Generate cache for the same musl-cross module definition if hash is not previously existing # CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names - key: heads-musl-cross-{{ checksum "./tmpDir/musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} + key: nix-heads-musl-cross-{{ checksum "./tmpDir/musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} paths: - build/ppc64/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1 - build/x86/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1 @@ -182,7 +182,7 @@ jobs: - save_cache: # Generate cache for the same coreboot mnd musl-cross-make modules definition if hash is not previously existing # CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names - key: heads-coreboot-musl-cross-{{ checksum "./tmpDir/coreboot_musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} + key: nix-heads-coreboot-musl-cross-{{ checksum "./tmpDir/coreboot_musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} paths: - build/ppc64/coreboot-talos_2 - build/ppc64/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1 @@ -199,7 +199,7 @@ jobs: - packages - save_cache: #Generate cache for the exact same modules definitions if hash is not previously existing - key: heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }} + key: nix-heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }} paths: - build - crossgcc From e5c55d79e36d18db97ac0486581951cc916d027b Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 23 Apr 2024 10:41:36 -0400 Subject: [PATCH 14/51] CircleCI: have nitropad-nv41 build on top of prep_env, not x230-hotp-maximized Signed-off-by: Thierry Laurion --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8044a330e..88eb5736e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -241,7 +241,7 @@ workflows: target: nitropad-nv41 subcommand: "" requires: - - x230-hotp-maximized + - prep_env # coreboot talos_2 - build_and_persist: From 75a5c2f1e65277d8b6333243036e8b59d5b2a6ec Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 23 Apr 2024 10:46:24 -0400 Subject: [PATCH 15/51] tpm2 modules: remove sysroot and unneeded duplicated Makefile tweaks now passed from golbal Makefile sysroot (TODO: generalize) Signed-off-by: Thierry Laurion --- modules/tpm2-tools | 3 --- modules/tpm2-tss | 1 - 2 files changed, 4 deletions(-) diff --git a/modules/tpm2-tools b/modules/tpm2-tools index 9d4f1247c..bba25bc5d 100644 --- a/modules/tpm2-tools +++ b/modules/tpm2-tools @@ -32,9 +32,6 @@ tpm2-tools_configure := \ --prefix "/" \ --disable-fapi \ CFLAGS="-fdebug-prefix-map=$(INSTALL)=." \ - --with-sysroot=$(INSTALL) \ - TSS2_ESYS_3_0_CFLAGS="-I$(INSTALL)/include" \ - TSS2_ESYS_3_0_LIBS="-ltss2-esys -L$(INSTALL)/lib" \ tpm2-tools_target := $(MAKE_JOBS) \ DESTDIR="$(INSTALL)" \ diff --git a/modules/tpm2-tss b/modules/tpm2-tss index 91e1a26cd..c8c1c84e8 100644 --- a/modules/tpm2-tss +++ b/modules/tpm2-tss @@ -21,7 +21,6 @@ tpm2-tss_configure := aclocal && automake --add-missing && autoreconf -fi \ $(CROSS_TOOLS) \ --host $(MUSL_ARCH)-elf-linux \ --prefix "/" \ - --with-sysroot=$(INSTALL) \ --disable-doxygen-doc \ --disable-doxygen-html \ --disable-doxygen-man \ From 9ab5260871d2b9c6b4fd58053f3b59ff9dc7ef54 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 23 Apr 2024 11:58:33 -0400 Subject: [PATCH 16/51] Makefile: add yet another developer helper: real.gitclean_keep_packages to not delete already downloaded packages to economize bandwidth Signed-off-by: Thierry Laurion --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 84f5af112..340070674 100644 --- a/Makefile +++ b/Makefile @@ -796,3 +796,5 @@ real.clean: cd install && rm -rf -- * real.gitclean: git clean -fxd +real.gitclean_keep_packages: + git clean -fxd -e "packages" From cbf984ad7c7e18fb542dcc8b74388c1cf30aadc0 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 23 Apr 2024 13:37:17 -0400 Subject: [PATCH 17/51] WiP modules/tpm2* : removed Makefile build instructions too quick /nix/store/5lr5n3qa4day8l1ivbwlcby2nknczqkq-bash-5.2p26/bin/bash ./libtool --tag=CC --mode=link /home/user/heads/crossgcc/x86/bin/x86_64-linux-musl-gcc -fdebug-prefix-map=/home/user/heads=heads -gno-record-gcc-switches -D__MUSL__ --sysroot /home/user/heads/install/x86 -isystem /home/user/heads/install/x86/include -L/home/user/heads/install/x86/lib -I./tools -I./lib -Wall -Wextra -Wformat -Wformat-security -Wstack-protector -fstack-protector-all -Wstrict-overflow=5 -O2 -fPIC -fPIE -D_GNU_SOURCE -std=gnu99 -Wstringop-overflow=4 -Wstringop-truncation -Wduplicated-branches -Wduplicated-cond -Wbool-compare -fdata-sections -ffunction-sections -I/home/user/heads/install/x86/include -I/home/user/heads/install/x86//include -I/home/user/heads/install/x86//include/tss2 -I/home/user/heads/install/x86/nix/store/yg75achq89wgqn2fi3gglgsd77kjpi03-openssl-3.0.13-dev/include -I/home/user/heads/install/x86//include -I/home/user/heads/install/x86//include/tss2 -I/home/user/heads/install/x86//include -I/home/user/heads/install/x86//include/tss2 -I/home/user/heads/install/x86//include -I/home/user/heads/install/x86//include/tss2 -DTPM2_TOOLS_MAX="101" -fdebug-prefix-map=/home/user/heads/install/x86=. -shared -pie -Wl,-z,relro -Wl,-z,now -Wl,--gc-sections -o tools/tpm2 tools/tpm2-tpm2_tool.o tools/misc/tpm2-tpm2_certifyX509certutil.o tools/misc/tpm2-tpm2_checkquote.o tools/misc/tpm2-tpm2_encodeobject.o tools/misc/tpm2-tpm2_eventlog.o tools/misc/tpm2-tpm2_print.o tools/misc/tpm2-tpm2_rc_decode.o tools/misc/tpm2-tpm2_tr_encode.o tools/tpm2-tpm2_activatecredential.o tools/tpm2-tpm2_certify.o tools/tpm2-tpm2_changeauth.o tools/tpm2-tpm2_changeeps.o tools/tpm2-tpm2_changepps.o tools/tpm2-tpm2_clear.o tools/tpm2-tpm2_clearcontrol.o tools/tpm2-tpm2_clockrateadjust.o tools/tpm2-tpm2_create.o tools/tpm2-tpm2_createak.o tools/tpm2-tpm2_createek.o tools/tpm2-tpm2_createpolicy.o tools/tpm2-tpm2_setprimarypolicy.o tools/tpm2-tpm2_createprimary.o tools/tpm2-tpm2_dictionarylockout.o tools/tpm2-tpm2_duplicate.o tools/tpm2-tpm2_getcap.o tools/tpm2-tpm2_gettestresult.o tools/tpm2-tpm2_encryptdecrypt.o tools/tpm2-tpm2_evictcontrol.o tools/tpm2-tpm2_flushcontext.o tools/tpm2-tpm2_getrandom.o tools/tpm2-tpm2_gettime.o tools/tpm2-tpm2_hash.o tools/tpm2-tpm2_hierarchycontrol.o tools/tpm2-tpm2_hmac.o tools/tpm2-tpm2_import.o tools/tpm2-tpm2_incrementalselftest.o tools/tpm2-tpm2_load.o tools/tpm2-tpm2_loadexternal.o tools/tpm2-tpm2_makecredential.o tools/tpm2-tpm2_nvdefine.o tools/tpm2-tpm2_nvextend.o tools/tpm2-tpm2_nvincrement.o tools/tpm2-tpm2_nvreadpublic.o tools/tpm2-tpm2_nvread.o tools/tpm2-tpm2_nvreadlock.o tools/tpm2-tpm2_nvundefine.o tools/tpm2-tpm2_nvwrite.o tools/tpm2-tpm2_nvwritelock.o tools/tpm2-tpm2_nvsetbits.o tools/tpm2-tpm2_pcrallocate.o tools/tpm2-tpm2_pcrevent.o tools/tpm2-tpm2_pcrextend.o tools/tpm2-tpm2_pcrread.o tools/tpm2-tpm2_pcrreset.o tools/tpm2-tpm2_policypcr.o tools/tpm2-tpm2_policyauthorize.o tools/tpm2-tpm2_policyauthorizenv.o tools/tpm2-tpm2_policynv.o tools/tpm2-tpm2_policycountertimer.o tools/tpm2-tpm2_policyor.o tools/tpm2-tpm2_policynamehash.o tools/tpm2-tpm2_policytemplate.o tools/tpm2-tpm2_policycphash.o tools/tpm2-tpm2_policypassword.o tools/tpm2-tpm2_policysigned.o tools/tpm2-tpm2_policyticket.o tools/tpm2-tpm2_policyauthvalue.o tools/tpm2-tpm2_policysecret.o tools/tpm2-tpm2_policyrestart.o tools/tpm2-tpm2_policycommandcode.o tools/tpm2-tpm2_policynvwritten.o tools/tpm2-tpm2_policyduplicationselect.o tools/tpm2-tpm2_policylocality.o tools/tpm2-tpm2_quote.o tools/tpm2-tpm2_readclock.o tools/tpm2-tpm2_readpublic.o tools/tpm2-tpm2_rsadecrypt.o tools/tpm2-tpm2_rsaencrypt.o tools/tpm2-tpm2_send.o tools/tpm2-tpm2_selftest.o tools/tpm2-tpm2_setclock.o tools/tpm2-tpm2_shutdown.o tools/tpm2-tpm2_sign.o tools/tpm2-tpm2_certifycreation.o tools/tpm2-tpm2_nvcertify.o tools/tpm2-tpm2_startauthsession.o tools/tpm2-tpm2_startup.o tools/tpm2-tpm2_stirrandom.o tools/tpm2-tpm2_testparms.o tools/tpm2-tpm2_unseal.o tools/tpm2-tpm2_verifysignature.o tools/tpm2-tpm2_setcommandauditstatus.o tools/tpm2-tpm2_getcommandauditdigest.o tools/tpm2-tpm2_getsessionauditdigest.o tools/tpm2-tpm2_geteccparameters.o tools/tpm2-tpm2_ecephemeral.o tools/tpm2-tpm2_commit.o tools/tpm2-tpm2_ecdhkeygen.o tools/tpm2-tpm2_ecdhzgen.o tools/tpm2-tpm2_zgen2phase.o tools/tpm2-tpm2_sessionconfig.o tools/tpm2-tpm2_getpolicydigest.o lib/libcommon.a -ltss2-esys -L/home/user/heads/install/x86/lib -L/home/user/heads/install/x86//lib -ltss2-mu -L/home/user/heads/install/x86/nix/store/7nmrrad8skxr47f9hfl3xc0pfqmwq51b-openssl-3.0.13/lib -lcrypto -L/home/user/heads/install/x86//lib -ltss2-tctildr -L/home/user/heads/install/x86//lib -ltss2-rc -L/home/user/heads/install/x86//lib -ltss2-sys libtool: error: cannot find the library '//lib/libtss2-sys.la' or unhandled argument '//lib/libtss2-sys.la' make[1]: *** [Makefile:2478: tools/tpm2] Error 1 make[1]: Leaving directory '/home/user/heads/build/x86/tpm2-tools-5.6' make: *** [Makefile:521: /home/user/heads/build/x86/tpm2-tools-5.6/.build] Error 1 Signed-off-by: Thierry Laurion --- modules/tpm2-tools | 3 +++ modules/tpm2-tss | 1 + 2 files changed, 4 insertions(+) diff --git a/modules/tpm2-tools b/modules/tpm2-tools index bba25bc5d..dbc86ebf9 100644 --- a/modules/tpm2-tools +++ b/modules/tpm2-tools @@ -31,6 +31,9 @@ tpm2-tools_configure := \ --host $(MUSL_ARCH)-elf-linux \ --prefix "/" \ --disable-fapi \ + --with-sysroot=$(INSTALL) \ + TSS2_ESYS_3_0_CFLAGS="-I$(INSTALL)/include" \ + TSS2_ESYS_3_0_LIBS="-ltss2-esys -L$(INSTALL)/lib" \ CFLAGS="-fdebug-prefix-map=$(INSTALL)=." \ tpm2-tools_target := $(MAKE_JOBS) \ diff --git a/modules/tpm2-tss b/modules/tpm2-tss index c8c1c84e8..91e1a26cd 100644 --- a/modules/tpm2-tss +++ b/modules/tpm2-tss @@ -21,6 +21,7 @@ tpm2-tss_configure := aclocal && automake --add-missing && autoreconf -fi \ $(CROSS_TOOLS) \ --host $(MUSL_ARCH)-elf-linux \ --prefix "/" \ + --with-sysroot=$(INSTALL) \ --disable-doxygen-doc \ --disable-doxygen-html \ --disable-doxygen-man \ From 05223ca6a37075553f9f52ffcdff39a01287fc01 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 23 Apr 2024 14:31:54 -0400 Subject: [PATCH 18/51] CircleCI + Makefile: remove limitation to loadavg of 16 in Makefile, test CPUS=8 to maximize loadavg on CircleCI with 4 CPUs & 8GB ram See first lines of output of any make command. Change aimed to be respectful of CI resource (8GB ram 4CPUs) With CPUS=8 AVAILABLE_MEM_GB=4, CircleCI outputs: !!!!!! BUILD SYSTEM INFO !!!!!! System CPUS: 36 System Available Memory: 4 GB System Load Average: 12.99 ---------------------------------------------------------------------- Used **CPUS**: 8 Used **LOADAVG**: 8 Used **AVAILABLE_MEM_GB**: 4 GB ---------------------------------------------------------------------- **MAKE_JOBS**: -j8 --max-load 8 Variables available for override (use 'make VAR_NAME=value'): **CPUS** (default: number of processors, e.g., 'make CPUS=4') **LOADAVG** (default: same as CPUS, e.g., 'make LOADAVG=4') **AVAILABLE_MEM_GB** (default: memory available on the system in GB, e.g., 'make AVAILABLE_MEM_GB=4') **MEM_PER_JOB_GB** (default: 1GB per job, e.g., 'make MEM_PER_JOB_GB=2') ---------------------------------------------------------------------- Let's try without any limitation... Signed-off-by: Thierry Laurion --- .circleci/config.yml | 2 +- Makefile | 44 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 88eb5736e..da6280b90 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,7 @@ commands: type: string steps: - run: - name: Make Board (FULL ORDERED BUILD LOGS HERE UNTIL JOB FAILED) + name: Make Board (console logs not contiguous because multiple CPUS used to build under CircleCI, see logs in artifacts or Output build failing logs below in case of failed step) command: | rm -rf build/<>/<>/* build/<>/log/* nix --print-build-logs --verbose develop --ignore-environment --command make V=1 BOARD=<> <> || touch ./tmpDir/failed_build diff --git a/Makefile b/Makefile index 340070674..ac3d6b42f 100644 --- a/Makefile +++ b/Makefile @@ -25,11 +25,49 @@ INSTALL = $(pwd)/install/$(CONFIG_TARGET_ARCH) log_dir = $(build)/log board_build = $(build)/$(BOARD) + +# Estimated memory required per job in GB (e.g., 1GB for gcc) +MEM_PER_JOB_GB ?= 1 + # Controls how many parallel jobs are invoked in subshells -CPUS ?= $(shell nproc) -MAKE_JOBS ?= -j$(CPUS) --max-load 16 +CPUS ?= $(shell getconf _NPROCESSORS_ONLN) +AVAILABLE_MEM_GB ?= $(shell awk '/MemAvailable/ {print int($$2 / 1024 / 1024)}' /proc/meminfo) + +# Calculate the maximum number of jobs based on available memory +MAX_JOBS_MEM := $(shell echo $$(( $(AVAILABLE_MEM_GB) / $(MEM_PER_JOB_GB) ))) + +# Use the minimum of the system's CPUs and the calculated max jobs based on memory +CPUS := $(shell echo $$(($(CPUS) < $(MAX_JOBS_MEM) ? $(CPUS) : $(MAX_JOBS_MEM)))) + +# Load average can be adjusted to be higher than CPUS to allow for some CPU overcommit +# Multiply by 3 and then divide by 2 to achieve the effect of multiplying by 1.5 using integer arithmetic +LOADAVG ?= $(shell echo $$(( ($(CPUS) * 3) / 2 ))) + +# Construct MAKE_JOBS with dynamic CPU count and load average +MAKE_JOBS := -j$(CPUS) --load-average=$(LOADAVG) # Add other flags as needed + +# Print out the settings and compare system values with actual ones used +$(info ----------------------------------------------------------------------) +$(info !!!!!! BUILD SYSTEM INFO !!!!!!) +$(info System CPUS: $(shell getconf _NPROCESSORS_ONLN)) +$(info System Available Memory: $(AVAILABLE_MEM_GB) GB) +$(info System Load Average: $(shell uptime | awk '{print $$10}')) +$(info ----------------------------------------------------------------------) +$(info Used **CPUS**: $(CPUS)) +$(info Used **LOADAVG**: $(LOADAVG)) +$(info Used **AVAILABLE_MEM_GB**: $(AVAILABLE_MEM_GB) GB) +$(info ----------------------------------------------------------------------) +$(info **MAKE_JOBS**: $(MAKE_JOBS)) +$(info ) +$(info Variables available for override (use 'make VAR_NAME=value'):) +$(info **CPUS** (default: number of processors, e.g., 'make CPUS=4')) +$(info **LOADAVG** (default: 1.5 times CPUS, e.g., 'make LOADAVG=54')) +$(info **AVAILABLE_MEM_GB** (default: memory available on the system in GB, e.g., 'make AVAILABLE_MEM_GB=4')) +$(info **MEM_PER_JOB_GB** (default: 1GB per job, e.g., 'make MEM_PER_JOB_GB=2')) +$(info ----------------------------------------------------------------------) +$(info !!!!!! Build starts !!!!!!) + -WGET ?= wget # Timestamps should be in ISO format DATE=`date --rfc-3339=seconds` From e841f9bc0d0587ed84b4dc7463febeba9e0d7abb Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 23 Apr 2024 17:56:24 -0400 Subject: [PATCH 19/51] modules/* : Make sure MAKE_JOBS is passed down Signed-off-by: Thierry Laurion --- modules/flashtools | 1 + modules/json-c | 2 +- modules/tpmtotp | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/flashtools b/modules/flashtools index edbf136d9..94386ba7c 100644 --- a/modules/flashtools +++ b/modules/flashtools @@ -10,6 +10,7 @@ flashtools_hash := a68cdb4a2e312f96862119a6d829ac900b53d0cbc80caa5632efd43b5b7ee flashtools_target := \ $(CROSS_TOOLS) \ + $(MAKE_JOBS) \ CFLAGS="-Os -I$(INSTALL)/include" \ LDFLAGS="-L$(INSTALL)/lib" \ diff --git a/modules/json-c b/modules/json-c index 158e320fd..cf310c3cf 100644 --- a/modules/json-c +++ b/modules/json-c @@ -31,7 +31,7 @@ json-c_configure := \ cmake .. -DCMAKE_INSTALL_PREFIX="$(INSTALL)" -DCMAKE_TOOLCHAIN_FILE=../toolchain -DCMAKE_BUILD_TYPE=minsizerel json-c_target := \ - $(CROSS_TOOLS) -C $(build)/$(json-c_dir)/build \ + $(CROSS_TOOLS) $(MAKE_JOBS) -C $(build)/$(json-c_dir)/build \ all install json-c_libraries := build/libjson-c.so.5 diff --git a/modules/tpmtotp b/modules/tpmtotp index 57b3c54ae..eb52f03c6 100644 --- a/modules/tpmtotp +++ b/modules/tpmtotp @@ -10,6 +10,7 @@ tpmtotp_hash := eaac1e8f652f1da7f5a1ed6a8cfefb6511f1e5e1dabf93b44db3b29c18c5ae53 tpmtotp_target := \ $(CROSS_TOOLS) \ + $(MAKE_JOBS) \ CFLAGS="-I$(INSTALL)/include -Os" \ LDFLAGS="-L$(INSTALL)/lib" \ From a29f92a26d8e3ca9b1b1e392948fc8603821219f Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 23 Apr 2024 19:14:31 -0400 Subject: [PATCH 20/51] modules/* : WiP for tpm2-* while having added MAKE_JOBS to modules that were missing it to propogate build optimizations per module, while still impossible to call make -j 12 on main make call Signed-off-by: Thierry Laurion --- modules/tpm2-tools | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/tpm2-tools b/modules/tpm2-tools index dbc86ebf9..0051a28a1 100644 --- a/modules/tpm2-tools +++ b/modules/tpm2-tools @@ -32,9 +32,6 @@ tpm2-tools_configure := \ --prefix "/" \ --disable-fapi \ --with-sysroot=$(INSTALL) \ - TSS2_ESYS_3_0_CFLAGS="-I$(INSTALL)/include" \ - TSS2_ESYS_3_0_LIBS="-ltss2-esys -L$(INSTALL)/lib" \ - CFLAGS="-fdebug-prefix-map=$(INSTALL)=." \ tpm2-tools_target := $(MAKE_JOBS) \ DESTDIR="$(INSTALL)" \ From 975a2424cddbaa18ba2cab676b36c143ec2c2955 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 23 Apr 2024 19:34:32 -0400 Subject: [PATCH 21/51] Makefile: add yet another helper real.remove_canary_files-extract_patch_rebuild_what_changed: reextract packages, repatch sources (might fail, easy to fix) and rebuild only what changed (not a lot) if patch fails to apply, its because patch file creates a file and doesn't expect it to exist. just call rm on the file reported to exist, and relaunch build. Deletes ./install/*/* and permits to rebuild all dependencies in order, just based on freshly extracted and patched code. Bonus, this saves your SDD from unneeded wear and rebuilds faster then all other Mafile helpers. That's my favorite. Signed-off-by: Thierry Laurion --- Makefile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Makefile b/Makefile index ac3d6b42f..68a601e6c 100644 --- a/Makefile +++ b/Makefile @@ -833,6 +833,20 @@ real.clean: done cd install && rm -rf -- * real.gitclean: + #Use git ignore file as a base to wipe everything not in tree. Keeps coreboot forks downloaded since detected as git repos, wipes the rest. git clean -fxd real.gitclean_keep_packages: + #Same as above but keep the packages downloaded to save bandwidth git clean -fxd -e "packages" +real.remove_canary_files-extract_patch_rebuild_what_changed: + #Another approach is to remove the "canary" files + # This forces Heads to restart building a board config by checking packages integrity, extracting them, redoing patching on files and rebuilding what needs to be rebuilt + # reinstalling what is needed under ./install as well which is what we normally want on a development cycle. + #Limitations: if for whatever reason, a patch creates a file in an extracted package dir, this approach will fail without further manual actions + # This is not so bad though: git patch apply tells you exactly which file couldn't be created as expected. Just delete those files and relaunch the build and it will succeed. + #This approach economizes a lot of time since most of the build artifacts do not need to be rebuilt since the dates of the files should be the same as when you originally built them. + # So only a minimal time is needed to rebuild, and this is also good for your SSD. + #**** USE THIS APPROACH FIRST *** + find ./build/ -type f -name ".canary" | xargs rm || echo "All .carnary files already deleted" + find ./install/*/* | xargs rm -rf || echo "All install/ARCH/* dirs and files already deleted" + echo "you can now call make BOARD=desired_board, and if any patch fails to apply because file exists; just rm that build/path_to_file and continue testing!" From c197d6193aae4162ab3003066f70a26dbf5e87aa Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 23 Apr 2024 20:25:21 -0400 Subject: [PATCH 22/51] NIX_REPRO_NOTES: Add notes to help repro and go farther faster without loosing traces of reproduction notes Signed-off-by: Thierry Laurion --- NIX_REPRO_NOTES | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 NIX_REPRO_NOTES diff --git a/NIX_REPRO_NOTES b/NIX_REPRO_NOTES new file mode 100644 index 000000000..50aa3fe27 --- /dev/null +++ b/NIX_REPRO_NOTES @@ -0,0 +1,60 @@ +Notes to repro on top of QubesOS debian-12-xfce template based qube + +- Clone a debian-12-xfce on top of Q4.2.1 to be specialized +- Deploy QubesOS nix requirements in template: https://dataswamp.org/~solene/2023-05-15-qubes-os-install-nix.html +- Deploy QubesOS requirements for docker: https://gist.github.com/tlaurion/9113983bbdead492735c8438cd14d6cd +- Create a qube based on the cloned template above +- Open qube +- Clone whatever repo is most recent between https://github.com/mmlb/osresearch-heads/tree/wip-nix-for-build or https://github.com/tlaurion/heads/tree/wip-nix-for-build + - git clone Above_repo + - cd /home/user/heads or cd /home/user/osresearch-heads + - git checkout wip-nix-for-build + - git reset --hard + + + +Now, replication traces + +Install nix cleanly: +sudo rm -rf /nix/* || echo "cannot delete /nix" && sh <(curl -L https://nixos.org/nix/install) --no-daemon + +Source nix prior of creating nix cache and jumping in it +. /home/user/.nix-profile/etc/profile.d/nix.sh + +Configure nix for local builds until we have proper docker image instead: +mkdir -p ~/.config/nix +echo 'experimental-features = nix-command flakes' >~/.config/nix/nix.conf + +Build nix developer local env with flakes locks to specified versions and jump into it: +nix --print-build-logs --verbose develop --ignore-environment + +Build a board: +make BOARD=nitropad-nv41 + +In case you need to rebuild from packages+ patches + a clean ./install (where libraries and headers are found): +make real.remove_canary_files-extract_patch_rebuild_what_changed + +IF that doesn;t help you, look at the end of the global Helper for more helper. Iterate. + + + +build docker out of nix develop environement and jump into it: +nix build .#dockerImage && docker load < result && docker run --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env + +From there, rebuild with +make real.remove_canary_files-extract_patch_rebuild_chat_changed +make BOARD=nitropad-nv41 + +Make changes until taht part works in both nix layer on top of OS and withing docker image. +...... + +SUCCESSSSS?!?!?!?! CELEBRATION! +Tell @mmlb and @insurgo on matrix channel!!!! + + +------- + +Some raw and incomplete notes on how to push that docker to dockerhub +docker tag linuxboot/heads:dev-env tlaurion/heads-dev-env:v0.0.3 +docker push tlaurion/heads-dev-env:v0.0.3 +(to be continued) From 093fc4a73ee5b001de2ce5a8309d08f795d5bb6f Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 24 Apr 2024 14:49:13 -0400 Subject: [PATCH 23/51] Makefile: reduce dependencies on external tools for dynamic MAKE_JOBS creation. Cleaner NIX_REPRO_NOTES Signed-off-by: Thierry Laurion --- Makefile | 9 ++++----- NIX_REPRO_NOTES | 16 +++++++--------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 68a601e6c..9c38f7d90 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,8 @@ board_build = $(build)/$(BOARD) MEM_PER_JOB_GB ?= 1 # Controls how many parallel jobs are invoked in subshells -CPUS ?= $(shell getconf _NPROCESSORS_ONLN) -AVAILABLE_MEM_GB ?= $(shell awk '/MemAvailable/ {print int($$2 / 1024 / 1024)}' /proc/meminfo) +CPUS ?= $(shell nproc) +AVAILABLE_MEM_GB ?= $(shell cat /proc/meminfo | grep MemAvailable | awk '{print int($$2 / 1024)}') # Calculate the maximum number of jobs based on available memory MAX_JOBS_MEM := $(shell echo $$(( $(AVAILABLE_MEM_GB) / $(MEM_PER_JOB_GB) ))) @@ -44,12 +44,12 @@ CPUS := $(shell echo $$(($(CPUS) < $(MAX_JOBS_MEM) ? $(CPUS) : $(MAX_ LOADAVG ?= $(shell echo $$(( ($(CPUS) * 3) / 2 ))) # Construct MAKE_JOBS with dynamic CPU count and load average -MAKE_JOBS := -j$(CPUS) --load-average=$(LOADAVG) # Add other flags as needed +MAKE_JOBS := -j$(CPUS) --load-average=$(LOADAVG) # Add other flags as needed to be more adaptive to CIs # Print out the settings and compare system values with actual ones used $(info ----------------------------------------------------------------------) $(info !!!!!! BUILD SYSTEM INFO !!!!!!) -$(info System CPUS: $(shell getconf _NPROCESSORS_ONLN)) +$(info System CPUS: $(shell nproc)) $(info System Available Memory: $(AVAILABLE_MEM_GB) GB) $(info System Load Average: $(shell uptime | awk '{print $$10}')) $(info ----------------------------------------------------------------------) @@ -68,7 +68,6 @@ $(info ----------------------------------------------------------------------) $(info !!!!!! Build starts !!!!!!) - # Timestamps should be in ISO format DATE=`date --rfc-3339=seconds` diff --git a/NIX_REPRO_NOTES b/NIX_REPRO_NOTES index 50aa3fe27..bf9af50bb 100644 --- a/NIX_REPRO_NOTES +++ b/NIX_REPRO_NOTES @@ -15,22 +15,20 @@ Notes to repro on top of QubesOS debian-12-xfce template based qube Now, replication traces -Install nix cleanly: +#Install nix cleanly: sudo rm -rf /nix/* || echo "cannot delete /nix" && sh <(curl -L https://nixos.org/nix/install) --no-daemon - -Source nix prior of creating nix cache and jumping in it +#Source nix prior of creating nix cache and jumping in it . /home/user/.nix-profile/etc/profile.d/nix.sh - -Configure nix for local builds until we have proper docker image instead: +#Configure nix for local builds until we have proper docker image instead: mkdir -p ~/.config/nix echo 'experimental-features = nix-command flakes' >~/.config/nix/nix.conf - -Build nix developer local env with flakes locks to specified versions and jump into it: +#Build nix developer local env with flakes locks to specified versions and jump into it: nix --print-build-logs --verbose develop --ignore-environment - -Build a board: +#Build a board make BOARD=nitropad-nv41 + + In case you need to rebuild from packages+ patches + a clean ./install (where libraries and headers are found): make real.remove_canary_files-extract_patch_rebuild_what_changed From b45fc960cf47c879e51fb5f8c3193f5d55fafc35 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 30 Apr 2024 15:20:38 -0400 Subject: [PATCH 24/51] CircleCI: Test tlaurion/heads-dev-env:v.0.0.5 (created from flake develop) which fails at tpm2-tss - switch cache to nix-docker-heads to not interfere with nixos develop layer on same PR - remove nix develop calls; replace by direct script calls and make calls - make sure save/restore/root is ~/heads Signed-off-by: Thierry Laurion Signed-off-by: Thierry Laurion --- .circleci/config.yml | 54 ++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index da6280b90..f4602c743 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ commands: name: Make Board (console logs not contiguous because multiple CPUS used to build under CircleCI, see logs in artifacts or Output build failing logs below in case of failed step) command: | rm -rf build/<>/<>/* build/<>/log/* - nix --print-build-logs --verbose develop --ignore-environment --command make V=1 BOARD=<> <> || touch ./tmpDir/failed_build + make V=1 BOARD=<> <> || touch ./tmpDir/failed_build no_output_timeout: 3h - run: name: Output hashes @@ -43,7 +43,7 @@ commands: jobs: prep_env: docker: - - image: nixos/nix:2.16.1 + - image: tlaurion/heads-dev-env:v0.0.5 resource_class: large working_directory: ~/heads steps: @@ -52,12 +52,6 @@ jobs: name: git reset command: | git reset --hard "$CIRCLE_SHA1" - - run: - name: Fetch nix dependencies - command: | - mkdir -p ~/.config/nix - echo 'experimental-features = nix-command flakes' >~/.config/nix/nix.conf - nix --print-build-logs --verbose develop --ignore-environment --command true - run: name: Make tmp dir command: | @@ -80,42 +74,42 @@ jobs: # Cache for matching modules digest, validated to be exactly the same as in github current commit. # This cache was made on top of below caches, if previously existing. # If no module definition changed, we reuse this one - - nix-heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }} + - nix-docker-heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }} # Cache for coreboot module (and patches) and musl-cross-make digests (coreboot: triannual release) - - nix-heads-coreboot-musl-cross-{{ checksum "./tmpDir/coreboot_musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} + - nix-docker-heads-coreboot-musl-cross-{{ checksum "./tmpDir/coreboot_musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} # Cache for musl-cross-make module digest (rarely modified). - - nix-heads-musl-cross-{{ checksum "./tmpDir/musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} + - nix-docker-heads-musl-cross-{{ checksum "./tmpDir/musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} - run: name: Download and neuter xx20 ME (keep generated GBE and extracted IFD in tree) command: | - nix --print-build-logs --verbose develop --ignore-environment --command ./blobs/xx20/download_parse_me.sh + ./blobs/xx20/download_parse_me.sh - run: # me_cleaner.py present under heads xx30 blobs dir comes from https://github.com/corna/me_cleaner/blob/43612a630c79f3bc6f2653bfe90dfe0b7b137e08/me_cleaner.py name: Download and neuter xx30 ME (keep generated GBE and extracted IFD in tree) command: | - nix --print-build-logs --verbose develop --ignore-environment --command ./blobs/xx30/download_clean_me_manually.sh -m $(readlink -f ./blobs/xx30/me_cleaner.py) + ./blobs/xx30/download_clean_me_manually.sh -m $(readlink -f ./blobs/xx30/me_cleaner.py) - run: name: Download and extract t530 vbios roms for dgpu boards command: | echo skipping for now exit 0 - nix --print-build-logs --verbose develop --ignore-environment --command ./blobs/xx30/vbios_t530.sh + ./blobs/xx30/vbios_t530.sh - run: name: Download and extract w530 vbios roms for dgpu boards command: | echo skipping for now exit 0 - nix --print-build-logs --verbose develop --ignore-environment --command ./blobs/xx30/vbios_w530.sh + ./blobs/xx30/vbios_w530.sh - persist_to_workspace: - root: ~/ + root: ~/heads paths: - . build_and_persist: docker: - - image: nixos/nix:2.16.1 + - image: tlaurion/heads-dev-env:v0.0.5 resource_class: large working_directory: ~/heads parameters: @@ -128,22 +122,22 @@ jobs: type: string steps: - attach_workspace: - at: ~/ + at: ~/heads - build_board: arch: <> target: <> subcommand: <> - persist_to_workspace: - root: ~/ + root: ~/heads paths: - - heads/packages/<> - - heads/build/<> - - heads/crossgcc/<> - - heads/install/<> + - packages/<> + - build/<> + - crossgcc/<> + - install/<> build: docker: - - image: nixos/nix:2.16.1 + - image: tlaurion/heads-dev-env:v0.0.5 resource_class: large working_directory: ~/heads parameters: @@ -156,7 +150,7 @@ jobs: type: string steps: - attach_workspace: - at: ~/ + at: ~/heads - build_board: arch: <> target: <> @@ -164,16 +158,16 @@ jobs: save_cache: docker: - - image: nixos/nix:2.16.1 + - image: tlaurion/heads-dev-env:v0.0.5 resource_class: large working_directory: ~/heads steps: - attach_workspace: - at: ~/ + at: ~/heads - save_cache: # Generate cache for the same musl-cross module definition if hash is not previously existing # CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names - key: nix-heads-musl-cross-{{ checksum "./tmpDir/musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} + key: nix-docker-heads-musl-cross-{{ checksum "./tmpDir/musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} paths: - build/ppc64/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1 - build/x86/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1 @@ -182,7 +176,7 @@ jobs: - save_cache: # Generate cache for the same coreboot mnd musl-cross-make modules definition if hash is not previously existing # CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names - key: nix-heads-coreboot-musl-cross-{{ checksum "./tmpDir/coreboot_musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} + key: nix-docker-heads-coreboot-musl-cross-{{ checksum "./tmpDir/coreboot_musl-cross.sha256sums" }}{{ .Environment.CACHE_VERSION }} paths: - build/ppc64/coreboot-talos_2 - build/ppc64/musl-cross-38e52db8358c043ae82b346a2e6e66bc86a53bc1 @@ -199,7 +193,7 @@ jobs: - packages - save_cache: #Generate cache for the exact same modules definitions if hash is not previously existing - key: nix-heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }} + key: nix-docker-heads-modules-and-patches-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }} paths: - build - crossgcc From 32e7082f094c5eb01bba7fe58ab0fadc4245e7b2 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 1 May 2024 11:09:38 -0400 Subject: [PATCH 25/51] flake.nix: change zlib and zlib.dev for zlib-ng? https://app.circleci.com/pipelines/github/tlaurion/heads/2500/workflows/23674215-8b22-4852-adf4-2a6df9e44353/jobs/45080?invite=true#step-102-16530_106 zlib-dev not found on coreboot buildstack buildstep... Signed-off-by: Thierry Laurion --- flake.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 8796f9290..8168cca55 100644 --- a/flake.nix +++ b/flake.nix @@ -61,8 +61,7 @@ which xz zip - zlib - zlib.dev + zlib-ng ] ++ [ # blobs/xx30/vbios_[tw]530.sh From 53ca8d3554e493b5cb5c5271417cae889f6f865a Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 1 May 2024 11:22:56 -0400 Subject: [PATCH 26/51] CirlceCI: use docker v0.0.6 which flake.nix jumped from zlib/zlib.dev to zlib-ng Signed-off-by: Thierry Laurion --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f4602c743..38453d77a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,7 @@ commands: jobs: prep_env: docker: - - image: tlaurion/heads-dev-env:v0.0.5 + - image: tlaurion/heads-dev-env:v0.0.6 resource_class: large working_directory: ~/heads steps: @@ -109,7 +109,7 @@ jobs: build_and_persist: docker: - - image: tlaurion/heads-dev-env:v0.0.5 + - image: tlaurion/heads-dev-env:v0.0.6 resource_class: large working_directory: ~/heads parameters: @@ -137,7 +137,7 @@ jobs: build: docker: - - image: tlaurion/heads-dev-env:v0.0.5 + - image: tlaurion/heads-dev-env:v0.0.6 resource_class: large working_directory: ~/heads parameters: @@ -158,7 +158,7 @@ jobs: save_cache: docker: - - image: tlaurion/heads-dev-env:v0.0.5 + - image: tlaurion/heads-dev-env:v0.0.6 resource_class: large working_directory: ~/heads steps: From f676405ae16d725042444409bd827a80ba98e96a Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 1 May 2024 12:39:06 -0400 Subject: [PATCH 27/51] Revert "flake.nix: change zlib and zlib.dev for zlib-ng? https://app.circleci.com/pipelines/github/tlaurion/heads/2500/workflows/23674215-8b22-4852-adf4-2a6df9e44353/jobs/45080?invite=true#step-102-16530_106 zlib-dev not found on coreboot buildstack buildstep..." This reverts commit d6c499676ed07037bcb4ca66db3d3876a87de23b. Signed-off-by: Thierry Laurion --- flake.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 8168cca55..8796f9290 100644 --- a/flake.nix +++ b/flake.nix @@ -61,7 +61,8 @@ which xz zip - zlib-ng + zlib + zlib.dev ] ++ [ # blobs/xx30/vbios_[tw]530.sh From 70a9f93ddf2ef429a6724097cafe58faf109f466 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 1 May 2024 12:39:41 -0400 Subject: [PATCH 28/51] Revert "CirlceCI: use docker v0.0.6 which flake.nix jumped from zlib/zlib.dev to zlib-ng" This reverts commit 9052d2b562162183fa201ebf89c75be904d87281. Signed-off-by: Thierry Laurion --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 38453d77a..f4602c743 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,7 @@ commands: jobs: prep_env: docker: - - image: tlaurion/heads-dev-env:v0.0.6 + - image: tlaurion/heads-dev-env:v0.0.5 resource_class: large working_directory: ~/heads steps: @@ -109,7 +109,7 @@ jobs: build_and_persist: docker: - - image: tlaurion/heads-dev-env:v0.0.6 + - image: tlaurion/heads-dev-env:v0.0.5 resource_class: large working_directory: ~/heads parameters: @@ -137,7 +137,7 @@ jobs: build: docker: - - image: tlaurion/heads-dev-env:v0.0.6 + - image: tlaurion/heads-dev-env:v0.0.5 resource_class: large working_directory: ~/heads parameters: @@ -158,7 +158,7 @@ jobs: save_cache: docker: - - image: tlaurion/heads-dev-env:v0.0.6 + - image: tlaurion/heads-dev-env:v0.0.5 resource_class: large working_directory: ~/heads steps: From 76c20847da66ead24659bcf56b5aec98cdb23bfb Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 1 May 2024 12:10:14 -0400 Subject: [PATCH 29/51] CircleCI: add CircleCI step to source manually /devenv.sh in build_board additional step Signed-off-by: Thierry Laurion --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index f4602c743..7c5d28488 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,8 @@ commands: - run: name: Make Board (console logs not contiguous because multiple CPUS used to build under CircleCI, see logs in artifacts or Output build failing logs below in case of failed step) command: | + echo "Sourcing /devenv.sh since docker entrypoint doesn't do it as expected" + source /devenv.sh rm -rf build/<>/<>/* build/<>/log/* make V=1 BOARD=<> <> || touch ./tmpDir/failed_build no_output_timeout: 3h From fa60bf7dfb31c6440473d7c4a91dc701b12604bc Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 1 May 2024 13:23:10 -0400 Subject: [PATCH 30/51] modules/tpm2-tss: just remove LT_LIB_DLLOAD from aclocal generated file since there is no easy way of fixing this nix doesn't provide an equivalent of libltdl-dev, so just wipe the remnant of old ages if present https://github.com/tpm2-software/tpm2-tss/issues/2161 Signed-off-by: Thierry Laurion --- modules/tpm2-tss | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/tpm2-tss b/modules/tpm2-tss index 91e1a26cd..e3d004bb6 100644 --- a/modules/tpm2-tss +++ b/modules/tpm2-tss @@ -15,7 +15,9 @@ tpm2-tss_hash := ba9e52117f254f357ff502e7d60fce652b3bfb26327d236bbf5ab634235e40f #sed -i 's/hardcode_into_libs=yes/hardcode_into_libs=no/g' configure # needed otherwise library build/x86/tpm2-tss-3.2.2/src/tss2-tcti/.libs/libtss2-tcti-pcap.so.0.0.0: # contains: /home/user/heads/build/x86/tpm2-tss-3.2.2/src/tss2-tcti/.libs:/home/user/heads/build/x86/tpm2-tss-3.2.2/src/tss2-mu/.libs://lib -tpm2-tss_configure := aclocal && automake --add-missing && autoreconf -fi \ +tpm2-tss_configure := \ + sed -i '/LT_LIB_DLLOAD/d' configure.ac \ + && aclocal && automake --add-missing && autoreconf -fi \ && sed -i 's/hardcode_into_libs=yes/hardcode_into_libs=no/g' configure \ && ./configure \ $(CROSS_TOOLS) \ From 35530f9115e9bbe3afbe04890af3dd3598b832ac Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 1 May 2024 14:30:03 -0400 Subject: [PATCH 31/51] modules/msrtools : add missing MAKE_JOBS for parallel builds Signed-off-by: Thierry Laurion --- modules/msrtools | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/msrtools b/modules/msrtools index 8a9b6b5b2..25d2a3bc8 100644 --- a/modules/msrtools +++ b/modules/msrtools @@ -13,6 +13,7 @@ msrtools_url := https://github.com/osresearch/msr-tools/archive/$(msrtools_versi msrtools_hash := 80554790d0a404205fe215c9ae8d2de159e980ec23821d636f201f12550e6ac0 msrtools_target := \ + $(MAKE_JOBS) \ $(CROSS_TOOLS) \ CFLAGS="-I$(INSTALL)/include" \ LDFLAGS="-L$(INSTALL)/lib" \ From 89181181df52312230bad64f8a4f76acbd90ada1 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 1 May 2024 14:30:41 -0400 Subject: [PATCH 32/51] flake.nix: add openssl requirement to build talos-2 board's kernel Signed-off-by: Thierry Laurion --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 8796f9290..a53ef508d 100644 --- a/flake.nix +++ b/flake.nix @@ -48,6 +48,7 @@ libtool m4 ncurses5 # make menuconfig and slang + openssl #needed for talos-2 kernel build parted patch perl From 9a72d9545ae2578549de057e3fbcf98c6d63d679 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 1 May 2024 14:47:03 -0400 Subject: [PATCH 33/51] CircleCI: use tlaurion/heads-dev-env:v0.0.7 which includes openssl in flake.nix for talos-2 board's linux config Signed-off-by: Thierry Laurion --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c5d28488..a6673a6a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,7 +45,7 @@ commands: jobs: prep_env: docker: - - image: tlaurion/heads-dev-env:v0.0.5 + - image: tlaurion/heads-dev-env:v0.0.7 resource_class: large working_directory: ~/heads steps: @@ -111,7 +111,7 @@ jobs: build_and_persist: docker: - - image: tlaurion/heads-dev-env:v0.0.5 + - image: tlaurion/heads-dev-env:v0.0.7 resource_class: large working_directory: ~/heads parameters: @@ -139,7 +139,7 @@ jobs: build: docker: - - image: tlaurion/heads-dev-env:v0.0.5 + - image: tlaurion/heads-dev-env:v0.0.7 resource_class: large working_directory: ~/heads parameters: @@ -160,7 +160,7 @@ jobs: save_cache: docker: - - image: tlaurion/heads-dev-env:v0.0.5 + - image: tlaurion/heads-dev-env:v0.0.7 resource_class: large working_directory: ~/heads steps: From 50ab1c5b88982afef283df9d54a3d78051bedb81 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 1 May 2024 16:22:01 -0400 Subject: [PATCH 34/51] modules/patches slang: bump to version 2.3.3 so we can disable termcap without hacking around Signed-off-by: Thierry Laurion --- modules/slang | 5 +++-- .../unhardcode-ln-in-Makefile.patch | 0 2 files changed, 3 insertions(+), 2 deletions(-) rename patches/{slang-2.3.1a => slang-2.3.3}/unhardcode-ln-in-Makefile.patch (100%) diff --git a/modules/slang b/modules/slang index c99c64497..6a8f5d3c5 100644 --- a/modules/slang +++ b/modules/slang @@ -1,10 +1,10 @@ modules-$(CONFIG_SLANG) += slang -slang_version := 2.3.1a +slang_version := 2.3.3 slang_dir := slang-$(slang_version) slang_tar := slang-$(slang_version).tar.bz2 slang_url := https://www.jedsoft.org/releases/slang/$(slang_tar) -slang_hash := 54f0c3007fde918039c058965dffdfd6c5aec0bad0f4227192cc486021f08c36 +slang_hash := f9145054ae131973c61208ea82486d5dd10e3c5cdad23b7c4a0617743c8f5a18 slang_configure := ./configure \ $(CROSS_TOOLS) \ @@ -15,6 +15,7 @@ slang_configure := ./configure \ --with-pcre=no \ --with-png=no \ --with-z=no \ + --disable-termcap \ && mkdir -p src/elfobjs # Disable parallel make for the install target diff --git a/patches/slang-2.3.1a/unhardcode-ln-in-Makefile.patch b/patches/slang-2.3.3/unhardcode-ln-in-Makefile.patch similarity index 100% rename from patches/slang-2.3.1a/unhardcode-ln-in-Makefile.patch rename to patches/slang-2.3.3/unhardcode-ln-in-Makefile.patch From 5b2b4dc0fd0b81c0fc6996328643b176ae23e949 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Thu, 2 May 2024 09:54:14 -0400 Subject: [PATCH 35/51] flake.nix: add exporting of AC_LOCAL which was not exported in deveenv.sh. Was causing another ac macro misbehavior since host ac was not considered by aclocal and autoreconf TODO: Might want to revert 6a1791112de451509d81e03bce5bdd6b1a49a79f if talos-2 board is able to build 3rdparty/sb-sign-tool Signed-off-by: Thierry Laurion --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index a53ef508d..bd9a96f67 100644 --- a/flake.nix +++ b/flake.nix @@ -92,6 +92,7 @@ -e NIX_CFLAGS_COMPILE_FOR_TARGET \ -e NIX_LDFLAGS_FOR_TARGET \ -e PKG_CONFIG_PATH_FOR_TARGET \ + -e ACLOCAL_PATH \ ${self.devShell.${system}} >$out ''; packages.dockerImage = pkgs.dockerTools.buildLayeredImage { @@ -117,6 +118,7 @@ -e NIX_LDFLAGS_FOR_TARGET \ -e NIX_PKG_CONFIG_WRAPPER_TARGET \ -e PKG_CONFIG_PATH_FOR_TARGET \ + -e ACLOCAL_PATH \ ${self.devShell.${system}} >/devenv.sh printf '[safe]\n\tdirectory = *\n' >/.gitconfig From 19bccf0cbd12382903f8feaa98cad9d848bf515d Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Thu, 2 May 2024 09:56:55 -0400 Subject: [PATCH 36/51] Revert "modules/tpm2-tss: just remove LT_LIB_DLLOAD from aclocal generated file since there is no easy way of fixing this" This reverts commit 6a1791112de451509d81e03bce5bdd6b1a49a79f. Signed-off-by: Thierry Laurion --- modules/tpm2-tss | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/tpm2-tss b/modules/tpm2-tss index e3d004bb6..91e1a26cd 100644 --- a/modules/tpm2-tss +++ b/modules/tpm2-tss @@ -15,9 +15,7 @@ tpm2-tss_hash := ba9e52117f254f357ff502e7d60fce652b3bfb26327d236bbf5ab634235e40f #sed -i 's/hardcode_into_libs=yes/hardcode_into_libs=no/g' configure # needed otherwise library build/x86/tpm2-tss-3.2.2/src/tss2-tcti/.libs/libtss2-tcti-pcap.so.0.0.0: # contains: /home/user/heads/build/x86/tpm2-tss-3.2.2/src/tss2-tcti/.libs:/home/user/heads/build/x86/tpm2-tss-3.2.2/src/tss2-mu/.libs://lib -tpm2-tss_configure := \ - sed -i '/LT_LIB_DLLOAD/d' configure.ac \ - && aclocal && automake --add-missing && autoreconf -fi \ +tpm2-tss_configure := aclocal && automake --add-missing && autoreconf -fi \ && sed -i 's/hardcode_into_libs=yes/hardcode_into_libs=no/g' configure \ && ./configure \ $(CROSS_TOOLS) \ From 6070d8f6f0319ead8de3f3b676c137f6ade0236c Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Thu, 2 May 2024 11:09:11 -0400 Subject: [PATCH 37/51] CircleCI: use tlaurion/heads-dev-env:v0.0.8 which includes AC_LOCAL export of develop env into the docker image. Works locally for talos-2 board build. Signed-off-by: Thierry Laurion --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a6673a6a3..327574ca3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,7 +45,7 @@ commands: jobs: prep_env: docker: - - image: tlaurion/heads-dev-env:v0.0.7 + - image: tlaurion/heads-dev-env:v0.0.8 resource_class: large working_directory: ~/heads steps: @@ -111,7 +111,7 @@ jobs: build_and_persist: docker: - - image: tlaurion/heads-dev-env:v0.0.7 + - image: tlaurion/heads-dev-env:v0.0.8 resource_class: large working_directory: ~/heads parameters: @@ -139,7 +139,7 @@ jobs: build: docker: - - image: tlaurion/heads-dev-env:v0.0.7 + - image: tlaurion/heads-dev-env:v0.0.8 resource_class: large working_directory: ~/heads parameters: @@ -160,7 +160,7 @@ jobs: save_cache: docker: - - image: tlaurion/heads-dev-env:v0.0.7 + - image: tlaurion/heads-dev-env:v0.0.8 resource_class: large working_directory: ~/heads steps: From 46cad549ef5b1de40dd36c61b824735d4217f8a1 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Thu, 2 May 2024 14:57:05 -0400 Subject: [PATCH 38/51] WiP flake.nix: make docker image usable for testing as well, target: qemu-coreboot-whiptail-tpm2 with swtpm and canokey for smartcard - include nix tools inside of the docker to be able to call the garbage collector prior of creating docker. - protect roots from garbage collection (WiP) - Requires external preparation call so that nix (the binary) is not wiped as well. See NIX_REPRO_NOTES at the end of the file for repro notes - Could probably be improved. Works as of now and created a 4Gb vs 3.02Gb docker image I'm uploading now. - CircleCI bumped to use v0.0.9 version including this - CircleCI now depending on flake.lock for all cache layers. Will rebuild clean once again So now we have qemu with canokey support in image, nix basic tools inside of container. Possible to call docker with DISPLAY, see NIX_REPRO_NOTES as of now. That feels nice. No need of USB security dongle to have TPM based TPMTOTP nor detach sign? Not tested but feature is there TODO: - make docker creating nicer in the Nix way. - Add canokey support under targets/qemu.mk - add canokey board version At least we have reproducible stack and testing stack being in same docker image. Docker image moved from 991.18MB (v0.0.8) to 1.18GB (v0.0.9) - And I tried to clean binaries of symbols here! Seems like I do not know enough of the Nix way here. Signed-off-by: Thierry Laurion --- .circleci/config.yml | 14 +-- NIX_REPRO_NOTES | 19 ++++ flake.lock | 6 +- flake.nix | 228 ++++++++++++++++++++++++------------------- targets/qemu.mk | 2 +- 5 files changed, 160 insertions(+), 109 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 327574ca3..1e2af08b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,7 +45,7 @@ commands: jobs: prep_env: docker: - - image: tlaurion/heads-dev-env:v0.0.8 + - image: tlaurion/heads-dev-env:v0.0.9 resource_class: large working_directory: ~/heads steps: @@ -61,15 +61,15 @@ jobs: - run: name: Creating all modules and patches digest (All modules cache digest) command: | - find ./Makefile ./patches/ ./modules/ -type f | sort -h |xargs sha256sum > ./tmpDir/all_modules_and_patches.sha256sums + find ./Makefile ./flake.lock ./patches/ ./modules/ -type f | sort -h |xargs sha256sum > ./tmpDir/all_modules_and_patches.sha256sums - run: name: Creating coreboot (and associated patches) and musl-cross-make modules digest (musl-cross-make and coreboot cache digest) command: | - find ./Makefile ./modules/coreboot ./modules/musl-cross* ./patches/coreboot* -type f | sort -h | xargs sha256sum > ./tmpDir/coreboot_musl-cross.sha256sums + find ./Makefile ./flake.lock ./modules/coreboot ./modules/musl-cross* ./patches/coreboot* -type f | sort -h | xargs sha256sum > ./tmpDir/coreboot_musl-cross.sha256sums - run: name: Creating musl-cross-make and musl-cross-make patches digest (musl-cross-make cache digest) command: | - find ./Makefile modules/musl-cross* -type f | sort -h | xargs sha256sum > ./tmpDir/musl-cross.sha256sums + find ./Makefile ./flake.lock modules/musl-cross* -type f | sort -h | xargs sha256sum > ./tmpDir/musl-cross.sha256sums - restore_cache: # First matched/found key wins and following keys are not tried keys: @@ -111,7 +111,7 @@ jobs: build_and_persist: docker: - - image: tlaurion/heads-dev-env:v0.0.8 + - image: tlaurion/heads-dev-env:v0.0.9 resource_class: large working_directory: ~/heads parameters: @@ -139,7 +139,7 @@ jobs: build: docker: - - image: tlaurion/heads-dev-env:v0.0.8 + - image: tlaurion/heads-dev-env:v0.0.9 resource_class: large working_directory: ~/heads parameters: @@ -160,7 +160,7 @@ jobs: save_cache: docker: - - image: tlaurion/heads-dev-env:v0.0.8 + - image: tlaurion/heads-dev-env:v0.0.9 resource_class: large working_directory: ~/heads steps: diff --git a/NIX_REPRO_NOTES b/NIX_REPRO_NOTES index bf9af50bb..7b7b9b0c8 100644 --- a/NIX_REPRO_NOTES +++ b/NIX_REPRO_NOTES @@ -56,3 +56,22 @@ Some raw and incomplete notes on how to push that docker to dockerhub docker tag linuxboot/heads:dev-env tlaurion/heads-dev-env:v0.0.3 docker push tlaurion/heads-dev-env:v0.0.3 (to be continued) + +------------------- + +#To create the docker image, I was not able to set gcroot dynamically, so here are the manual steps so nix is not wiped per garbage collector +mkdir -p /nix/var/nix/gcroots/per-user/$(whoami) +ln -sfn $(readlink -f $HOME/.nix-profile) /nix/var/nix/gcroots/per-user/$(whoami)/profile + +#And then build derivates and cal garbarrge collection (not yet successfull since docker build fails now) +nix build .#dockerImage && docker load < result && docker run --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env + +#Use it +docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env -- make BOARD=qemu-coreboot-whiptail-tpm2 +docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env -- make BOARD=qemu-coreboot-whiptail-tpm2 run + +#push it +docker login +docker tag linuxboot/heads:dev-env tlaurion/heads-dev-env:v0.0.9 +docker push tlaurion/heads-dev-env:v0.0.9 + diff --git a/flake.lock b/flake.lock index 491e53f71..6fb25794b 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1711703276, - "narHash": "sha256-iMUFArF0WCatKK6RzfUJknjem0H9m4KgorO/p3Dopkk=", + "lastModified": 1714253743, + "narHash": "sha256-mdTQw2XlariysyScCv2tTE45QSU9v/ezLcHJ22f0Nxc=", "owner": "nixos", "repo": "nixpkgs", - "rev": "d8fe5e6c92d0d190646fb9f1056741a229980089", + "rev": "58a1abdbae3217ca6b702f03d3b35125d88a2994", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index bd9a96f67..e8e9e396b 100644 --- a/flake.nix +++ b/flake.nix @@ -1,87 +1,111 @@ { - description = "heads flake, mostly for devshell for now"; + description = "Optimized heads flake for Docker image with garbage collection protection"; + # Inputs define external dependencies and their sources. inputs = { - nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; - flake-utils.url = "github:numtide/flake-utils"; + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; # Using the unstable channel for the latest packages, while flake.lock fixates the commit reused until changed. + flake-utils.url = "github:numtide/flake-utils"; # Utilities for flake functionality. }; - outputs = { - self, - flake-utils, - nixpkgs, - ... - }: + # Outputs are the result of the flake, including the development environment and Docker image. + outputs = { self, nixpkgs, flake-utils, ... }: flake-utils.lib.eachDefaultSystem (system: let - pkgs = nixpkgs.legacyPackages.${system}; - lib = pkgs.lib; - deps = with pkgs; - [ - autoconf - automake - bashInteractive - coreutils - bc - bison # Generate flashmap descriptor parser - bzip2 - cacert - ccache - cmake - cpio - curl - diffutils - dtc - e2fsprogs - elfutils - findutils - flex - gawk - git - gnat - gnugrep - gnumake - gnused - gnutar - gzip - imagemagick - innoextract - libtool - m4 - ncurses5 # make menuconfig and slang - openssl #needed for talos-2 kernel build - parted - patch - perl - pkg-config - python3 - rsync - sharutils - texinfo - unzip - wget - which - xz - zip - zlib - zlib.dev - ] - ++ [ - # blobs/xx30/vbios_[tw]530.sh - bundler - p7zip - ruby - sudo # ( °-° ) - upx - ] - ++ [ - # debugging/fixing/testing - qemu - vim - ]; + pkgs = nixpkgs.legacyPackages.${system}; # Accessing the legacy package set. + lib = pkgs.lib; # The standard Nix packages library. + + # Dependencies are the packages required for the Heads project. + # Organized into subsets for clarity and maintainability. + deps = with pkgs; [ + # Core build utilities + autoconf + automake + bashInteractive + coreutils + bc + bison + bzip2 + cacert + ccache + cmake + cpio + curl + diffutils + dtc + e2fsprogs + elfutils + findutils + flex + gawk + git + gnat + gnugrep + gnumake + gnused + gnutar + gzip + imagemagick # For bootsplash manipulation. + innoextract # ROM extraction for dGPU. + libtool + m4 + ncurses5 + openssl + parted + patch + perl + pkg-config + python3 # me_cleaner, coreboot. + rsync # coreboot. + sharutils + texinfo + unzip + wget + which + xz + zip + zlib + zlib.dev + ] ++ [ + # Packages for qemu support with Canokey integration. + qemu # To test make BOARD=qemu-coreboot-* boards and then call make BOARD=qemu-coreboot-* with inject_gpg statement, and then run statement (RTFM). + canokey-qemu # Canokey lib for qemu build-time compilation. + (qemu.override { + canokeySupport = true; # This override enables Canokey support in QEMU, resulting in -device canokey being available. + }) + ] ++ [ + # Additional tools for editing and testing. + vim # Mostly used amongst us, sorry if you'd like something else, open issue. + swtpm # QEMU requirement to emulate tpm1/tpm2. + dosfstools # QEMU requirement to produce valid fs to store exported public key to be fused through inject_key on qemu (so qemu flashrom emulated SPI support). + ] ++ [ + # Tools for handling binary blobs and compression. + bundler + p7zip + ruby + sudo + upx + ]; + + # Stripping binaries to reduce size, while ensuring functionality is not affected. + stripBinaries = map (pkg: if pkg?isDerivation then pkg.overrideAttrs (oldAttrs: { + postInstall = oldAttrs.postInstall or "" + '' + strip $out/bin/* || true + ''; + }) else pkg) deps; + in { + # The development shell includes all the dependencies. devShell = pkgs.mkShellNoCC { - buildInputs = deps; + buildInputs = stripBinaries ++ [ pkgs.nix ]; # Include the Nix package to provide nix-collect-garbage. + shellHook = '' + # Create a garbage collection root for the Nix profile + mkdir -p /nix/var/nix/gcroots/per-user/$(whoami) + echo $(readlink -f $HOME/.nix-profile) > /nix/var/nix/gcroots/per-user/$(whoami)/profile + # Perform garbage collection to clean up any unnecessary files. + nix-collect-garbage -d + ''; }; + + # myDevShell outputs environment variables necessary for development. packages.myDevShell = pkgs.runCommand "my-dev-shell" {} #bash @@ -95,35 +119,43 @@ -e ACLOCAL_PATH \ ${self.devShell.${system}} >$out ''; + + # Docker image configuration for the Heads project. packages.dockerImage = pkgs.dockerTools.buildLayeredImage { - name = "linuxboot/heads"; - tag = "dev-env"; - config.Entrypoint = ["bash" "-c" ''source /devenv.sh; if (( $# == 0 )); then exec bash; else exec "$0" "$@"; fi'']; - contents = - deps - ++ [ - pkgs.dockerTools.binSh - pkgs.dockerTools.caCertificates - pkgs.dockerTools.usrBinEnv - ]; - enableFakechroot = true; + name = "linuxboot/heads"; # Image name. + tag = "dev-env"; # Image tag. + config.Entrypoint = ["bash" "-c" ''source /devenv.sh; if (( $# == 0 )); then exec bash; else exec "$@"; fi'']; # Entrypoint configuration. + + # Contents of the Docker image, including stripped binaries for size optimization. + contents = stripBinaries ++ [ + pkgs.dockerTools.binSh + pkgs.dockerTools.caCertificates + pkgs.dockerTools.usrBinEnv + ]; + + enableFakechroot = true; # Enable fakechroot for compatibility. + + # Fake root commands to set up the environment inside the Docker image. fakeRootCommands = #bash '' - set -e + set -e - grep \ - -e NIX_CC_WRAPPER_TARGET_TARGET \ - -e NIX_CFLAGS_COMPILE_FOR_TARGET \ - -e NIX_LDFLAGS_FOR_TARGET \ - -e NIX_PKG_CONFIG_WRAPPER_TARGET \ - -e PKG_CONFIG_PATH_FOR_TARGET \ - -e ACLOCAL_PATH \ - ${self.devShell.${system}} >/devenv.sh + # Environment setup for the development shell. + grep \ + -e NIX_CC_WRAPPER_TARGET_TARGET \ + -e NIX_CFLAGS_COMPILE_FOR_TARGET \ + -e NIX_LDFLAGS_FOR_TARGET \ + -e NIX_PKG_CONFIG_WRAPPER_TARGET \ + -e PKG_CONFIG_PATH_FOR_TARGET \ + -e ACLOCAL_PATH \ + ${self.devShell.${system}} >/devenv.sh - printf '[safe]\n\tdirectory = *\n' >/.gitconfig - mkdir /tmp; - ''; + # Git configuration for safe directory access. + printf '[safe]\n\tdirectory = *\n' >/.gitconfig + mkdir /tmp; # Temporary directory for various operations. + ''; }; }); } + diff --git a/targets/qemu.mk b/targets/qemu.mk index 59286a3e1..051361229 100644 --- a/targets/qemu.mk +++ b/targets/qemu.mk @@ -16,7 +16,7 @@ endif ifeq "$(CONFIG_TPM2_TSS)" "y" SWTPM_TPMVER := --tpm2 -SWTPM_PRESETUP := swtpm_setup --create-config-files skip-if-exist +SWTPM_PRESETUP := swtpm_setup --create-config-files root skip-if-exist else # TPM1 is the default SWTPM_TPMVER := From 973e905ef6d4cadeae0f71f9edab9f99a7abbf12 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Fri, 3 May 2024 10:06:16 -0400 Subject: [PATCH 39/51] flake.lock: revert to old package pinning Signed-off-by: Thierry Laurion --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 6fb25794b..491e53f71 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1714253743, - "narHash": "sha256-mdTQw2XlariysyScCv2tTE45QSU9v/ezLcHJ22f0Nxc=", + "lastModified": 1711703276, + "narHash": "sha256-iMUFArF0WCatKK6RzfUJknjem0H9m4KgorO/p3Dopkk=", "owner": "nixos", "repo": "nixpkgs", - "rev": "58a1abdbae3217ca6b702f03d3b35125d88a2994", + "rev": "d8fe5e6c92d0d190646fb9f1056741a229980089", "type": "github" }, "original": { From 3d0991f6c148c77d141d3bf747d1bde485e86d48 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Fri, 3 May 2024 10:13:09 -0400 Subject: [PATCH 40/51] flake.nix: revert to mmlb state and testing Signed-off-by: Thierry Laurion --- flake.nix | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/flake.nix b/flake.nix index e8e9e396b..8f110f67d 100644 --- a/flake.nix +++ b/flake.nix @@ -22,7 +22,7 @@ bashInteractive coreutils bc - bison + bison # Generate flashmap descriptor parser bzip2 cacert ccache @@ -47,8 +47,8 @@ innoextract # ROM extraction for dGPU. libtool m4 - ncurses5 - openssl + ncurses5 # make menuconfig and slang + openssl #needed for talos-2 kernel build parted patch perl @@ -67,21 +67,22 @@ ] ++ [ # Packages for qemu support with Canokey integration. qemu # To test make BOARD=qemu-coreboot-* boards and then call make BOARD=qemu-coreboot-* with inject_gpg statement, and then run statement (RTFM). - canokey-qemu # Canokey lib for qemu build-time compilation. - (qemu.override { - canokeySupport = true; # This override enables Canokey support in QEMU, resulting in -device canokey being available. - }) + #canokey doesn;t work still even if compiled in, so no reason to add 1Gb of stuff in the image + #canokey-qemu # Canokey lib for qemu build-time compilation. + #(qemu.override { + # canokeySupport = true; # This override enables Canokey support in QEMU, resulting in -device canokey being available. + #}) ] ++ [ - # Additional tools for editing and testing. + # Additional tools for debugging/editing/testing. vim # Mostly used amongst us, sorry if you'd like something else, open issue. swtpm # QEMU requirement to emulate tpm1/tpm2. dosfstools # QEMU requirement to produce valid fs to store exported public key to be fused through inject_key on qemu (so qemu flashrom emulated SPI support). ] ++ [ - # Tools for handling binary blobs and compression. + # Tools for handling binary blobs in their compressed state. (blobs/xx30/vbios_[tw]530.sh) bundler p7zip ruby - sudo + sudo # ( °-° ) upx ]; @@ -124,7 +125,7 @@ packages.dockerImage = pkgs.dockerTools.buildLayeredImage { name = "linuxboot/heads"; # Image name. tag = "dev-env"; # Image tag. - config.Entrypoint = ["bash" "-c" ''source /devenv.sh; if (( $# == 0 )); then exec bash; else exec "$@"; fi'']; # Entrypoint configuration. + config.Entrypoint = ["bash" "-c" ''source /devenv.sh; if (( $# == 0 )); then exec bash; else exec "$0" "$@"; fi'']; # Entrypoint configuration. # Contents of the Docker image, including stripped binaries for size optimization. contents = stripBinaries ++ [ @@ -151,9 +152,9 @@ -e ACLOCAL_PATH \ ${self.devShell.${system}} >/devenv.sh - # Git configuration for safe directory access. + # Git configuration for safe directory access. printf '[safe]\n\tdirectory = *\n' >/.gitconfig - mkdir /tmp; # Temporary directory for various operations. + mkdir /tmp; # Temporary directory for various operations. ''; }; }); From 0b7ce534a8f662d31fc4cd494a216e08551b5509 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Fri, 3 May 2024 14:13:39 -0400 Subject: [PATCH 41/51] WiP: revert garbage collector within nix environment. Doesn;t help and makes the docker image bigger TODO: push v0.1.2 with those changes pointing circleci to use it Signed-off-by: Thierry Laurion --- NIX_REPRO_NOTES | 13 +++++++------ flake.nix | 34 ++++++++-------------------------- 2 files changed, 15 insertions(+), 32 deletions(-) diff --git a/NIX_REPRO_NOTES b/NIX_REPRO_NOTES index 7b7b9b0c8..af127a8e8 100644 --- a/NIX_REPRO_NOTES +++ b/NIX_REPRO_NOTES @@ -59,11 +59,12 @@ docker push tlaurion/heads-dev-env:v0.0.3 ------------------- -#To create the docker image, I was not able to set gcroot dynamically, so here are the manual steps so nix is not wiped per garbage collector -mkdir -p /nix/var/nix/gcroots/per-user/$(whoami) -ln -sfn $(readlink -f $HOME/.nix-profile) /nix/var/nix/gcroots/per-user/$(whoami)/profile +#To create the docker image, using garbage collector didn't help and made the size of the image grow bigger?! not using it reverting to what worked +# produced image without garbage collector was 990 Mb image, with it (and stripping which is already done by nix builder...) its 1006Mb. Over my head +sh <(curl -L https://nixos.org/nix/install) --no-daemon + . /home/user/.nix-profile/etc/profile.d/nix.sh -#And then build derivates and cal garbarrge collection (not yet successfull since docker build fails now) +#And then build derivates nix build .#dockerImage && docker load < result && docker run --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env #Use it @@ -72,6 +73,6 @@ docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd #push it docker login -docker tag linuxboot/heads:dev-env tlaurion/heads-dev-env:v0.0.9 -docker push tlaurion/heads-dev-env:v0.0.9 +docker tag linuxboot/heads:dev-env tlaurion/heads-dev-env:v0.1.2 +docker push tlaurion/heads-dev-env:v0.1.2 diff --git a/flake.nix b/flake.nix index 8f110f67d..325193a03 100644 --- a/flake.nix +++ b/flake.nix @@ -85,25 +85,10 @@ sudo # ( °-° ) upx ]; - - # Stripping binaries to reduce size, while ensuring functionality is not affected. - stripBinaries = map (pkg: if pkg?isDerivation then pkg.overrideAttrs (oldAttrs: { - postInstall = oldAttrs.postInstall or "" + '' - strip $out/bin/* || true - ''; - }) else pkg) deps; - in { # The development shell includes all the dependencies. devShell = pkgs.mkShellNoCC { - buildInputs = stripBinaries ++ [ pkgs.nix ]; # Include the Nix package to provide nix-collect-garbage. - shellHook = '' - # Create a garbage collection root for the Nix profile - mkdir -p /nix/var/nix/gcroots/per-user/$(whoami) - echo $(readlink -f $HOME/.nix-profile) > /nix/var/nix/gcroots/per-user/$(whoami)/profile - # Perform garbage collection to clean up any unnecessary files. - nix-collect-garbage -d - ''; + buildInputs = deps; }; # myDevShell outputs environment variables necessary for development. @@ -123,20 +108,17 @@ # Docker image configuration for the Heads project. packages.dockerImage = pkgs.dockerTools.buildLayeredImage { - name = "linuxboot/heads"; # Image name. - tag = "dev-env"; # Image tag. - config.Entrypoint = ["bash" "-c" ''source /devenv.sh; if (( $# == 0 )); then exec bash; else exec "$0" "$@"; fi'']; # Entrypoint configuration. - - # Contents of the Docker image, including stripped binaries for size optimization. - contents = stripBinaries ++ [ + name = "linuxboot/heads"; + tag = "dev-env"; + config.Entrypoint = ["bash" "-c" ''source /devenv.sh; if (( $# == 0 )); then exec bash; else exec "$0" "$@"; fi'']; + contents = + deps + ++ [ pkgs.dockerTools.binSh pkgs.dockerTools.caCertificates pkgs.dockerTools.usrBinEnv ]; - - enableFakechroot = true; # Enable fakechroot for compatibility. - - # Fake root commands to set up the environment inside the Docker image. + enableFakechroot = true; fakeRootCommands = #bash '' From 2b2356e87e3df9569e74dd45f4bf79dd14f21287 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Fri, 3 May 2024 15:00:26 -0400 Subject: [PATCH 42/51] CircleCI: use tlaurion/heads-dev-env:v0.1.1 which reverts nix attempt of garbage collection inside of nix prior of making the docker Signed-off-by: Thierry Laurion --- .circleci/config.yml | 8 ++++---- NIX_REPRO_NOTES | 11 +++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e2af08b0..62751f4c3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,7 +45,7 @@ commands: jobs: prep_env: docker: - - image: tlaurion/heads-dev-env:v0.0.9 + - image: tlaurion/heads-dev-env:v0.1.1 resource_class: large working_directory: ~/heads steps: @@ -111,7 +111,7 @@ jobs: build_and_persist: docker: - - image: tlaurion/heads-dev-env:v0.0.9 + - image: tlaurion/heads-dev-env:v0.1.1 resource_class: large working_directory: ~/heads parameters: @@ -139,7 +139,7 @@ jobs: build: docker: - - image: tlaurion/heads-dev-env:v0.0.9 + - image: tlaurion/heads-dev-env:v0.1.1 resource_class: large working_directory: ~/heads parameters: @@ -160,7 +160,7 @@ jobs: save_cache: docker: - - image: tlaurion/heads-dev-env:v0.0.9 + - image: tlaurion/heads-dev-env:v0.1.1 resource_class: large working_directory: ~/heads steps: diff --git a/NIX_REPRO_NOTES b/NIX_REPRO_NOTES index af127a8e8..b96382183 100644 --- a/NIX_REPRO_NOTES +++ b/NIX_REPRO_NOTES @@ -61,6 +61,9 @@ docker push tlaurion/heads-dev-env:v0.0.3 #To create the docker image, using garbage collector didn't help and made the size of the image grow bigger?! not using it reverting to what worked # produced image without garbage collector was 990 Mb image, with it (and stripping which is already done by nix builder...) its 1006Mb. Over my head +# TODO: +# - revisit later, should not block merge of pr +# - do doc under heads's README.md and then heads-wiki sh <(curl -L https://nixos.org/nix/install) --no-daemon . /home/user/.nix-profile/etc/profile.d/nix.sh @@ -73,6 +76,10 @@ docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd #push it docker login -docker tag linuxboot/heads:dev-env tlaurion/heads-dev-env:v0.1.2 -docker push tlaurion/heads-dev-env:v0.1.2 +docker tag linuxboot/heads:dev-env tlaurion/heads-dev-env:v0.1.1 +docker push tlaurion/heads-dev-env:v0.1.1 + +# For end-users, CircleCI +docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) laurion/heads-dev-env:v0.1.1 -- make BOARD=qemu-coreboot-whiptail-tpm2 +docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) laurion/heads-dev-env:v0.1.1 -- make BOARD=qemu-coreboot-whiptail-tpm2 run From f4db4b791c7abfd049835cf9487c56152e825976 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Mon, 6 May 2024 14:12:05 -0400 Subject: [PATCH 43/51] README.md qemu.md + CircleCI: point to images for building and using nix developed created docker image - push v0.1.3 and have latest point to the same image, add repro notes inside of README.md - modify qemu.md to also refer to using docker images TODO: remove NIX_REPRO_NOTES prior of merging Signed-off-by: Thierry Laurion --- .circleci/config.yml | 8 ++++---- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ targets/qemu.md | 13 ++++++++++++- 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 62751f4c3..ba0d02eda 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,7 +45,7 @@ commands: jobs: prep_env: docker: - - image: tlaurion/heads-dev-env:v0.1.1 + - image: tlaurion/heads-dev-env:v0.1.3 resource_class: large working_directory: ~/heads steps: @@ -111,7 +111,7 @@ jobs: build_and_persist: docker: - - image: tlaurion/heads-dev-env:v0.1.1 + - image: tlaurion/heads-dev-env:v0.1.3 resource_class: large working_directory: ~/heads parameters: @@ -139,7 +139,7 @@ jobs: build: docker: - - image: tlaurion/heads-dev-env:v0.1.1 + - image: tlaurion/heads-dev-env:v0.1.3 resource_class: large working_directory: ~/heads parameters: @@ -160,7 +160,7 @@ jobs: save_cache: docker: - - image: tlaurion/heads-dev-env:v0.1.1 + - image: tlaurion/heads-dev-env:v0.1.3 resource_class: large working_directory: ~/heads steps: diff --git a/README.md b/README.md index a0819f48a..d1505693b 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,49 @@ Please refer to [Heads-wiki](https://osresearch.net) for your Heads' documentati Building heads === + +Build docker from nix develop layer locally +==== + +``` +sh <(curl -L https://nixos.org/nix/install) --no-daemon + . /home/user/.nix-profile/etc/profile.d/nix.sh +nix build .#dockerImage && docker load < result +``` + +Jump into docker image +===== +`docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env` + +Build a board from docker image +===== `make BOARD=board_name` where board_name is the name of the board directory under `./boards` directory. + +Use prepared docker image from docker hub +==== +``` +docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) tlaurion/heads-dev-env:latest -- make BOARD=qemu-coreboot-whiptail-tpm2 +docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) tlaurion/heads-dev-env:latest -- make BOARD=qemu-coreboot-whiptail-tpm2 run +``` + +Maintenance notes on docker image +=== +Redo the steps above in case the flake.nix or nix.lock changes. Then publish on docker hub: + +``` +docker tag tlaurion/heads-dev-env:vx.y.z tlaurion/heads-dev-env:latest +docker push tlaurion/heads-dev-env:latest +``` + +Notes: +- Local builds can use ":latest" +- To reproduce CirlceCI results, make sure to use the same versioned tag declared under .circleci/config.yml's "image:" + + + +General notes on reproducible builds +=== In order to build reproducible firmware images, Heads builds a specific version of gcc and uses it to compile the Linux kernel and various tools that go into the initrd. Unfortunately this means the first step is a diff --git a/targets/qemu.md b/targets/qemu.md index 0dfdd63e3..1587fa882 100644 --- a/targets/qemu.md +++ b/targets/qemu.md @@ -1,4 +1,4 @@ -qemu-coreboot-(fb)whiptail-tpm[1,2](-hotp) boards +qemu-coreboot-(fb)whiptail-tpmX(-hotp) boards === The `qemu-coreboot-fbwhiptail-tpm1-hotp` configuration (and their variants) permits testing of most features of Heads. @@ -87,3 +87,14 @@ swtpm on Debian bookworm === 1. Install dependencies * `sudo apt install swtpm swtpm-tools` + +swtpm on nix docker image +=== +Nothing to do. Everything needed is in the docker image. + +Just make sure to pass DISPLAY environement variable on your docker command line. eg: +* Remotely downloaded docker image (doing make command only inside of docker example): + * `docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) tlaurion/heads-dev-env:latest -- make BOARD=qemu-coreboot-whiptail-tpm2` + * `docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) tlaurion/heads-dev-env:latest -- make BOARD=qemu-coreboot-whiptail-tpm2 run` +* Locally created docker image from nix develop environment (jumping into docker image variation of the above, where developer does what he wants within): + * `docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env` From 03e861ea489e2dd7531946d4b945339b3ae7bc85 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Tue, 7 May 2024 11:58:06 -0400 Subject: [PATCH 44/51] README.md: Add docs refs to setup docker and nix persistence over QubesOS Template/AppVM for usage. Expand on nix repro instructions fro NIX_REPRO_NOTES for review Signed-off-by: Thierry Laurion --- README.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d1505693b..e87aa90a0 100644 --- a/README.md +++ b/README.md @@ -29,12 +29,29 @@ Please refer to [Heads-wiki](https://osresearch.net) for your Heads' documentati Building heads === +Under QubesOS? +==== +* Setup nix persistent layer under QubesOS (Thanks @rapenne-s !) + * https://dataswamp.org/~solene/2023-05-15-qubes-os-install-nix.html +* Install docker under QubesOS (imperfect old article of mine. Better somewhere?) + * https://gist.github.com/tlaurion/9113983bbdead492735c8438cd14d6cd + Build docker from nix develop layer locally ==== ``` -sh <(curl -L https://nixos.org/nix/install) --no-daemon - . /home/user/.nix-profile/etc/profile.d/nix.sh +# DANGER: remove /nix store and recreates a fresh one. Skip if you use Nix already: +sudo rm -rf /nix/* || echo "cannot delete /nix" && sh <(curl -L https://nixos.org/nix/install) --no-daemon +# Configure nix for local builds for nix-commands and flakes usage under nix which are considered experimental features +mkdir -p ~/.config/nix +echo 'experimental-features = nix-command flakes' >~/.config/nix/nix.conf +# Source nix prior of anything else: +. /home/user/.nix-profile/etc/profile.d/nix.sh +# END OF DANGER SECTION TO BE REVIEWED +# [...] +# Build nix developer local env with flakes locks to specified versions and exits just running "true" command: +nix --print-build-logs --verbose develop --ignore-environment -- true +# Build docker image with current develop created environment (this will take a while and create "linuxboot/heads:dev-env" local docker image: nix build .#dockerImage && docker load < result ``` From 1bef1083e047c8f78b47ae8fd8eed33eea770d39 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 8 May 2024 11:26:34 -0400 Subject: [PATCH 45/51] README.md: update repro notes. flake.nix: qemu_kvm was not included for native kvm support: added Signed-off-by: Thierry Laurion --- README.md | 33 +++++++++++++++++++++++++-------- flake.nix | 18 +++++++++++++----- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index e87aa90a0..15b1527c5 100644 --- a/README.md +++ b/README.md @@ -50,25 +50,42 @@ echo 'experimental-features = nix-command flakes' >~/.config/nix/nix.conf # END OF DANGER SECTION TO BE REVIEWED # [...] # Build nix developer local env with flakes locks to specified versions and exits just running "true" command: -nix --print-build-logs --verbose develop --ignore-environment -- true +nix --print-build-logs --verbose develop --ignore-environment --command true # Build docker image with current develop created environment (this will take a while and create "linuxboot/heads:dev-env" local docker image: nix build .#dockerImage && docker load < result ``` -Jump into docker image + +Jump into nix develop created docker image for interactive workflow ===== `docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env` -Build a board from docker image -===== + +From there you can use the docker image interactively. + `make BOARD=board_name` where board_name is the name of the board directory under `./boards` directory. -Use prepared docker image from docker hub +One such useful example is to build and test qemu board roms and test them through qemu/kvm/swtpm provided in the docker image. +Please refer to [qemu documentation](targets/qemu.md) for more information. + +Eg: +``` +make BOARD=qemu-coreboot-fbwhiptail-tpm2 # Build rom, export public key to emulated usb storage from qemu runtime +make BOARD=qemu-coreboot-fbwhiptail-tpm2 PUBKEY_ASC=~/pubkey.asc inject_gpg # Inject pubkey into rom image +make BOARD=qemu-coreboot-fbwhiptail-tpm2 USB_TOKEN=Nitrokey3NFC PUBKEY_ASC=~/pubkey.asc ROOT_DISK_IMG=~/qemu-disks/debian-9.cow2 INSTALL_IMG=~/Downloads/debian-9.13.0-amd64-xfce-CD-1.iso run # Install +``` + +Alternatively, you can use locally built docker image to build a board ROM image in a single call. + +Eg: +`docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env -- make BOARD=nitropad-nv41` + +Pull docker hub image to prepare reproducible ROMs as CircleCI in one call ==== ``` -docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) tlaurion/heads-dev-env:latest -- make BOARD=qemu-coreboot-whiptail-tpm2 -docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) tlaurion/heads-dev-env:latest -- make BOARD=qemu-coreboot-whiptail-tpm2 run +docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) tlaurion/heads-dev-env:latest -- make BOARD=x230-hotp-maximized +docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) tlaurion/heads-dev-env:latest -- make BOARD=nitropad-nv41 ``` Maintenance notes on docker image @@ -81,7 +98,7 @@ docker push tlaurion/heads-dev-env:latest ``` Notes: -- Local builds can use ":latest" +- Local builds can use ":latest" tag, which will use latest tested successful CircleCI run - To reproduce CirlceCI results, make sure to use the same versioned tag declared under .circleci/config.yml's "image:" diff --git a/flake.nix b/flake.nix index 325193a03..94c947f63 100644 --- a/flake.nix +++ b/flake.nix @@ -6,9 +6,13 @@ nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; # Using the unstable channel for the latest packages, while flake.lock fixates the commit reused until changed. flake-utils.url = "github:numtide/flake-utils"; # Utilities for flake functionality. }; - # Outputs are the result of the flake, including the development environment and Docker image. - outputs = { self, nixpkgs, flake-utils, ... }: + outputs = { + self, + flake-utils, + nixpkgs, + ... + }: flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; # Accessing the legacy package set. lib = pkgs.lib; # The standard Nix packages library. @@ -66,8 +70,13 @@ zlib.dev ] ++ [ # Packages for qemu support with Canokey integration. - qemu # To test make BOARD=qemu-coreboot-* boards and then call make BOARD=qemu-coreboot-* with inject_gpg statement, and then run statement (RTFM). - #canokey doesn;t work still even if compiled in, so no reason to add 1Gb of stuff in the image + #qemu_full #Heavier but contains qemu-img, kvm and everything else needed to do development cycles under docker + qemu # To test make BOARD=qemu-coreboot-* boards and then call make BOARD=qemu-coreboot-* with inject_gpg statement, and then run statement. + qemu_kvm # kvm additional support for qemu without all the qemu-img and everything else under qemu_full + # + # TODO: make work qemu-canokey not existing in caches: + # Below are overrides to make canokey-qemu library availabe to qemu built derivative through override) + #canokey doesn't work still even if compiled in, so no reason to add 1Gb of stuff in the image (qemu -device canokey not exposed even if configured in) #canokey-qemu # Canokey lib for qemu build-time compilation. #(qemu.override { # canokeySupport = true; # This override enables Canokey support in QEMU, resulting in -device canokey being available. @@ -141,4 +150,3 @@ }; }); } - From b4936ea42c7d0acea797fafaa26347fc8ab610fc Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 8 May 2024 11:35:13 -0400 Subject: [PATCH 46/51] CircleCI: use v.0.1.4 produced with latest flake.nix which includes qemu_kvm for kvm testing in docker image Signed-off-by: Thierry Laurion --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba0d02eda..e8188d3b3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,7 +45,7 @@ commands: jobs: prep_env: docker: - - image: tlaurion/heads-dev-env:v0.1.3 + - image: tlaurion/heads-dev-env:v0.1.4 resource_class: large working_directory: ~/heads steps: @@ -111,7 +111,7 @@ jobs: build_and_persist: docker: - - image: tlaurion/heads-dev-env:v0.1.3 + - image: tlaurion/heads-dev-env:v0.1.4 resource_class: large working_directory: ~/heads parameters: @@ -139,7 +139,7 @@ jobs: build: docker: - - image: tlaurion/heads-dev-env:v0.1.3 + - image: tlaurion/heads-dev-env:v0.1.4 resource_class: large working_directory: ~/heads parameters: @@ -160,7 +160,7 @@ jobs: save_cache: docker: - - image: tlaurion/heads-dev-env:v0.1.3 + - image: tlaurion/heads-dev-env:v0.1.4 resource_class: large working_directory: ~/heads steps: From e4976e7882992c6b5182d2593f5d34bd4028e229 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Wed, 8 May 2024 15:25:57 -0400 Subject: [PATCH 47/51] Re-add kgpe-d16 as UNMAINTAINED_* boards, still built by CircleCI (since cosntant interest in the builds) Modify .circleci/config.yml to also not reuse past caches if CircleCI config changes as part of calculated hashes for the 3 layers Signed-off-by: Thierry Laurion --- .circleci/config.yml | 42 ++++++++++++++++--- ...MAINTAINED_kgpe-d16_server-whiptail.config | 0 .../UNMAINTAINED_kgpe-d16_server.config | 0 ...D_kgpe-d16_workstation-usb_keyboard.config | 0 .../UNMAINTAINED_kgpe-d16_workstation.config | 0 5 files changed, 37 insertions(+), 5 deletions(-) rename {unmaintained_boards => boards}/UNMAINTAINED_kgpe-d16_server-whiptail/UNMAINTAINED_kgpe-d16_server-whiptail.config (100%) rename {unmaintained_boards => boards}/UNMAINTAINED_kgpe-d16_server/UNMAINTAINED_kgpe-d16_server.config (100%) rename {unmaintained_boards => boards}/UNMAINTAINED_kgpe-d16_workstation-usb_keyboard/UNMAINTAINED_kgpe-d16_workstation-usb_keyboard.config (100%) rename {unmaintained_boards => boards}/UNMAINTAINED_kgpe-d16_workstation/UNMAINTAINED_kgpe-d16_workstation.config (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index e8188d3b3..3fd260dcd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -61,15 +61,15 @@ jobs: - run: name: Creating all modules and patches digest (All modules cache digest) command: | - find ./Makefile ./flake.lock ./patches/ ./modules/ -type f | sort -h |xargs sha256sum > ./tmpDir/all_modules_and_patches.sha256sums + find .circleci/config.yml ./Makefile ./flake.lock ./patches/ ./modules/ -type f | sort -h |xargs sha256sum > ./tmpDir/all_modules_and_patches.sha256sums - run: name: Creating coreboot (and associated patches) and musl-cross-make modules digest (musl-cross-make and coreboot cache digest) command: | - find ./Makefile ./flake.lock ./modules/coreboot ./modules/musl-cross* ./patches/coreboot* -type f | sort -h | xargs sha256sum > ./tmpDir/coreboot_musl-cross.sha256sums + find .circleci/config.yml ./Makefile ./flake.lock ./modules/coreboot ./modules/musl-cross* ./patches/coreboot* -type f | sort -h | xargs sha256sum > ./tmpDir/coreboot_musl-cross.sha256sums - run: name: Creating musl-cross-make and musl-cross-make patches digest (musl-cross-make cache digest) command: | - find ./Makefile ./flake.lock modules/musl-cross* -type f | sort -h | xargs sha256sum > ./tmpDir/musl-cross.sha256sums + find .circleci/config.yml ./Makefile ./flake.lock modules/musl-cross* -type f | sort -h | xargs sha256sum > ./tmpDir/musl-cross.sha256sums - restore_cache: # First matched/found key wins and following keys are not tried keys: @@ -248,6 +248,14 @@ workflows: requires: - prep_env + # coreboot 4.11 + - build_and_persist: + name: UNMAINTAINED_kgpe-d16_workstation + target: UNMAINTAINED_kgpe-d16_workstation + subcommand: "" + requires: + - prep_env + # Cache one workspace per architecture # Make sure workspace caches are chainloaded and the last in chain for an arch is saved - save_cache: @@ -458,14 +466,38 @@ workflows: requires: - librem_14 - # coreboot purism - # dasharo clevo_release + staging IASL patch + # dasharo release - build: name: nitropad-ns50 target: nitropad-ns50 subcommand: "" requires: - nitropad-nv41 + + # coreboot 4.11 + - build: + name: UNMAINTAINED_kgpe-d16_workstation-usb_keyboard + target: UNMAINTAINED_kgpe-d16_workstation-usb_keyboard + subcommand: "" + requires: + - UNMAINTAINED_kgpe-d16_workstation + + # coreboot 4.11 + - build: + name: UNMAINTAINED_kgpe-d16_server + target: UNMAINTAINED_kgpe-d16_server + subcommand: "" + requires: + - UNMAINTAINED_kgpe-d16_workstation + + # coreboot 4.11 + - build: + name: UNMAINTAINED_kgpe-d16_server-whiptail + target: UNMAINTAINED_kgpe-d16_server-whiptail + subcommand: "" + requires: + - UNMAINTAINED_kgpe-d16_workstation + # - build: # name: UNMAINTAINED_kgpe-d16_workstation-usb_keyboard # target: UNMAINTAINED_kgpe-d16_workstation-usb_keyboard diff --git a/unmaintained_boards/UNMAINTAINED_kgpe-d16_server-whiptail/UNMAINTAINED_kgpe-d16_server-whiptail.config b/boards/UNMAINTAINED_kgpe-d16_server-whiptail/UNMAINTAINED_kgpe-d16_server-whiptail.config similarity index 100% rename from unmaintained_boards/UNMAINTAINED_kgpe-d16_server-whiptail/UNMAINTAINED_kgpe-d16_server-whiptail.config rename to boards/UNMAINTAINED_kgpe-d16_server-whiptail/UNMAINTAINED_kgpe-d16_server-whiptail.config diff --git a/unmaintained_boards/UNMAINTAINED_kgpe-d16_server/UNMAINTAINED_kgpe-d16_server.config b/boards/UNMAINTAINED_kgpe-d16_server/UNMAINTAINED_kgpe-d16_server.config similarity index 100% rename from unmaintained_boards/UNMAINTAINED_kgpe-d16_server/UNMAINTAINED_kgpe-d16_server.config rename to boards/UNMAINTAINED_kgpe-d16_server/UNMAINTAINED_kgpe-d16_server.config diff --git a/unmaintained_boards/UNMAINTAINED_kgpe-d16_workstation-usb_keyboard/UNMAINTAINED_kgpe-d16_workstation-usb_keyboard.config b/boards/UNMAINTAINED_kgpe-d16_workstation-usb_keyboard/UNMAINTAINED_kgpe-d16_workstation-usb_keyboard.config similarity index 100% rename from unmaintained_boards/UNMAINTAINED_kgpe-d16_workstation-usb_keyboard/UNMAINTAINED_kgpe-d16_workstation-usb_keyboard.config rename to boards/UNMAINTAINED_kgpe-d16_workstation-usb_keyboard/UNMAINTAINED_kgpe-d16_workstation-usb_keyboard.config diff --git a/unmaintained_boards/UNMAINTAINED_kgpe-d16_workstation/UNMAINTAINED_kgpe-d16_workstation.config b/boards/UNMAINTAINED_kgpe-d16_workstation/UNMAINTAINED_kgpe-d16_workstation.config similarity index 100% rename from unmaintained_boards/UNMAINTAINED_kgpe-d16_workstation/UNMAINTAINED_kgpe-d16_workstation.config rename to boards/UNMAINTAINED_kgpe-d16_workstation/UNMAINTAINED_kgpe-d16_workstation.config From 6ce3d21e4c4f90452396882654a4f00662fae663 Mon Sep 17 00:00:00 2001 From: Jonathon Hall Date: Thu, 9 May 2024 10:40:21 -0400 Subject: [PATCH 48/51] modules/flashrom: Remove LIBS_BASE to stop linking in RPATH. Specifying LIBS_BASE causes flashrom's Makefile to link in an RPATH, using the Heads workspace path, which is not what we want. It does other things too, but we already pass the parts we need to the make invocation for flashrom. Signed-off-by: Jonathon Hall Signed-off-by: Thierry Laurion --- modules/flashrom | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/flashrom b/modules/flashrom index 52db5ec6e..63143d9dd 100644 --- a/modules/flashrom +++ b/modules/flashrom @@ -33,7 +33,6 @@ flashrom_target := \ DESTDIR="$(INSTALL)" \ INSTALL="$(INSTALL)" \ LDFLAGS="-L$(INSTALL)/lib" \ - LIBS_BASE="$(INSTALL)" \ PREFIX="$(INSTALL)" \ $(CROSS_TOOLS) \ $(flashrom_cfg) \ From c7f652bf897faf2e4110806efd28d15e8a37d53c Mon Sep 17 00:00:00 2001 From: Jonathon Hall Date: Thu, 9 May 2024 17:00:38 -0400 Subject: [PATCH 49/51] Makefile: Use relative paths in configs generated from templates Use relative paths in configs generated from templates, so the final build doesn't depend on the absolute location of the repository. The coreboot config is part of the final ROM. Signed-off-by: Jonathon Hall Signed-off-by: Thierry Laurion --- Makefile | 8 +++----- bin/prepare_module_config.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) create mode 100755 bin/prepare_module_config.sh diff --git a/Makefile b/Makefile index 9c38f7d90..2498f374d 100644 --- a/Makefile +++ b/Makefile @@ -268,12 +268,10 @@ all payload: FORCE: # Copies config while replacing predefined placeholders with actual values +# This is used in a command like 'this && $(call install_config ...) && that' +# so it needs to evaluate to a shell command. define install_config = - sed -e 's!@BOARD_BUILD_DIR@!$(board_build)!g' \ - -e 's!@BLOB_DIR@!$(pwd)/blobs!g' \ - -e 's!@BRAND_DIR@!$(pwd)/branding/$(BRAND_NAME)!g' \ - -e 's!@BRAND_NAME@!$(BRAND_NAME)!g' \ - "$1" > "$2" + $(pwd)/bin/prepare_module_config.sh "$1" "$2" "$(board_build)" "$(BRAND_NAME)" endef # Make helpers to operate on lists of things diff --git a/bin/prepare_module_config.sh b/bin/prepare_module_config.sh new file mode 100755 index 000000000..ba312ebfc --- /dev/null +++ b/bin/prepare_module_config.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash + +TEMPLATE="$1" +RESULT="$2" +BOARD_BUILD="$3" +BRAND_NAME="$4" + +repo="$(realpath "$(dirname "${BASH_SOURCE[0]}")/..")" +# For both coreboot and Linux, the config file is in a board- +# specific build directory, but the build occurs from the +# parent of that directory. +module_dir="$(realpath "$(dirname "$2")/..")" + +# Use relative paths since the config may be part of the ROM +# artifacts, and relative paths won't depend on the workspace +# absolute path. +board_build_rel="$(realpath --relative-to "$module_dir" "$BOARD_BUILD")" +repo_rel="$(realpath --relative-to "$module_dir" "$repo")" + +echo "board_build_rel=$board_build_rel" +echo "repo_rel=$repo_rel" + +sed -e "s!@BOARD_BUILD_DIR@!${board_build_rel}!g" \ + -e "s!@BLOB_DIR@!${repo_rel}/blobs!g" \ + -e "s!@BRAND_DIR@!${repo_rel}/branding/$BRAND_NAME!g" \ + -e "s!@BRAND_NAME@!$BRAND_NAME!g" \ + "$TEMPLATE" > "$RESULT" From 181ce621bb913d3fd8ed8f9c13222850555f6615 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Fri, 10 May 2024 11:37:05 -0400 Subject: [PATCH 50/51] README.md Makefile: address comments in PR review for daily/non-daily Nix users, remove NIX_REPRO_NOTES, Makefile dev helpers self-explain themselves Signed-off-by: Thierry Laurion --- Makefile | 36 +++++++++++++-------- NIX_REPRO_NOTES | 85 ------------------------------------------------- README.md | 23 +++++++++++-- 3 files changed, 44 insertions(+), 100 deletions(-) delete mode 100644 NIX_REPRO_NOTES diff --git a/Makefile b/Makefile index 2498f374d..38cc9f685 100644 --- a/Makefile +++ b/Makefile @@ -819,6 +819,8 @@ $(board_build)/$(CB_OUTPUT_BASENAME)-gpg-injected.rom: $(board_build)/$(CB_OUTPU ./bin/inject_gpg_key.sh --cbfstool "$(build)/$(coreboot_dir)/cbfstool" \ "$(board_build)/$(CB_OUTPUT_FILE_GPG_INJ)" "$(PUBKEY_ASC)" + +#Dev cycles helpers: real.clean: for dir in \ $(module_dirs) \ @@ -830,20 +832,28 @@ real.clean: done cd install && rm -rf -- * real.gitclean: - #Use git ignore file as a base to wipe everything not in tree. Keeps coreboot forks downloaded since detected as git repos, wipes the rest. + @echo "Cleaning the repository using Git ignore file as a base..." + @echo "This will wipe everything not in the Git tree, but keep downloaded coreboot forks (detected as Git repos)." git clean -fxd + real.gitclean_keep_packages: - #Same as above but keep the packages downloaded to save bandwidth + @echo "Cleaning the repository using Git ignore file as a base..." + @echo "This will wipe everything not in the Git tree, but keep the 'packages' directory." git clean -fxd -e "packages" + real.remove_canary_files-extract_patch_rebuild_what_changed: - #Another approach is to remove the "canary" files - # This forces Heads to restart building a board config by checking packages integrity, extracting them, redoing patching on files and rebuilding what needs to be rebuilt - # reinstalling what is needed under ./install as well which is what we normally want on a development cycle. - #Limitations: if for whatever reason, a patch creates a file in an extracted package dir, this approach will fail without further manual actions - # This is not so bad though: git patch apply tells you exactly which file couldn't be created as expected. Just delete those files and relaunch the build and it will succeed. - #This approach economizes a lot of time since most of the build artifacts do not need to be rebuilt since the dates of the files should be the same as when you originally built them. - # So only a minimal time is needed to rebuild, and this is also good for your SSD. - #**** USE THIS APPROACH FIRST *** - find ./build/ -type f -name ".canary" | xargs rm || echo "All .carnary files already deleted" - find ./install/*/* | xargs rm -rf || echo "All install/ARCH/* dirs and files already deleted" - echo "you can now call make BOARD=desired_board, and if any patch fails to apply because file exists; just rm that build/path_to_file and continue testing!" + @echo "Removing 'canary' files to force Heads to restart building board configurations..." + @echo "This will check package integrity, extract them, redo patching on files, and rebuild what needs to be rebuilt." + @echo "It will also reinstall the necessary files under './install'." + @echo "Limitations: If a patch creates a file in an extracted package directory, this approach may fail without further manual actions." + @echo "In such cases, Git will inform you about the file that couldn't be created as expected. Simply delete those files and relaunch the build." + @echo "This approach economizes time since most build artifacts do not need to be rebuilt, as the file dates should be the same as when you originally built them." + @echo "Only a minimal time is needed for rebuilding, which is also good for your SSD." + @echo "*** USE THIS APPROACH FIRST ***" + find ./build/ -type f -name ".canary" -print -delete + find ./install/*/* -print -exec rm -rf {} + + +real.gitclean_keep_packages_and_build: + @echo "Cleaning the repository using Git ignore file as a base..." + @echo "This will wipe everything not in the Git tree, but keep the 'packages' and 'build' directories." + git clean -fxd -e "packages" -e "build" diff --git a/NIX_REPRO_NOTES b/NIX_REPRO_NOTES deleted file mode 100644 index b96382183..000000000 --- a/NIX_REPRO_NOTES +++ /dev/null @@ -1,85 +0,0 @@ -Notes to repro on top of QubesOS debian-12-xfce template based qube - -- Clone a debian-12-xfce on top of Q4.2.1 to be specialized -- Deploy QubesOS nix requirements in template: https://dataswamp.org/~solene/2023-05-15-qubes-os-install-nix.html -- Deploy QubesOS requirements for docker: https://gist.github.com/tlaurion/9113983bbdead492735c8438cd14d6cd -- Create a qube based on the cloned template above -- Open qube -- Clone whatever repo is most recent between https://github.com/mmlb/osresearch-heads/tree/wip-nix-for-build or https://github.com/tlaurion/heads/tree/wip-nix-for-build - - git clone Above_repo - - cd /home/user/heads or cd /home/user/osresearch-heads - - git checkout wip-nix-for-build - - git reset --hard - - - -Now, replication traces - -#Install nix cleanly: -sudo rm -rf /nix/* || echo "cannot delete /nix" && sh <(curl -L https://nixos.org/nix/install) --no-daemon -#Source nix prior of creating nix cache and jumping in it -. /home/user/.nix-profile/etc/profile.d/nix.sh -#Configure nix for local builds until we have proper docker image instead: -mkdir -p ~/.config/nix -echo 'experimental-features = nix-command flakes' >~/.config/nix/nix.conf -#Build nix developer local env with flakes locks to specified versions and jump into it: -nix --print-build-logs --verbose develop --ignore-environment -#Build a board -make BOARD=nitropad-nv41 - - - -In case you need to rebuild from packages+ patches + a clean ./install (where libraries and headers are found): -make real.remove_canary_files-extract_patch_rebuild_what_changed - -IF that doesn;t help you, look at the end of the global Helper for more helper. Iterate. - - - -build docker out of nix develop environement and jump into it: -nix build .#dockerImage && docker load < result && docker run --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env - -From there, rebuild with -make real.remove_canary_files-extract_patch_rebuild_chat_changed -make BOARD=nitropad-nv41 - -Make changes until taht part works in both nix layer on top of OS and withing docker image. -...... - -SUCCESSSSS?!?!?!?! CELEBRATION! -Tell @mmlb and @insurgo on matrix channel!!!! - - -------- - -Some raw and incomplete notes on how to push that docker to dockerhub -docker tag linuxboot/heads:dev-env tlaurion/heads-dev-env:v0.0.3 -docker push tlaurion/heads-dev-env:v0.0.3 -(to be continued) - -------------------- - -#To create the docker image, using garbage collector didn't help and made the size of the image grow bigger?! not using it reverting to what worked -# produced image without garbage collector was 990 Mb image, with it (and stripping which is already done by nix builder...) its 1006Mb. Over my head -# TODO: -# - revisit later, should not block merge of pr -# - do doc under heads's README.md and then heads-wiki -sh <(curl -L https://nixos.org/nix/install) --no-daemon - . /home/user/.nix-profile/etc/profile.d/nix.sh - -#And then build derivates -nix build .#dockerImage && docker load < result && docker run --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env - -#Use it -docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env -- make BOARD=qemu-coreboot-whiptail-tpm2 -docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env -- make BOARD=qemu-coreboot-whiptail-tpm2 run - -#push it -docker login -docker tag linuxboot/heads:dev-env tlaurion/heads-dev-env:v0.1.1 -docker push tlaurion/heads-dev-env:v0.1.1 - - -# For end-users, CircleCI -docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) laurion/heads-dev-env:v0.1.1 -- make BOARD=qemu-coreboot-whiptail-tpm2 -docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) laurion/heads-dev-env:v0.1.1 -- make BOARD=qemu-coreboot-whiptail-tpm2 run diff --git a/README.md b/README.md index 15b1527c5..98fea8020 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ Under QubesOS? Build docker from nix develop layer locally ==== +If you do not use Nix on a daily basis: ``` # DANGER: remove /nix store and recreates a fresh one. Skip if you use Nix already: sudo rm -rf /nix/* || echo "cannot delete /nix" && sh <(curl -L https://nixos.org/nix/install) --no-daemon @@ -47,14 +48,31 @@ mkdir -p ~/.config/nix echo 'experimental-features = nix-command flakes' >~/.config/nix/nix.conf # Source nix prior of anything else: . /home/user/.nix-profile/etc/profile.d/nix.sh -# END OF DANGER SECTION TO BE REVIEWED -# [...] +``` + +If you use Nix on a daily basis: +``` +# Make sure your nix setup supports both nix-command and flakes experimental features: +mkdir -p ~/.config/nix +echo 'experimental-features = nix-command flakes' >> ~/.config/nix/nix.conf +# Review ~/.config/nix/nix.conf for inconsistencies in your favorite editor (vim, vi, gedit etc) # Build nix developer local env with flakes locks to specified versions and exits just running "true" command: nix --print-build-logs --verbose develop --ignore-environment --command true # Build docker image with current develop created environment (this will take a while and create "linuxboot/heads:dev-env" local docker image: nix build .#dockerImage && docker load < result ``` +Common steps to follow to build local doscker image from nix develop environment: +``` +# Build nix developer local env with flakes locks to specified versions and exits just running "true" command: +nix --print-build-logs --verbose develop --ignore-environment --command true +# Build docker image with current develop created environment (this will take a while and create "linuxboot/heads:dev-env" local docker image: +nix build .#dockerImage && docker load < result +``` + +Done! + +Your local docker image "linuxboot/heads:dev-env" is ready to use, reproducible for the specific Heads commit used and will produce ROMs reproducible for that Heads commit ID. Jump into nix develop created docker image for interactive workflow ===== @@ -81,6 +99,7 @@ Alternatively, you can use locally built docker image to build a board ROM image Eg: `docker run -e DISPLAY=$DISPLAY --network host --rm -ti -v $(pwd):$(pwd) -w $(pwd) linuxboot/heads:dev-env -- make BOARD=nitropad-nv41` + Pull docker hub image to prepare reproducible ROMs as CircleCI in one call ==== ``` From ecbfdbc57b23ef0b884b394e1ad97491b8d2f8b6 Mon Sep 17 00:00:00 2001 From: Thierry Laurion Date: Fri, 10 May 2024 15:44:34 -0400 Subject: [PATCH 51/51] README.md Simplify Setup of Nix and flakes and docker image creation instructions Signed-off-by: Thierry Laurion --- README.md | 41 +++++++++++++---------------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 98fea8020..dbc082988 100644 --- a/README.md +++ b/README.md @@ -39,36 +39,21 @@ Under QubesOS? Build docker from nix develop layer locally ==== -If you do not use Nix on a daily basis: -``` -# DANGER: remove /nix store and recreates a fresh one. Skip if you use Nix already: -sudo rm -rf /nix/* || echo "cannot delete /nix" && sh <(curl -L https://nixos.org/nix/install) --no-daemon -# Configure nix for local builds for nix-commands and flakes usage under nix which are considered experimental features -mkdir -p ~/.config/nix -echo 'experimental-features = nix-command flakes' >~/.config/nix/nix.conf -# Source nix prior of anything else: -. /home/user/.nix-profile/etc/profile.d/nix.sh -``` +#### Set up Nix and flakes -If you use Nix on a daily basis: -``` -# Make sure your nix setup supports both nix-command and flakes experimental features: -mkdir -p ~/.config/nix -echo 'experimental-features = nix-command flakes' >> ~/.config/nix/nix.conf -# Review ~/.config/nix/nix.conf for inconsistencies in your favorite editor (vim, vi, gedit etc) -# Build nix developer local env with flakes locks to specified versions and exits just running "true" command: -nix --print-build-logs --verbose develop --ignore-environment --command true -# Build docker image with current develop created environment (this will take a while and create "linuxboot/heads:dev-env" local docker image: -nix build .#dockerImage && docker load < result -``` +* If you don't already have Nix, install it: + * `[ -d /nix ] || sh <(curl -L https://nixos.org/nix/install) --no-daemon` + * `. /home/user/.nix-profile/etc/profile.d/nix.sh` +* Enable flake support in nix + * `mkdir -p ~/.config/nix` + * `echo 'experimental-features = nix-command flakes' >>~/.config/nix/nix.conf` -Common steps to follow to build local doscker image from nix develop environment: -``` -# Build nix developer local env with flakes locks to specified versions and exits just running "true" command: -nix --print-build-logs --verbose develop --ignore-environment --command true -# Build docker image with current develop created environment (this will take a while and create "linuxboot/heads:dev-env" local docker image: -nix build .#dockerImage && docker load < result -``` +#### Build image + +* Build nix developer local environment with flakes locked to specified versions + * `nix --print-build-logs --verbose develop --ignore-environment --command true` +* Build docker image with current develop created environment (this will take a while and create "linuxboot/heads:dev-env" local docker image: + * `nix build .#dockerImage && docker load < result` Done!