test(dockerfile: toolchain): wip

Signed-off-by: Alexandre Rulleau <[email protected]>

appsec/tests/integration/build.gradle

@@ -329,7 +329,6 @@ def buildAppSecTask = { String version, String variant ->
                     '''
                     git config --global --add safe.directory '*'
                     cd /appsec
-                    // test -f CMakeCache.txt || \\
                         cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \\
                               -DCMAKE_INSTALL_PREFIX=/appsec \\
                               -DDD_APPSEC_ENABLE_PATCHELF_LIBC=ON \\

appsec/tests/integration/src/docker/toolchain/CHECKSUMS

@@ -1,2 +1 @@
-9591360672ba6192c606404caf70101538728a1cd5d548efcbb952f663f182bd1954d63743ffc9dd18f5c649a62a042c5e36d1ff423634dfd074f672dd1f4af9  cmake-3.28.0-linux-x86_64.tar.gz
-48a20095711870b23bd5db342de0e058a7c6876bafad4c6ce9ff9bce672ca1e95ed9ac890d519b0884cd277d091575eda7e60a97cad377ee57c1e20dee25feb1  cmake-3.28.0-linux-aarch64.tar.gz
+89a67ebfbbc764cc456e8825ecfa90707741f8835b1b2adffae0b227ab1fe5ca9cce75b0efaffc9ca8431cae528dc54fd838867a56a2b645344d9e82d19ab1b7  llvm-project-16.0.6.src.tar.xz

appsec/tests/integration/src/docker/toolchain/Dockerfile

@@ -1,20 +1,52 @@
-FROM debian@sha256:e11072c1614c08bf88b543fcfe09d75a0426d90896408e926454e88078274fcb AS toolchain
+FROM debian:latest AS toolchain
 
-ARG LLVM_VERSION=16
+ARG LLVM_VERSION=16.0.6
 ARG ARCH
 
-COPY . /build
+COPY Toolchain.cmake /build/Toolchain.cmake
+COPY CHECKSUMS /CHECKSUMS
 
 RUN echo "Building LLVM ${LLVM_VERSION} on ${ARCH}"
 
-RUN apt-get update -y && \
-  apt-get install -y git wget lsb-release software-properties-common gnupg curl xz-utils make file lld patchelf gcc libgcc-s1 sed autoconf libssl-dev libxml2
-
-RUN wget "https://github.com/Kitware/CMake/releases/download/v3.28.0/cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" && \
-  grep -F "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" ./build/CHECKSUMS | sha512sum --check && \
-  tar --strip-components=1 -C /usr/local -xvzf "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" && \
-  rm "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz"
+RUN apt-get update && apt-get install -y \
+  wget cmake binutils lld libncurses5-dev git patchelf xz-utils curl lsb-release wget software-properties-common gnupg
 
 RUN wget https://apt.llvm.org/llvm.sh && \
   chmod +x llvm.sh && \
-  ./llvm.sh ${LLVM_VERSION} all
+  ./llvm.sh 16 all
+
+RUN wget https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-project-${LLVM_VERSION}.src.tar.xz && \
+  grep -F llvm-project-${LLVM_VERSION}.src.tar.xz /CHECKSUMS | sha512sum --check && \
+  tar -xvf llvm-project-${LLVM_VERSION}.src.tar.xz
+
+COPY wchar.h.diff /wchar.h.diff
+RUN patch /usr/include/wchar.h < /wchar.h.diff
+
+RUN cd llvm-project-${LLVM_VERSION}.src && mkdir -p build && cd build && \
+  cmake \
+  -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+  -DCMAKE_INSTALL_PREFIX=/usr \
+  -DCMAKE_C_COMPILER=clang-16 \
+  -DCMAKE_C_FLAGS="-fno-omit-frame-pointer -D_LIBCPP_HAS_NO_C11_ALIGNED_ALLOC=1" \
+  -DCMAKE_CXX_COMPILER=clang++-16 \
+  -DCMAKE_CXX_FLAGS="-fno-omit-frame-pointer -D_LIBCPP_HAS_NO_C11_ALIGNED_ALLOC=1" \
+  -DLIBUNWIND_ENABLE_SHARED=OFF \
+  -DLIBUNWIND_ENABLE_STATIC=ON \
+  -DLIBUNWIND_USE_COMPILER_RT=ON \
+  -DLIBCXXABI_ENABLE_SHARED=ON \
+  -DLIBCXXABI_USE_LLVM_UNWINDER=ON \
+  -DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON \
+  -DLIBCXXABI_USE_COMPILER_RT=ON \
+  -DLIBCXX_ENABLE_SHARED=OFF \
+  -DLIBCXX_HAS_MUSL_LIBC=ON \
+  -DLIBCXX_USE_COMPILER_RT=ON \
+  -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \
+  -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
+  -DLLVM_EXTERNAL_LIT=/usr/bin/lit ../runtimes && \
+  make -j$(nproc) install-unwind install
+
+RUN cd /usr/lib && ln -s gcc/*/*/ resource_dir
+RUN cd /usr/lib && ln -s clang/${LLVM_VERSION%%.*}/lib/linux/libclang_rt.builtins-*.a libclang_rt.builtins.a
+
+RUN rm -rf /llvm-project-${LLVM_VERSION}.src
+RUN rm -f llvm-project-${LLVM_VERSION}.src.tar.xz

appsec/tests/integration/src/docker/toolchain/Dockerfile.bak

@@ -0,0 +1,15 @@
+FROM debian@sha256:e11072c1614c08bf88b543fcfe09d75a0426d90896408e926454e88078274fcb AS toolchain
+
+ARG LLVM_VERSION=16
+ARG ARCH
+
+COPY . /build
+
+RUN echo "Building LLVM ${LLVM_VERSION} on ${ARCH}"
+
+RUN apt-get update -y && \
+  apt-get install -y git wget lsb-release software-properties-common gnupg curl xz-utils make file lld patchelf gcc libgcc-s1 sed autoconf libssl-dev libxml2 cmake
+
+RUN wget https://apt.llvm.org/llvm.sh && \
+  chmod +x llvm.sh && \
+  ./llvm.sh ${LLVM_VERSION} all

appsec/tests/integration/src/docker/toolchain/Toolchain.cmake

@@ -9,22 +9,21 @@ if(ARCHITECTURE MATCHES "x86_64")
 else()
     set(ARCH aarch64)
 endif()
-set(CMAKE_SYSROOT /build/muslsysroot)
-set(CMAKE_AR /usr/bin/llvm-ar-11)
+set(CMAKE_AR /usr/bin/llvm-ar-16)
 set(triple ${ARCH}-none-linux-musl)
 set(CMAKE_ASM_COMPILER_TARGET ${triple})
-set(CMAKE_C_COMPILER /usr/bin/clang-11)
+set(CMAKE_C_COMPILER /usr/bin/clang-16)
 set(CMAKE_C_COMPILER_TARGET ${triple})
-set(c_cxx_flags "-nostdinc -isystem${CMAKE_SYSROOT}/include -isystem/usr/lib/llvm-11/lib/clang/11.0.1/include -resource-dir ${CMAKE_SYSROOT} -Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc")
+set(c_cxx_flags "-Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc -fno-omit-frame-pointer")
 set(CMAKE_C_FLAGS_INIT ${c_cxx_flags})
-set(CMAKE_CXX_COMPILER /usr/bin/clang++-11)
+set(CMAKE_CXX_COMPILER /usr/bin/clang++-16)
 set(CMAKE_CXX_COMPILER_TARGET ${triple})
-set(CMAKE_CXX_FLAGS_INIT "-stdlib=libc++ -isystem${CMAKE_SYSROOT}/include/c++/v1 ${c_cxx_flags}")
-set(CMAKE_EXE_LINKER_FLAGS_INIT "-v -fuse-ld=lld -static -nodefaultlibs -lc++ -lc++abi ${CMAKE_SYSROOT}/lib/linux/libclang_rt.builtins-${ARCH}.a -lunwind -lc ${CMAKE_SYSROOT}/lib/linux/libclang_rt.builtins-${ARCH}.a")
-set(CMAKE_SHARED_LINKER_FLAGS_INIT "-v -fuse-ld=lld -nodefaultlibs -Wl,-Bstatic -lc++ -lc++abi ${CMAKE_SYSROOT}/lib/linux/libclang_rt.builtins-${ARCH}.a -lunwind -lglibc_compat -Wl,-Bdynamic ${CMAKE_SYSROOT}/lib/linux/libclang_rt.builtins-${ARCH}.a")
+set(CMAKE_CXX_FLAGS_INIT "-stdlib=libc++ -isystem/usr/lib/clang/16.0.6/include/c++/v1 ${c_cxx_flags}")
+set(CMAKE_EXE_LINKER_FLAGS_INIT "-v -fuse-ld=lld -static -nodefaultlibs -lc++ -lc++abi /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a -lunwind -lc /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a")
+set(CMAKE_SHARED_LINKER_FLAGS_INIT "-v -fuse-ld=lld -nodefaultlibs -Wl,-Bstatic -lc++ -lc++abi /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a -lunwind -lglibc_compat -Wl,-Bdynamic /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a")
 set(CMAKE_C_STANDARD_LIBRARIES "-Wl,-Bdynamic -lc")
 set(CMAKE_CXX_STANDARD_LIBRARIES "-Wl,-Bdynamic -lc")
 
-set(CMAKE_NM /usr/bin/llvm-nm-11)
-set(CMAKE_RANLIB /usr/bin/llvm-ranlib-11)
+set(CMAKE_NM /usr/bin/llvm-nm-16)
+set(CMAKE_RANLIB /usr/bin/llvm-ranlib-16)
 set(CMAKE_STRIP /usr/bin/strip) # llvm-strip doesn't seem to work correctly

appsec/tests/integration/src/docker/toolchain/locale.h.diff

@@ -1,11 +1,23 @@
--- locale.h
-+++ locale.h
-@@ -71,7 +71,7 @@
- #define LC_COLLATE_MASK  (1<<LC_COLLATE)
- #define LC_MONETARY_MASK (1<<LC_MONETARY)
- #define LC_MESSAGES_MASK (1<<LC_MESSAGES)
--#define LC_ALL_MASK      0x7fffffff
-+#define LC_ALL_MASK      0x1fbf
+--- /usr/include/locale.h
++++ /usr/include/locale.h
+@@ -157,19 +157,7 @@
+ # define LC_TELEPHONE_MASK	(1 << __LC_TELEPHONE)
+ # define LC_MEASUREMENT_MASK	(1 << __LC_MEASUREMENT)
+ # define LC_IDENTIFICATION_MASK	(1 << __LC_IDENTIFICATION)
+-# define LC_ALL_MASK		(LC_CTYPE_MASK \
+-				 | LC_NUMERIC_MASK \
+-				 | LC_TIME_MASK \
+-				 | LC_COLLATE_MASK \
+-				 | LC_MONETARY_MASK \
+-				 | LC_MESSAGES_MASK \
+-				 | LC_PAPER_MASK \
+-				 | LC_NAME_MASK \
+-				 | LC_ADDRESS_MASK \
+-				 | LC_TELEPHONE_MASK \
+-				 | LC_MEASUREMENT_MASK \
+-				 | LC_IDENTIFICATION_MASK \
+-				 )
++# define LC_ALL_MASK		0x1fbf
 
- locale_t duplocale(locale_t);
- void freelocale(locale_t);
+ /* Return a duplicate of the set of locale in DATASET.  All usage
+    counters are increased if necessary.  */

appsec/tests/integration/src/docker/toolchain/wchar.h.diff

@@ -0,0 +1,11 @@
+--- /usr/include/wchar.h	2024-10-31 15:25:55.995768004 +0000
++++ /usr/include/wchar.h.fix	2024-10-31 15:25:35.478137009 +0000
+@@ -485,7 +485,7 @@
+ 				int __base, locale_t __loc) __THROW;
+
+ __extension__
+-extern unsigned long long int wcstoull_l (const wchar_t *__restrict __nptr,
++extern long long wcstoull_l (const wchar_t *__restrict __nptr,
+ 					  wchar_t **__restrict __endptr,
+ 					  int __base, locale_t __loc)
+      __THROW;