From 9cf2e7795350f99b722b0a2c2723f9d269791eea Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Mon, 4 Nov 2024 13:56:40 +0100 Subject: [PATCH] test(dockerfile: toolchain): wip Signed-off-by: Alexandre Rulleau --- appsec/tests/integration/build.gradle | 1 - .../src/docker/toolchain/CHECKSUMS | 3 +- .../src/docker/toolchain/Dockerfile | 54 +++++++++++++++---- .../src/docker/toolchain/Dockerfile.bak | 15 ++++++ .../src/docker/toolchain/Toolchain.cmake | 19 ++++--- .../src/docker/toolchain/Toolchain.env | 8 --- .../src/docker/toolchain/locale.h.diff | 32 +++++++---- .../src/docker/toolchain/wchar.h.diff | 11 ++++ appsec/third_party/libddwaf | 2 +- 9 files changed, 102 insertions(+), 43 deletions(-) create mode 100644 appsec/tests/integration/src/docker/toolchain/Dockerfile.bak delete mode 100644 appsec/tests/integration/src/docker/toolchain/Toolchain.env create mode 100644 appsec/tests/integration/src/docker/toolchain/wchar.h.diff diff --git a/appsec/tests/integration/build.gradle b/appsec/tests/integration/build.gradle index a4d13593a1..e6e371eb7e 100644 --- a/appsec/tests/integration/build.gradle +++ b/appsec/tests/integration/build.gradle @@ -329,7 +329,6 @@ def buildAppSecTask = { String version, String variant -> ''' git config --global --add safe.directory '*' cd /appsec - // test -f CMakeCache.txt || \\ cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \\ -DCMAKE_INSTALL_PREFIX=/appsec \\ -DDD_APPSEC_ENABLE_PATCHELF_LIBC=ON \\ diff --git a/appsec/tests/integration/src/docker/toolchain/CHECKSUMS b/appsec/tests/integration/src/docker/toolchain/CHECKSUMS index f44d511fd9..d62c3ab33f 100644 --- a/appsec/tests/integration/src/docker/toolchain/CHECKSUMS +++ b/appsec/tests/integration/src/docker/toolchain/CHECKSUMS @@ -1,2 +1 @@ -9591360672ba6192c606404caf70101538728a1cd5d548efcbb952f663f182bd1954d63743ffc9dd18f5c649a62a042c5e36d1ff423634dfd074f672dd1f4af9 cmake-3.28.0-linux-x86_64.tar.gz -48a20095711870b23bd5db342de0e058a7c6876bafad4c6ce9ff9bce672ca1e95ed9ac890d519b0884cd277d091575eda7e60a97cad377ee57c1e20dee25feb1 cmake-3.28.0-linux-aarch64.tar.gz +89a67ebfbbc764cc456e8825ecfa90707741f8835b1b2adffae0b227ab1fe5ca9cce75b0efaffc9ca8431cae528dc54fd838867a56a2b645344d9e82d19ab1b7 llvm-project-16.0.6.src.tar.xz diff --git a/appsec/tests/integration/src/docker/toolchain/Dockerfile b/appsec/tests/integration/src/docker/toolchain/Dockerfile index b5d9aabf91..6c1fe3949c 100644 --- a/appsec/tests/integration/src/docker/toolchain/Dockerfile +++ b/appsec/tests/integration/src/docker/toolchain/Dockerfile @@ -1,20 +1,52 @@ -FROM debian@sha256:e11072c1614c08bf88b543fcfe09d75a0426d90896408e926454e88078274fcb AS toolchain +FROM debian:latest AS toolchain -ARG LLVM_VERSION=16 +ARG LLVM_VERSION=16.0.6 ARG ARCH -COPY . /build +COPY Toolchain.cmake /build/Toolchain.cmake +COPY CHECKSUMS /CHECKSUMS RUN echo "Building LLVM ${LLVM_VERSION} on ${ARCH}" -RUN apt-get update -y && \ - apt-get install -y git wget lsb-release software-properties-common gnupg curl xz-utils make file lld patchelf gcc libgcc-s1 sed autoconf libssl-dev libxml2 - -RUN wget "https://github.com/Kitware/CMake/releases/download/v3.28.0/cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" && \ - grep -F "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" ./build/CHECKSUMS | sha512sum --check && \ - tar --strip-components=1 -C /usr/local -xvzf "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" && \ - rm "cmake-3.28.0-linux-$(arch | sed s/arm/aarch/).tar.gz" +RUN apt-get update && apt-get install -y \ + wget cmake binutils lld libncurses5-dev git patchelf xz-utils curl lsb-release wget software-properties-common gnupg RUN wget https://apt.llvm.org/llvm.sh && \ chmod +x llvm.sh && \ - ./llvm.sh ${LLVM_VERSION} all + ./llvm.sh 16 all + +RUN wget https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-project-${LLVM_VERSION}.src.tar.xz && \ + grep -F llvm-project-${LLVM_VERSION}.src.tar.xz /CHECKSUMS | sha512sum --check && \ + tar -xvf llvm-project-${LLVM_VERSION}.src.tar.xz + +COPY wchar.h.diff /wchar.h.diff +RUN patch /usr/include/wchar.h < /wchar.h.diff + +RUN cd llvm-project-${LLVM_VERSION}.src && mkdir -p build && cd build && \ + cmake \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_C_COMPILER=clang-16 \ + -DCMAKE_C_FLAGS="-fno-omit-frame-pointer -D_LIBCPP_HAS_NO_C11_ALIGNED_ALLOC=1" \ + -DCMAKE_CXX_COMPILER=clang++-16 \ + -DCMAKE_CXX_FLAGS="-fno-omit-frame-pointer -D_LIBCPP_HAS_NO_C11_ALIGNED_ALLOC=1" \ + -DLIBUNWIND_ENABLE_SHARED=OFF \ + -DLIBUNWIND_ENABLE_STATIC=ON \ + -DLIBUNWIND_USE_COMPILER_RT=ON \ + -DLIBCXXABI_ENABLE_SHARED=ON \ + -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ + -DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON \ + -DLIBCXXABI_USE_COMPILER_RT=ON \ + -DLIBCXX_ENABLE_SHARED=OFF \ + -DLIBCXX_HAS_MUSL_LIBC=ON \ + -DLIBCXX_USE_COMPILER_RT=ON \ + -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ + -DLLVM_EXTERNAL_LIT=/usr/bin/lit ../runtimes && \ + make -j$(nproc) install-unwind install + +RUN cd /usr/lib && ln -s gcc/*/*/ resource_dir +RUN cd /usr/lib && ln -s clang/${LLVM_VERSION%%.*}/lib/linux/libclang_rt.builtins-*.a libclang_rt.builtins.a + +RUN rm -rf /llvm-project-${LLVM_VERSION}.src +RUN rm -f llvm-project-${LLVM_VERSION}.src.tar.xz diff --git a/appsec/tests/integration/src/docker/toolchain/Dockerfile.bak b/appsec/tests/integration/src/docker/toolchain/Dockerfile.bak new file mode 100644 index 0000000000..904a99093e --- /dev/null +++ b/appsec/tests/integration/src/docker/toolchain/Dockerfile.bak @@ -0,0 +1,15 @@ +FROM debian@sha256:e11072c1614c08bf88b543fcfe09d75a0426d90896408e926454e88078274fcb AS toolchain + +ARG LLVM_VERSION=16 +ARG ARCH + +COPY . /build + +RUN echo "Building LLVM ${LLVM_VERSION} on ${ARCH}" + +RUN apt-get update -y && \ + apt-get install -y git wget lsb-release software-properties-common gnupg curl xz-utils make file lld patchelf gcc libgcc-s1 sed autoconf libssl-dev libxml2 cmake + +RUN wget https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + ./llvm.sh ${LLVM_VERSION} all diff --git a/appsec/tests/integration/src/docker/toolchain/Toolchain.cmake b/appsec/tests/integration/src/docker/toolchain/Toolchain.cmake index a444fc21c1..b88959a21f 100644 --- a/appsec/tests/integration/src/docker/toolchain/Toolchain.cmake +++ b/appsec/tests/integration/src/docker/toolchain/Toolchain.cmake @@ -9,22 +9,21 @@ if(ARCHITECTURE MATCHES "x86_64") else() set(ARCH aarch64) endif() -set(CMAKE_SYSROOT /build/muslsysroot) -set(CMAKE_AR /usr/bin/llvm-ar-11) +set(CMAKE_AR /usr/bin/llvm-ar-16) set(triple ${ARCH}-none-linux-musl) set(CMAKE_ASM_COMPILER_TARGET ${triple}) -set(CMAKE_C_COMPILER /usr/bin/clang-11) +set(CMAKE_C_COMPILER /usr/bin/clang-16) set(CMAKE_C_COMPILER_TARGET ${triple}) -set(c_cxx_flags "-nostdinc -isystem${CMAKE_SYSROOT}/include -isystem/usr/lib/llvm-11/lib/clang/11.0.1/include -resource-dir ${CMAKE_SYSROOT} -Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc") +set(c_cxx_flags "-Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc -fno-omit-frame-pointer") set(CMAKE_C_FLAGS_INIT ${c_cxx_flags}) -set(CMAKE_CXX_COMPILER /usr/bin/clang++-11) +set(CMAKE_CXX_COMPILER /usr/bin/clang++-16) set(CMAKE_CXX_COMPILER_TARGET ${triple}) -set(CMAKE_CXX_FLAGS_INIT "-stdlib=libc++ -isystem${CMAKE_SYSROOT}/include/c++/v1 ${c_cxx_flags}") -set(CMAKE_EXE_LINKER_FLAGS_INIT "-v -fuse-ld=lld -static -nodefaultlibs -lc++ -lc++abi ${CMAKE_SYSROOT}/lib/linux/libclang_rt.builtins-${ARCH}.a -lunwind -lc ${CMAKE_SYSROOT}/lib/linux/libclang_rt.builtins-${ARCH}.a") -set(CMAKE_SHARED_LINKER_FLAGS_INIT "-v -fuse-ld=lld -nodefaultlibs -Wl,-Bstatic -lc++ -lc++abi ${CMAKE_SYSROOT}/lib/linux/libclang_rt.builtins-${ARCH}.a -lunwind -lglibc_compat -Wl,-Bdynamic ${CMAKE_SYSROOT}/lib/linux/libclang_rt.builtins-${ARCH}.a") +set(CMAKE_CXX_FLAGS_INIT "-stdlib=libc++ -isystem/usr/lib/clang/16.0.6/include/c++/v1 ${c_cxx_flags}") +set(CMAKE_EXE_LINKER_FLAGS_INIT "-v -fuse-ld=lld -static -nodefaultlibs -lc++ -lc++abi /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a -lunwind -lc /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a") +set(CMAKE_SHARED_LINKER_FLAGS_INIT "-v -fuse-ld=lld -nodefaultlibs -Wl,-Bstatic -lc++ -lc++abi /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a -lunwind -lglibc_compat -Wl,-Bdynamic /usr/lib/clang/16.0.6/lib/linux/libclang_rt.builtins-${ARCH}.a") set(CMAKE_C_STANDARD_LIBRARIES "-Wl,-Bdynamic -lc") set(CMAKE_CXX_STANDARD_LIBRARIES "-Wl,-Bdynamic -lc") -set(CMAKE_NM /usr/bin/llvm-nm-11) -set(CMAKE_RANLIB /usr/bin/llvm-ranlib-11) +set(CMAKE_NM /usr/bin/llvm-nm-16) +set(CMAKE_RANLIB /usr/bin/llvm-ranlib-16) set(CMAKE_STRIP /usr/bin/strip) # llvm-strip doesn't seem to work correctly diff --git a/appsec/tests/integration/src/docker/toolchain/Toolchain.env b/appsec/tests/integration/src/docker/toolchain/Toolchain.env deleted file mode 100644 index 7403fda0d3..0000000000 --- a/appsec/tests/integration/src/docker/toolchain/Toolchain.env +++ /dev/null @@ -1,8 +0,0 @@ -export CXXFLAGS="-stdlib=libc++ -isystem/build/muslsysroot/include/c++/v1 -nostdinc -isystem/build/muslsysroot/include -isystem/usr/lib/llvm-11/lib/clang/11.0.1/include -resource-dir /build/muslsysroot -Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc" -export CFLAGS="-nostdinc -isystem/build/muslsysroot/include -isystem/usr/lib/llvm-11/lib/clang/11.0.1/include -resource-dir /build/muslsysroot -Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc" -export LDFLAGS="-v -fuse-ld=lld -static -nodefaultlibs -lc++ -lc++abi /build/muslsysroot/lib/linux/libclang_rt.builtins-x86_64.a -lunwind -lc /build/muslsysroot/lib/linux/libclang_rt.builtins-x86_64.a" -export CC="/usr/bin/clang-11" -export CXX="/usr/bin/clang++-11" -export AR="/usr/bin/llvm-ar-11" -export NM="/usr/bin/llvm-nm-11" -export RANLIB="/usr/bin/llvm-ranlib-11" diff --git a/appsec/tests/integration/src/docker/toolchain/locale.h.diff b/appsec/tests/integration/src/docker/toolchain/locale.h.diff index 408367ec34..52d046665a 100644 --- a/appsec/tests/integration/src/docker/toolchain/locale.h.diff +++ b/appsec/tests/integration/src/docker/toolchain/locale.h.diff @@ -1,11 +1,23 @@ --- locale.h -+++ locale.h -@@ -71,7 +71,7 @@ - #define LC_COLLATE_MASK (1<