From e39e3c5e7a739778e239a5681e996ec5515f4b47 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 12 Aug 2024 11:39:26 -0500 Subject: [PATCH] CI: heFFTe for tests (#5121) * CI: heFFTe for tests Add heFFTe to Azure runners to enable runtime tests that need it. * CUDA & HIP: heFFTe Builds --- .azure-pipelines.yml | 11 +++++++++++ .github/workflows/cuda.yml | 16 ++++++++++++++++ .github/workflows/dependencies/hip.sh | 20 ++++++++++++++++++++ .github/workflows/hip.yml | 6 ++++-- 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 7c71599bbf3..fa14902283c 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -110,6 +110,17 @@ jobs: -DCMAKE_CXX_STANDARD=17 \ -Duse_cmake_find_lapack=ON -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON fi + if [[ "${WARPX_CI_REGULAR_CARTESIAN_3D:-FALSE}" == "TRUE" ]]; then + cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_ENABLE_FFTW=ON -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=OFF \ + -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_MAGMA=OFF \ + -DCMAKE_VERBOSE_MAKEFILE=ON + fi rm -rf ${CEI_TMP} df -h displayName: 'Install dependencies' diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 22c09561062..95d9c148b97 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -43,6 +43,11 @@ jobs: export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" + + export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} + export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} + which nvcc || echo "nvcc not in PATH!" + cmake-easyinstall --prefix=/usr/local \ git+https://github.com/openPMD/openPMD-api.git@0.15.1 \ -DopenPMD_USE_PYTHON=OFF \ @@ -51,6 +56,16 @@ jobs: -DBUILD_CLI_TOOLS=OFF \ -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ -DCMAKE_VERBOSE_MAKEFILE=ON + cmake-easyinstall --prefix=/usr/local \ + git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_CUDA=ON -DHeffte_ENABLE_ROCM=OFF \ + -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_MAGMA=OFF \ + -DCMAKE_VERBOSE_MAKEFILE=ON - name: build WarpX run: | export CCACHE_COMPRESS=1 @@ -71,6 +86,7 @@ jobs: -DWarpX_openpmd_internal=OFF \ -DWarpX_PRECISION=SINGLE \ -DWarpX_FFT=ON \ + -DWarpX_HEFFTE=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON cmake --build build_sp -j 4 diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh index ae897786acd..2a1b4d090bc 100755 --- a/.github/workflows/dependencies/hip.sh +++ b/.github/workflows/dependencies/hip.sh @@ -65,6 +65,13 @@ source /etc/profile.d/rocm.sh hipcc --version which clang which clang++ +export CXX=$(which clang++) +export CC=$(which clang) + +# "mpic++ --showme" forgets open-pal in Ubuntu 20.04 + OpenMPI 4.0.3 +# https://bugs.launchpad.net/ubuntu/+source/openmpi/+bug/1941786 +# https://github.com/open-mpi/ompi/issues/9317 +export LDFLAGS="-lopen-pal" # cmake-easyinstall # @@ -72,3 +79,16 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" + +# heFFTe +# +cmake-easyinstall --prefix=/usr/local \ + git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ + -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=ON \ + -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_MAGMA=OFF \ + -DCMAKE_VERBOSE_MAKEFILE=ON diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 0f6710b1405..8b89e3dc4d0 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -55,7 +55,8 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE \ - -DWarpX_FFT=ON + -DWarpX_FFT=ON \ + -DWarpX_HEFFTE=ON cmake --build build_sp -j 4 export WARPX_MPI=OFF @@ -115,7 +116,8 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=DOUBLE \ - -DWarpX_FFT=ON + -DWarpX_FFT=ON \ + -DWarpX_HEFFTE=ON cmake --build build_2d -j 4 export WARPX_MPI=OFF