From 27e6365e8217b06ab6edad4efff27cf48cae450d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 8 Mar 2023 12:22:46 -0800 Subject: [PATCH 1/3] CI: Intel Compilers Add CI entries that build with: - ICC (Intel Classic Compilers `icc` and `icpc`) - Intel OneAPI (new Intel LLVM-based compilers) - Intel OneAPI GPU (new Intel DPC++/SYCL GPU compilers) --- .github/workflows/dependencies/dpcpp.sh | 50 ++++++++ .github/workflows/dependencies/icc.sh | 55 ++++++++ .github/workflows/intel.yml | 162 ++++++++++++++++++++++++ 3 files changed, 267 insertions(+) create mode 100755 .github/workflows/dependencies/dpcpp.sh create mode 100755 .github/workflows/dependencies/icc.sh create mode 100644 .github/workflows/intel.yml diff --git a/.github/workflows/dependencies/dpcpp.sh b/.github/workflows/dependencies/dpcpp.sh new file mode 100755 index 000000000..0e2dea4d0 --- /dev/null +++ b/.github/workflows/dependencies/dpcpp.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# Copyright 2020-2021 The WarpX Community +# +# License: BSD-3-Clause-LBNL +# Authors: Axel Huebl + +set -eu -o pipefail + +# Ref.: https://github.com/rscohn2/oneapi-ci +# intel-basekit intel-hpckit are too large in size +wget -q -O - https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB \ + | sudo apt-key add - +echo "deb https://apt.repos.intel.com/oneapi all main" \ + | sudo tee /etc/apt/sources.list.d/oneAPI.list + +sudo apt-get update + +df -h +# Install and reduce disk space +# https://github.com/ECP-WarpX/WarpX/pull/1566#issuecomment-790934878 +sudo apt-get install -y --no-install-recommends \ + build-essential \ + ccache \ + cmake \ + intel-oneapi-dpcpp-cpp-compiler intel-oneapi-mkl-devel \ + g++ gfortran \ + libopenmpi-dev \ + openmpi-bin && \ +sudo apt-get clean + +du -sh /opt/intel/oneapi/ +du -sh /opt/intel/oneapi/*/* +echo "+++ REDUCING oneAPI install size +++" +sudo rm -rf /opt/intel/oneapi/mkl/latest/lib/intel64/*.a \ + /opt/intel/oneapi/compiler/latest/linux/lib/oclfpga \ + /opt/intel/oneapi/compiler/latest/linux/lib/emu \ + /opt/intel/oneapi/compiler/latest/linux/bin/intel64 \ + /opt/intel/oneapi/compiler/latest/linux/bin/lld \ + /opt/intel/oneapi/compiler/latest/linux/bin/lld-link \ + /opt/intel/oneapi/compiler/latest/linux/bin/wasm-ld +du -sh /opt/intel/oneapi/ +du -sh /opt/intel/oneapi/*/* +df -h + +# Python +python3 -m pip install -U pip setuptools wheel +python3 -m pip install -U cmake pytest +#python3 -m pip install -r requirements_mpi.txt +python3 -m pip install -r examples/requirements.txt diff --git a/.github/workflows/dependencies/icc.sh b/.github/workflows/dependencies/icc.sh new file mode 100755 index 000000000..841cb0a89 --- /dev/null +++ b/.github/workflows/dependencies/icc.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# +# Copyright 2020-2021 The WarpX Community +# +# License: BSD-3-Clause-LBNL +# Authors: Axel Huebl + +set -eu -o pipefail + +export DEBIAN_FRONTEND=noninteractive +sudo apt-get -qqq update +sudo apt-get install -y \ + build-essential \ + ca-certificates \ + ccache \ + cmake \ + gnupg \ + pkg-config \ + wget + +# Ref.: https://github.com/rscohn2/oneapi-ci +sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB +sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB +echo "deb https://apt.repos.intel.com/oneapi all main" | \ + sudo tee /etc/apt/sources.list.d/oneAPI.list +sudo apt-get update +sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic + +# activate now via +set +eu +source /opt/intel/oneapi/setvars.sh +set -eu + +# cmake-easyinstall +sudo curl -L -o /usr/local/bin/cmake-easyinstall https://git.io/JvLxY +sudo chmod a+x /usr/local/bin/cmake-easyinstall +export CEI_SUDO="sudo" +export CEI_TMP="/tmp/cei" + +# openPMD-api +CXX=$(which icpc) CC=$(which icc) \ + cmake-easyinstall \ + --prefix=/usr/local \ + git+https://github.com/openPMD/openPMD-api.git@0.14.3 \ + -DopenPMD_USE_PYTHON=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_CLI_TOOLS=OFF \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) + +# Python +python3 -m pip install -U pip setuptools wheel +python3 -m pip install -U cmake pytest +#python3 -m pip install -r requirements_mpi.txt +python3 -m pip install -r examples/requirements.txt diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml new file mode 100644 index 000000000..9ff0b8416 --- /dev/null +++ b/.github/workflows/intel.yml @@ -0,0 +1,162 @@ +name: 🐧 Intel + +on: [push, pull_request] + +concurrency: + group: ${{ github.ref }}-${{ github.head_ref }}-intel + cancel-in-progress: true + +jobs: +# Ref.: https://github.com/rscohn2/oneapi-ci +# intel-basekit intel-hpckit are too large in size + build_icc: + name: oneAPI ICC + runs-on: ubuntu-20.04 + if: github.event.pull_request.draft == false + # For oneAPI, Ninja is slower than the default: + #env: + # CMAKE_GENERATOR: Ninja + steps: + - uses: actions/checkout@v3 + - name: install dependencies + run: | + .github/workflows/dependencies/icc.sh + - name: CCache Cache + uses: actions/cache@v3 + # - once stored under a key, they become immutable (even if local cache path content changes) + # - for a refresh the key has to change, e.g., hash of a tracked file in the key + with: + path: | + ~/.ccache + ~/.cache/ccache + key: ccache-intel-icc-${{ hashFiles('.github/workflows/intel.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + restore-keys: | + ccache-intel-icc-${{ hashFiles('.github/workflows/intel.yml') }}- + ccache-intel-icc- + - name: build ImpactX_ + run: | + set +eu + source /opt/intel/oneapi/setvars.sh + python3 -m pip install --upgrade pip setuptools wheel + set -eu + export CXX=$(which icpc) + export CC=$(which icc) + + cmake -S . -B build_sp \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DImpactX_PYTHON=ON \ + -DImpactX_MPI=OFF + cmake --build build_sp -j 2 + cmake --build build_sp -j 2 --target pip_install + - name: run tests + run: | + set +eu + source /opt/intel/oneapi/setvars.sh + set -eu + ctest --test-dir build_sp --output-on-failure + + build_icpx: + name: oneAPI ICX SP + runs-on: ubuntu-20.04 + # Since 2021.4.0, AMReX_GpuUtility.H: error: comparison with NaN always evaluates to false in fast floating point modes + # oneAPI 2022.2.0 hangs for -O2 and higher: + # https://github.com/ECP-WarpX/WarpX/issues/3442 + env: + CXXFLAGS: "-Werror -Wno-error=pass-failed -Wno-tautological-constant-compare" + # For oneAPI, Ninja is slower than the default: + # CMAKE_GENERATOR: Ninja + if: github.event.pull_request.draft == false + steps: + - uses: actions/checkout@v3 + - name: install dependencies + shell: bash + run: | + .github/workflows/dependencies/dpcpp.sh + - name: CCache Cache + uses: actions/cache@v3 + # - once stored under a key, they become immutable (even if local cache path content changes) + # - for a refresh the key has to change, e.g., hash of a tracked file in the key + with: + path: | + ~/.ccache + ~/.cache/ccache + key: ccache-intel-icpx-${{ hashFiles('.github/workflows/intel.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + restore-keys: | + ccache-intel-icpx-${{ hashFiles('.github/workflows/intel.yml') }}- + ccache-intel-icpx- + - name: build ImpactX_ + shell: bash + run: | + set +e + source /opt/intel/oneapi/setvars.sh + python3 -m pip install --upgrade pip setuptools wheel + set -e + export CXX=$(which icpx) + export CC=$(which icx) + + cmake -S . -B build_sp \ + -DCMAKE_CXX_FLAGS_RELEASE="-O1 -DNDEBUG" \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DImpactX_PYTHON=ON \ + -DImpactX_MPI=OFF \ + -DImpactX_PRECISION=SINGLE + cmake --build build_sp -j 2 + cmake --build build_sp -j 2 --target pip_install + + - name: run tests + run: | + set +eu + source /opt/intel/oneapi/setvars.sh + set -eu + ctest --test-dir build_sp --output-on-failure + + build_dpcc: + name: oneAPI DPC++ + runs-on: ubuntu-20.04 + # Since 2021.4.0, AMReX_GpuUtility.H: error: comparison with NaN always evaluates to false in fast floating point modes + # oneAPI 2022.2.0 hangs for -O2 and higher: + # https://github.com/ECP-ImpactX_/ImpactX_/issues/3442 + env: + CXXFLAGS: "-Werror -Wno-tautological-constant-compare" + # For oneAPI, Ninja is slower than the default: + # CMAKE_GENERATOR: Ninja + if: github.event.pull_request.draft == false + steps: + - uses: actions/checkout@v3 + - name: install dependencies + shell: bash + run: | + .github/workflows/dependencies/dpcpp.sh + - name: CCache Cache + uses: actions/cache@v3 + # - once stored under a key, they become immutable (even if local cache path content changes) + # - for a refresh the key has to change, e.g., hash of a tracked file in the key + with: + path: | + ~/.ccache + ~/.cache/ccache + key: ccache-intel-dpcc-${{ hashFiles('.github/workflows/intel.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + restore-keys: | + ccache-intel-dpcc-${{ hashFiles('.github/workflows/intel.yml') }}- + ccache-intel-dpcc- + - name: build ImpactX_ + shell: bash + run: | + set +e + source /opt/intel/oneapi/setvars.sh + python3 -m pip install --upgrade pip setuptools wheel + set -e + export CXX=$(which icpx) + export CC=$(which icx) + export CXXFLAGS="-fsycl ${CXXFLAGS}" + + cmake -S . -B build_sp \ + -DCMAKE_CXX_FLAGS_RELEASE="-O1 -DNDEBUG" \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DImpactX_COMPUTE=SYCL \ + -DImpactX_PYTHON=ON \ + -DImpactX_MPI=OFF + cmake --build build_sp -j 2 + # Skip this as it will copy the binary artifacts and we are tight on disk space + # cmake --build build_sp -j 2 --target pip_install From 4a811bdb11453ea727af08ab5e5a8353dcd07de1 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 17 Jul 2023 11:18:50 -0700 Subject: [PATCH 2/3] oneAPI 2023.2.0 Update --- .github/workflows/dependencies/dpcpp.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dependencies/dpcpp.sh b/.github/workflows/dependencies/dpcpp.sh index 0e2dea4d0..191654592 100755 --- a/.github/workflows/dependencies/dpcpp.sh +++ b/.github/workflows/dependencies/dpcpp.sh @@ -9,10 +9,12 @@ set -eu -o pipefail # Ref.: https://github.com/rscohn2/oneapi-ci # intel-basekit intel-hpckit are too large in size -wget -q -O - https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB \ - | sudo apt-key add - -echo "deb https://apt.repos.intel.com/oneapi all main" \ - | sudo tee /etc/apt/sources.list.d/oneAPI.list +# download the key to system keyring +wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ +| gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null + +# add signed entry to apt sources and configure the APT client to use Intel repository: +echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update @@ -23,7 +25,7 @@ sudo apt-get install -y --no-install-recommends \ build-essential \ ccache \ cmake \ - intel-oneapi-dpcpp-cpp-compiler intel-oneapi-mkl-devel \ + intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mkl-devel \ g++ gfortran \ libopenmpi-dev \ openmpi-bin && \ From c1ae7ce14bb34aeed692f476e1f09e6e9ab76e78 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 17 Jul 2023 11:24:34 -0700 Subject: [PATCH 3/3] add a space --- .github/workflows/dependencies/dpcpp.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dependencies/dpcpp.sh b/.github/workflows/dependencies/dpcpp.sh index 191654592..82c5c6d1c 100755 --- a/.github/workflows/dependencies/dpcpp.sh +++ b/.github/workflows/dependencies/dpcpp.sh @@ -9,6 +9,7 @@ set -eu -o pipefail # Ref.: https://github.com/rscohn2/oneapi-ci # intel-basekit intel-hpckit are too large in size + # download the key to system keyring wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null