Building and Running on Summit

The following instructions apply to the OLCF Summit system with NVIDIA V100 GPUs.

initial setup

Create a working directory to contain the build script, environment script, the source code, and build directories.

See https://docs.olcf.ornl.gov/data/project_centric.html#project-work-areas for information on possible locations.

mkdir $PROJ/pumipicDev
cd $_

environment script

Create a file envGnu74.sh with the following contents:

module swap xl gcc/7.4.0
module load cuda/10.2.89
module load cmake/3.20.2

export root=$PWD
export OMPI_CXX=$root/kokkos/bin/nvcc_wrapper #spectrum is based on openmpi
export OMPI_CC=gcc

build script

Create a script named buildAllSummit.sh with the following contents:

#!/bin/bash -e

[ "$root" != "$PWD" ] && exit 1
build=build-summit-gcc74
export kk=$root/${build}-kokkos/install
export engpar=$root/${build}-engpar/install
export oh=$root/${build}-omegah/install
export pp=$root/${build}-pumipic/install
export cab=$root/${build}-cabana/install
export CMAKE_PREFIX_PATH=$kk:$oh:$engpar:$pp:$cab:$CMAKE_PREFIX_PATH

#kokkos
cd $root
git clone https://github.com/kokkos/kokkos.git
cd kokkos
git checkout 3.1.00
cd -
[ -d $kk ] && rm -rf ${kk%%install}
mkdir -p $kk
cd ${kk%%install}
cmake ../kokkos \
  -DCMAKE_CXX_COMPILER=$root/kokkos/bin/nvcc_wrapper \
  -DKokkos_ARCH_VOLTA70=ON \
  -DKokkos_ENABLE_SERIAL=ON \
  -DKokkos_ENABLE_OPENMP=off \
  -DKokkos_ENABLE_CUDA=on \
  -DKokkos_ENABLE_CUDA_LAMBDA=on \
  -DKokkos_ENABLE_DEBUG=on \
  -DKokkos_ENABLE_PROFILING=on \
  -DCMAKE_INSTALL_PREFIX=$kk
make -j 24 install

#engpar
cd $root
git clone https://github.com/SCOREC/EnGPar.git
[ -d $engpar ] && rm -rf ${engpar%%install}
mkdir -p $engpar
cd ${engpar%%install}
cmake ../EnGPar \
  -DCMAKE_INSTALL_PREFIX=$engpar \
  -DCMAKE_C_COMPILER="mpicc" \
  -DCMAKE_CXX_COMPILER="mpicxx" \
  -DCMAKE_CXX_FLAGS="-std=c++11" \
  -DENABLE_PARMETIS=OFF \
  -DENABLE_PUMI=OFF \
  -DIS_TESTING=OFF
make install -j8 

# Omega_h
cd $root
git clone https://github.com/SCOREC/omega_h.git
[ -d $oh ] && rm -rf ${oh%%install}
mkdir -p $oh 
cd ${oh%%install}
cmake ../omega_h \
  -DCMAKE_INSTALL_PREFIX=$oh \
  -DBUILD_SHARED_LIBS=OFF \
  -DOmega_h_USE_Kokkos=ON \
  -DOmega_h_USE_CUDA=on \
  -DOmega_h_CUDA_ARCH=70 \
  -DOmega_h_USE_MPI=on  \
  -DBUILD_TESTING=on  \
  -DCMAKE_CXX_COMPILER=`which mpicxx` \
  -DMPIEXEC_EXECUTABLE=`which mpiexec` \
  -DKokkos_PREFIX=$kk/lib64/cmake
make VERBOSE=1 -j8 install

#Cabana
cd $root
git clone https://github.com/ECP-copa/Cabana.git cabana
cd cabana
git checkout 0.3.0
[ -d $cab ] && rm -rf ${cab%%install}
mkdir -p $cab
cd ${cab%%install}
cmake ../cabana \
  -DCMAKE_BUILD_TYPE="Release" \
  -DCMAKE_CXX_COMPILER=$root/kokkos/bin/nvcc_wrapper \
  -DCabana_ENABLE_TESTING=ON \
  -DCabana_ENABLE_EXAMPLES=ON \
  -DCabana_ENABLE_Cuda=ON \
  -DCMAKE_INSTALL_PREFIX=$cab
make -j 24 install
ctest

#PUMIPic
cd $root
git clone https://github.com/SCOREC/pumi-pic.git
cd pumi-pic
sed -i -e '[email protected]:!https://github.com/!g' .gitmodules #use https
git submodule init 
git submodule update # download test data
cd -
[ -d $pp ] && rm -rf ${pp%%install}
mkdir -p $pp
cd ${pp%%install}
echo $CMAKE_PREFIX_PATH
cmake ../pumi-pic/  \
  -DCMAKE_CXX_COMPILER=mpicxx  \
  -DIS_TESTING=ON  \
  -DPS_IS_TESTING=ON  \
  -DTEST_DATA_DIR=../pumi-pic/pumipic-data  \
  -DOmega_h_PREFIX=$oh  \
  -DKokkos_PREFIX=$kk  \
  -DEnGPar_PREFIX=$engpar  \
  -DCabana_INSTALL_DIR=$cab  \
  -DENABLE_CABANA=on \
  -DCMAKE_INSTALL_PREFIX=$pp
make -j8 install

Make the script executable:

chmod +x buildAllSummit.sh

build

Source the environment file, and run the build script:

source envGnu74.sh
./buildAllSummit.sh