Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Base for librom-mgmol interface development (#219)
Browse files Browse the repository at this point in the history
* changed the Signal.h header file name.

* additional routine to find scalapack library, if nothing is found.

* ci workflow with test and format

* changed branch name

* ctest verbose

* added --oversubscribe to cmake MPIEXEC_PREFLAGS variable.

* parallel hdf5 package fix. also librom is included.

* ci workflow only checks the first 20 tests.

* minor fix

* rom-fpmd driver and librom dependency in cmake. librom must be compiled before mgmol cmake.

* cmake fix and toolchain file for LC quartz.

* bug fix on FindSCALAPACK.cmake

* librom-mgmol installation script for LC quartz.
dreamer2368 authored and jeanlucf22 committed Apr 23, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent f0149cb commit eeb3de8
Showing 15 changed files with 274 additions and 78 deletions.
45 changes: 35 additions & 10 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@ on:
pull_request:
types: [opened, labeled, synchronize]
branches:
- release
- 'ROMFPMD'
# push:
# branches:
# - release
@@ -25,23 +25,48 @@ jobs:
uses: styfle/[email protected]
with:
access_token: ${{ github.token }}
- name: Set Swap Space
uses: pierotofy/set-swap-space@master
with:
swap-size-gb: 10
- name: Check out pylibROM
# - name: Set Swap Space
# uses: pierotofy/set-swap-space@master
# with:
# swap-size-gb: 10
- name: Check out mgmol
uses: actions/checkout@v1
with:
submodules: 'true'
- name: Git Submodules status
run: |
git submodule status
- name: cmake
run: |
mkdir build
cd build
cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90
cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe"
- name: make
run: |
cd build && make -j 4
- name: test
run: |
cd build && ctest --no-compress-output -V -T Test -I 1,20,1
# code-style:
# runs-on: ubuntu-latest
# needs: [docker-image]
# container:
# image: ghcr.io/llnl/mgmol/mgmol_env:latest
# options: --user 1001 --privileged
# volumes:
# - /mnt:/mnt
# steps:
# - name: Cancel previous runs
# uses: styfle/[email protected]
# with:
# access_token: ${{ github.token }}
# - name: Check out mgmol
# uses: actions/checkout@v1
# with:
# submodules: 'true'
# - name: cmake
# run: |
# mkdir build
# cd build
# cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMGMOL_WITH_CLANG_FORMAT=ON
# - name: make
# run: |
# cd build && make format

20 changes: 4 additions & 16 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -127,24 +127,12 @@ endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT)
set(USE_LIBROM False CACHE BOOL "Build with libROM")
set(LIBROM_PATH "" CACHE STRING "Path of libROM")
if(USE_LIBROM)
message(STATUS "LIBROM_PATH: ${LIBROM_PATH}")
if(NOT LIBROM_PATH)
message(FATAL_ERROR "libROM PATH not specified.")
else(NOT LIBROM_PATH)
set(LIBROM_SCRIPTS_PATH ${LIBROM_PATH}/scripts)
message(STATUS "LIBROM_PATH: ${LIBROM_PATH}")
message(STATUS "LIBROM_SCRIPTS_PATH: ${LIBROM_SCRIPTS_PATH}")
include(ExternalProject)
ExternalProject_Add(
libROM
SOURCE_DIR ${LIBROM_SCRIPTS_PATH}
CONFIGURE_COMMAND ""
BINARY_DIR ${LIBROM_PATH}
BUILD_COMMAND ${LIBROM_SCRIPTS_PATH}/compile.sh -t ${LIBROM_PATH}/cmake/toolchains/simple.cmake
INSTALL_COMMAND ""
)
include_directories(${LIBROM_PATH}/lib)
link_directories(${LIBROM_PATH}/build/lib)
message(FATAL_ERROR "Cmake is asked to use libROM, but LIBROM_PATH not specified.")
endif(NOT LIBROM_PATH)

find_package(libROM REQUIRED)
endif(USE_LIBROM)

# ARPACK (optional)
21 changes: 18 additions & 3 deletions cmake_modules/FindSCALAPACK.cmake
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@ if(DEFINED ENV{SCALAPACK_ROOT})
endif(DEFINED ENV{SCALAPACK_ROOT})

if(SCALAPACK_ROOT)
set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT})
set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT} ${SCALAPACK_ROOT}/lib/intel64)
list(APPEND _SCALAPACK_SEARCHES ${_SCALAPACK_SEARCH_DIR})
endif(SCALAPACK_ROOT)

@@ -29,13 +29,28 @@ if(NOT SCALAPACK_LIBRARY)
endforeach()
endif()

unset(SCALAPACK_NAMES)

mark_as_advanced(SCALAPACK_LIBRARY SCALAPACK_INCLUDE_DIR)

include(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY)

# Search for some default library paths
if (NOT SCALAPACK_FOUND)
find_library(SCALAPACK_LIBRARY
NAMES ${SCALAPACK_NAMES}
PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib
/opt/local/lib /opt/sw/lib /sw/lib
ENV LD_LIBRARY_PATH
ENV DYLD_FALLBACK_LIBRARY_PATH
ENV DYLD_LIBRARY_PATH
ENV SCALAPACKDIR
ENV BLACSDIR)

FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY)
endif()

unset(SCALAPACK_NAMES)

if(SCALAPACK_FOUND)
# Only Intel's scalapack requires an include directory
if(SCALAPACK_INCLUDE_DIR)
11 changes: 11 additions & 0 deletions cmake_modules/FindlibROM.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
if(NOT LIBROM_PATH)
message(FATAL_ERROR "LIBROM_PATH not specified.")
endif(NOT LIBROM_PATH)

find_library(LIBROM_LIB libROM.so HINTS "${LIBROM_PATH}/build/lib")
find_path(LIBROM_INCLUDES librom.h HINTS "${LIBROM_PATH}/lib")

mark_as_advanced(LIBROM_LIB LIBROM_INCLUDES)

include(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(libROM REQUIRED_VARS LIBROM_LIB LIBROM_INCLUDES)
6 changes: 6 additions & 0 deletions cmake_toolchains/quartz.default.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
set(CMAKE_C_COMPILER mpicc)
set(CMAKE_CXX_COMPILER mpicxx)
set(CMAKE_Fortran_COMPILER mpif90)

set(SCALAPACK_ROOT $ENV{MKLROOT})
set(SCALAPACK_BLACS_LIBRARY $ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.so)
24 changes: 18 additions & 6 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
@@ -10,23 +10,35 @@ WORKDIR /$ENVDIR
# install packages
RUN sudo apt-get install -yq git
RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake
RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpich-dev
RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev
RUN sudo apt-get install -yq libboost-all-dev
RUN sudo apt-get install -yq vim
RUN sudo apt-get install -yq git-lfs
RUN sudo apt-get install -yq valgrind
RUN sudo apt-get install -yq wget
RUN sudo apt-get install -yq astyle
### clang-format seems to be updated to 14.0. Not using it for now.
# RUN sudo apt-get install -yq clang-format

# install lldb and gdb for debugging
RUN sudo apt-get install -yq lldb gdb

RUN sudo apt-get clean -q

# NOTE: currently docker does not have a way to set environment variable with a command output.
# The following environment variable should use $(uname -m) for different architecture.
# ENV SCALAPACK_ROOT=/usr/lib/aarch64-linux-gnu/
ENV SCALAPACK_ROOT=/usr/lib/x86_64-linux-gnu/
ENV LIB_DIR=/$ENVDIR/dependencies
WORKDIR $LIB_DIR

# cmake toolchain file for librom
RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\
set(CMAKE_CXX_COMPILER mpicxx)\n\
set(CMAKE_Fortran_COMPILER mpif90)' > ./librom_env.cmake
ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake

# install libROM for scaleupROM
RUN sudo git clone https://github.com/LLNL/libROM.git
WORKDIR ./libROM/build
# libROM without MFEM.
RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=OFF
RUN sudo make -j 16

# create and switch to a user
ENV USERNAME=test
69 changes: 39 additions & 30 deletions scripts/build_quartz_libROM.sh
Original file line number Diff line number Diff line change
@@ -1,52 +1,61 @@
##! /bin/csh -f
#!/bin/bash
## An example script to build on LLNL Peloton systems.
## For now, this script assumes intel/ mkl libraries are being used.

# load some modules
## load some modules
source scripts/modules.quartz

# set some environment variables. Set them explicitly or use loaded module path (preferred)
# Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid
# benign cmake warnings
#setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib
setenv SCALAPACK_ROOT ${MKLROOT}
setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0

# We need to define the cmake blas vendor option here to find the right one.
set BLAS_VENDOR = Intel10_64lp

# manually set the location of BLACS libraries for scalapack
set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64

set MGMOL_ROOT = `pwd`

set INSTALL_DIR = ${MGMOL_ROOT}/install_quartz
## set some environment variables. Set them explicitly or use loaded module path (preferred)
## Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid
## benign cmake warnings
##setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib
#setenv SCALAPACK_ROOT ${MKLROOT}
#setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0
#
## We need to define the cmake blas vendor option here to find the right one.
#set BLAS_VENDOR = Intel10_64lp
#
## manually set the location of BLACS libraries for scalapack
#set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64

MGMOL_ROOT="$(pwd)"

INSTALL_DIR=${MGMOL_ROOT}/install_quartz
mkdir -p ${INSTALL_DIR}

set BUILD_DIR = ${MGMOL_ROOT}/build_quartz
BUILD_DIR=${MGMOL_ROOT}/build_quartz
mkdir -p ${BUILD_DIR}
cd ${BUILD_DIR}

# clone the libROM GitHub repo in BUILD_DIR
set USE_LIBROM="On"
set LIBROM_PATH = ${BUILD_DIR}/libROM
USE_LIBROM="On"
LIBROM_PATH=${BUILD_DIR}/libROM
git clone https://github.com/LLNL/libROM
cd libROM
#./scripts/compile.sh -t ./cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake
./scripts/compile.sh
cd ${BUILD_DIR}

# call cmake
cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-DCMAKE_CXX_COMPILER=mpic++ \
-DCMAKE_Fortran_COMPILER=mpif77 \
-DMPIEXEC_NUMPROC_FLAG="-n" \
-DBLA_VENDOR=${BLAS_VENDOR} \
-DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \
cmake -DCMAKE_TOOLCHAIN_FILE=${MGMOL_ROOT}/cmake_toolchains/quartz.default.cmake \
-DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-DUSE_LIBROM=${USE_LIBROM} \
-DLIBROM_PATH=${LIBROM_PATH} \
-DCMAKE_BUILD_TYPE=DEBUG \
..

# -DCMAKE_CXX_COMPILER=mpic++ \
# -DCMAKE_Fortran_COMPILER=mpif77 \
# -DMPIEXEC_NUMPROC_FLAG="-n" \
# -DBLA_VENDOR=${BLAS_VENDOR} \
# -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \
# -DCMAKE_BUILD_TYPE=DEBUG \

# call make install
make -j
make install
make -j 16
### Currently libROM does not have the installation procedure,
### so copying binary file to installation directory will disrupt the relative path to libROM.so,
### causing a run-time error.
#make install

# -DBLAS_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \
# -DLAPACK_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \
2 changes: 1 addition & 1 deletion scripts/build_ubuntu22_openmpi.sh
Original file line number Diff line number Diff line change
@@ -25,7 +25,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-DMPIEXEC_PREFLAGS="--oversubscribe" \
-DMGMOL_WITH_CLANG_FORMAT=ON \
-DCMAKE_PREFIX_PATH=${HOME}/bin \
-D CMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \
-DCMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \
..

# call make install
21 changes: 14 additions & 7 deletions scripts/modules.quartz
Original file line number Diff line number Diff line change
@@ -1,12 +1,19 @@
module load intel/2022.1.0
### choose either gcc or intel
#module load intel/2022.1.0
module load gcc/11.2.1

module load cmake
module load hdf5-parallel/1.14.0
#module load hdf5-parallel
#module load boost
module load mkl
module load cmake/3.14.5
module load boost

### choose either one
module load mkl-interfaces
#module load mkl

module load python
## manually add boost path
setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH

### manually add boost path
#setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH

#setenv MKLROOT $LIBRARY_PATH
#setenv MKLROOT /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0
21 changes: 21 additions & 0 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -193,3 +193,24 @@ endif (${MGMOL_WITH_LIBXC})

install(TARGETS mgmol-opt DESTINATION bin)

# build ROM executable
if(USE_LIBROM)
add_executable(mgmol-rom rom_main.cc)
target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS} ${LIBROM_INCLUDES})

target_link_libraries(mgmol-rom mgmol_src ${link_libs})
target_link_libraries(mgmol-rom ${SCALAPACK_LIBRARIES})
target_link_libraries(mgmol-rom ${HDF5_LIBRARIES})
target_link_libraries(mgmol-rom ${HDF5_HL_LIBRARIES})
target_link_libraries(mgmol-rom ${BLAS_LIBRARIES})
target_link_libraries(mgmol-rom ${LAPACK_LIBRARIES})
target_link_libraries(mgmol-rom ${Boost_LIBRARIES})
target_link_libraries(mgmol-rom ${LIBROM_LIB})
if (${OPENMP_CXX_FOUND})
target_link_libraries(mgmol-rom OpenMP::OpenMP_CXX)
endif()
if(${MGMOL_WITH_LIBXC})
target_link_libraries(mgmol-rom ${LIBXC_DIR}/lib/libxc.a)
endif (${MGMOL_WITH_LIBXC})
install(TARGETS mgmol-rom DESTINATION bin)
endif(USE_LIBROM)
2 changes: 1 addition & 1 deletion src/MGmol.cc
Original file line number Diff line number Diff line change
@@ -106,7 +106,7 @@ extern Timer ions_setupInteractingIons_tm;
extern Timer ions_setup_tm;
extern Timer updateCenters_tm;

#include "Signal.h"
#include "Signal_mgmol.h"
std::set<int> Signal::recv_;

template <class OrbitalsType>
2 changes: 1 addition & 1 deletion src/md.cc
Original file line number Diff line number Diff line change
@@ -31,7 +31,7 @@
#include "ProjectedMatricesMehrstellen.h"
#include "ProjectedMatricesSparse.h"
#include "Rho.h"
#include "Signal.h"
#include "Signal_mgmol.h"
#include "SpreadsAndCenters.h"
#include "tools.h"

102 changes: 102 additions & 0 deletions src/rom_main.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
// Copyright (c) 2017, Lawrence Livermore National Security, LLC and
// UT-Battelle, LLC.
// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge
// National Laboratory.
// LLNL-CODE-743438
// All rights reserved.
// This file is part of MGmol. For details, see https://github.com/llnl/mgmol.
// Please also read this link https://github.com/llnl/mgmol/LICENSE

//
// main.cc
//
// Description:
// Real grid, finite difference, molecular dynamics program
// for with nonorthogonal localized orbitals.
//
// Uses Mehrstellen operators, multigrid accelerations, and
// non-local pseudopotentials.
//
// Includes LDA and PBE exchange and correlation functionals.
//
// Units:
// Potentials, eigenvalues and operators in Rydberg
// Energies in Hartree
//
#include <cassert>
#include <iostream>
#include <iterator>
#include <vector>
using namespace std;

#ifdef _OPENMP
#include <omp.h>
#endif

#ifdef USE_CNR
#include <mkl.h>
#endif

#include <mpi.h>

#include "Control.h"
#include "DistMatrix.h"
#include "ExtendedGridOrbitals.h"
#include "LocGridOrbitals.h"
#include "MGmol.h"
#include "MGmol_MPI.h"
#include "MPIdata.h"
#include "MatricesBlacsContext.h"
#include "Mesh.h"
#include "PackedCommunicationBuffer.h"
#include "ReplicatedWorkSpace.h"
#include "SparseDistMatrix.h"
#include "magma_singleton.h"
#include "tools.h"

#include <fenv.h>
#include <sys/cdefs.h>
#include <time.h>

#include <boost/program_options.hpp>
namespace po = boost::program_options;

#include "librom.h"

//#include "MemTrack.h"

int main(int argc, char** argv)
{
// change handling of memory allocation errors
set_new_handler(noMoreMemory);

cout.sync_with_stdio();

int mpirc = MPI_Init(&argc, &argv);
if (mpirc != MPI_SUCCESS)
{
cerr << "MPI Initialization failed!!!" << endl;
MPI_Abort(MPI_COMM_WORLD, 0);
}
MPI_Comm_rank(MPI_COMM_WORLD, &mype);
assert(mype > -1);
onpe0 = (mype == 0);

CAROM::Vector librom_vector(10, false);

mpirc = MPI_Finalize();
if (mpirc != MPI_SUCCESS)
{
cerr << "MPI Finalize failed!!!" << endl;
}

time_t tt;
time(&tt);
if (onpe0) cout << " Run ended at " << ctime(&tt) << endl;

// MemTrack::TrackDumpBlocks();

// MemTrack::TrackListMemoryUsage();

return 0;
}
4 changes: 2 additions & 2 deletions src/tools/Signal.h → src/tools/Signal_mgmol.h
Original file line number Diff line number Diff line change
@@ -7,15 +7,15 @@
// This file is part of MGmol. For details, see https://github.com/llnl/mgmol.
// Please also read this link https://github.com/llnl/mgmol/LICENSE

// Adapted from Jeep: Signal.h,v 1.5 2002/06/28 20:50:33
// Adapted from Jeep: Signal_mgmol.h,v 1.5 2002/06/28 20:50:33

// The Signal class is a utility to catch UNIX signals.
// A set of flags is maintained to remeber which signals were caught.
// Member functions are provided to enable or disable signals,
// access the flag set, reset flags, or interrogate flags.
// The Signal class can be used in an application by declaring
//
// #include "Signal.h"
// #include "Signal_mgmol.h"
// set<int> Signal::recv_;
//
// A signal can be registered using, e.g.
2 changes: 1 addition & 1 deletion src/tools/Timeout.h
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@
#include <stdlib.h>

#include "MPIdata.h"
#include "Signal.h"
#include "Signal_mgmol.h"

#if PCS
#include <csignal>

0 comments on commit eeb3de8

Please sign in to comment.