From b9687013a8fd8c2e552d6a63786e3869e070c977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20K=C3=B6cher?= Date: Thu, 29 Jul 2021 21:33:54 +0200 Subject: [PATCH 1/7] petsc: enhances petsc with parmetis then scalapack and mumps through download and export --- deal.II-toolchain/packages/petsc.package | 57 +++++++++++++++++------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/deal.II-toolchain/packages/petsc.package b/deal.II-toolchain/packages/petsc.package index f409cce6..0b66a97d 100644 --- a/deal.II-toolchain/packages/petsc.package +++ b/deal.II-toolchain/packages/petsc.package @@ -73,30 +73,44 @@ else fi fi -# Add ParMETIS, if present -if [ ! -z "${PARMETIS_DIR}" ]; then +# ParMETIS +if [ -n "${PARMETIS_DIR}" ]; then cecho ${INFO} "PETSc: configuration with ParMETIS" - CONFOPTS="\ - ${CONFOPTS} \ + cecho ${INFO} "PETSc: setting --with-parmetis-dir=${PARMETIS_DIR}" + cecho ${INFO} "PETSc: setting --with-metis-dir=${PARMETIS_DIR}" + CONFOPTS="${CONFOPTS} \ --with-parmetis-dir=${PARMETIS_DIR} \ - --with-metis-dir=${PARMETIS_DIR}" + --with-metis-dir=${PARMETIS_DIR}" fi -if [ ! -z "${SCALAPACK_DIR}" ]; then +# ScaLAPACK +if [ -n "${SCALAPACK_DIR}" ]; then cecho ${INFO} "PETSc: configuration with SCALAPACK" - CONFOPTS="\ - ${CONFOPTS} \ + cecho ${INFO} "PETSc: setting --with-scalapack-dir=${SCALAPACK_DIR}" + CONFOPTS="${CONFOPTS} \ --with-scalapack-dir=${SCALAPACK_DIR}" else - CONFOPTS="${CONFOPTS} --download-scalapack=1" + % TODO: petsc install scalapack, mumps if parmetis is present + if [ -n "${PARMETIS_DIR}" ]; then + cecho ${INFO} "PETSc: setting --download-scalapack=1" + CANDI_PETSC_INSTALLS_SCALAPACK=ON + CONFOPTS="${CONFOPTS} --download-scalapack=1" + fi fi -if [ ! -z "${MUMPS_DIR}" ]; then +# MUMPS +if [ -n "${MUMPS_DIR}" ]; then cecho ${INFO} "PETSc: configuration with MUMPS" + cecho ${INFO} "PETSc: setting --with-mumps-dir=${MUMPS_DIR}" CONFOPTS="${CONFOPTS} \ --with-mumps-dir=${MUMPS_DIR}" else - CONFOPTS="${CONFOPTS} --download-mumps=1" + % TODO: petsc install scalapack, mumps if parmetis is present + if [ -n "${PARMETIS_DIR}" ]; then + cecho ${INFO} "PETSc: setting --download-mumps=1" + CANDI_PETSC_INSTALLS_MUMPS=ON + CONFOPTS="${CONFOPTS} --download-mumps=1" + fi fi ######################################################################### @@ -117,19 +131,28 @@ package_specific_setup () { package_specific_register () { export PETSC_DIR=${INSTALL_PATH} - if [ ! -z "${SCALAPACK_DIR}" ]; then + + if [ ${CANDI_PETSC_INSTALLS_SCALAPACK} = "ON" ]; then export SCALAPACK_DIR=${INSTALL_PATH} fi + + if [ ${CANDI_PETSC_INSTALLS_MUMPS} = "ON" ]; then + export MUMPS_DIR=${INSTALL_PATH} + fi } package_specific_conf () { # Generate configuration file CONFIG_FILE=${CONFIGURATION_PATH}/${EXTRACTSTO} rm -f $CONFIG_FILE - echo " -export PETSC_DIR=${INSTALL_PATH} -" >> $CONFIG_FILE - if [ ! -z "${SCALAPACK_DIR}" ]; then - echo "export SCALAPACK_DIR=${INSTALL_PATH}" >> $CONFIG_FILE + + echo "export PETSC_DIR=${INSTALL_PATH}" >> $CONFIG_FILE + + if [ ${CANDI_PETSC_INSTALLS_SCALAPACK} = "ON" ]; then + echo "export SCALAPACK_DIR=${INSTALL_PATH}" >> $CONFIG_FILE + fi + + if [ ${CANDI_PETSC_INSTALLS_MUMPS} = "ON" ]; then + echo "export MUMPS_DIR=${INSTALL_PATH}" >> $CONFIG_FILE fi } From f118f3742bcd2aad9dc74413cf7fc4c2d21ec27b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20K=C3=B6cher?= Date: Thu, 29 Jul 2021 23:30:42 +0200 Subject: [PATCH 2/7] updates scalapack --- deal.II-toolchain/packages/scalapack.package | 102 +++++++++---------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/deal.II-toolchain/packages/scalapack.package b/deal.II-toolchain/packages/scalapack.package index 6d6c251b..5784b4f8 100644 --- a/deal.II-toolchain/packages/scalapack.package +++ b/deal.II-toolchain/packages/scalapack.package @@ -1,85 +1,86 @@ ################################################################################ -## ScaLAPACK ## +# ScaLAPACK ################################################################################ +# TODO: +# BLACS is provided by ScaLAPACK + VERSION=2.1.0 +CHECKSUM=e9ae970238030fa1e8ee022a9d3e6fbb + EXTRACTSTO=scalapack-${VERSION} NAME=v${VERSION} PACKING=.tar.gz SOURCE=https://github.com/Reference-ScaLAPACK/scalapack/archive/ -CHECKSUM=e9ae970238030fa1e8ee022a9d3e6fbb - BUILDCHAIN=cmake BUILDDIR=${BUILD_PATH}/scalapack-${VERSION} INSTALL_PATH=${INSTALL_PATH}/scalapack-${VERSION} - - # Set compilers & compiler options -if [ ! -z "${CC}" ]; then - CONFOPTS="${CONFOPTS} \ - -D CMAKE_C_COMPILER=${CC}" -fi +#if [ ! -z "${CC}" ]; then +# CONFOPTS="${CONFOPTS} \ +# -D CMAKE_C_COMPILER=${CC}" +#fi -if [ ! -z "${CXX}" ]; then - CONFOPTS="${CONFOPTS} \ - -D CMAKE_CXX_COMPILER=${CXX}" -fi +#if [ ! -z "${CXX}" ]; then +# CONFOPTS="${CONFOPTS} \ +# -D CMAKE_CXX_COMPILER=${CXX}" +#fi -if [ ! -z "${FC}" ]; then - CONFOPTS="${CONFOPTS} \ - -D CMAKE_Fortran_COMPILER=${FC}" -fi +#if [ ! -z "${FC}" ]; then +# CONFOPTS="${CONFOPTS} \ +# -D CMAKE_Fortran_COMPILER=${FC}" +#fi # Set Fortran flags if [ -n ${FC} ]; then - FORTRAN_MAJOR_VERSION=$(echo __GNUC__ | ${FC} -E -xc - | tail -n 1) - if [ ${FORTRAN_MAJOR_VERSION} > 9 ]; then - CONFOPTS="${CONFOPTS} \ - -D CMAKE_Fortran_FLAGS='-g -O3 -fallow-argument-mismatch' " - else - CONFOPTS="${CONFOPTS} \ - -D CMAKE_Fortran_FLAGS='-g -O3' " - fi + FORTRAN_MAJOR_VERSION=$(echo __GNUC__ | ${FC} -E -xc - | tail -n 1) + if [ ${FORTRAN_MAJOR_VERSION} > 9 ]; then + CONFOPTS="${CONFOPTS} \ + -D CMAKE_Fortran_FLAGS='-fPIC -g -O3 -fallow-argument-mismatch' " + else + CONFOPTS="${CONFOPTS} \ + -D CMAKE_Fortran_FLAGS='-fPIC -g -O3' " + fi fi - # Set C flags -CONFOPTS="${CONFOPTS} \ - -D CMAKE_C_FLAGS:STRING='-fPIC -g -O3 ' \ - -D CMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON" +CONFOPTS="${CONFOPTS} -D CMAKE_C_FLAGS:STRING='-fPIC -g -O3 ' " + +CONFOPTS="${CONFOPTS} -D CMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON" # Set intel MKL options if [ "${MKL}" = "ON" ]; then cecho ${INFO} "MUMPS: configuration with MKL" + CONFOPTS="${CONFOPTS} -D MKLROOT=${MKL_DIR}" +else + # Set BLAS and LAPACK directories (if MKL is not used) + if [ -n "${BLAS_DIR}" ]; then + cecho ${INFO} "ScaLAPACK: configuration with BLAS_DIR=${BLAS_DIR}" + CONFOPTS="${CONFOPTS} -D BLAS_LIBRARY_DIRS:STRING=${BLAS_DIR}" + fi - CONFOPTS="\ - ${CONFOPTS} \ - -D MKLROOT=${MKL_DIR}" -else - # Set blas and lapack directories (if MKL is not used) - if [ ! -z "${BLAS_DIR}" ]; then - cecho ${INFO} "SCALAPACK: configuration with BLAS_DIR=${BLAS_DIR}" - CONFOPTS="${CONFOPTS} \ - -D BLAS_LIBRARY_DIRS:STRING=${BLAS_DIR}" - fi - - if [ ! -z "${LAPACK_DIR}" ]; then - cecho ${INFO} "SCALAPACK: configuration with LAPACK_DIR=${LAPACK_DIR}" - CONFOPTS="${CONFOPTS} \ - -D LAPACK_LIBRARY_DIRS:STRING=${LAPACK_DIR}" - fi + if [ -n "${LAPACK_DIR}" ]; then + cecho ${INFO} "ScaLAPACK: configuration with LAPACK_DIR=${LAPACK_DIR}" + CONFOPTS="${CONFOPTS} -D LAPACK_LIBRARY_DIRS:STRING=${LAPACK_DIR}" + fi fi -# BLACS is provided by ScaLAPACK package_specific_register () { export SCALAPACK_DIR=${INSTALL_PATH} - # only export the BLACS dir when MKL is not provided, so we do not override the MKL BLACS_DIR + + # BLACS is provided by ScaLAPACK + + # TODO: it seems that BLACS is not installed through scalapack + # TODO: check if */lib is useful and consider specific_conf + # + # only export the BLACS dir when MKL is not provided, + # so we do not override the MKL BLACS_DIR if [ ${MKL} = OFF ]; then - export BLACS_DIR=${INSTALL_PATH}/lib + export BLACS_DIR=${INSTALL_PATH}/lib fi } @@ -87,7 +88,6 @@ package_specific_conf () { # Generate configuration file CONFIG_FILE=${CONFIGURATION_PATH}/${EXTRACTSTO} rm -f $CONFIG_FILE - echo " -export SCALAPACK_DIR=${INSTALL_PATH} -" >> $CONFIG_FILE + + echo "export SCALAPACK_DIR=${INSTALL_PATH}" >> $CONFIG_FILE } From f4944d1cde806f0319049184b5bda12fabf2848b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20K=C3=B6cher?= Date: Thu, 29 Jul 2021 23:44:00 +0200 Subject: [PATCH 3/7] removes exporting BLACS from scalapack --- deal.II-toolchain/packages/scalapack.package | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deal.II-toolchain/packages/scalapack.package b/deal.II-toolchain/packages/scalapack.package index 5784b4f8..b8c3b093 100644 --- a/deal.II-toolchain/packages/scalapack.package +++ b/deal.II-toolchain/packages/scalapack.package @@ -79,9 +79,9 @@ package_specific_register () { # # only export the BLACS dir when MKL is not provided, # so we do not override the MKL BLACS_DIR - if [ ${MKL} = OFF ]; then - export BLACS_DIR=${INSTALL_PATH}/lib - fi + #if [ ${MKL} = OFF ]; then + # export BLACS_DIR=${INSTALL_PATH}/lib + #fi } package_specific_conf () { From 26fc00775c54b0aa0ab7ba6b8dd35bf9d26cbea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20K=C3=B6cher?= Date: Fri, 30 Jul 2021 00:25:13 +0200 Subject: [PATCH 4/7] scalapack --- deal.II-toolchain/packages/scalapack.package | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/deal.II-toolchain/packages/scalapack.package b/deal.II-toolchain/packages/scalapack.package index b8c3b093..134d7f92 100644 --- a/deal.II-toolchain/packages/scalapack.package +++ b/deal.II-toolchain/packages/scalapack.package @@ -74,14 +74,13 @@ package_specific_register () { # BLACS is provided by ScaLAPACK - # TODO: it seems that BLACS is not installed through scalapack # TODO: check if */lib is useful and consider specific_conf # # only export the BLACS dir when MKL is not provided, # so we do not override the MKL BLACS_DIR - #if [ ${MKL} = OFF ]; then - # export BLACS_DIR=${INSTALL_PATH}/lib - #fi + if [ ${MKL} = OFF ]; then + export BLACS_DIR=${INSTALL_PATH}/lib + fi } package_specific_conf () { From dcf759dc4069149cf303eb176066d1eaa11d93f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20K=C3=B6cher?= Date: Fri, 30 Jul 2021 00:25:38 +0200 Subject: [PATCH 5/7] mumps --- deal.II-toolchain/packages/mumps.package | 93 +++++++++++++----------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/deal.II-toolchain/packages/mumps.package b/deal.II-toolchain/packages/mumps.package index dd73efc5..14c8d982 100644 --- a/deal.II-toolchain/packages/mumps.package +++ b/deal.II-toolchain/packages/mumps.package @@ -1,15 +1,20 @@ ################################################################################ -## MUMPS ## +# MUMPS ################################################################################ +# TODO: +# PETSc 3.13.1: Warning: Using version 5.4.0 of package mumps PETSc is tested with 5.2 +# PETSc 3.15.2: Warning: Using version 5.4.0 of package mumps PETSc is tested with 5.3 + VERSION=5.4.0.5 +CHECKSUM=0cd3bd9ad9860f7c0092cd561931a36e + EXTRACTSTO=mumps-${VERSION} NAME=v${VERSION} PACKING=.tar.gz SOURCE=https://github.com/scivision/mumps/archive/ -CHECKSUM=0cd3bd9ad9860f7c0092cd561931a36e BUILDCHAIN=cmake @@ -17,59 +22,60 @@ BUILDDIR=${BUILD_PATH}/mumps-${VERSION} INSTALL_PATH=${INSTALL_PATH}/mumps-${VERSION} # Set compilers & compiler options -if [ ! -z "${CC}" ]; then - CONFOPTS="${CONFOPTS} \ - -D CMAKE_C_COMPILER=${CC}" -fi +#if [ ! -z "${CC}" ]; then +# CONFOPTS="${CONFOPTS} \ +# -D CMAKE_C_COMPILER=${CC}" +#fi -if [ ! -z "${CXX}" ]; then - CONFOPTS="${CONFOPTS} \ - -D CMAKE_CXX_COMPILER=${CXX}" -fi +#if [ ! -z "${CXX}" ]; then +# CONFOPTS="${CONFOPTS} \ +# -D CMAKE_CXX_COMPILER=${CXX}" +#fi -if [ ! -z "${FC}" ]; then - CONFOPTS="${CONFOPTS} \ - -D CMAKE_Fortran_COMPILER=${FC}" -fi +#if [ ! -z "${FC}" ]; then +# CONFOPTS="${CONFOPTS} \ +# -D CMAKE_Fortran_COMPILER=${FC}" +#fi + +CONFOPTS="${CONFOPTS} -D CMAKE_C_FLAGS:STRING='-fPIC -g -O3' " -CONFOPTS="${CONFOPTS} \ - -D CMAKE_C_FLAGS:STRING='-fPIC -g -O3' \ - -D CMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON" +CONFOPTS="${CONFOPTS} -D CMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON" if [ ${MKL} = ON ]; then # When MKL is provided, MUMPS will search for the required packages by itself. # Therefore we only need to set MKLROOT. - CONFOPTS="${CONFOPTS} \ - -D MKLROOT=${MKL_DIR}" + CONFOPTS="${CONFOPTS} -D MKLROOT=${MKL_DIR}" else - if [ ! -z "${BLAS_DIR}" ]; then - CONFOPTS="${CONFOPTS} \ - -D pc_blas_LIBRARY_DIRS=${BLAS_DIR}/lib " - fi - - if [ ! -z "${LAPACK_DIR}" ]; then - CONFOPTS="${CONFOPTS} \ - -D pc_lapack_LIBRARY_DIRS=${LAPACK_DIR}/lib" - fi - - if [ ! -z "${SCALAPACK_DIR}" ]; then - CONFOPTS="${CONFOPTS} \ - -D pc_scalapack_LIBRARY_DIRS=${SCALAPACK_DIR}/lib" - else - # auto build missing packages - CONFOPTS="${CONFOPTS} -D autobuild=true" - fi + if [ -n "${BLAS_DIR}" ]; then + CONFOPTS="${CONFOPTS} -D pc_blas_LIBRARY_DIRS=${BLAS_DIR}/lib" + fi + + if [ -n "${LAPACK_DIR}" ]; then + CONFOPTS="${CONFOPTS} -D pc_lapack_LIBRARY_DIRS=${LAPACK_DIR}/lib" + fi + + if [ -n "${SCALAPACK_DIR}" ]; then + CONFOPTS="${CONFOPTS} -D pc_scalapack_LIBRARY_DIRS=${SCALAPACK_DIR}/lib" + fi + + # TODO: mumps installs scalapack and blacs now! + # auto build missing packages + CONFOPTS="${CONFOPTS} -D autobuild=true" fi -# Set metis directories -if [ ! -z "${METIS_DIR}" ]; then +# Metis +if [ -n "${METIS_DIR}" ]; then CONFOPTS="${CONFOPTS} \ -D metis=true \ - -D METIS_LIBRARY=${METIS_DIR}/lib/libmetis.so \ - -D METIS_INCLUDE_DIR=${METIS_DIR}/include \ - -D PARMETIS_LIBRARY=${PARMETIS_DIR}/lib/libparmetis.so " + -D METIS_LIBRARY=${METIS_DIR}/lib/libmetis.${LDSUFFIX} \ + -D METIS_INCLUDE_DIR=${METIS_DIR}/include" fi +# Parmetis +if [ -n "${PARMETIS_DIR}" ]; then + CONFOPTS="${CONFOPTS} \ + -D PARMETIS_LIBRARY=${PARMETIS_DIR}/lib/libparmetis.${LDSUFFIX}" +fi package_specific_register () { export MUMPS_DIR=${INSTALL_PATH} @@ -79,7 +85,6 @@ package_specific_conf () { # Generate configuration file CONFIG_FILE=${CONFIGURATION_PATH}/${EXTRACTSTO} rm -f $CONFIG_FILE - echo " -export MUMPS_DIR=${INSTALL_PATH} -" >> $CONFIG_FILE + + echo "export MUMPS_DIR=${INSTALL_PATH}" >> $CONFIG_FILE } From f5d27e414ef6cd6f71631f104f87ff8f2bba9868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20K=C3=B6cher?= Date: Fri, 30 Jul 2021 00:26:10 +0200 Subject: [PATCH 6/7] petsc: v3.15.2 with updated scalapack and mumps --- deal.II-toolchain/packages/petsc.package | 34 +++++++++++------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/deal.II-toolchain/packages/petsc.package b/deal.II-toolchain/packages/petsc.package index 0b66a97d..5c3d10f8 100644 --- a/deal.II-toolchain/packages/petsc.package +++ b/deal.II-toolchain/packages/petsc.package @@ -1,5 +1,8 @@ -VERSION=3.13.1 -CHECKSUM=b8b99c4a16fbc7d6694a70af044dbaac +VERSION=3.15.2 +CHECKSUM=3f657db3a8d25835fb8a7c292f42aa17 + +#VERSION=3.13.1 +#CHECKSUM=b8b99c4a16fbc7d6694a70af044dbaac #VERSION=3.11.3 #CHECKSUM=e04cbb1aaccde4a33c8843e75a1fba15 @@ -87,30 +90,25 @@ fi if [ -n "${SCALAPACK_DIR}" ]; then cecho ${INFO} "PETSc: configuration with SCALAPACK" cecho ${INFO} "PETSc: setting --with-scalapack-dir=${SCALAPACK_DIR}" - CONFOPTS="${CONFOPTS} \ - --with-scalapack-dir=${SCALAPACK_DIR}" -else - % TODO: petsc install scalapack, mumps if parmetis is present - if [ -n "${PARMETIS_DIR}" ]; then - cecho ${INFO} "PETSc: setting --download-scalapack=1" - CANDI_PETSC_INSTALLS_SCALAPACK=ON - CONFOPTS="${CONFOPTS} --download-scalapack=1" - fi + CONFOPTS="${CONFOPTS} --with-scalapack-dir=${SCALAPACK_DIR}" fi # MUMPS if [ -n "${MUMPS_DIR}" ]; then cecho ${INFO} "PETSc: configuration with MUMPS" cecho ${INFO} "PETSc: setting --with-mumps-dir=${MUMPS_DIR}" - CONFOPTS="${CONFOPTS} \ - --with-mumps-dir=${MUMPS_DIR}" + CONFOPTS="${CONFOPTS} --with-mumps-dir=${MUMPS_DIR}" else - % TODO: petsc install scalapack, mumps if parmetis is present - if [ -n "${PARMETIS_DIR}" ]; then - cecho ${INFO} "PETSc: setting --download-mumps=1" - CANDI_PETSC_INSTALLS_MUMPS=ON - CONFOPTS="${CONFOPTS} --download-mumps=1" + # PETSc: --download-mumps needs scalapack + if [ -z "${SCALAPACK_DIR}" ]; then + cecho ${INFO} "PETSc: setting --download-scalapack=1" + CANDI_PETSC_INSTALLS_SCALAPACK=ON + CONFOPTS="${CONFOPTS} --download-scalapack=1" fi + + cecho ${INFO} "PETSc: setting --download-mumps=1" + CANDI_PETSC_INSTALLS_MUMPS=ON + CONFOPTS="${CONFOPTS} --download-mumps=1" fi ######################################################################### From 580a712dd6ca86bbde2bb526d08496db8ef75c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20K=C3=B6cher?= Date: Fri, 30 Jul 2021 00:27:09 +0200 Subject: [PATCH 7/7] candi.cfg: reorders mumps after parmetis due to dependency --- candi.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/candi.cfg b/candi.cfg index 647b57d5..cae32c76 100644 --- a/candi.cfg +++ b/candi.cfg @@ -74,9 +74,9 @@ PACKAGES="load:dealii-prepare" #PACKAGES="${PACKAGES} once:ginkgo" #PACKAGES="${PACKAGES} once:gmsh" #PACKAGES="${PACKAGES} once:gsl" -#PACKAGES="${PACKAGES} once:mumps" PACKAGES="${PACKAGES} once:opencascade" PACKAGES="${PACKAGES} once:parmetis" +#PACKAGES="${PACKAGES} once:mumps" #PACKAGES="${PACKAGES} once:sundials" #PACKAGES="${PACKAGES} once:superlu_dist" PACKAGES="${PACKAGES} once:hdf5"