From 32868695e93d47173e982752a8e738be31eb4ba3 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 13 Jul 2023 15:28:01 -0500 Subject: [PATCH] Attempt to simplify build system support for the gasnet pp --- .cmake-format.py | 3 +- CMakeLists.txt | 53 +- cmake/FindGasnet.cmake | 497 ---------------- cmake/HPX_AddTest.cmake | 105 +--- cmake/HPX_SetupGasnet.cmake | 550 ++++++++++++++++++ cmake/templates/HPXConfig.cmake.in | 12 +- cmake/templates/hpxrun.py.in | 16 +- libs/core/gasnet_base/CMakeLists.txt | 65 +-- .../include/hpx/gasnet_base/gasnet.hpp | 2 +- .../hpx/gasnet_base/gasnet_environment.hpp | 10 +- .../gasnet_base/src/gasnet_environment.cpp | 410 +++++++------ libs/core/gasnet_base/tests/CMakeLists.txt | 4 +- .../full/command_line_handling/CMakeLists.txt | 21 +- libs/full/parcelport_gasnet/CMakeLists.txt | 59 +- .../parcelport_gasnet/examples/CMakeLists.txt | 4 +- .../include/hpx/parcelport_gasnet/header.hpp | 2 +- .../hpx/parcelport_gasnet/locality.hpp | 4 +- .../hpx/parcelport_gasnet/receiver.hpp | 28 +- .../parcelport_gasnet/receiver_connection.hpp | 13 +- .../include/hpx/parcelport_gasnet/sender.hpp | 6 +- .../parcelport_gasnet/sender_connection.hpp | 88 +-- libs/full/parcelport_gasnet/src/locality.cpp | 2 +- .../src/parcelport_gasnet.cpp | 20 +- 23 files changed, 962 insertions(+), 1012 deletions(-) delete mode 100644 cmake/FindGasnet.cmake create mode 100644 cmake/HPX_SetupGasnet.cmake diff --git a/.cmake-format.py b/.cmake-format.py index a84ef9e1025c..c58cb99e0dd9 100644 --- a/.cmake-format.py +++ b/.cmake-format.py @@ -393,7 +393,8 @@ 'RUN_SERIAL', 'NO_PARCELPORT_TCP', 'NO_PARCELPORT_LCI', - 'NO_PARCELPORT_MPI'], + 'NO_PARCELPORT_MPI', + 'NO_PARCELPORT_GASNET'], 'nargs': '2+'}}, 'add_hpx_source_group': { 'kwargs': { 'CLASS': 1, 'NAME': 1, diff --git a/CMakeLists.txt b/CMakeLists.txt index acd22b6c7c13..1edb5aed9b77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -588,7 +588,6 @@ if(APPLE OR HPX_WITH_CUDA OR HPX_WITH_HIP OR HPX_WITH_PARCELPORT_LCI - OR HPX_WITH_PARCELPORT_GASNET ) set(HPX_WITH_PKGCONFIG_DEFAULT OFF) endif() @@ -1150,37 +1149,11 @@ if(HPX_WITH_NETWORKING) ) if(HPX_WITH_PARCELPORT_GASNET) hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET) - endif() - - hpx_option( - HPX_WITH_PARCELPORT_GASNET_CONDUIT - STRING - "Define which conduit to use for the gasnet parcelport" - "udp" - STRINGS "smp;udp;mpi;ofi;ucx;ibv" - ) - - if(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "mpi") - hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_MPI) - set(HPX_WITH_PARCELPORT_GASNET_MPI ON) - elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "smp") - hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_SMP) - set(HPX_WITH_PARCELPORT_GASNET_SMP ON) - elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "ibv") - hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_IBV) - set(HPX_WITH_PARCELPORT_GASNET_IBV ON) - elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "ofi") - hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_OFI) - set(HPX_WITH_PARCELPORT_GASNET_OFI ON) - elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "ucx") - hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_UCX) - set(HPX_WITH_PARCELPORT_GASNET_UCX ON) - elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "udp") - hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_UDP) - set(HPX_WITH_PARCELPORT_GASNET_UDP ON) - else() - hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_UDP) - set(HPX_WITH_PARCELPORT_GASNET_UDP ON) + hpx_option( + HPX_WITH_PARCELPORT_GASNET_CONDUIT STRING + "Define which conduit to use for the gasnet parcelport" "udp" + STRINGS "smp;udp;mpi;ofi;ucx;ibv" + ) endif() hpx_option( @@ -1220,7 +1193,10 @@ else(HPX_WITH_NETWORKING) ) endif(HPX_WITH_NETWORKING) -if((HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) OR HPX_WITH_ASYNC_MPI OR HPX_WITH_PARCELPORT_GASNET_MPI) +if((HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) + OR HPX_WITH_ASYNC_MPI + OR ("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") +) # mpi parcelport settings hpx_option( HPX_WITH_PARCELPORT_MPI_ENV @@ -2165,13 +2141,20 @@ add_hpx_library_headers_noglob(hpx_external) # Setup plugins (set here cause if we include it inside plugins, it will not be # defined in src/CMakeLists.txt where we call add_static_parcelports) include(HPX_SetupMPI) # must come before APEX -if((HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) OR HPX_WITH_ASYNC_MPI OR HPX_WITH_PARCELPORT_GASNET_MPI) +if((HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) + OR HPX_WITH_ASYNC_MPI + OR ("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") +) hpx_setup_mpi() endif() -include(HPX_SetupLCI) if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) + include(HPX_SetupLCI) hpx_setup_lci() endif() +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) + include(HPX_SetupGasnet) + hpx_setup_gasnet() +endif() # Setup packages and subprojects include(HPX_SetupSYCL) diff --git a/cmake/FindGasnet.cmake b/cmake/FindGasnet.cmake deleted file mode 100644 index 2dbcb190eb77..000000000000 --- a/cmake/FindGasnet.cmake +++ /dev/null @@ -1,497 +0,0 @@ -# Copyright (c) 2017 Thomas Heller -# Copyright (c) 2023 Christopher Taylor -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -# -macro(find_gasnet) - -find_package(PkgConfig REQUIRED QUIET COMPONENTS) - -if(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "mpi") - message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") - pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-mpi-par) - set(GASNET_MPI_FOUND TRUE) - hpx_setup_mpi() -else() - message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") - pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par) -endif() - -target_link_directories(hpx_core PUBLIC ${GASNET_LIBRARY_DIRS}) - -if(GASNET_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_CFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_CFLAGS "${X}") - endforeach() -endif() - -if(GASNET_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_CFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_CFLAGS_OTHER "${X}") - endforeach() -endif() - -if(GASNET_LDFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-l" LIDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IDX}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_LDFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_LDFLAGS "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library(FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} - NO_CACHE - NO_CMAKE_FIND_ROOT_PATH - NO_DEFAULT_PATH) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" "${NEWLINK}" IDX) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() -endif() - -if(GASNET_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IDX}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_LDFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_LDFLAGS_OTHER "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library(FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} - NO_CACHE - NO_CMAKE_FIND_ROOT_PATH - NO_DEFAULT_PATH) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" "${NEWLINK}" IDX) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") - endif() - endif() - -endif() - -if(GASNET_STATIC_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_STATIC_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_CFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_CFLAGS "${X}") - endforeach() -endif() - -if(GASNET_STATIC_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_STATIC_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_CFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_CFLAGS_OTHER "${X}") - endforeach() -endif() - -if(GASNET_STATIC_LDFLAGS) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IDX}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_STATIC_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_LDFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_LDFLAGS "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library(FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} - NO_CACHE - NO_CMAKE_FIND_ROOT_PATH - NO_DEFAULT_PATH) - - list(LENGTH FOUND_LIB IDX) - - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" "${NEWLINK}" IDX) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") - endif() - endif() -endif() - -if(GASNET_STATIC_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IDX}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_STATIC_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_LDFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library(FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} - NO_CACHE - NO_CMAKE_FIND_ROOT_PATH - NO_DEFAULT_PATH) - - list(LENGTH FOUND_LIB IDX) - - message(STATUS "${FOUND_LIB} ${X}") - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" "${NEWLINK}" IDX) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") - endif() - endif() -endif() - -set_target_properties(PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}") -set_target_properties(PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}") - -endmacro() diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index ce884f5cc0fe..6dfb45989c2b 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -6,8 +6,7 @@ function(add_hpx_test category name) set(options FAILURE_EXPECTED RUN_SERIAL NO_PARCELPORT_TCP NO_PARCELPORT_MPI - NO_PARCELPORT_LCI NO_PARCELPORT_GASNET_SMP NO_PARCELPORT_GASNET_UDP - NO_PARCELPORT_GASNET_UCX NO_PARCELPORT_GASNET_OFI NO_PARCELPORT_GASNET_IBV + NO_PARCELPORT_LCI NO_PARCELPORT_GASNET ) set(one_value_args EXECUTABLE LOCALITIES THREADS_PER_LOCALITY TIMEOUT RUNWRAPPER @@ -173,7 +172,7 @@ function(add_hpx_test category name) endif() endif() endif() - if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_SMP AND NOT ${${name}_NO_PARCELPORT_GASNET_SMP}) + if(HPX_WITH_PARCELPORT_GASNET AND NOT ${name}_NO_PARCELPORT_GASNET) set(_add_test FALSE) if(DEFINED ${name}_PARCELPORTS) set(PP_FOUND -1) @@ -186,104 +185,8 @@ function(add_hpx_test category name) endif() if(_add_test) set(_full_name "${category}.distributed.gasnet.${name}") - add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" - ${args} - ) - set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) - if(${name}_TIMEOUT) - set_tests_properties( - "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} - ) - endif() - endif() - endif() - if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_UDP AND NOT ${${name}_NO_PARCELPORT_GASNET_UDP}) - set(_add_test FALSE) - if(DEFINED ${name}_PARCELPORTS) - set(PP_FOUND -1) - list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) - if(NOT PP_FOUND EQUAL -1) - set(_add_test TRUE) - endif() - else() - set(_add_test TRUE) - endif() - if(_add_test) - set(_full_name "${category}.distributed.gasnet.${name}") - add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" - ${args} - ) - set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) - if(${name}_TIMEOUT) - set_tests_properties( - "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} - ) - endif() - endif() - endif() - if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_OFI AND NOT ${${name}_NO_PARCELPORT_GASNET_OFI}) - set(_add_test FALSE) - if(DEFINED ${name}_PARCELPORTS) - set(PP_FOUND -1) - list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) - if(NOT PP_FOUND EQUAL -1) - set(_add_test TRUE) - endif() - else() - set(_add_test TRUE) - endif() - if(_add_test) - set(_full_name "${category}.distributed.gasnet.${name}") - add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" - ${args} - ) - set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) - if(${name}_TIMEOUT) - set_tests_properties( - "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} - ) - endif() - endif() - endif() - if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_UCX AND NOT ${${name}_NO_PARCELPORT_GASNET_UCX}) - set(_add_test FALSE) - if(DEFINED ${name}_PARCELPORTS) - set(PP_FOUND -1) - list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) - if(NOT PP_FOUND EQUAL -1) - set(_add_test TRUE) - endif() - else() - set(_add_test TRUE) - endif() - if(_add_test) - set(_full_name "${category}.distributed.gasnet.${name}") - add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" - ${args} - ) - set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) - if(${name}_TIMEOUT) - set_tests_properties( - "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} - ) - endif() - endif() - endif() - if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_IBV AND NOT ${${name}_NO_PARCELPORT_GASNET_IBV}) - set(_add_test FALSE) - if(DEFINED ${name}_PARCELPORTS) - set(PP_FOUND -1) - list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) - if(NOT PP_FOUND EQUAL -1) - set(_add_test TRUE) - endif() - else() - set(_add_test TRUE) - endif() - if(_add_test) - set(_full_name "${category}.distributed.gasnet.${name}") - add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" - ${args} + add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" + "gasnet" ${args} ) set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) if(${name}_TIMEOUT) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake new file mode 100644 index 000000000000..01bf57ad22d9 --- /dev/null +++ b/cmake/HPX_SetupGasnet.cmake @@ -0,0 +1,550 @@ +# Copyright (c) 2017 Thomas Heller +# Copyright (c) 2023 Christopher Taylor +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# + +macro(hpx_setup_gasnet) + + if(NOT TARGET PkgConfig::GASNET) + + find_package(PkgConfig REQUIRED QUIET COMPONENTS) + + hpx_info( + STATUS + "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`" + ) + pkg_search_module( + GASNET REQUIRED IMPORTED_TARGET GLOBAL + gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par + ) + + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") + set(GASNET_MPI_FOUND TRUE) + include(HPX_SetupMPI) + hpx_setup_mpi() + endif() + + target_link_directories(hpx_core PUBLIC ${GASNET_LIBRARY_DIRS}) + + if(GASNET_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_CFLAGS "${X}") + endforeach() + endif() + + if(GASNET_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(GASNET_LDFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-l" LIDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + + if(GASNET_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") + endif() + endif() + + endif() + + if(GASNET_STATIC_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_STATIC_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_CFLAGS "${X}") + endforeach() + endif() + + if(GASNET_STATIC_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_STATIC_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(GASNET_STATIC_LDFLAGS) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_STATIC_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + + if(GASNET_STATIC_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_STATIC_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + message(STATUS "${FOUND_LIB} ${X}") + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") + endif() + endif() + endif() + + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES "${GASNET_LIBRARY_DIRS}" + ) + + endif() + +endmacro() diff --git a/cmake/templates/HPXConfig.cmake.in b/cmake/templates/HPXConfig.cmake.in index 7a68ee418a21..28cc6b872304 100644 --- a/cmake/templates/HPXConfig.cmake.in +++ b/cmake/templates/HPXConfig.cmake.in @@ -52,15 +52,9 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) endif() endif() -if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND - ( HPX_WITH_PARCELPORT_GASNET_SMP OR - HPX_WITH_PARCELPORT_GASNET_UDP OR - HPX_WITH_PARCELPORT_GASNET_OFI OR - HPX_WITH_PARCELPORT_GASNET_UCX OR - HPX_WITH_PARCELPORT_GASNET_IBV ) ) - set(GASNET_ROOT "@GASNET_ROOT@") - include(FindGasnet) - find_gasnet() +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) + include(HPX_SetupGasnet) + hpx_setup_gasnet() endif() # Eve can be installed by HPX or externally installed. In the first case we use diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index 4955cdf75537..76eed0195555 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -196,17 +196,9 @@ def run_mpi(cmd, localities, verbose): subproc(exec_cmd) # Run with amudprun -# This is executing mpiexec with the "-np" option set to the number of localities +# This is executing amudprun with the "-np" option set to the number of localities def run_gasnet(cmd, localities, verbose): - mpiexec = 'amudprun' - if mpiexec == '': - mpiexec = 'amudprun' - if mpiexec == '': - msg = 'amudprun not available on this platform. ' - msg += 'Please rerun CMake with HPX_PARCELPORT_GASNET=True.' - print(msg, sys.stderr) - sys.exit(1) - exec_cmd = [mpiexec, '-np', str(localities)] + cmd + exec_cmd = ['amudprun', '-np', str(localities)] + cmd if verbose: print('Executing command: ' + ' '.join(exec_cmd)) subproc(exec_cmd) @@ -381,13 +373,13 @@ Used by the tcp parcelport only. parser.add_option('-p', '--parcelport' , action='store', type='string' , dest='parcelport', default=default_env('HPXRUN_PARCELPORT', 'tcp') - , help='Which parcelport to use (Options are: mpi, lci, gaset, tcp) ' + , help='Which parcelport to use (Options are: mpi, lci, gasnet, tcp) ' '(environment variable HPXRUN_PARCELPORT') parser.add_option('-r', '--runwrapper' , action='store', type='string' , dest='runwrapper', default=default_env('HPXRUN_RUNWRAPPER', 'none') - , help='Which runwrapper to use (Options are: none, mpi, srun, jsrun) ' + , help='Which runwrapper to use (Options are: none, mpi, gasnet, srun, jsrun) ' '(environment variable HPXRUN_RUNWRAPPER)') parser.add_option('-e', '--expected' diff --git a/libs/core/gasnet_base/CMakeLists.txt b/libs/core/gasnet_base/CMakeLists.txt index 2f4c4486f50a..0a5736df497b 100644 --- a/libs/core/gasnet_base/CMakeLists.txt +++ b/libs/core/gasnet_base/CMakeLists.txt @@ -1,57 +1,38 @@ -# Copyright (c) 2019-2023 The STE||AR-Group +# Copyright (c) 2023 Chistopher Taylor # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_CONDUIT)) - message(STATUS "*** PARCELPORT_GASNET not set") +if(NOT + (HPX_WITH_NETWORKING + AND HPX_WITH_PARCELPORT_GASNET + AND HPX_WITH_PARCELPORT_GASNET_CONDUIT) +) return() endif() -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") - -include(FindGasnet) -find_gasnet() +include(HPX_SetupGasnet) +hpx_setup_gasnet() # Default location is $HPX_ROOT/libs/gasnet_base/include -set(gasnet_base_headers - hpx/gasnet_base/gasnet.hpp - hpx/gasnet_base/gasnet_environment.hpp +set(gasnet_base_headers hpx/gasnet_base/gasnet.hpp + hpx/gasnet_base/gasnet_environment.hpp ) -# Default location is $HPX_ROOT/libs/gasnet_base/include -# cmake-format: off -set(gasnet_base_compat_headers - hpx/plugins/parcelport/gasnet/gasnet.hpp => hpx/modules/gasnet_base.hpp - hpx/plugins/parcelport/gasnet/gasnet_environment.hpp => hpx/modules/gasnet_base.hpp -) -# cmake-format: on - set(gasnet_base_sources gasnet_environment.cpp) -if(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "mpi") - include(HPX_AddModule) - add_hpx_module( - core gasnet_base - GLOBAL_HEADER_GEN ON - SOURCES ${gasnet_base_sources} - HEADERS ${gasnet_base_headers} - COMPAT_HEADERS ${gasnet_base_compat_headers} - DEPENDENCIES PkgConfig::GASNET Mpi::mpi - MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util - CMAKE_SUBDIRS examples tests - ) -else() - include(HPX_AddModule) - add_hpx_module( - core gasnet_base - GLOBAL_HEADER_GEN ON - SOURCES ${gasnet_base_sources} - HEADERS ${gasnet_base_headers} - COMPAT_HEADERS ${gasnet_base_compat_headers} - DEPENDENCIES PkgConfig::GASNET - MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util - CMAKE_SUBDIRS examples tests - ) +if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") + set(gasnet_additional_dependencies Mpi::mpi) endif() + +include(HPX_AddModule) +add_hpx_module( + core gasnet_base + GLOBAL_HEADER_GEN ON + SOURCES ${gasnet_base_sources} + HEADERS ${gasnet_base_headers} + DEPENDENCIES PkgConfig::GASNET ${gasnet_additional_dependencies} + MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util + CMAKE_SUBDIRS examples tests +) diff --git a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp index e76245428664..b13044c7ad59 100644 --- a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp +++ b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp @@ -6,7 +6,7 @@ #pragma once -#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) #if defined(__clang__) diff --git a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp index 98c7310ab124..fe66baaeca11 100644 --- a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp +++ b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp @@ -9,13 +9,13 @@ #include -#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) #include #include -#include #include +#include #include #include @@ -44,9 +44,11 @@ namespace hpx { namespace util { static bool gettable(const int node, void* start, const size_t len); - static void put(std::uint8_t * addr, const int rank, std::uint8_t * raddr, const std::size_t size); + static void put(std::uint8_t* addr, const int rank, std::uint8_t* raddr, + const std::size_t size); - static void get(std::uint8_t * addr, const int rank, std::uint8_t * raddr, const std::size_t size); + static void get(std::uint8_t* addr, const int rank, std::uint8_t* raddr, + const std::size_t size); struct HPX_CORE_EXPORT scoped_lock { diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 44436a8934c6..1f8accd5837a 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -9,8 +9,8 @@ #include -#include #include +#include #include #include #include @@ -19,76 +19,77 @@ #include #include -#include #include +#include // // AM functions // -typedef enum { - SIGNAL = 136, // ack to a done_t via gasnet_AMReplyShortM() - SIGNAL_LONG, // ack to a done_t via gasnet_AMReplyLongM() - DO_REPLY_PUT = 143, // do a PUT here from another locale - DO_COPY_PAYLOAD // copy AM payload to another address +typedef enum +{ + SIGNAL = 136, // ack to a done_t via gasnet_AMReplyShortM() + SIGNAL_LONG, // ack to a done_t via gasnet_AMReplyLongM() + DO_REPLY_PUT = 143, // do a PUT here from another locale + DO_COPY_PAYLOAD // copy AM payload to another address } AM_handler_function_idx_t; -typedef struct { - void* ack; // acknowledgement object - void* tgt; // target memory address - void* src; // source memory address - size_t size; // number of bytes. +typedef struct +{ + void* ack; // acknowledgement object + void* tgt; // target memory address + void* src; // source memory address + size_t size; // number of bytes. } xfer_info_t; // Gasnet AM handler arguments are only 32 bits, so here we have // functions to get the 2 arguments for a 64-bit pointer, // and a function to reconstitute the pointer from the 2 arguments. // -static inline -gasnet_handlerarg_t get_arg_from_ptr0(uintptr_t addr) +static inline gasnet_handlerarg_t get_arg_from_ptr0(uintptr_t addr) { - // This one returns the bottom 32 bits. - return ((gasnet_handlerarg_t) - ((((uint64_t) (addr)) << 32UL) >> 32UL)); + // This one returns the bottom 32 bits. + return ((gasnet_handlerarg_t)((((uint64_t)(addr)) << 32UL) >> 32UL)); } -static inline -gasnet_handlerarg_t get_arg_from_ptr1(uintptr_t addr) +static inline gasnet_handlerarg_t get_arg_from_ptr1(uintptr_t addr) { - // this one returns the top 32 bits. - return ((gasnet_handlerarg_t) - (((uint64_t) (addr)) >> 32UL)); + // this one returns the top 32 bits. + return ((gasnet_handlerarg_t)(((uint64_t)(addr)) >> 32UL)); } -static inline -uintptr_t get_uintptr_from_args(gasnet_handlerarg_t a0, gasnet_handlerarg_t a1 ) +static inline uintptr_t get_uintptr_from_args( + gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { - return (uintptr_t) - (((uint64_t) (uint32_t) a0) - | (((uint64_t) (uint32_t) a1) << 32UL)); + return (uintptr_t)( + ((uint64_t)(uint32_t) a0) | (((uint64_t)(uint32_t) a1) << 32UL)); } -static inline -void* get_ptr_from_args(gasnet_handlerarg_t a0, gasnet_handlerarg_t a1 ) +static inline void* get_ptr_from_args( + gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { - return (void*) get_uintptr_from_args(a0, a1); + return (void*) get_uintptr_from_args(a0, a1); } // Build acknowledgement address arguments for gasnetAMRequest*() calls. // -#define Arg0(addr) get_arg_from_ptr0((uintptr_t)addr) -#define Arg1(addr) get_arg_from_ptr1((uintptr_t)addr) +#define Arg0(addr) get_arg_from_ptr0((uintptr_t) addr) +#define Arg1(addr) get_arg_from_ptr1((uintptr_t) addr) // The following macro is from the GASNet test.h distribution // -#define GASNET_Safe(fncall) do { \ - int _retval; \ - if ((_retval = fncall) != GASNET_OK) { \ - fprintf(stderr, "ERROR calling: %s\n" \ - " at: %s:%i\n" \ - " error: %s (%s)\n", \ - #fncall, __FILE__, __LINE__, \ - gasnet_ErrorName(_retval), gasnet_ErrorDesc(_retval)); \ - fflush(stderr); \ - gasnet_exit(_retval); \ - } \ - } while(0) +#define GASNET_Safe(fncall) \ + do \ + { \ + int _retval; \ + if ((_retval = fncall) != GASNET_OK) \ + { \ + fprintf(stderr, \ + "ERROR calling: %s\n" \ + " at: %s:%i\n" \ + " error: %s (%s)\n", \ + #fncall, __FILE__, __LINE__, gasnet_ErrorName(_retval), \ + gasnet_ErrorDesc(_retval)); \ + fflush(stderr); \ + gasnet_exit(_retval); \ + } \ + } while (0) // This is the type of object we use to manage GASNet acknowledgements. // @@ -100,101 +101,105 @@ void* get_ptr_from_args(gasnet_handlerarg_t a0, gasnet_handlerarg_t a1 ) // the condition for a BLOCKUNTIL can only be changed by the execution // of an AM handler.) // -typedef struct { - std::atomic count; - std::uint32_t target; - volatile int flag; +typedef struct +{ + std::atomic count; + std::uint32_t target; + volatile int flag; } done_t; -static void AM_signal(gasnet_token_t token, gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { - done_t* done = (done_t*) get_ptr_from_args(a0, a1); - uint_least32_t prev; - prev = done->count.fetch_add(1, std::memory_order_seq_cst); - if (prev + 1 == done->target) - done->flag = 1; +static void AM_signal( + gasnet_token_t token, gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) +{ + done_t* done = (done_t*) get_ptr_from_args(a0, a1); + uint_least32_t prev; + prev = done->count.fetch_add(1, std::memory_order_seq_cst); + if (prev + 1 == done->target) + done->flag = 1; } -static void AM_signal_long(gasnet_token_t token, void *buf, size_t nbytes, - gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { - done_t* done = (done_t*) get_ptr_from_args(a0, a1); - uint_least32_t prev; - prev = done->count.fetch_add(1, std::memory_order_seq_cst); - if (prev + 1 == done->target) - done->flag = 1; +static void AM_signal_long(gasnet_token_t token, void* buf, size_t nbytes, + gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) +{ + done_t* done = (done_t*) get_ptr_from_args(a0, a1); + uint_least32_t prev; + prev = done->count.fetch_add(1, std::memory_order_seq_cst); + if (prev + 1 == done->target) + done->flag = 1; } // Put from arg->src (which is local to the AM handler) back to // arg->dst (which is local to the caller of this AM). // nbytes is < gasnet_AMMaxLongReply here (see chpl_comm_get). // -static void AM_reply_put(gasnet_token_t token, void* buf, size_t nbytes) { - xfer_info_t* x = static_cast(buf); +static void AM_reply_put(gasnet_token_t token, void* buf, size_t nbytes) +{ + xfer_info_t* x = static_cast(buf); - assert(nbytes == sizeof(xfer_info_t)); + assert(nbytes == sizeof(xfer_info_t)); - GASNET_Safe(gasnet_AMReplyLong2(token, SIGNAL_LONG, - x->src, x->size, x->tgt, - Arg0(x->ack), Arg1(x->ack))); + GASNET_Safe(gasnet_AMReplyLong2(token, SIGNAL_LONG, x->src, x->size, x->tgt, + Arg0(x->ack), Arg1(x->ack))); } // Copy from the payload in this active message to dst. // -static -void AM_copy_payload(gasnet_token_t token, void* buf, size_t nbytes, - gasnet_handlerarg_t ack0, gasnet_handlerarg_t ack1, - gasnet_handlerarg_t dst0, gasnet_handlerarg_t dst1) +static void AM_copy_payload(gasnet_token_t token, void* buf, size_t nbytes, + gasnet_handlerarg_t ack0, gasnet_handlerarg_t ack1, + gasnet_handlerarg_t dst0, gasnet_handlerarg_t dst1) { - void* dst = get_ptr_from_args(dst0, dst1); - { - // would prefer to protect the memory segments - // associated with each node (n-node mutex) - // will require future work - // - std::lock_guard lk(hpx::util::gasnet_environment::dshm_mut); - std::memcpy(dst, buf, nbytes); - } - - GASNET_Safe(gasnet_AMReplyShort2(token, SIGNAL, ack0, ack1)); + void* dst = get_ptr_from_args(dst0, dst1); + { + // would prefer to protect the memory segments + // associated with each node (n-node mutex) + // will require future work + // + std::lock_guard lk(hpx::util::gasnet_environment::dshm_mut); + std::memcpy(dst, buf, nbytes); + } + + GASNET_Safe(gasnet_AMReplyShort2(token, SIGNAL, ack0, ack1)); } -static gasnet_handlerentry_t ftable[] = { - {SIGNAL, (void (*)())&AM_signal}, - {SIGNAL_LONG, (void (*)())&AM_signal_long}, - {DO_REPLY_PUT, (void (*)())&AM_reply_put}, - {DO_COPY_PAYLOAD, (void (*)())&AM_copy_payload} -}; +static gasnet_handlerentry_t ftable[] = {{SIGNAL, (void (*)()) & AM_signal}, + {SIGNAL_LONG, (void (*)()) & AM_signal_long}, + {DO_REPLY_PUT, (void (*)()) & AM_reply_put}, + {DO_COPY_PAYLOAD, (void (*)()) & AM_copy_payload}}; // // Initialize one of the above. // -static inline -void init_done_obj(done_t* done, int target) { - done->count.store(0, std::memory_order_seq_cst); - done->target = target; - done->flag = 0; +static inline void init_done_obj(done_t* done, int target) +{ + done->count.store(0, std::memory_order_seq_cst); + done->target = target; + done->flag = 0; } -static inline void am_poll_try() { - // Serialize polling for IBV, UCX, Aries, and OFI. Concurrent polling causes - // contention in these configurations. For other configurations that are - // AM-based (udp/amudp, mpi/ammpi) serializing can hurt performance. - // -#if defined(GASNET_CONDUIT_IBV) || defined(GASNET_CONDUIT_UCX) || defined(GASNET_CONDUIT_ARIES) || defined(GASNET_CONDUIT_OFI) - std::lock_guard lk(hpx::util::gasnet_environment::pollingLock); +static inline void am_poll_try() +{ + // Serialize polling for IBV, UCX, Aries, and OFI. Concurrent polling causes + // contention in these configurations. For other configurations that are + // AM-based (udp/amudp, mpi/ammpi) serializing can hurt performance. + // +#if defined(GASNET_CONDUIT_IBV) || defined(GASNET_CONDUIT_UCX) || \ + defined(GASNET_CONDUIT_ARIES) || defined(GASNET_CONDUIT_OFI) + std::lock_guard lk( + hpx::util::gasnet_environment::pollingLock); (void) gasnet_AMPoll(); #else (void) gasnet_AMPoll(); #endif } -static inline -void wait_done_obj(done_t* done, bool do_yield) +static inline void wait_done_obj(done_t* done, bool do_yield) { - while (!done->flag) { - am_poll_try(); - if (do_yield) - hpx::this_thread::yield(); - } + while (!done->flag) + { + am_poll_try(); + if (do_yield) + hpx::this_thread::yield(); + } } /////////////////////////////////////////////////////////////////////////////// @@ -223,8 +228,9 @@ namespace hpx { namespace util { } } - LBT_(info) << "No known GASNET environment variable found, disabling " - "GASNET support\n"; + LBT_(info) + << "No known GASNET environment variable found, disabling " + "GASNET support\n"; return false; } } // namespace detail @@ -238,7 +244,7 @@ namespace hpx { namespace util { // - The parcelport is explicitly disabled // - The application is not run in an GASNET environment // - The TCP parcelport is enabled and has higher priority - // + // if (get_entry_as(cfg, "hpx.parcel.gasnet.enable", 1) == 0 || (get_entry_as(cfg, "hpx.parcel.tcp.enable", 1) && (get_entry_as(cfg, "hpx.parcel.tcp.priority", 1) > @@ -247,7 +253,8 @@ namespace hpx { namespace util { (get_entry_as(cfg, "hpx.parcel.gasnet.priority", 1) > get_entry_as(cfg, "hpx.parcel.mpi.priority", 0)))) { - LBT_(info) << "GASNET support disabled via configuration settings\n"; + LBT_(info) + << "GASNET support disabled via configuration settings\n"; return false; } @@ -272,59 +279,69 @@ namespace hpx { namespace util { gasnet_seginfo_t* gasnet_environment::segments = nullptr; /////////////////////////////////////////////////////////////////////////// - int gasnet_environment::init( - int* argc, char*** argv, const int minimal, const int required, int& provided) + int gasnet_environment::init(int* argc, char*** argv, const int minimal, + const int required, int& provided) { - if(!has_called_init_) { - gasnet_environment::init_val_ = gasnet_init(argc, argv); - has_called_init_ = true; + if (!has_called_init_) + { + gasnet_environment::init_val_ = gasnet_init(argc, argv); + has_called_init_ = true; } - if(gasnet_environment::init_val_ == GASNET_ERR_NOT_INIT) { - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET initialization error"); + if (gasnet_environment::init_val_ == GASNET_ERR_NOT_INIT) + { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET initialization error"); } - else if(gasnet_environment::init_val_ == GASNET_ERR_RESOURCE) { - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET resource error"); + else if (gasnet_environment::init_val_ == GASNET_ERR_RESOURCE) + { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", "GASNET resource error"); } - else if(gasnet_environment::init_val_ == GASNET_ERR_BAD_ARG) { - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET bad argument error"); + else if (gasnet_environment::init_val_ == GASNET_ERR_BAD_ARG) + { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET bad argument error"); } - else if(gasnet_environment::init_val_ == GASNET_ERR_NOT_READY) { - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET not ready error"); + else if (gasnet_environment::init_val_ == GASNET_ERR_NOT_READY) + { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET not ready error"); } - if (provided < minimal) { - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET doesn't provide minimal requested thread level"); + if (provided < minimal) + { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET doesn't provide minimal requested thread level"); } - if(gasnet_attach(nullptr, 0, gasnet_getMaxLocalSegmentSize(), 0) != GASNET_OK) { - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET failed to attach to memory"); + if (gasnet_attach(nullptr, 0, gasnet_getMaxLocalSegmentSize(), 0) != + GASNET_OK) + { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET failed to attach to memory"); } // create a number of segments equal to the number of hardware // threads per machine (locality) // gasnet_environment::segments = new gasnet_seginfo_t[size()]; - gasnet_environment::segment_mutex = new hpx::mutex[size()]; + gasnet_environment::segment_mutex = new hpx::mutex[size()]; GASNET_Safe(gasnet_getSegmentInfo(segments, size())); int retval; gasnet_barrier_notify(0, GASNET_BARRIERFLAG_ANONYMOUS); - while ((retval = gasnet_barrier_try(0, GASNET_BARRIERFLAG_ANONYMOUS)) == GASNET_ERR_NOT_READY) {} + while ((retval = gasnet_barrier_try(0, GASNET_BARRIERFLAG_ANONYMOUS)) == + GASNET_ERR_NOT_READY) + { + } gasnet_barrier_try(0, GASNET_BARRIERFLAG_ANONYMOUS); gasnet_set_waitmode(GASNET_WAIT_BLOCK); @@ -364,8 +381,8 @@ namespace hpx { namespace util { int msglen = 0; char message[1024 + 1]; - std::snprintf(message, 1024+1, "%s\n", gasnet_ErrorDesc(retval)); - msglen = strnlen(message, 1025+1); + std::snprintf(message, 1024 + 1, "%s\n", gasnet_ErrorDesc(retval)); + msglen = strnlen(message, 1025 + 1); std::string msg("gasnet_environment::init: gasnet_init failed: "); msg = msg + message + "."; @@ -389,10 +406,12 @@ namespace hpx { namespace util { gasnet_environment::segment_mutex = new hpx::mutex[size()]; #if defined(HPX_HAVE_NETWORKING) - if (this_rank == 0) { + if (this_rank == 0) + { rtcfg.mode_ = hpx::runtime_mode::console; } - else { + else + { rtcfg.mode_ = hpx::runtime_mode::worker; } #elif defined(HPX_HAVE_DISTRIBUTED_RUNTIME) @@ -402,7 +421,8 @@ namespace hpx { namespace util { #endif rtcfg.add_entry("hpx.parcel.gasnet.rank", std::to_string(this_rank)); - rtcfg.add_entry("hpx.parcel.gasnet.processorname", get_processor_name()); + rtcfg.add_entry( + "hpx.parcel.gasnet.processorname", get_processor_name()); } std::string gasnet_environment::get_processor_name() @@ -410,51 +430,60 @@ namespace hpx { namespace util { char name[1024 + 1] = {'\0'}; const std::string rnkstr = std::to_string(rank()); const int len = rnkstr.size(); - if(1025 < len) { - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::get_processor_name", - "GASNET processor name is larger than 1025"); + if (1025 < len) + { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::get_processor_name", + "GASNET processor name is larger than 1025"); } std::copy(std::begin(rnkstr), std::end(rnkstr), name); return name; } - bool gasnet_environment::gettable(const int node, void* start, const size_t len) + bool gasnet_environment::gettable( + const int node, void* start, const size_t len) { - const uintptr_t segstart = (uintptr_t) gasnet_environment::segments[node].addr; - const uintptr_t segend = segstart + gasnet_environment::segments[node].size; + const uintptr_t segstart = + (uintptr_t) gasnet_environment::segments[node].addr; + const uintptr_t segend = + segstart + gasnet_environment::segments[node].size; const uintptr_t reqstart = (uintptr_t) start; const uintptr_t reqend = reqstart + len; - return ( segstart <= reqstart && - reqstart <= segend && - segstart <= reqend && - reqend <= segend); + return (segstart <= reqstart && reqstart <= segend && + segstart <= reqend && reqend <= segend); } - void gasnet_environment::put(std::uint8_t * addr, const int node, std::uint8_t * raddr, const std::size_t size) { + void gasnet_environment::put(std::uint8_t* addr, const int node, + std::uint8_t* raddr, const std::size_t size) + { const bool in_remote_seg = gettable(node, raddr, size); - if(in_remote_seg) { + if (in_remote_seg) + { const std::lock_guard lk(segment_mutex[node]); - gasnet_put(node, static_cast(raddr), static_cast(addr), size); + gasnet_put(node, static_cast(raddr), + static_cast(addr), size); } - else { + else + { // tell the remote node to copy the data being sent // size_t max_chunk = gasnet_AMMaxMedium(); size_t start = 0; // AMRequestMedium will send put; the active message handler - // will memcpy on the remote host + // will memcpy on the remote host // - for(start = 0; start < size; start += max_chunk) { + for (start = 0; start < size; start += max_chunk) + { size_t this_size; void* addr_chunk; void* raddr_chunk; done_t done; this_size = size - start; - if( this_size > max_chunk ) { + if (this_size > max_chunk) + { this_size = max_chunk; } @@ -466,10 +495,8 @@ namespace hpx { namespace util { // Send an AM over to ask for a them to copy the data // passed in the active message (addr_chunk) to raddr_chunk. GASNET_Safe(gasnet_AMRequestMedium4(node, DO_COPY_PAYLOAD, - addr_chunk, this_size, - Arg0(&done), Arg1(&done), - Arg0(raddr_chunk), - Arg1(raddr_chunk))); + addr_chunk, this_size, Arg0(&done), Arg1(&done), + Arg0(raddr_chunk), Arg1(raddr_chunk))); // Wait for the PUT to complete. wait_done_obj(&done, false); @@ -477,13 +504,17 @@ namespace hpx { namespace util { } } - void gasnet_environment::get(std::uint8_t * addr, const int node, std::uint8_t * raddr, const std::size_t size) { + void gasnet_environment::get(std::uint8_t* addr, const int node, + std::uint8_t* raddr, const std::size_t size) + { bool remote_in_segment = false; - if (rank() == node) { + if (rank() == node) + { std::memmove(addr, raddr, size); } - else { + else + { // Handle remote address not in remote segment. // The GASNet Spec says: // The source memory address for all gets and the target memory address @@ -496,13 +527,15 @@ namespace hpx { namespace util { // remote_in_segment = gettable(node, raddr, size); - if( remote_in_segment ) { + if (remote_in_segment) + { // If raddr is in the remote segment, do a normal gasnet_get. // GASNet will handle the local portion not being in the segment. // - gasnet_get(addr, node, raddr, size); // dest, node, src, size + gasnet_get(addr, node, raddr, size); // dest, node, src, size } - else { + else + { // If raddr is not in the remote segment, we need to send an // active message; the other node will PUT back to us. // The local side has to be in the registered memory segment. @@ -518,9 +551,11 @@ namespace hpx { namespace util { // do the GET into a temporary buffer instead, and then // copy the result back. // - if( ! local_in_segment ) { + if (!local_in_segment) + { size_t buf_sz = size; - if( buf_sz > max_chunk ) { + if (buf_sz > max_chunk) + { buf_sz = max_chunk; } @@ -531,14 +566,16 @@ namespace hpx { namespace util { // do a PUT on the remote locale back to here. // But do it in chunks of size gasnet_AMMaxLongReply() // since we use gasnet_AMReplyLong to do the PUT. - for(start = 0; start < size; start += max_chunk) { + for (start = 0; start < size; start += max_chunk) + { size_t this_size; void* addr_chunk; xfer_info_t info; done_t done; this_size = size - start; - if( this_size > max_chunk ) { + if (this_size > max_chunk) + { this_size = max_chunk; } @@ -547,25 +584,27 @@ namespace hpx { namespace util { init_done_obj(&done, 1); info.ack = &done; - info.tgt = local_buf?local_buf:addr_chunk; + info.tgt = local_buf ? local_buf : addr_chunk; info.src = ((char*) raddr) + start; info.size = this_size; // Send an AM over to ask for a PUT back to us - GASNET_Safe(gasnet_AMRequestMedium0(node, DO_REPLY_PUT, - &info, sizeof(info))); + GASNET_Safe(gasnet_AMRequestMedium0( + node, DO_REPLY_PUT, &info, sizeof(info))); // Wait for the PUT to complete. wait_done_obj(&done, false); // Now copy from local_buf back to addr if necessary. - if( local_buf ) { + if (local_buf) + { std::memcpy(addr_chunk, local_buf, this_size); } } // If we were using a temporary local buffer free it - if( local_buf ) { + if (local_buf) + { free(local_buf); } } @@ -574,7 +613,8 @@ namespace hpx { namespace util { void gasnet_environment::finalize() { - if (enabled() && has_called_init()) { + if (enabled() && has_called_init()) + { gasnet_exit(1); } } diff --git a/libs/core/gasnet_base/tests/CMakeLists.txt b/libs/core/gasnet_base/tests/CMakeLists.txt index e6593cda9ac9..765589ee57bd 100644 --- a/libs/core/gasnet_base/tests/CMakeLists.txt +++ b/libs/core/gasnet_base/tests/CMakeLists.txt @@ -10,7 +10,9 @@ include(HPX_Option) if(HPX_WITH_TESTS) if(HPX_WITH_TESTS_UNIT) add_hpx_pseudo_target(tests.unit.modules.gasnet_base) - add_hpx_pseudo_dependencies(tests.unit.modules tests.unit.modules.gasnet_base) + add_hpx_pseudo_dependencies( + tests.unit.modules tests.unit.modules.gasnet_base + ) add_subdirectory(unit) endif() diff --git a/libs/full/command_line_handling/CMakeLists.txt b/libs/full/command_line_handling/CMakeLists.txt index 1247a71b61de..966ba221406e 100644 --- a/libs/full/command_line_handling/CMakeLists.txt +++ b/libs/full/command_line_handling/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2022 The STE||AR-Group +# Copyright (c) 2019-2023 The STE||AR-Group # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -34,18 +34,13 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) hpx_setup_lci() set(additional_dependencies ${additional_dependencies} LCI::LCI) endif() -if(HPX_WITH_PARCELPORT_GASNET_MPI) - set(additional_dependencies ${additional_dependencies} Mpi::mpi PkgConfig::GASNET) -elseif(HPX_WITH_PARCELPORT_GASNET_UDP) - set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) -elseif(HPX_WITH_PARCELPORT_GASNET_SMP) - set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) -elseif(HPX_WITH_PARCELPORT_GASNET_OFI) - set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) -elseif(HPX_WITH_PARCELPORT_GASNET_UCX) - set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) -elseif(HPX_WITH_PARCELPORT_GASNET_IBV) - set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) +if(HPX_WITH_PARCELPORT_GASNET) + include(HPX_SetupGasnet) + hpx_setup_gasnet() + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") + set(additional_dependencies ${additional_dependencies} Mpi::mpi) + endif() + set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) endif() include(HPX_AddModule) diff --git a/libs/full/parcelport_gasnet/CMakeLists.txt b/libs/full/parcelport_gasnet/CMakeLists.txt index 1cf8df58efab..50c4380e81f2 100644 --- a/libs/full/parcelport_gasnet/CMakeLists.txt +++ b/libs/full/parcelport_gasnet/CMakeLists.txt @@ -1,59 +1,48 @@ -# Copyright (c) 2019-2021 The STE||AR-Group +# Copyright (c) 2023 Chistopher Taylor # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_CONDUIT)) +if(NOT + (HPX_WITH_NETWORKING + AND HPX_WITH_PARCELPORT_GASNET + AND HPX_WITH_PARCELPORT_GASNET_CONDUIT) +) return() endif() -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") -include(FindGasnet) -find_gasnet() +include(HPX_SetupGasnet) +hpx_setup_gasnet() set(parcelport_gasnet_headers hpx/parcelport_gasnet/header.hpp hpx/parcelport_gasnet/locality.hpp hpx/parcelport_gasnet/receiver.hpp + hpx/parcelport_gasnet/receiver_connection.hpp hpx/parcelport_gasnet/sender.hpp hpx/parcelport_gasnet/sender_connection.hpp + hpx/parcelport_gasnet/tag_provider.hpp ) -# cmake-format: off -set(parcelport_gasnet_compat_headers) -# cmake-format: on - set(parcelport_gasnet_sources locality.cpp parcelport_gasnet.cpp) -if(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "mpi") -link_directories(${GASNET_LIBRARY_DIRS}) - include(HPX_AddModule) - add_hpx_module( - full parcelport_gasnet - GLOBAL_HEADER_GEN ON - SOURCES ${parcelport_gasnet_sources} - HEADERS ${parcelport_gasnet_headers} - COMPAT_HEADERS ${parcelport_gasnet_compat_headers} - DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET Mpi::mpi - MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - CMAKE_SUBDIRS examples tests - ) -else() -link_directories(${GASNET_LIBRARY_DIRS}) - include(HPX_AddModule) - add_hpx_module( - full parcelport_gasnet - GLOBAL_HEADER_GEN ON - SOURCES ${parcelport_gasnet_sources} - HEADERS ${parcelport_gasnet_headers} - COMPAT_HEADERS ${parcelport_gasnet_compat_headers} - DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET - MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - CMAKE_SUBDIRS examples tests - ) +if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") + set(gasnet_additional_dependencies Mpi::mpi) endif() +include(HPX_AddModule) +add_hpx_module( + full parcelport_gasnet + GLOBAL_HEADER_GEN ON + SOURCES ${parcelport_gasnet_sources} + HEADERS ${parcelport_gasnet_headers} + DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET + ${gasnet_additional_dependencies} + MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset + CMAKE_SUBDIRS examples tests +) + set(HPX_STATIC_PARCELPORT_PLUGINS ${HPX_STATIC_PARCELPORT_PLUGINS} parcelport_gasnet CACHE INTERNAL "" FORCE diff --git a/libs/full/parcelport_gasnet/examples/CMakeLists.txt b/libs/full/parcelport_gasnet/examples/CMakeLists.txt index 57dd90f4214e..24c8de88643c 100644 --- a/libs/full/parcelport_gasnet/examples/CMakeLists.txt +++ b/libs/full/parcelport_gasnet/examples/CMakeLists.txt @@ -6,7 +6,9 @@ if(HPX_WITH_EXAMPLES) add_hpx_pseudo_target(examples.modules.parcelport_gasnet) - add_hpx_pseudo_dependencies(examples.modules examples.modules.parcelport_gasnet) + add_hpx_pseudo_dependencies( + examples.modules examples.modules.parcelport_gasnet + ) if(HPX_WITH_TESTS AND HPX_WITH_TESTS_EXAMPLES) add_hpx_pseudo_target(tests.examples.modules.parcelport_gasnet) add_hpx_pseudo_dependencies( diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp index d0282a2b0ed7..886118b244cf 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2023 Christopher Taylor // Copyright (c) 2013-2021 Hartmut Kaiser // Copyright (c) 2013-2015 Thomas Heller // diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp index 82c9b6c97892..98d315b97c5b 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp @@ -11,8 +11,8 @@ #include #if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) -#include #include +#include #include @@ -43,7 +43,7 @@ namespace hpx::parcelset::policies::gasnet { explicit constexpr operator bool() const noexcept { - return rank_ != ((unsigned int)-1); + return rank_ != ((unsigned int) -1); } HPX_EXPORT void save(serialization::output_archive& ar) const; diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp index 6797de62e7bb..5d81749e0b7f 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -24,7 +25,6 @@ #include #include #include -#include namespace hpx::parcelset::policies::gasnet { @@ -37,23 +37,29 @@ namespace hpx::parcelset::policies::gasnet { using connection_ptr = std::shared_ptr; using connection_list = std::deque; - struct exp_backoff { + struct exp_backoff + { int numTries; const static int maxRetries = 10; - void operator()() { - if(numTries <= maxRetries) { + void operator()() + { + if (numTries <= maxRetries) + { gasnet_AMPoll(); - hpx::this_thread::suspend(std::chrono::microseconds(1 << numTries)); + hpx::this_thread::suspend( + std::chrono::microseconds(1 << numTries)); } - else { + else + { numTries = 0; } } }; explicit constexpr receiver(Parcelport& pp) noexcept - : pp_(pp), bo() + : pp_(pp) + , bo() { } @@ -122,7 +128,8 @@ namespace hpx::parcelset::policies::gasnet { // remote localities 'put' into the gasnet shared memory segment on this machine // - res.reset(new connection_type(hpx::util::gasnet_environment::rank(), h, pp_)); + res.reset(new connection_type( + hpx::util::gasnet_environment::rank(), h, pp_)); return res; } return res; @@ -133,8 +140,9 @@ namespace hpx::parcelset::policies::gasnet { header h = rcv_header_; rcv_header_.reset(); - while(rcv_header_.data() == 0) { - bo(); + while (rcv_header_.data() == 0) + { + bo(); } return h; diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp index edc74460fb17..512b18339c24 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp @@ -138,8 +138,8 @@ namespace hpx::parcelset::policies::gasnet { auto self_ = hpx::util::gasnet_environment::rank(); hpx::util::gasnet_environment::scoped_lock l; std::memcpy(buffer_.data_.data(), - hpx::util::gasnet_environment::segments[self_].addr, - buffer_.data_.size()); + hpx::util::gasnet_environment::segments[self_].addr, + buffer_.data_.size()); request_ptr_ = true; } @@ -150,7 +150,6 @@ namespace hpx::parcelset::policies::gasnet { bool receive_chunks(std::size_t num_thread = -1) { - while (chunks_idx_ < buffer_.chunks_.size()) { if (!request_done()) @@ -168,8 +167,8 @@ namespace hpx::parcelset::policies::gasnet { auto self_ = hpx::util::gasnet_environment::rank(); hpx::util::gasnet_environment::scoped_lock l; std::memcpy(c.data(), - hpx::util::gasnet_environment::segments[self_].addr, - c.size()); + hpx::util::gasnet_environment::segments[self_].addr, + c.size()); request_ptr_ = true; } } @@ -193,8 +192,8 @@ namespace hpx::parcelset::policies::gasnet { auto self_ = hpx::util::gasnet_environment::rank(); hpx::util::gasnet_environment::scoped_lock l; std::memcpy(&tag_, - hpx::util::gasnet_environment::segments[self_].addr, - sizeof(int)); + hpx::util::gasnet_environment::segments[self_].addr, + sizeof(int)); request_ptr_ = true; } diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp index 744ac3a6ab95..faee1482d804 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp @@ -140,10 +140,12 @@ namespace hpx::parcelset::policies::gasnet { int get_next_free_tag() noexcept { int next_free = next_free_tag_; - + hpx::util::gasnet_environment::scoped_lock l; std::memcpy(&next_free, - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, + hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr, sizeof(int)); return next_free; diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp index c7b7b3d84b80..1520400b41f9 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp @@ -16,13 +16,13 @@ #include #include +#include #include #include #include #include #include #include -#include #include #include @@ -152,25 +152,27 @@ namespace hpx::parcelset::policies::gasnet { // compute + send the number of GASNET_PAGEs to send and the remainder number of bytes to a GASNET_PAGE // - const std::size_t chunks[] = { static_cast(header_.data_size_ / GASNET_PAGESIZE), static_cast(header_.data_size_ % GASNET_PAGESIZE) }; + const std::size_t chunks[] = { + static_cast(header_.data_size_ / GASNET_PAGESIZE), + static_cast(header_.data_size_ % GASNET_PAGESIZE)}; const std::size_t sizeof_chunks = sizeof(chunks); - std::memcpy( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, - chunks, sizeof_chunks); + std::memcpy(hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr, + chunks, sizeof_chunks); // put from this localities gasnet shared memory segment // into the remote locality (dst_)'s shared memory segment // hpx::util::gasnet_environment::put( static_cast( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr - ), + hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr), dst_, static_cast( - hpx::util::gasnet_environment::segments[dst_].addr - ), - sizeof_chunks - ); + hpx::util::gasnet_environment::segments[dst_].addr), + sizeof_chunks); } state_ = sent_header; @@ -190,22 +192,22 @@ namespace hpx::parcelset::policies::gasnet { if (!chunks.empty()) { hpx::util::gasnet_environment::scoped_lock l; - std::memcpy( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, - chunks.data(), - static_cast(chunks.size() * - sizeof(parcel_buffer_type::transmission_chunk_type)) - ); - - gasnet_put_bulk(dst_, + std::memcpy(hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr, + chunks.data(), + static_cast(chunks.size() * + sizeof(parcel_buffer_type::transmission_chunk_type))); + + gasnet_put_bulk(dst_, static_cast( - hpx::util::gasnet_environment::segments[dst_].addr - ), + hpx::util::gasnet_environment::segments[dst_].addr), static_cast( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr - ), - static_cast(chunks.size() * sizeof(parcel_buffer_type::transmission_chunk_type)) - ); + hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr), + static_cast(chunks.size() * + sizeof(parcel_buffer_type::transmission_chunk_type))); } state_ = sent_transmission_chunks; @@ -223,20 +225,20 @@ namespace hpx::parcelset::policies::gasnet { if (!header_.piggy_back()) { hpx::util::gasnet_environment::scoped_lock l; - std::memcpy( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, + std::memcpy(hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr, buffer_.data_.data(), buffer_.data_.size()); hpx::util::gasnet_environment::put( static_cast( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr - ), + hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr), dst_, static_cast( - hpx::util::gasnet_environment::segments[dst_].addr - ), - buffer_.data_.size() - ); + hpx::util::gasnet_environment::segments[dst_].addr), + buffer_.data_.size()); } state_ = sent_data; @@ -260,22 +262,20 @@ namespace hpx::parcelset::policies::gasnet { hpx::util::gasnet_environment::scoped_lock l; - std::memcpy( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, - c.data_.cpos_, - static_cast(c.size_) - ); + std::memcpy(hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr, + c.data_.cpos_, static_cast(c.size_)); hpx::util::gasnet_environment::put( static_cast( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr - ), + hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr), dst_, static_cast( - hpx::util::gasnet_environment::segments[dst_].addr - ), - static_cast(c.size_) - ); + hpx::util::gasnet_environment::segments[dst_].addr), + static_cast(c.size_)); } ++chunks_idx_; diff --git a/libs/full/parcelport_gasnet/src/locality.cpp b/libs/full/parcelport_gasnet/src/locality.cpp index 150ce6e82834..5e9e309c1375 100644 --- a/libs/full/parcelport_gasnet/src/locality.cpp +++ b/libs/full/parcelport_gasnet/src/locality.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2023 Christopher Taylor // Copyright (c) 2007-2021 Hartmut Kaiser // Copyright (c) 2013-2014 Thomas Heller // diff --git a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp index 728d107f6258..b29d21006ac6 100644 --- a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp +++ b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp @@ -106,7 +106,7 @@ namespace hpx::parcelset { static std::size_t background_threads( util::runtime_configuration const& ini) { -/* + /* return hpx::util::get_entry_as(ini, "hpx.parcel.gasnet.background_threads", HPX_HAVE_PARCELPORT_GASNET_BACKGROUND_THREADS); @@ -160,18 +160,20 @@ namespace hpx::parcelset { "gasnet::parcelport::do_stop"); } - bool expected = false; if (stopped_.compare_exchange_strong(expected, true)) { stopped_ = true; gasnet_barrier_notify(0, GASNET_BARRIERFLAG_ANONYMOUS); - if((retval = gasnet_barrier_wait(0, GASNET_BARRIERFLAG_ANONYMOUS)) != GASNET_OK) { + if ((retval = gasnet_barrier_wait( + 0, GASNET_BARRIERFLAG_ANONYMOUS)) != GASNET_OK) + { // throw exception HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET failed ", std::string{gasnet_ErrorName(retval)}, " ", - std::string{gasnet_ErrorDesc(retval)}); + "hpx::util::gasnet_environment::init", + "GASNET failed ", + std::string{gasnet_ErrorName(retval)}, " ", + std::string{gasnet_ErrorDesc(retval)}); gasnet_exit(retval); } } @@ -291,12 +293,14 @@ namespace hpx::traits { return "1"; } - static void init(int* argc, char*** argv, util::command_line_handling& cfg) + static void init( + int* argc, char*** argv, util::command_line_handling& cfg) { util::gasnet_environment::init(argc, argv, cfg.rtcfg_); cfg.num_localities_ = static_cast(util::gasnet_environment::size()); - cfg.node_ = static_cast(util::gasnet_environment::rank()); + cfg.node_ = + static_cast(util::gasnet_environment::rank()); } // by default no additional initialization using the resource