Skip to content

Commit

Permalink
Merge remote-tracking branch 'mainline/development' into heffte
Browse files Browse the repository at this point in the history
  • Loading branch information
@Haavaan
Haavaan committed Aug 14, 2024
2 parents 18dd29b + 502e5ea commit 70493c9
Show file tree
Hide file tree
Showing 6 changed files with 87 additions and 81 deletions.
12 changes: 7 additions & 5 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -711,7 +711,9 @@ endforeach()
# pip helpers for the pywarpx package #########################################
#
if(WarpX_PYTHON)
set(PYINSTALLOPTIONS "" CACHE STRING
set(PY_PIP_OPTIONS "-v" CACHE STRING
"Additional parameters to pass to `pip`")
set(PY_PIP_INSTALL_OPTIONS "" CACHE STRING
"Additional parameters to pass to `pip install`")

# ensure all targets are built before we package them in a wheel
Expand All @@ -735,7 +737,7 @@ if(WarpX_PYTHON)
${CMAKE_COMMAND} -E rm -f -r warpx-whl
COMMAND
${CMAKE_COMMAND} -E env PYWARPX_LIB_DIR=$<TARGET_FILE_DIR:pyWarpX_${WarpX_DIMS_LAST}>
${Python_EXECUTABLE} -m pip wheel -v --no-build-isolation --no-deps --wheel-dir=warpx-whl ${WarpX_SOURCE_DIR}
${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} wheel --no-build-isolation --no-deps --wheel-dir=warpx-whl ${WarpX_SOURCE_DIR}
WORKING_DIRECTORY
${WarpX_BINARY_DIR}
DEPENDS
Expand All @@ -749,7 +751,7 @@ if(WarpX_PYTHON)
set(pyWarpX_REQUIREMENT_FILE "requirements.txt")
endif()
add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_requirements
${Python_EXECUTABLE} -m pip install ${PYINSTALLOPTIONS} -r "${WarpX_SOURCE_DIR}/${pyWarpX_REQUIREMENT_FILE}"
${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install ${PY_PIP_INSTALL_OPTIONS} -r "${WarpX_SOURCE_DIR}/${pyWarpX_REQUIREMENT_FILE}"
WORKING_DIRECTORY
${WarpX_BINARY_DIR}
)
Expand All @@ -766,7 +768,7 @@ if(WarpX_PYTHON)
# because otherwise pip would also force reinstall all dependencies.
add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install
${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI}
${Python_EXECUTABLE} -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx
${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install --force-reinstall --no-index --no-deps ${PY_PIP_INSTALL_OPTIONS} --find-links=warpx-whl pywarpx
WORKING_DIRECTORY
${WarpX_BINARY_DIR}
DEPENDS
Expand All @@ -779,7 +781,7 @@ if(WarpX_PYTHON)
# this is for package managers only
add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_nodeps
${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI}
${Python_EXECUTABLE} -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx
${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install --force-reinstall --no-index --no-deps ${PY_PIP_INSTALL_OPTIONS} --find-links=warpx-whl pywarpx
WORKING_DIRECTORY
${WarpX_BINARY_DIR}
DEPENDS
Expand Down
6 changes: 4 additions & 2 deletions Docs/source/install/cmake.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,6 @@ CMake Option Default & Values Descr
``CMAKE_BUILD_TYPE`` RelWithDebInfo/**Release**/Debug `Type of build, symbols & optimizations <https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html>`__
``CMAKE_INSTALL_PREFIX`` system-dependent path `Install path prefix <https://cmake.org/cmake/help/latest/variable/CMAKE_INSTALL_PREFIX.html>`__
``CMAKE_VERBOSE_MAKEFILE`` ON/**OFF** `Print all compiler commands to the terminal during build <https://cmake.org/cmake/help/latest/variable/CMAKE_VERBOSE_MAKEFILE.html>`__
``PYINSTALLOPTIONS`` Additional options for ``pip install``, e.g., ``-v --user``
``WarpX_APP`` **ON**/OFF Build the WarpX executable application
``WarpX_ASCENT`` ON/**OFF** Ascent in situ visualization
``WarpX_COMPUTE`` NOACC/**OMP**/CUDA/SYCL/HIP On-node, accelerated computing backend
Expand All @@ -104,6 +103,9 @@ CMake Option Default & Values Descr
``WarpX_QED_TOOLS`` ON/**OFF** Build external tool to generate QED lookup tables (requires PICSAR and Boost)
``WarpX_QED_TABLES_GEN_OMP`` **AUTO**/ON/OFF Enables OpenMP support for QED lookup tables generation
``WarpX_SENSEI`` ON/**OFF** SENSEI in situ visualization
``Python_EXECUTABLE`` (newest found) Path to Python executable
``PY_PIP_OPTIONS`` ``-v`` Additional options for ``pip``, e.g., ``-vvv``
``PY_PIP_INSTALL_OPTIONS`` Additional options for ``pip install``, e.g., ``--user``
============================= ============================================ =========================================================

WarpX can be configured in further detail with options from AMReX, which are documented in the AMReX manual:
Expand Down Expand Up @@ -239,7 +241,7 @@ Developers could now change the WarpX source code and then call the build line a
.. tip::

If you do *not* develop with :ref:`a user-level package manager <install-dependencies>`, e.g., because you rely on a HPC system's environment modules, then consider to set up a virtual environment via `Python venv <https://docs.python.org/3/library/venv.html>`__.
Otherwise, without a virtual environment, you likely need to add the CMake option ``-DPYINSTALLOPTIONS="--user"``.
Otherwise, without a virtual environment, you likely need to add the CMake option ``-DPY_PIP_INSTALL_OPTIONS="--user"``.


.. _building-pip-python:
Expand Down
2 changes: 1 addition & 1 deletion Regression/WarpX-tests.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ branch = 24.08
dir = /home/regtester/AMReX_RegTesting/warpx
branch = development
cmakeSetupOpts = -DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON -DWarpX_PYTHON_IPO=OFF -DpyAMReX_IPO=OFF
# -DPYINSTALLOPTIONS="--disable-pip-version-check"
# -DPY_PIP_INSTALL_OPTIONS="--disable-pip-version-check"

# individual problems follow

Expand Down
141 changes: 73 additions & 68 deletions Source/Initialization/WarpXInitData.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,71 @@ namespace
WARPX_ALWAYS_ASSERT_WITH_MESSAGE(vc.allGT(gc), ss_msg.str());
}
}

/**
* \brief Check the requested resources and write performance hints
*
* @param[in] total_nboxes total number of boxes in the simulation
* @param[in] nprocs number of MPI processes
*/
void PerformanceHints (const amrex::Long total_nboxes, const amrex::Long nprocs)
{
// Check: are there more MPI ranks than Boxes?
if (nprocs > total_nboxes) {
std::stringstream warnMsg;
warnMsg << "Too many resources / too little work!\n"
<< " It looks like you requested more compute resources than "
<< "there are total number of boxes of cells available ("
<< total_nboxes << "). "
<< "You started with (" << nprocs
<< ") MPI ranks, so (" << nprocs - total_nboxes
<< ") rank(s) will have no work.\n"
#ifdef AMREX_USE_GPU
<< " On GPUs, consider using 1-8 boxes per GPU that together fill "
<< "each GPU's memory sufficiently. If you do not rely on dynamic "
<< "load-balancing, then one large box per GPU is ideal.\n"
#endif
<< "Consider decreasing the amr.blocking_factor and "
<< "amr.max_grid_size parameters and/or using fewer MPI ranks.\n"
<< " More information:\n"
<< " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n";

ablastr::warn_manager::WMRecordWarning(
"Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high);
}

#ifdef AMREX_USE_GPU
// Check: Are there more than 12 boxes per GPU?
if (total_nboxes > nprocs * 12) {
std::stringstream warnMsg;
warnMsg << "Too many boxes per GPU!\n"
<< " It looks like you split your simulation domain "
<< "in too many boxes (" << total_nboxes << "), which "
<< "results in an average number of ("
<< amrex::Long(total_nboxes/nprocs) << ") per GPU. "
<< "This causes severe overhead in the communication of "
<< "border/guard regions.\n"
<< " On GPUs, consider using 1-8 boxes per GPU that together fill "
<< "each GPU's memory sufficiently. If you do not rely on dynamic "
<< "load-balancing, then one large box per GPU is ideal.\n"
<< "Consider increasing the amr.blocking_factor and "
<< "amr.max_grid_size parameters and/or using more MPI ranks.\n"
<< " More information:\n"
<< " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n";

ablastr::warn_manager::WMRecordWarning(
"Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high);
}
#endif

// TODO: warn if some ranks have disproportionally more work than all others
// tricky: it can be ok to assign "vacuum" boxes to some ranks w/o slowing down
// all other ranks; we need to measure this with our load-balancing
// routines and issue a warning only of some ranks stall all other ranks
// TODO: check MPI-rank to GPU ratio (should be 1:1)
// TODO: check memory per MPI rank, especially if GPUs are underutilized
// TODO: CPU tiling hints with OpenMP
}
}

void
Expand Down Expand Up @@ -527,7 +592,14 @@ WarpX::InitData ()
}
}

PerformanceHints();
// Computes available boxes on all levels.
amrex::Long total_nboxes = 0;
for (int ilev = 0; ilev <= finestLevel(); ++ilev) {
total_nboxes += boxArray(ilev).size();
}
auto const nprocs = ParallelDescriptor::NProcs();

::PerformanceHints(total_nboxes, nprocs);

CheckKnownIssues();
}
Expand Down Expand Up @@ -1115,73 +1187,6 @@ WarpX::InitializeExternalFieldsOnGridUsingParser (
}
}

void
WarpX::PerformanceHints ()
{
// Check requested MPI ranks and available boxes
amrex::Long total_nboxes = 0; // on all MPI ranks
for (int ilev = 0; ilev <= finestLevel(); ++ilev) {
total_nboxes += boxArray(ilev).size();
}
auto const nprocs = ParallelDescriptor::NProcs();

// Check: are there more MPI ranks than Boxes?
if (nprocs > total_nboxes) {
std::stringstream warnMsg;
warnMsg << "Too many resources / too little work!\n"
<< " It looks like you requested more compute resources than "
<< "there are total number of boxes of cells available ("
<< total_nboxes << "). "
<< "You started with (" << nprocs
<< ") MPI ranks, so (" << nprocs - total_nboxes
<< ") rank(s) will have no work.\n"
#ifdef AMREX_USE_GPU
<< " On GPUs, consider using 1-8 boxes per GPU that together fill "
<< "each GPU's memory sufficiently. If you do not rely on dynamic "
<< "load-balancing, then one large box per GPU is ideal.\n"
#endif
<< "Consider decreasing the amr.blocking_factor and "
<< "amr.max_grid_size parameters and/or using fewer MPI ranks.\n"
<< " More information:\n"
<< " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n";

ablastr::warn_manager::WMRecordWarning(
"Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high);
}

#ifdef AMREX_USE_GPU
// Check: Are there more than 12 boxes per GPU?
if (total_nboxes > nprocs * 12) {
std::stringstream warnMsg;
warnMsg << "Too many boxes per GPU!\n"
<< " It looks like you split your simulation domain "
<< "in too many boxes (" << total_nboxes << "), which "
<< "results in an average number of ("
<< amrex::Long(total_nboxes/nprocs) << ") per GPU. "
<< "This causes severe overhead in the communication of "
<< "border/guard regions.\n"
<< " On GPUs, consider using 1-8 boxes per GPU that together fill "
<< "each GPU's memory sufficiently. If you do not rely on dynamic "
<< "load-balancing, then one large box per GPU is ideal.\n"
<< "Consider increasing the amr.blocking_factor and "
<< "amr.max_grid_size parameters and/or using more MPI ranks.\n"
<< " More information:\n"
<< " https://warpx.readthedocs.io/en/latest/usage/workflows/parallelization.html\n";

ablastr::warn_manager::WMRecordWarning(
"Performance", warnMsg.str(), ablastr::warn_manager::WarnPriority::high);
}
#endif

// TODO: warn if some ranks have disproportionally more work than all others
// tricky: it can be ok to assign "vacuum" boxes to some ranks w/o slowing down
// all other ranks; we need to measure this with our load-balancing
// routines and issue a warning only of some ranks stall all other ranks
// TODO: check MPI-rank to GPU ratio (should be 1:1)
// TODO: check memory per MPI rank, especially if GPUs are underutilized
// TODO: CPU tiling hints with OpenMP
}

void WarpX::CheckGuardCells()
{
for (int lev = 0; lev <= finest_level; ++lev)
Expand Down
3 changes: 0 additions & 3 deletions Source/WarpX.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1399,9 +1399,6 @@ private:
*/
void CheckKnownIssues();

/** Check the requested resources and write performance hints */
void PerformanceHints ();

void BuildBufferMasks ();

[[nodiscard]] const amrex::iMultiFab* getCurrentBufferMasks (int lev) const {
Expand Down
4 changes: 2 additions & 2 deletions run_test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ curl -sOL https://github.com/openPMD/openPMD-example-datasets/raw/4ba1d257c5b489
cd -

# Clone the AMReX regression test utility
git clone -b EZoni_rm_testdir https://github.com/EZoni/regression_testing.git
git clone https://github.com/AMReX-Codes/regression_testing.git

# Prepare regression tests
mkdir -p rt-WarpX/WarpX-benchmarks
Expand All @@ -99,7 +99,7 @@ echo "cd $PWD"
if [ -z "${WARPX_CI_CLEAN_TESTS}" ]; then
test_rm_dir=""
else
test_rm_dir="--rm_testdir"
test_rm_dir="--clean_testdir"
fi
# run only tests specified in variable tests_arg (single test or multiple tests)
if [[ ! -z "${tests_arg}" ]]; then
Expand Down

0 comments on commit 70493c9

Please sign in to comment.