From ced43dc9db3cfe11309bcbe68d414505286cb5b7 Mon Sep 17 00:00:00 2001 From: PhilipDeegan Date: Sun, 12 May 2024 22:43:03 +0200 Subject: [PATCH] Generate report zip (#801) * report zip gen * consistent cmake version required * bind to none for better // of multiple mpirun calls * report zip gen * more * correct python cmake var name * revert default configurator option * PR comments / move ompi bind to none to configurator * default report writing on / disabled if testing / simulator retval consistency --- .gitignore | 3 +- ISSUES.TXT | 18 +- pyphare/pyphare/core/__init__.py | 59 +++++++ pyphare/pyphare/cpp/__init__.py | 15 ++ pyphare/pyphare/cpp/validate.py | 109 ++++++++++++ .../pyphare/pharein/maxwellian_fluid_model.py | 3 +- pyphare/pyphare/pharein/simulation.py | 9 + pyphare/pyphare/simulator/simulator.py | 27 ++- res/cmake/def.cmake | 17 +- res/cmake/dep/samrai.cmake | 1 + src/amr/CMakeLists.txt | 2 +- src/amr/solvers/solver_ppc.hpp | 3 +- src/core/def/phare_config.hpp | 23 +++ src/core/def/phare_mpi.hpp | 6 +- src/hdf5/detail/h5/h5_file.hpp | 5 + src/python3/cpp_etc.cpp | 4 + tests/core/numerics/ohm/CMakeLists.txt | 2 +- tests/simulator/__init__.py | 8 +- tests/simulator/advance/__init__.py | 0 tests/simulator/initialize/__init__.py | 0 tests/simulator/refinement/__init__.py | 0 tests/simulator/test_validation.py | 4 +- tools/config/CMakeLists.txt | 3 + tools/config/cmake.sh | 21 +-- tools/config/config.py | 157 +++++++++++++----- tools/config/main.cpp | 23 ++- tools/report.py | 61 +++++++ tools/report.sh | 13 ++ 28 files changed, 514 insertions(+), 82 deletions(-) create mode 100644 pyphare/pyphare/cpp/validate.py create mode 100644 src/core/def/phare_config.hpp create mode 100644 tests/simulator/advance/__init__.py create mode 100644 tests/simulator/initialize/__init__.py create mode 100644 tests/simulator/refinement/__init__.py create mode 100644 tools/report.py create mode 100755 tools/report.sh diff --git a/.gitignore b/.gitignore index b270b8a47..e31337989 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ tools/cmake.sh perf.* **/*.h5 .vscode - +.phare* +PHARE_REPORT.zip diff --git a/ISSUES.TXT b/ISSUES.TXT index 175018faa..d4e3d3a9d 100644 --- a/ISSUES.TXT +++ b/ISSUES.TXT @@ -1,5 +1,20 @@ +# Having Issues + +Please run: + +> ./tools/report.sh # or "python3 tools/report.py" with the correct PYTHONPATH + +This will build a zip archive "REPORT_INFO.zip", which you should either email to us, or +Log an issue on github via https://github.com/PHAREHUB/PHARE/issues/new +Outline the context of your issue, and upload the zip + + + + +# Known Issues + 1. OMPI symbol resolution with python3. Affects: OMPI versions < 3 Source: https://github.com/open-mpi/ompi/issues/3705 @@ -10,4 +25,5 @@ 2. Python launching, with C++ MPI init, doesn't work so well with MPI4PY Source: https://bitbucket.org/mpi4py/mpi4py/issues/154/attempting-to-use-an-mpi-routine-before Solution: - LD_PRELOAD=/path/to/your/libmpi.so python3 $SCRIPT \ No newline at end of file + LD_PRELOAD=/path/to/your/libmpi.so python3 $SCRIPT + diff --git a/pyphare/pyphare/core/__init__.py b/pyphare/pyphare/core/__init__.py index e69de29bb..1e3f7500b 100644 --- a/pyphare/pyphare/core/__init__.py +++ b/pyphare/pyphare/core/__init__.py @@ -0,0 +1,59 @@ +# +# +# +# program help looks like +""" +usage: phare_sim.py [-h] [-d] + +options: + -h, --help show this help message and exit + -d, --dry-run Validate but do not run simulations +""" + +import sys +import dataclasses + + +def disabled_for_testing(): + # check if any module is loaded from PHARE tests directory + from pathlib import Path + + test_dir = Path(__file__).resolve().parent.parent.parent.parent / "tests" + if test_dir.exists(): + test_dir = str(test_dir) + for k, mod in sys.modules.items(): + if hasattr(mod, "__file__") and mod.__file__ and test_dir in mod.__file__: + return True + return False + + +@dataclasses.dataclass +class CliArgs: + dry_run: bool = dataclasses.field(default_factory=lambda: False) + write_reports: bool = dataclasses.field(default_factory=lambda: False) + + +def parse_cli_args(): + default_off = len(sys.argv) == 1 and disabled_for_testing() + if default_off: + return CliArgs() + + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "-d", + "--dry-run", + help="Validate but do not run simulations", + action="store_true", + default=False, + ) + parser.add_argument( + "-w", + "--write_reports", + help="Write build and runtime configs to disk", + action="store_true", + default=True, + ) + + return CliArgs(**vars(parser.parse_args())) diff --git a/pyphare/pyphare/cpp/__init__.py b/pyphare/pyphare/cpp/__init__.py index 223890426..79d0c8cf5 100644 --- a/pyphare/pyphare/cpp/__init__.py +++ b/pyphare/pyphare/cpp/__init__.py @@ -1,3 +1,10 @@ +# +# +# + + +import json + # continue to use override if set _cpp_lib_override = None @@ -25,6 +32,14 @@ def cpp_etc_lib(): return importlib.import_module("pybindlibs.cpp_etc") +def build_config(): + return cpp_etc_lib().phare_build_config() + + +def build_config_as_json(): + return json.dumps(build_config()) + + def splitter_type(dim, interp, n_particles): return getattr(cpp_lib(), f"Splitter_{dim}_{interp}_{n_particles}") diff --git a/pyphare/pyphare/cpp/validate.py b/pyphare/pyphare/cpp/validate.py new file mode 100644 index 000000000..1a614778b --- /dev/null +++ b/pyphare/pyphare/cpp/validate.py @@ -0,0 +1,109 @@ +""" +PHARE build and runtime validation checks +""" + +import os +import sys +import json +import dataclasses +from pathlib import Path + +from pyphare.core import phare_utilities +from pyphare import cpp + +DOT_PHARE_DIR = Path(os.getcwd()) / ".phare" + + +ERROR_MESSAGES = dict( + on_phare_config_error=""" + Warning: PHARE was not built with the configurator active""", + on_python3_version_mismatch=""" + Inconsistency detected! + Python during build and now are not the same! + Build python version : {} + Current python version: {}""", + on_build_config_check_runtime_error=""" + Could not interrogate python versions + Please see 'Having Issues' Section of ISSUES.TXT + Actual error (consider adding to issue report): {}""", +) + + +def print_error(key, *args): + print(ERROR_MESSAGES[key].format(*args).strip()) + + +def python_version_from(binary): + return phare_utilities.decode_bytes( + phare_utilities.run_cli_cmd(f"{binary} -V", check=True).stdout.strip() + ) + + +def check_build_config_is_runtime_compatible(strict=True): + try: + build_config: dict = cpp.build_config() + + if "PHARE_CONFIG_ERROR" in build_config: + print_error("on_phare_config_error") + return + + build_python_version = build_config["PYTHON_VERSION"] + current_python_version = python_version_from(sys.executable) + if build_python_version != current_python_version: + print_error( + "on_python3_version_mismatch", + build_python_version, + current_python_version, + ) + + raise ValueError("Python version mismatch!") + + except RuntimeError as e: + print_error("on_build_config_check_runtime_error", e) + + except ValueError as e: + print(e) + if strict: + raise e + + +@dataclasses.dataclass +class RuntimeSettings: + python_version: str + python_binary: str + + +def try_system_binary(cli, log_to): + with open(log_to, "w") as f: + try: + proc = phare_utilities.run_cli_cmd(cli, check=True) + f.write(phare_utilities.decode_bytes(proc.stdout).strip()) + except Exception as e: + f.write(f"failed to run cli command {cli}\n") + f.write(f"error {e}") + + +def try_system_binaries(log_dir): + try_system_binary("free -g", log_dir / "free_dash_g.txt") + try_system_binary("lscpu", log_dir / "lscpu.txt") + try_system_binary("hwloc-info", log_dir / "hwloc_info.txt") + + +def log_runtime_config(): + cpp_lib = cpp.cpp_lib() + + settings = RuntimeSettings( + python_binary=sys.executable, python_version=python_version_from(sys.executable) + ) + + if cpp_lib.mpi_rank() == 0: + DOT_PHARE_DIR.mkdir(exist_ok=True, parents=True) + cpp_lib.mpi_barrier() + + rank_dir = DOT_PHARE_DIR / f"rank_{cpp_lib.mpi_rank()}" + rank_dir.mkdir(exist_ok=True) + + with open(rank_dir / "runtime_config.json", "w") as f: + json.dump(dataclasses.asdict(settings), f) + + try_system_binaries(rank_dir) diff --git a/pyphare/pyphare/pharein/maxwellian_fluid_model.py b/pyphare/pyphare/pharein/maxwellian_fluid_model.py index ecb7ee7ff..6bff2fe24 100644 --- a/pyphare/pyphare/pharein/maxwellian_fluid_model.py +++ b/pyphare/pyphare/pharein/maxwellian_fluid_model.py @@ -43,7 +43,8 @@ def __init__(self, bx=None, by=None, bz=None, **kwargs): for population in self.populations: self.add_population(population, **kwargs[population]) - self.validate(global_vars.sim) + if not global_vars.sim.dry_run: + self.validate(global_vars.sim) global_vars.sim.set_model(self) diff --git a/pyphare/pyphare/pharein/simulation.py b/pyphare/pyphare/pharein/simulation.py index 1855004d5..17c6e45bc 100644 --- a/pyphare/pyphare/pharein/simulation.py +++ b/pyphare/pyphare/pharein/simulation.py @@ -5,6 +5,12 @@ from . import global_vars from ..core import box as boxm from ..core.box import Box +from ..core import parse_cli_args + +CLI_ARGS = parse_cli_args() + + +# ------------------------------------------------------------------------------ def supported_dimensions(): @@ -671,6 +677,9 @@ def wrapper(simulation_object, **kwargs): kwargs["hyper_resistivity"] = check_hyper_resistivity(**kwargs) + # kwargs["dry_run"] = CLI_ARGS.dry_run + # kwargs["write_reports"] = CLI_ARGS.reports + return func(simulation_object, **kwargs) return wrapper diff --git a/pyphare/pyphare/simulator/simulator.py b/pyphare/pyphare/simulator/simulator.py index 8e7f0110f..784593e67 100644 --- a/pyphare/pyphare/simulator/simulator.py +++ b/pyphare/pyphare/simulator/simulator.py @@ -1,7 +1,14 @@ +# +# +# + + import atexit import time as timem import numpy as np +import pyphare.pharein as ph +CLI_ARGS = ph.simulation.CLI_ARGS life_cycles = {} @@ -31,8 +38,6 @@ def startMPI(): class Simulator: def __init__(self, simulation, auto_dump=True, **kwargs): - import pyphare.pharein as ph - assert isinstance(simulation, ph.Simulation) # pylint: disable=no-member self.simulation = simulation self.cpp_hier = None # HERE @@ -58,9 +63,16 @@ def setup(self): # mostly to detach C++ class construction/dict parsing from C++ Simulator::init try: from pyphare.cpp import cpp_lib - import pyphare.pharein as ph startMPI() + + import pyphare.cpp.validate as validate_cpp + + if all([not CLI_ARGS.dry_run, CLI_ARGS.write_reports]): + # not necessary during testing + validate_cpp.log_runtime_config() + validate_cpp.check_build_config_is_runtime_compatible() + if self.log_to_file: self._log_to_file() ph.populateDict() @@ -90,6 +102,9 @@ def initialize(self): if self.cpp_hier is None: self.setup() + if CLI_ARGS.dry_run: + return self + self.cpp_sim.initialize() self._auto_dump() # first dump might be before first advance return self @@ -113,6 +128,8 @@ def _throw(self, e): def advance(self, dt=None): self._check_init() + if CLI_ARGS.dry_run: + return self if dt is None: dt = self.timeStep() @@ -138,6 +155,8 @@ def run(self): from pyphare.cpp import cpp_lib self._check_init() + if CLI_ARGS.dry_run: + return self perf = [] end_time = self.cpp_sim.endTime() t = self.cpp_sim.currentTime() @@ -180,8 +199,6 @@ def data_wrangler(self): def reset(self): if self.cpp_sim is not None: - import pyphare.pharein as ph - ph.clearDict() if self.cpp_dw is not None: self.cpp_dw.kill() diff --git a/res/cmake/def.cmake b/res/cmake/def.cmake index 2d68b4c96..79f9d272d 100644 --- a/res/cmake/def.cmake +++ b/res/cmake/def.cmake @@ -27,12 +27,12 @@ endif() set (PHARE_WERROR_FLAGS ${PHARE_FLAGS} ${PHARE_WERROR_FLAGS}) set (PHARE_PYTHONPATH "${CMAKE_BINARY_DIR}:${CMAKE_SOURCE_DIR}/pyphare") - +set (PHARE_MPIRUN_POSTFIX ${PHARE_MPIRUN_POSTFIX}) # now we see if we are running with configurator if (phare_configurator) execute_process( - COMMAND ./tools/config/cmake.sh + COMMAND ./tools/config/cmake.sh "${CMAKE_COMMAND}" "${CMAKE_CXX_COMPILER}" "${Python_EXECUTABLE}" WORKING_DIRECTORY ${PHARE_PROJECT_DIR} COMMAND_ERROR_IS_FATAL ANY ) @@ -145,6 +145,7 @@ if (test AND ${PHARE_EXEC_LEVEL_MIN} GREATER 0) # 0 = no tests set_property(TEST ${binary} PROPERTY ENVIRONMENT "PYTHONPATH=${PHARE_PYTHONPATH}") # ASAN detects leaks by default, even in system/third party libraries set_property(TEST ${binary} APPEND PROPERTY ENVIRONMENT "ASAN_OPTIONS=detect_leaks=0") + set_property(TEST ${binary} APPEND PROPERTY ENVIRONMENT PHARE_SKIP_CLI=1 ) endfunction(set_exe_paths_) function(add_phare_test_ binary directory) @@ -181,17 +182,17 @@ if (test AND ${PHARE_EXEC_LEVEL_MIN} GREATER 0) # 0 = no tests if(testMPI) function(add_phare_test binary directory) - add_test(NAME ${binary} COMMAND mpirun -n ${PHARE_MPI_PROCS} ./${binary} WORKING_DIRECTORY ${directory}) + add_test(NAME ${binary} COMMAND mpirun -n ${PHARE_MPI_PROCS} ${PHARE_MPIRUN_POSTFIX} ./${binary} WORKING_DIRECTORY ${directory}) add_phare_test_(${binary} ${directory}) endfunction(add_phare_test) function(add_python3_test name file directory) - add_test(NAME py3_${name} COMMAND mpirun -n ${PHARE_MPI_PROCS} python3 -u ${file} WORKING_DIRECTORY ${directory}) + add_test(NAME py3_${name} COMMAND mpirun -n ${PHARE_MPI_PROCS} ${PHARE_MPIRUN_POSTFIX} python3 -u ${file} WORKING_DIRECTORY ${directory}) set_exe_paths_(py3_${name}) endfunction(add_python3_test) function(add_mpi_python3_test N name file directory) - add_test(NAME py3_${name}_mpi_n_${N} COMMAND mpirun -n ${N} python3 ${file} WORKING_DIRECTORY ${directory}) + add_test(NAME py3_${name}_mpi_n_${N} COMMAND mpirun -n ${N} ${PHARE_MPIRUN_POSTFIX} python3 ${file} WORKING_DIRECTORY ${directory}) set_exe_paths_(py3_${name}_mpi_n_${N}) endfunction(add_mpi_python3_test) @@ -254,7 +255,7 @@ if (test AND ${PHARE_EXEC_LEVEL_MIN} GREATER 0) # 0 = no tests else() add_test( NAME py3_${target}_mpi_n_${N} - COMMAND mpirun -n ${N} python3 -u ${file} ${CLI_ARGS} + COMMAND mpirun -n ${N} ${PHARE_MPIRUN_POSTFIX} python3 -u ${file} ${CLI_ARGS} WORKING_DIRECTORY ${directory}) set_exe_paths_(py3_${target}_mpi_n_${N}) endif() @@ -288,11 +289,11 @@ if (test AND ${PHARE_EXEC_LEVEL_MIN} GREATER 0) # 0 = no tests endif() # useful to see what's available after importing a package -function(phare_print_all_cmake_vars) +function(phare_print_all_vars) get_cmake_property(_variableNames VARIABLES) list (SORT _variableNames) foreach (_variableName ${_variableNames}) message(STATUS "${_variableName}=${${_variableName}}") endforeach() -endfunction(phare_print_all_cmake_vars) +endfunction(phare_print_all_vars) diff --git a/res/cmake/dep/samrai.cmake b/res/cmake/dep/samrai.cmake index 303861da5..d099b1565 100644 --- a/res/cmake/dep/samrai.cmake +++ b/res/cmake/dep/samrai.cmake @@ -35,3 +35,4 @@ else() message("SAMRAI HAS BEEN FOUND") message(${SAMRAI_INCLUDE_DIRS}) endif() + diff --git a/src/amr/CMakeLists.txt b/src/amr/CMakeLists.txt index 82c52e220..4eca5d5bb 100644 --- a/src/amr/CMakeLists.txt +++ b/src/amr/CMakeLists.txt @@ -77,7 +77,7 @@ add_library(${PROJECT_NAME} ${SOURCES_INC} ${SOURCES_CPP}) target_compile_options(${PROJECT_NAME} PRIVATE ${PHARE_WERROR_FLAGS}) set_property(TARGET ${PROJECT_NAME} PROPERTY INTERPROCEDURAL_OPTIMIZATION ${PHARE_INTERPROCEDURAL_OPTIMIZATION}) -target_link_directories(${PROJECT_NAME} PUBLIC ${MPI_LIBRARY_PATH}) +target_link_directories(${PROJECT_NAME} PUBLIC MPI::MPI_C) target_link_libraries(${PROJECT_NAME} PUBLIC phare_core SAMRAI_algs diff --git a/src/amr/solvers/solver_ppc.hpp b/src/amr/solvers/solver_ppc.hpp index c6143daff..1e1dcc4c4 100644 --- a/src/amr/solvers/solver_ppc.hpp +++ b/src/amr/solvers/solver_ppc.hpp @@ -568,7 +568,6 @@ void SolverPPC::moveIons_(level_t& level, Ions& ions, rm.setTime(ions, *patch, newTime); } - fromCoarser.fillIonGhostParticles(ions, level, newTime); fromCoarser.fillIonPopMomentGhosts(ions, level, newTime); @@ -584,6 +583,8 @@ void SolverPPC::moveIons_(level_t& level, Ions& ions, // these were not completed by the deposition of patch and levelghost particles fromCoarser.fillIonMomentGhosts(ions, level, newTime); } + + } // namespace PHARE::solver diff --git a/src/core/def/phare_config.hpp b/src/core/def/phare_config.hpp new file mode 100644 index 000000000..139c6bb49 --- /dev/null +++ b/src/core/def/phare_config.hpp @@ -0,0 +1,23 @@ +#ifndef PHARE_CORE_DEF_PHARE_CONFIG_HPP +#define PHARE_CORE_DEF_PHARE_CONFIG_HPP + +#include +#include + +// if __gen_sys.hpp exists, it has been generated by the phare_configurator (config.py) +// if it does not, the build_config() version below reflects that phare_configurator is off +#if __has_include("core/def/_gen_sys.hpp") +#include "core/def/_gen_sys.hpp" +#else + +namespace PHARE +{ +std::unordered_map build_config() +{ + return {{"PHARE_CONFIG_ERROR", "PHARE NOT BUILT WITH CONFIGURATOR!"}}; +} +} // namespace PHARE + +#endif + +#endif /*PHARE_CORE_DEF_PHARE_CONFIG_HPP*/ diff --git a/src/core/def/phare_mpi.hpp b/src/core/def/phare_mpi.hpp index 582f14fbc..4588dde90 100644 --- a/src/core/def/phare_mpi.hpp +++ b/src/core/def/phare_mpi.hpp @@ -1,5 +1,5 @@ -#ifndef PHARE_CORE_DEF_MPI_HPP -#define PHARE_CORE_DEF_MPI_HPP +#ifndef PHARE_CORE_DEF_PHARE_MPI_HPP +#define PHARE_CORE_DEF_PHARE_MPI_HPP // DO NOT INCLUDE MPI MANUALLY! USE THIS FILE! @@ -20,4 +20,4 @@ ENABLE_WARNING(cast-function-type, bad-function-cast, 42) -#endif /*PHARE_CORE_DEF_MPI_HPP*/ \ No newline at end of file +#endif /*PHARE_CORE_DEF_PHARE_MPI_HPP*/ diff --git a/src/hdf5/detail/h5/h5_file.hpp b/src/hdf5/detail/h5/h5_file.hpp index 52eb7fbac..d52b8f3f2 100644 --- a/src/hdf5/detail/h5/h5_file.hpp +++ b/src/hdf5/detail/h5/h5_file.hpp @@ -2,6 +2,7 @@ #define PHARE_HDF5_H5FILE_HPP #include "core/def.hpp" +#include "core/def/phare_mpi.hpp" #include "highfive/H5File.hpp" #include "highfive/H5Easy.hpp" @@ -47,6 +48,10 @@ class HighFiveFile fapl.add(HighFive::MPIOFileAccess{MPI_COMM_WORLD, MPI_INFO_NULL}); #else std::cout << "WARNING: PARALLEL HDF5 not available" << std::endl; + if (core::mpi_size() > 1) + { + throw std::runtime_error("HDF5 NOT PARALLEL!"); + } #endif } return HiFile{path, flags, fapl}; diff --git a/src/python3/cpp_etc.cpp b/src/python3/cpp_etc.cpp index 294767b8e..0550c5a0a 100644 --- a/src/python3/cpp_etc.cpp +++ b/src/python3/cpp_etc.cpp @@ -2,6 +2,8 @@ #include "python3/pybind_def.hpp" #include "simulator/simulator.hpp" +#include "core/def/phare_config.hpp" + #include "amr/wrappers/hierarchy.hpp" // for HierarchyRestarter::getRestartFileFullPath @@ -51,5 +53,7 @@ PYBIND11_MODULE(cpp_etc, m) m.def("restart_path_for_time", [](std::string path, double timestamp) { return PHARE::amr::Hierarchy::restartFilePathForTime(path, timestamp); }); + + m.def("phare_build_config", []() { return PHARE::build_config(); }); } } // namespace PHARE::pydata diff --git a/tests/core/numerics/ohm/CMakeLists.txt b/tests/core/numerics/ohm/CMakeLists.txt index 302b65c92..1d4f136a2 100644 --- a/tests/core/numerics/ohm/CMakeLists.txt +++ b/tests/core/numerics/ohm/CMakeLists.txt @@ -1,4 +1,4 @@ - cmake_minimum_required (VERSION 3.20.1) +cmake_minimum_required (VERSION 3.20.1) project(test-ohm) diff --git a/tests/simulator/__init__.py b/tests/simulator/__init__.py index 56ca9e9d9..e0e04ec61 100644 --- a/tests/simulator/__init__.py +++ b/tests/simulator/__init__.py @@ -249,14 +249,12 @@ def unique_diag_dir_for_test_case(self, base_path, ndim, interp, post_path=""): def clean_up_diags_dirs(self): from pyphare.cpp import cpp_lib - cpp_lib().mpi_barrier() # synchronize first - if cpp_lib().mpi_rank() > 0: - return # only delete h5 files for rank 0 - - if self.success: + cpp_lib().mpi_barrier() + if cpp_lib().mpi_rank() == 0 and self.success: import os import shutil for diag_dir in self.diag_dirs: if os.path.exists(diag_dir): shutil.rmtree(diag_dir) + cpp_lib().mpi_barrier() diff --git a/tests/simulator/advance/__init__.py b/tests/simulator/advance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/simulator/initialize/__init__.py b/tests/simulator/initialize/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/simulator/refinement/__init__.py b/tests/simulator/refinement/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/simulator/test_validation.py b/tests/simulator/test_validation.py index b08a5f9ca..7bcd5a9a5 100644 --- a/tests/simulator/test_validation.py +++ b/tests/simulator/test_validation.py @@ -13,6 +13,7 @@ from pyphare.simulator.simulator import Simulator from tests.simulator import NoOverwriteDict, populate_simulation +from tests.simulator import SimulatorTest out = "phare_outputs/valid/refinement_boxes/" diags = { @@ -29,12 +30,13 @@ def dup(dic): @ddt -class SimulatorValidation(unittest.TestCase): +class SimulatorValidation(SimulatorTest): def __init__(self, *args, **kwargs): super(SimulatorValidation, self).__init__(*args, **kwargs) self.simulator = None def tearDown(self): + super(SimulatorValidation, self).tearDown() if self.simulator is not None: self.simulator.reset() diff --git a/tools/config/CMakeLists.txt b/tools/config/CMakeLists.txt index a98b026ed..fa48d47bc 100644 --- a/tools/config/CMakeLists.txt +++ b/tools/config/CMakeLists.txt @@ -2,6 +2,9 @@ cmake_minimum_required (VERSION 3.20.1) # released April 8, 2021 - https://www.k project(phare_configurator) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + find_package(MPI REQUIRED COMPONENTS C) find_package(HDF5 REQUIRED COMPONENTS C) diff --git a/tools/config/cmake.sh b/tools/config/cmake.sh index d6bbf41ee..24f771bb6 100755 --- a/tools/config/cmake.sh +++ b/tools/config/cmake.sh @@ -6,19 +6,14 @@ CWD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" && cd "$CWD" ROOT_DIR=$(cd ../.. && pwd) [ -f "${ROOT_DIR}/res/cmake/def.cmake" ] || (echo "Corruption detected" && exit 1) # confirm correct directory +export CMAKE_COMMAND=${1} +export CMAKE_CXX_COMPILER=${2} +export PYTHON_EXECUTABLE=${3} + [ -d build ] || ( mkdir -p build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. # 2>&1 > /dev/null - make && ./phare_configurator + "$CMAKE_COMMAND" -DCMAKE_BUILD_TYPE=Release .. \ + -DCMAKE_CXX_COMPILER="${CMAKE_CXX_COMPILER}" # 2>&1 > /dev/null + make VERBOSE=1 && ./phare_configurator ) -python3 config.py - -# here as a placeholder while we flesh this out, but this does hookin to the normal cmake with -Dphare_configurator=ON -# probably to be generated by python (config.py) eventually -cat > local.cmake <<- EOM -cmake_minimum_required (VERSION 3.20.1) -project(configured_phare) -message("") -message("!!PHARE CONFIGURATED!!") -message("") -EOM +$PYTHON_EXECUTABLE config.py diff --git a/tools/config/config.py b/tools/config/config.py index 7a20a0b44..2408c8b09 100644 --- a/tools/config/config.py +++ b/tools/config/config.py @@ -1,13 +1,14 @@ +import os import json import subprocess -from pathlib import Path -from dataclasses import dataclass, field import dataclasses +from pathlib import Path FILE_DIR = Path(__file__).resolve().parent BUILD_DIR = FILE_DIR / "build" ROOT_DIR = FILE_DIR.parent.parent -FILES = [f for f in BUILD_DIR.glob("PHARE_*")] +DOT_PHARE_DIR = ROOT_DIR / ".phare" +FILES = list(BUILD_DIR.glob("PHARE_*")) DEF_DIR = ROOT_DIR / "src" / "core" / "def" GENERATED_CONFIGS = dict( mpi=DEF_DIR / "_gen_mpi.hpp", @@ -15,37 +16,66 @@ ) -@dataclass +@dataclasses.dataclass class SystemSettings: - cmake_binary: str # field(default_factory=lambda: "Couldn't parse cmake binary path") - cmake_version: str # field(default_factory=lambda: "Couldn't parse cmake version") - hdf5_version: str # field(default_factory=lambda: "Couldn't parse cmake hdf5 version") - mpi_version: str # field(default_factory=lambda: "Couldn't parse cmake mpi version") - uname: str # field(default_factory=lambda: "Couldn't parse uname") - - -system_cpp_ = """ + cmake_binary: str + cmake_version: str + cxx_compiler: str + cxx_compiler_version: str + hdf5_version: str + hdf5_is_parallel: str + mpi_version: str + python_binary: str + python_version: str + uname: str + + +SYSTEM_CPP_ = """ #include +#include namespace PHARE {{ struct SystemConfig {{ - constexpr static std::string_view UNAME = R"({})"; - constexpr static std::string_view MPI_VERSION = R"({})"; - constexpr static std::string_view HDF5_VERSION = R"({})"; - constexpr static std::string_view CMAKE_VERSION = R"({})"; constexpr static std::string_view CMAKE_BINARY = R"({})"; + constexpr static std::string_view CMAKE_VERSION = R"({})"; + constexpr static std::string_view CXX_COMPILER = R"({})"; + constexpr static std::string_view CXX_COMPILER_VERSION = R"({})"; + constexpr static std::string_view HDF5_VERSION = R"({})"; + constexpr static std::string_view HDF5_IS_PARALLEL = R"({})"; + constexpr static std::string_view _MPI_VERSION_ = R"({})"; + constexpr static std::string_view PYTHON_BINARY = R"({})"; + constexpr static std::string_view PYTHON_VERSION = R"({})"; + constexpr static std::string_view UNAME = R"({})"; }}; +std::unordered_map build_config(){{ + return {{ + {{"CMAKE_BINARY", std::string{{SystemConfig::CMAKE_BINARY}}}}, + {{"CMAKE_VERSION", std::string{{SystemConfig::CMAKE_VERSION}}}}, + {{"CXX_COMPILER", std::string{{SystemConfig::CXX_COMPILER}}}}, + {{"CXX_COMPILER_VERSION", std::string{{SystemConfig::CXX_COMPILER_VERSION}}}}, + {{"HDF5_VERSION", std::string{{SystemConfig::HDF5_VERSION}}}}, + {{"HDF5_IS_PARALLEL", std::string{{SystemConfig::HDF5_IS_PARALLEL}}}}, + {{"MPI_VERSION", std::string{{SystemConfig::_MPI_VERSION_}}}}, + {{"PYTHON_BINARY", std::string{{SystemConfig::PYTHON_BINARY}}}}, + {{"PYTHON_VERSION", std::string{{SystemConfig::PYTHON_VERSION}}}}, + {{"UNAME", std::string{{SystemConfig::UNAME}}}} + }}; +}} + }} """ -def exec(cmd): - proc = subprocess.run(cmd.split(" "), check=False, capture_output=True) - return (proc.stdout + proc.stderr).decode().strip() +def subprocess_run(cmd, on_error="error message"): + try: + proc = subprocess.run(cmd.split(" "), check=True, capture_output=True) + return (proc.stdout + proc.stderr).decode().strip() + except Exception: + return on_error def file_string_or(filename, fail=None): @@ -92,15 +122,6 @@ def cmake_version_from(cmake_cache: dict) -> str: return "Failed to find version in cache file" -def cmake_binary_path_from(cmake_cache: dict) -> str: - """With this we should be able to guarantee future cmake - calls by us if needed for whatever reason""" - path_key = "CMAKE_COMMAND:INTERNAL" - if path_key in cmake_cache: - return cmake_cache[path_key] - return "Failed to find cmake binary path in cache file" - - def local_file_format(filename): return filename[6:-4] @@ -119,20 +140,41 @@ def ifndef_define(var, val, buf, comment=None): return buf +def deduce_mpi_type_from(version): + mpi_type = "unknown" + if any([s in version for s in ["Open MPI", "OpenMPI"]]): + return "OMPI" + if any([s in version for s in ["MPICH"]]): + return "MPICH" + return "unknown" + + def config_mpi_version(txtfile, h_file): with open(txtfile) as f: version = f.readline() buf = "\n" - if any([s in version for s in ["Open MPI", "OpenMPI"]]): + mpi_type = deduce_mpi_type_from(version) + if mpi_type == "OMPI": buf = ifndef_define( "OMPI_SKIP_MPICXX", 1, buf, "avoids default including mpicxx" ) - if any([s in version for s in ["MPICH"]]): + elif mpi_type == "MPICH": buf = ifndef_define( "MPICH_SKIP_MPICXX", 1, buf, "avoids default including mpicxx" ) with open(h_file, "w") as f: f.write(buf) + return version + + +def try_compiler_dash_v(compiler): + return subprocess_run(f"{compiler} -v", "compiler does not support '-v' argument") + + +def get_python_version(py3_binary): + return subprocess_run( + f"{py3_binary} -V", "python3 interpreter does not support '-V' argument" + ) def gen_system_file(): @@ -140,25 +182,37 @@ def gen_system_file(): cmake_cache: dict = parse_cmake_cache_file() settings = SystemSettings( - cmake_binary=cmake_binary_path_from(cmake_cache), + cmake_binary=os.environ["CMAKE_COMMAND"], cmake_version=cmake_version_from(cmake_cache), + cxx_compiler=os.environ["CMAKE_CXX_COMPILER"], + cxx_compiler_version=try_compiler_dash_v(os.environ["CMAKE_CXX_COMPILER"]), hdf5_version=file_string_or("PHARE_HDF5_version.txt", "HDF5 version failed"), + hdf5_is_parallel=file_string_or( + "PHARE_HDF5_is_parallel.txt", "HDF5 is parallel check failed" + ), mpi_version=file_string_or( "PHARE_MPI_Get_library_version.txt", "MPI version failed" ), - uname=exec("uname -a"), + python_binary=os.environ["PYTHON_EXECUTABLE"], + python_version=get_python_version(os.environ["PYTHON_EXECUTABLE"]), + uname=subprocess_run("uname -a"), ) - with open(ROOT_DIR/".phare_config.json", "w") as f: + with open(DOT_PHARE_DIR / "build_config.json", "w") as f: json.dump(dataclasses.asdict(settings), f) with open(out_file, "w") as f: f.write( - system_cpp_.format( - settings.uname, + SYSTEM_CPP_.format( + settings.cmake_binary, + settings.cmake_version, + settings.cxx_compiler, + settings.cxx_compiler_version, settings.hdf5_version, + settings.hdf5_is_parallel, settings.mpi_version, - settings.cmake_version, - settings.cmake_binary, + settings.python_binary, + settings.python_version, + settings.uname, ) ) @@ -167,15 +221,40 @@ def config_mpi(): h_file = GENERATED_CONFIGS["mpi"] create_file(h_file) operators = dict(MPI_Get_library_version=config_mpi_version) + results = {} for f in FILES: local_name = local_file_format(f.name) if local_name in operators: - operators[local_name](f, h_file) + results[local_name] = operators[local_name](f, h_file) + return results + + +def write_local_cmake_file(mpi_results): + with open("local.cmake", "w") as file: + file.write( + """cmake_minimum_required (VERSION 3.20.1) +project(configured_phare) +message("") +message("!!PHARE CONFIGURATED!!") +message("") + +""" + ) + + mpi_type = deduce_mpi_type_from(mpi_results["MPI_Get_library_version"]) + if mpi_type == "OMPI": + # work around for https://github.com/open-mpi/ompi/issues/10761#issuecomment-1236909802 + file.write( + """set (PHARE_MPIRUN_POSTFIX "${PHARE_MPIRUN_POSTFIX} -q --bind-to none") + """ + ) def main(): - config_mpi() + DOT_PHARE_DIR.mkdir(exist_ok=True, parents=True) + mpi_results = config_mpi() gen_system_file() + write_local_cmake_file(mpi_results) if __name__ == "__main__": diff --git a/tools/config/main.cpp b/tools/config/main.cpp index 0719a5922..506374d84 100644 --- a/tools/config/main.cpp +++ b/tools/config/main.cpp @@ -11,8 +11,16 @@ #include "hdf5.h" constexpr static std::string_view hdf5_version = H5_VERSION; #else -constexpr static std::string_view hdf5_version = "HDF5 was not found!"; +constexpr static std::string_view hdf5_version = "HDF5 was not found!"; +constexpr static std::string_view H5_HAVE_PARALLEL = false; #endif +constexpr std::string_view hdf5_is_parallel() +{ + if constexpr (H5_HAVE_PARALLEL) + return "yes"; + return "no"; +} + void write_string_to_file(std::string const& buff, std::string const& filename) { @@ -27,6 +35,17 @@ void write_hdf5_version() write_string_to_file(std::string{hdf5_version}, "PHARE_HDF5_version.txt"); } +void write_hdf5_is_parallel() +{ + write_string_to_file(std::string{hdf5_is_parallel()}, "PHARE_HDF5_is_parallel.txt"); +} + +void write_hdf5_info() +{ + write_hdf5_version(); + write_hdf5_is_parallel(); +} + void config_mpi() { int initialized; @@ -51,7 +70,7 @@ int main(int argc, char** argv) try { config_mpi(); - write_hdf5_version(); + write_hdf5_info(); } catch (const std::exception& e) { diff --git a/tools/report.py b/tools/report.py new file mode 100644 index 000000000..5549ef9c5 --- /dev/null +++ b/tools/report.py @@ -0,0 +1,61 @@ +import os +import sys +import shutil +import tempfile +import datetime +from pathlib import Path +from zipfile import ZipFile + +from tools.python3 import pushd + +SEARCH_PATHS = list(set([os.getcwd()] + sys.path)) + + +def find_phare_dirs(): + phare_dirs = [] + for path in SEARCH_PATHS: + dot_phare_dir = Path(path) / ".phare" + if dot_phare_dir.exists(): + phare_dirs += [dot_phare_dir] + return phare_dirs + + +def main(): + phare_dirs = find_phare_dirs() + assert find_phare_dirs + cwd = Path(os.getcwd()) + with tempfile.TemporaryDirectory() as tmpdirname: + tmp_dir = Path(tmpdirname) + for dir_idx, phare_dir in enumerate(phare_dirs): + with pushd(phare_dir.parent): + shutil.copytree(phare_dir.stem, tmp_dir / phare_dir.stem) + shutil.move( + tmp_dir / phare_dir.stem, f"{tmpdirname}/{phare_dir.stem}_{dir_idx}" + ) + + zip_name = "PHARE_REPORT.zip" + final_report_zip = cwd / zip_name + if final_report_zip.exists(): + """move existing to subdirectory with creation timestamp in name""" + reports_dir = cwd / ".phare_reports" + reports_dir.mkdir(exist_ok=True, parents=True) + timestamp = datetime.datetime.fromtimestamp( + final_report_zip.stat().st_ctime + ).isoformat() + shutil.move( + final_report_zip, + reports_dir / f"{final_report_zip.stem}.{timestamp}.zip", + ) + + tmp_report_zip = tmp_dir / zip_name + with pushd(tmpdirname): + """shutil.make_archive doesn't like multiple directory inputs""" + with ZipFile(tmp_report_zip, "w") as zip_object: + for phare_dir in Path(tmpdirname).glob(".phare_*"): + for item in phare_dir.glob("**/*"): + zip_object.write(item, arcname=item.relative_to(tmpdirname)) + shutil.move(tmp_report_zip, cwd) + + +if __name__ == "__main__": + main() diff --git a/tools/report.sh b/tools/report.sh new file mode 100755 index 000000000..fc474c6c7 --- /dev/null +++ b/tools/report.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# usage: +# ./tools/report.sh +# generates zip archive for logging issues + +set -eu +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT=$(cd "$SCRIPT_DIR/.." && pwd) +( + export PYTHONPATH="${PWD}:${PWD}/build:${PWD}/pyphare:${PYTHONPATH}" + python3 "$ROOT/tools/report.py" +)