Replace internal dependencies by FetchContent (openPMD#1583)

* Remove specific version from openPMD json find_package() command * Replace internal dependency on nlohmann_json by FetchContent * Update required cmake version for fetchContent and add openPMD interface library for nlohmann_json * Add nlohmann_json inculde directories * Revert "Add nlohmann_json inculde directories" This reverts commit 41a4658. * Revert "Update required cmake version for fetchContent and add openPMD interface library for nlohmann_json" This reverts commit 6136a97. * Revert "Replace internal dependency on nlohmann_json by FetchContent" This reverts commit 620d861. * Replace internal JSON by CMake FetchContent command. * Replace the old json directory by FetchContent data. Add the directory filled by FetchContent to .gitignore Add CMakeUserPresets.json to .gitignore * Replace internal TOML11 library by a FetchContent call * Replace internal Catch2 library by a FetchContent call * Update versions within FetchContent to stay with the latest version * Set C++ Standard to 17 and do not move Catch2 to v3 yet. * Remove version updates. Update README.md * Replace PyBind11 dependency by FetchContent * Fix pybind11 not found * Remove pybind11 from repo. Add pybind11 path to gitignore. Simplify pybind11 configuration. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Correct Download message for Catch2 Co-authored-by: Franz Pöschel <[email protected]> * More detailed description of FetchContent in README.md * Request cmake>=3.24 in appveyor CI build * Remove OVERRIDE_FIND_PACKAGE to reduce cmake to version 3.22. Revert previous commit to use cmake 3.22 in Win32 CI runner. * Revert "Request cmake>=3.24 in appveyor CI build" This reverts commit 5976cc9. * Change to trigger CI. * Use SOURCE_DIR only in CMake >= 3.23 * Keep Thirdparty in this PR * Doc: CMake 3.22.0+ * Keep Explicit Superbuild Control * FetchContent: Repo & Branch/Tag Control * Nested Superbuild Support Dependent projects might have the same dependencies and build as a superbuild as well. * Update `.gitignore` --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Franz Pöschel <[email protected]> Co-authored-by: Axel Huebl <[email protected]>
franzpoeschel · Sep 5, 2024 · 570f155 · 570f155
1 parent 1c7d7ff
commit 570f155
Show file tree

Hide file tree

Showing 8 changed files with 145 additions and 84 deletions.
diff --git a/.gitignore b/.gitignore
@@ -96,6 +96,11 @@ Temporary Items
 .spack-env/
 spack.lock
 
+#########
+# CMake #
+#########
+CMakeUserPresets.json
+
 #########
 # Tools #
 #########

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,6 +1,6 @@
 # Preamble ####################################################################
 #
-cmake_minimum_required(VERSION 3.15.0)
+cmake_minimum_required(VERSION 3.22.0)
 
 project(openPMD VERSION 0.16.0) # LANGUAGES CXX
 
@@ -12,18 +12,7 @@ include(${openPMD_SOURCE_DIR}/cmake/openPMDFunctions.cmake)
 
 # CMake policies ##############################################################
 #
-# Search in <PackageName>_ROOT:
-#   https://cmake.org/cmake/help/v3.12/policy/CMP0074.html
-if(POLICY CMP0074)
-    cmake_policy(SET CMP0074 NEW)
-endif()
-
-# We use simple syntax in cmake_dependent_option, so we are compatible with the
-# extended syntax in CMake 3.22+
-# https://cmake.org/cmake/help/v3.22/policy/CMP0127.html
-if(POLICY CMP0127)
-    cmake_policy(SET CMP0127 NEW)
-endif()
+# not needed right now
 
 
 # No in-Source builds #########################################################
@@ -148,10 +137,13 @@ openpmd_option(PYTHON         "Enable Python bindings"                    AUTO)
 option(openPMD_INSTALL               "Add installation targets"             ON)
 option(openPMD_INSTALL_RPATH         "Add RPATHs to installed binaries"     ON)
 option(openPMD_HAVE_PKGCONFIG        "Generate a .pc file for pkg-config"   ON)
-option(openPMD_USE_INTERNAL_CATCH    "Use internally shipped Catch2"        ON)
-option(openPMD_USE_INTERNAL_PYBIND11 "Use internally shipped pybind11"      ON)
-option(openPMD_USE_INTERNAL_JSON     "Use internally shipped nlohmann-json" ON)
-option(openPMD_USE_INTERNAL_TOML11   "Use internally shipped toml11"        ON)
+
+# superbuild control
+option(openPMD_SUPERBUILD            "Download & build extra dependencies"  ON)
+option(openPMD_USE_INTERNAL_CATCH    "Use internally shipped Catch2"        ${openPMD_SUPERBUILD})
+option(openPMD_USE_INTERNAL_PYBIND11 "Use internally shipped pybind11"      ${openPMD_SUPERBUILD})
+option(openPMD_USE_INTERNAL_JSON     "Use internally shipped nlohmann-json" ${openPMD_SUPERBUILD})
+option(openPMD_USE_INTERNAL_TOML11   "Use internally shipped toml11"        ${openPMD_SUPERBUILD})
 
 option(openPMD_USE_INVASIVE_TESTS "Enable unit tests that modify source code" OFF)
 option(openPMD_USE_VERIFY "Enable internal VERIFY (assert) macro independent of build type" ON)
@@ -223,6 +215,10 @@ endfunction()
 
 # Dependencies ################################################################
 #
+include(FetchContent)
+
+message(STATUS "openPMD-api superbuild: ${openPMD_SUPERBUILD}")
+
 # external library: MPI (optional)
 # Implementation quirks for BullMPI, Clang+MPI and Brew's MPICH
 #   definitely w/o MPI::MPI_C:
@@ -259,11 +255,30 @@ endif()
 
 
 # external library: nlohmann-json (required)
-if(openPMD_USE_INTERNAL_JSON)
-    set(JSON_BuildTests OFF CACHE INTERNAL "")
-    set(JSON_Install OFF CACHE INTERNAL "")  # only used PRIVATE
-    add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/json")
-    message(STATUS "nlohmann-json: Using INTERNAL version '3.9.1'")
+if(TARGET nlohmann_json::nlohmann_json)
+    # nothing to do, target already exists in the superbuild
+    message(STATUS "nlohmann_json::nlohmann_json target already imported")
+elseif(openPMD_USE_INTERNAL_JSON)
+    set(JSON_BuildTests OFF CACHE INTERNAL "NLohmann JSON option defiend internally by openPMD")
+    set(JSON_Install OFF CACHE INTERNAL "NLohmann JSON option defiend internally by openPMD")  # only used PRIVATE
+
+    # Git fetcher
+    set(openPMD_json_repo "https://github.com/nlohmann/json.git"
+        CACHE STRING
+        "Repository URI to pull and build nlohmann-json from if(openPMD_USE_INTERNAL_JSON)")
+    set(openPMD_json_branch "v3.11.3"
+        CACHE STRING
+        "Repository branch for openPMD_json_repo if(openPMD_USE_INTERNAL_JSON)")
+
+    message(STATUS "Downloading nlohmann-json ...")
+    message(STATUS "nlohmann-json repository: ${openPMD_json_repo} (${openPMD_json_branch})")
+
+    FetchContent_Declare(nlohmann_json
+      GIT_REPOSITORY  ${openPMD_json_repo}
+      GIT_TAG         ${openPMD_json_branch}
+      BUILD_IN_SOURCE 0
+    )
+    FetchContent_MakeAvailable(nlohmann_json)
 else()
     find_package(nlohmann_json 3.9.1 CONFIG REQUIRED)
     message(STATUS "nlohmann-json: Found version '${nlohmann_json_VERSION}'")
@@ -273,10 +288,30 @@ target_link_libraries(openPMD::thirdparty::nlohmann_json
     INTERFACE nlohmann_json::nlohmann_json)
 
 # external library: toml11
-if(openPMD_USE_INTERNAL_TOML11)
-    set(toml11_INSTALL OFF CACHE INTERNAL "")
-    add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/toml11")
-    message(STATUS "toml11: Using INTERNAL version '3.7.1'")
+if(TARGET toml11::toml11)
+    # nothing to do, target already exists in the superbuild
+    message(STATUS "toml11::toml11 target already imported")
+elseif(openPMD_USE_INTERNAL_TOML11)
+    set(toml11_INSTALL OFF CACHE INTERNAL "toml11 option defined internally by openPMD")
+
+    # Git fetcher
+    set(openPMD_toml11_repo "https://github.com/ToruNiina/toml11.git"
+        CACHE STRING
+        "Repository URI to pull and build toml11 from if(openPMD_USE_INTERNAL_TOML11)")
+    set(openPMD_toml11_branch "v3.7.1"
+        CACHE STRING
+        "Repository branch for openPMD_toml11_branch if(openPMD_USE_INTERNAL_TOML11)")
+
+    message(STATUS "Downloading toml11 ...")
+    message(STATUS "toml11 repository: ${openPMD_toml11_repo} (${openPMD_toml11_branch})")
+
+    FetchContent_Declare(
+      toml11
+      GIT_REPOSITORY  ${openPMD_toml11_repo}
+      GIT_TAG         ${openPMD_toml11_branch}
+      BUILD_IN_SOURCE 0
+    )
+    FetchContent_MakeAvailable(toml11)
 else()
     # toml11 4.0 was a breaking change. This is reflected in the library's CMake
     # logic: version 4.0 is not accepted by a call to find_package(toml11 3.7).
@@ -403,48 +438,51 @@ endif()
 # TODO: Check if ADIOS2 is parallel when openPMD_HAVE_MPI is ON
 
 # external library: pybind11 (optional)
-set(_PY_DEV_MODULE Development.Module)
-if(CMAKE_VERSION VERSION_LESS 3.18.0)
-    # over-specification needed for CMake<3.18
-    #   https://pybind11.readthedocs.io/en/latest/compiling.html#findpython-mode
-    #   https://cmake.org/cmake/help/v3.18/module/FindPython.html
-    set(_PY_DEV_MODULE Development)
-endif()
 if(openPMD_USE_PYTHON STREQUAL AUTO)
-    find_package(Python 3.8.0 COMPONENTS Interpreter ${_PY_DEV_MODULE})
-    if(Python_FOUND)
-        if(openPMD_USE_INTERNAL_PYBIND11)
-            add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/pybind11")
+    find_package(Python 3.8.0 COMPONENTS Interpreter Development.Module)
+elseif(openPMD_USE_PYTHON)
+    find_package(Python COMPONENTS Interpreter Development.Module REQUIRED)
+else()
+    set(openPMD_HAVE_PYTHON FALSE)
+endif()
+if(Python_FOUND)
+    if(TARGET pybind11::module)
+        # nothing to do, target already exists in the superbuild
+        message(STATUS "pybind11::module target already imported")
+    elseif(openPMD_USE_INTERNAL_PYBIND11)
+        # Git fetcher
+        set(openPMD_pybind11_repo "https://github.com/pybind/pybind11.git"
+            CACHE STRING
+            "Repository URI to pull and build pybind11 from if(openPMD_USE_INTERNAL_PYBIND11)")
+        set(openPMD_pybind11_branch "v2.12.0"
+            CACHE STRING
+            "Repository branch for openPMD_pybind11_repo if(openPMD_USE_INTERNAL_PYBIND11)")
+
+        message(STATUS "Downloading pybind11 ...")
+        message(STATUS "pybind11 repository: ${openPMD_pybind11_repo} (${openPMD_pybind11_branch})")
+
+        FetchContent_Declare(
+            pybind11
+            GIT_REPOSITORY  ${openPMD_pybind11_repo}
+            GIT_TAG         ${openPMD_pybind11_branch}
+            BUILD_IN_SOURCE 0
+        )
+        FetchContent_MakeAvailable(pybind11)
+    else()
+        find_package(pybind11 2.12.0 CONFIG)
+        if(pybind11_FOUND)
             set(openPMD_HAVE_PYTHON TRUE)
-            message(STATUS "pybind11: Using INTERNAL version 2.12.0")
+            message(STATUS "pybind11: Found version '${pybind11_VERSION}'")
         else()
-            find_package(pybind11 2.12.0 CONFIG)
-            if(pybind11_FOUND)
-                set(openPMD_HAVE_PYTHON TRUE)
-                message(STATUS "pybind11: Found version '${pybind11_VERSION}'")
-            else()
-                set(openPMD_HAVE_PYTHON FALSE)
-            endif()
+            set(openPMD_HAVE_PYTHON FALSE)
         endif()
-    else()
-        set(openPMD_HAVE_PYTHON FALSE)
-    endif()
-elseif(openPMD_USE_PYTHON)
-    find_package(Python COMPONENTS Interpreter ${_PY_DEV_MODULE} REQUIRED)
-    if(openPMD_USE_INTERNAL_PYBIND11)
-        add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/pybind11")
-        set(openPMD_HAVE_PYTHON TRUE)
-        message(STATUS "pybind11: Using INTERNAL version 2.12.0")
-    else()
-        find_package(pybind11 2.12.0 REQUIRED CONFIG)
-        set(openPMD_HAVE_PYTHON TRUE)
-        message(STATUS "pybind11: Found version '${pybind11_VERSION}'")
+
     endif()
+    set(openPMD_HAVE_PYTHON TRUE)
 else()
     set(openPMD_HAVE_PYTHON FALSE)
 endif()
 
-
 # Targets #####################################################################
 #
 set(CORE_SOURCE
@@ -560,18 +598,37 @@ target_include_directories(openPMD PUBLIC
 
 # Catch2 for unit tests
 if(openPMD_BUILD_TESTING)
-    add_library(openPMD::thirdparty::Catch2 INTERFACE IMPORTED)
-    if(openPMD_USE_INTERNAL_CATCH)
-        target_include_directories(openPMD::thirdparty::Catch2 SYSTEM INTERFACE
-            $<BUILD_INTERFACE:${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/catch2/include>
+    if(TARGET Catch2::Catch2)
+        # nothing to do, target already exists in the superbuild
+        message(STATUS "Catch2::Catch2 target already imported")
+    elseif(openPMD_USE_INTERNAL_CATCH)
+        # Git fetcher
+        set(openPMD_catch_repo "https://github.com/catchorg/Catch2.git"
+            CACHE STRING
+            "Repository URI to pull and build catch2 from if(openPMD_USE_INTERNAL_CATCH)")
+        set(openPMD_catch_branch "v2.13.10"  # ToDo Migrate to v3 and latest release
+            CACHE STRING
+            "Repository branch for openPMD_catch_branch if(openPMD_USE_INTERNAL_CATCH)")
+
+        message(STATUS "Downloading Catch2 ...")
+        message(STATUS "Catch2 repository: ${openPMD_catch_repo} (${openPMD_catch_branch})")
+
+        FetchContent_Declare(
+            Catch2
+            GIT_REPOSITORY  ${openPMD_catch_repo}
+            GIT_TAG         ${openPMD_catch_branch}
+            BUILD_IN_SOURCE 0
         )
-        message(STATUS "Catch2: Using INTERNAL version '2.13.10'")
+        FetchContent_MakeAvailable(Catch2)
     else()
         find_package(Catch2 2.13.10 REQUIRED CONFIG)
         target_link_libraries(openPMD::thirdparty::Catch2
             INTERFACE Catch2::Catch2)
         message(STATUS "Catch2: Found version '${Catch2_VERSION}'")
     endif()
+    add_library(openPMD::thirdparty::Catch2 INTERFACE IMPORTED)
+    target_link_libraries(openPMD::thirdparty::Catch2
+            INTERFACE Catch2::Catch2)
 endif()
 
 if(openPMD_HAVE_MPI)
@@ -1458,7 +1515,6 @@ if(openPMD_BUILD_TESTING)
     endif()
 endif()
 
-
 # Status Message for Build Options ############################################
 #
 openpmd_print_summary()
diff --git a/NEWS.rst b/NEWS.rst
@@ -13,6 +13,7 @@ Please transition to ADIOS2.
 For reading legacy ADIOS1 BP3 files, either use an older version of openPMD-api or the BP3 backend in ADIOS2.
 Note that ADIOS2 does not support compression in BP3 files.
 
+CMake 3.22.0 is now the minimally supported version for CMake.
 pybind11 2.12.0 is now the minimally supported version for Python support.
 
 

diff --git a/README.md b/README.md
@@ -96,7 +96,7 @@ Our manual shows full [read & write examples](https://openpmd-api.readthedocs.io
 ## Dependencies
 
 Required:
-* CMake 3.15.0+
+* CMake 3.22.0+
 * C++17 capable compiler, e.g., g++ 7+, clang 7+, MSVC 19.15+, icpc 19+, icpx
 
 Shipped internally in `share/openPMD/thirdParty/`:
@@ -261,15 +261,13 @@ CMake controls options with prefixed `-D`, e.g. `-DopenPMD_USE_MPI=OFF`:
 <sup>1</sup> *e.g. changes C++ visibility keywords, breaks MSVC*
 <sup>2</sup> *this includes most pre-/post-condition checks, disabling without specific cause is highly discouraged*
 
-Additionally, the following libraries are shipped internally.
-The following options allow to switch to external installs:
 
-| CMake Option                    | Values     | Library       | Version  |
-|---------------------------------|------------|---------------|----------|
-| `openPMD_USE_INTERNAL_CATCH`    | **ON**/OFF | Catch2        | 2.13.10+ |
-| `openPMD_USE_INTERNAL_PYBIND11` | **ON**/OFF | pybind11      |  2.12.0+ |
-| `openPMD_USE_INTERNAL_JSON`     | **ON**/OFF | NLohmann-JSON |   3.9.1+ |
-| `openPMD_USE_INTERNAL_TOML11`   | **ON**/OFF | toml11        |   3.7.1+ |
+Additionally, the following libraries are downloaded via [FetchContent](https://cmake.org/cmake/help/latest/module/FetchContent.html)
+during the configuration of the project or, if the corresponding `<PACKAGENAME>_ROOT` variable is provided, can be provided externally:
+* [Catch2](https://github.com/catchorg/Catch2) (2.13.10+)
+* [PyBind11](https://github.com/pybind/pybind11) (2.12.0+)
+* [NLohmann-JSON](https://github.com/nlohmann/json) (3.9.1+)
+* [toml11](https://github.com/ToruNiina/toml11) (3.7.1+)
 
 By default, this will build as a shared library (`libopenPMD.[so|dylib|dll]`) and installs also its headers.
 In order to build a static library, append `-DBUILD_SHARED_LIBS=OFF` to the `cmake` command.

diff --git a/docs/source/dev/buildoptions.rst b/docs/source/dev/buildoptions.rst
@@ -59,18 +59,19 @@ In order to build with debug symbols, pass ``-DCMAKE_BUILD_TYPE=Debug`` to your
 Shipped Dependencies
 --------------------
 
-Additionally, the following libraries are shipped internally for convenience.
+Additionally, the following libraries are downloaded and built internally for convenience, with defaults controlled by the ``openPMD_SUPERBUILD`` option.
 These might get installed in your :ref:`CMAKE_INSTALL_PREFIX <install-cmake>` if the option is ``ON``.
 
 The following options allow to switch to external installs of dependencies:
 
 ================================= =========== ======== ============= ========
 CMake Option                      Values      Installs Library       Version
 ================================= =========== ======== ============= ========
-``openPMD_USE_INTERNAL_CATCH``    **ON**/OFF  No       Catch2        2.13.10+
-``openPMD_USE_INTERNAL_PYBIND11`` **ON**/OFF  No       pybind11       2.12.0+
-``openPMD_USE_INTERNAL_JSON``     **ON**/OFF  No       NLohmann-JSON   3.9.1+
-``openPMD_USE_INTERNAL_TOML11``   **ON**/OFF  No       toml11          3.7.1+
+``openPMD_SUPERBUILD``            **ON**/OFF
+``openPMD_USE_INTERNAL_CATCH``    ON/OFF      No       Catch2        2.13.10+
+``openPMD_USE_INTERNAL_PYBIND11`` ON/OFF      No       pybind11       2.12.0+
+``openPMD_USE_INTERNAL_JSON``     ON/OFF      No       NLohmann-JSON   3.9.1+
+``openPMD_USE_INTERNAL_TOML11``   ON/OFF      No       toml11          3.7.1+
 ================================= =========== ======== ============= ========
 
 

diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst
@@ -9,7 +9,7 @@ These are currently:
 Required
 --------
 
-* CMake 3.15.0+
+* CMake 3.22.0+
 * C++17 capable compiler, e.g., g++ 7+, clang 7+, MSVC 19.15+, icpc 19+, icpx
 
 Shipped internally

diff --git a/pyproject.toml b/pyproject.toml
@@ -2,7 +2,7 @@
 requires = [
     "setuptools>=42",
     "wheel",
-    "cmake>=3.15.0,<4.0.0",
+    "cmake>=3.22.0,<4.0.0",
     "packaging>=23",
     "pybind11>=2.12.0,<3.0.0"
 ]

diff --git a/setup.py b/setup.py
@@ -22,16 +22,16 @@ def run(self):
             out = subprocess.check_output(['cmake', '--version'])
         except OSError:
             raise RuntimeError(
-                "CMake 3.15.0+ must be installed to build the following " +
+                "CMake 3.22.0+ must be installed to build the following " +
                 "extensions: " +
                 ", ".join(e.name for e in self.extensions))
 
         cmake_version = parse(re.search(
             r'version\s*([\d.]+)',
             out.decode()
         ).group(1))
-        if cmake_version < parse('3.15.0'):
-            raise RuntimeError("CMake >= 3.15.0 is required")
+        if cmake_version < parse('3.22.0'):
+            raise RuntimeError("CMake >= 3.22.0 is required")
 
         for ext in self.extensions:
             self.build_extension(ext)