Fxm json (#471)

Add a plugin for outputting with JSON format

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "vendor/run-clang-format"]
 	path = vendor/run-clang-format
 	url = https://github.com/Sarcasm/run-clang-format.git
+[submodule "vendor/json"]
+	path = vendor/json
+	url = https://github.com/nlohmann/json.git

AUTHORS

@@ -18,6 +18,10 @@ Benoit Martin - CEA ([email protected])
 * CMake maintainer (Oct. 2024 - ...)
 * Deisa plugin refactoring
 
+François-Xavier Mordant - CEA ([email protected])
+* Fixed CMake issues, internal API enhancement
+* Bug fix, JSON plugin
+
 Tomasz Paluszkiewicz - PSNC ([email protected])
 The scientific/academic work is financed from financial resources for science
 in the years 2016 - 2018 granted for the realization of the international

CHANGELOG.md

@@ -12,6 +12,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Added
 
+* Added new JSON plugin. Enables users to write data in JSON format, 
+  using the [nlohmann/json](https://github.com/nlohmann/json) library.
+  Users can also write data in other format thanks to taking JSON as input for
+  the dedicated tools.
+  [#440](https://gitlab.maisondelasimulation.fr/pdidev/pdi/-/issues/440)
+
 ### Changed
 
 ### Deprecated
@@ -26,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [1.8.0] - 2024-11-28
 
 ### Added
+
 * NetCDF plugin now support the size_of attribute, fixes
   [#446](https://gitlab.maisondelasimulation.fr/pdidev/pdi/-/issues/446)
 * Deisa plugin for in-situ analysis using Dask

CMakeLists.txt

@@ -77,10 +77,12 @@ option(BUILD_SERIALIZE_PLUGIN   "Build Serialize plug-in" ON)
 option(BUILD_SHARED_LIBS        "Build shared libraries rather than static ones" ON)
 option(BUILD_TRACE_PLUGIN       "Build Trace plugin" ON)
 option(BUILD_USER_CODE_PLUGIN   "Build User-code plugin" ON)
+option(BUILD_JSON_PLUGIN        "Build JSON plugin" OFF)
 option(BUILD_DEISA_PLUGIN   	"Build Deisa plug-in" OFF)
 
 
 
+
 ### Default build type
 
 if(NOT "${CMAKE_BUILD_TYPE}")
@@ -123,6 +125,9 @@ endif()
 if("${BUILD_PYCALL_PLUGIN}")
 	list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/plugins/pycall/cmake")
 endif()
+if ("${BUILD_JSON_PLUGIN}")
+    list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/plugins/json/cmake")
+endif()
 if("${BUILD_TESTING}")
 	list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/example/cmake")
 endif()
@@ -476,6 +481,15 @@ sbuild_add_module(DECL_NETCDF_PLUGIN
 	SUBSTEPS test
 )
 
+
+sbuild_add_module(JSON_PLUGIN
+	ENABLE_BUILD_FLAG BUILD_JSON_PLUGIN
+	SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/plugins/json"
+	DEPENDS PDI
+	SUBSTEPS indent test
+)
+
+
 sbuild_add_module(PYCALL_PLUGIN
 	ENABLE_BUILD_FLAG BUILD_PYCALL_PLUGIN
 	SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/plugins/pycall"
@@ -512,14 +526,12 @@ sbuild_add_module(USER_CODE_PLUGIN
 	SUBSTEPS test
 )
 
-
 sbuild_add_module(DEISA_PLUGIN
 	ENABLE_BUILD_FLAG BUILD_DEISA_PLUGIN
 	SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/plugins/deisa"
 	DEPENDS PDI
 )
 
-
 sbuild_add_module(PDI_EXAMPLE
 	ENABLE_BUILD_FLAG BUILD_TESTING
 	SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/example"

example/CMakeLists.txt

@@ -116,6 +116,18 @@ set_property(TEST PDI_example_decl_netcdf_C PROPERTY PROCESSORS 3)
 endif("${BUILD_NETCDF_PARALLEL}")
 endif("${BUILD_DECL_NETCDF_PLUGIN}")
 
+if("${BUILD_JSON_PLUGIN}")
+add_test(NAME PDI_example_json_C COMMAND "${RUNTEST_DIR}" "${MPIEXEC}" "${MPIEXEC_NUMPROC_FLAG}" 4 ${MPIEXEC_PREFLAGS} "$<TARGET_FILE:PDI_example_C>" ${MPIEXEC_POSTFLAGS} "${CMAKE_CURRENT_SOURCE_DIR}/json.yml")
+set_property(TEST PDI_example_json_C PROPERTY TIMEOUT 30)
+set_property(TEST PDI_example_json_C PROPERTY PROCESSORS 6)
+
+if("${BUILD_FORTRAN}")
+add_test(NAME PDI_example_json_F COMMAND "${RUNTEST_DIR}" "${MPIEXEC}" "${MPIEXEC_NUMPROC_FLAG}" 3 ${MPIEXEC_PREFLAGS} "$<TARGET_FILE:PDI_example_F>" ${MPIEXEC_POSTFLAGS} "${CMAKE_CURRENT_SOURCE_DIR}/json.yml")
+set_property(TEST PDI_example_json_F PROPERTY TIMEOUT 30)
+set_property(TEST PDI_example_json_F PROPERTY PROCESSORS 6)
+endif("${BUILD_FORTRAN}")
+
+endif("${BUILD_JSON_PLUGIN}")
 
 add_test(NAME PDI_example_trace_C COMMAND "${MPIEXEC}" "${MPIEXEC_NUMPROC_FLAG}" 3 ${MPIEXEC_PREFLAGS} "$<TARGET_FILE:PDI_example_C>" ${MPIEXEC_POSTFLAGS} "${CMAKE_CURRENT_SOURCE_DIR}/trace.yml")
 set_property(TEST PDI_example_trace_C PROPERTY TIMEOUT 15)

example/json.yml

@@ -0,0 +1,64 @@
+#=============================================================================
+# Copyright (C) 2023-2024 Commissariat a l'energie atomique et aux energies alternatives (CEA)
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the names of CEA, nor the names of the contributors may be used to
+#   endorse or promote products derived from this software without specific
+#   prior written  permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#=============================================================================
+
+# duration in seconds
+duration: 0.75
+# global [height, width] (excluding boundary conditions or ghosts)
+datasize: [60, 12]
+# degree of parallelism
+parallelism: { height: 2, width: 2 }
+
+# only the following config is passed to PDI
+pdi:
+  metadata: # type of small values for which PDI keeps a copy
+    iter:   int                    # current iteration id
+    dsize:  { size: 2, type: array, subtype: int } # local data size including ghosts/boundary
+    psize:  { size: 2, type: array, subtype: int } # number of processes in each dimension
+    pcoord: { size: 2, type: array, subtype: int } # coordinate of the process
+    mpi_rank: int
+    mpi_size: int
+  data: # type of values for which PDI does not keep a copy
+    main_field: { size: [ '$dsize[0]', '$dsize[1]' ], type: array, subtype: double }
+
+#! [example_config_json_plugin]
+  plugins:
+    json:
+    - pcoord: params_pcoord_proc_${mpi_rank}.json
+    - file : params1.json
+      write : [dsize, psize]
+      when: ${mpi_rank}=0
+    - file : params2.json
+      when: ${mpi_rank}=0
+      write : [mpi_rank, mpi_size, dsize, psize, pcoord]
+    - file : main_field_proc_${mpi_rank}_iter_${iter}.json
+      write : [main_field]
+      when: $iter % 100 = 0
+
+
+#! [example_config_json_plugin]

pdi/docs/CMakeLists.txt

@@ -100,7 +100,8 @@ doxygen_add_docs(doxygen_doc
 		"${PDI_SOURCE_DIR}/../plugins/user_code/README.md"
 		"${PDI_SOURCE_DIR}/../plugins/pycall/README.md"
 		"${PDI_SOURCE_DIR}/../plugins/serialize/README.md"
-		"${PDI_SOURCE_DIR}/../plugins/set_value/README.md")
+		"${PDI_SOURCE_DIR}/../plugins/set_value/README.md"
+		"${PDI_SOURCE_DIR}/../plugins/json/README.md")
 add_subdirectory(First_steps)
 add_dependencies(doc doxygen_doc)
 install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/html/" DESTINATION "${CMAKE_INSTALL_DOCDIR}/html")

pdi/docs/Plugins.md

@@ -7,6 +7,7 @@
 |Deisa plugin                                         |Expose data to a Dask analysis script.                             |
 |\subpage mpi_plugin "MPI plugin"                     |Enables MPI support in %PDI and plugins.                           |
 |\subpage pycall_plugin "Pycall plugin"               |Call python scripts from C application                             |
+|\subpage JSON_plugin "JSON plugin"                   |Export data in JSON format.                                        |
 |\subpage serialize_plugin "Serialize plugin"         |Serializes and deserializes shared data.                           |
 |\subpage set_value_plugin "Set Value plugin"         |Set values to data and metadata from yaml file.                    |
 |\subpage trace_plugin "Trace plugin"                 |Generate a trace of what happens in %PDI data store.               |

plugins/json/AUTHORS

@@ -0,0 +1,13 @@
+Multiple people have contributed to PDI Json plugin. To show our appreciation
+for their public spirit, we list here in alphabetical order a condensed list of
+their contributions.
+
+
+François-Xavier Mordant - CEA ([email protected])
+* Design and initial implementation
+* Plugin documentation
+* Apr. 2023 - Oct. 2023
+
+Yushan Wang - CEA ([email protected])
+* Plugin documentation and implementation
+* Maintainer (Oct. 2024 - ...)

plugins/json/CMakeLists.txt

@@ -0,0 +1,50 @@
+#=============================================================================
+# Copyright (C) 2023-2024 Commissariat a l'energie atomique et aux energies alternatives (CEA)
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# * Redistributions of source code must retain the above copyright
+#   notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+#   notice, this list of conditions and the following disclaimer in the
+#   documentation and/or other materials provided with the distribution.
+# * Neither the name of CEA nor the names of its contributors may be used to
+#   endorse or promote products derived from this software without specific
+#   prior written permission.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#=============================================================================
+
+cmake_minimum_required(VERSION 3.16...3.29)
+project(pdi_json_plugin LANGUAGES C CXX)
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
+
+# Includes
+include(CTest)
+include(GNUInstallDirs)
+
+# PDI
+find_package(PDI REQUIRED COMPONENTS plugins)
+add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/../../vendor/json" nlohmann_json)
+
+# The plugin
+add_library(pdi_json_plugin MODULE json.cxx)
+target_link_libraries(pdi_json_plugin PUBLIC PDI::PDI_plugins nlohmann_json)
+
+# Installation
+set(INSTALL_PDIPLUGINDIR "${PDI_DEFAULT_PLUGINDIR}" CACHE PATH "PDI plugins (${PDI_DEFAULT_PLUGINDIR})")
+install(TARGETS pdi_json_plugin
+		LIBRARY DESTINATION "${INSTALL_PDIPLUGINDIR}"
+)
+
+# Tests
+if("${BUILD_TESTING}")
+	add_subdirectory(tests/)
+endif()