decl_hdf5 MPIO Collective or Independent, fix #419

example/decl_hdf5.yml

@@ -28,6 +28,7 @@ pdi:
       write:
         main_field: # the name of the data to write
           dataset: data
+          mpio: COLLECTIVE # or INDEPENDENT
           when: '$iter<10'   # do only write the first 10 iterations (0...9)
           memory_selection:  # exclude ghosts from the data in memory
             size:  ['$dsize[0]-2', '$dsize[1]-2']

plugins/decl_hdf5/AUTHORS

@@ -11,6 +11,9 @@ Julien Bigot - CEA ([email protected])
 * Run tests that depend on the filesystem in their own temporary directory
 * Buildsystem
 
+Yacine Ould Rouis - CNRS ([email protected])
+* contribution to feature design, validation
+
 Thomas Padioleau - CEA ([email protected])
 * Fixed a bug with parallel file deletion
 

plugins/decl_hdf5/README.md

@@ -145,6 +145,8 @@ The possible values for the keys are as follow:
   See
   https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetFletcher32
   for more information.
+* `mpio` : a string expression to define the type of MPIIO parallel pointer 
+for the operation among two choices : `COLLECTIVE` (default) and `INDEPENDENT`.
 
 ### SELECTION_DESC
 
@@ -162,7 +164,6 @@ Memory selection default values:
 * If the `size` is not specified, it defaults to size of the whole data in each
   dimension.
 * If the `start` is not specified it defaults to 0 in all dimensions.
-
 Dataset selection default values:
 * If the `size` is not specified:
   * if the number of dimensions match that of the memory, the size defaults to

plugins/decl_hdf5/dataset_op.cxx

@@ -1,6 +1,6 @@
 /*******************************************************************************
  * Copyright (C) 2015-2024 Commissariat a l'energie atomique et aux energies alternatives (CEA)
- * Copyright (C) 2021 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC)
+ * Copyright (C) 2022 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC)
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -164,6 +164,14 @@ Dataset_op::Dataset_op(Direction dir, string name, Expression default_when, PC_t
 				m_fletcher = value;
 			} else if (key == "attributes") {
 				// pass
+			} else if (key == "mpio") {
+				if (to_string(value) == "INDEPENDENT") {
+					m_mpio = H5FD_MPIO_INDEPENDENT;
+				} else if (to_string(value) == "COLLECTIVE") {
+					m_mpio = H5FD_MPIO_COLLECTIVE;
+				} else {
+					throw Config_error{key_tree, "Not valid MPIO value: `{}'", to_string(value)};
+				}
 			} else if (key == "collision_policy") {
 				m_collision_policy = to_collision_policy(to_string(value));
 			} else {
@@ -203,8 +211,12 @@ void Dataset_op::fletcher(Context& ctx, Expression value)
 	}
 }
 
-void Dataset_op::execute(Context& ctx, hid_t h5_file, hid_t xfer_lst, const unordered_map<string, Datatype_template_sptr>& dsets)
+void Dataset_op::execute(Context& ctx, hid_t h5_file, bool use_mpio, const unordered_map<string, Datatype_template_ptr>& dsets)
 {
+	Raii_hid xfer_lst = make_raii_hid(H5Pcreate(H5P_DATASET_XFER), H5Pclose);
+	if (use_mpio) {
+		if (0 > H5Pset_dxpl_mpio(xfer_lst, m_mpio)) handle_hdf5_err();
+	}
 	if (m_direction == READ)
 		do_read(ctx, h5_file, xfer_lst);
 	else

plugins/decl_hdf5/dataset_op.h

@@ -1,6 +1,6 @@
 /*******************************************************************************
  * Copyright (C) 2015-2024 Commissariat a l'energie atomique et aux energies alternatives (CEA)
- * Copyright (C) 2021 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC)
+ * Copyright (C) 2021-2022 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC)
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -67,6 +67,8 @@ class Dataset_op
 	/// direction of the transfer (read or write)
 	Direction m_direction;
 
+	H5FD_mpio_xfer_t m_mpio = H5FD_MPIO_COLLECTIVE;
+
 	/// the name of the dataset where to transfer
 	PDI::Expression m_dataset;
 
@@ -182,11 +184,10 @@ class Dataset_op
 	 *
 	 * \param ctx the context in which to operate
 	 * \param h5_file the already opened HDF5 file id
-	 * \param xfer_lst the already created transfer property list including any
-	 *                 parallel HDF5 required property.
+	 * \param use_mpi whether the hdf5 read/write is parallel
 	 * \param dsets the type of the explicitly typed datasets
 	 */
-	void execute(PDI::Context& ctx, hid_t h5_file, hid_t xfer_lst, const std::unordered_map<std::string, PDI::Datatype_template_sptr>& dsets);
+	void execute(PDI::Context& ctx, hid_t h5_file, bool use_mpio, const std::unordered_map<std::string, PDI::Datatype_template_ptr>& dsets);
 
 private:
 	void do_read(PDI::Context& ctx, hid_t h5_file, hid_t read_lst);

plugins/decl_hdf5/file_op.cxx

@@ -1,6 +1,6 @@
 /*******************************************************************************
  * Copyright (C) 2015-2024 Commissariat a l'energie atomique et aux energies alternatives (CEA)
- * Copyright (C) 2021 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC)
+ * Copyright (C) 2021-2022 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC)
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -280,15 +280,15 @@ void File_op::execute(Context& ctx)
 	std::string filename = m_file.to_string(ctx);
 
 	Raii_hid file_lst = make_raii_hid(H5Pcreate(H5P_FILE_ACCESS), H5Pclose);
-	Raii_hid xfer_lst = make_raii_hid(H5Pcreate(H5P_DATASET_XFER), H5Pclose);
+	bool use_mpio = false;
 #ifdef H5_HAVE_PARALLEL
 	MPI_Comm comm = MPI_COMM_SELF;
 	if (communicator()) {
 		comm = *(static_cast<const MPI_Comm*>(Ref_r{communicator().to_ref(ctx)}.get()));
 	}
 	if (comm != MPI_COMM_SELF) {
 		if (0 > H5Pset_fapl_mpio(file_lst, comm, MPI_INFO_NULL)) handle_hdf5_err();
-		if (0 > H5Pset_dxpl_mpio(xfer_lst, H5FD_MPIO_COLLECTIVE)) handle_hdf5_err();
+		use_mpio = true;
 		ctx.logger().debug("Opening `{}' file in parallel mode", filename);
 	}
 #endif
@@ -337,10 +337,10 @@ void File_op::execute(Context& ctx)
 	Raii_hid h5_file = make_raii_hid(h5_file_raw, H5Fclose, ("Cannot open `" + filename + "' file").c_str());
 
 	for (auto&& one_dset_op: dset_writes) {
-		one_dset_op.execute(ctx, h5_file, xfer_lst, m_datasets);
+		one_dset_op.execute(ctx, h5_file, use_mpio, m_datasets);
 	}
 	for (auto&& one_dset_op: dset_reads) {
-		one_dset_op.execute(ctx, h5_file, xfer_lst, m_datasets);
+		one_dset_op.execute(ctx, h5_file, use_mpio, m_datasets);
 	}
 	for (auto&& one_attr_op: attr_writes) {
 		one_attr_op.execute(ctx, h5_file);

plugins/decl_hdf5/tests/CMakeLists.txt

@@ -1,7 +1,7 @@
 #=============================================================================
 # Copyright (C) 2015-2024 Commissariat a l'energie atomique et aux energies alternatives (CEA)
 # Copyright (C) 2021 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC)
-#
+# Copyright (C) 2022 Centre National de Recherche Scientifique (CNRS)
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -92,7 +92,7 @@ if("${BUILD_HDF5_PARALLEL}")
 	set_property(TEST decl_hdf5_mpi_04_C PROPERTY PROCESSORS 4)
 endif()
 
-# bcommunicator as a reference
+# communicator as a reference
 if("${BUILD_HDF5_PARALLEL}")
 	add_executable(decl_hdf5_mpi_05_C decl_hdf5_mpi_test_05.c)
 	target_link_libraries(decl_hdf5_mpi_05_C PDI::PDI_C MPI::MPI_C)
@@ -110,6 +110,15 @@ if("${BUILD_HDF5_PARALLEL}")
 	set_property(TEST decl_hdf5_mpi_06_C PROPERTY PROCESSORS 4)
 endif()
 
+# mpio dataset attribute
+if("${BUILD_HDF5_PARALLEL}")
+	add_executable(decl_hdf5_mpi_07_C decl_hdf5_mpi_test_07.c)
+	target_link_libraries(decl_hdf5_mpi_07_C PDI::PDI_C MPI::MPI_C)
+	add_test(NAME decl_hdf5_mpi_07_C COMMAND "${RUNTEST_DIR}" "${MPIEXEC}" "${MPIEXEC_NUMPROC_FLAG}" 4 ${MPIEXEC_PREFLAGS} "$<TARGET_FILE:decl_hdf5_mpi_07_C>" ${MPIEXEC_POSTFLAGS})
+	set_property(TEST decl_hdf5_mpi_07_C PROPERTY TIMEOUT 15)
+	set_property(TEST decl_hdf5_mpi_07_C PROPERTY PROCESSORS 4)
+endif()
+
 add_executable(decl_hdf5_IO_options_C decl_hdf5_test_IO_options.c)
 target_link_libraries(decl_hdf5_IO_options_C PDI::PDI_C  ${HDF5_DEPS})
 add_test(NAME decl_hdf5_IO_options_C COMMAND "${RUNTEST_DIR}" "$<TARGET_FILE:decl_hdf5_IO_options_C>")