From 9552ab6af6d109c124e3fd4c75397bfcd86f8576 Mon Sep 17 00:00:00 2001 From: Bartosz Kostrzewa Date: Tue, 15 Dec 2020 11:57:31 +0100 Subject: [PATCH 1/2] disable AC_FUNC_MALLOC to make sure that cross-compilation works --- configure.ac | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index c58f0f3..3eccaf7 100644 --- a/configure.ac +++ b/configure.ac @@ -28,8 +28,10 @@ AC_CHECK_HEADERS([stdint.h], ) # Checks for library functions. -AC_FUNC_MALLOC -AC_CHECK_FUNCS([gettimeofday]) +# AC_FUNC_MALLOC disabled to ensure that cross-compilation works and autoconf does +# not attempt to substitute 'rpl_malloc' +##AC_FUNC_MALLOC +AC_CHECK_FUNCS([malloc realloc calloc gettimeofday]) AC_CONFIG_FILES([Makefile]) AC_CONFIG_FILES([include/Makefile]) From 638a89287ccaa5f5222067b6d1dcd80deef5f989 Mon Sep 17 00:00:00 2001 From: Bartosz Kostrzewa Date: Sat, 27 May 2023 17:25:02 +0200 Subject: [PATCH 2/2] attempt at chunked reading and writing for latticeParallelMapped --- src/internal_LemonSetup.ih | 1 + src/internal_setupIOTypes.static | 3 ++- src/reader_latticeParallelMapped.c | 36 +++++++++++++++++++++-------- src/writer_latticeParallelMapped.c | 37 +++++++++++++++++++----------- 4 files changed, 53 insertions(+), 24 deletions(-) diff --git a/src/internal_LemonSetup.ih b/src/internal_LemonSetup.ih index 385103d..cc6934e 100644 --- a/src/internal_LemonSetup.ih +++ b/src/internal_LemonSetup.ih @@ -36,6 +36,7 @@ typedef struct int ndims; int localVol; int totalVol; + int nranks; int *starts; int *localDims; diff --git a/src/internal_setupIOTypes.static b/src/internal_setupIOTypes.static index 9541eeb..181ae22 100644 --- a/src/internal_setupIOTypes.static +++ b/src/internal_setupIOTypes.static @@ -42,6 +42,7 @@ static void lemonSetupIOTypes(LemonSetup *setup, MPI_Comm cartesian, MPI_Offset /* Gathering of the required MPI data from the cartesian communicator. */ MPI_Cartdim_get(cartesian, &setup->ndims); + MPI_Comm_size(cartesian, &setup->nranks); setup->starts = (int*)malloc(setup->ndims * sizeof(int)); setup->localDims = (int*)malloc(setup->ndims * sizeof(int)); @@ -72,4 +73,4 @@ static void lemonSetupIOTypes(LemonSetup *setup, MPI_Comm cartesian, MPI_Offset MPI_Type_create_subarray(setup->ndims, setup->mappedDims, setup->localDims, setup->starts, MPI_ORDER_C, setup->etype, &setup->ftype); MPI_Type_commit(&setup->ftype); -} \ No newline at end of file +} diff --git a/src/reader_latticeParallelMapped.c b/src/reader_latticeParallelMapped.c index ac1dde8..0774481 100644 --- a/src/reader_latticeParallelMapped.c +++ b/src/reader_latticeParallelMapped.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "internal_LemonSetup.ih" #include "internal_clearReaderState.static" @@ -45,18 +46,33 @@ int lemonReadLatticeParallelMapped(LemonReader *reader, void *data, MPI_Offset s return error; lemonSetupIOTypes(&setup, reader->cartesian, siteSize, latticeDims, mapping); - + /* Install the data organization we worked out above on the file as a view. - We keep the individual file pointers synchronized explicitly, so assume they are here. */ - MPI_File_set_view(*reader->fp, reader->off + reader->pos, setup.etype, setup.ftype, "native", MPI_INFO_NULL); + We keep the individual file pointers synchronized explicitly, so assume they are here. */ + MPI_File_set_view(*reader->fp, reader->off + reader->pos, + setup.etype, setup.ftype, "native", MPI_INFO_NULL); - /* Blast away! */ - MPI_File_read_at_all(*reader->fp, reader->pos, data, setup.localVol, setup.etype, &status); - MPI_Barrier(reader->cartesian); + long int bytes = (long int)setup.localVol * (long int)siteSize; + long int buffer_offset = 0; + while( bytes > 0 ){ + int chunk = bytes > (long int)INT_MAX ? (INT_MAX/siteSize)*siteSize : bytes; + bytes -= (long int)chunk; - /* Synchronize the file pointer */ - MPI_Get_count(&status, MPI_BYTE, &read); - reader->pos += setup.totalVol * siteSize; + /* Blast away! */ + MPI_File_read_at_all(*reader->fp, reader->pos, &data[buffer_offset], chunk/siteSize, setup.etype, &status); + MPI_Barrier(reader->cartesian); + + /* Synchronize the file pointer */ + MPI_Get_count(&status, MPI_BYTE, &read); + buffer_offset += (long int)read; + if (read != chunk) + { + fprintf(stderr, "[LEMON] Node %d reports in lemonReadLatticeParallel:\n" + " Could not read the required amount of data.\n", reader->my_rank); + return LEMON_ERR_READ; + } + reader->pos += (long int)chunk * (long int)setup.nranks; + } /* We want to leave the file in a well-defined state, so we reset the view to a default. */ /* We don't want to reread any data, so we maximize the file pointer globally. */ @@ -66,7 +82,7 @@ int lemonReadLatticeParallelMapped(LemonReader *reader, void *data, MPI_Offset s lemonFreeIOTypes(&setup); /* Doing a data read should never get us to EOF, only header scanning -- any shortfall is an error */ - if (read != siteSize * setup.localVol) + if (buffer_offset != (long int)siteSize * (long int)setup.localVol) { fprintf(stderr, "[LEMON] Node %d reports in lemonReadLatticeParallel:\n" " Could not read the required amount of data.\n", reader->my_rank); diff --git a/src/writer_latticeParallelMapped.c b/src/writer_latticeParallelMapped.c index 72c11ad..7a41971 100644 --- a/src/writer_latticeParallelMapped.c +++ b/src/writer_latticeParallelMapped.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "internal_LemonSetup.ih" #include "internal_clearWriterState.static" @@ -50,13 +51,31 @@ int lemonWriteLatticeParallelMapped(LemonWriter *writer, void *data, MPI_Offset MPI_Barrier(writer->cartesian); MPI_File_set_view(*writer->fp, writer->off + writer->pos, setup.etype, setup.ftype, "native", MPI_INFO_NULL); - /* Blast away! */ - MPI_File_write_at_all(*writer->fp, writer->pos, data, setup.localVol, setup.etype, &status); - MPI_File_sync(*writer->fp); + long int bytes = (long int)setup.localVol * (long int)siteSize; + long int buffer_offset = 0; - MPI_Barrier(writer->cartesian); + while( bytes > 0 ){ + int chunk = bytes > (long int)INT_MAX ? (INT_MAX/siteSize)*siteSize : bytes; + bytes -= (long int)chunk; + + /* Blast away! */ + MPI_File_write_at_all(*writer->fp, writer->pos, &data[buffer_offset], chunk/siteSize, setup.etype, &status); + MPI_File_sync(*writer->fp); + + MPI_Barrier(writer->cartesian); + + writer->pos += (long int)chunk * (long int)setup.nranks; + + MPI_Get_count(&status, MPI_BYTE, &written); - writer->pos += setup.totalVol * siteSize; + buffer_offset += (long int)written; + if (written != chunk) + { + fprintf(stderr, "[LEMON] Node %d reports in lemonWriteLatticeParallel:\n" + " Could not write the required amount of data.\n", writer->my_rank); + return LEMON_ERR_WRITE; + } + } /* We should reset the shared file pointer, in an MPI_BYTE based view... */ MPI_Barrier(writer->cartesian); @@ -65,13 +84,5 @@ int lemonWriteLatticeParallelMapped(LemonWriter *writer, void *data, MPI_Offset /* Free up the resources we claimed for this operation. */ lemonFreeIOTypes(&setup); - MPI_Get_count(&status, MPI_BYTE, &written); - if (written != siteSize * setup.localVol) - { - fprintf(stderr, "[LEMON] Node %d reports in lemonWriteLatticeParallel:\n" - " Could not write the required amount of data.\n", writer->my_rank); - return LEMON_ERR_WRITE; - } - return LEMON_SUCCESS; }