From 44c2316e514b7b0f14b91889f7d34d8c6d8584a3 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Thu, 17 Oct 2024 11:53:46 -0400 Subject: [PATCH 01/11] Adds scripts and Dockerfiles for autoformatting and format checks --- scripts/formatting/Dockerfile.check | 32 +++++++++++ scripts/formatting/Dockerfile.format | 32 +++++++++++ scripts/formatting/autoformat-docker.sh | 24 +++++++++ scripts/formatting/autoformat.sh | 53 +++++++++++++++++++ scripts/formatting/check-formatting-docker.sh | 24 +++++++++ scripts/formatting/check-formatting.sh | 53 +++++++++++++++++++ 6 files changed, 218 insertions(+) create mode 100644 scripts/formatting/Dockerfile.check create mode 100644 scripts/formatting/Dockerfile.format create mode 100755 scripts/formatting/autoformat-docker.sh create mode 100755 scripts/formatting/autoformat.sh create mode 100755 scripts/formatting/check-formatting-docker.sh create mode 100755 scripts/formatting/check-formatting.sh diff --git a/scripts/formatting/Dockerfile.check b/scripts/formatting/Dockerfile.check new file mode 100644 index 00000000..2f6cdd7d --- /dev/null +++ b/scripts/formatting/Dockerfile.check @@ -0,0 +1,32 @@ +FROM ubuntu:24.04 + +USER root + +ENV LOCAL_USER=jovyan \ + LOCAL_UID=1001 \ + HOME=/home/jovyan + +RUN apt-get update \ + && apt-get -qq install -y --no-install-recommends \ + adduser \ + build-essential \ + clang-format-17 \ + && rm -rf /var/lib/apt/lists/* + +RUN adduser \ + --disabled-password \ + --gecos "Default user" \ + --uid ${LOCAL_UID} \ + --home ${HOME} \ + --force-badname \ + ${LOCAL_USER} + +COPY --chown=${LOCAL_USER} --chmod=777 ./scripts /home/jovyan/ +COPY --chown=${LOCAL_USER} --chmod=666 ./src /home/jovyan/ +COPY --chown=${LOCAL_USER} --chmod=666 ./include /home/jovyan/ +COPY --chown=${LOCAL_USER} --chmod=666 ./examples /home/jovyan/ +COPY --chown=${LOCAL_USER} --chmod=666 ./test /home/jovyan/ + +USER ${LOCAL_USER} + +ENTRYPOINT [ "/home/jovyan/scripts/formatting/check-formatting.sh", "/usr/bin/clang-format-17" ] \ No newline at end of file diff --git a/scripts/formatting/Dockerfile.format b/scripts/formatting/Dockerfile.format new file mode 100644 index 00000000..20fb4159 --- /dev/null +++ b/scripts/formatting/Dockerfile.format @@ -0,0 +1,32 @@ +FROM ubuntu:24.04 + +USER root + +ENV LOCAL_USER=jovyan \ + LOCAL_UID=1001 \ + HOME=/home/jovyan + +RUN apt-get update \ + && apt-get -qq install -y --no-install-recommends \ + adduser \ + build-essential \ + clang-format-17 \ + && rm -rf /var/lib/apt/lists/* + +RUN adduser \ + --disabled-password \ + --gecos "Default user" \ + --uid ${LOCAL_UID} \ + --home ${HOME} \ + --force-badname \ + ${LOCAL_USER} + +COPY --chown=${LOCAL_USER} --chmod=777 ./scripts /home/jovyan/ +COPY --chown=${LOCAL_USER} --chmod=666 ./src /home/jovyan/ +COPY --chown=${LOCAL_USER} --chmod=666 ./include /home/jovyan/ +COPY --chown=${LOCAL_USER} --chmod=666 ./examples /home/jovyan/ +COPY --chown=${LOCAL_USER} --chmod=666 ./test /home/jovyan/ + +USER ${LOCAL_USER} + +ENTRYPOINT [ "/home/jovyan/scripts/formatting/autoformat.sh", "/usr/bin/clang-format-17" ] \ No newline at end of file diff --git a/scripts/formatting/autoformat-docker.sh b/scripts/formatting/autoformat-docker.sh new file mode 100755 index 00000000..15bea2c3 --- /dev/null +++ b/scripts/formatting/autoformat-docker.sh @@ -0,0 +1,24 @@ +#! /bin/bash + +if [ command -v docker >/dev/null 2>&1 ]; then + echo "Autoformatting with Docker requires Docker to be available" + exit 1 +fi + +set -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +curr_dir=$(pwd) + +cd $SCRIPT_DIR +cd .. +cd .. + +echo "Bulding Docker image" +docker build -t caliper-autoformat -f ./scripts/formatting/Dockerfile.format . + +echo "Running Docker container to autoformat code with clang-format 17.0.6" +docker run --rm -v $(pwd):/home/jovyan --name caliper-autoformat-container caliper-autoformat + +cd $curr_dir \ No newline at end of file diff --git a/scripts/formatting/autoformat.sh b/scripts/formatting/autoformat.sh new file mode 100755 index 00000000..10ff6624 --- /dev/null +++ b/scripts/formatting/autoformat.sh @@ -0,0 +1,53 @@ +#! /bin/bash + +clang_format_exe="clang-format" +if [ $# -ge 1 ]; then + clang_format_exe="$1" +fi + +SUPPORTED_CLANG_FORMAT_VERSION="17.0.6" + +if [ command -v $clang_format_exe >/dev/null 2>&1 ]; then + echo "You must have 'clang-format' in PATH to use 'autoformat.sh'" + exit 1 +fi + +clang_format_version_str=$($clang_format_exe --version) +clang_format_version=$(echo "$clang_format_version_str" | grep -oP 'clang-format version \K\d+(\.\d+)+') + +if [ "$clang_format_version" != "$SUPPORTED_CLANG_FORMAT_VERSION" ]; then + echo "WARNING: the .clang-format file in this repo is designed for version 16.0.0." + echo " You are running with clang-format v$clang_format_version." + echo " The resulting formatting is highly likely to be incorrect." +fi + +if [ command -v find >/dev/null 2>&1 ]; then + echo "You must have 'find' in PATH to use 'autoformat.sh'" + exit 1 +fi + +if [ command -v dirname >/dev/null 2>&1 ]; then + echo "You must have 'dirname' in PATH to use 'autoformat.sh'" + exit 1 +fi + +if [ command -v xargs >/dev/null 2>&1 ]; then + echo "You must have 'dirname' in PATH to use 'autoformat.sh'" + exit 1 +fi + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +curr_dir=$(pwd) + +cd $SCRIPT_DIR +cd .. +cd .. + +echo "Formatting C/C++ code in 'src'" +find src -name "*.c" -or -name "*.cpp" -or -name "*.h" -or -name "*.hpp" | xargs $clang_format_exe -i + +echo "Formatting C/C++ code in 'include'" +find include -name "*.c" -or -name "*.cpp" -or -name "*.h" -or -name "*.hpp" | xargs $clang_format_exe -i + +cd $curr_dir \ No newline at end of file diff --git a/scripts/formatting/check-formatting-docker.sh b/scripts/formatting/check-formatting-docker.sh new file mode 100755 index 00000000..bc3b97b1 --- /dev/null +++ b/scripts/formatting/check-formatting-docker.sh @@ -0,0 +1,24 @@ +#! /bin/bash + +if [ command -v docker >/dev/null 2>&1 ]; then + echo "Formatting check with Docker requires Docker to be available" + exit 1 +fi + +set -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +curr_dir=$(pwd) + +cd $SCRIPT_DIR +cd .. +cd .. + +echo "Bulding Docker image" +docker build -t caliper-check-format -f ./scripts/formatting/Dockerfile.check . + +echo "Running Docker container to check formatting of code with clang-format 17.0.6" +docker run --rm -v $(pwd):/home/jovyan --name caliper-check-format-container caliper-check-format + +cd $curr_dir \ No newline at end of file diff --git a/scripts/formatting/check-formatting.sh b/scripts/formatting/check-formatting.sh new file mode 100755 index 00000000..bc931c3f --- /dev/null +++ b/scripts/formatting/check-formatting.sh @@ -0,0 +1,53 @@ +#! /bin/bash + +clang_format_exe="clang-format" +if [ $# -ge 1 ]; then + clang_format_exe="$1" +fi + +SUPPORTED_CLANG_FORMAT_VERSION="17.0.6" + +if [ command -v $clang_format_exe >/dev/null 2>&1 ]; then + echo "You must have 'clang-format' in PATH to use 'check-formatting.sh'" + exit 1 +fi + +clang_format_version_str=$($clang_format_exe --version) +clang_format_version=$(echo "$clang_format_version_str" | grep -oP 'clang-format version \K\d+(\.\d+)+') + +if [ "$clang_format_version" != "$SUPPORTED_CLANG_FORMAT_VERSION" ]; then + echo "WARNING: the .clang-format file in this repo is designed for version $SUPPORTED_CLANG_FORMAT_VERSION." + echo " You are running with clang-format v$clang_format_version." + echo " The resulting check is highly likely to be incorrect." +fi + +if [ command -v find >/dev/null 2>&1 ]; then + echo "You must have 'find' in PATH to use 'check-formatting.sh'" + exit 1 +fi + +if [ command -v dirname >/dev/null 2>&1 ]; then + echo "You must have 'dirname' in PATH to use 'check-formatting.sh'" + exit 1 +fi + +if [ command -v xargs >/dev/null 2>&1 ]; then + echo "You must have 'dirname' in PATH to use 'check-formatting.sh'" + exit 1 +fi + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +curr_dir=$(pwd) + +cd $SCRIPT_DIR +cd .. +cd .. + +echo "Check formatting of C/C++ code in 'src'" +find src -name "*.c" -or -name "*.cpp" -or -name "*.h" -or -name "*.hpp" | xargs $clang_format_exe --dry-run -Werror + +echo "Check formatting of C/C++ code in 'include'" +find include -name "*.c" -or -name "*.cpp" -or -name "*.h" -or -name "*.hpp" | xargs $clang_format_exe --dry-run -Werror + +cd $curr_dir \ No newline at end of file From 50edfa6215c2c411443168cdf2fe66c04eabd171 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Thu, 17 Oct 2024 11:54:45 -0400 Subject: [PATCH 02/11] Adds GitHub Actions workflow for checking if PRs are correctly formatted --- .github/workflows/format-check.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/format-check.yml diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml new file mode 100644 index 00000000..49e2d4d0 --- /dev/null +++ b/.github/workflows/format-check.yml @@ -0,0 +1,25 @@ +name: Check code formatting + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + cpp-format-check: + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v4 + + - name: Install clang-format + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential \ + clang-format-17 + + - name: Test formatting + run: | + ./scripts/formatting/check-formatting.sh /usr/bin/clang-format-17 \ No newline at end of file From c8e5f9e75c726c0b25f7730ef371035057045820 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Thu, 17 Oct 2024 12:05:05 -0400 Subject: [PATCH 03/11] Removes directories that existed in Caliper but don't in DYAD --- scripts/formatting/Dockerfile.check | 2 -- scripts/formatting/Dockerfile.format | 2 -- 2 files changed, 4 deletions(-) diff --git a/scripts/formatting/Dockerfile.check b/scripts/formatting/Dockerfile.check index 2f6cdd7d..68b10fba 100644 --- a/scripts/formatting/Dockerfile.check +++ b/scripts/formatting/Dockerfile.check @@ -24,8 +24,6 @@ RUN adduser \ COPY --chown=${LOCAL_USER} --chmod=777 ./scripts /home/jovyan/ COPY --chown=${LOCAL_USER} --chmod=666 ./src /home/jovyan/ COPY --chown=${LOCAL_USER} --chmod=666 ./include /home/jovyan/ -COPY --chown=${LOCAL_USER} --chmod=666 ./examples /home/jovyan/ -COPY --chown=${LOCAL_USER} --chmod=666 ./test /home/jovyan/ USER ${LOCAL_USER} diff --git a/scripts/formatting/Dockerfile.format b/scripts/formatting/Dockerfile.format index 20fb4159..f7ff997d 100644 --- a/scripts/formatting/Dockerfile.format +++ b/scripts/formatting/Dockerfile.format @@ -24,8 +24,6 @@ RUN adduser \ COPY --chown=${LOCAL_USER} --chmod=777 ./scripts /home/jovyan/ COPY --chown=${LOCAL_USER} --chmod=666 ./src /home/jovyan/ COPY --chown=${LOCAL_USER} --chmod=666 ./include /home/jovyan/ -COPY --chown=${LOCAL_USER} --chmod=666 ./examples /home/jovyan/ -COPY --chown=${LOCAL_USER} --chmod=666 ./test /home/jovyan/ USER ${LOCAL_USER} From 74a36f7aadf2a2783ba6744759249b6e710b35ce Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Thu, 17 Oct 2024 12:06:47 -0400 Subject: [PATCH 04/11] Fixes a bug in .clang-format --- .clang-format | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.clang-format b/.clang-format index b18dc481..d7dc8504 100644 --- a/.clang-format +++ b/.clang-format @@ -23,4 +23,4 @@ TabWidth: '4' UseTab: Never AlwaysBreakAfterReturnType: None AlwaysBreakAfterDefinitionReturnType: None -ContinuationIndentWidth: Always \ No newline at end of file +ContinuationIndentWidth: '4' \ No newline at end of file From c680ced351eac38ddc4aad045f3ec6574655785e Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Thu, 17 Oct 2024 12:09:43 -0400 Subject: [PATCH 05/11] Adds logic to fail if any check fails in check-formatting.sh --- .github/workflows/format-check.yml | 4 ++-- scripts/formatting/check-formatting.sh | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml index 49e2d4d0..56b3f452 100644 --- a/.github/workflows/format-check.yml +++ b/.github/workflows/format-check.yml @@ -2,9 +2,9 @@ name: Check code formatting on: push: - branches: [ master ] + branches: [ main ] pull_request: - branches: [ master ] + branches: [ main ] jobs: cpp-format-check: diff --git a/scripts/formatting/check-formatting.sh b/scripts/formatting/check-formatting.sh index bc931c3f..2b2bfeb6 100755 --- a/scripts/formatting/check-formatting.sh +++ b/scripts/formatting/check-formatting.sh @@ -46,8 +46,14 @@ cd .. echo "Check formatting of C/C++ code in 'src'" find src -name "*.c" -or -name "*.cpp" -or -name "*.h" -or -name "*.hpp" | xargs $clang_format_exe --dry-run -Werror +src_check_rc=$? echo "Check formatting of C/C++ code in 'include'" find include -name "*.c" -or -name "*.cpp" -or -name "*.h" -or -name "*.hpp" | xargs $clang_format_exe --dry-run -Werror +include_check_rc=$? -cd $curr_dir \ No newline at end of file +cd $curr_dir + +if [ $src_check_rc -ne 0 ] || [ $include_check_rc -ne 0 ]; then + exit 2 +fi \ No newline at end of file From 85e99f255994514e8277dae2d8278222f7a725d9 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Fri, 18 Oct 2024 15:30:34 -0400 Subject: [PATCH 06/11] Adds paragraph to docs about ordering of #include directives --- docs/developer_guide.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/developer_guide.rst b/docs/developer_guide.rst index df9e3f49..f4394899 100644 --- a/docs/developer_guide.rst +++ b/docs/developer_guide.rst @@ -12,3 +12,10 @@ etiquette: * `The 'Contributing' Page from the Flux Framework's ReadTheDocs `_ * `The Flux Coding Style Guide `_ (used for C code) * `The black Coding Style Guide `_ (used for Python code) + +Besides the broader Flux contribution guidelines above, we also expect C/C++ :code:`#include` +directives to be grouped. Each group should be separated by one newline. +Includes for external tools, system headers, and the C++ +Standard Library should be placed in one group, and includes for internal DYAD headers +should be placed in another group. The group for external headers should be placed +before the group for internal headers. From 81de2971f034fa76fc88ea001ac04fe2eea755ec Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Fri, 18 Oct 2024 15:37:57 -0400 Subject: [PATCH 07/11] Tweaks .clang-format to add newline to end of file and fix include grouping and ordering --- .clang-format | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.clang-format b/.clang-format index d7dc8504..c6c25e9a 100644 --- a/.clang-format +++ b/.clang-format @@ -23,4 +23,7 @@ TabWidth: '4' UseTab: Never AlwaysBreakAfterReturnType: None AlwaysBreakAfterDefinitionReturnType: None -ContinuationIndentWidth: '4' \ No newline at end of file +ContinuationIndentWidth: '4' +IncludeBlocks: 'Preserve' +SortIncludes: 'CaseSensitive' +InsertNewlineAtEOF: 'true' \ No newline at end of file From a7c4ec987785e0b1a61999365039c5fec2b5d944 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Fri, 18 Oct 2024 15:56:10 -0400 Subject: [PATCH 08/11] Updates Docker stuff to use multi-stage builds --- scripts/formatting/Dockerfile.check | 30 ------------------- scripts/formatting/Dockerfile.format | 10 +++++-- scripts/formatting/autoformat-docker.sh | 6 ++-- scripts/formatting/check-formatting-docker.sh | 6 ++-- 4 files changed, 16 insertions(+), 36 deletions(-) delete mode 100644 scripts/formatting/Dockerfile.check diff --git a/scripts/formatting/Dockerfile.check b/scripts/formatting/Dockerfile.check deleted file mode 100644 index 68b10fba..00000000 --- a/scripts/formatting/Dockerfile.check +++ /dev/null @@ -1,30 +0,0 @@ -FROM ubuntu:24.04 - -USER root - -ENV LOCAL_USER=jovyan \ - LOCAL_UID=1001 \ - HOME=/home/jovyan - -RUN apt-get update \ - && apt-get -qq install -y --no-install-recommends \ - adduser \ - build-essential \ - clang-format-17 \ - && rm -rf /var/lib/apt/lists/* - -RUN adduser \ - --disabled-password \ - --gecos "Default user" \ - --uid ${LOCAL_UID} \ - --home ${HOME} \ - --force-badname \ - ${LOCAL_USER} - -COPY --chown=${LOCAL_USER} --chmod=777 ./scripts /home/jovyan/ -COPY --chown=${LOCAL_USER} --chmod=666 ./src /home/jovyan/ -COPY --chown=${LOCAL_USER} --chmod=666 ./include /home/jovyan/ - -USER ${LOCAL_USER} - -ENTRYPOINT [ "/home/jovyan/scripts/formatting/check-formatting.sh", "/usr/bin/clang-format-17" ] \ No newline at end of file diff --git a/scripts/formatting/Dockerfile.format b/scripts/formatting/Dockerfile.format index f7ff997d..9131c6cf 100644 --- a/scripts/formatting/Dockerfile.format +++ b/scripts/formatting/Dockerfile.format @@ -1,4 +1,4 @@ -FROM ubuntu:24.04 +FROM ubuntu:24.04 as base USER root @@ -27,4 +27,10 @@ COPY --chown=${LOCAL_USER} --chmod=666 ./include /home/jovyan/ USER ${LOCAL_USER} -ENTRYPOINT [ "/home/jovyan/scripts/formatting/autoformat.sh", "/usr/bin/clang-format-17" ] \ No newline at end of file +FROM base as autoformat + +ENTRYPOINT [ "/home/jovyan/scripts/formatting/autoformat.sh", "/usr/bin/clang-format-17" ] + +FROM base as check + +ENTRYPOINT [ "/home/jovyan/scripts/formatting/check-formatting.sh", "/usr/bin/clang-format-17" ] diff --git a/scripts/formatting/autoformat-docker.sh b/scripts/formatting/autoformat-docker.sh index 15bea2c3..ee0007fc 100755 --- a/scripts/formatting/autoformat-docker.sh +++ b/scripts/formatting/autoformat-docker.sh @@ -15,10 +15,12 @@ cd $SCRIPT_DIR cd .. cd .. +export DOCKER_BUILDKIT=1 + echo "Bulding Docker image" -docker build -t caliper-autoformat -f ./scripts/formatting/Dockerfile.format . +docker build --target autoformat -t dyad-autoformat -f ./scripts/formatting/Dockerfile.format . echo "Running Docker container to autoformat code with clang-format 17.0.6" -docker run --rm -v $(pwd):/home/jovyan --name caliper-autoformat-container caliper-autoformat +docker run --rm -v $(pwd):/home/jovyan --name dyad-autoformat-container dyad-autoformat cd $curr_dir \ No newline at end of file diff --git a/scripts/formatting/check-formatting-docker.sh b/scripts/formatting/check-formatting-docker.sh index bc3b97b1..307103b9 100755 --- a/scripts/formatting/check-formatting-docker.sh +++ b/scripts/formatting/check-formatting-docker.sh @@ -15,10 +15,12 @@ cd $SCRIPT_DIR cd .. cd .. +export DOCKER_BUILDKIT=1 + echo "Bulding Docker image" -docker build -t caliper-check-format -f ./scripts/formatting/Dockerfile.check . +docker build --target check -t dyad-check-format -f ./scripts/formatting/Dockerfile.format . echo "Running Docker container to check formatting of code with clang-format 17.0.6" -docker run --rm -v $(pwd):/home/jovyan --name caliper-check-format-container caliper-check-format +docker run --rm -v $(pwd):/home/jovyan --name dyad-check-format-container dyad-check-format cd $curr_dir \ No newline at end of file From c0ea2b21a19eab39773f8cd202092d0a35d22cc2 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Fri, 18 Oct 2024 16:06:07 -0400 Subject: [PATCH 09/11] Applies results of autoformatting to repo + manually divides includes into groups --- include/dyad/common/dyad_dtl.h | 16 +- include/dyad/common/dyad_rc.h | 43 +- include/dyad/core/dyad_ctx.h | 46 +- include/dyad/stream/dyad_params.hpp | 2 +- include/dyad/stream/dyad_stream_api.hpp | 169 +- include/dyad/stream/dyad_stream_core.hpp | 10 +- src/dyad/common/dyad_logging.h | 74 +- src/dyad/core/dyad_ctx.c | 1444 +++++++++-------- src/dyad/dtl/dyad_dtl_api.c | 30 +- src/dyad/dtl/dyad_dtl_api.h | 56 +- src/dyad/dtl/flux_dtl.c | 74 +- src/dyad/dtl/ucx_dtl.c | 262 +-- src/dyad/dtl/ucx_dtl.h | 11 +- src/dyad/dtl/ucx_ep_cache.cpp | 392 ++--- src/dyad/dtl/ucx_ep_cache.h | 50 +- src/dyad/modules/dyad.c | 793 +++++---- src/dyad/modules/test_opt_parse.c | 321 ++-- src/dyad/stream/dyad_stream_core.cpp | 365 +++-- src/dyad/utils/base64/base64.c | 4 +- src/dyad/utils/base64/base64.h | 14 +- .../utils/test_cmp_canonical_path_prefix.c | 80 +- src/dyad/utils/test_murmur3.c | 14 +- src/dyad/utils/utils.c | 178 +- src/dyad/utils/utils.h | 85 +- src/dyad/wrapper/flux_barrier.c | 14 +- 25 files changed, 2324 insertions(+), 2223 deletions(-) diff --git a/include/dyad/common/dyad_dtl.h b/include/dyad/common/dyad_dtl.h index f9acbfa7..5bcae248 100644 --- a/include/dyad/common/dyad_dtl.h +++ b/include/dyad/common/dyad_dtl.h @@ -11,15 +11,16 @@ extern "C" { #endif - -enum dyad_dtl_mode { DYAD_DTL_UCX = 0, - DYAD_DTL_FLUX_RPC = 1, - DYAD_DTL_DEFAULT = 1, - DYAD_DTL_END = 2 }; +enum dyad_dtl_mode { + DYAD_DTL_UCX = 0, + DYAD_DTL_FLUX_RPC = 1, + DYAD_DTL_DEFAULT = 1, + DYAD_DTL_END = 2 +}; typedef enum dyad_dtl_mode dyad_dtl_mode_t; -static const char* dyad_dtl_mode_name[DYAD_DTL_END+1] __attribute__((unused)) - = {"UCX", "FLUX_RPC", "DTL_UNKNOWN"}; +static const char* dyad_dtl_mode_name[DYAD_DTL_END + 1] + __attribute__ ((unused)) = {"UCX", "FLUX_RPC", "DTL_UNKNOWN"}; enum dyad_dtl_comm_mode { DYAD_COMM_NONE = 0, // Sanity check value for when @@ -30,7 +31,6 @@ enum dyad_dtl_comm_mode { }; typedef enum dyad_dtl_comm_mode dyad_dtl_comm_mode_t; - #define DYAD_DTL_RPC_NAME "dyad.fetch" struct dyad_dtl; diff --git a/include/dyad/common/dyad_rc.h b/include/dyad/common/dyad_rc.h index 5921141c..aa657254 100644 --- a/include/dyad/common/dyad_rc.h +++ b/include/dyad/common/dyad_rc.h @@ -26,9 +26,9 @@ extern "C" { #endif enum dyad_core_return_codes { - DYAD_RC_OK = 0, // Operation worked correctly - DYAD_RC_SYSFAIL = -1, // Some sys call or C standard - // library call failed + DYAD_RC_OK = 0, // Operation worked correctly + DYAD_RC_SYSFAIL = -1, // Some sys call or C standard + // library call failed // Internal DYAD_RC_NOCTX = -1000, // No DYAD Context found DYAD_RC_BADMETADATA = -1001, // Cannot create/populate a DYAD response @@ -42,27 +42,26 @@ enum dyad_core_return_codes { DYAD_RC_BAD_CLI_PARSE = -1009, // Trying to parse CLI arguments failed DYAD_RC_BADBUF = -1010, // Invalid buffer/pointer passed to function - // FLUX - DYAD_RC_FLUXFAIL = -2000, // Some Flux function failed - DYAD_RC_BADCOMMIT = -2001, // Flux KVS commit didn't work - DYAD_RC_NOTFOUND = -2002, // Flux KVS lookup didn't work - DYAD_RC_BADFETCH = -2003, // Flux KVS commit didn't work - DYAD_RC_BADRPC = -2004, // Flux RPC pack or get didn't work - DYAD_RC_BADPACK = -2005, // JSON packing failed - DYAD_RC_BADUNPACK = -2006, // JSON unpacking failed - DYAD_RC_RPC_FINISHED = -2007, // The Flux RPC responded with ENODATA (i.e., - // end of stream) sooner than expected + DYAD_RC_FLUXFAIL = -2000, // Some Flux function failed + DYAD_RC_BADCOMMIT = -2001, // Flux KVS commit didn't work + DYAD_RC_NOTFOUND = -2002, // Flux KVS lookup didn't work + DYAD_RC_BADFETCH = -2003, // Flux KVS commit didn't work + DYAD_RC_BADRPC = -2004, // Flux RPC pack or get didn't work + DYAD_RC_BADPACK = -2005, // JSON packing failed + DYAD_RC_BADUNPACK = -2006, // JSON unpacking failed + DYAD_RC_RPC_FINISHED = -2007, // The Flux RPC responded with ENODATA (i.e., + // end of stream) sooner than expected - //UCX - DYAD_RC_UCXINIT_FAIL = -3001, // UCX initialization failed - DYAD_RC_UCXWAIT_FAIL = -3002, // UCX wait (either custom or - // 'ucp_worker_wait') failed - DYAD_RC_UCXEP_FAIL = -3003, // An operation on a ucp_ep_h failed - DYAD_RC_UCXCOMM_FAIL = -3004, // UCX communication routine failed - DYAD_RC_UCXMMAP_FAIL = -3005, // Failed to perform operations with ucp_mem_map - DYAD_RC_UCXRKEY_PACK_FAILED = -3006, // Failed to perform operations with ucp_mem_map - DYAD_RC_UCXRKEY_UNPACK_FAILED = -3007, // Failed to perform operations with ucp_mem_map + // UCX + DYAD_RC_UCXINIT_FAIL = -3001, // UCX initialization failed + DYAD_RC_UCXWAIT_FAIL = -3002, // UCX wait (either custom or + // 'ucp_worker_wait') failed + DYAD_RC_UCXEP_FAIL = -3003, // An operation on a ucp_ep_h failed + DYAD_RC_UCXCOMM_FAIL = -3004, // UCX communication routine failed + DYAD_RC_UCXMMAP_FAIL = -3005, // Failed to perform operations with ucp_mem_map + DYAD_RC_UCXRKEY_PACK_FAILED = -3006, // Failed to perform operations with ucp_mem_map + DYAD_RC_UCXRKEY_UNPACK_FAILED = -3007, // Failed to perform operations with ucp_mem_map }; diff --git a/include/dyad/core/dyad_ctx.h b/include/dyad/core/dyad_ctx.h index a09be760..e40c7a9b 100644 --- a/include/dyad/core/dyad_ctx.h +++ b/include/dyad/core/dyad_ctx.h @@ -17,10 +17,9 @@ extern "C" { DYAD_DLL_EXPORTED extern const struct dyad_ctx dyad_ctx_default; -DYAD_DLL_EXPORTED dyad_ctx_t *dyad_ctx_get(); -DYAD_DLL_EXPORTED void dyad_ctx_init(dyad_dtl_comm_mode_t dtl_comm_mode, - void *flux_handle); -DYAD_DLL_EXPORTED void dyad_ctx_fini(); +DYAD_DLL_EXPORTED dyad_ctx_t *dyad_ctx_get (); +DYAD_DLL_EXPORTED void dyad_ctx_init (dyad_dtl_comm_mode_t dtl_comm_mode, void *flux_handle); +DYAD_DLL_EXPORTED void dyad_ctx_fini (); /** * @brief Intialize the DYAD context @@ -47,21 +46,30 @@ DYAD_DLL_EXPORTED void dyad_ctx_fini(); * * @return An error code from dyad_rc.h */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_init( - bool debug, bool check, bool shared_storage, bool reinit, - bool async_publish, bool fsync_write, unsigned int key_depth, - unsigned int key_bins, unsigned int service_mux, const char *kvs_namespace, - const char *prod_managed_path, const char *cons_managed_path, - bool relative_to_managed_path, const char *dtl_mode_str, - const dyad_dtl_comm_mode_t dtl_comm_mode, void *flux_handle); +DYAD_DLL_EXPORTED dyad_rc_t dyad_init (bool debug, + bool check, + bool shared_storage, + bool reinit, + bool async_publish, + bool fsync_write, + unsigned int key_depth, + unsigned int key_bins, + unsigned int service_mux, + const char *kvs_namespace, + const char *prod_managed_path, + const char *cons_managed_path, + bool relative_to_managed_path, + const char *dtl_mode_str, + const dyad_dtl_comm_mode_t dtl_comm_mode, + void *flux_handle); /** * @brief Intialize the DYAD context using environment variables * * @return An error code from dyad_rc.h */ -DYAD_DLL_EXPORTED dyad_rc_t -dyad_init_env(const dyad_dtl_comm_mode_t dtl_comm_mode, void *flux_handle); +DYAD_DLL_EXPORTED dyad_rc_t dyad_init_env (const dyad_dtl_comm_mode_t dtl_comm_mode, + void *flux_handle); /** * @brief Reset producer path. Can be used by the module @@ -69,7 +77,7 @@ dyad_init_env(const dyad_dtl_comm_mode_t dtl_comm_mode, void *flux_handle); * * @return An error code from dyad_rc.h */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_set_prod_path(const char *path); +DYAD_DLL_EXPORTED dyad_rc_t dyad_set_prod_path (const char *path); /** * @brief Reset consumer path. Can be used by the module @@ -77,7 +85,7 @@ DYAD_DLL_EXPORTED dyad_rc_t dyad_set_prod_path(const char *path); * * @return An error code from dyad_rc.h */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_set_cons_path(const char *path); +DYAD_DLL_EXPORTED dyad_rc_t dyad_set_cons_path (const char *path); /** * @brief Reset dtl mode. Can be used by the module @@ -86,8 +94,8 @@ DYAD_DLL_EXPORTED dyad_rc_t dyad_set_cons_path(const char *path); * * @return An error code from dyad_rc.h */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_set_and_init_dtl_mode( - const char *dtl_mode_name, dyad_dtl_comm_mode_t dtl_comm_mode); +DYAD_DLL_EXPORTED dyad_rc_t dyad_set_and_init_dtl_mode (const char *dtl_mode_name, + dyad_dtl_comm_mode_t dtl_comm_mode); /** * Reset the contents of the ctx to the default values and deallocate @@ -95,14 +103,14 @@ DYAD_DLL_EXPORTED dyad_rc_t dyad_set_and_init_dtl_mode( * This is needed for wrapper to handle dyad exceptions as the wrapper requires * ctx for it's lifetime */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_clear(); +DYAD_DLL_EXPORTED dyad_rc_t dyad_clear (); /** * @brief Finalizes the DYAD instance and deallocates the context * * @return An error code from dyad_rc.h */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_finalize(); +DYAD_DLL_EXPORTED dyad_rc_t dyad_finalize (); #ifdef __cplusplus } diff --git a/include/dyad/stream/dyad_params.hpp b/include/dyad/stream/dyad_params.hpp index c322a12c..1a3424f1 100644 --- a/include/dyad/stream/dyad_params.hpp +++ b/include/dyad/stream/dyad_params.hpp @@ -24,7 +24,7 @@ namespace dyad struct dyad_params { bool m_debug; /** Indicate if the storage associated with the managed path is shared - * (i.e. visible to all ranks) */ + * (i.e. visible to all ranks) */ bool m_shared_storage; /// Indicate if reinitialization is required even if already initinialized bool m_reinit; diff --git a/include/dyad/stream/dyad_stream_api.hpp b/include/dyad/stream/dyad_stream_api.hpp index d640cb2d..251d95b0 100644 --- a/include/dyad/stream/dyad_stream_api.hpp +++ b/include/dyad/stream/dyad_stream_api.hpp @@ -19,7 +19,7 @@ #include // fsync -#include // realpath +#include // realpath #include #include #include @@ -28,7 +28,6 @@ #include - namespace dyad { @@ -36,8 +35,7 @@ namespace dyad // Enable if _Path is a filesystem::path or experimental::filesystem::path template ().make_preferred ().filename ())> + typename _Path2 = decltype (std::declval<_Path&> ().make_preferred ().filename ())> using dyad_if_fs_path = std::enable_if_t, _Result>; #endif // c++17 filesystem @@ -63,7 +61,7 @@ void fsync_ofstream (std::basic_ofstream<_CharT, _Traits>& os) if (os.is_open ()) { os.flush (); } -#endif // DYAD_HAS_STD_FSTREAM_FD +#endif // DYAD_HAS_STD_FSTREAM_FD } template @@ -87,14 +85,15 @@ void fsync_fstream (std::basic_fstream<_CharT, _Traits>& os) if (os.is_open ()) { os.flush (); } -#endif // DYAD_HAS_STD_FSTREAM_FD +#endif // DYAD_HAS_STD_FSTREAM_FD } //---------------------------------------------------------------------- #if defined(DYAD_HAS_STD_FSTREAM_FD) //---------------------------------------------------------------------- template -void lock_exclusive_ofstream (std::basic_ofstream<_CharT, _Traits>& os, const dyad_stream_core& core) +void lock_exclusive_ofstream (std::basic_ofstream<_CharT, _Traits>& os, + const dyad_stream_core& core) { class my_filebuf : public std::basic_filebuf<_CharT> { @@ -219,25 +218,25 @@ void unlock_fstream (std::basic_fstream<_CharT, _Traits>& os, const dyad_stream_ } } -#define DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM(_os_,_core_) lock_exclusive_ofstream(_os_,_core_) -#define DYAD_EXCLUSIVE_LOCK_CPP_FSTREAM(_os_,_core_) lock_exclusive_fstream(_os_,_core_) -#define DYAD_SHARED_LOCK_CPP_IFSTREAM(_os_,_core_) lock_shared_ifstream(_os_,_core_) -#define DYAD_SHARED_LOCK_CPP_FSTREAM(_os_,_core_) lock_shared_fstream(_os_,_core_) -#define DYAD_UNLOCK_CPP_OFSTREAM(_os_,_core_) unlock_ofstream(_os_,_core_) -#define DYAD_UNLOCK_CPP_IFSTREAM(_os_,_core_) unlock_ifstream(_os_,_core_) -#define DYAD_UNLOCK_CPP_FSTREAM(_os_,_core_) unlock_fstream(_os_,_core_) +#define DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM(_os_, _core_) lock_exclusive_ofstream (_os_, _core_) +#define DYAD_EXCLUSIVE_LOCK_CPP_FSTREAM(_os_, _core_) lock_exclusive_fstream (_os_, _core_) +#define DYAD_SHARED_LOCK_CPP_IFSTREAM(_os_, _core_) lock_shared_ifstream (_os_, _core_) +#define DYAD_SHARED_LOCK_CPP_FSTREAM(_os_, _core_) lock_shared_fstream (_os_, _core_) +#define DYAD_UNLOCK_CPP_OFSTREAM(_os_, _core_) unlock_ofstream (_os_, _core_) +#define DYAD_UNLOCK_CPP_IFSTREAM(_os_, _core_) unlock_ifstream (_os_, _core_) +#define DYAD_UNLOCK_CPP_FSTREAM(_os_, _core_) unlock_fstream (_os_, _core_) -#else // DYAD_HAS_STD_FSTREAM_FD +#else // DYAD_HAS_STD_FSTREAM_FD -#define DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM(_os_,_core_) -#define DYAD_EXCLUSIVE_LOCK_CPP_FSTREAM(_os_,_core_) -#define DYAD_SHARED_LOCK_CPP_IFSTREAM(_os_,_core_) -#define DYAD_SHARED_LOCK_CPP_FSTREAM(_os_,_core_) -#define DYAD_UNLOCK_CPP_OFSTREAM(_os_,_core_) -#define DYAD_UNLOCK_CPP_IFSTREAM(_os_,_core_) -#define DYAD_UNLOCK_CPP_FSTREAM(_os_,_core_) +#define DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM(_os_, _core_) +#define DYAD_EXCLUSIVE_LOCK_CPP_FSTREAM(_os_, _core_) +#define DYAD_SHARED_LOCK_CPP_IFSTREAM(_os_, _core_) +#define DYAD_SHARED_LOCK_CPP_FSTREAM(_os_, _core_) +#define DYAD_UNLOCK_CPP_OFSTREAM(_os_, _core_) +#define DYAD_UNLOCK_CPP_IFSTREAM(_os_, _core_) +#define DYAD_UNLOCK_CPP_FSTREAM(_os_, _core_) //---------------------------------------------------------------------- -#endif // DYAD_HAS_STD_FSTREAM_FD +#endif // DYAD_HAS_STD_FSTREAM_FD //---------------------------------------------------------------------- //============================================================================= @@ -255,8 +254,7 @@ class basic_ifstream_dyad basic_ifstream_dyad (const dyad_stream_core& core); basic_ifstream_dyad (); - explicit basic_ifstream_dyad (const char* filename, - ios_base::openmode mode = ios_base::in); + explicit basic_ifstream_dyad (const char* filename, ios_base::openmode mode = ios_base::in); ~basic_ifstream_dyad (); void open (const char* filename, ios_base::openmode mode = ios_base::in); @@ -264,14 +262,12 @@ class basic_ifstream_dyad #if __cplusplus < 201103L bool is_open (); #else - explicit basic_ifstream_dyad (const string& filename, - ios_base::openmode mode = ios_base::in); + explicit basic_ifstream_dyad (const string& filename, ios_base::openmode mode = ios_base::in); basic_ifstream_dyad (const basic_ifstream_dyad&) = delete; basic_ifstream_dyad (basic_ifstream_dyad&& rhs); #if (__cplusplus >= 201703L) && __has_include() template > - basic_ifstream_dyad (const _Path& filepath, - std::ios_base::openmode mode = std::ios_base::in); + basic_ifstream_dyad (const _Path& filepath, std::ios_base::openmode mode = std::ios_base::in); #endif // c++17 filesystem void open (const string& filename, ios_base::openmode mode = ios_base::in); @@ -289,7 +285,10 @@ class basic_ifstream_dyad void init (const dyad_stream_core& core); - const dyad_stream_core& core () const { return m_core; } + const dyad_stream_core& core () const + { + return m_core; + } private: /// context info @@ -392,8 +391,7 @@ basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad (const _Path& filepath { m_core.init (); m_core.open_sync (filepath.c_str ()); - m_stream = - std::unique_ptr (new basic_ifstream (filepath.c_str (), mode)); + m_stream = std::unique_ptr (new basic_ifstream (filepath.c_str (), mode)); if ((m_stream != nullptr) && (*m_stream)) { m_filename = std::string{filepath.c_str ()}; } @@ -448,8 +446,7 @@ void basic_ifstream_dyad<_CharT, _Traits>::swap (basic_ifstream_dyad& rhs) #endif //----------------------------------------------------------------------- template -void basic_ifstream_dyad<_CharT, _Traits>::open (const char* filename, - std::ios_base::openmode mode) +void basic_ifstream_dyad<_CharT, _Traits>::open (const char* filename, std::ios_base::openmode mode) { if (m_stream == nullptr) { // TODO: set fail bit if nullptr @@ -483,8 +480,7 @@ std::filebuf* basic_ifstream_dyad<_CharT, _Traits>::rdbuf () const } template -std::basic_ifstream<_CharT, _Traits>& basic_ifstream_dyad<_CharT, - _Traits>::get_stream () +std::basic_ifstream<_CharT, _Traits>& basic_ifstream_dyad<_CharT, _Traits>::get_stream () { if (m_stream == nullptr) { // TODO: throw @@ -515,8 +511,7 @@ class basic_ofstream_dyad basic_ofstream_dyad (const dyad_stream_core& core); basic_ofstream_dyad (); - explicit basic_ofstream_dyad (const char* filename, - ios_base::openmode mode = ios_base::out); + explicit basic_ofstream_dyad (const char* filename, ios_base::openmode mode = ios_base::out); ~basic_ofstream_dyad (); void open (const char* filename, ios_base::openmode mode = ios_base::out); @@ -524,14 +519,12 @@ class basic_ofstream_dyad #if __cplusplus < 201103L bool is_open (); #else - explicit basic_ofstream_dyad (const string& filename, - ios_base::openmode mode = ios_base::out); + explicit basic_ofstream_dyad (const string& filename, ios_base::openmode mode = ios_base::out); basic_ofstream_dyad (const basic_ofstream_dyad&) = delete; basic_ofstream_dyad (basic_ofstream_dyad&& rhs); #if (__cplusplus >= 201703L) && __has_include() template > - basic_ofstream_dyad (const _Path& filepath, - std::ios_base::openmode mode = std::ios_base::out); + basic_ofstream_dyad (const _Path& filepath, std::ios_base::openmode mode = std::ios_base::out); #endif // c++18 filesystem void open (const string& filename, ios_base::openmode mode = ios_base::out); @@ -549,7 +542,10 @@ class basic_ofstream_dyad void init (const dyad_stream_core& core); - const dyad_stream_core& core () const { return m_core; } + const dyad_stream_core& core () const + { + return m_core; + } private: /// context info @@ -590,11 +586,10 @@ basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (const char* filename, m_stream = new basic_ofstream (filename, mode); if ((m_stream != nullptr) && (*m_stream) - #if DYAD_HAS_STD_FSTREAM_FD +#if DYAD_HAS_STD_FSTREAM_FD && m_core.cmp_canonical_path_prefix (true, filename) - #endif // DYAD_HAS_STD_FSTREAM_FD - ) - { +#endif // DYAD_HAS_STD_FSTREAM_FD + ) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); m_filename = std::string{filename}; } @@ -627,11 +622,10 @@ basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (const char* filename, m_core.init (); m_stream = std::unique_ptr (new basic_ofstream (filename, mode)); if ((m_stream != nullptr) && (*m_stream) - #if DYAD_HAS_STD_FSTREAM_FD +#if DYAD_HAS_STD_FSTREAM_FD && m_core.cmp_canonical_path_prefix (true, filename) - #endif // DYAD_HAS_STD_FSTREAM_FD - ) - { +#endif // DYAD_HAS_STD_FSTREAM_FD + ) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); m_filename = std::string{filename}; } @@ -644,11 +638,10 @@ basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (const string& filenam m_core.init (); m_stream = std::unique_ptr (new basic_ofstream (filename, mode)); if ((m_stream != nullptr) && (*m_stream) - #if DYAD_HAS_STD_FSTREAM_FD - && m_core.cmp_canonical_path_prefix (true, (const char* const) filename.c_str ()) - #endif // DYAD_HAS_STD_FSTREAM_FD - ) - { +#if DYAD_HAS_STD_FSTREAM_FD + && m_core.cmp_canonical_path_prefix (true, (const char* const)filename.c_str ()) +#endif // DYAD_HAS_STD_FSTREAM_FD + ) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); m_filename = filename; } @@ -667,15 +660,14 @@ basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (const _Path& filepath std::ios_base::openmode mode) { m_core.init (); - m_stream = - std::unique_ptr (new basic_ofstream ((const char* const) filepath.c_str (), mode)); + m_stream = std::unique_ptr ( + new basic_ofstream ((const char* const)filepath.c_str (), mode)); if ((m_stream != nullptr) && (*m_stream) - #if DYAD_HAS_STD_FSTREAM_FD +#if DYAD_HAS_STD_FSTREAM_FD && m_core.cmp_canonical_path_prefix (true, filename.c_str ()) - #endif // DYAD_HAS_STD_FSTREAM_FD - ) - { +#endif // DYAD_HAS_STD_FSTREAM_FD + ) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); m_filename = std::string{filepath.c_str ()}; } @@ -744,8 +736,7 @@ void basic_ofstream_dyad<_CharT, _Traits>::swap (basic_ofstream_dyad& rhs) #endif //----------------------------------------------------------------------- template -void basic_ofstream_dyad<_CharT, _Traits>::open (const char* filename, - std::ios_base::openmode mode) +void basic_ofstream_dyad<_CharT, _Traits>::open (const char* filename, std::ios_base::openmode mode) { if (m_stream == nullptr) { // TODO: set fail bit if nullptr @@ -753,11 +744,10 @@ void basic_ofstream_dyad<_CharT, _Traits>::open (const char* filename, } m_stream->open (filename, mode); if ((*m_stream) - #if DYAD_HAS_STD_FSTREAM_FD +#if DYAD_HAS_STD_FSTREAM_FD && m_core.cmp_canonical_path_prefix (true, filename) - #endif // DYAD_HAS_STD_FSTREAM_FD - ) - { +#endif // DYAD_HAS_STD_FSTREAM_FD + ) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); m_filename = std::string{filename}; } @@ -787,8 +777,7 @@ std::filebuf* basic_ofstream_dyad<_CharT, _Traits>::rdbuf () const } template -std::basic_ofstream<_CharT, _Traits>& basic_ofstream_dyad<_CharT, - _Traits>::get_stream () +std::basic_ofstream<_CharT, _Traits>& basic_ofstream_dyad<_CharT, _Traits>::get_stream () { if (m_stream == nullptr) { // TODO: throw @@ -820,29 +809,24 @@ class basic_fstream_dyad basic_fstream_dyad (const dyad_stream_core& core); basic_fstream_dyad (); explicit basic_fstream_dyad (const char* filename, - ios_base::openmode mode = ios_base::in - | ios_base::out); + ios_base::openmode mode = ios_base::in | ios_base::out); ~basic_fstream_dyad (); - void open (const char* filename, - ios_base::openmode mode = ios_base::in | ios_base::out); + void open (const char* filename, ios_base::openmode mode = ios_base::in | ios_base::out); #if __cplusplus < 201103L bool is_open (); #else explicit basic_fstream_dyad (const string& filename, - ios_base::openmode mode = ios_base::in - | ios_base::out); + ios_base::openmode mode = ios_base::in | ios_base::out); basic_fstream_dyad (const basic_fstream_dyad&) = delete; basic_fstream_dyad (basic_fstream_dyad&& rhs); #if (__cplusplus >= 201703L) && __has_include() template > - basic_fstream_dyad (const _Path& filepath, - std::ios_base::openmode mode = std::ios_base::out); + basic_fstream_dyad (const _Path& filepath, std::ios_base::openmode mode = std::ios_base::out); #endif // c++18 filesystem - void open (const string& filename, - ios_base::openmode mode = ios_base::in | ios_base::out); + void open (const string& filename, ios_base::openmode mode = ios_base::in | ios_base::out); bool is_open () const; basic_fstream_dyad& operator= (basic_fstream_dyad&& rhs); @@ -857,7 +841,10 @@ class basic_fstream_dyad void init (const dyad_stream_core& core); - const dyad_stream_core& core () const { return m_core; } + const dyad_stream_core& core () const + { + return m_core; + } private: /// context info @@ -899,11 +886,10 @@ basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad (const char* filename, m_stream = new basic_fstream (filename, mode); if ((m_stream != nullptr) && (*m_stream)) { if (m_core.is_dyad_producer () - #if DYAD_HAS_STD_FSTREAM_FD +#if DYAD_HAS_STD_FSTREAM_FD && m_core.cmp_canonical_path_prefix (true, filename) - #endif // DYAD_HAS_STD_FSTREAM_FD - ) - { +#endif // DYAD_HAS_STD_FSTREAM_FD + ) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); } m_filename = std::string{filename}; @@ -974,8 +960,7 @@ basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad (const _Path& filepath, { m_core.init (); m_core.open_sync (filepath.c_str ()); - m_stream = - std::unique_ptr (new basic_fstream (filepath.c_str (), mode)); + m_stream = std::unique_ptr (new basic_fstream (filepath.c_str (), mode)); if ((m_stream != nullptr) && (*m_stream)) { if (m_core.is_dyad_producer ()) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); @@ -1049,8 +1034,7 @@ void basic_fstream_dyad<_CharT, _Traits>::swap (basic_fstream_dyad& rhs) #endif //----------------------------------------------------------------------- template -void basic_fstream_dyad<_CharT, _Traits>::open (const char* filename, - std::ios_base::openmode mode) +void basic_fstream_dyad<_CharT, _Traits>::open (const char* filename, std::ios_base::openmode mode) { if (m_stream == nullptr) { // TODO: set fail bit if nullptr @@ -1060,11 +1044,10 @@ void basic_fstream_dyad<_CharT, _Traits>::open (const char* filename, m_stream->open (filename, mode); if ((*m_stream)) { if (m_core.is_dyad_producer () - #if DYAD_HAS_STD_FSTREAM_FD +#if DYAD_HAS_STD_FSTREAM_FD && m_core.cmp_canonical_path_prefix (true, filename) - #endif // DYAD_HAS_STD_FSTREAM_FD - ) - { +#endif // DYAD_HAS_STD_FSTREAM_FD + ) { DYAD_EXCLUSIVE_LOCK_CPP_OFSTREAM (*m_stream, m_core); } m_filename = std::string{filename}; diff --git a/include/dyad/stream/dyad_stream_core.hpp b/include/dyad/stream/dyad_stream_core.hpp index 55e0106b..3ef30416 100644 --- a/include/dyad/stream/dyad_stream_core.hpp +++ b/include/dyad/stream/dyad_stream_core.hpp @@ -17,9 +17,9 @@ #error "no config" #endif +#include #include #include -#include #include @@ -52,12 +52,12 @@ class dyad_stream_core bool chk_initialized () const; bool chk_fsync_write () const; - bool cmp_canonical_path_prefix (bool is_prod, const char* const __restrict__ path); + bool cmp_canonical_path_prefix (bool is_prod, const char *const __restrict__ path); std::string get_upath () const; - int file_lock_exclusive(int fd) const; - int file_lock_shared(int fd) const; - int file_unlock(int fd) const; + int file_lock_exclusive (int fd) const; + int file_lock_shared (int fd) const; + int file_unlock (int fd) const; private: const dyad_ctx *m_ctx; diff --git a/src/dyad/common/dyad_logging.h b/src/dyad/common/dyad_logging.h index 31204fad..77b974f9 100644 --- a/src/dyad/common/dyad_logging.h +++ b/src/dyad/common/dyad_logging.h @@ -10,6 +10,8 @@ #include #include +#include + #ifdef __cplusplus extern "C" { #endif @@ -31,50 +33,50 @@ extern "C" { //============================================================================= #else //============================================================================= -#define DYAD_LOG_STDERR(...) fprintf(stderr, __VA_ARGS__); -#define DYAD_LOG_STDOUT(...) fprintf(stdout, __VA_ARGS__); +#define DYAD_LOG_STDERR(...) fprintf (stderr, __VA_ARGS__); +#define DYAD_LOG_STDOUT(...) fprintf (stdout, __VA_ARGS__); -#ifdef DYAD_LOGGER_FLUX // FLUX ----------------------------------------------- +#ifdef DYAD_LOGGER_FLUX // FLUX ----------------------------------------------- #define DYAD_LOGGER_INIT() ; -#define DYAD_LOG_STDOUT_REDIRECT(fpath) freopen((fpath), "a+", stdout); -#define DYAD_LOG_STDERR_REDIRECT(fpath) freopen((fpath), "a+", stderr); +#define DYAD_LOG_STDOUT_REDIRECT(fpath) freopen ((fpath), "a+", stdout); +#define DYAD_LOG_STDERR_REDIRECT(fpath) freopen ((fpath), "a+", stderr); #ifdef DYAD_UTIL_LOGGER #ifdef DYAD_LOGGER_LEVEL_DEBUG -#define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_LOG_STDERR(__VA_ARGS__) +#define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_LOG_STDERR (__VA_ARGS__) #else #define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO #endif #ifdef DYAD_LOGGER_LEVEL_INFO -#define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_LOG_STDOUT(__VA_ARGS__) +#define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_LOG_STDOUT (__VA_ARGS__) #else #define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO #endif #ifdef DYAD_LOGGER_LEVEL_WARN -#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_LOG_STDOUT(__VA_ARGS__) +#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_LOG_STDOUT (__VA_ARGS__) #else #define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO #endif #ifdef DYAD_LOGGER_LEVEL_ERROR -#define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_LOG_STDERR(__VA_ARGS__) +#define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_LOG_STDERR (__VA_ARGS__) #else #define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO #endif -#else // DYAD_UTIL_LOGGER +#else // DYAD_UTIL_LOGGER #include #ifdef DYAD_LOGGER_LEVEL_DEBUG -#define DYAD_LOG_DEBUG(dyad_ctx, ...) \ - flux_log((flux_t *)(((dyad_ctx_t *)dyad_ctx)->h), LOG_DEBUG, __VA_ARGS__); +#define DYAD_LOG_DEBUG(dyad_ctx, ...) \ + flux_log ((flux_t *)(((dyad_ctx_t *)dyad_ctx)->h), LOG_DEBUG, __VA_ARGS__); #else #define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO #endif #ifdef DYAD_LOGGER_LEVEL_INFO -#define DYAD_LOG_INFO(dyad_ctx, ...) \ - flux_log((flux_t *)(((dyad_ctx_t *)dyad_ctx)->h), LOG_INFO, __VA_ARGS__); +#define DYAD_LOG_INFO(dyad_ctx, ...) \ + flux_log ((flux_t *)(((dyad_ctx_t *)dyad_ctx)->h), LOG_INFO, __VA_ARGS__); #else #define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO #endif @@ -86,64 +88,60 @@ extern "C" { #endif #ifdef DYAD_LOGGER_LEVEL_ERROR -#define DYAD_LOG_ERROR(dyad_ctx, ...) \ - flux_log_error((flux_t *)(((dyad_ctx_t *)dyad_ctx)->h), __VA_ARGS__); +#define DYAD_LOG_ERROR(dyad_ctx, ...) \ + flux_log_error ((flux_t *)(((dyad_ctx_t *)dyad_ctx)->h), __VA_ARGS__); #else #define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO #endif -#endif // DYAD_UTIL_LOGGER -#elif defined(DYAD_LOGGER_CPP_LOGGER) // CPP_LOGGER --------------------------- +#endif // DYAD_UTIL_LOGGER +#elif defined(DYAD_LOGGER_CPP_LOGGER) // CPP_LOGGER --------------------------- #include #define DYAD_LOGGER_NAME "DYAD" #ifdef DYAD_LOGGER_LEVEL_DEBUG -#define DYAD_LOGGER_INIT() \ - cpp_logger_clog_level(CPP_LOGGER_DEBUG, DYAD_LOGGER_NAME); +#define DYAD_LOGGER_INIT() cpp_logger_clog_level (CPP_LOGGER_DEBUG, DYAD_LOGGER_NAME); #elif defined(DYAD_LOGGER_LEVEL_INFO) -#define DYAD_LOGGER_INIT() \ - cpp_logger_clog_level(CPP_LOGGER_INFO, DYAD_LOGGER_NAME); +#define DYAD_LOGGER_INIT() cpp_logger_clog_level (CPP_LOGGER_INFO, DYAD_LOGGER_NAME); #elif defined(DYAD_LOGGER_LEVEL_WARN) -#define DYAD_LOGGER_INIT() \ - cpp_logger_clog_level(CPP_LOGGER_WARN, DYAD_LOGGER_NAME); +#define DYAD_LOGGER_INIT() cpp_logger_clog_level (CPP_LOGGER_WARN, DYAD_LOGGER_NAME); #else -#define DYAD_LOGGER_INIT() \ - cpp_logger_clog_level(CPP_LOGGER_ERROR, DYAD_LOGGER_NAME); +#define DYAD_LOGGER_INIT() cpp_logger_clog_level (CPP_LOGGER_ERROR, DYAD_LOGGER_NAME); #endif -#define DYAD_LOG_STDOUT_REDIRECT(fpath) freopen((fpath), "a+", stdout); -#define DYAD_LOG_STDERR_REDIRECT(fpath) freopen((fpath), "a+", stderr); +#define DYAD_LOG_STDOUT_REDIRECT(fpath) freopen ((fpath), "a+", stdout); +#define DYAD_LOG_STDERR_REDIRECT(fpath) freopen ((fpath), "a+", stderr); #ifdef DYAD_LOGGER_LEVEL_DEBUG -#define DYAD_LOG_DEBUG(dyad_ctx, ...) \ - cpp_logger_clog(CPP_LOGGER_DEBUG, DYAD_LOGGER_NAME, __VA_ARGS__); +#define DYAD_LOG_DEBUG(dyad_ctx, ...) \ + cpp_logger_clog (CPP_LOGGER_DEBUG, DYAD_LOGGER_NAME, __VA_ARGS__); #else #define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO #endif #ifdef DYAD_LOGGER_LEVEL_INFO -#define DYAD_LOG_INFO(dyad_ctx, ...) \ - cpp_logger_clog(CPP_LOGGER_INFO, DYAD_LOGGER_NAME, __VA_ARGS__); +#define DYAD_LOG_INFO(dyad_ctx, ...) \ + cpp_logger_clog (CPP_LOGGER_INFO, DYAD_LOGGER_NAME, __VA_ARGS__); #else #define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO #endif #ifdef DYAD_LOGGER_LEVEL_WARN -#define DYAD_LOG_WARN(dyad_ctx, ...) \ - cpp_logger_clog(CPP_LOGGER_WARN, DYAD_LOGGER_NAME, __VA_ARGS__); +#define DYAD_LOG_WARN(dyad_ctx, ...) \ + cpp_logger_clog (CPP_LOGGER_WARN, DYAD_LOGGER_NAME, __VA_ARGS__); #else #define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO #endif #ifdef DYAD_LOGGER_LEVEL_ERROR -#define DYAD_LOG_ERROR(dyad_ctx, ...) \ - cpp_logger_clog(CPP_LOGGER_ERROR, DYAD_LOGGER_NAME, __VA_ARGS__); +#define DYAD_LOG_ERROR(dyad_ctx, ...) \ + cpp_logger_clog (CPP_LOGGER_ERROR, DYAD_LOGGER_NAME, __VA_ARGS__); #else #define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO #endif -#endif // DYAD_LOGGER_FLUX ---------------------------------------------------- +#endif // DYAD_LOGGER_FLUX ---------------------------------------------------- //============================================================================= -#endif // DYAD_LOGGER_NO_LOG +#endif // DYAD_LOGGER_NO_LOG //============================================================================= #ifdef __cplusplus diff --git a/src/dyad/core/dyad_ctx.c b/src/dyad/core/dyad_ctx.c index 2b5af118..f64b1de9 100644 --- a/src/dyad/core/dyad_ctx.c +++ b/src/dyad/core/dyad_ctx.c @@ -26,7 +26,13 @@ #include #endif -#include +#include +#include +#include +#include +#include +#include +#include // Note: // To ensure we don't have multiple initialization, we need the following: @@ -36,781 +42,791 @@ static __thread dyad_ctx_t *ctx = NULL; const struct dyad_ctx dyad_ctx_default = { // Internal - NULL, // h - NULL, // dtl_handle - NULL, // fname - false, // use_fs_locks - NULL, // prod_real_path - NULL, // cons_real_path - 0u, // prod_managed_len - 0u, // cons_managed_len - 0u, // prod_real_len - 0u, // cons_real_len - 0u, // prod_managed_hash - 0u, // cons_managed_hash - 0u, // prod_real_hash - 0u, // cons_real_hash - 0u, // delim_len + NULL, // h + NULL, // dtl_handle + NULL, // fname + false, // use_fs_locks + NULL, // prod_real_path + NULL, // cons_real_path + 0u, // prod_managed_len + 0u, // cons_managed_len + 0u, // prod_real_len + 0u, // cons_real_len + 0u, // prod_managed_hash + 0u, // cons_managed_hash + 0u, // prod_real_hash + 0u, // cons_real_hash + 0u, // delim_len // User facing - false, // debug - false, // check - false, // reenter - true, // initialized - false, // shared_storage - false, // async_publish - false, // fsync_write - 3u, // key_depth - 1024u, // key_bins - 0u, // rank - 1u, // service_mux - 0u, // node_idx - -1, // pid - NULL, // kvs_namespace - NULL, // prod_managed_path - NULL, // cons_managed_path - false // relative_to_managed_path + false, // debug + false, // check + false, // reenter + true, // initialized + false, // shared_storage + false, // async_publish + false, // fsync_write + 3u, // key_depth + 1024u, // key_bins + 0u, // rank + 1u, // service_mux + 0u, // node_idx + -1, // pid + NULL, // kvs_namespace + NULL, // prod_managed_path + NULL, // cons_managed_path + false // relative_to_managed_path }; -DYAD_DLL_EXPORTED dyad_ctx_t *dyad_ctx_get() { return ctx; } +DYAD_DLL_EXPORTED dyad_ctx_t *dyad_ctx_get () +{ + return ctx; +} -DYAD_DLL_EXPORTED void dyad_ctx_init(const dyad_dtl_comm_mode_t dtl_comm_mode, - void *flux_handle) { +DYAD_DLL_EXPORTED void dyad_ctx_init (const dyad_dtl_comm_mode_t dtl_comm_mode, void *flux_handle) +{ #if DYAD_PROFILER == 3 - DFTRACER_C_FINI(); + DFTRACER_C_FINI (); #endif - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - - rc = dyad_init_env(dtl_comm_mode, flux_handle); - - if (DYAD_IS_ERROR(rc)) { - fprintf(stderr, "Failed to initialize DYAD (code = %d)", rc); - if (ctx != NULL) { - ctx->initialized = false; - ctx->reenter = false; - } - DYAD_C_FUNCTION_END(); - return; - } else { - DYAD_LOG_INFO(ctx, "DYAD Initialized on loading using env variables"); - } - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_START (); + dyad_rc_t rc = DYAD_RC_OK; + + rc = dyad_init_env (dtl_comm_mode, flux_handle); + + if (DYAD_IS_ERROR (rc)) { + fprintf (stderr, "Failed to initialize DYAD (code = %d)", rc); + if (ctx != NULL) { + ctx->initialized = false; + ctx->reenter = false; + } + DYAD_C_FUNCTION_END (); + return; + } else { + DYAD_LOG_INFO (ctx, "DYAD Initialized on loading using env variables"); + } + DYAD_C_FUNCTION_END (); } -DYAD_DLL_EXPORTED void dyad_ctx_fini() { - DYAD_C_FUNCTION_START(); - if (ctx == NULL) { - DYAD_C_FUNCTION_END(); - goto dyad_wrapper_fini_done; - } - dyad_finalize(); +DYAD_DLL_EXPORTED void dyad_ctx_fini () +{ + DYAD_C_FUNCTION_START (); + if (ctx == NULL) { + DYAD_C_FUNCTION_END (); + goto dyad_wrapper_fini_done; + } + dyad_finalize (); dyad_wrapper_fini_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); #if DYAD_PROFILER == 3 - DFTRACER_C_FINI(); + DFTRACER_C_FINI (); #endif } -dyad_rc_t dyad_clear(); +dyad_rc_t dyad_clear (); DYAD_DLL_EXPORTED -dyad_rc_t dyad_init(bool debug, bool check, bool shared_storage, bool reinit, - bool async_publish, bool fsync_write, - unsigned int key_depth, unsigned int key_bins, - unsigned int service_mux, const char *kvs_namespace, - const char *prod_managed_path, - const char *cons_managed_path, - bool relative_to_managed_path, const char *dtl_mode_str, - const dyad_dtl_comm_mode_t dtl_comm_mode, - void *flux_handle) { - DYAD_LOGGER_INIT(); - unsigned my_rank = 0u; - size_t namespace_len = 0ul; +dyad_rc_t dyad_init (bool debug, + bool check, + bool shared_storage, + bool reinit, + bool async_publish, + bool fsync_write, + unsigned int key_depth, + unsigned int key_bins, + unsigned int service_mux, + const char *kvs_namespace, + const char *prod_managed_path, + const char *cons_managed_path, + bool relative_to_managed_path, + const char *dtl_mode_str, + const dyad_dtl_comm_mode_t dtl_comm_mode, + void *flux_handle) +{ + DYAD_LOGGER_INIT (); + unsigned my_rank = 0u; + size_t namespace_len = 0ul; #ifdef DYAD_PROFILER_DFTRACER - const char *file_prefix = getenv(DFTRACER_LOG_FILE); - if (file_prefix == NULL) - file_prefix = "./dyad_"; - char log_file[4096] = {'\0'}; - sprintf(log_file, "%s_core-%d.pfw", file_prefix, getpid()); - DFTRACER_C_INIT_NO_BIND(log_file, NULL, NULL); + const char *file_prefix = getenv (DFTRACER_LOG_FILE); + if (file_prefix == NULL) + file_prefix = "./dyad_"; + char log_file[4096] = {'\0'}; + sprintf (log_file, "%s_core-%d.pfw", file_prefix, getpid ()); + DFTRACER_C_INIT_NO_BIND (log_file, NULL, NULL); #endif - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - // Check if the actual dyad_ctx_t object is not NULL. - // If it is not NULL, that means the dyad_ctx_t object - // has either been allocated or fully initialized. - // If it's initialized, simply print a message and - // return DYAD_OK. - if (!reinit && (ctx != NULL)) { - if ((ctx)->initialized) { - DYAD_LOG_DEBUG((ctx), "DYAD context already initialized\n"); - rc = DYAD_RC_OK; - goto init_region_finish; - } - } else { - if (reinit) - dyad_finalize(); - // Allocate the dyad_ctx_t object and make sure the allocation - // worked successfully - ctx = (dyad_ctx_t *)malloc(sizeof(struct dyad_ctx)); - if (ctx == NULL) { - fprintf(stderr, "Could not allocate DYAD context!\n"); - rc = DYAD_RC_NOCTX; - goto init_region_finish; - } - } - // Set the initial contents of the dyad_ctx_t object to dyad_ctx_default. - *ctx = dyad_ctx_default; - // If neither managed path is provided, DYAD will not do anything. - // So, simply print a warning and return DYAD_OK. - if (prod_managed_path == NULL && cons_managed_path == NULL) { - fprintf(stderr, "Warning: no managed path provided! DYAD will not do " - "anything!\n"); - rc = DYAD_RC_OK; - goto init_region_finish; - } + DYAD_C_FUNCTION_START (); + dyad_rc_t rc = DYAD_RC_OK; + // Check if the actual dyad_ctx_t object is not NULL. + // If it is not NULL, that means the dyad_ctx_t object + // has either been allocated or fully initialized. + // If it's initialized, simply print a message and + // return DYAD_OK. + if (!reinit && (ctx != NULL)) { + if ((ctx)->initialized) { + DYAD_LOG_DEBUG ((ctx), "DYAD context already initialized\n"); + rc = DYAD_RC_OK; + goto init_region_finish; + } + } else { + if (reinit) + dyad_finalize (); + // Allocate the dyad_ctx_t object and make sure the allocation + // worked successfully + ctx = (dyad_ctx_t *)malloc (sizeof (struct dyad_ctx)); + if (ctx == NULL) { + fprintf (stderr, "Could not allocate DYAD context!\n"); + rc = DYAD_RC_NOCTX; + goto init_region_finish; + } + } + // Set the initial contents of the dyad_ctx_t object to dyad_ctx_default. + *ctx = dyad_ctx_default; + // If neither managed path is provided, DYAD will not do anything. + // So, simply print a warning and return DYAD_OK. + if (prod_managed_path == NULL && cons_managed_path == NULL) { + fprintf (stderr, + "Warning: no managed path provided! DYAD will not do " + "anything!\n"); + rc = DYAD_RC_OK; + goto init_region_finish; + } #ifdef DYAD_PATH_DELIM - if (DYAD_PATH_DELIM == NULL || strlen(DYAD_PATH_DELIM) == 0ul) { - fprintf(stderr, "Invalid DYAD_PATH_DELIM defined in 'dyad_config.h'!\n"); + if (DYAD_PATH_DELIM == NULL || strlen (DYAD_PATH_DELIM) == 0ul) { + fprintf (stderr, "Invalid DYAD_PATH_DELIM defined in 'dyad_config.h'!\n"); + rc = DYAD_RC_NOCTX; + goto init_region_failed; + } + ctx->delim_len = strlen (DYAD_PATH_DELIM); +#else + fprintf (stderr, "DYAD_PATH_DELIM is not defined in 'dyad_config.h'!\n"); rc = DYAD_RC_NOCTX; goto init_region_failed; - } - ctx->delim_len = strlen(DYAD_PATH_DELIM); -#else - fprintf(stderr, "DYAD_PATH_DELIM is not defined in 'dyad_config.h'!\n"); - rc = DYAD_RC_NOCTX; - goto init_region_failed; #endif - // Set the values in dyad_ctx_t that don't need allocation - ctx->debug = debug; - ctx->check = check; - ctx->shared_storage = shared_storage; - ctx->async_publish = async_publish; - ctx->fsync_write = fsync_write; - ctx->key_depth = key_depth; - ctx->key_bins = key_bins; - - // Open a Flux handle and store it in the dyad_ctx_t - // object. If the open operation failed, return DYAD_FLUXFAIL - // In case of module, we use the flux handle passed to mod_main() - // instead of getting it by flux_open() - - if (flux_handle != NULL) { - ctx->h = (flux_t *)flux_handle; - } else { - ctx->h = flux_open(NULL, 0); - } - if (ctx->h == NULL) { - fprintf(stderr, "Could not open Flux handle!\n"); - rc = DYAD_RC_FLUXFAIL; - goto init_region_finish; - } - - // Get the rank of the Flux broker corresponding - // to the handle. If this fails, return DYAD_FLUXFAIL - DYAD_LOG_INFO(ctx, "DYAD_CORE: getting Flux rank"); - if (flux_get_rank(ctx->h, &(ctx->rank)) < 0) { - DYAD_LOG_INFO(ctx, "Could not get Flux rank!\n"); - rc = DYAD_RC_FLUXFAIL; - goto init_region_finish; - } - my_rank = ctx->rank; - ctx->service_mux = (service_mux < 1u) ? 1u : service_mux; - ctx->node_idx = ctx->rank / ctx->service_mux; - ctx->pid = getpid(); - if (my_rank == 0) { - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: debug %s", - ctx->debug ? "true" : "false"); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: shared_storage %s", - ctx->shared_storage ? "true" : "false"); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: async_publish %s", - ctx->async_publish ? "true" : "false"); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: fsync_write %s", - ctx->fsync_write ? "true" : "false"); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: kvs key depth %u", ctx->key_depth); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: kvs key bins %u", ctx->key_bins); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: broker rank %u", my_rank); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: pid %u", ctx->pid); - } - - // If the namespace is provided, copy it into the dyad_ctx_t object - DYAD_LOG_INFO(ctx, "DYAD_CORE: saving KVS namespace"); - if (kvs_namespace == NULL) { - DYAD_LOG_ERROR(ctx, "No KVS namespace provided!\n"); - // TODO see if we want a different return val - goto init_region_failed; - } - namespace_len = strlen(kvs_namespace); - ctx->kvs_namespace = (char *)calloc(namespace_len + 1, sizeof(char)); - if (ctx->kvs_namespace == NULL) { - DYAD_LOG_ERROR(ctx, "Could not allocate buffer for KVS namespace!\n"); - goto init_region_failed; - } - strncpy(ctx->kvs_namespace, kvs_namespace, namespace_len + 1); - if (my_rank == 0) { - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: kvs_namespace %s", ctx->kvs_namespace); - } - - // Initialize the DTL based on the value of dtl_mode - // If an error occurs, log it and return an error - DYAD_LOG_INFO(ctx, "DYAD_CORE: inintializing DYAD DTL %s", dtl_mode_str); - rc = dyad_set_and_init_dtl_mode(dtl_mode_str, dtl_comm_mode); - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, "Cannot initialize the DTL %s\n", dtl_mode_str); - goto init_region_failed; - } - if (my_rank == 0) { - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: dtl_mode %s", dtl_mode_str); - } - - // If the producer-managed path is provided, copy it into the dyad_ctx_t - // object - if (dyad_set_prod_path(prod_managed_path) != DYAD_RC_OK) { - goto init_region_failed; - } + // Set the values in dyad_ctx_t that don't need allocation + ctx->debug = debug; + ctx->check = check; + ctx->shared_storage = shared_storage; + ctx->async_publish = async_publish; + ctx->fsync_write = fsync_write; + ctx->key_depth = key_depth; + ctx->key_bins = key_bins; + + // Open a Flux handle and store it in the dyad_ctx_t + // object. If the open operation failed, return DYAD_FLUXFAIL + // In case of module, we use the flux handle passed to mod_main() + // instead of getting it by flux_open() + + if (flux_handle != NULL) { + ctx->h = (flux_t *)flux_handle; + } else { + ctx->h = flux_open (NULL, 0); + } + if (ctx->h == NULL) { + fprintf (stderr, "Could not open Flux handle!\n"); + rc = DYAD_RC_FLUXFAIL; + goto init_region_finish; + } - // If the consumer-managed path is provided, copy it into the dyad_ctx_t - // object - if (dyad_set_cons_path(cons_managed_path) != DYAD_RC_OK) { - goto init_region_failed; - } - - ctx->relative_to_managed_path = relative_to_managed_path; - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: relative_to_managed_path %s", - ctx->relative_to_managed_path ? "true" : "false"); - - DYAD_C_FUNCTION_UPDATE_STR("prod_managed_path", ctx->prod_managed_path); - DYAD_C_FUNCTION_UPDATE_STR("cons_managed_path", ctx->cons_managed_path); - ctx->use_fs_locks = - true; // This is default value except for streams which dont have a fp. - // TODO Print logging info - rc = DYAD_RC_OK; - // TODO: Add folder option here. + // Get the rank of the Flux broker corresponding + // to the handle. If this fails, return DYAD_FLUXFAIL + DYAD_LOG_INFO (ctx, "DYAD_CORE: getting Flux rank"); + if (flux_get_rank (ctx->h, &(ctx->rank)) < 0) { + DYAD_LOG_INFO (ctx, "Could not get Flux rank!\n"); + rc = DYAD_RC_FLUXFAIL; + goto init_region_finish; + } + my_rank = ctx->rank; + ctx->service_mux = (service_mux < 1u) ? 1u : service_mux; + ctx->node_idx = ctx->rank / ctx->service_mux; + ctx->pid = getpid (); + if (my_rank == 0) { + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: debug %s", ctx->debug ? "true" : "false"); + DYAD_LOG_INFO (ctx, + "DYAD_CORE INIT: shared_storage %s", + ctx->shared_storage ? "true" : "false"); + DYAD_LOG_INFO (ctx, + "DYAD_CORE INIT: async_publish %s", + ctx->async_publish ? "true" : "false"); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: fsync_write %s", ctx->fsync_write ? "true" : "false"); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: kvs key depth %u", ctx->key_depth); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: kvs key bins %u", ctx->key_bins); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: broker rank %u", my_rank); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: pid %u", ctx->pid); + } + + // If the namespace is provided, copy it into the dyad_ctx_t object + DYAD_LOG_INFO (ctx, "DYAD_CORE: saving KVS namespace"); + if (kvs_namespace == NULL) { + DYAD_LOG_ERROR (ctx, "No KVS namespace provided!\n"); + // TODO see if we want a different return val + goto init_region_failed; + } + namespace_len = strlen (kvs_namespace); + ctx->kvs_namespace = (char *)calloc (namespace_len + 1, sizeof (char)); + if (ctx->kvs_namespace == NULL) { + DYAD_LOG_ERROR (ctx, "Could not allocate buffer for KVS namespace!\n"); + goto init_region_failed; + } + strncpy (ctx->kvs_namespace, kvs_namespace, namespace_len + 1); + if (my_rank == 0) { + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: kvs_namespace %s", ctx->kvs_namespace); + } + + // Initialize the DTL based on the value of dtl_mode + // If an error occurs, log it and return an error + DYAD_LOG_INFO (ctx, "DYAD_CORE: inintializing DYAD DTL %s", dtl_mode_str); + rc = dyad_set_and_init_dtl_mode (dtl_mode_str, dtl_comm_mode); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, "Cannot initialize the DTL %s\n", dtl_mode_str); + goto init_region_failed; + } + if (my_rank == 0) { + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: dtl_mode %s", dtl_mode_str); + } + + // If the producer-managed path is provided, copy it into the dyad_ctx_t + // object + if (dyad_set_prod_path (prod_managed_path) != DYAD_RC_OK) { + goto init_region_failed; + } + + // If the consumer-managed path is provided, copy it into the dyad_ctx_t + // object + if (dyad_set_cons_path (cons_managed_path) != DYAD_RC_OK) { + goto init_region_failed; + } + + ctx->relative_to_managed_path = relative_to_managed_path; + DYAD_LOG_INFO (ctx, + "DYAD_CORE INIT: relative_to_managed_path %s", + ctx->relative_to_managed_path ? "true" : "false"); + + DYAD_C_FUNCTION_UPDATE_STR ("prod_managed_path", ctx->prod_managed_path); + DYAD_C_FUNCTION_UPDATE_STR ("cons_managed_path", ctx->cons_managed_path); + ctx->use_fs_locks = true; // This is default value except for streams which dont have a fp. + // TODO Print logging info + rc = DYAD_RC_OK; + // TODO: Add folder option here. #ifndef DYAD_LOGGER_NO_LOG - char log_file_name[4096] = {'\0'}; - char err_file_name[4096] = {'\0'}; - mkdir_as_needed("logs", (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID)); - sprintf(log_file_name, "logs/dyad_core_%d.out", ctx->pid); - sprintf(err_file_name, "logs/dyad_core_%d.err", ctx->pid); - DYAD_LOG_STDERR_REDIRECT(err_file_name); - DYAD_LOG_STDERR_REDIRECT(log_file_name); -#endif // DYAD_LOGGER_NO_LOG - // Initialization is now complete! - ctx->initialized = true; - goto init_region_finish; + char log_file_name[4096] = {'\0'}; + char err_file_name[4096] = {'\0'}; + mkdir_as_needed ("logs", (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID)); + sprintf (log_file_name, "logs/dyad_core_%d.out", ctx->pid); + sprintf (err_file_name, "logs/dyad_core_%d.err", ctx->pid); + DYAD_LOG_STDERR_REDIRECT (err_file_name); + DYAD_LOG_STDERR_REDIRECT (log_file_name); +#endif // DYAD_LOGGER_NO_LOG + // Initialization is now complete! + ctx->initialized = true; + goto init_region_finish; init_region_failed:; - dyad_clear(); - // Set the initial contents of the dyad_ctx_t object to dyad_ctx_default. - *ctx = dyad_ctx_default; - if (rc == DYAD_RC_NOCTX) { - ctx->initialized = false; - ctx->reenter = false; - } - return DYAD_RC_NOCTX; + dyad_clear (); + // Set the initial contents of the dyad_ctx_t object to dyad_ctx_default. + *ctx = dyad_ctx_default; + if (rc == DYAD_RC_NOCTX) { + ctx->initialized = false; + ctx->reenter = false; + } + return DYAD_RC_NOCTX; init_region_finish:; - ctx->reenter = true; - DYAD_C_FUNCTION_END(); - return rc; + ctx->reenter = true; + DYAD_C_FUNCTION_END (); + return rc; } -DYAD_DLL_EXPORTED dyad_rc_t -dyad_init_env(const dyad_dtl_comm_mode_t dtl_comm_mode, void *flux_handle) { - DYAD_C_FUNCTION_START(); - const char *e = NULL; - bool debug = false; - bool check = false; - bool shared_storage = false; - bool reinit = false; - bool async_publish = false; - bool fsync_write = false; - bool relative_to_managed_path = false; - unsigned int key_depth = 0u; - unsigned int key_bins = 0u; - unsigned int service_mux = 1u; - const char *kvs_namespace = NULL; - const char *prod_managed_path = NULL; - const char *cons_managed_path = NULL; - const char *dtl_mode = NULL; - - if ((e = getenv(DYAD_SYNC_DEBUG_ENV))) { - debug = true; - } else { - debug = false; - } - - if (debug) { - DYAD_LOG_STDERR("DYAD_CORE: Initializing with environment " - "variables\n"); - } - - if ((e = getenv(DYAD_SYNC_CHECK_ENV))) { - check = true; - } else { - check = false; - } - - if ((e = getenv(DYAD_SHARED_STORAGE_ENV))) { - shared_storage = true; - } else { - shared_storage = false; - } - - if ((e = getenv(DYAD_REINIT_ENV))) { - reinit = true; - } else { - reinit = false; - } - - if ((e = getenv(DYAD_ASYNC_PUBLISH_ENV))) { - async_publish = true; - } else { - async_publish = false; - } - - if ((e = getenv(DYAD_FSYNC_WRITE_ENV))) { - fsync_write = true; - } else { - fsync_write = false; - } - - if ((e = getenv(DYAD_KEY_DEPTH_ENV))) { - key_depth = atoi(e); - } else { - key_depth = 3; - } - - if ((e = getenv(DYAD_KEY_BINS_ENV))) { - key_bins = atoi(e); - } else { - key_bins = 1024; - } - - if ((e = getenv(DYAD_SERVICE_MUX_ENV))) { - service_mux = atoi(e); - } else { - service_mux = 1u; - } - - if ((e = getenv(DYAD_KVS_NAMESPACE_ENV))) { - kvs_namespace = e; - } else { - kvs_namespace = NULL; - } - - if ((e = getenv(DYAD_PATH_CONSUMER_ENV))) { - cons_managed_path = e; - } else { - cons_managed_path = NULL; - } - - if ((e = getenv(DYAD_PATH_PRODUCER_ENV))) { - prod_managed_path = e; - } else { - prod_managed_path = NULL; - } - - if ((e = getenv(DYAD_PATH_RELATIVE_ENV))) { - relative_to_managed_path = true; - } else { - relative_to_managed_path = false; - } - - if ((e = getenv(DYAD_DTL_MODE_ENV))) { - dtl_mode = e; - } else { - dtl_mode = dyad_dtl_mode_name[DYAD_DTL_DEFAULT]; - DYAD_LOG_STDERR("%s is not set. Defaulting to %s\n", DYAD_DTL_MODE_ENV, - dtl_mode); - } - if (debug) { - DYAD_LOG_STDERR("DYAD_CORE: retrieved configuration from environment. " - "Now initializing DYAD\n"); - } - dyad_rc_t rc = - dyad_init(debug, check, shared_storage, reinit, async_publish, - fsync_write, key_depth, key_bins, service_mux, kvs_namespace, - prod_managed_path, cons_managed_path, relative_to_managed_path, - dtl_mode, dtl_comm_mode, flux_handle); - DYAD_C_FUNCTION_END(); - return rc; +DYAD_DLL_EXPORTED dyad_rc_t dyad_init_env (const dyad_dtl_comm_mode_t dtl_comm_mode, + void *flux_handle) +{ + DYAD_C_FUNCTION_START (); + const char *e = NULL; + bool debug = false; + bool check = false; + bool shared_storage = false; + bool reinit = false; + bool async_publish = false; + bool fsync_write = false; + bool relative_to_managed_path = false; + unsigned int key_depth = 0u; + unsigned int key_bins = 0u; + unsigned int service_mux = 1u; + const char *kvs_namespace = NULL; + const char *prod_managed_path = NULL; + const char *cons_managed_path = NULL; + const char *dtl_mode = NULL; + + if ((e = getenv (DYAD_SYNC_DEBUG_ENV))) { + debug = true; + } else { + debug = false; + } + + if (debug) { + DYAD_LOG_STDERR ( + "DYAD_CORE: Initializing with environment " + "variables\n"); + } + + if ((e = getenv (DYAD_SYNC_CHECK_ENV))) { + check = true; + } else { + check = false; + } + + if ((e = getenv (DYAD_SHARED_STORAGE_ENV))) { + shared_storage = true; + } else { + shared_storage = false; + } + + if ((e = getenv (DYAD_REINIT_ENV))) { + reinit = true; + } else { + reinit = false; + } + + if ((e = getenv (DYAD_ASYNC_PUBLISH_ENV))) { + async_publish = true; + } else { + async_publish = false; + } + + if ((e = getenv (DYAD_FSYNC_WRITE_ENV))) { + fsync_write = true; + } else { + fsync_write = false; + } + + if ((e = getenv (DYAD_KEY_DEPTH_ENV))) { + key_depth = atoi (e); + } else { + key_depth = 3; + } + + if ((e = getenv (DYAD_KEY_BINS_ENV))) { + key_bins = atoi (e); + } else { + key_bins = 1024; + } + + if ((e = getenv (DYAD_SERVICE_MUX_ENV))) { + service_mux = atoi (e); + } else { + service_mux = 1u; + } + + if ((e = getenv (DYAD_KVS_NAMESPACE_ENV))) { + kvs_namespace = e; + } else { + kvs_namespace = NULL; + } + + if ((e = getenv (DYAD_PATH_CONSUMER_ENV))) { + cons_managed_path = e; + } else { + cons_managed_path = NULL; + } + + if ((e = getenv (DYAD_PATH_PRODUCER_ENV))) { + prod_managed_path = e; + } else { + prod_managed_path = NULL; + } + + if ((e = getenv (DYAD_PATH_RELATIVE_ENV))) { + relative_to_managed_path = true; + } else { + relative_to_managed_path = false; + } + + if ((e = getenv (DYAD_DTL_MODE_ENV))) { + dtl_mode = e; + } else { + dtl_mode = dyad_dtl_mode_name[DYAD_DTL_DEFAULT]; + DYAD_LOG_STDERR ("%s is not set. Defaulting to %s\n", DYAD_DTL_MODE_ENV, dtl_mode); + } + if (debug) { + DYAD_LOG_STDERR ( + "DYAD_CORE: retrieved configuration from environment. " + "Now initializing DYAD\n"); + } + dyad_rc_t rc = dyad_init (debug, + check, + shared_storage, + reinit, + async_publish, + fsync_write, + key_depth, + key_bins, + service_mux, + kvs_namespace, + prod_managed_path, + cons_managed_path, + relative_to_managed_path, + dtl_mode, + dtl_comm_mode, + flux_handle); + DYAD_C_FUNCTION_END (); + return rc; } -DYAD_DLL_EXPORTED dyad_rc_t dyad_set_and_init_dtl_mode( - const char *dtl_name, dyad_dtl_comm_mode_t dtl_comm_mode) { - int rc = DYAD_RC_OK; - size_t dtl_name_len = 0ul; - dyad_dtl_mode_t dtl_mode = DYAD_DTL_DEFAULT; - - DYAD_C_FUNCTION_START(); - - if (!dtl_name) { - return DYAD_RC_BADDTLMODE; - } - - dtl_name_len = strlen(dtl_name); - if (strncmp(dtl_name, dyad_dtl_mode_name[DYAD_DTL_DEFAULT], dtl_name_len) == - 0) { - dtl_mode = DYAD_DTL_DEFAULT; - } else if (strncmp(dtl_name, dyad_dtl_mode_name[DYAD_DTL_UCX], - dtl_name_len) == 0) { - dtl_mode = DYAD_DTL_UCX; - } else if (strncmp(dtl_name, dyad_dtl_mode_name[DYAD_DTL_FLUX_RPC], - dtl_name_len) == 0) { - dtl_mode = DYAD_DTL_FLUX_RPC; - } else { - DYAD_LOG_STDERR("Invalid env %s = %s.\n", DYAD_DTL_MODE_ENV, dtl_name); - return DYAD_RC_BADDTLMODE; - } - - DYAD_LOG_INFO(ctx, "DYAD_CORE: DYAD DTL mode to initialize: %s", - dyad_dtl_mode_name[dtl_mode]); - - // If the ctx->dtl_handle is already null, it will just return success - rc = dyad_dtl_finalize(ctx); - if (DYAD_IS_ERROR(rc)) { - goto set_and_init_dtl_mode_region_finish; - } - - rc = dyad_dtl_init(ctx, dtl_mode, dtl_comm_mode, ctx->debug); - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, "Cannot initialize the DTL %s\n", - dyad_dtl_mode_name[dtl_mode]); - } +DYAD_DLL_EXPORTED dyad_rc_t dyad_set_and_init_dtl_mode (const char *dtl_name, + dyad_dtl_comm_mode_t dtl_comm_mode) +{ + int rc = DYAD_RC_OK; + size_t dtl_name_len = 0ul; + dyad_dtl_mode_t dtl_mode = DYAD_DTL_DEFAULT; + + DYAD_C_FUNCTION_START (); + + if (!dtl_name) { + return DYAD_RC_BADDTLMODE; + } + + dtl_name_len = strlen (dtl_name); + if (strncmp (dtl_name, dyad_dtl_mode_name[DYAD_DTL_DEFAULT], dtl_name_len) == 0) { + dtl_mode = DYAD_DTL_DEFAULT; + } else if (strncmp (dtl_name, dyad_dtl_mode_name[DYAD_DTL_UCX], dtl_name_len) == 0) { + dtl_mode = DYAD_DTL_UCX; + } else if (strncmp (dtl_name, dyad_dtl_mode_name[DYAD_DTL_FLUX_RPC], dtl_name_len) == 0) { + dtl_mode = DYAD_DTL_FLUX_RPC; + } else { + DYAD_LOG_STDERR ("Invalid env %s = %s.\n", DYAD_DTL_MODE_ENV, dtl_name); + return DYAD_RC_BADDTLMODE; + } + + DYAD_LOG_INFO (ctx, "DYAD_CORE: DYAD DTL mode to initialize: %s", dyad_dtl_mode_name[dtl_mode]); + + // If the ctx->dtl_handle is already null, it will just return success + rc = dyad_dtl_finalize (ctx); + if (DYAD_IS_ERROR (rc)) { + goto set_and_init_dtl_mode_region_finish; + } + + rc = dyad_dtl_init (ctx, dtl_mode, dtl_comm_mode, ctx->debug); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, "Cannot initialize the DTL %s\n", dyad_dtl_mode_name[dtl_mode]); + } set_and_init_dtl_mode_region_finish:; - DYAD_C_FUNCTION_END(); - return rc; + DYAD_C_FUNCTION_END (); + return rc; } -DYAD_DLL_EXPORTED dyad_rc_t dyad_set_prod_path(const char *prod_managed_path) { - dyad_rc_t rc = DYAD_RC_OK; - bool reenter_backup = false; - size_t prod_path_len = 0ul; - size_t prod_realpath_len = 0ul; - char prod_real_path[PATH_MAX + 1] = {'\0'}; - - if (!ctx) { - DYAD_LOG_STDERR("Empty Producer managed path to allocate!\n"); - rc = DYAD_RC_NOCTX; - goto set_prod_path_region_finish; - } else { - reenter_backup = ctx->reenter; - ctx->reenter = false; - } - DYAD_C_FUNCTION_START(); - - if (ctx->prod_managed_path) { - free(ctx->prod_managed_path); - ctx->prod_managed_path = NULL; - ctx->prod_managed_len = 0u; - ctx->prod_managed_hash = 0u; - } - - if (ctx->prod_real_path) { - free(ctx->prod_real_path); - ctx->prod_real_path = NULL; - ctx->prod_real_len = 0u; - ctx->prod_real_hash = 0u; - } - - DYAD_LOG_INFO(ctx, "DYAD_CORE: saving producer path"); - if (prod_managed_path == NULL) { - DYAD_LOG_INFO(ctx, "DYAD_CORE: producer path is not provided"); - ctx->prod_managed_path = NULL; - ctx->prod_managed_len = 0u; - ctx->prod_managed_hash = 0u; - ctx->prod_real_path = NULL; - ctx->prod_real_len = 0u; - ctx->prod_real_hash = 0u; - } else { - if ((prod_path_len = strlen(prod_managed_path)) == 0ul) { - DYAD_LOG_ERROR(ctx, "Empty Producer managed path to allocate!\n"); - rc = DYAD_RC_BADMANAGEDPATH; - goto set_prod_path_region_failed; - } - ctx->prod_managed_path = (char *)calloc(prod_path_len + 1, sizeof(char)); - if (ctx->prod_managed_path == NULL) { - DYAD_LOG_ERROR(ctx, - "Could not allocate buffer for Producer managed path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_prod_path_region_failed; - } - if (!memcpy(ctx->prod_managed_path, prod_managed_path, prod_path_len + 1)) { - DYAD_LOG_ERROR(ctx, "Could not copy Producer managed path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_prod_path_region_failed; - } - ctx->prod_managed_len = prod_path_len; - ctx->prod_managed_hash = hash_str(ctx->prod_managed_path, DYAD_SEED); - if (ctx->prod_managed_hash == 0u) { - DYAD_LOG_ERROR(ctx, "Could not hash Producer managed path!\n"); - rc = DYAD_RC_BADMANAGEDPATH; - goto set_prod_path_region_failed; - } - - if (!realpath(prod_managed_path, prod_real_path)) { - DYAD_LOG_DEBUG(ctx, "Could not get Producer real path!\n"); - // perror ("Could not get Producer real path!\n"); - // This is not necessarily an error. This can happen when the environment - // variables for both managed paths are set but consumer does not create - // the managed path of producer or vice versa. `realpath()' fails if a - // path does not exist. +DYAD_DLL_EXPORTED dyad_rc_t dyad_set_prod_path (const char *prod_managed_path) +{ + dyad_rc_t rc = DYAD_RC_OK; + bool reenter_backup = false; + size_t prod_path_len = 0ul; + size_t prod_realpath_len = 0ul; + char prod_real_path[PATH_MAX + 1] = {'\0'}; + + if (!ctx) { + DYAD_LOG_STDERR ("Empty Producer managed path to allocate!\n"); + rc = DYAD_RC_NOCTX; + goto set_prod_path_region_finish; } else { - prod_realpath_len = strlen(prod_real_path); + reenter_backup = ctx->reenter; + ctx->reenter = false; } + DYAD_C_FUNCTION_START (); - // If the realpath is the same, we turn off checking against it as it is - // redundant - if (prod_realpath_len == 0u || - strncmp(prod_managed_path, prod_real_path, prod_path_len) == 0) { - DYAD_LOG_INFO(ctx, "DYAD_CORE: producer real path is redundant"); - ctx->prod_real_path = NULL; - ctx->prod_real_len = 0u; - ctx->prod_real_hash = 0u; + if (ctx->prod_managed_path) { + free (ctx->prod_managed_path); + ctx->prod_managed_path = NULL; + ctx->prod_managed_len = 0u; + ctx->prod_managed_hash = 0u; + } + + if (ctx->prod_real_path) { + free (ctx->prod_real_path); + ctx->prod_real_path = NULL; + ctx->prod_real_len = 0u; + ctx->prod_real_hash = 0u; + } + + DYAD_LOG_INFO (ctx, "DYAD_CORE: saving producer path"); + if (prod_managed_path == NULL) { + DYAD_LOG_INFO (ctx, "DYAD_CORE: producer path is not provided"); + ctx->prod_managed_path = NULL; + ctx->prod_managed_len = 0u; + ctx->prod_managed_hash = 0u; + ctx->prod_real_path = NULL; + ctx->prod_real_len = 0u; + ctx->prod_real_hash = 0u; } else { - ctx->prod_real_path = (char *)calloc(prod_realpath_len + 1, sizeof(char)); - if (ctx->prod_real_path == NULL || prod_realpath_len == 0ul) { - DYAD_LOG_ERROR( - ctx, "Could not allocate buffer for Producer managed path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_prod_path_region_failed; - } - - if (!memcpy(ctx->prod_real_path, prod_real_path, prod_realpath_len)) { - DYAD_LOG_ERROR(ctx, "Could not copy Producer real path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_prod_path_region_failed; - } - ctx->prod_real_path[prod_realpath_len] = '\0'; - ctx->prod_real_len = prod_realpath_len; - ctx->prod_real_hash = hash_str(ctx->prod_real_path, DYAD_SEED); - if (ctx->prod_real_hash == 0u) { - DYAD_LOG_ERROR(ctx, "Could not hash Producer real path!\n"); - rc = DYAD_RC_BADMANAGEDPATH; - goto set_prod_path_region_failed; - } - } - - if (ctx->rank == 0) { - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: prod_managed_path %s", - ctx->prod_managed_path); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: prod_managed_len %u", - ctx->prod_managed_len); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: prod_managed_hash %u", - ctx->prod_managed_hash); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: prod_real_path %s", - ctx->prod_real_path); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: prod_real_len %u", - ctx->prod_real_len); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: prod_real_hash %u", - ctx->prod_real_hash); - } - } + if ((prod_path_len = strlen (prod_managed_path)) == 0ul) { + DYAD_LOG_ERROR (ctx, "Empty Producer managed path to allocate!\n"); + rc = DYAD_RC_BADMANAGEDPATH; + goto set_prod_path_region_failed; + } + ctx->prod_managed_path = (char *)calloc (prod_path_len + 1, sizeof (char)); + if (ctx->prod_managed_path == NULL) { + DYAD_LOG_ERROR (ctx, "Could not allocate buffer for Producer managed path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_prod_path_region_failed; + } + if (!memcpy (ctx->prod_managed_path, prod_managed_path, prod_path_len + 1)) { + DYAD_LOG_ERROR (ctx, "Could not copy Producer managed path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_prod_path_region_failed; + } + ctx->prod_managed_len = prod_path_len; + ctx->prod_managed_hash = hash_str (ctx->prod_managed_path, DYAD_SEED); + if (ctx->prod_managed_hash == 0u) { + DYAD_LOG_ERROR (ctx, "Could not hash Producer managed path!\n"); + rc = DYAD_RC_BADMANAGEDPATH; + goto set_prod_path_region_failed; + } + + if (!realpath (prod_managed_path, prod_real_path)) { + DYAD_LOG_DEBUG (ctx, "Could not get Producer real path!\n"); + // perror ("Could not get Producer real path!\n"); + // This is not necessarily an error. This can happen when the environment + // variables for both managed paths are set but consumer does not create + // the managed path of producer or vice versa. `realpath()' fails if a + // path does not exist. + } else { + prod_realpath_len = strlen (prod_real_path); + } + + // If the realpath is the same, we turn off checking against it as it is + // redundant + if (prod_realpath_len == 0u + || strncmp (prod_managed_path, prod_real_path, prod_path_len) == 0) { + DYAD_LOG_INFO (ctx, "DYAD_CORE: producer real path is redundant"); + ctx->prod_real_path = NULL; + ctx->prod_real_len = 0u; + ctx->prod_real_hash = 0u; + } else { + ctx->prod_real_path = (char *)calloc (prod_realpath_len + 1, sizeof (char)); + if (ctx->prod_real_path == NULL || prod_realpath_len == 0ul) { + DYAD_LOG_ERROR (ctx, "Could not allocate buffer for Producer managed path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_prod_path_region_failed; + } + + if (!memcpy (ctx->prod_real_path, prod_real_path, prod_realpath_len)) { + DYAD_LOG_ERROR (ctx, "Could not copy Producer real path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_prod_path_region_failed; + } + ctx->prod_real_path[prod_realpath_len] = '\0'; + ctx->prod_real_len = prod_realpath_len; + ctx->prod_real_hash = hash_str (ctx->prod_real_path, DYAD_SEED); + if (ctx->prod_real_hash == 0u) { + DYAD_LOG_ERROR (ctx, "Could not hash Producer real path!\n"); + rc = DYAD_RC_BADMANAGEDPATH; + goto set_prod_path_region_failed; + } + } + + if (ctx->rank == 0) { + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: prod_managed_path %s", ctx->prod_managed_path); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: prod_managed_len %u", ctx->prod_managed_len); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: prod_managed_hash %u", ctx->prod_managed_hash); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: prod_real_path %s", ctx->prod_real_path); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: prod_real_len %u", ctx->prod_real_len); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: prod_real_hash %u", ctx->prod_real_hash); + } + } set_prod_path_region_failed:; set_prod_path_region_finish:; - if (ctx) { - ctx->reenter = reenter_backup; - } - DYAD_C_FUNCTION_END(); - return rc; + if (ctx) { + ctx->reenter = reenter_backup; + } + DYAD_C_FUNCTION_END (); + return rc; } -DYAD_DLL_EXPORTED dyad_rc_t dyad_set_cons_path(const char *cons_managed_path) { - dyad_rc_t rc = DYAD_RC_OK; - bool reenter_backup = false; - size_t cons_path_len = 0ul; - size_t cons_realpath_len = 0ul; - char cons_real_path[PATH_MAX + 1] = {'\0'}; - - if (!ctx) { - DYAD_LOG_STDERR("Empty Consumer managed path to allocate!\n"); - rc = DYAD_RC_NOCTX; - goto set_cons_path_region_finish; - } else { - reenter_backup = ctx->reenter; - ctx->reenter = false; - } - DYAD_C_FUNCTION_START(); - - if (ctx->cons_managed_path) { - free(ctx->cons_managed_path); - ctx->cons_managed_path = NULL; - ctx->cons_managed_len = 0u; - ctx->cons_managed_hash = 0u; - } - - if (ctx->cons_real_path) { - free(ctx->cons_real_path); - ctx->cons_real_path = NULL; - ctx->cons_real_len = 0u; - ctx->cons_real_hash = 0u; - } - - DYAD_LOG_INFO(ctx, "DYAD_CORE: saving consucer path"); - if (cons_managed_path == NULL) { - DYAD_LOG_INFO(ctx, "DYAD_CORE: consucer path is not provided"); - ctx->cons_managed_path = NULL; - ctx->cons_managed_len = 0u; - ctx->cons_managed_hash = 0u; - ctx->cons_real_path = NULL; - ctx->cons_real_len = 0u; - ctx->cons_real_hash = 0u; - } else { - if ((cons_path_len = strlen(cons_managed_path)) == 0ul) { - DYAD_LOG_ERROR(ctx, "Empty Consumer managed path to allocate!\n"); - rc = DYAD_RC_BADMANAGEDPATH; - goto set_cons_path_region_failed; - } - ctx->cons_managed_path = (char *)calloc(cons_path_len + 1, sizeof(char)); - if (ctx->cons_managed_path == NULL) { - DYAD_LOG_ERROR(ctx, - "Could not allocate buffer for Consumer managed path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_cons_path_region_failed; - } - if (!memcpy(ctx->cons_managed_path, cons_managed_path, cons_path_len + 1)) { - DYAD_LOG_ERROR(ctx, "Could not copy Consumer managed path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_cons_path_region_failed; - } - ctx->cons_managed_len = cons_path_len; - ctx->cons_managed_hash = hash_str(ctx->cons_managed_path, DYAD_SEED); - if (ctx->cons_managed_hash == 0u) { - DYAD_LOG_ERROR(ctx, "Could not hash Consumer managed path!\n"); - rc = DYAD_RC_BADMANAGEDPATH; - goto set_cons_path_region_failed; - } - - if (!realpath(cons_managed_path, cons_real_path)) { - DYAD_LOG_DEBUG(ctx, "Could not get Consumer real path!\n"); - // perror ("Could not get Consumer real path!\n"); - // This is not necessarily an error. This can happen when the environment - // variables for both managed paths are set but consumer does not create - // the managed path of consucer or vice versa. `realpath()' fails if a - // path does not exist. +DYAD_DLL_EXPORTED dyad_rc_t dyad_set_cons_path (const char *cons_managed_path) +{ + dyad_rc_t rc = DYAD_RC_OK; + bool reenter_backup = false; + size_t cons_path_len = 0ul; + size_t cons_realpath_len = 0ul; + char cons_real_path[PATH_MAX + 1] = {'\0'}; + + if (!ctx) { + DYAD_LOG_STDERR ("Empty Consumer managed path to allocate!\n"); + rc = DYAD_RC_NOCTX; + goto set_cons_path_region_finish; } else { - cons_realpath_len = strlen(cons_real_path); + reenter_backup = ctx->reenter; + ctx->reenter = false; } + DYAD_C_FUNCTION_START (); - // If the realpath is the same, we turn off checking against it as it is - // redundant - if (cons_realpath_len == 0u || - strncmp(cons_managed_path, cons_real_path, cons_path_len) == 0) { - DYAD_LOG_INFO(ctx, "DYAD_CORE: consucer real path is redundant"); - ctx->cons_real_path = NULL; - ctx->cons_real_len = 0u; - ctx->cons_real_hash = 0u; + if (ctx->cons_managed_path) { + free (ctx->cons_managed_path); + ctx->cons_managed_path = NULL; + ctx->cons_managed_len = 0u; + ctx->cons_managed_hash = 0u; + } + + if (ctx->cons_real_path) { + free (ctx->cons_real_path); + ctx->cons_real_path = NULL; + ctx->cons_real_len = 0u; + ctx->cons_real_hash = 0u; + } + + DYAD_LOG_INFO (ctx, "DYAD_CORE: saving consucer path"); + if (cons_managed_path == NULL) { + DYAD_LOG_INFO (ctx, "DYAD_CORE: consucer path is not provided"); + ctx->cons_managed_path = NULL; + ctx->cons_managed_len = 0u; + ctx->cons_managed_hash = 0u; + ctx->cons_real_path = NULL; + ctx->cons_real_len = 0u; + ctx->cons_real_hash = 0u; } else { - ctx->cons_real_path = (char *)calloc(cons_realpath_len + 1, sizeof(char)); - if (ctx->cons_real_path == NULL || cons_realpath_len == 0ul) { - DYAD_LOG_ERROR( - ctx, "Could not allocate buffer for Consumer managed path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_cons_path_region_failed; - } - - if (!memcpy(ctx->cons_real_path, cons_real_path, cons_realpath_len)) { - DYAD_LOG_ERROR(ctx, "Could not copy Consumer real path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_cons_path_region_failed; - } - ctx->cons_real_path[cons_realpath_len] = '\0'; - ctx->cons_real_len = cons_realpath_len; - ctx->cons_real_hash = hash_str(ctx->cons_real_path, DYAD_SEED); - if (ctx->cons_real_hash == 0u) { - DYAD_LOG_ERROR(ctx, "Could not hash Consumer real path!\n"); - rc = DYAD_RC_BADMANAGEDPATH; - goto set_cons_path_region_failed; - } - } - - if (ctx->rank == 0) { - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: cons_managed_path %s", - ctx->cons_managed_path); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: cons_managed_len %u", - ctx->cons_managed_len); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: cons_managed_hash %u", - ctx->cons_managed_hash); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: cons_real_path %s", - ctx->cons_real_path); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: cons_real_len %u", - ctx->cons_real_len); - DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: cons_real_hash %u", - ctx->cons_real_hash); - } - } + if ((cons_path_len = strlen (cons_managed_path)) == 0ul) { + DYAD_LOG_ERROR (ctx, "Empty Consumer managed path to allocate!\n"); + rc = DYAD_RC_BADMANAGEDPATH; + goto set_cons_path_region_failed; + } + ctx->cons_managed_path = (char *)calloc (cons_path_len + 1, sizeof (char)); + if (ctx->cons_managed_path == NULL) { + DYAD_LOG_ERROR (ctx, "Could not allocate buffer for Consumer managed path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_cons_path_region_failed; + } + if (!memcpy (ctx->cons_managed_path, cons_managed_path, cons_path_len + 1)) { + DYAD_LOG_ERROR (ctx, "Could not copy Consumer managed path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_cons_path_region_failed; + } + ctx->cons_managed_len = cons_path_len; + ctx->cons_managed_hash = hash_str (ctx->cons_managed_path, DYAD_SEED); + if (ctx->cons_managed_hash == 0u) { + DYAD_LOG_ERROR (ctx, "Could not hash Consumer managed path!\n"); + rc = DYAD_RC_BADMANAGEDPATH; + goto set_cons_path_region_failed; + } + + if (!realpath (cons_managed_path, cons_real_path)) { + DYAD_LOG_DEBUG (ctx, "Could not get Consumer real path!\n"); + // perror ("Could not get Consumer real path!\n"); + // This is not necessarily an error. This can happen when the environment + // variables for both managed paths are set but consumer does not create + // the managed path of consucer or vice versa. `realpath()' fails if a + // path does not exist. + } else { + cons_realpath_len = strlen (cons_real_path); + } + + // If the realpath is the same, we turn off checking against it as it is + // redundant + if (cons_realpath_len == 0u + || strncmp (cons_managed_path, cons_real_path, cons_path_len) == 0) { + DYAD_LOG_INFO (ctx, "DYAD_CORE: consucer real path is redundant"); + ctx->cons_real_path = NULL; + ctx->cons_real_len = 0u; + ctx->cons_real_hash = 0u; + } else { + ctx->cons_real_path = (char *)calloc (cons_realpath_len + 1, sizeof (char)); + if (ctx->cons_real_path == NULL || cons_realpath_len == 0ul) { + DYAD_LOG_ERROR (ctx, "Could not allocate buffer for Consumer managed path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_cons_path_region_failed; + } + + if (!memcpy (ctx->cons_real_path, cons_real_path, cons_realpath_len)) { + DYAD_LOG_ERROR (ctx, "Could not copy Consumer real path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_cons_path_region_failed; + } + ctx->cons_real_path[cons_realpath_len] = '\0'; + ctx->cons_real_len = cons_realpath_len; + ctx->cons_real_hash = hash_str (ctx->cons_real_path, DYAD_SEED); + if (ctx->cons_real_hash == 0u) { + DYAD_LOG_ERROR (ctx, "Could not hash Consumer real path!\n"); + rc = DYAD_RC_BADMANAGEDPATH; + goto set_cons_path_region_failed; + } + } + + if (ctx->rank == 0) { + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: cons_managed_path %s", ctx->cons_managed_path); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: cons_managed_len %u", ctx->cons_managed_len); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: cons_managed_hash %u", ctx->cons_managed_hash); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: cons_real_path %s", ctx->cons_real_path); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: cons_real_len %u", ctx->cons_real_len); + DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: cons_real_hash %u", ctx->cons_real_hash); + } + } set_cons_path_region_failed:; set_cons_path_region_finish:; - if (ctx) { - ctx->reenter = reenter_backup; - } - DYAD_C_FUNCTION_END(); - return rc; + if (ctx) { + ctx->reenter = reenter_backup; + } + DYAD_C_FUNCTION_END (); + return rc; } -DYAD_DLL_EXPORTED dyad_rc_t dyad_clear() { - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - DYAD_LOG_STDERR("DYAD context is being cleared!\n"); - if (ctx == NULL) { +DYAD_DLL_EXPORTED dyad_rc_t dyad_clear () +{ + DYAD_C_FUNCTION_START (); + dyad_rc_t rc = DYAD_RC_OK; + DYAD_LOG_STDERR ("DYAD context is being cleared!\n"); + if (ctx == NULL) { + rc = DYAD_RC_OK; + goto clear_region_finish; + } + dyad_dtl_finalize (ctx); + if (ctx->h != NULL) { + flux_close (ctx->h); + ctx->h = NULL; + } + if (ctx->kvs_namespace != NULL) { + free (ctx->kvs_namespace); + ctx->kvs_namespace = NULL; + } + if (ctx->prod_managed_path != NULL) { + free (ctx->prod_managed_path); + ctx->prod_managed_path = NULL; + } + if (ctx->prod_real_path != NULL) { + free (ctx->prod_real_path); + ctx->prod_real_path = NULL; + } + if (ctx->cons_managed_path != NULL) { + free (ctx->cons_managed_path); + ctx->cons_managed_path = NULL; + } + if (ctx->cons_real_path != NULL) { + free (ctx->cons_real_path); + ctx->cons_real_path = NULL; + } rc = DYAD_RC_OK; - goto clear_region_finish; - } - dyad_dtl_finalize(ctx); - if (ctx->h != NULL) { - flux_close(ctx->h); - ctx->h = NULL; - } - if (ctx->kvs_namespace != NULL) { - free(ctx->kvs_namespace); - ctx->kvs_namespace = NULL; - } - if (ctx->prod_managed_path != NULL) { - free(ctx->prod_managed_path); - ctx->prod_managed_path = NULL; - } - if (ctx->prod_real_path != NULL) { - free(ctx->prod_real_path); - ctx->prod_real_path = NULL; - } - if (ctx->cons_managed_path != NULL) { - free(ctx->cons_managed_path); - ctx->cons_managed_path = NULL; - } - if (ctx->cons_real_path != NULL) { - free(ctx->cons_real_path); - ctx->cons_real_path = NULL; - } - rc = DYAD_RC_OK; clear_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); #ifdef DYAD_PROFILER_DFTRACER - DFTRACER_C_FINI(); + DFTRACER_C_FINI (); #endif - return rc; + return rc; } -DYAD_DLL_EXPORTED dyad_rc_t dyad_finalize() { - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - // DYAD_LOG_STDERR ("DYAD context is being destroyed!\n"); - if (ctx == NULL) { +DYAD_DLL_EXPORTED dyad_rc_t dyad_finalize () +{ + DYAD_C_FUNCTION_START (); + dyad_rc_t rc = DYAD_RC_OK; + // DYAD_LOG_STDERR ("DYAD context is being destroyed!\n"); + if (ctx == NULL) { + rc = DYAD_RC_OK; + goto finalize_region_finish; + } + dyad_clear (); + free (ctx); + ctx = NULL; rc = DYAD_RC_OK; - goto finalize_region_finish; - } - dyad_clear(); - free(ctx); - ctx = NULL; - rc = DYAD_RC_OK; finalize_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); #ifdef DYAD_PROFILER_DFTRACER - DFTRACER_C_FINI(); + DFTRACER_C_FINI (); #endif - return rc; + return rc; } diff --git a/src/dyad/dtl/dyad_dtl_api.c b/src/dyad/dtl/dyad_dtl_api.c index bc35a7ce..268af4bd 100644 --- a/src/dyad/dtl/dyad_dtl_api.c +++ b/src/dyad/dtl/dyad_dtl_api.c @@ -4,27 +4,27 @@ #error "no config" #endif -#include -#include #include #include +#include +#include #if DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA #include "ucx_dtl.h" -#endif // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA dyad_rc_t dyad_dtl_init (dyad_ctx_t* ctx, dyad_dtl_mode_t mode, dyad_dtl_comm_mode_t comm_mode, bool debug) { - DYAD_C_FUNCTION_START(); - DYAD_LOG_DEBUG(ctx, "Initializing DTL ..."); + DYAD_C_FUNCTION_START (); + DYAD_LOG_DEBUG (ctx, "Initializing DTL ..."); dyad_rc_t rc = DYAD_RC_OK; ctx->dtl_handle = malloc (sizeof (struct dyad_dtl)); if (ctx->dtl_handle == NULL) { rc = DYAD_RC_SYSFAIL; - DYAD_LOG_ERROR(ctx, "Cannot allocate Memory"); + DYAD_LOG_ERROR (ctx, "Cannot allocate Memory"); goto dtl_init_done; } ctx->dtl_handle->mode = mode; @@ -36,9 +36,9 @@ dyad_rc_t dyad_dtl_init (dyad_ctx_t* ctx, goto dtl_init_done; } } else if (mode == DYAD_DTL_FLUX_RPC) { -#else // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA +#else // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA if (mode == DYAD_DTL_FLUX_RPC) { -#endif // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA rc = dyad_dtl_flux_init (ctx, mode, comm_mode, debug); if (DYAD_IS_ERROR (rc)) { DYAD_LOG_ERROR (ctx, "dyad_dtl_flux_init initialization failed rc %d", rc); @@ -46,19 +46,21 @@ dyad_rc_t dyad_dtl_init (dyad_ctx_t* ctx, } } else { rc = DYAD_RC_BADDTLMODE; - DYAD_LOG_ERROR (ctx, "dyad_dtl_flux_init initialization failed with incorrect mode %d", mode); + DYAD_LOG_ERROR (ctx, + "dyad_dtl_flux_init initialization failed with incorrect mode %d", + mode); goto dtl_init_done; } rc = DYAD_RC_OK; DYAD_LOG_DEBUG (ctx, "Finished dyad_dtl_init successfully"); dtl_init_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_finalize (dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; if (ctx->dtl_handle == NULL) { // We should only reach this line if the user has passed @@ -77,9 +79,9 @@ dyad_rc_t dyad_dtl_finalize (dyad_ctx_t* ctx) } } } else if ((ctx->dtl_handle)->mode == DYAD_DTL_FLUX_RPC) { -#else // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA +#else // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA if ((ctx->dtl_handle)->mode == DYAD_DTL_FLUX_RPC) { -#endif // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_DTL || DYAD_ENABLE_UCX_RMA if ((ctx->dtl_handle)->private_dtl.flux_dtl_handle != NULL) { rc = dyad_dtl_flux_finalize (ctx); if (DYAD_IS_ERROR (rc)) { @@ -95,6 +97,6 @@ dyad_rc_t dyad_dtl_finalize (dyad_ctx_t* ctx) dtl_finalize_done:; free (ctx->dtl_handle); ctx->dtl_handle = NULL; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } diff --git a/src/dyad/dtl/dyad_dtl_api.h b/src/dyad/dtl/dyad_dtl_api.h index 62c6ca5d..0305fa9b 100644 --- a/src/dyad/dtl/dyad_dtl_api.h +++ b/src/dyad/dtl/dyad_dtl_api.h @@ -18,46 +18,54 @@ #ifdef __cplusplus #include - -extern "C" { #else #include #endif +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + // Forward declarations of DTL contexts for the underlying implementations struct dyad_dtl_ucx; struct dyad_dtl_flux; // Union type to store the underlying DTL contexts union dyad_dtl_private { - struct dyad_dtl_ucx *ucx_dtl_handle; - struct dyad_dtl_flux *flux_dtl_handle; -} __attribute__((aligned(16))); + struct dyad_dtl_ucx *ucx_dtl_handle; + struct dyad_dtl_flux *flux_dtl_handle; +} __attribute__ ((aligned (16))); typedef union dyad_dtl_private dyad_dtl_private_t; struct dyad_dtl { - dyad_dtl_private_t private_dtl; - dyad_dtl_mode_t mode; - dyad_rc_t (*rpc_pack)(const dyad_ctx_t *ctx, const char *upath, - uint32_t producer_rank, json_t **packed_obj); - dyad_rc_t (*rpc_unpack)(const dyad_ctx_t *ctx, const flux_msg_t *packed_obj, - char **upath); - dyad_rc_t (*rpc_respond)(const dyad_ctx_t *ctx, const flux_msg_t *orig_msg); - dyad_rc_t (*rpc_recv_response)(const dyad_ctx_t *ctx, flux_future_t *f); - dyad_rc_t (*get_buffer)(const dyad_ctx_t *ctx, size_t data_size, - void **data_buf); - dyad_rc_t (*return_buffer)(const dyad_ctx_t *ctx, void **data_buf); - dyad_rc_t (*establish_connection)(const dyad_ctx_t *ctx); - dyad_rc_t (*send)(const dyad_ctx_t *ctx, void *buf, size_t buflen); - dyad_rc_t (*recv)(const dyad_ctx_t *ctx, void **buf, size_t *buflen); - dyad_rc_t (*close_connection)(const dyad_ctx_t *ctx); -} __attribute__((aligned(256))); + dyad_dtl_private_t private_dtl; + dyad_dtl_mode_t mode; + dyad_rc_t (*rpc_pack) (const dyad_ctx_t *ctx, + const char *upath, + uint32_t producer_rank, + json_t **packed_obj); + dyad_rc_t (*rpc_unpack) (const dyad_ctx_t *ctx, const flux_msg_t *packed_obj, char **upath); + dyad_rc_t (*rpc_respond) (const dyad_ctx_t *ctx, const flux_msg_t *orig_msg); + dyad_rc_t (*rpc_recv_response) (const dyad_ctx_t *ctx, flux_future_t *f); + dyad_rc_t (*get_buffer) (const dyad_ctx_t *ctx, size_t data_size, void **data_buf); + dyad_rc_t (*return_buffer) (const dyad_ctx_t *ctx, void **data_buf); + dyad_rc_t (*establish_connection) (const dyad_ctx_t *ctx); + dyad_rc_t (*send) (const dyad_ctx_t *ctx, void *buf, size_t buflen); + dyad_rc_t (*recv) (const dyad_ctx_t *ctx, void **buf, size_t *buflen); + dyad_rc_t (*close_connection) (const dyad_ctx_t *ctx); +} __attribute__ ((aligned (256))); typedef struct dyad_dtl dyad_dtl_t; -dyad_rc_t dyad_dtl_init(dyad_ctx_t *ctx, dyad_dtl_mode_t mode, - dyad_dtl_comm_mode_t comm_mode, bool debug); +dyad_rc_t dyad_dtl_init (dyad_ctx_t *ctx, + dyad_dtl_mode_t mode, + dyad_dtl_comm_mode_t comm_mode, + bool debug); -dyad_rc_t dyad_dtl_finalize(dyad_ctx_t *ctx); +dyad_rc_t dyad_dtl_finalize (dyad_ctx_t *ctx); #ifdef __cplusplus } diff --git a/src/dyad/dtl/flux_dtl.c b/src/dyad/dtl/flux_dtl.c index 80c7aeda..a959a2eb 100644 --- a/src/dyad/dtl/flux_dtl.c +++ b/src/dyad/dtl/flux_dtl.c @@ -4,10 +4,11 @@ #error "no config" #endif -#include +#include // sysconf + #include #include -#include // sysconf +#include dyad_rc_t dyad_dtl_flux_init (const dyad_ctx_t* ctx, dyad_dtl_mode_t mode, @@ -15,14 +16,14 @@ dyad_rc_t dyad_dtl_flux_init (const dyad_ctx_t* ctx, bool debug) { dyad_rc_t rc = DYAD_RC_OK; - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); ctx->dtl_handle->private_dtl.flux_dtl_handle = malloc (sizeof (struct dyad_dtl_flux)); if (ctx->dtl_handle->private_dtl.flux_dtl_handle == NULL) { DYAD_LOG_ERROR (ctx, "Cannot allocate the Flux DTL handle"); rc = DYAD_RC_SYSFAIL; goto dtl_flux_init_region_finish; } - ctx->dtl_handle->private_dtl.flux_dtl_handle->h = (flux_t*) ctx->h; + ctx->dtl_handle->private_dtl.flux_dtl_handle->h = (flux_t*)ctx->h; ctx->dtl_handle->private_dtl.flux_dtl_handle->comm_mode = comm_mode; ctx->dtl_handle->private_dtl.flux_dtl_handle->debug = debug; ctx->dtl_handle->private_dtl.flux_dtl_handle->f = NULL; @@ -40,7 +41,7 @@ dyad_rc_t dyad_dtl_flux_init (const dyad_ctx_t* ctx, ctx->dtl_handle->close_connection = dyad_dtl_flux_close_connection; dtl_flux_init_region_finish: - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } @@ -49,7 +50,7 @@ dyad_rc_t dyad_dtl_flux_rpc_pack (const dyad_ctx_t* ctx, uint32_t producer_rank, json_t** restrict packed_obj) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); DYAD_C_FUNCTION_UPDATE_INT ("producer_rank", producer_rank); dyad_rc_t rc = DYAD_RC_OK; @@ -60,19 +61,18 @@ dyad_rc_t dyad_dtl_flux_rpc_pack (const dyad_ctx_t* ctx, goto dtl_flux_rpc_pack; } dtl_flux_rpc_pack: - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_flux_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg, char** upath) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); int rc = 0; dyad_rc_t dyad_rc = DYAD_RC_OK; rc = flux_request_unpack (msg, NULL, "{s:s}", "upath", upath); if (FLUX_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, - "Could not unpack Flux message from consumer"); + DYAD_LOG_ERROR (ctx, "Could not unpack Flux message from consumer"); // TODO create new RC for this dyad_rc = DYAD_RC_BADUNPACK; goto dtl_flux_rpc_unpack_region_finish; @@ -81,28 +81,28 @@ dyad_rc_t dyad_dtl_flux_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg dyad_rc = DYAD_RC_OK; DYAD_C_FUNCTION_UPDATE_STR ("upath", *upath); dtl_flux_rpc_unpack_region_finish: - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return dyad_rc; } dyad_rc_t dyad_dtl_flux_rpc_respond (const dyad_ctx_t* ctx, const flux_msg_t* orig_msg) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; } dyad_rc_t dyad_dtl_flux_rpc_recv_response (const dyad_ctx_t* ctx, flux_future_t* f) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); ctx->dtl_handle->private_dtl.flux_dtl_handle->f = f; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; } dyad_rc_t dyad_dtl_flux_get_buffer (const dyad_ctx_t* ctx, size_t data_size, void** data_buf) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; if (data_buf == NULL || *data_buf != NULL) { @@ -116,7 +116,7 @@ dyad_rc_t dyad_dtl_flux_get_buffer (const dyad_ctx_t* ctx, size_t data_size, voi goto flux_get_buf_done; } #else - rc = posix_memalign (data_buf, sysconf(_SC_PAGESIZE), data_size); + rc = posix_memalign (data_buf, sysconf (_SC_PAGESIZE), data_size); if (rc != 0 || *data_buf == NULL) { rc = DYAD_RC_SYSFAIL; goto flux_get_buf_done; @@ -125,13 +125,13 @@ dyad_rc_t dyad_dtl_flux_get_buffer (const dyad_ctx_t* ctx, size_t data_size, voi rc = DYAD_RC_OK; flux_get_buf_done: - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_flux_return_buffer (const dyad_ctx_t* ctx, void** data_buf) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; if (data_buf == NULL || *data_buf == NULL) { rc = DYAD_RC_BADBUF; @@ -141,49 +141,47 @@ dyad_rc_t dyad_dtl_flux_return_buffer (const dyad_ctx_t* ctx, void** data_buf) rc = DYAD_RC_OK; flux_ret_buf_done: - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_flux_establish_connection (const dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; } dyad_rc_t dyad_dtl_flux_send (const dyad_ctx_t* ctx, void* buf, size_t buflen) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t dyad_rc = DYAD_RC_OK; int rc = 0; - DYAD_LOG_INFO (ctx, - "Send data to consumer using a Flux RPC response"); + DYAD_LOG_INFO (ctx, "Send data to consumer using a Flux RPC response"); rc = flux_respond_raw (ctx->dtl_handle->private_dtl.flux_dtl_handle->h, ctx->dtl_handle->private_dtl.flux_dtl_handle->msg, buf, (int)buflen); if (FLUX_IS_ERROR (rc)) { DYAD_LOG_ERROR (ctx, - "Could not send Flux RPC response containing file " - "contents"); + "Could not send Flux RPC response containing file " + "contents"); dyad_rc = DYAD_RC_FLUXFAIL; goto dtl_flux_send_region_finish; } if (ctx->dtl_handle->private_dtl.flux_dtl_handle->debug) { - DYAD_LOG_INFO (ctx, - "Successfully sent file contents to consumer"); + DYAD_LOG_INFO (ctx, "Successfully sent file contents to consumer"); } dyad_rc = DYAD_RC_OK; dtl_flux_send_region_finish: DYAD_C_FUNCTION_UPDATE_INT ("buflen", buflen); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return dyad_rc; } dyad_rc_t dyad_dtl_flux_recv (const dyad_ctx_t* ctx, void** buf, size_t* buflen) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); int rc = 0; dyad_rc_t dyad_rc = DYAD_RC_OK; errno = 0; @@ -206,7 +204,7 @@ dyad_rc_t dyad_dtl_flux_recv (const dyad_ctx_t* ctx, void** buf, size_t* buflen) dyad_rc = DYAD_RC_BADRPC; goto finish_recv; } - *buflen = (size_t) tmp_buflen; + *buflen = (size_t)tmp_buflen; dyad_rc = ctx->dtl_handle->get_buffer (ctx, *buflen, buf); if (DYAD_IS_ERROR (dyad_rc)) { *buf = NULL; @@ -216,27 +214,27 @@ dyad_rc_t dyad_dtl_flux_recv (const dyad_ctx_t* ctx, void** buf, size_t* buflen) memcpy (*buf, tmp_buf, *buflen); dyad_rc = DYAD_RC_OK; finish_recv: - if (dtl_handle->f != NULL) + if (dtl_handle->f != NULL) flux_future_reset (dtl_handle->f); DYAD_C_FUNCTION_UPDATE_INT ("tmp_buflen", tmp_buflen); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return dyad_rc; } dyad_rc_t dyad_dtl_flux_close_connection (const dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); if (ctx->dtl_handle->private_dtl.flux_dtl_handle->f != NULL) ctx->dtl_handle->private_dtl.flux_dtl_handle->f = NULL; if (ctx->dtl_handle->private_dtl.flux_dtl_handle->msg != NULL) ctx->dtl_handle->private_dtl.flux_dtl_handle->msg = NULL; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; } dyad_rc_t dyad_dtl_flux_finalize (const dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; if (ctx->dtl_handle == NULL) { goto dtl_flux_finalize_done; @@ -247,6 +245,6 @@ dyad_rc_t dyad_dtl_flux_finalize (const dyad_ctx_t* ctx) free (ctx->dtl_handle->private_dtl.flux_dtl_handle); ctx->dtl_handle->private_dtl.flux_dtl_handle = NULL; dtl_flux_finalize_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } diff --git a/src/dyad/dtl/ucx_dtl.c b/src/dyad/dtl/ucx_dtl.c index 23b28fb4..f281c8c5 100644 --- a/src/dyad/dtl/ucx_dtl.c +++ b/src/dyad/dtl/ucx_dtl.c @@ -5,17 +5,18 @@ #endif #include -#include -#include -#include -#include +#include #include #include #include #include -#include -#include #include +#include + +#include +#include +#include +#include extern const base64_maps_t base64_maps_rfc4648; @@ -35,11 +36,11 @@ typedef struct ucx_request dyad_ucx_request_t; // initialize our request struct static void dyad_ucx_request_init (void* request) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_ucx_request_t* real_request = NULL; real_request = (dyad_ucx_request_t*)request; real_request->completed = 0; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); } #ifndef DYAD_ENABLE_UCX_RMA // Define a function that ucp_tag_msg_recv_nbx will use @@ -50,36 +51,35 @@ static void dyad_recv_callback (void* request, ucs_status_t status, const ucp_tag_recv_info_t* tag_info, void* user_data) -#else // UCP_API_VERSION +#else // UCP_API_VERSION static void dyad_recv_callback (void* request, ucs_status_t status, ucp_tag_recv_info_t* tag_info) -#endif // UCP_API_VERSION +#endif // UCP_API_VERSION { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_ucx_request_t* real_request = NULL; real_request = (dyad_ucx_request_t*)request; real_request->completed = 1; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); } -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA #if UCP_API_VERSION >= UCP_VERSION(1, 10) static void dyad_send_callback (void* req, ucs_status_t status, void* ctx) -#else // UCP_API_VERSION +#else // UCP_API_VERSION static void dyad_send_callback (void* req, ucs_status_t status) -#endif // UCP_API_VERSION +#endif // UCP_API_VERSION { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_LOG_STDERR ("Calling send callback"); dyad_ucx_request_t* real_req = (dyad_ucx_request_t*)req; real_req->completed = 1; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); } // Simple function used to wait on the async receive -static ucs_status_t dyad_ucx_request_wait (const dyad_ctx_t* ctx, - dyad_ucx_request_t* request) +static ucs_status_t dyad_ucx_request_wait (const dyad_ctx_t* ctx, dyad_ucx_request_t* request) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); ucs_status_t final_request_status = UCS_OK; if (UCS_OK == request) { DYAD_LOG_INFO (ctx, "Finished Immediately for Request"); @@ -118,15 +118,15 @@ static ucs_status_t dyad_ucx_request_wait (const dyad_ctx_t* ctx, // So, we simply set the status to UCS_OK final_request_status = UCS_OK; dtl_ucx_request_wait_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return final_request_status; } -static dyad_rc_t ucx_allocate_buffer (const dyad_ctx_t *ctx, +static dyad_rc_t ucx_allocate_buffer (const dyad_ctx_t* ctx, dyad_dtl_ucx_t* dtl_handle, dyad_dtl_comm_mode_t comm_mode) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_BADBUF; ucs_status_t status; ucp_mem_map_params_t mmap_params; @@ -146,7 +146,7 @@ static dyad_rc_t ucx_allocate_buffer (const dyad_ctx_t *ctx, | UCP_MEM_MAP_PARAM_FIELD_PROT; mmap_params.address = NULL; mmap_params.memory_type = UCS_MEMORY_TYPE_HOST; - mmap_params.length = dtl_handle->max_transfer_size + sizeof(size_t); + mmap_params.length = dtl_handle->max_transfer_size + sizeof (size_t); mmap_params.flags = UCP_MEM_MAP_ALLOCATE; if (dtl_handle->comm_mode == DYAD_COMM_SEND) { mmap_params.prot = UCP_MEM_MAP_PROT_LOCAL_READ; @@ -183,16 +183,16 @@ static dyad_rc_t ucx_allocate_buffer (const dyad_ctx_t *ctx, rc = DYAD_RC_OK; ucx_allocate_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -static dyad_rc_t ucx_free_buffer (const dyad_ctx_t *ctx, +static dyad_rc_t ucx_free_buffer (const dyad_ctx_t* ctx, ucp_context_h ucp_ctx, ucp_mem_h mem_handle, void** buf) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; if (ucp_ctx == NULL) { rc = DYAD_RC_NOCTX; @@ -215,17 +215,20 @@ static dyad_rc_t ucx_free_buffer (const dyad_ctx_t *ctx, rc = DYAD_RC_OK; ucx_free_buf_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -static inline ucs_status_ptr_t ucx_send_no_wait (const dyad_ctx_t* ctx, bool is_warmup, void* buf, size_t buflen) +static inline ucs_status_ptr_t ucx_send_no_wait (const dyad_ctx_t* ctx, + bool is_warmup, + void* buf, + size_t buflen) { /** * The warmup is not used but is passed for consistency with the recv_no_wait calls. */ - (void) is_warmup; - DYAD_C_FUNCTION_START(); + (void)is_warmup; + DYAD_C_FUNCTION_START (); ucs_status_ptr_t stat_ptr = NULL; dyad_dtl_ucx_t* dtl_handle = ctx->dtl_handle->private_dtl.ucx_dtl_handle; if (dtl_handle->ep == NULL) { @@ -245,18 +248,18 @@ static inline ucs_status_ptr_t ucx_send_no_wait (const dyad_ctx_t* ctx, bool is_ params.cb.send = dyad_send_callback; DYAD_LOG_INFO (ctx, "Sending data to consumer with ucp_tag_send_nbx"); stat_ptr = ucp_tag_send_nbx (dtl_handle->ep, buf, buflen, dtl_handle->comm_tag, ¶ms); -#else // UCP_API_VERSION - DYAD_LOG_INFO (ctx, "Sending %lu bytes of data to consumer with ucp_tag_send_nb", \ - buflen); +#else // UCP_API_VERSION + DYAD_LOG_INFO (ctx, "Sending %lu bytes of data to consumer with ucp_tag_send_nb", buflen); stat_ptr = ucp_tag_send_nb (dtl_handle->ep, buf, buflen, UCP_DATATYPE_CONTIG, dtl_handle->comm_tag, dyad_send_callback); -#endif // UCP_API_VERSION -#else // DYAD_ENABLE_UCX_RMA - ucs_status_t status = ucp_ep_rkey_unpack (dtl_handle->ep, dtl_handle->rkey_buf, &(dtl_handle->rkey)); +#endif // UCP_API_VERSION +#else // DYAD_ENABLE_UCX_RMA + ucs_status_t status = + ucp_ep_rkey_unpack (dtl_handle->ep, dtl_handle->rkey_buf, &(dtl_handle->rkey)); if (UCX_STATUS_FAIL (status)) { DYAD_LOG_ERROR (ctx, "ucp_ep_rkey_unpack failed"); stat_ptr = (void*)UCS_ERR_NOT_CONNECTED; @@ -265,18 +268,24 @@ static inline ucs_status_ptr_t ucx_send_no_wait (const dyad_ctx_t* ctx, bool is_ ucp_request_param_t params; params.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK; params.cb.send = dyad_send_callback; - stat_ptr = ucp_put_nbx(dtl_handle->ep, buf, buflen, dtl_handle->cons_buf_ptr, dtl_handle->rkey, ¶ms); - if (UCS_PTR_IS_ERR(stat_ptr)) { - DYAD_LOG_ERROR (ctx, "ucp_put_nbx() failed %s (%d)\n", \ - ucs_status_string(UCS_PTR_STATUS(stat_ptr)), \ - UCS_PTR_STATUS(stat_ptr)); + stat_ptr = ucp_put_nbx (dtl_handle->ep, + buf, + buflen, + dtl_handle->cons_buf_ptr, + dtl_handle->rkey, + ¶ms); + if (UCS_PTR_IS_ERR (stat_ptr)) { + DYAD_LOG_ERROR (ctx, + "ucp_put_nbx() failed %s (%d)\n", + ucs_status_string (UCS_PTR_STATUS (stat_ptr)), + UCS_PTR_STATUS (stat_ptr)); stat_ptr = (void*)UCS_ERR_NOT_CONNECTED; goto ucx_send_no_wait_done; } DYAD_LOG_INFO (ctx, "written data buf of length %lu", buflen); -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA ucx_send_no_wait_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return stat_ptr; } @@ -285,7 +294,7 @@ static inline ucs_status_ptr_t ucx_recv_no_wait (const dyad_ctx_t* ctx, void** buf, size_t* buflen) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); ucs_status_ptr_t stat_ptr = NULL; #ifndef DYAD_ENABLE_UCX_RMA dyad_rc_t rc = DYAD_RC_OK; @@ -355,8 +364,10 @@ static inline ucs_status_ptr_t ucx_recv_no_wait (const dyad_ctx_t* ctx, // The metadata retrived from the probed tag recv event contains // the size of the data to be sent. // So, use that size to allocate a buffer - DYAD_LOG_INFO (ctx, "Got message with tag %lu and size %lu\n", \ - msg_info.sender_tag, msg_info.length); + DYAD_LOG_INFO (ctx, + "Got message with tag %lu and size %lu\n", + msg_info.sender_tag, + msg_info.length); *buflen = msg_info.length; if (is_warmup) { if (*buflen > dtl_handle->max_transfer_size) { @@ -391,36 +402,36 @@ static inline ucs_status_ptr_t ucx_recv_no_wait (const dyad_ctx_t* ctx, recv_params.memory_type = UCS_MEMORY_TYPE_HOST; // Perform the async recv operation using the probed tag recv event stat_ptr = ucp_tag_msg_recv_nbx (dtl_handle->ucx_worker, *buf, *buflen, msg, &recv_params); -#else // UCP_API_VERSION +#else // UCP_API_VERSION stat_ptr = ucp_tag_msg_recv_nb (dtl_handle->ucx_worker, *buf, *buflen, UCP_DATATYPE_CONTIG, msg, dyad_recv_callback); -#endif // UCP_API_VERSION -#else // DYAD_ENABLE_UCX_RMA +#endif // UCP_API_VERSION +#else // DYAD_ENABLE_UCX_RMA dyad_dtl_ucx_t* dtl_handle = ctx->dtl_handle->private_dtl.ucx_dtl_handle; ssize_t temp = 0l; do { - memcpy (&temp, dtl_handle->net_buf, sizeof(temp)); - ucp_worker_progress(ctx->dtl_handle->private_dtl.ucx_dtl_handle->ucx_worker); - nanosleep((const struct timespec[]){{0, 10000L}}, NULL); + memcpy (&temp, dtl_handle->net_buf, sizeof (temp)); + ucp_worker_progress (ctx->dtl_handle->private_dtl.ucx_dtl_handle->ucx_worker); + nanosleep ((const struct timespec[]){{0, 10000L}}, NULL); DYAD_LOG_DEBUG (ctx, "Consumer Waiting for worker to finsih all work"); } while (temp == 0l); -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA DYAD_LOG_DEBUG (ctx, "Consumer finsihed all work"); #ifndef DYAD_ENABLE_UCX_RMA ucx_recv_no_wait_done:; -#endif // DYAD_ENABLE_UCX_RMA - DYAD_C_FUNCTION_END(); +#endif // DYAD_ENABLE_UCX_RMA + DYAD_C_FUNCTION_END (); return stat_ptr; } static dyad_rc_t ucx_warmup (const dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; void* send_buf = NULL; void* recv_buf = NULL; @@ -428,7 +439,7 @@ static dyad_rc_t ucx_warmup (const dyad_ctx_t* ctx) #ifndef DYAD_ENABLE_UCX_RMA ucs_status_ptr_t recv_stat_ptr = NULL; size_t recv_buf_len = 0; -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA ucs_status_t send_status = UCS_OK; ucs_status_t recv_status = UCS_OK; DYAD_LOG_INFO (ctx, "Starting warmup for UCX DTL"); @@ -468,12 +479,10 @@ static dyad_rc_t ucx_warmup (const dyad_ctx_t* ctx) DYAD_LOG_INFO (ctx, "Starting non-blocking recv for warmup"); recv_stat_ptr = ucx_recv_no_wait (ctx, true, &recv_buf, &recv_buf_len); DYAD_LOG_INFO (ctx, "Waiting on warmup recv to finish"); - recv_status = - dyad_ucx_request_wait (ctx, recv_stat_ptr); -#endif // DYAD_ENABLE_UCX_RMA + recv_status = dyad_ucx_request_wait (ctx, recv_stat_ptr); +#endif // DYAD_ENABLE_UCX_RMA DYAD_LOG_INFO (ctx, "Waiting on warmup send to finish"); - send_status = - dyad_ucx_request_wait (ctx, send_stat_ptr); + send_status = dyad_ucx_request_wait (ctx, send_stat_ptr); DYAD_LOG_INFO (ctx, "Disconnecting from self"); ucx_disconnect (ctx, ctx->dtl_handle->private_dtl.ucx_dtl_handle->ucx_worker, @@ -487,13 +496,13 @@ static dyad_rc_t ucx_warmup (const dyad_ctx_t* ctx) DYAD_LOG_INFO (ctx, "Communication succeeded (according to UCX)"); #ifndef DYAD_ENABLE_UCX_RMA assert (recv_buf_len == 1); -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA DYAD_LOG_INFO (ctx, "Correct amount of data received in warmup"); free (recv_buf); rc = DYAD_RC_OK; warmup_region_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } @@ -502,7 +511,7 @@ dyad_rc_t dyad_dtl_ucx_init (const dyad_ctx_t* ctx, dyad_dtl_comm_mode_t comm_mode, bool debug) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); ucp_params_t ucx_params; ucp_worker_params_t worker_params; ucp_config_t* config; @@ -513,13 +522,13 @@ dyad_rc_t dyad_dtl_ucx_init (const dyad_ctx_t* ctx, ctx->dtl_handle->private_dtl.ucx_dtl_handle = malloc (sizeof (struct dyad_dtl_ucx)); if (ctx->dtl_handle->private_dtl.ucx_dtl_handle == NULL) { DYAD_LOG_ERROR (ctx, "Could not allocate UCX DTL context\n"); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return DYAD_RC_SYSFAIL; } dtl_handle = ctx->dtl_handle->private_dtl.ucx_dtl_handle; // Allocation/Freeing of the Flux handle should be // handled by the DYAD context - dtl_handle->h = (flux_t*) ctx->h; + dtl_handle->h = (flux_t*)ctx->h; dtl_handle->comm_mode = comm_mode; dtl_handle->debug = debug; dtl_handle->ucx_ctx = NULL; @@ -552,8 +561,7 @@ dyad_rc_t dyad_dtl_ucx_init (const dyad_ctx_t* ctx, // * Remote Memory Access communication // * Auto initialization of request objects // * Worker sleep, wakeup, poll, etc. features - ucx_params.field_mask = - UCP_PARAM_FIELD_FEATURES | UCP_PARAM_FIELD_REQUEST_SIZE; + ucx_params.field_mask = UCP_PARAM_FIELD_FEATURES | UCP_PARAM_FIELD_REQUEST_SIZE; ucx_params.features = UCP_FEATURE_RMA | UCP_FEATURE_AMO32 | UCP_FEATURE_TAG; ucx_params.request_size = sizeof (struct ucx_request); ucx_params.request_init = dyad_ucx_request_init; @@ -585,9 +593,9 @@ dyad_rc_t dyad_dtl_ucx_init (const dyad_ctx_t* ctx, } // Query the worker for its address DYAD_LOG_INFO (ctx, "Get address of UCP worker\n"); - status = ucp_worker_get_address(dtl_handle->ucx_worker, - &(dtl_handle->local_address), - &(dtl_handle->local_addr_len)); + status = ucp_worker_get_address (dtl_handle->ucx_worker, + &(dtl_handle->local_address), + &(dtl_handle->local_addr_len)); // Initialize endpoint cache rc = dyad_ucx_ep_cache_init (ctx, &(dtl_handle->ep_cache)); @@ -617,7 +625,7 @@ dyad_rc_t dyad_dtl_ucx_init (const dyad_ctx_t* ctx, } dtl_handle->ep = NULL; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; @@ -625,7 +633,7 @@ error:; // If an error occured, finalize the DTL handle and // return a failing error code dyad_dtl_ucx_finalize (ctx); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return DYAD_RC_UCXINIT_FAIL; } dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, @@ -633,7 +641,7 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, uint32_t producer_rank, json_t** restrict packed_obj) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); DYAD_C_FUNCTION_UPDATE_INT ("producer_rank", producer_rank); DYAD_C_FUNCTION_UPDATE_INT ("pid", ctx->pid); @@ -643,7 +651,7 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, * as we expect a size_t lets put int -1 as a check. */ ssize_t temp = 0; - memcpy (dtl_handle->net_buf, &temp, sizeof(temp)); + memcpy (dtl_handle->net_buf, &temp, sizeof (temp)); dyad_rc_t rc = DYAD_RC_OK; size_t cons_enc_len = 0; char* cons_enc_buf = NULL; @@ -670,8 +678,8 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, cons_enc_size = base64_encode_using_maps (&base64_maps_rfc4648, cons_enc_buf, cons_enc_len + 1, - (const char*)dtl_handle->local_address, - dtl_handle->local_addr_len); + (const char*)dtl_handle->local_address, + dtl_handle->local_addr_len); if (cons_enc_size < 0) { DYAD_LOG_ERROR (ctx, "Unable to encode address\n"); // TODO log error @@ -699,8 +707,8 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, rkey_enc_size = base64_encode_using_maps (&base64_maps_rfc4648, rkey_enc_buf, rkey_enc_len + 1, - (const char*)dtl_handle->rkey_buf, - dtl_handle->rkey_size); + (const char*)dtl_handle->rkey_buf, + dtl_handle->rkey_size); if (rkey_enc_size < 0) { DYAD_LOG_ERROR (ctx, "Unable to encode rkey\n"); // TODO log error @@ -717,7 +725,7 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, // Instead, we use this function to create the tag that will be // used for the upcoming communication. tag_val = 0; - if (flux_get_rank (dtl_handle->h, (uint32_t *)&tag_val) < 0) { + if (flux_get_rank (dtl_handle->h, (uint32_t*)&tag_val) < 0) { DYAD_LOG_ERROR (ctx, "Cannot get consumer rank\n"); rc = DYAD_RC_FLUXFAIL; goto dtl_ucx_rpc_pack_region_finish; @@ -730,13 +738,13 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, // Use Jansson to pack the tag and UCX address into // the payload to be sent via RPC to the producer plugin DYAD_LOG_INFO (ctx, "Packing RPC payload for UCX DTL\n"); -#else // DYAD_ENABLE_UCX_RMA +#else // DYAD_ENABLE_UCX_RMA char* tag_name = "cons_buf"; uint64_t tag_val = dtl_handle->cons_buf_ptr; -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA char tag_val_buf[128]; - memset(tag_val_buf, 0x00, 128); - sprintf( tag_val_buf, "%" PRIu64, tag_val); + memset (tag_val_buf, 0x00, 128); + sprintf (tag_val_buf, "%" PRIu64, tag_val); DYAD_LOG_INFO (ctx, "Creating Json object %lu with buf %s", tag_val, tag_val_buf); *packed_obj = json_pack ("{s:s, s:i, s:s, s:i, s:s%, s:s%}", "upath", @@ -751,8 +759,8 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, cons_enc_buf, cons_enc_len, "rkey", - rkey_enc_buf, - rkey_enc_len); + rkey_enc_buf, + rkey_enc_len); free (cons_enc_buf); free (rkey_enc_buf); // If the packing failed, log an error @@ -763,13 +771,13 @@ dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, } rc = DYAD_RC_OK; dtl_ucx_rpc_pack_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg, char** upath) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; char* enc_addr = NULL; size_t enc_addr_len = 0; @@ -785,9 +793,9 @@ dyad_rc_t dyad_dtl_ucx_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg, uint64_t tag_val; #ifndef DYAD_ENABLE_UCX_RMA char* tag_name = "tag_cons"; -#else // DYAD_ENABLE_UCX_RMA +#else // DYAD_ENABLE_UCX_RMA char* tag_name = "cons_buf"; -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA char* tag_value_str = NULL; errcode = flux_request_unpack (msg, NULL, @@ -799,7 +807,7 @@ dyad_rc_t dyad_dtl_ucx_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg, tag_name, &tag_value_str, "pid_cons", - &pid, + &pid, "addr", &enc_addr, &enc_addr_len, @@ -815,9 +823,9 @@ dyad_rc_t dyad_dtl_ucx_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg, } #ifndef DYAD_ENABLE_UCX_RMA tag_cons = tag_val; -#else // DYAD_ENABLE_UCX_RMA +#else // DYAD_ENABLE_UCX_RMA dtl_handle->cons_buf_ptr = tag_val; -#endif // DYAD_ENABLE_UCX_RMA +#endif // DYAD_ENABLE_UCX_RMA DYAD_C_FUNCTION_UPDATE_INT ("pid", pid); DYAD_C_FUNCTION_UPDATE_INT ("tag_cons", tag_cons); dtl_handle->comm_tag = tag_prod << 32 | tag_cons; @@ -870,27 +878,27 @@ dyad_rc_t dyad_dtl_ucx_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg, } rc = DYAD_RC_OK; dtl_ucx_rpc_unpack_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_rpc_respond (const dyad_ctx_t* ctx, const flux_msg_t* orig_msg) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; } dyad_rc_t dyad_dtl_ucx_rpc_recv_response (const dyad_ctx_t* ctx, flux_future_t* f) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; } dyad_rc_t dyad_dtl_ucx_get_buffer (const dyad_ctx_t* ctx, size_t data_size, void** data_buf) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; dyad_dtl_ucx_t* dtl_handle = ctx->dtl_handle->private_dtl.ucx_dtl_handle; DYAD_LOG_INFO (dtl_handle, "Validating data_buf in get_buffer"); @@ -907,7 +915,7 @@ dyad_rc_t dyad_dtl_ucx_get_buffer (const dyad_ctx_t* ctx, size_t data_size, void // } DYAD_LOG_INFO (dtl_handle, "Validating data_size in get_buffer"); if (data_size > ctx->dtl_handle->private_dtl.ucx_dtl_handle->max_transfer_size) { - DYAD_LOG_ERROR (dtl_handle, \ + DYAD_LOG_ERROR (dtl_handle, "Requested a data size that's larger than the pre-allocated UCX buffer"); rc = DYAD_RC_BADBUF; goto ucx_get_buffer_done; @@ -917,13 +925,13 @@ dyad_rc_t dyad_dtl_ucx_get_buffer (const dyad_ctx_t* ctx, size_t data_size, void rc = DYAD_RC_OK; ucx_get_buffer_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_return_buffer (const dyad_ctx_t* ctx, void** data_buf) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; if (data_buf == NULL || *data_buf == NULL) { rc = DYAD_RC_BADBUF; @@ -931,19 +939,20 @@ dyad_rc_t dyad_dtl_ucx_return_buffer (const dyad_ctx_t* ctx, void** data_buf) } *data_buf = NULL; dtl_ucx_return_buffer_done:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_establish_connection (const dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; dyad_dtl_ucx_t* dtl_handle = ctx->dtl_handle->private_dtl.ucx_dtl_handle; dyad_dtl_comm_mode_t comm_mode = dtl_handle->comm_mode; if (comm_mode == DYAD_COMM_SEND) { DYAD_LOG_INFO (ctx, "Create UCP endpoint for communication with consumer\n"); - rc = dyad_ucx_ep_cache_find (ctx, dtl_handle->ep_cache, + rc = dyad_ucx_ep_cache_find (ctx, + dtl_handle->ep_cache, dtl_handle->remote_address, dtl_handle->remote_addr_len, &(dtl_handle->ep)); @@ -972,13 +981,13 @@ dyad_rc_t dyad_dtl_ucx_establish_connection (const dyad_ctx_t* ctx) rc = DYAD_RC_BAD_COMM_MODE; } dtl_ucx_establish_connection_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_send (const dyad_ctx_t* ctx, void* buf, size_t buflen) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; ucs_status_ptr_t stat_ptr; ucs_status_t status = UCS_OK; @@ -986,24 +995,22 @@ dyad_rc_t dyad_dtl_ucx_send (const dyad_ctx_t* ctx, void* buf, size_t buflen) DYAD_LOG_INFO (ctx, "Processing UCP send request\n"); status = dyad_ucx_request_wait (ctx, stat_ptr); if (status != UCS_OK) { - DYAD_LOG_ERROR (ctx, "UCP Put failed (status = %d)!\n", - (int)status); + DYAD_LOG_ERROR (ctx, "UCP Put failed (status = %d)!\n", (int)status); rc = DYAD_RC_UCXCOMM_FAIL; goto dtl_ucx_send_region_finish; } DYAD_LOG_INFO (ctx, "Data send with UCP succeeded\n"); rc = DYAD_RC_OK; dtl_ucx_send_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_recv (const dyad_ctx_t* ctx, void** buf, size_t* buflen) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; - ucs_status_ptr_t stat_ptr = NULL; // Wait on the recv operation to complete stat_ptr = ucx_recv_no_wait (ctx, false, buf, buflen); @@ -1018,9 +1025,9 @@ dyad_rc_t dyad_dtl_ucx_recv (const dyad_ctx_t* ctx, void** buf, size_t* buflen) rc = DYAD_RC_UCXCOMM_FAIL; goto dtl_ucx_recv_region_finish; } -#else // DYAD_ENABLE_UCX_RMA - (void)stat_ptr; -#endif // DYAD_ENABLE_UCX_RMA +#else // DYAD_ENABLE_UCX_RMA + (void)stat_ptr; +#endif // DYAD_ENABLE_UCX_RMA DYAD_LOG_INFO (ctx, "Data receive using UCX is successful\n"); DYAD_LOG_INFO (ctx, "Received %lu bytes from producer\n", *buflen); @@ -1028,14 +1035,14 @@ dyad_rc_t dyad_dtl_ucx_recv (const dyad_ctx_t* ctx, void** buf, size_t* buflen) rc = DYAD_RC_OK; #ifndef DYAD_ENABLE_UCX_RMA dtl_ucx_recv_region_finish:; -#endif // DYAD_ENABLE_UCX_RMA - DYAD_C_FUNCTION_END(); +#endif // DYAD_ENABLE_UCX_RMA + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_close_connection (const dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_rc_t rc = DYAD_RC_OK; dyad_dtl_ucx_t* dtl_handle = ctx->dtl_handle->private_dtl.ucx_dtl_handle; dyad_dtl_comm_mode_t comm_mode = dtl_handle->comm_mode; @@ -1049,8 +1056,8 @@ dyad_rc_t dyad_dtl_ucx_close_connection (const dyad_ctx_t* ctx) // "released."); // } #ifdef DYAD_ENABLE_UCX_RMA - ucp_rkey_destroy(dtl_handle->rkey); -#endif // DYAD_ENABLE_UCX_RMA + ucp_rkey_destroy (dtl_handle->rkey); +#endif // DYAD_ENABLE_UCX_RMA dtl_handle->ep = NULL; // Sender doesn't have a consumer address at this time // So, free the consumer address when closing the connection @@ -1078,13 +1085,13 @@ dyad_rc_t dyad_dtl_ucx_close_connection (const dyad_ctx_t* ctx) // TODO create new RC for this case rc = DYAD_RC_BAD_COMM_MODE; } - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } dyad_rc_t dyad_dtl_ucx_finalize (const dyad_ctx_t* ctx) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); dyad_dtl_ucx_t* dtl_handle = NULL; dyad_rc_t rc = DYAD_RC_OK; if (ctx->dtl_handle == NULL || ctx->dtl_handle->private_dtl.ucx_dtl_handle == NULL) { @@ -1108,10 +1115,7 @@ dyad_rc_t dyad_dtl_ucx_finalize (const dyad_ctx_t* ctx) } // Free memory buffer if not already freed if (dtl_handle->mem_handle != NULL) { - ucx_free_buffer (ctx, - dtl_handle->ucx_ctx, - dtl_handle->mem_handle, - &(dtl_handle->net_buf)); + ucx_free_buffer (ctx, dtl_handle->ucx_ctx, dtl_handle->mem_handle, &(dtl_handle->net_buf)); } // Release worker if not already released if (dtl_handle->ucx_worker != NULL) { @@ -1131,6 +1135,6 @@ dyad_rc_t dyad_dtl_ucx_finalize (const dyad_ctx_t* ctx) ctx->dtl_handle->private_dtl.ucx_dtl_handle = NULL; rc = DYAD_RC_OK; dtl_ucx_finalize_region_finish:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } diff --git a/src/dyad/dtl/ucx_dtl.h b/src/dyad/dtl/ucx_dtl.h index f8af6e8f..1572dc9c 100644 --- a/src/dyad/dtl/ucx_dtl.h +++ b/src/dyad/dtl/ucx_dtl.h @@ -7,10 +7,11 @@ #error "no config" #endif +#include +#include + #include #include -#include -#include struct dyad_dtl_ucx { flux_t* h; @@ -34,7 +35,7 @@ struct dyad_dtl_ucx { size_t rkey_size; uint64_t cons_buf_ptr; // Internal for Sender - ucp_rkey_h rkey; + ucp_rkey_h rkey; }; typedef struct dyad_dtl_ucx dyad_dtl_ucx_t; @@ -45,9 +46,9 @@ dyad_rc_t dyad_dtl_ucx_init (const dyad_ctx_t* ctx, bool debug); dyad_rc_t dyad_dtl_ucx_rpc_pack (const dyad_ctx_t* ctx, - const char* upath, + const char* upath, uint32_t producer_rank, - json_t** packed_obj); + json_t** packed_obj); dyad_rc_t dyad_dtl_ucx_rpc_unpack (const dyad_ctx_t* ctx, const flux_msg_t* msg, char** upath); diff --git a/src/dyad/dtl/ucx_ep_cache.cpp b/src/dyad/dtl/ucx_ep_cache.cpp index 8493793b..bfc3d61e 100644 --- a/src/dyad/dtl/ucx_ep_cache.cpp +++ b/src/dyad/dtl/ucx_ep_cache.cpp @@ -16,224 +16,236 @@ #include #include +#include +#include +#include +#include + using key_type = uint64_t; using cache_type = std::unordered_map; -static void __attribute__((unused)) -dyad_ucx_ep_err_handler(void *arg, ucp_ep_h ep, ucs_status_t status) { - DYAD_C_FUNCTION_START(); - dyad_ctx_t __attribute__((unused)) *ctx = (dyad_ctx_t *)arg; - DYAD_LOG_ERROR(ctx, "An error occured on the UCP endpoint (status = %d)", - status); - DYAD_C_FUNCTION_END(); +static void __attribute__ ((unused)) +dyad_ucx_ep_err_handler (void *arg, ucp_ep_h ep, ucs_status_t status) +{ + DYAD_C_FUNCTION_START (); + dyad_ctx_t __attribute__ ((unused)) *ctx = (dyad_ctx_t *)arg; + DYAD_LOG_ERROR (ctx, "An error occured on the UCP endpoint (status = %d)", status); + DYAD_C_FUNCTION_END (); } -dyad_rc_t ucx_connect(const dyad_ctx_t *ctx, ucp_worker_h worker, - const ucp_address_t *addr, ucp_ep_h *ep) { - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - ucp_ep_params_t params; - ucs_status_t status = UCS_OK; - params.field_mask = UCP_EP_PARAM_FIELD_REMOTE_ADDRESS; - params.address = addr; - status = ucp_ep_create(worker, ¶ms, ep); - if (UCX_STATUS_FAIL(status)) { - DYAD_LOG_ERROR(ctx, "ucp_ep_create failed with status %d", (int)status); - rc = DYAD_RC_UCXCOMM_FAIL; - goto ucx_connect_done; - } - if (*ep == NULL) { - DYAD_LOG_ERROR(ctx, - "ucp_ep_create succeeded, but returned a NULL endpoint"); - rc = DYAD_RC_UCXCOMM_FAIL; - goto ucx_connect_done; - } +dyad_rc_t ucx_connect (const dyad_ctx_t *ctx, + ucp_worker_h worker, + const ucp_address_t *addr, + ucp_ep_h *ep) +{ + DYAD_C_FUNCTION_START (); + dyad_rc_t rc = DYAD_RC_OK; + ucp_ep_params_t params; + ucs_status_t status = UCS_OK; + params.field_mask = UCP_EP_PARAM_FIELD_REMOTE_ADDRESS; + params.address = addr; + status = ucp_ep_create (worker, ¶ms, ep); + if (UCX_STATUS_FAIL (status)) { + DYAD_LOG_ERROR (ctx, "ucp_ep_create failed with status %d", (int)status); + rc = DYAD_RC_UCXCOMM_FAIL; + goto ucx_connect_done; + } + if (*ep == NULL) { + DYAD_LOG_ERROR (ctx, "ucp_ep_create succeeded, but returned a NULL endpoint"); + rc = DYAD_RC_UCXCOMM_FAIL; + goto ucx_connect_done; + } ucx_connect_done:; - DYAD_C_FUNCTION_END(); - return rc; + DYAD_C_FUNCTION_END (); + return rc; } -dyad_rc_t ucx_disconnect(const dyad_ctx_t *ctx, ucp_worker_h worker, - ucp_ep_h ep) { - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - ucs_status_t status = UCS_OK; - ucs_status_ptr_t stat_ptr; - if (ep != NULL) { - // ucp_tag_send_sync_nbx is the prefered version of this send - // since UCX 1.9 However, some systems (e.g., Lassen) may have - // an older verison This conditional compilation will use - // ucp_tag_send_sync_nbx if using UCX 1.9+, and it will use the - // deprecated ucp_tag_send_sync_nb if using UCX < 1.9. +dyad_rc_t ucx_disconnect (const dyad_ctx_t *ctx, ucp_worker_h worker, ucp_ep_h ep) +{ + DYAD_C_FUNCTION_START (); + dyad_rc_t rc = DYAD_RC_OK; + ucs_status_t status = UCS_OK; + ucs_status_ptr_t stat_ptr; + if (ep != NULL) { + // ucp_tag_send_sync_nbx is the prefered version of this send + // since UCX 1.9 However, some systems (e.g., Lassen) may have + // an older verison This conditional compilation will use + // ucp_tag_send_sync_nbx if using UCX 1.9+, and it will use the + // deprecated ucp_tag_send_sync_nb if using UCX < 1.9. #if UCP_API_VERSION >= UCP_VERSION(1, 10) - ucp_request_param_t close_params; - close_params.op_attr_mask = UCP_OP_ATTR_FIELD_FLAGS; - close_params.flags = UCP_EP_CLOSE_FLAG_FORCE; - stat_ptr = ucp_ep_close_nbx(ep, &close_params); + ucp_request_param_t close_params; + close_params.op_attr_mask = UCP_OP_ATTR_FIELD_FLAGS; + close_params.flags = UCP_EP_CLOSE_FLAG_FORCE; + stat_ptr = ucp_ep_close_nbx (ep, &close_params); #else - // TODO change to FORCE if we decide to enable err handleing - // mode - stat_ptr = ucp_ep_close_nb(ep, UCP_EP_CLOSE_MODE_FORCE); + // TODO change to FORCE if we decide to enable err handleing + // mode + stat_ptr = ucp_ep_close_nb (ep, UCP_EP_CLOSE_MODE_FORCE); #endif - // Don't use dyad_ucx_request_wait here because ep_close behaves - // differently than other UCX calls - if (stat_ptr != NULL) { - // Endpoint close is in-progress. - // Wait until finished - if (UCS_PTR_IS_PTR(stat_ptr)) { - do { - ucp_worker_progress(worker); - status = ucp_request_check_status(stat_ptr); - } while (status == UCS_INPROGRESS); - ucp_request_free(stat_ptr); - } - // An error occurred during endpoint closure - // However, the endpoint can no longer be used - // Get the status code for reporting - else { - status = UCS_PTR_STATUS(stat_ptr); - } - if (UCX_STATUS_FAIL(status)) { - rc = DYAD_RC_UCXEP_FAIL; - goto ucx_disconnect_region_finish; - } + // Don't use dyad_ucx_request_wait here because ep_close behaves + // differently than other UCX calls + if (stat_ptr != NULL) { + // Endpoint close is in-progress. + // Wait until finished + if (UCS_PTR_IS_PTR (stat_ptr)) { + do { + ucp_worker_progress (worker); + status = ucp_request_check_status (stat_ptr); + } while (status == UCS_INPROGRESS); + ucp_request_free (stat_ptr); + } + // An error occurred during endpoint closure + // However, the endpoint can no longer be used + // Get the status code for reporting + else { + status = UCS_PTR_STATUS (stat_ptr); + } + if (UCX_STATUS_FAIL (status)) { + rc = DYAD_RC_UCXEP_FAIL; + goto ucx_disconnect_region_finish; + } + } } - } ucx_disconnect_region_finish: - DYAD_C_FUNCTION_END(); - return rc; + DYAD_C_FUNCTION_END (); + return rc; } -dyad_rc_t dyad_ucx_ep_cache_init(const dyad_ctx_t *ctx, ucx_ep_cache_h *cache) { - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - if (cache == nullptr || *cache != nullptr) { - rc = DYAD_RC_BADBUF; - goto ucx_ep_cache_init_done; - } - *cache = reinterpret_cast(new (std::nothrow) cache_type()); - if (*cache == nullptr) { - rc = DYAD_RC_SYSFAIL; - goto ucx_ep_cache_init_done; - } +dyad_rc_t dyad_ucx_ep_cache_init (const dyad_ctx_t *ctx, ucx_ep_cache_h *cache) +{ + DYAD_C_FUNCTION_START (); + dyad_rc_t rc = DYAD_RC_OK; + if (cache == nullptr || *cache != nullptr) { + rc = DYAD_RC_BADBUF; + goto ucx_ep_cache_init_done; + } + *cache = reinterpret_cast (new (std::nothrow) cache_type ()); + if (*cache == nullptr) { + rc = DYAD_RC_SYSFAIL; + goto ucx_ep_cache_init_done; + } ucx_ep_cache_init_done:; - DYAD_C_FUNCTION_END(); - return rc; + DYAD_C_FUNCTION_END (); + return rc; } -dyad_rc_t dyad_ucx_ep_cache_find(const dyad_ctx_t *ctx, - const ucx_ep_cache_h cache, - const ucp_address_t *addr, - const size_t addr_size, ucp_ep_h *ep) { - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - if (ep == nullptr || *ep != nullptr) { - rc = DYAD_RC_BADBUF; - goto ucx_ep_cache_find_done; - } - try { - const auto *cpp_cache = reinterpret_cast(cache); - auto key = ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key; - auto cache_it = cpp_cache->find(key); - if (cache_it == cpp_cache->cend()) { - *ep = nullptr; - rc = DYAD_RC_NOTFOUND; - } else { - *ep = cache_it->second; - rc = DYAD_RC_OK; +dyad_rc_t dyad_ucx_ep_cache_find (const dyad_ctx_t *ctx, + const ucx_ep_cache_h cache, + const ucp_address_t *addr, + const size_t addr_size, + ucp_ep_h *ep) +{ + DYAD_C_FUNCTION_START (); + dyad_rc_t rc = DYAD_RC_OK; + if (ep == nullptr || *ep != nullptr) { + rc = DYAD_RC_BADBUF; + goto ucx_ep_cache_find_done; + } + try { + const auto *cpp_cache = reinterpret_cast (cache); + auto key = ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key; + auto cache_it = cpp_cache->find (key); + if (cache_it == cpp_cache->cend ()) { + *ep = nullptr; + rc = DYAD_RC_NOTFOUND; + } else { + *ep = cache_it->second; + rc = DYAD_RC_OK; + } + } catch (...) { + *ep = nullptr; + rc = DYAD_RC_SYSFAIL; } - } catch (...) { - *ep = nullptr; - rc = DYAD_RC_SYSFAIL; - } ucx_ep_cache_find_done:; - DYAD_C_FUNCTION_END(); - return rc; + DYAD_C_FUNCTION_END (); + return rc; } -dyad_rc_t dyad_ucx_ep_cache_insert(const dyad_ctx_t *ctx, ucx_ep_cache_h cache, - const ucp_address_t *addr, - const size_t addr_size, - ucp_worker_h worker) { - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - try { - cache_type *cpp_cache = reinterpret_cast(cache); - uint64_t key = - ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key; - DYAD_C_FUNCTION_UPDATE_INT( - "cons_key", - ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key) - auto cache_it = cpp_cache->find(key); - if (cache_it != cpp_cache->end()) { - rc = DYAD_RC_OK; - } else { - DYAD_LOG_INFO(ctx, "No cache entry found. Creating new connection"); - rc = ucx_connect(ctx, worker, addr, - &ctx->dtl_handle->private_dtl.ucx_dtl_handle->ep); - if (!DYAD_IS_ERROR(rc)) { - cpp_cache->insert_or_assign( - key, ctx->dtl_handle->private_dtl.ucx_dtl_handle->ep); - rc = DYAD_RC_OK; - } +dyad_rc_t dyad_ucx_ep_cache_insert (const dyad_ctx_t *ctx, + ucx_ep_cache_h cache, + const ucp_address_t *addr, + const size_t addr_size, + ucp_worker_h worker) +{ + DYAD_C_FUNCTION_START (); + dyad_rc_t rc = DYAD_RC_OK; + try { + cache_type *cpp_cache = reinterpret_cast (cache); + uint64_t key = ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key; + DYAD_C_FUNCTION_UPDATE_INT ("cons_key", + ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key) + auto cache_it = cpp_cache->find (key); + if (cache_it != cpp_cache->end ()) { + rc = DYAD_RC_OK; + } else { + DYAD_LOG_INFO (ctx, "No cache entry found. Creating new connection"); + rc = ucx_connect (ctx, worker, addr, &ctx->dtl_handle->private_dtl.ucx_dtl_handle->ep); + if (!DYAD_IS_ERROR (rc)) { + cpp_cache->insert_or_assign (key, ctx->dtl_handle->private_dtl.ucx_dtl_handle->ep); + rc = DYAD_RC_OK; + } + } + } catch (...) { + rc = DYAD_RC_SYSFAIL; } - } catch (...) { - rc = DYAD_RC_SYSFAIL; - } - DYAD_C_FUNCTION_END(); - return rc; + DYAD_C_FUNCTION_END (); + return rc; } -static inline cache_type::iterator cache_remove_impl(const dyad_ctx_t *ctx, - cache_type *cache, - cache_type::iterator it, - ucp_worker_h worker) { - DYAD_C_FUNCTION_START(); - if (it != cache->end()) { - ucx_disconnect(ctx, worker, it->second); - // The UCP address was allocated with 'malloc' while unpacking - // the RPC message. So, we extract it from the key and free - // it after erasing the iterator - auto next_it = cache->erase(it); - DYAD_C_FUNCTION_END(); - return next_it; - } - DYAD_C_FUNCTION_END(); - return cache->end(); +static inline cache_type::iterator cache_remove_impl (const dyad_ctx_t *ctx, + cache_type *cache, + cache_type::iterator it, + ucp_worker_h worker) +{ + DYAD_C_FUNCTION_START (); + if (it != cache->end ()) { + ucx_disconnect (ctx, worker, it->second); + // The UCP address was allocated with 'malloc' while unpacking + // the RPC message. So, we extract it from the key and free + // it after erasing the iterator + auto next_it = cache->erase (it); + DYAD_C_FUNCTION_END (); + return next_it; + } + DYAD_C_FUNCTION_END (); + return cache->end (); } -dyad_rc_t dyad_ucx_ep_cache_remove(const dyad_ctx_t *ctx, ucx_ep_cache_h cache, - const ucp_address_t *addr, - const size_t addr_size, - ucp_worker_h worker) { - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - try { - cache_type *cpp_cache = reinterpret_cast(cache); - auto key = ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key; - cache_type::iterator cache_it = cpp_cache->find(key); - cache_remove_impl(ctx, cpp_cache, cache_it, worker); - rc = DYAD_RC_OK; - } catch (...) { - rc = DYAD_RC_SYSFAIL; - } - DYAD_C_FUNCTION_END(); - return rc; +dyad_rc_t dyad_ucx_ep_cache_remove (const dyad_ctx_t *ctx, + ucx_ep_cache_h cache, + const ucp_address_t *addr, + const size_t addr_size, + ucp_worker_h worker) +{ + DYAD_C_FUNCTION_START (); + dyad_rc_t rc = DYAD_RC_OK; + try { + cache_type *cpp_cache = reinterpret_cast (cache); + auto key = ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key; + cache_type::iterator cache_it = cpp_cache->find (key); + cache_remove_impl (ctx, cpp_cache, cache_it, worker); + rc = DYAD_RC_OK; + } catch (...) { + rc = DYAD_RC_SYSFAIL; + } + DYAD_C_FUNCTION_END (); + return rc; } -dyad_rc_t dyad_ucx_ep_cache_finalize(const dyad_ctx_t *ctx, - ucx_ep_cache_h *cache, - ucp_worker_h worker) { - DYAD_C_FUNCTION_START(); - if (cache == nullptr || *cache == nullptr) { +dyad_rc_t dyad_ucx_ep_cache_finalize (const dyad_ctx_t *ctx, + ucx_ep_cache_h *cache, + ucp_worker_h worker) +{ + DYAD_C_FUNCTION_START (); + if (cache == nullptr || *cache == nullptr) { + return DYAD_RC_OK; + } + cache_type *cpp_cache = reinterpret_cast (*cache); + for (cache_type::iterator it = cpp_cache->begin (); it != cpp_cache->end ();) { + it = cache_remove_impl (ctx, cpp_cache, it, worker); + } + delete cpp_cache; + *cache = nullptr; + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; - } - cache_type *cpp_cache = reinterpret_cast(*cache); - for (cache_type::iterator it = cpp_cache->begin(); it != cpp_cache->end();) { - it = cache_remove_impl(ctx, cpp_cache, it, worker); - } - delete cpp_cache; - *cache = nullptr; - DYAD_C_FUNCTION_END(); - return DYAD_RC_OK; } diff --git a/src/dyad/dtl/ucx_ep_cache.h b/src/dyad/dtl/ucx_ep_cache.h index 5ada63b8..72e6811e 100644 --- a/src/dyad/dtl/ucx_ep_cache.h +++ b/src/dyad/dtl/ucx_ep_cache.h @@ -9,6 +9,8 @@ // clang-format off #include +#include + #include #include #include @@ -23,32 +25,38 @@ extern "C" { // of UCX operations #define UCX_STATUS_FAIL(status) (status != UCS_OK) -dyad_rc_t ucx_connect(const dyad_ctx_t *ctx, ucp_worker_h worker, - const ucp_address_t *addr, ucp_ep_h *ep); +dyad_rc_t ucx_connect (const dyad_ctx_t *ctx, + ucp_worker_h worker, + const ucp_address_t *addr, + ucp_ep_h *ep); -dyad_rc_t ucx_disconnect(const dyad_ctx_t *ctx, ucp_worker_h worker, - ucp_ep_h ep); +dyad_rc_t ucx_disconnect (const dyad_ctx_t *ctx, ucp_worker_h worker, ucp_ep_h ep); // NOTE: in future, add option to configure replacement strategy -dyad_rc_t dyad_ucx_ep_cache_init(const dyad_ctx_t *ctx, ucx_ep_cache_h *cache); +dyad_rc_t dyad_ucx_ep_cache_init (const dyad_ctx_t *ctx, ucx_ep_cache_h *cache); // NOTE: not positive if UCP addresses are 100% unique by worker -dyad_rc_t dyad_ucx_ep_cache_find(const dyad_ctx_t *ctx, - const ucx_ep_cache_h cache, - const ucp_address_t *addr, - const size_t addr_size, ucp_ep_h *ep); - -dyad_rc_t dyad_ucx_ep_cache_insert(const dyad_ctx_t *ctx, ucx_ep_cache_h cache, - const ucp_address_t *addr, - const size_t addr_size, ucp_worker_h worker); - -dyad_rc_t dyad_ucx_ep_cache_remove(const dyad_ctx_t *ctx, ucx_ep_cache_h cache, - const ucp_address_t *addr, - const size_t addr_size, ucp_worker_h worker); - -dyad_rc_t dyad_ucx_ep_cache_finalize(const dyad_ctx_t *ctx, - ucx_ep_cache_h *cache, - ucp_worker_h worker); +dyad_rc_t dyad_ucx_ep_cache_find (const dyad_ctx_t *ctx, + const ucx_ep_cache_h cache, + const ucp_address_t *addr, + const size_t addr_size, + ucp_ep_h *ep); + +dyad_rc_t dyad_ucx_ep_cache_insert (const dyad_ctx_t *ctx, + ucx_ep_cache_h cache, + const ucp_address_t *addr, + const size_t addr_size, + ucp_worker_h worker); + +dyad_rc_t dyad_ucx_ep_cache_remove (const dyad_ctx_t *ctx, + ucx_ep_cache_h cache, + const ucp_address_t *addr, + const size_t addr_size, + ucp_worker_h worker); + +dyad_rc_t dyad_ucx_ep_cache_finalize (const dyad_ctx_t *ctx, + ucx_ep_cache_h *cache, + ucp_worker_h worker); #ifdef __cplusplus } diff --git a/src/dyad/modules/dyad.c b/src/dyad/modules/dyad.c index 1e922cab..ecb80eb6 100644 --- a/src/dyad/modules/dyad.c +++ b/src/dyad/modules/dyad.c @@ -43,7 +43,7 @@ #include #include #include -#endif // defined(__cplusplus) +#endif // defined(__cplusplus) #include #include @@ -51,482 +51,473 @@ #include #include -#define TIME_DIFF(Tstart, Tend) \ - ((double)(1000000000L * ((Tend).tv_sec - (Tstart).tv_sec) + (Tend).tv_nsec - \ - (Tstart).tv_nsec) / \ - 1000000000L) +#define TIME_DIFF(Tstart, Tend) \ + ((double)(1000000000L * ((Tend).tv_sec - (Tstart).tv_sec) + (Tend).tv_nsec - (Tstart).tv_nsec) \ + / 1000000000L) struct dyad_mod_ctx { - flux_msg_handler_t **handlers; - dyad_ctx_t *ctx; + flux_msg_handler_t **handlers; + dyad_ctx_t *ctx; }; const struct dyad_mod_ctx dyad_mod_ctx_default = {NULL, NULL}; typedef struct dyad_mod_ctx dyad_mod_ctx_t; -static void dyad_mod_fini(void) __attribute__((destructor)); +static void dyad_mod_fini (void) __attribute__ ((destructor)); -void dyad_mod_fini(void) { - flux_t *h = flux_open(NULL, 0); +void dyad_mod_fini (void) +{ + flux_t *h = flux_open (NULL, 0); - if (h != NULL) { - } + if (h != NULL) { + } #ifdef DYAD_PROFILER_DFTRACER - DFTRACER_C_FINI(); + DFTRACER_C_FINI (); #endif } -static void freectx(void *arg) { - dyad_mod_ctx_t *mod_ctx = (dyad_mod_ctx_t *)arg; - flux_msg_handler_delvec(mod_ctx->handlers); - if (mod_ctx->ctx) { - dyad_ctx_fini(); - mod_ctx->ctx = NULL; - } - free(mod_ctx); -} - -static dyad_mod_ctx_t *get_mod_ctx(flux_t *h) { - dyad_mod_ctx_t *mod_ctx = (dyad_mod_ctx_t *)flux_aux_get(h, "dyad"); - - if (!mod_ctx) { - mod_ctx = (dyad_mod_ctx_t *)malloc(sizeof(*mod_ctx)); - if (mod_ctx == NULL) { - DYAD_LOG_STDERR("DYAD_MOD: could not allocate memory for context"); - goto getctx_error; +static void freectx (void *arg) +{ + dyad_mod_ctx_t *mod_ctx = (dyad_mod_ctx_t *)arg; + flux_msg_handler_delvec (mod_ctx->handlers); + if (mod_ctx->ctx) { + dyad_ctx_fini (); + mod_ctx->ctx = NULL; } - mod_ctx->handlers = NULL; - mod_ctx->ctx = NULL; + free (mod_ctx); +} - if (flux_aux_set(h, "dyad", mod_ctx, freectx) < 0) { - DYAD_LOG_STDERR("DYAD_MOD: flux_aux_set() failed!"); - goto getctx_error; +static dyad_mod_ctx_t *get_mod_ctx (flux_t *h) +{ + dyad_mod_ctx_t *mod_ctx = (dyad_mod_ctx_t *)flux_aux_get (h, "dyad"); + + if (!mod_ctx) { + mod_ctx = (dyad_mod_ctx_t *)malloc (sizeof (*mod_ctx)); + if (mod_ctx == NULL) { + DYAD_LOG_STDERR ("DYAD_MOD: could not allocate memory for context"); + goto getctx_error; + } + mod_ctx->handlers = NULL; + mod_ctx->ctx = NULL; + + if (flux_aux_set (h, "dyad", mod_ctx, freectx) < 0) { + DYAD_LOG_STDERR ("DYAD_MOD: flux_aux_set() failed!"); + goto getctx_error; + } } - } - goto getctx_done; + goto getctx_done; getctx_error:; - return NULL; + return NULL; getctx_done: - return mod_ctx; + return mod_ctx; } /* request callback called when dyad.fetch request is invoked */ #if DYAD_PERFFLOW -__attribute__((annotate("@critical_path()"))) +__attribute__ ((annotate ("@critical_path()"))) #endif static void -dyad_fetch_request_cb(flux_t *h, flux_msg_handler_t *w, const flux_msg_t *msg, - void *arg) { - DYAD_C_FUNCTION_START(); - dyad_mod_ctx_t *mod_ctx = get_mod_ctx(h); - DYAD_LOG_INFO(mod_ctx->ctx, "Launched callback for %s", DYAD_DTL_RPC_NAME); - ssize_t inlen = 0l; - char *inbuf = NULL; - int fd = -1; - uint32_t userid = 0u; - char *upath = NULL; - char fullpath[PATH_MAX + 1] = {'\0'}; - int saved_errno = errno; - ssize_t file_size = 0l; - dyad_rc_t rc = 0; - struct flock shared_lock; - if (!flux_msg_is_streaming(msg)) { - errno = EPROTO; - goto fetch_error_wo_flock; - } - - if (flux_msg_get_userid(msg, &userid) < 0) - goto fetch_error_wo_flock; - - DYAD_LOG_INFO(mod_ctx->ctx, "DYAD_MOD: unpacking RPC message"); - - rc = mod_ctx->ctx->dtl_handle->rpc_unpack(mod_ctx->ctx, msg, &upath); - - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(mod_ctx->ctx, "Could not unpack message from client"); - errno = EPROTO; - goto fetch_error_wo_flock; - } - DYAD_C_FUNCTION_UPDATE_STR("upath", upath); - DYAD_LOG_DEBUG(mod_ctx->ctx, "DYAD_MOD: requested user_path: %s", upath); - DYAD_LOG_DEBUG(mod_ctx->ctx, - "DYAD_MOD: sending initial response to consumer"); - - rc = mod_ctx->ctx->dtl_handle->rpc_respond(mod_ctx->ctx, msg); - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(mod_ctx->ctx, - "Could not send primary RPC response to client"); - goto fetch_error_wo_flock; - } - - strncpy(fullpath, mod_ctx->ctx->prod_managed_path, PATH_MAX - 1); - concat_str(fullpath, upath, "/", PATH_MAX); - DYAD_C_FUNCTION_UPDATE_STR("fullpath", fullpath); +dyad_fetch_request_cb (flux_t *h, flux_msg_handler_t *w, const flux_msg_t *msg, void *arg) +{ + DYAD_C_FUNCTION_START (); + dyad_mod_ctx_t *mod_ctx = get_mod_ctx (h); + DYAD_LOG_INFO (mod_ctx->ctx, "Launched callback for %s", DYAD_DTL_RPC_NAME); + ssize_t inlen = 0l; + char *inbuf = NULL; + int fd = -1; + uint32_t userid = 0u; + char *upath = NULL; + char fullpath[PATH_MAX + 1] = {'\0'}; + int saved_errno = errno; + ssize_t file_size = 0l; + dyad_rc_t rc = 0; + struct flock shared_lock; + if (!flux_msg_is_streaming (msg)) { + errno = EPROTO; + goto fetch_error_wo_flock; + } + + if (flux_msg_get_userid (msg, &userid) < 0) + goto fetch_error_wo_flock; + + DYAD_LOG_INFO (mod_ctx->ctx, "DYAD_MOD: unpacking RPC message"); + + rc = mod_ctx->ctx->dtl_handle->rpc_unpack (mod_ctx->ctx, msg, &upath); + + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (mod_ctx->ctx, "Could not unpack message from client"); + errno = EPROTO; + goto fetch_error_wo_flock; + } + DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); + DYAD_LOG_DEBUG (mod_ctx->ctx, "DYAD_MOD: requested user_path: %s", upath); + DYAD_LOG_DEBUG (mod_ctx->ctx, "DYAD_MOD: sending initial response to consumer"); + + rc = mod_ctx->ctx->dtl_handle->rpc_respond (mod_ctx->ctx, msg); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (mod_ctx->ctx, "Could not send primary RPC response to client"); + goto fetch_error_wo_flock; + } + + strncpy (fullpath, mod_ctx->ctx->prod_managed_path, PATH_MAX - 1); + concat_str (fullpath, upath, "/", PATH_MAX); + DYAD_C_FUNCTION_UPDATE_STR ("fullpath", fullpath); #if DYAD_SPIN_WAIT - if (!get_stat(fullpath, 1000U, 1000L)) { - DYAD_LOG_ERR(mod_ctx->ctx, "DYAD_MOD: Failed to access info on \"%s\".", - fullpath); - // goto error; - } -#endif // DYAD_SPIN_WAIT - - DYAD_LOG_INFO(mod_ctx->ctx, "DYAD_MOD: Reading file %s for transfer", - fullpath); - fd = open(fullpath, O_RDONLY); - - if (fd < 0) { - DYAD_LOG_ERROR(mod_ctx->ctx, "DYAD_MOD: Failed to open file \"%s\".", - fullpath); - goto fetch_error_wo_flock; - } - rc = dyad_shared_flock(mod_ctx->ctx, fd, &shared_lock); - if (DYAD_IS_ERROR(rc)) { - goto fetch_error; - } - file_size = get_file_size(fd); - DYAD_LOG_DEBUG(mod_ctx->ctx, "DYAD_MOD: file %s has size %zd", fullpath, - file_size); - rc = mod_ctx->ctx->dtl_handle->get_buffer(mod_ctx->ctx, file_size, - (void **)&inbuf); + if (!get_stat (fullpath, 1000U, 1000L)) { + DYAD_LOG_ERR (mod_ctx->ctx, "DYAD_MOD: Failed to access info on \"%s\".", fullpath); + // goto error; + } +#endif // DYAD_SPIN_WAIT + + DYAD_LOG_INFO (mod_ctx->ctx, "DYAD_MOD: Reading file %s for transfer", fullpath); + fd = open (fullpath, O_RDONLY); + + if (fd < 0) { + DYAD_LOG_ERROR (mod_ctx->ctx, "DYAD_MOD: Failed to open file \"%s\".", fullpath); + goto fetch_error_wo_flock; + } + rc = dyad_shared_flock (mod_ctx->ctx, fd, &shared_lock); + if (DYAD_IS_ERROR (rc)) { + goto fetch_error; + } + file_size = get_file_size (fd); + DYAD_LOG_DEBUG (mod_ctx->ctx, "DYAD_MOD: file %s has size %zd", fullpath, file_size); + rc = mod_ctx->ctx->dtl_handle->get_buffer (mod_ctx->ctx, file_size, (void **)&inbuf); #ifdef DYAD_ENABLE_UCX_RMA - // To reduce the number of RMA calls, we are encoding file size at the start - // of the buffer - memcpy(inbuf, &file_size, sizeof(file_size)); + // To reduce the number of RMA calls, we are encoding file size at the start + // of the buffer + memcpy (inbuf, &file_size, sizeof (file_size)); #endif - if (file_size > 0l) { + if (file_size > 0l) { #ifdef DYAD_ENABLE_UCX_RMA - inlen = read(fd, inbuf + sizeof(file_size), file_size); + inlen = read (fd, inbuf + sizeof (file_size), file_size); #else - inlen = read(fd, inbuf, file_size); + inlen = read (fd, inbuf, file_size); #endif - if (inlen != file_size) { - DYAD_LOG_ERROR( - mod_ctx->ctx, - "DYAD_MOD: Failed to load file \"%s\" only read %zd of %zd.", - fullpath, inlen, file_size); - goto fetch_error; - } + if (inlen != file_size) { + DYAD_LOG_ERROR (mod_ctx->ctx, + "DYAD_MOD: Failed to load file \"%s\" only read %zd of %zd.", + fullpath, + inlen, + file_size); + goto fetch_error; + } #ifdef DYAD_ENABLE_UCX_RMA - inlen = file_size + sizeof(file_size); + inlen = file_size + sizeof (file_size); #endif - DYAD_C_FUNCTION_UPDATE_INT("file_size", file_size); - DYAD_LOG_DEBUG(mod_ctx->ctx, "Closing file pointer"); - dyad_release_flock(mod_ctx->ctx, fd, &shared_lock); - close(fd); - DYAD_LOG_DEBUG(mod_ctx->ctx, "Is inbuf NULL? -> %i", (int)(inbuf == NULL)); - DYAD_LOG_DEBUG(mod_ctx->ctx, "Establish DTL connection with consumer"); - rc = mod_ctx->ctx->dtl_handle->establish_connection(mod_ctx->ctx); - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(mod_ctx->ctx, - "Could not establish DTL connection with client"); - errno = ECONNREFUSED; - goto fetch_error_wo_flock; + DYAD_C_FUNCTION_UPDATE_INT ("file_size", file_size); + DYAD_LOG_DEBUG (mod_ctx->ctx, "Closing file pointer"); + dyad_release_flock (mod_ctx->ctx, fd, &shared_lock); + close (fd); + DYAD_LOG_DEBUG (mod_ctx->ctx, "Is inbuf NULL? -> %i", (int)(inbuf == NULL)); + DYAD_LOG_DEBUG (mod_ctx->ctx, "Establish DTL connection with consumer"); + rc = mod_ctx->ctx->dtl_handle->establish_connection (mod_ctx->ctx); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (mod_ctx->ctx, "Could not establish DTL connection with client"); + errno = ECONNREFUSED; + goto fetch_error_wo_flock; + } + DYAD_LOG_DEBUG (mod_ctx->ctx, "Send file to consumer with DTL"); + rc = mod_ctx->ctx->dtl_handle->send (mod_ctx->ctx, inbuf, inlen); + DYAD_LOG_DEBUG (mod_ctx->ctx, "Close DTL connection with consumer"); + mod_ctx->ctx->dtl_handle->close_connection (mod_ctx->ctx); + mod_ctx->ctx->dtl_handle->return_buffer (mod_ctx->ctx, (void **)&inbuf); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (mod_ctx->ctx, "Could not send data to client via DTL\n"); + errno = ECOMM; + goto fetch_error_wo_flock; + } + } else { + goto fetch_error; } - DYAD_LOG_DEBUG(mod_ctx->ctx, "Send file to consumer with DTL"); - rc = mod_ctx->ctx->dtl_handle->send(mod_ctx->ctx, inbuf, inlen); - DYAD_LOG_DEBUG(mod_ctx->ctx, "Close DTL connection with consumer"); - mod_ctx->ctx->dtl_handle->close_connection(mod_ctx->ctx); - mod_ctx->ctx->dtl_handle->return_buffer(mod_ctx->ctx, (void **)&inbuf); - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(mod_ctx->ctx, "Could not send data to client via DTL\n"); - errno = ECOMM; - goto fetch_error_wo_flock; + DYAD_LOG_DEBUG (mod_ctx->ctx, "Close RPC message stream with an ENODATA (%d) message", ENODATA); + if (flux_respond_error (h, msg, ENODATA, NULL) < 0) { + DYAD_LOG_ERROR (mod_ctx->ctx, + "DYAD_MOD: %s: flux_respond_error with ENODATA failed\n", + __func__); } - } else { - goto fetch_error; - } - DYAD_LOG_DEBUG(mod_ctx->ctx, - "Close RPC message stream with an ENODATA (%d) message", - ENODATA); - if (flux_respond_error(h, msg, ENODATA, NULL) < 0) { - DYAD_LOG_ERROR(mod_ctx->ctx, - "DYAD_MOD: %s: flux_respond_error with ENODATA failed\n", - __func__); - } - DYAD_LOG_INFO(mod_ctx->ctx, "Finished %s module invocation\n", - DYAD_DTL_RPC_NAME); - goto end_fetch_cb; + DYAD_LOG_INFO (mod_ctx->ctx, "Finished %s module invocation\n", DYAD_DTL_RPC_NAME); + goto end_fetch_cb; fetch_error:; - dyad_release_flock(mod_ctx->ctx, fd, &shared_lock); - close(fd); + dyad_release_flock (mod_ctx->ctx, fd, &shared_lock); + close (fd); fetch_error_wo_flock:; - DYAD_LOG_ERROR(mod_ctx->ctx, - "Close RPC message stream with an error (errno = %d)\n", - errno); - if (flux_respond_error(h, msg, errno, NULL) < 0) { - DYAD_LOG_ERROR(mod_ctx->ctx, "DYAD_MOD: %s: flux_respond_error", __func__); - } - errno = saved_errno; - DYAD_C_FUNCTION_END(); - return; + DYAD_LOG_ERROR (mod_ctx->ctx, "Close RPC message stream with an error (errno = %d)\n", errno); + if (flux_respond_error (h, msg, errno, NULL) < 0) { + DYAD_LOG_ERROR (mod_ctx->ctx, "DYAD_MOD: %s: flux_respond_error", __func__); + } + errno = saved_errno; + DYAD_C_FUNCTION_END (); + return; end_fetch_cb:; - errno = saved_errno; - DYAD_C_FUNCTION_END(); - return; + errno = saved_errno; + DYAD_C_FUNCTION_END (); + return; } -static const struct flux_msg_handler_spec htab[] = { - {FLUX_MSGTYPE_REQUEST, DYAD_DTL_RPC_NAME, dyad_fetch_request_cb, 0}, - FLUX_MSGHANDLER_TABLE_END}; - -static void show_help(void) { - DYAD_LOG_STDOUT("dyad module options and arguments\n"); - DYAD_LOG_STDOUT(" -h, --help: Show help.\n"); - DYAD_LOG_STDOUT(" -d, --debug: Enable debugging log message.\n"); - DYAD_LOG_STDOUT(" -m, --mode: DTL mode. Need an argument.\n" - " Either 'FLUX_RPC' (default) or 'UCX'.\n"); - DYAD_LOG_STDOUT( - " -i, --info_log: Specify the file into which to redirect\n" - " info logging. Does nothing if DYAD was not\n" - " configured with '-DDYAD_LOGGER=PRINTF'.\n" - " Need a filename as an argument.\n"); - DYAD_LOG_STDOUT( - " -e, --error_log: Specify the file into which to redirect\n" - " error logging. Does nothing if DYAD was\n" - " not configured with '-DDYAD_LOGGER=PRINTF'\n" - " Need a filename as an argument.\n"); +static const struct flux_msg_handler_spec htab[] = + {{FLUX_MSGTYPE_REQUEST, DYAD_DTL_RPC_NAME, dyad_fetch_request_cb, 0}, + FLUX_MSGHANDLER_TABLE_END}; + +static void show_help (void) +{ + DYAD_LOG_STDOUT ("dyad module options and arguments\n"); + DYAD_LOG_STDOUT (" -h, --help: Show help.\n"); + DYAD_LOG_STDOUT (" -d, --debug: Enable debugging log message.\n"); + DYAD_LOG_STDOUT ( + " -m, --mode: DTL mode. Need an argument.\n" + " Either 'FLUX_RPC' (default) or 'UCX'.\n"); + DYAD_LOG_STDOUT ( + " -i, --info_log: Specify the file into which to redirect\n" + " info logging. Does nothing if DYAD was not\n" + " configured with '-DDYAD_LOGGER=PRINTF'.\n" + " Need a filename as an argument.\n"); + DYAD_LOG_STDOUT ( + " -e, --error_log: Specify the file into which to redirect\n" + " error logging. Does nothing if DYAD was\n" + " not configured with '-DDYAD_LOGGER=PRINTF'\n" + " Need a filename as an argument.\n"); } struct opt_parse_out { - const char *prod_managed_path; - const char *dtl_mode; - bool debug; + const char *prod_managed_path; + const char *dtl_mode; + bool debug; }; typedef struct opt_parse_out opt_parse_out_t; -static int opt_parse(opt_parse_out_t *restrict opt, const unsigned broker_rank, - dyad_dtl_mode_t *restrict dtl_mode, int _argc, - char **restrict _argv) { +static int opt_parse (opt_parse_out_t *restrict opt, + const unsigned broker_rank, + dyad_dtl_mode_t *restrict dtl_mode, + int _argc, + char **restrict _argv) +{ #ifndef DYAD_LOGGER_NO_LOG - char log_file_name[PATH_MAX + 1] = {'\0'}; - char err_file_name[PATH_MAX + 1] = {'\0'}; - sprintf(log_file_name, "dyad_mod_%u.out", broker_rank); - sprintf(err_file_name, "dyad_mod_%d.err", broker_rank); -#endif // DYAD_LOGGER_NO_LOG - *dtl_mode = DYAD_DTL_END; - - int rc = DYAD_RC_OK; - int argc = 0; - char *argv[PATH_MAX] = {NULL}; - char *prod_managed_path = NULL; - - if (opt == NULL) - return rc; - - if ((argc = _argc + 1) > PATH_MAX) { - DYAD_LOG_STDERR("DYAD_MOD: too many options.\n"); - return DYAD_RC_SYSFAIL; - } - - for (int i = 1; i < argc; ++i) { - argv[i] = _argv[i - 1]; - // DYAD_LOG_STDERR ("DYAD_MOD: argv[%d] = '%s'\n", i, argv[i]); - } - - while (1) { - static struct option long_options[] = { - {"help", no_argument, 0, 'h'}, - {"debug", no_argument, 0, 'd'}, - {"mode", required_argument, 0, 'm'}, - {"info_log", required_argument, 0, 'i'}, - {"error_log", required_argument, 0, 'e'}, - {0, 0, 0, 0}}; - /* getopt_long stores the option index here. */ - int option_index = 0; - int c = -1; - - c = getopt_long(argc, argv, "hdm:i:e:", long_options, &option_index); - - /* Detect the end of the options. */ - if (c == -1) { - // DYAD_LOG_STDERR ("DYAD_MOD: no more option.\n"); - break; + char log_file_name[PATH_MAX + 1] = {'\0'}; + char err_file_name[PATH_MAX + 1] = {'\0'}; + sprintf (log_file_name, "dyad_mod_%u.out", broker_rank); + sprintf (err_file_name, "dyad_mod_%d.err", broker_rank); +#endif // DYAD_LOGGER_NO_LOG + *dtl_mode = DYAD_DTL_END; + + int rc = DYAD_RC_OK; + int argc = 0; + char *argv[PATH_MAX] = {NULL}; + char *prod_managed_path = NULL; + + if (opt == NULL) + return rc; + + if ((argc = _argc + 1) > PATH_MAX) { + DYAD_LOG_STDERR ("DYAD_MOD: too many options.\n"); + return DYAD_RC_SYSFAIL; } - DYAD_LOG_STDERR("DYAD_MOD: opt %c, index %d\n", (char)c, optind); - - switch (c) { - case 'h': - show_help(); - break; - case 'd': - DYAD_LOG_STDERR("DYAD_MOD: 'debug' option -d \n"); - opt->debug = true; - break; - case 'm': - // If the DTL is already initialized and it is set to the same - // mode as the option, then skip reinitializing - DYAD_LOG_STDERR("DYAD_MOD: DTL 'mode' option -m with value `%s'\n", - optarg); - opt->dtl_mode = optarg; - if (strcmp("UCX", optarg) == 0) - *dtl_mode = DYAD_DTL_UCX; - else if (strcmp("FLUX_RPC", optarg) == 0) - *dtl_mode = DYAD_DTL_FLUX_RPC; - break; - case 'i': + + for (int i = 1; i < argc; ++i) { + argv[i] = _argv[i - 1]; + // DYAD_LOG_STDERR ("DYAD_MOD: argv[%d] = '%s'\n", i, argv[i]); + } + + while (1) { + static struct option long_options[] = {{"help", no_argument, 0, 'h'}, + {"debug", no_argument, 0, 'd'}, + {"mode", required_argument, 0, 'm'}, + {"info_log", required_argument, 0, 'i'}, + {"error_log", required_argument, 0, 'e'}, + {0, 0, 0, 0}}; + /* getopt_long stores the option index here. */ + int option_index = 0; + int c = -1; + + c = getopt_long (argc, argv, "hdm:i:e:", long_options, &option_index); + + /* Detect the end of the options. */ + if (c == -1) { + // DYAD_LOG_STDERR ("DYAD_MOD: no more option.\n"); + break; + } + DYAD_LOG_STDERR ("DYAD_MOD: opt %c, index %d\n", (char)c, optind); + + switch (c) { + case 'h': + show_help (); + break; + case 'd': + DYAD_LOG_STDERR ("DYAD_MOD: 'debug' option -d \n"); + opt->debug = true; + break; + case 'm': + // If the DTL is already initialized and it is set to the same + // mode as the option, then skip reinitializing + DYAD_LOG_STDERR ("DYAD_MOD: DTL 'mode' option -m with value `%s'\n", optarg); + opt->dtl_mode = optarg; + if (strcmp ("UCX", optarg) == 0) + *dtl_mode = DYAD_DTL_UCX; + else if (strcmp ("FLUX_RPC", optarg) == 0) + *dtl_mode = DYAD_DTL_FLUX_RPC; + break; + case 'i': #ifndef DYAD_LOGGER_NO_LOG - DYAD_LOG_STDERR("DYAD_MOD: 'info_log' option -i with value `%s'\n", - optarg); - sprintf(log_file_name, "%s_%u.out", optarg, broker_rank); -#endif // DYAD_LOGGER_NO_LOG - break; - case 'e': + DYAD_LOG_STDERR ("DYAD_MOD: 'info_log' option -i with value `%s'\n", optarg); + sprintf (log_file_name, "%s_%u.out", optarg, broker_rank); +#endif // DYAD_LOGGER_NO_LOG + break; + case 'e': #ifndef DYAD_LOGGER_NO_LOG - DYAD_LOG_STDERR("DYAD_MOD: 'error_log' option -e with value `%s'\n", - optarg); - sprintf(err_file_name, "%s_%d.err", optarg, broker_rank); -#endif // DYAD_LOGGER_NO_LOG - break; - case '?': - /* getopt_long already printed an error message. */ - break; - default: - DYAD_LOG_STDERR("DYAD_MOD: option parsing failed %d\n", c); - return DYAD_RC_SYSFAIL; + DYAD_LOG_STDERR ("DYAD_MOD: 'error_log' option -e with value `%s'\n", optarg); + sprintf (err_file_name, "%s_%d.err", optarg, broker_rank); +#endif // DYAD_LOGGER_NO_LOG + break; + case '?': + /* getopt_long already printed an error message. */ + break; + default: + DYAD_LOG_STDERR ("DYAD_MOD: option parsing failed %d\n", c); + return DYAD_RC_SYSFAIL; + } } - } #ifndef DYAD_LOGGER_NO_LOG - DYAD_LOG_STDOUT_REDIRECT(log_file_name); - DYAD_LOG_STDERR_REDIRECT(err_file_name); -#endif // DYAD_LOGGER_NO_LOG - - if (*dtl_mode == DYAD_DTL_END) { - opt->dtl_mode = NULL; - } - - /* Print any remaining command line arguments (not options). */ - while (optind < argc) { - DYAD_LOG_STDERR("DYAD_MOD: positional arguments %s\n", argv[optind]); - prod_managed_path = argv[optind++]; - } - opt->prod_managed_path = prod_managed_path; - - return DYAD_RC_OK; + DYAD_LOG_STDOUT_REDIRECT (log_file_name); + DYAD_LOG_STDERR_REDIRECT (err_file_name); +#endif // DYAD_LOGGER_NO_LOG + + if (*dtl_mode == DYAD_DTL_END) { + opt->dtl_mode = NULL; + } + + /* Print any remaining command line arguments (not options). */ + while (optind < argc) { + DYAD_LOG_STDERR ("DYAD_MOD: positional arguments %s\n", argv[optind]); + prod_managed_path = argv[optind++]; + } + opt->prod_managed_path = prod_managed_path; + + return DYAD_RC_OK; } -dyad_rc_t dyad_module_ctx_init(const opt_parse_out_t *opt, - flux_t *h) { // Initialize DYAD context - dyad_mod_ctx_t *mod_ctx = get_mod_ctx(h); - - if (mod_ctx == NULL || opt == NULL || h == NULL) { - return DYAD_RC_NOCTX; - } - - if (opt->prod_managed_path) { - setenv(DYAD_PATH_PRODUCER_ENV, opt->prod_managed_path, 1); - const mode_t m = (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID); - mkdir_as_needed(opt->prod_managed_path, m); - DYAD_LOG_STDERR("DYAD_MOD: Loading DYAD Module with Path %s", - opt->prod_managed_path); - } - - if (opt->dtl_mode) { - setenv(DYAD_DTL_MODE_ENV, opt->dtl_mode, 1); - DYAD_LOG_STDERR("DYAD_MOD: DTL 'mode' option set. " - "Setting env %s=%s", - DYAD_DTL_MODE_ENV, opt->dtl_mode); - } else { - DYAD_LOG_STDERR("DYAD_MOD: Did not find DTL 'mode' option. " - "Using env %s=%s", - DYAD_DTL_MODE_ENV, getenv(DYAD_DTL_MODE_ENV)); - } - char *kvs = getenv(DYAD_KVS_NAMESPACE_ENV); - if (kvs != NULL) { - DYAD_LOG_STDERR("DYAD_MOD: %s is set to `%s'\n", DYAD_KVS_NAMESPACE_ENV, - kvs); - } else { - DYAD_LOG_STDERR("DYAD_MOD: %s is not set\n", DYAD_KVS_NAMESPACE_ENV); - // Required so that dyad_ctx_init can pass - setenv(DYAD_KVS_NAMESPACE_ENV, "dyad_module_dummy_env", 1); - } - - dyad_ctx_init(DYAD_COMM_SEND, h); - mod_ctx->ctx = dyad_ctx_get(); - dyad_ctx_t *ctx = mod_ctx->ctx; - - if (ctx == NULL) { - DYAD_LOG_STDERR("DYAD_MOD: dyad_ctx_init() failed!"); - return DYAD_RC_NOCTX; - } - ctx->h = h; - ctx->debug = opt->debug; - - if (ctx->dtl_handle == NULL) { - DYAD_LOG_STDERR("DYAD_MOD: dyad_ctx_init() failed to initialize DTL!"); - return DYAD_RC_NOCTX; - } - - return DYAD_RC_OK; +dyad_rc_t dyad_module_ctx_init (const opt_parse_out_t *opt, flux_t *h) +{ // Initialize DYAD context + dyad_mod_ctx_t *mod_ctx = get_mod_ctx (h); + + if (mod_ctx == NULL || opt == NULL || h == NULL) { + return DYAD_RC_NOCTX; + } + + if (opt->prod_managed_path) { + setenv (DYAD_PATH_PRODUCER_ENV, opt->prod_managed_path, 1); + const mode_t m = (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID); + mkdir_as_needed (opt->prod_managed_path, m); + DYAD_LOG_STDERR ("DYAD_MOD: Loading DYAD Module with Path %s", opt->prod_managed_path); + } + + if (opt->dtl_mode) { + setenv (DYAD_DTL_MODE_ENV, opt->dtl_mode, 1); + DYAD_LOG_STDERR ( + "DYAD_MOD: DTL 'mode' option set. " + "Setting env %s=%s", + DYAD_DTL_MODE_ENV, + opt->dtl_mode); + } else { + DYAD_LOG_STDERR ( + "DYAD_MOD: Did not find DTL 'mode' option. " + "Using env %s=%s", + DYAD_DTL_MODE_ENV, + getenv (DYAD_DTL_MODE_ENV)); + } + char *kvs = getenv (DYAD_KVS_NAMESPACE_ENV); + if (kvs != NULL) { + DYAD_LOG_STDERR ("DYAD_MOD: %s is set to `%s'\n", DYAD_KVS_NAMESPACE_ENV, kvs); + } else { + DYAD_LOG_STDERR ("DYAD_MOD: %s is not set\n", DYAD_KVS_NAMESPACE_ENV); + // Required so that dyad_ctx_init can pass + setenv (DYAD_KVS_NAMESPACE_ENV, "dyad_module_dummy_env", 1); + } + + dyad_ctx_init (DYAD_COMM_SEND, h); + mod_ctx->ctx = dyad_ctx_get (); + dyad_ctx_t *ctx = mod_ctx->ctx; + + if (ctx == NULL) { + DYAD_LOG_STDERR ("DYAD_MOD: dyad_ctx_init() failed!"); + return DYAD_RC_NOCTX; + } + ctx->h = h; + ctx->debug = opt->debug; + + if (ctx->dtl_handle == NULL) { + DYAD_LOG_STDERR ("DYAD_MOD: dyad_ctx_init() failed to initialize DTL!"); + return DYAD_RC_NOCTX; + } + + return DYAD_RC_OK; } -DYAD_DLL_EXPORTED int mod_main(flux_t *h, int argc, char **argv) { - DYAD_LOGGER_INIT(); - DYAD_LOG_STDOUT("Loading mod_main\n"); - dyad_mod_ctx_t *mod_ctx = NULL; - dyad_dtl_mode_t dtl_mode = DYAD_DTL_DEFAULT; +DYAD_DLL_EXPORTED int mod_main (flux_t *h, int argc, char **argv) +{ + DYAD_LOGGER_INIT (); + DYAD_LOG_STDOUT ("Loading mod_main\n"); + dyad_mod_ctx_t *mod_ctx = NULL; + dyad_dtl_mode_t dtl_mode = DYAD_DTL_DEFAULT; - if (!h) { - DYAD_LOG_STDERR("Failed to get flux handle\n"); - goto mod_done; - } + if (!h) { + DYAD_LOG_STDERR ("Failed to get flux handle\n"); + goto mod_done; + } - mod_ctx = get_mod_ctx(h); + mod_ctx = get_mod_ctx (h); - uint32_t broker_rank; - flux_get_rank(h, &broker_rank); + uint32_t broker_rank; + flux_get_rank (h, &broker_rank); #ifdef DYAD_PROFILER_DFTRACER - int pid = broker_rank; - DFTRACER_C_INIT(NULL, NULL, &pid); + int pid = broker_rank; + DFTRACER_C_INIT (NULL, NULL, &pid); #endif - DYAD_C_FUNCTION_START(); - - opt_parse_out_t opt = {NULL, NULL, false}; - DYAD_LOG_STDERR("DYAD_MOD: Parsing command line options"); - - if (DYAD_IS_ERROR(opt_parse(&opt, broker_rank, &dtl_mode, argc, argv))) { - DYAD_LOG_STDERR("DYAD_MOD: Cannot parse command line arguments"); - goto mod_error; - } - - if (DYAD_IS_ERROR(dyad_module_ctx_init(&opt, h))) { - goto mod_error; - } - - if (flux_msg_handler_addvec(mod_ctx->ctx->h, htab, (void *)h, - &mod_ctx->handlers) < 0) { - DYAD_LOG_ERROR(mod_ctx->ctx, "DYAD_MOD: flux_msg_handler_addvec: %s\n", - strerror(errno)); - goto mod_error; - } - - if (flux_reactor_run(flux_get_reactor(mod_ctx->ctx->h), 0) < 0) { - DYAD_LOG_DEBUG(mod_ctx->ctx, "DYAD_MOD: flux_reactor_run: %s", - strerror(errno)); - goto mod_error; - } - DYAD_LOG_DEBUG(mod_ctx->ctx, "DYAD_MOD: Finished"); - goto mod_done; + DYAD_C_FUNCTION_START (); + + opt_parse_out_t opt = {NULL, NULL, false}; + DYAD_LOG_STDERR ("DYAD_MOD: Parsing command line options"); + + if (DYAD_IS_ERROR (opt_parse (&opt, broker_rank, &dtl_mode, argc, argv))) { + DYAD_LOG_STDERR ("DYAD_MOD: Cannot parse command line arguments"); + goto mod_error; + } + + if (DYAD_IS_ERROR (dyad_module_ctx_init (&opt, h))) { + goto mod_error; + } + + if (flux_msg_handler_addvec (mod_ctx->ctx->h, htab, (void *)h, &mod_ctx->handlers) < 0) { + DYAD_LOG_ERROR (mod_ctx->ctx, "DYAD_MOD: flux_msg_handler_addvec: %s\n", strerror (errno)); + goto mod_error; + } + + if (flux_reactor_run (flux_get_reactor (mod_ctx->ctx->h), 0) < 0) { + DYAD_LOG_DEBUG (mod_ctx->ctx, "DYAD_MOD: flux_reactor_run: %s", strerror (errno)); + goto mod_error; + } + DYAD_LOG_DEBUG (mod_ctx->ctx, "DYAD_MOD: Finished"); + goto mod_done; mod_error:; - DYAD_C_FUNCTION_END(); - return EXIT_FAILURE; + DYAD_C_FUNCTION_END (); + return EXIT_FAILURE; mod_done:; - DYAD_C_FUNCTION_END(); - return EXIT_SUCCESS; + DYAD_C_FUNCTION_END (); + return EXIT_SUCCESS; } -DYAD_DLL_EXPORTED MOD_NAME("dyad"); +DYAD_DLL_EXPORTED MOD_NAME ("dyad"); /* * vi:tabstop=4 shiftwidth=4 expandtab diff --git a/src/dyad/modules/test_opt_parse.c b/src/dyad/modules/test_opt_parse.c index cfc61666..9caf6276 100644 --- a/src/dyad/modules/test_opt_parse.c +++ b/src/dyad/modules/test_opt_parse.c @@ -31,7 +31,7 @@ #include #include #include -#endif // defined(__cplusplus) +#endif // defined(__cplusplus) #include #include @@ -41,179 +41,184 @@ /** This is a temporary measure until environment variable based initialization * is implemented */ -static dyad_dtl_mode_t get_dtl_mode_env() { - char *e = NULL; - - size_t dtl_mode_env_len = 0ul; - - if ((e = getenv(DYAD_DTL_MODE_ENV))) { - dtl_mode_env_len = strlen(e); - if (strncmp(e, "FLUX_RPC", dtl_mode_env_len) == 0) { - DYAD_LOG_STDERR("DYAD MOD: FLUX_RPC DTL mode found in environment\n"); - return DYAD_DTL_FLUX_RPC; - } else if (strncmp(e, "UCX", dtl_mode_env_len) == 0) { - DYAD_LOG_STDERR("DYAD MOD: UCX DTL mode found in environment\n"); - return DYAD_DTL_UCX; +static dyad_dtl_mode_t get_dtl_mode_env () +{ + char *e = NULL; + + size_t dtl_mode_env_len = 0ul; + + if ((e = getenv (DYAD_DTL_MODE_ENV))) { + dtl_mode_env_len = strlen (e); + if (strncmp (e, "FLUX_RPC", dtl_mode_env_len) == 0) { + DYAD_LOG_STDERR ("DYAD MOD: FLUX_RPC DTL mode found in environment\n"); + return DYAD_DTL_FLUX_RPC; + } else if (strncmp (e, "UCX", dtl_mode_env_len) == 0) { + DYAD_LOG_STDERR ("DYAD MOD: UCX DTL mode found in environment\n"); + return DYAD_DTL_UCX; + } else { + DYAD_LOG_STDERR ("DYAD MOD: Invalid env %s = %s. Defaulting to %s\n", + DYAD_DTL_MODE_ENV, + e, + dyad_dtl_mode_name[DYAD_DTL_DEFAULT]); + return DYAD_DTL_DEFAULT; + } } else { - DYAD_LOG_STDERR("DYAD MOD: Invalid env %s = %s. Defaulting to %s\n", - DYAD_DTL_MODE_ENV, e, - dyad_dtl_mode_name[DYAD_DTL_DEFAULT]); - return DYAD_DTL_DEFAULT; + DYAD_LOG_STDERR ("DYAD MOD: %s is not set. Defaulting to %s\n", + DYAD_DTL_MODE_ENV, + dyad_dtl_mode_name[DYAD_DTL_DEFAULT]); + return DYAD_DTL_DEFAULT; } - } else { - DYAD_LOG_STDERR("DYAD MOD: %s is not set. Defaulting to %s\n", - DYAD_DTL_MODE_ENV, dyad_dtl_mode_name[DYAD_DTL_DEFAULT]); return DYAD_DTL_DEFAULT; - } - return DYAD_DTL_DEFAULT; } -static void show_help(void) { - DYAD_LOG_STDOUT("dyad module options and arguments\n"); - DYAD_LOG_STDOUT(" -h, --help: Show help.\n"); - DYAD_LOG_STDOUT(" -d, --debug: Enable debugging log message.\n"); - DYAD_LOG_STDOUT(" -m, --mode: DTL mode. Need an argument.\n" - " Either 'FLUX_RPC' (default) or 'UCX'.\n"); - DYAD_LOG_STDOUT( - " -i, --info_log: Specify the file into which to redirect\n" - " info logging. Does nothing if DYAD was not\n" - " configured with '-DDYAD_LOGGER=PRINTF'.\n" - " Need a filename as an argument.\n"); - DYAD_LOG_STDOUT( - " -e, --error_log: Specify the file into which to redirect\n" - " error logging. Does nothing if DYAD was\n" - " not configured with '-DDYAD_LOGGER=PRINTF'\n" - " Need a filename as an argument.\n"); +static void show_help (void) +{ + DYAD_LOG_STDOUT ("dyad module options and arguments\n"); + DYAD_LOG_STDOUT (" -h, --help: Show help.\n"); + DYAD_LOG_STDOUT (" -d, --debug: Enable debugging log message.\n"); + DYAD_LOG_STDOUT ( + " -m, --mode: DTL mode. Need an argument.\n" + " Either 'FLUX_RPC' (default) or 'UCX'.\n"); + DYAD_LOG_STDOUT ( + " -i, --info_log: Specify the file into which to redirect\n" + " info logging. Does nothing if DYAD was not\n" + " configured with '-DDYAD_LOGGER=PRINTF'.\n" + " Need a filename as an argument.\n"); + DYAD_LOG_STDOUT ( + " -e, --error_log: Specify the file into which to redirect\n" + " error logging. Does nothing if DYAD was\n" + " not configured with '-DDYAD_LOGGER=PRINTF'\n" + " Need a filename as an argument.\n"); } -static int opt_parse(dyad_ctx_t *ctx, const unsigned broker_rank, - dyad_dtl_mode_t *dtl_mode, int argc, char **argv) { - size_t arglen = 0ul; +static int opt_parse (dyad_ctx_t *ctx, + const unsigned broker_rank, + dyad_dtl_mode_t *dtl_mode, + int argc, + char **argv) +{ + size_t arglen = 0ul; #ifndef DYAD_LOGGER_NO_LOG - char log_file_name[PATH_MAX + 1] = {'\0'}; - char err_file_name[PATH_MAX + 1] = {'\0'}; - sprintf(log_file_name, "dyad_mod_%u.out", broker_rank); - sprintf(err_file_name, "dyad_mod_%d.err", broker_rank); -#endif // DYAD_LOGGER_NO_LOG - *dtl_mode = DYAD_DTL_END; - char *prod_managed_path = NULL; - bool debug = false; - (void)debug; - - while (1) { - static struct option long_options[] = { - {"help", no_argument, 0, 'h'}, - {"debug", no_argument, 0, 'd'}, - {"mode", required_argument, 0, 'm'}, - {"info_log", required_argument, 0, 'i'}, - {"error_log", required_argument, 0, 'e'}, - {0, 0, 0, 0}}; - /* getopt_long stores the option index here. */ - int option_index = 0; - int c = -1; - - c = getopt_long(argc, argv, "hdm:i:e:", long_options, &option_index); - - /* Detect the end of the options. */ - if (c == -1) { - DYAD_LOG_DEBUG(ctx, "DYAD_MOD: no more option\n"); - break; - } - DYAD_LOG_DEBUG(ctx, "DYAD_MOD: opt %c, index %d\n", (char)c, optind); - - switch (c) { - case 'h': - show_help(); - break; - case 'd': - DYAD_LOG_DEBUG(ctx, "DYAD_MOD: 'debug' option -d \n"); - debug = true; - break; - case 'm': - DYAD_LOG_DEBUG(ctx, "DYAD_MOD: DTL 'mode' option -m with value `%s'\n", - optarg); - arglen = strlen(optarg); - if (strncmp(optarg, "FLUX_RPC", arglen) == 0) { - *dtl_mode = DYAD_DTL_FLUX_RPC; - } else if (strncmp(optarg, "UCX", arglen) == 0) { - *dtl_mode = DYAD_DTL_UCX; - } else { - DYAD_LOG_ERROR(ctx, "DYAD_MOD: Invalid DTL 'mode' (%s) provided\n", - optarg); - *dtl_mode = DYAD_DTL_END; - show_help(); - return DYAD_RC_SYSFAIL; - } - break; - case 'i': + char log_file_name[PATH_MAX + 1] = {'\0'}; + char err_file_name[PATH_MAX + 1] = {'\0'}; + sprintf (log_file_name, "dyad_mod_%u.out", broker_rank); + sprintf (err_file_name, "dyad_mod_%d.err", broker_rank); +#endif // DYAD_LOGGER_NO_LOG + *dtl_mode = DYAD_DTL_END; + char *prod_managed_path = NULL; + bool debug = false; + (void)debug; + + while (1) { + static struct option long_options[] = {{"help", no_argument, 0, 'h'}, + {"debug", no_argument, 0, 'd'}, + {"mode", required_argument, 0, 'm'}, + {"info_log", required_argument, 0, 'i'}, + {"error_log", required_argument, 0, 'e'}, + {0, 0, 0, 0}}; + /* getopt_long stores the option index here. */ + int option_index = 0; + int c = -1; + + c = getopt_long (argc, argv, "hdm:i:e:", long_options, &option_index); + + /* Detect the end of the options. */ + if (c == -1) { + DYAD_LOG_DEBUG (ctx, "DYAD_MOD: no more option\n"); + break; + } + DYAD_LOG_DEBUG (ctx, "DYAD_MOD: opt %c, index %d\n", (char)c, optind); + + switch (c) { + case 'h': + show_help (); + break; + case 'd': + DYAD_LOG_DEBUG (ctx, "DYAD_MOD: 'debug' option -d \n"); + debug = true; + break; + case 'm': + DYAD_LOG_DEBUG (ctx, "DYAD_MOD: DTL 'mode' option -m with value `%s'\n", optarg); + arglen = strlen (optarg); + if (strncmp (optarg, "FLUX_RPC", arglen) == 0) { + *dtl_mode = DYAD_DTL_FLUX_RPC; + } else if (strncmp (optarg, "UCX", arglen) == 0) { + *dtl_mode = DYAD_DTL_UCX; + } else { + DYAD_LOG_ERROR (ctx, "DYAD_MOD: Invalid DTL 'mode' (%s) provided\n", optarg); + *dtl_mode = DYAD_DTL_END; + show_help (); + return DYAD_RC_SYSFAIL; + } + break; + case 'i': #ifndef DYAD_LOGGER_NO_LOG - DYAD_LOG_DEBUG(ctx, "DYAD_MOD: 'info_log' option -i with value `%s'\n", - optarg); - sprintf(log_file_name, "%s_%u.out", optarg, broker_rank); -#endif // DYAD_LOGGER_NO_LOG - break; - case 'e': + DYAD_LOG_DEBUG (ctx, "DYAD_MOD: 'info_log' option -i with value `%s'\n", optarg); + sprintf (log_file_name, "%s_%u.out", optarg, broker_rank); +#endif // DYAD_LOGGER_NO_LOG + break; + case 'e': #ifndef DYAD_LOGGER_NO_LOG - DYAD_LOG_DEBUG(ctx, "DYAD_MOD: 'error_log' option -e with value `%s'\n", - optarg); - sprintf(err_file_name, "%s_%d.err", optarg, broker_rank); -#endif // DYAD_LOGGER_NO_LOG - break; - case '?': - /* getopt_long already printed an error message. */ - break; - default: - DYAD_LOG_DEBUG(ctx, "DYAD_MOD: option parsing failed %d\n", c); - return DYAD_RC_SYSFAIL; + DYAD_LOG_DEBUG (ctx, "DYAD_MOD: 'error_log' option -e with value `%s'\n", optarg); + sprintf (err_file_name, "%s_%d.err", optarg, broker_rank); +#endif // DYAD_LOGGER_NO_LOG + break; + case '?': + /* getopt_long already printed an error message. */ + break; + default: + DYAD_LOG_DEBUG (ctx, "DYAD_MOD: option parsing failed %d\n", c); + return DYAD_RC_SYSFAIL; + } } - } #ifndef DYAD_LOGGER_NO_LOG - DYAD_LOG_STDOUT_REDIRECT(log_file_name); - DYAD_LOG_STDERR_REDIRECT(err_file_name); -#endif // DYAD_LOGGER_NO_LOG - - if (*dtl_mode == DYAD_DTL_END) { - DYAD_LOG_DEBUG(ctx, "DYAD_MOD: Did not find DTL 'mode' option"); - *dtl_mode = get_dtl_mode_env(); - } - - DYAD_LOG_DEBUG(ctx, "DYAD_MOD: optind %d argc %d\n", optind, argc); - /* Print any remaining command line arguments (not options). */ - while (optind < argc) { - DYAD_LOG_DEBUG(ctx, "DYAD_MOD: positional arguments %s\n", argv[optind]); - prod_managed_path = argv[optind++]; - } - if (prod_managed_path) { - const size_t prod_path_len = strlen(prod_managed_path); - const char *tmp_ptr = prod_managed_path; - prod_managed_path = (char *)malloc(prod_path_len + 1); - if (prod_managed_path == NULL) { - DYAD_LOG_ERROR(ctx, "DYAD_MOD: Could not allocate buffer for " - "Producer managed path!\n"); - return DYAD_RC_SYSFAIL; + DYAD_LOG_STDOUT_REDIRECT (log_file_name); + DYAD_LOG_STDERR_REDIRECT (err_file_name); +#endif // DYAD_LOGGER_NO_LOG + + if (*dtl_mode == DYAD_DTL_END) { + DYAD_LOG_DEBUG (ctx, "DYAD_MOD: Did not find DTL 'mode' option"); + *dtl_mode = get_dtl_mode_env (); + } + + DYAD_LOG_DEBUG (ctx, "DYAD_MOD: optind %d argc %d\n", optind, argc); + /* Print any remaining command line arguments (not options). */ + while (optind < argc) { + DYAD_LOG_DEBUG (ctx, "DYAD_MOD: positional arguments %s\n", argv[optind]); + prod_managed_path = argv[optind++]; + } + if (prod_managed_path) { + const size_t prod_path_len = strlen (prod_managed_path); + const char *tmp_ptr = prod_managed_path; + prod_managed_path = (char *)malloc (prod_path_len + 1); + if (prod_managed_path == NULL) { + DYAD_LOG_ERROR (ctx, + "DYAD_MOD: Could not allocate buffer for " + "Producer managed path!\n"); + return DYAD_RC_SYSFAIL; + } + strncpy (prod_managed_path, tmp_ptr, prod_path_len + 1); + DYAD_LOG_INFO (ctx, + "DYAD_MOD: Loading DYAD Module with Path %s and DTL Mode %s", + prod_managed_path, + dyad_dtl_mode_name[*dtl_mode]); } - strncpy(prod_managed_path, tmp_ptr, prod_path_len + 1); - DYAD_LOG_INFO(ctx, - "DYAD_MOD: Loading DYAD Module with Path %s and DTL Mode %s", - prod_managed_path, dyad_dtl_mode_name[*dtl_mode]); - } - DYAD_LOG_INFO(NULL, "DYAD_MOD: debug flag set to %s\n", - debug ? "true" : "false"); - DYAD_LOG_INFO(NULL, "DYAD_MOD: prod_managed_path set to %s\n", - prod_managed_path); - return DYAD_RC_OK; + DYAD_LOG_INFO (NULL, "DYAD_MOD: debug flag set to %s\n", debug ? "true" : "false"); + DYAD_LOG_INFO (NULL, "DYAD_MOD: prod_managed_path set to %s\n", prod_managed_path); + return DYAD_RC_OK; } -int main(int argc, char **argv) { - dyad_dtl_mode_t dtl_mode = DYAD_DTL_DEFAULT; - uint32_t broker_rank = 0u; +int main (int argc, char **argv) +{ + dyad_dtl_mode_t dtl_mode = DYAD_DTL_DEFAULT; + uint32_t broker_rank = 0u; - DYAD_LOG_DEBUG(NULL, "DYAD_MOD: Parsing command line options"); - if (opt_parse(NULL, broker_rank, &dtl_mode, argc, argv) != DYAD_RC_OK) { - DYAD_LOG_ERROR(NULL, "DYAD_MOD: Cannot parse command line arguments"); - return EXIT_FAILURE; - } + DYAD_LOG_DEBUG (NULL, "DYAD_MOD: Parsing command line options"); + if (opt_parse (NULL, broker_rank, &dtl_mode, argc, argv) != DYAD_RC_OK) { + DYAD_LOG_ERROR (NULL, "DYAD_MOD: Cannot parse command line arguments"); + return EXIT_FAILURE; + } - return EXIT_SUCCESS; + return EXIT_SUCCESS; } diff --git a/src/dyad/stream/dyad_stream_core.cpp b/src/dyad/stream/dyad_stream_core.cpp index 5a94bb9d..275f324d 100644 --- a/src/dyad/stream/dyad_stream_core.cpp +++ b/src/dyad/stream/dyad_stream_core.cpp @@ -23,8 +23,11 @@ #ifndef _GNU_SOURCE #define _GNU_SOURCE #include -#endif // _GNU_SOURCE +#endif // _GNU_SOURCE +#include +#include // dirname +#include #include #include #include @@ -32,225 +35,273 @@ #include #include #include -using namespace std; // std::clock () +using namespace std; // std::clock () // #include // c++11 #include + +#include + #include #include #include #include #include -#include // dirname +#include // dirname #include -namespace dyad { +namespace dyad +{ /***************************************************************************** * * * dyad_stream_core API * * * *****************************************************************************/ -dyad_stream_core::dyad_stream_core() - : m_ctx(NULL), m_ctx_mutable(NULL), m_initialized(false), m_is_prod(false), - m_is_cons(false) { - DYAD_CPP_FUNCTION(); +dyad_stream_core::dyad_stream_core () + : m_ctx (NULL), + m_ctx_mutable (NULL), + m_initialized (false), + m_is_prod (false), + m_is_cons (false) +{ + DYAD_CPP_FUNCTION (); } -dyad_stream_core::~dyad_stream_core() { finalize(); } +dyad_stream_core::~dyad_stream_core () +{ + finalize (); +} -void dyad_stream_core::finalize() { - DYAD_CPP_FUNCTION(); - if (m_ctx != NULL) { - // dyad_finalize (&m_ctx); - m_ctx = m_ctx_mutable = NULL; - m_initialized = false; - } +void dyad_stream_core::finalize () +{ + DYAD_CPP_FUNCTION (); + if (m_ctx != NULL) { + // dyad_finalize (&m_ctx); + m_ctx = m_ctx_mutable = NULL; + m_initialized = false; + } } -void dyad_stream_core::init(const bool reinit) { - DYAD_CPP_FUNCTION(); - bool reinit_env = false; - char *e = NULL; - - if ((e = getenv(DYAD_REINIT_ENV))) { - reinit_env = true; - } else { - reinit_env = false; - } - - if (!(reinit || reinit_env) && m_initialized) { - return; - } - - if ((e = getenv(DYAD_PATH_CONSUMER_ENV))) { - m_is_cons = (strlen(e) != 0); - } else { - m_is_cons = false; - } - if ((e = getenv(DYAD_PATH_PRODUCER_ENV))) { - m_is_prod = (strlen(e) != 0); - } else { - m_is_prod = false; - } - - if (reinit || reinit_env || !(m_ctx = m_ctx_mutable = dyad_ctx_get())) { - dyad_ctx_init(DYAD_COMM_RECV, NULL); - m_ctx = m_ctx_mutable = dyad_ctx_get(); - log_info("Stream core is initialized by env variables."); - } else { - log_info( - "Stream core skips initialization as it has already been initialized."); - } - - // TODO figure out if we want to error if init fails - m_initialized = true; +void dyad_stream_core::init (const bool reinit) +{ + DYAD_CPP_FUNCTION (); + bool reinit_env = false; + char *e = NULL; + + if ((e = getenv (DYAD_REINIT_ENV))) { + reinit_env = true; + } else { + reinit_env = false; + } + + if (!(reinit || reinit_env) && m_initialized) { + return; + } + + if ((e = getenv (DYAD_PATH_CONSUMER_ENV))) { + m_is_cons = (strlen (e) != 0); + } else { + m_is_cons = false; + } + if ((e = getenv (DYAD_PATH_PRODUCER_ENV))) { + m_is_prod = (strlen (e) != 0); + } else { + m_is_prod = false; + } + + if (reinit || reinit_env || !(m_ctx = m_ctx_mutable = dyad_ctx_get ())) { + dyad_ctx_init (DYAD_COMM_RECV, NULL); + m_ctx = m_ctx_mutable = dyad_ctx_get (); + log_info ("Stream core is initialized by env variables."); + } else { + log_info ("Stream core skips initialization as it has already been initialized."); + } + + // TODO figure out if we want to error if init fails + m_initialized = true; } -void dyad_stream_core::init(const dyad_params &p) { - DYAD_CPP_FUNCTION(); - DYAD_LOG_DEBUG(m_ctx, "DYAD_WRAPPER: Initializeing DYAD wrapper"); - dyad_rc_t rc = dyad_init( - p.m_debug, false, p.m_shared_storage, p.m_reinit, p.m_async_publish, - p.m_fsync_write, p.m_key_depth, p.m_key_bins, p.m_service_mux, - p.m_kvs_namespace.c_str(), p.m_prod_managed_path.c_str(), - p.m_cons_managed_path.c_str(), p.m_relative_to_managed_path, - dyad_dtl_mode_name[static_cast(p.m_dtl_mode)], - DYAD_COMM_RECV, NULL); +void dyad_stream_core::init (const dyad_params &p) +{ + DYAD_CPP_FUNCTION (); + DYAD_LOG_DEBUG (m_ctx, "DYAD_WRAPPER: Initializeing DYAD wrapper"); + dyad_rc_t rc = dyad_init (p.m_debug, + false, + p.m_shared_storage, + p.m_reinit, + p.m_async_publish, + p.m_fsync_write, + p.m_key_depth, + p.m_key_bins, + p.m_service_mux, + p.m_kvs_namespace.c_str (), + p.m_prod_managed_path.c_str (), + p.m_cons_managed_path.c_str (), + p.m_relative_to_managed_path, + dyad_dtl_mode_name[static_cast (p.m_dtl_mode)], + DYAD_COMM_RECV, + NULL); #if defined(DYAD_HAS_STD_FSTREAM_FD) - m_ctx_mutable->use_fs_locks = true; + m_ctx_mutable->use_fs_locks = true; #else - // Rely on the KVS-based synchronization and disable checking for fs lock - // based logic. - m_ctx_mutable->use_fs_locks = false; + // Rely on the KVS-based synchronization and disable checking for fs lock + // based logic. + m_ctx_mutable->use_fs_locks = false; #endif - (void)rc; - // TODO figure out if we want to error if init fails - m_initialized = true; - log_info("Stream core is initialized by parameters"); + (void)rc; + // TODO figure out if we want to error if init fails + m_initialized = true; + log_info ("Stream core is initialized by parameters"); } -void dyad_stream_core::log_info(const std::string &msg_head) const { - DYAD_CPP_FUNCTION(); - DYAD_LOG_INFO(m_ctx, "=== %s ===", msg_head.c_str()); - DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_PATH_CONSUMER_ENV, - m_ctx->cons_managed_path); - DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_PATH_PRODUCER_ENV, - m_ctx->prod_managed_path); - DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_PATH_RELATIVE_ENV, - (m_ctx->relative_to_managed_path) ? "true" : "false"); - DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_SYNC_DEBUG_ENV, - (m_ctx->debug) ? "true" : "false"); - DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_SHARED_STORAGE_ENV, - (m_ctx->shared_storage) ? "true" : "false"); - DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_ASYNC_PUBLISH_ENV, - (m_ctx->async_publish) ? "true" : "false"); - DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_FSYNC_WRITE_ENV, - (m_ctx->fsync_write) ? "true" : "false"); - DYAD_LOG_INFO(m_ctx, "%s=%u", DYAD_KEY_DEPTH_ENV, m_ctx->key_depth); - DYAD_LOG_INFO(m_ctx, "%s=%u", DYAD_KEY_BINS_ENV, m_ctx->key_bins); - DYAD_LOG_INFO(m_ctx, "%s=%u", DYAD_SERVICE_MUX_ENV, m_ctx->service_mux); - DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_KVS_NAMESPACE_ENV, m_ctx->kvs_namespace); - DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_DTL_MODE_ENV, getenv(DYAD_DTL_MODE_ENV)); +void dyad_stream_core::log_info (const std::string &msg_head) const +{ + DYAD_CPP_FUNCTION (); + DYAD_LOG_INFO (m_ctx, "=== %s ===", msg_head.c_str ()); + DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_PATH_CONSUMER_ENV, m_ctx->cons_managed_path); + DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_PATH_PRODUCER_ENV, m_ctx->prod_managed_path); + DYAD_LOG_INFO (m_ctx, + "%s=%s", + DYAD_PATH_RELATIVE_ENV, + (m_ctx->relative_to_managed_path) ? "true" : "false"); + DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_SYNC_DEBUG_ENV, (m_ctx->debug) ? "true" : "false"); + DYAD_LOG_INFO (m_ctx, + "%s=%s", + DYAD_SHARED_STORAGE_ENV, + (m_ctx->shared_storage) ? "true" : "false"); + DYAD_LOG_INFO (m_ctx, + "%s=%s", + DYAD_ASYNC_PUBLISH_ENV, + (m_ctx->async_publish) ? "true" : "false"); + DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_FSYNC_WRITE_ENV, (m_ctx->fsync_write) ? "true" : "false"); + DYAD_LOG_INFO (m_ctx, "%s=%u", DYAD_KEY_DEPTH_ENV, m_ctx->key_depth); + DYAD_LOG_INFO (m_ctx, "%s=%u", DYAD_KEY_BINS_ENV, m_ctx->key_bins); + DYAD_LOG_INFO (m_ctx, "%s=%u", DYAD_SERVICE_MUX_ENV, m_ctx->service_mux); + DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_KVS_NAMESPACE_ENV, m_ctx->kvs_namespace); + DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_DTL_MODE_ENV, getenv (DYAD_DTL_MODE_ENV)); } -bool dyad_stream_core::is_dyad_producer() const { return m_is_prod; } +bool dyad_stream_core::is_dyad_producer () const +{ + return m_is_prod; +} -bool dyad_stream_core::is_dyad_consumer() const { return m_is_cons; } +bool dyad_stream_core::is_dyad_consumer () const +{ + return m_is_cons; +} -bool dyad_stream_core::open_sync(const char *path) { - DYAD_CPP_FUNCTION(); - DYAD_CPP_FUNCTION_UPDATE("path", path); - DYAD_LOG_DEBUG(m_ctx, "DYAD_SYNC OPEN: enters sync (\"%s\").", path); - if (!m_initialized) { - // TODO log - return true; - } +bool dyad_stream_core::open_sync (const char *path) +{ + DYAD_CPP_FUNCTION (); + DYAD_CPP_FUNCTION_UPDATE ("path", path); + DYAD_LOG_DEBUG (m_ctx, "DYAD_SYNC OPEN: enters sync (\"%s\").", path); + if (!m_initialized) { + // TODO log + return true; + } - if (!is_dyad_consumer()) { - return true; - } + if (!is_dyad_consumer ()) { + return true; + } - dyad_rc_t rc = dyad_consume(m_ctx_mutable, path); + dyad_rc_t rc = dyad_consume (m_ctx_mutable, path); - if (DYAD_IS_ERROR(rc)) { - DPRINTF(m_ctx, "DYAD_SYNC OPEN: failed sync (\"%s\").", path); - return false; - } + if (DYAD_IS_ERROR (rc)) { + DPRINTF (m_ctx, "DYAD_SYNC OPEN: failed sync (\"%s\").", path); + return false; + } - DYAD_LOG_DEBUG(m_ctx, "DYAD_SYNC OEPN: exists sync (\"%s\").\n", path); - return true; + DYAD_LOG_DEBUG (m_ctx, "DYAD_SYNC OEPN: exists sync (\"%s\").\n", path); + return true; } -bool dyad_stream_core::close_sync(const char *path) { - DYAD_CPP_FUNCTION(); - DYAD_CPP_FUNCTION_UPDATE("path", path); - DYAD_LOG_DEBUG(m_ctx, "DYAD_SYNC CLOSE: enters sync (\"%s\").\n", path); - if (!m_initialized) { - // TODO log - return true; - } +bool dyad_stream_core::close_sync (const char *path) +{ + DYAD_CPP_FUNCTION (); + DYAD_CPP_FUNCTION_UPDATE ("path", path); + DYAD_LOG_DEBUG (m_ctx, "DYAD_SYNC CLOSE: enters sync (\"%s\").\n", path); + if (!m_initialized) { + // TODO log + return true; + } - if (!is_dyad_producer()) { - return true; - } + if (!is_dyad_producer ()) { + return true; + } - dyad_rc_t rc = dyad_produce(m_ctx_mutable, path); + dyad_rc_t rc = dyad_produce (m_ctx_mutable, path); - if (DYAD_IS_ERROR(rc)) { - DPRINTF(m_ctx, "DYAD_SYNC CLOSE: failed sync (\"%s\").\n", path); - return false; - } + if (DYAD_IS_ERROR (rc)) { + DPRINTF (m_ctx, "DYAD_SYNC CLOSE: failed sync (\"%s\").\n", path); + return false; + } - DYAD_LOG_DEBUG(m_ctx, "DYAD_SYNC CLOSE: exists sync (\"%s\").\n", path); - return true; + DYAD_LOG_DEBUG (m_ctx, "DYAD_SYNC CLOSE: exists sync (\"%s\").\n", path); + return true; } -void dyad_stream_core::set_initialized() { m_initialized = true; } +void dyad_stream_core::set_initialized () +{ + m_initialized = true; +} -bool dyad_stream_core::chk_initialized() const { return m_initialized; } +bool dyad_stream_core::chk_initialized () const +{ + return m_initialized; +} -bool dyad_stream_core::chk_fsync_write() const { return m_ctx->fsync_write; } +bool dyad_stream_core::chk_fsync_write () const +{ + return m_ctx->fsync_write; +} -int dyad_stream_core::file_lock_exclusive(int fd) const { - struct flock exclusive_flock; +int dyad_stream_core::file_lock_exclusive (int fd) const +{ + struct flock exclusive_flock; - dyad_rc_t rc = dyad_excl_flock(m_ctx, fd, &exclusive_flock); + dyad_rc_t rc = dyad_excl_flock (m_ctx, fd, &exclusive_flock); - if (DYAD_IS_ERROR(rc)) { - dyad_release_flock(m_ctx, fd, &exclusive_flock); - } + if (DYAD_IS_ERROR (rc)) { + dyad_release_flock (m_ctx, fd, &exclusive_flock); + } - return rc; + return rc; } -int dyad_stream_core::file_lock_shared(int fd) const { - struct flock shared_flock; +int dyad_stream_core::file_lock_shared (int fd) const +{ + struct flock shared_flock; - dyad_rc_t rc = dyad_excl_flock(m_ctx, fd, &shared_flock); + dyad_rc_t rc = dyad_excl_flock (m_ctx, fd, &shared_flock); - if (DYAD_IS_ERROR(rc)) { - dyad_release_flock(m_ctx, fd, &shared_flock); - } + if (DYAD_IS_ERROR (rc)) { + dyad_release_flock (m_ctx, fd, &shared_flock); + } - return rc; + return rc; } -int dyad_stream_core::file_unlock(int fd) const { - struct flock a_flock; +int dyad_stream_core::file_unlock (int fd) const +{ + struct flock a_flock; - return dyad_release_flock(m_ctx, fd, &a_flock); + return dyad_release_flock (m_ctx, fd, &a_flock); } -bool dyad_stream_core::cmp_canonical_path_prefix( - bool is_prod, const char *const __restrict__ path) { - memset(upath, '\0', PATH_MAX); - return ::cmp_canonical_path_prefix(m_ctx, is_prod, path, upath, PATH_MAX); +bool dyad_stream_core::cmp_canonical_path_prefix (bool is_prod, const char *const __restrict__ path) +{ + memset (upath, '\0', PATH_MAX); + return ::cmp_canonical_path_prefix (m_ctx, is_prod, path, upath, PATH_MAX); } -std::string dyad_stream_core::get_upath() const { return std::string{upath}; } +std::string dyad_stream_core::get_upath () const +{ + return std::string{upath}; +} -} // end of namespace dyad +} // end of namespace dyad /* * vi: ts=4 sw=4 expandtab diff --git a/src/dyad/utils/base64/base64.c b/src/dyad/utils/base64/base64.c index 5fa7f114..2599287d 100644 --- a/src/dyad/utils/base64/base64.c +++ b/src/dyad/utils/base64/base64.c @@ -59,9 +59,7 @@ size_t base64_encoded_length (size_t srclen) return ((srclen + 2) / 3) * 4; } -void base64_encode_triplet_using_maps (const base64_maps_t *maps, - char dest[4], - const char src[3]) +void base64_encode_triplet_using_maps (const base64_maps_t *maps, char dest[4], const char src[3]) { char a = src[0]; char b = src[1]; diff --git a/src/dyad/utils/base64/base64.h b/src/dyad/utils/base64/base64.h index 898f3d2b..fb4011b0 100644 --- a/src/dyad/utils/base64/base64.h +++ b/src/dyad/utils/base64/base64.h @@ -53,9 +53,7 @@ void base64_init_maps (base64_maps_t *dest, const char src[64]); * @param dest Buffer containing 3 bytes * @param src Buffer containing 4 characters */ -void base64_encode_triplet_using_maps (const base64_maps_t *maps, - char dest[4], - const char src[3]); +void base64_encode_triplet_using_maps (const base64_maps_t *maps, char dest[4], const char src[3]); /** * base64_encode_tail_using_maps - encode the final bytes of a source using a specific @@ -164,10 +162,7 @@ extern const base64_maps_t base64_maps_rfc4648; * encoded_length = base64_encode(dest, sizeof(dest), src, strlen(src)); * printf("Returned data of length %zd @%p", encoded_length, &dest); */ -static inline ssize_t base64_encode (char *dest, - size_t destlen, - const char *src, - size_t srclen) +static inline ssize_t base64_encode (char *dest, size_t destlen, const char *src, size_t srclen) { return base64_encode_using_maps (&base64_maps_rfc4648, dest, destlen, src, srclen); } @@ -214,10 +209,7 @@ static inline void base64_encode_tail (char dest[4], const char *src, size_t src * decoded_length = base64_decode(ret, sizeof(ret), src, strlen(src)); * printf("Returned data of length %zd @%p", decoded_length, &ret); */ -static inline ssize_t base64_decode (char *dest, - size_t destlen, - const char *src, - size_t srclen) +static inline ssize_t base64_decode (char *dest, size_t destlen, const char *src, size_t srclen) { return base64_decode_using_maps (&base64_maps_rfc4648, dest, destlen, src, srclen); } diff --git a/src/dyad/utils/test_cmp_canonical_path_prefix.c b/src/dyad/utils/test_cmp_canonical_path_prefix.c index e71d90ce..48df4970 100644 --- a/src/dyad/utils/test_cmp_canonical_path_prefix.c +++ b/src/dyad/utils/test_cmp_canonical_path_prefix.c @@ -10,52 +10,52 @@ #include #include #include +#include #include // clang-format on -int main(int argc, char **argv) { - char *prefix = NULL; - char *can_prefix = NULL; - char *path = NULL; - bool ret = false; - char upath[PATH_MAX] = {'\0'}; +int main (int argc, char **argv) +{ + char *prefix = NULL; + char *can_prefix = NULL; + char *path = NULL; + bool ret = false; + char upath[PATH_MAX] = {'\0'}; - if (argc < 3 || 4 < argc) { - printf("Usage: %s prefix path [canonical_prefix]\n", argv[0]); - return EXIT_FAILURE; - } - if (argc >= 3) { - prefix = argv[1]; - path = argv[2]; - } - if (argc == 4) { - can_prefix = argv[3]; - } + if (argc < 3 || 4 < argc) { + printf ("Usage: %s prefix path [canonical_prefix]\n", argv[0]); + return EXIT_FAILURE; + } + if (argc >= 3) { + prefix = argv[1]; + path = argv[2]; + } + if (argc == 4) { + can_prefix = argv[3]; + } - const size_t prefix_len = strlen(prefix); - const size_t can_prefix_len = can_prefix ? strlen(can_prefix) : 0u; - ; - uint32_t prefix_hash = hash_str(prefix, DYAD_SEED); - uint32_t can_prefix_hash = hash_str(can_prefix, DYAD_SEED); + const size_t prefix_len = strlen (prefix); + const size_t can_prefix_len = can_prefix ? strlen (can_prefix) : 0u; + ; + uint32_t prefix_hash = hash_str (prefix, DYAD_SEED); + uint32_t can_prefix_hash = hash_str (can_prefix, DYAD_SEED); - dyad_ctx_t ctx; - ctx.cons_managed_path = prefix; - ctx.cons_real_path = can_prefix; - ctx.cons_managed_len = prefix_len; - ctx.cons_real_len = can_prefix_len; - ctx.cons_managed_hash = prefix_hash; - ctx.cons_real_hash = can_prefix_hash; - bool is_prod = false; + dyad_ctx_t ctx; + ctx.cons_managed_path = prefix; + ctx.cons_real_path = can_prefix; + ctx.cons_managed_len = prefix_len; + ctx.cons_real_len = can_prefix_len; + ctx.cons_managed_hash = prefix_hash; + ctx.cons_real_hash = can_prefix_hash; + bool is_prod = false; - ret = cmp_canonical_path_prefix(&ctx, is_prod, path, upath, PATH_MAX); - if (!ret) { - printf("path '%s' does not include prefix '%s' ('%s')\n", path, prefix, - can_prefix); - return EXIT_FAILURE; - } - printf("path '%s' include prefix '%s' (canonical form: '%s')\n", path, prefix, - can_prefix); - printf("'%s' is under '%s'\n", upath, prefix); + ret = cmp_canonical_path_prefix (&ctx, is_prod, path, upath, PATH_MAX); + if (!ret) { + printf ("path '%s' does not include prefix '%s' ('%s')\n", path, prefix, can_prefix); + return EXIT_FAILURE; + } + printf ("path '%s' include prefix '%s' (canonical form: '%s')\n", path, prefix, can_prefix); + printf ("'%s' is under '%s'\n", upath, prefix); - return EXIT_SUCCESS; + return EXIT_SUCCESS; } diff --git a/src/dyad/utils/test_murmur3.c b/src/dyad/utils/test_murmur3.c index 9bb64f78..6c9627fd 100644 --- a/src/dyad/utils/test_murmur3.c +++ b/src/dyad/utils/test_murmur3.c @@ -1,8 +1,9 @@ -#include "dyad/utils/murmur3.h" +#include #include -#include #include -#include +#include + +#include "dyad/utils/murmur3.h" static int gen_path_key (const char* restrict str, char* restrict path_key, @@ -42,7 +43,7 @@ static int gen_path_key (const char* restrict str, MurmurHash3_x64_128 (str_long, str_len, seed, hash); uint32_t bin = (hash[0] ^ hash[1] ^ hash[2] ^ hash[3]) % width; n = snprintf (path_key + cx, len - cx, "%x.", bin); - //n = snprintf (path_key + cx, len - cx, "%x%x%x%x.", hash[0], hash[1], hash[2], hash[3]); + // n = snprintf (path_key + cx, len - cx, "%x%x%x%x.", hash[0], hash[1], hash[2], hash[3]); cx += n; if (cx >= len || n < 0) { return -1; @@ -56,7 +57,6 @@ static int gen_path_key (const char* restrict str, return 0; } - int main (int argc, char** argv) { if (argc < 4) { @@ -67,9 +67,9 @@ int main (int argc, char** argv) int depth = atoi (argv[1]); int width = atoi (argv[2]); for (int i = 3; i < argc; i++) { - char path_key [PATH_MAX + 1] = {'\0'}; + char path_key[PATH_MAX + 1] = {'\0'}; gen_path_key (argv[i], path_key, PATH_MAX, depth, width); - printf("%s\t%s\n", argv[i], path_key); + printf ("%s\t%s\n", argv[i], path_key); } return EXIT_SUCCESS; diff --git a/src/dyad/utils/utils.c b/src/dyad/utils/utils.c index 4efec532..002fc04f 100644 --- a/src/dyad/utils/utils.c +++ b/src/dyad/utils/utils.c @@ -21,41 +21,41 @@ // logger for utils where it does not depend on flux #define DYAD_UTIL_LOGGER 1 -#include - -#include -#include -#include - -#include // open -#include // basename dirname +#include // open +#include // basename dirname +#include #include // open #include // open -#include #include // readlink #if defined(__cplusplus) #include #include #include +#include #include #include #include #include -#include // #include // c++11 #else #include #include // PATH_MAX #include #include +#include #include #include #include #include -#include #endif +#include + +#include +#include +#include + #ifndef DYAD_PATH_DELIM #define DYAD_PATH_DELIM "/" #endif @@ -63,10 +63,12 @@ /// If hashing is not possible, returns 0. Otherwise, returns a non-zero hash value. uint32_t hash_str (const char* str, const uint32_t seed) { - if (!str) return 0u; + if (!str) + return 0u; const char* str_long = str; size_t str_len = strlen (str); - if (str_len == 0ul) return 0u; + if (str_len == 0ul) + return 0u; // Just append the string so that it can be as large as 128 bytes. if (str_len < 128ul) { @@ -85,8 +87,7 @@ uint32_t hash_str (const char* str, const uint32_t seed) /** If hashing is not possible, returns 0. Otherwise, returns a non-zero hash value. * This only hashes the prefix of a given length */ -uint32_t hash_path_prefix (const char* str, const uint32_t seed, - const size_t len) +uint32_t hash_path_prefix (const char* str, const uint32_t seed, const size_t len) { if (!str || len == 0ul) { return 0u; @@ -94,7 +95,8 @@ uint32_t hash_path_prefix (const char* str, const uint32_t seed, const char* str_long = str; size_t str_len = strlen (str); - if (str_len < len) return 0u; + if (str_len < len) + return 0u; str_len = len; // Just append the string so that it can be as large as 128 bytes. @@ -139,9 +141,8 @@ char* concat_str (char* __restrict__ str, const size_t str_len = (con_end ? (str_len_org - con_len) : str_len_org); const char* const str_end = str + str_capacity; - bool no_overlap = - ((to_append + strlen (to_append) <= str) || (str_end <= to_append)) - && ((connector + strlen (connector) <= str) || (str_end <= connector)); + bool no_overlap = ((to_append + strlen (to_append) <= str) || (str_end <= to_append)) + && ((connector + strlen (connector) <= str) || (str_end <= connector)); if (!no_overlap) { DYAD_LOG_DEBUG (NULL, "DYAD UTIL: buffers overlap.\n"); @@ -188,10 +189,9 @@ bool extract_user_path (const char* __restrict__ prefix, { const char* const upath_end = upath + upath_capacity; - bool no_overlap = - ((prefix + prefix_len <= upath) || (upath_end <= prefix)) - && ((full + full_len <= upath) || (upath_end <= full)) - && ((delim + delim_len <= upath) || (upath_end <= delim)); + bool no_overlap = ((prefix + prefix_len <= upath) || (upath_end <= prefix)) + && ((full + full_len <= upath) || (upath_end <= full)) + && ((delim + delim_len <= upath) || (upath_end <= delim)); if (!no_overlap) { DYAD_LOG_DEBUG (NULL, "DYAD UTIL: buffers overlap.\n"); @@ -257,7 +257,7 @@ bool cmp_canonical_path_prefix (const dyad_ctx_t* __restrict__ ctx, const size_t upath_capacity) { // Only works when there are no multiple absolute paths via hardlinks - char can_path[PATH_MAX] = {'\0'}; // canonical form of the given path + char can_path[PATH_MAX] = {'\0'}; // canonical form of the given path if (!ctx) { DYAD_LOG_DEBUG (NULL, "DYAD UTIL: Invalid dyad context!\n"); return false; @@ -285,11 +285,10 @@ bool cmp_canonical_path_prefix (const dyad_ctx_t* __restrict__ ctx, can_prefix_hash = ctx->cons_real_hash; } - const uint32_t path_hash1 = hash_path_prefix (path, DYAD_SEED, prefix_len); - if ((path_hash1 == prefix_hash) && - extract_user_path (prefix, path, DYAD_PATH_DELIM, upath, upath_capacity)) { + if ((path_hash1 == prefix_hash) + && extract_user_path (prefix, path, DYAD_PATH_DELIM, upath, upath_capacity)) { return true; } @@ -311,15 +310,15 @@ bool cmp_canonical_path_prefix (const dyad_ctx_t* __restrict__ ctx, const uint32_t can_path_hash1 = hash_path_prefix (can_path, DYAD_SEED, prefix_len); - if ((can_path_hash1 == prefix_hash) && - extract_user_path (prefix, can_path, DYAD_PATH_DELIM, upath, upath_capacity)) { + if ((can_path_hash1 == prefix_hash) + && extract_user_path (prefix, can_path, DYAD_PATH_DELIM, upath, upath_capacity)) { return true; } if (can_prefix_len > 0u) { const uint32_t can_path_hash2 = hash_path_prefix (can_path, DYAD_SEED, can_prefix_len); - if ((can_path_hash2 == can_prefix_hash) && - extract_user_path (can_prefix, can_path, DYAD_PATH_DELIM, upath, upath_capacity)) { + if ((can_path_hash2 == can_prefix_hash) + && extract_user_path (can_prefix, can_path, DYAD_PATH_DELIM, upath, upath_capacity)) { return true; } } @@ -365,12 +364,12 @@ int mkdir_as_needed (const char* path, const mode_t m) return -2; } else if ((sb.st_mode & RWX_UGO) ^ (m & RWX_UGO)) { DYAD_LOG_DEBUG (NULL, - "Directory \"%s\" already exists with " - "different permission bits %o from " - "the requested %o\n", - path, - (sb.st_mode & RWX_UGO), - (m & RWX_UGO)); + "Directory \"%s\" already exists with " + "different permission bits %o from " + "the requested %o\n", + path, + (sb.st_mode & RWX_UGO), + (m & RWX_UGO)); return 5; // already exists but with different mode } return 1; // already exists @@ -381,16 +380,16 @@ int mkdir_as_needed (const char* path, const mode_t m) if (mkpath (path, m) != 0) { if (stat (path, &sb) == 0) { // already exist if (S_ISDIR (sb.st_mode) == 0) { // not a directory - DYAD_LOG_DEBUG (NULL,"\"%s\" already exists not as a directory\n", path); + DYAD_LOG_DEBUG (NULL, "\"%s\" already exists not as a directory\n", path); return -4; } else if ((sb.st_mode & RWX_UGO) ^ (m & RWX_UGO)) { DYAD_LOG_DEBUG (NULL, - "Directory \"%s\" already exists with " - "different permission bits %o from " - "the requested %o\n", - path, - (sb.st_mode & RWX_UGO), - (m & RWX_UGO)); + "Directory \"%s\" already exists with " + "different permission bits %o from " + "the requested %o\n", + path, + (sb.st_mode & RWX_UGO), + (m & RWX_UGO)); return 5; // already exists but with different mode } return 1; // already exists @@ -420,9 +419,10 @@ int get_path (const int fd, const size_t max_size, char* path) sprintf (proclink, "/proc/self/fd/%d", fd); ssize_t rc = readlink (proclink, path, max_size); if (rc < (ssize_t)0) { - DYAD_LOG_DEBUG (NULL, "DYAD UTIL: error reading the file link (%s): %s\n", - strerror (errno), - proclink); + DYAD_LOG_DEBUG (NULL, + "DYAD UTIL: error reading the file link (%s): %s\n", + strerror (errno), + proclink); return -1; } else if ((size_t)rc == max_size) { DYAD_LOG_DEBUG (NULL, "DYAD UTIL: truncation might have happend with %s\n", proclink); @@ -495,14 +495,19 @@ ssize_t get_file_size (int fd) return file_size; } -dyad_rc_t dyad_excl_flock (const dyad_ctx_t* __restrict__ ctx, int fd, +dyad_rc_t dyad_excl_flock (const dyad_ctx_t* __restrict__ ctx, + int fd, struct flock* __restrict__ lock) { dyad_rc_t rc = DYAD_RC_OK; - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_INT ("fd", fd); - DYAD_LOG_DEBUG (ctx, "[node %u rank %u pid %d] Applies an exclusive lock on fd %d\n", \ - ctx->node_idx, ctx->rank, ctx->pid, fd); + DYAD_LOG_DEBUG (ctx, + "[node %u rank %u pid %d] Applies an exclusive lock on fd %d\n", + ctx->node_idx, + ctx->rank, + ctx->pid, + fd); if (!lock) { rc = DYAD_RC_BADFIO; goto excl_flock_end; @@ -511,28 +516,37 @@ dyad_rc_t dyad_excl_flock (const dyad_ctx_t* __restrict__ ctx, int fd, lock->l_whence = SEEK_SET; lock->l_start = 0; lock->l_len = 0; - lock->l_pid = ctx->pid; //getpid(); - if (fcntl (fd, F_SETLKW, lock) == -1) { // will wait until able to lock + lock->l_pid = ctx->pid; // getpid(); + if (fcntl (fd, F_SETLKW, lock) == -1) { // will wait until able to lock DYAD_LOG_ERROR (ctx, "Cannot apply exclusive lock on fd %d\n", fd); rc = DYAD_RC_BADFIO; goto excl_flock_end; } - DYAD_LOG_DEBUG (ctx, "[node %u rank %u pid %d] Exclusive lock placed on fd %d\n", \ - ctx->node_idx, ctx->rank, ctx->pid, fd); + DYAD_LOG_DEBUG (ctx, + "[node %u rank %u pid %d] Exclusive lock placed on fd %d\n", + ctx->node_idx, + ctx->rank, + ctx->pid, + fd); rc = DYAD_RC_OK; excl_flock_end:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -dyad_rc_t dyad_shared_flock (const dyad_ctx_t* __restrict__ ctx, int fd, +dyad_rc_t dyad_shared_flock (const dyad_ctx_t* __restrict__ ctx, + int fd, struct flock* __restrict__ lock) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_INT ("fd", fd); dyad_rc_t rc = DYAD_RC_OK; - DYAD_LOG_DEBUG (ctx, "[node %u rank %u pid %d] Applies a shared lock on fd %d\n", \ - ctx->node_idx, ctx->rank, ctx->pid, fd); + DYAD_LOG_DEBUG (ctx, + "[node %u rank %u pid %d] Applies a shared lock on fd %d\n", + ctx->node_idx, + ctx->rank, + ctx->pid, + fd); if (!lock) { rc = DYAD_RC_BADFIO; goto shared_flock_end; @@ -541,43 +555,56 @@ dyad_rc_t dyad_shared_flock (const dyad_ctx_t* __restrict__ ctx, int fd, lock->l_whence = SEEK_SET; lock->l_start = 0; lock->l_len = 0; - lock->l_pid = ctx->pid; //getpid(); - if (fcntl (fd, F_SETLKW, lock) == -1) { // will wait until able to lock + lock->l_pid = ctx->pid; // getpid(); + if (fcntl (fd, F_SETLKW, lock) == -1) { // will wait until able to lock DYAD_LOG_ERROR (ctx, "Cannot apply shared lock on fd %d\n", fd); rc = DYAD_RC_BADFIO; goto shared_flock_end; } - DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] Shared lock placed on fd %d\n", \ - ctx->node_idx, ctx->rank, ctx->pid, fd); + DYAD_LOG_INFO (ctx, + "[node %u rank %u pid %d] Shared lock placed on fd %d\n", + ctx->node_idx, + ctx->rank, + ctx->pid, + fd); rc = DYAD_RC_OK; shared_flock_end:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } -dyad_rc_t dyad_release_flock (const dyad_ctx_t* __restrict__ ctx, int fd, +dyad_rc_t dyad_release_flock (const dyad_ctx_t* __restrict__ ctx, + int fd, struct flock* __restrict__ lock) { - DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_START (); DYAD_C_FUNCTION_UPDATE_INT ("fd", fd); dyad_rc_t rc = DYAD_RC_OK; - DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] Releases a lock on fd %d\n", \ - ctx->node_idx, ctx->rank, ctx->pid, fd); + DYAD_LOG_INFO (ctx, + "[node %u rank %u pid %d] Releases a lock on fd %d\n", + ctx->node_idx, + ctx->rank, + ctx->pid, + fd); if (!lock) { rc = DYAD_RC_BADFIO; goto release_flock_end; } lock->l_type = F_UNLCK; - if (fcntl (fd, F_SETLKW, lock) == -1) { // will just unlock + if (fcntl (fd, F_SETLKW, lock) == -1) { // will just unlock DYAD_LOG_ERROR (ctx, "Cannot release lock on fd %d\n", fd); rc = DYAD_RC_BADFIO; goto release_flock_end; } - DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] lock lifted from fd %d\n", \ - ctx->node_idx, ctx->rank, ctx->pid, fd); + DYAD_LOG_INFO (ctx, + "[node %u rank %u pid %d] lock lifted from fd %d\n", + ctx->node_idx, + ctx->rank, + ctx->pid, + fd); rc = DYAD_RC_OK; release_flock_end:; - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_END (); return rc; } @@ -597,10 +624,7 @@ int sync_containing_dir (const char* path) if (dir_fd < 0) { char errmsg[PATH_MAX + 256] = {'\0'}; - snprintf (errmsg, - PATH_MAX + 256, - "Failed to open directory %s\n", - containing_dir); + snprintf (errmsg, PATH_MAX + 256, "Failed to open directory %s\n", containing_dir); perror (errmsg); return -1; // exit (SYS_ERR); } diff --git a/src/dyad/utils/utils.h b/src/dyad/utils/utils.h index e259269e..86a648ad 100644 --- a/src/dyad/utils/utils.h +++ b/src/dyad/utils/utils.h @@ -19,7 +19,7 @@ #ifndef _GNU_SOURCE #define _GNU_SOURCE -#endif // _GNU_SOURCE +#endif // _GNU_SOURCE #if defined(__cplusplus) // #include // c++11 @@ -31,7 +31,7 @@ #include #include #include -#endif // defined(__cplusplus) +#endif // defined(__cplusplus) #include #include @@ -39,66 +39,71 @@ #if defined(__cplusplus) extern "C" { -#endif // defined(__cplusplus) +#endif // defined(__cplusplus) -bool file_in_read_mode(FILE *f); -bool fd_in_read_mode(int fd); -bool oflag_is_read(int oflag); -bool mode_is_read(const char *mode); +bool file_in_read_mode (FILE *f); +bool fd_in_read_mode (int fd); +bool oflag_is_read (int oflag); +bool mode_is_read (const char *mode); -void enable_debug_dyad_utils(void); -void disable_debug_dyad_utils(void); -bool check_debug_dyad_utils(void); +void enable_debug_dyad_utils (void); +void disable_debug_dyad_utils (void); +bool check_debug_dyad_utils (void); /** Return the hash value computed for the entire string with a given seed * The return value 0 means that the string was not valid or the length was 0. */ -uint32_t hash_str(const char *str, const uint32_t seed); +uint32_t hash_str (const char *str, const uint32_t seed); /** Return the hash value computed for the path string up to len characters * with a given seed. * The return value 0 means that the string was not valid or the length was 0. */ -uint32_t hash_path_prefix(const char *str, const uint32_t seed, - const size_t len); +uint32_t hash_path_prefix (const char *str, const uint32_t seed, const size_t len); -char *concat_str(char *__restrict__ str, const char *__restrict__ to_append, - const char *__restrict__ connector, size_t str_capacity); +char *concat_str (char *__restrict__ str, + const char *__restrict__ to_append, + const char *__restrict__ connector, + size_t str_capacity); -bool extract_user_path(const char *__restrict__ prefix, - const char *__restrict__ full, - const char *__restrict__ delim, char *__restrict__ upath, - const size_t upath_capacity); +bool extract_user_path (const char *__restrict__ prefix, + const char *__restrict__ full, + const char *__restrict__ delim, + char *__restrict__ upath, + const size_t upath_capacity); -bool cmp_canonical_path_prefix(const dyad_ctx_t *__restrict__ ctx, - const bool is_prod, - const char *__restrict__ path, - char *__restrict__ upath, - const size_t upath_capacity); +bool cmp_canonical_path_prefix (const dyad_ctx_t *__restrict__ ctx, + const bool is_prod, + const char *__restrict__ path, + char *__restrict__ upath, + const size_t upath_capacity); -int mkdir_as_needed(const char *path, const mode_t m); +int mkdir_as_needed (const char *path, const mode_t m); /// Obtain path from the file descriptor -int get_path(const int fd, const size_t max_size, char *path); +int get_path (const int fd, const size_t max_size, char *path); /// Check if the path is a directory -bool is_path_dir(const char *path); +bool is_path_dir (const char *path); /// Check if the file identified by the file descriptor is a directory -bool is_fd_dir(int fd); +bool is_fd_dir (int fd); #if DYAD_SPIN_WAIT /// Try access file info -bool get_stat(const char *path, unsigned int max_retry, long ns_sleep); -#endif // DYAD_SPIN_WAIT +bool get_stat (const char *path, unsigned int max_retry, long ns_sleep); +#endif // DYAD_SPIN_WAIT -ssize_t get_file_size(int fd); +ssize_t get_file_size (int fd); -dyad_rc_t dyad_excl_flock(const dyad_ctx_t *__restrict__ ctx, int fd, - struct flock *__restrict__ lock); -dyad_rc_t dyad_shared_flock(const dyad_ctx_t *__restrict__ ctx, int fd, - struct flock *__restrict__ lock); -dyad_rc_t dyad_release_flock(const dyad_ctx_t *__restrict__ ctx, int fd, +dyad_rc_t dyad_excl_flock (const dyad_ctx_t *__restrict__ ctx, + int fd, + struct flock *__restrict__ lock); +dyad_rc_t dyad_shared_flock (const dyad_ctx_t *__restrict__ ctx, + int fd, struct flock *__restrict__ lock); +dyad_rc_t dyad_release_flock (const dyad_ctx_t *__restrict__ ctx, + int fd, + struct flock *__restrict__ lock); #if DYAD_SYNC_DIR /** @@ -106,11 +111,11 @@ dyad_rc_t dyad_release_flock(const dyad_ctx_t *__restrict__ ctx, int fd, * For example, if path is "/a/b", then fsync on "/a". * This cannot be used with DYAD interception. */ -int sync_containing_dir(const char *path); -#endif // DYAD_SYNC_DIR +int sync_containing_dir (const char *path); +#endif // DYAD_SYNC_DIR #if defined(__cplusplus) } -#endif // defined(__cplusplus) +#endif // defined(__cplusplus) -#endif // DYAD_UTILS_UTILS_H +#endif // DYAD_UTILS_UTILS_H diff --git a/src/dyad/wrapper/flux_barrier.c b/src/dyad/wrapper/flux_barrier.c index d55ee230..e44e8266 100644 --- a/src/dyad/wrapper/flux_barrier.c +++ b/src/dyad/wrapper/flux_barrier.c @@ -8,15 +8,13 @@ * SPDX-License-Identifier: LGPL-3.0 \************************************************************/ - +#include #include #include #include #include -#include #include #include -#include #if DYAD_FULL_DEBUG #define IPRINTF DPRINTF #define IPRINTF_DEFINED @@ -94,12 +92,12 @@ int main (int argc, char** argv) flux_future_destroy (fb); -// struct timespec t_now; -// clock_gettime (CLOCK_MONOTONIC_RAW, &t_now); -// char tbuf[100]; -// strftime (tbuf, sizeof (tbuf), "%D %T", gmtime (&(t_now.tv_sec))); + // struct timespec t_now; + // clock_gettime (CLOCK_MONOTONIC_RAW, &t_now); + // char tbuf[100]; + // strftime (tbuf, sizeof (tbuf), "%D %T", gmtime (&(t_now.tv_sec))); -// FLUX_LOG_INFO ("Synchronized at %s.%09ld", tbuf, t_now.tv_nsec); + // FLUX_LOG_INFO ("Synchronized at %s.%09ld", tbuf, t_now.tv_nsec); return EXIT_SUCCESS; } From fc7ba50d9b3715d012989532545ba144a35a6cc9 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Mon, 16 Dec 2024 18:30:24 -0500 Subject: [PATCH 10/11] Fixes formatting after rebase --- include/dyad/client/dyad_client.h | 26 +- include/dyad/common/dyad_structures.h | 2 +- src/dyad/client/dyad_client.c | 1689 +++++++++++++------------ src/dyad/client/dyad_client_int.h | 64 +- src/dyad/common/dyad_structures_int.h | 71 +- src/dyad/wrapper/wrapper.c | 573 ++++----- 6 files changed, 1224 insertions(+), 1201 deletions(-) diff --git a/include/dyad/client/dyad_client.h b/include/dyad/client/dyad_client.h index d0304f0e..7d1d0e5c 100644 --- a/include/dyad/client/dyad_client.h +++ b/include/dyad/client/dyad_client.h @@ -28,14 +28,14 @@ extern "C" { #endif #if DYAD_PERFFLOW -#define DYAD_CORE_FUNC_MODS __attribute__((annotate("@critical_path()"))) static +#define DYAD_CORE_FUNC_MODS __attribute__ ((annotate ("@critical_path()"))) static #else #define DYAD_CORE_FUNC_MODS static inline #endif struct dyad_metadata { - char *fpath; - uint32_t owner_rank; + char *fpath; + uint32_t owner_rank; }; typedef struct dyad_metadata dyad_metadata_t; @@ -47,8 +47,7 @@ typedef struct dyad_metadata dyad_metadata_t; * * @return An error code from dyad_rc.h */ -DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_produce(dyad_ctx_t *ctx, - const char *fname); +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_produce (dyad_ctx_t *ctx, const char *fname); /** * @brief Obtain DYAD metadata for a file in the consumer-managed directory @@ -61,12 +60,12 @@ DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_produce(dyad_ctx_t *ctx, * * @return An error code from dyad_rc.h */ -DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t -dyad_get_metadata(dyad_ctx_t *ctx, const char *fname, bool should_wait, - dyad_metadata_t **mdata); +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_get_metadata (dyad_ctx_t *ctx, + const char *fname, + bool should_wait, + dyad_metadata_t **mdata); -DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t -dyad_free_metadata(dyad_metadata_t **mdata); +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_free_metadata (dyad_metadata_t **mdata); /** * @brief Wrapper function that performs all the common tasks needed @@ -76,8 +75,7 @@ dyad_free_metadata(dyad_metadata_t **mdata); * * @return An error code from dyad_rc.h */ -DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_consume(dyad_ctx_t *ctx, - const char *fname); +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_consume (dyad_ctx_t *ctx, const char *fname); /** * @brief Wrapper function that performs all the common tasks needed @@ -89,8 +87,8 @@ DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_consume(dyad_ctx_t *ctx, * * @return An error code from dyad_rc.h */ -DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_consume_w_metadata( - dyad_ctx_t *ctx, const char *fname, const dyad_metadata_t *mdata); +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t +dyad_consume_w_metadata (dyad_ctx_t *ctx, const char *fname, const dyad_metadata_t *mdata); #ifdef __cplusplus } diff --git a/include/dyad/common/dyad_structures.h b/include/dyad/common/dyad_structures.h index cdf96370..1afdf337 100644 --- a/include/dyad/common/dyad_structures.h +++ b/include/dyad/common/dyad_structures.h @@ -10,4 +10,4 @@ struct dyad_ctx; typedef struct dyad_ctx dyad_ctx_t; -#endif /* DYAD_COMMON_STRUCTURES_H */ \ No newline at end of file +#endif /* DYAD_COMMON_STRUCTURES_H */ diff --git a/src/dyad/client/dyad_client.c b/src/dyad/client/dyad_client.c index b6537920..a47b7900 100644 --- a/src/dyad/client/dyad_client.c +++ b/src/dyad/client/dyad_client.c @@ -29,535 +29,549 @@ #include #endif -DYAD_DLL_EXPORTED int gen_path_key(const char *restrict str, - char *restrict path_key, const size_t len, - const uint32_t depth, const uint32_t width) { - DYAD_C_FUNCTION_START(); - static const uint32_t seeds[10] = {104677u, 104681u, 104683u, 104693u, - 104701u, 104707u, 104711u, 104717u, - 104723u, 104729u}; - - uint32_t seed = 57u; - uint32_t hash[4] = {0u}; // Output for the hash - size_t cx = 0ul; - int n = 0; - if (str == NULL || path_key == NULL || len == 0ul) { - DYAD_C_FUNCTION_END(); - return -1; - } - size_t str_len = strlen(str); - if (str_len == 0ul) { - DYAD_C_FUNCTION_END(); - return -1; - } - const char *str_long = str; - - path_key[0] = '\0'; - - // Just append the string so that it can be as large as 128 bytes. - if (str_len < 128ul) { - char buf[256] = {'\0'}; - memcpy(buf, str, str_len); - memset(buf + str_len, '@', 128ul - str_len); - buf[128u] = '\0'; - str_len = 128ul; - str_long = buf; - } - - for (uint32_t d = 0u; d < depth; d++) { - seed += seeds[d % 10]; - MurmurHash3_x64_128(str_long, str_len, seed, hash); - uint32_t bin = (hash[0] ^ hash[1] ^ hash[2] ^ hash[3]) % width; - n = snprintf(path_key + cx, len - cx, "%x.", bin); - cx += n; - if (cx >= len || n < 0) { - DYAD_C_FUNCTION_END(); - return -1; - } - } - n = snprintf(path_key + cx, len - cx, "%s", str); - // FIXME: cx + n >= len fails for str_len > 256 - if (n < 0) { - DYAD_C_FUNCTION_END(); - return -1; - } - DYAD_C_FUNCTION_UPDATE_STR("path_key", path_key); - DYAD_C_FUNCTION_END(); - return 0; +DYAD_DLL_EXPORTED int gen_path_key (const char *restrict str, + char *restrict path_key, + const size_t len, + const uint32_t depth, + const uint32_t width) +{ + DYAD_C_FUNCTION_START (); + static const uint32_t seeds[10] = + {104677u, 104681u, 104683u, 104693u, 104701u, 104707u, 104711u, 104717u, 104723u, 104729u}; + + uint32_t seed = 57u; + uint32_t hash[4] = {0u}; // Output for the hash + size_t cx = 0ul; + int n = 0; + if (str == NULL || path_key == NULL || len == 0ul) { + DYAD_C_FUNCTION_END (); + return -1; + } + size_t str_len = strlen (str); + if (str_len == 0ul) { + DYAD_C_FUNCTION_END (); + return -1; + } + const char *str_long = str; + + path_key[0] = '\0'; + + // Just append the string so that it can be as large as 128 bytes. + if (str_len < 128ul) { + char buf[256] = {'\0'}; + memcpy (buf, str, str_len); + memset (buf + str_len, '@', 128ul - str_len); + buf[128u] = '\0'; + str_len = 128ul; + str_long = buf; + } + + for (uint32_t d = 0u; d < depth; d++) { + seed += seeds[d % 10]; + MurmurHash3_x64_128 (str_long, str_len, seed, hash); + uint32_t bin = (hash[0] ^ hash[1] ^ hash[2] ^ hash[3]) % width; + n = snprintf (path_key + cx, len - cx, "%x.", bin); + cx += n; + if (cx >= len || n < 0) { + DYAD_C_FUNCTION_END (); + return -1; + } + } + n = snprintf (path_key + cx, len - cx, "%s", str); + // FIXME: cx + n >= len fails for str_len > 256 + if (n < 0) { + DYAD_C_FUNCTION_END (); + return -1; + } + DYAD_C_FUNCTION_UPDATE_STR ("path_key", path_key); + DYAD_C_FUNCTION_END (); + return 0; } -static void future_cleanup_cb(flux_future_t *f, void *arg) { - if (flux_future_get(f, NULL) < 0) { - DYAD_LOG_STDERR("future_cleanup: future error detected with.%s", ""); - } - flux_future_destroy(f); +static void future_cleanup_cb (flux_future_t *f, void *arg) +{ + if (flux_future_get (f, NULL) < 0) { + DYAD_LOG_STDERR ("future_cleanup: future error detected with.%s", ""); + } + flux_future_destroy (f); } -DYAD_CORE_FUNC_MODS dyad_rc_t dyad_kvs_commit(const dyad_ctx_t *restrict ctx, - flux_kvs_txn_t *restrict txn) { - DYAD_C_FUNCTION_START(); - flux_future_t *f = NULL; - dyad_rc_t rc = DYAD_RC_OK; - DYAD_LOG_INFO(ctx, "Committing transaction to KVS"); - // Commit the transaction to the Flux KVS - f = flux_kvs_commit((flux_t *)ctx->h, ctx->kvs_namespace, 0, txn); - // If the commit failed, log an error and return DYAD_BADCOMMIT - if (f == NULL) { - DYAD_LOG_ERROR(ctx, "Could not commit transaction to Flux KVS"); - rc = DYAD_RC_BADCOMMIT; - goto kvs_commit_region_finish; - } - if (ctx->async_publish) { - if (flux_future_then(f, -1, future_cleanup_cb, NULL) < 0) { - DYAD_LOG_ERROR(ctx, "Error with flux_future_then"); - } - } else { - // If the commit is pending, wait for it to complete - flux_future_wait_for(f, -1.0); - // Once the commit is complete, destroy the future and transaction - flux_future_destroy(f); - f = NULL; - } - rc = DYAD_RC_OK; +DYAD_CORE_FUNC_MODS dyad_rc_t dyad_kvs_commit (const dyad_ctx_t *restrict ctx, + flux_kvs_txn_t *restrict txn) +{ + DYAD_C_FUNCTION_START (); + flux_future_t *f = NULL; + dyad_rc_t rc = DYAD_RC_OK; + DYAD_LOG_INFO (ctx, "Committing transaction to KVS"); + // Commit the transaction to the Flux KVS + f = flux_kvs_commit ((flux_t *)ctx->h, ctx->kvs_namespace, 0, txn); + // If the commit failed, log an error and return DYAD_BADCOMMIT + if (f == NULL) { + DYAD_LOG_ERROR (ctx, "Could not commit transaction to Flux KVS"); + rc = DYAD_RC_BADCOMMIT; + goto kvs_commit_region_finish; + } + if (ctx->async_publish) { + if (flux_future_then (f, -1, future_cleanup_cb, NULL) < 0) { + DYAD_LOG_ERROR (ctx, "Error with flux_future_then"); + } + } else { + // If the commit is pending, wait for it to complete + flux_future_wait_for (f, -1.0); + // Once the commit is complete, destroy the future and transaction + flux_future_destroy (f); + f = NULL; + } + rc = DYAD_RC_OK; kvs_commit_region_finish:; - DYAD_C_FUNCTION_END(); - return rc; + DYAD_C_FUNCTION_END (); + return rc; } -DYAD_CORE_FUNC_MODS dyad_rc_t publish_via_flux(const dyad_ctx_t *restrict ctx, - const char *restrict upath) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR("fname", ctx->fname); - DYAD_C_FUNCTION_UPDATE_STR("upath", upath); - dyad_rc_t rc = DYAD_RC_OK; - flux_kvs_txn_t *txn = NULL; - const size_t topic_len = PATH_MAX; - char topic[PATH_MAX + 1] = {'\0'}; - memset(topic, 0, topic_len + 1); - memset(topic, '\0', topic_len + 1); - // Generate the KVS key from the file path relative to - // the producer-managed directory - DYAD_LOG_INFO(ctx, "Generating KVS key from path (%s)", upath); - gen_path_key(upath, topic, topic_len, ctx->key_depth, ctx->key_bins); - // Crete and pack a Flux KVS transaction. - // The transaction will contain a single key-value pair - // with the previously generated key as the key and the - // producer's rank as the value - DYAD_LOG_INFO(ctx, "Creating KVS transaction under the key %s", topic); - txn = flux_kvs_txn_create(); - if (txn == NULL) { - DYAD_LOG_ERROR(ctx, "Could not create Flux KVS transaction"); - rc = DYAD_RC_FLUXFAIL; - goto publish_done; - } - if (flux_kvs_txn_pack(txn, 0, topic, "i", ctx->rank) < 0) { - DYAD_LOG_ERROR(ctx, "Could not pack Flux KVS transaction"); - rc = DYAD_RC_FLUXFAIL; - goto publish_done; - } - // Call dyad_kvs_commit to commit the transaction into the Flux KVS - rc = dyad_kvs_commit(ctx, txn); - // If dyad_kvs_commit failed, log an error and forward the return code - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, "dyad_kvs_commit failed!"); - goto publish_done; - } - rc = DYAD_RC_OK; +DYAD_CORE_FUNC_MODS dyad_rc_t publish_via_flux (const dyad_ctx_t *restrict ctx, + const char *restrict upath) +{ + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_UPDATE_STR ("fname", ctx->fname); + DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); + dyad_rc_t rc = DYAD_RC_OK; + flux_kvs_txn_t *txn = NULL; + const size_t topic_len = PATH_MAX; + char topic[PATH_MAX + 1] = {'\0'}; + memset (topic, 0, topic_len + 1); + memset (topic, '\0', topic_len + 1); + // Generate the KVS key from the file path relative to + // the producer-managed directory + DYAD_LOG_INFO (ctx, "Generating KVS key from path (%s)", upath); + gen_path_key (upath, topic, topic_len, ctx->key_depth, ctx->key_bins); + // Crete and pack a Flux KVS transaction. + // The transaction will contain a single key-value pair + // with the previously generated key as the key and the + // producer's rank as the value + DYAD_LOG_INFO (ctx, "Creating KVS transaction under the key %s", topic); + txn = flux_kvs_txn_create (); + if (txn == NULL) { + DYAD_LOG_ERROR (ctx, "Could not create Flux KVS transaction"); + rc = DYAD_RC_FLUXFAIL; + goto publish_done; + } + if (flux_kvs_txn_pack (txn, 0, topic, "i", ctx->rank) < 0) { + DYAD_LOG_ERROR (ctx, "Could not pack Flux KVS transaction"); + rc = DYAD_RC_FLUXFAIL; + goto publish_done; + } + // Call dyad_kvs_commit to commit the transaction into the Flux KVS + rc = dyad_kvs_commit (ctx, txn); + // If dyad_kvs_commit failed, log an error and forward the return code + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, "dyad_kvs_commit failed!"); + goto publish_done; + } + rc = DYAD_RC_OK; publish_done:; - if (txn != NULL) { - flux_kvs_txn_destroy(txn); - } - DYAD_C_FUNCTION_END(); - return rc; + if (txn != NULL) { + flux_kvs_txn_destroy (txn); + } + DYAD_C_FUNCTION_END (); + return rc; } -DYAD_DLL_EXPORTED dyad_rc_t dyad_commit(dyad_ctx_t *restrict ctx, - const char *restrict fname) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR("fname", ctx->fname); - dyad_rc_t rc = DYAD_RC_OK; - char upath[PATH_MAX + 1] = {'\0'}; +DYAD_DLL_EXPORTED dyad_rc_t dyad_commit (dyad_ctx_t *restrict ctx, const char *restrict fname) +{ + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_UPDATE_STR ("fname", ctx->fname); + dyad_rc_t rc = DYAD_RC_OK; + char upath[PATH_MAX + 1] = {'\0'}; #if 0 if (fname == NULL || strlen (fname) > PATH_MAX) { rc = DYAD_RC_SYSFAIL; goto get_metadata_done; } #endif - // As this is a function called for DYAD producer, ctx->prod_managed_path - // must be a valid string (!NULL). ctx->delim_len is verified to be greater - // than 0 during initialization. - if (ctx->relative_to_managed_path && - //(strlen (fname) > 0ul) && // checked where get_path() was - (strncmp(fname, DYAD_PATH_DELIM, ctx->delim_len) != - 0)) { // fname is a relative path that is relative to the - // prod_managed_path - memcpy(upath, fname, strlen(fname)); - } else if (!cmp_canonical_path_prefix(ctx, true, fname, upath, PATH_MAX)) { - // Extract the path to the file specified by fname relative to the - // producer-managed path - // This relative path will be stored in upath - DYAD_LOG_INFO(ctx, "%s is not in the Producer's managed path", fname); - rc = DYAD_RC_OK; - goto commit_done; - } - DYAD_C_FUNCTION_UPDATE_STR("upath", upath); - DYAD_LOG_INFO(ctx, "Obtained file path relative to producer directory: %s", - upath); - // Call publish_via_flux to actually store information about the file into - // the Flux KVS - // Fence this call with reassignments of reenter so that, if intercepting - // file I/O API calls, we will not get stuck in infinite recursion - ctx->reenter = false; - rc = publish_via_flux(ctx, upath); - ctx->reenter = true; + // As this is a function called for DYAD producer, ctx->prod_managed_path + // must be a valid string (!NULL). ctx->delim_len is verified to be greater + // than 0 during initialization. + if (ctx->relative_to_managed_path && + //(strlen (fname) > 0ul) && // checked where get_path() was + (strncmp (fname, DYAD_PATH_DELIM, ctx->delim_len) + != 0)) { // fname is a relative path that is relative to the + // prod_managed_path + memcpy (upath, fname, strlen (fname)); + } else if (!cmp_canonical_path_prefix (ctx, true, fname, upath, PATH_MAX)) { + // Extract the path to the file specified by fname relative to the + // producer-managed path + // This relative path will be stored in upath + DYAD_LOG_INFO (ctx, "%s is not in the Producer's managed path", fname); + rc = DYAD_RC_OK; + goto commit_done; + } + DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); + DYAD_LOG_INFO (ctx, "Obtained file path relative to producer directory: %s", upath); + // Call publish_via_flux to actually store information about the file into + // the Flux KVS + // Fence this call with reassignments of reenter so that, if intercepting + // file I/O API calls, we will not get stuck in infinite recursion + ctx->reenter = false; + rc = publish_via_flux (ctx, upath); + ctx->reenter = true; commit_done:; - // If "check" is set and the operation was successful, set the - // DYAD_CHECK_ENV environment variable to "ok" - if (rc == DYAD_RC_OK && (ctx && ctx->check)) { - setenv(DYAD_CHECK_ENV, "ok", 1); - } - DYAD_C_FUNCTION_END(); - return rc; + // If "check" is set and the operation was successful, set the + // DYAD_CHECK_ENV environment variable to "ok" + if (rc == DYAD_RC_OK && (ctx && ctx->check)) { + setenv (DYAD_CHECK_ENV, "ok", 1); + } + DYAD_C_FUNCTION_END (); + return rc; } -static void print_mdata(const dyad_ctx_t *restrict ctx, - const dyad_metadata_t *restrict mdata) { - if (mdata == NULL) { - DYAD_LOG_INFO(ctx, "Cannot print a NULL metadata object!"); - } else { - DYAD_LOG_INFO(ctx, "Printing contents of DYAD Metadata object"); - DYAD_LOG_INFO(ctx, "fpath = %s", mdata->fpath); - DYAD_LOG_INFO(ctx, "owner_rank = %u", mdata->owner_rank); - } +static void print_mdata (const dyad_ctx_t *restrict ctx, const dyad_metadata_t *restrict mdata) +{ + if (mdata == NULL) { + DYAD_LOG_INFO (ctx, "Cannot print a NULL metadata object!"); + } else { + DYAD_LOG_INFO (ctx, "Printing contents of DYAD Metadata object"); + DYAD_LOG_INFO (ctx, "fpath = %s", mdata->fpath); + DYAD_LOG_INFO (ctx, "owner_rank = %u", mdata->owner_rank); + } } -DYAD_DLL_EXPORTED dyad_rc_t dyad_kvs_read(const dyad_ctx_t *restrict ctx, - const char *restrict topic, - const char *restrict upath, - bool should_wait, - dyad_metadata_t **restrict mdata) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR("upath", upath); - dyad_rc_t rc = DYAD_RC_OK; - int kvs_lookup_flags = 0; - flux_future_t *f = NULL; - if (mdata == NULL) { - DYAD_LOG_ERROR(ctx, "Metadata double pointer is NULL. " +DYAD_DLL_EXPORTED dyad_rc_t dyad_kvs_read (const dyad_ctx_t *restrict ctx, + const char *restrict topic, + const char *restrict upath, + bool should_wait, + dyad_metadata_t **restrict mdata) +{ + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); + dyad_rc_t rc = DYAD_RC_OK; + int kvs_lookup_flags = 0; + flux_future_t *f = NULL; + if (mdata == NULL) { + DYAD_LOG_ERROR (ctx, + "Metadata double pointer is NULL. " "Cannot correctly create metadata object"); - rc = DYAD_RC_NOTFOUND; - goto kvs_read_end; - } - // Lookup information about the desired file (represented by kvs_topic) - // from the Flux KVS. If there is no information, wait for it to be - // made available - if (should_wait) - kvs_lookup_flags = FLUX_KVS_WAITCREATE; - DYAD_LOG_INFO(ctx, "Retrieving information from KVS under the key %s", topic); - f = flux_kvs_lookup((flux_t *)ctx->h, ctx->kvs_namespace, kvs_lookup_flags, - topic); - // If the KVS lookup failed, log an error and return DYAD_BADLOOKUP - if (f == NULL) { - DYAD_LOG_ERROR(ctx, "KVS lookup failed!\n"); - rc = DYAD_RC_NOTFOUND; - goto kvs_read_end; - } - // Extract the rank of the producer from the KVS response - DYAD_LOG_INFO(ctx, "Building metadata object from KVS entry\n"); - if (*mdata != NULL) { - DYAD_LOG_INFO(ctx, - "Metadata object is already allocated. Skipping allocation"); - } else { - *mdata = (dyad_metadata_t *)malloc(sizeof(struct dyad_metadata)); - if (*mdata == NULL) { - DYAD_LOG_ERROR(ctx, "Cannot allocate memory for metadata object"); - rc = DYAD_RC_SYSFAIL; - goto kvs_read_end; - } - } - size_t upath_len = strlen(upath); - (*mdata)->fpath = (char *)malloc(upath_len + 1); - if ((*mdata)->fpath == NULL) { - DYAD_LOG_ERROR(ctx, "Cannot allocate memory for fpath in metadata object"); - rc = DYAD_RC_SYSFAIL; - goto kvs_read_end; - } - memset((*mdata)->fpath, '\0', upath_len + 1); - memcpy((*mdata)->fpath, upath, upath_len); - rc = flux_kvs_lookup_get_unpack(f, "i", &((*mdata)->owner_rank)); - // If the extraction did not work, log an error and return DYAD_BADFETCH - if (rc < 0) { - DYAD_LOG_ERROR(ctx, "Could not unpack owner's rank from KVS response\n"); - rc = DYAD_RC_BADMETADATA; - goto kvs_read_end; - } - DYAD_LOG_INFO(ctx, "Successfully created DYAD Metadata object"); - print_mdata(ctx, *mdata); - DYAD_C_FUNCTION_UPDATE_STR("fpath", (*mdata)->fpath); - DYAD_C_FUNCTION_UPDATE_INT("owner_rank", (*mdata)->owner_rank); - rc = DYAD_RC_OK; + rc = DYAD_RC_NOTFOUND; + goto kvs_read_end; + } + // Lookup information about the desired file (represented by kvs_topic) + // from the Flux KVS. If there is no information, wait for it to be + // made available + if (should_wait) + kvs_lookup_flags = FLUX_KVS_WAITCREATE; + DYAD_LOG_INFO (ctx, "Retrieving information from KVS under the key %s", topic); + f = flux_kvs_lookup ((flux_t *)ctx->h, ctx->kvs_namespace, kvs_lookup_flags, topic); + // If the KVS lookup failed, log an error and return DYAD_BADLOOKUP + if (f == NULL) { + DYAD_LOG_ERROR (ctx, "KVS lookup failed!\n"); + rc = DYAD_RC_NOTFOUND; + goto kvs_read_end; + } + // Extract the rank of the producer from the KVS response + DYAD_LOG_INFO (ctx, "Building metadata object from KVS entry\n"); + if (*mdata != NULL) { + DYAD_LOG_INFO (ctx, "Metadata object is already allocated. Skipping allocation"); + } else { + *mdata = (dyad_metadata_t *)malloc (sizeof (struct dyad_metadata)); + if (*mdata == NULL) { + DYAD_LOG_ERROR (ctx, "Cannot allocate memory for metadata object"); + rc = DYAD_RC_SYSFAIL; + goto kvs_read_end; + } + } + size_t upath_len = strlen (upath); + (*mdata)->fpath = (char *)malloc (upath_len + 1); + if ((*mdata)->fpath == NULL) { + DYAD_LOG_ERROR (ctx, "Cannot allocate memory for fpath in metadata object"); + rc = DYAD_RC_SYSFAIL; + goto kvs_read_end; + } + memset ((*mdata)->fpath, '\0', upath_len + 1); + memcpy ((*mdata)->fpath, upath, upath_len); + rc = flux_kvs_lookup_get_unpack (f, "i", &((*mdata)->owner_rank)); + // If the extraction did not work, log an error and return DYAD_BADFETCH + if (rc < 0) { + DYAD_LOG_ERROR (ctx, "Could not unpack owner's rank from KVS response\n"); + rc = DYAD_RC_BADMETADATA; + goto kvs_read_end; + } + DYAD_LOG_INFO (ctx, "Successfully created DYAD Metadata object"); + print_mdata (ctx, *mdata); + DYAD_C_FUNCTION_UPDATE_STR ("fpath", (*mdata)->fpath); + DYAD_C_FUNCTION_UPDATE_INT ("owner_rank", (*mdata)->owner_rank); + rc = DYAD_RC_OK; kvs_read_end:; - if (DYAD_IS_ERROR(rc) && mdata != NULL && *mdata != NULL) { - dyad_free_metadata(mdata); - } - if (f != NULL) { - flux_future_destroy(f); - f = NULL; - } - DYAD_C_FUNCTION_END(); - return rc; + if (DYAD_IS_ERROR (rc) && mdata != NULL && *mdata != NULL) { + dyad_free_metadata (mdata); + } + if (f != NULL) { + flux_future_destroy (f); + f = NULL; + } + DYAD_C_FUNCTION_END (); + return rc; } -DYAD_CORE_FUNC_MODS dyad_rc_t dyad_fetch_metadata( - const dyad_ctx_t *restrict ctx, const char *restrict fname, - const char *restrict upath, dyad_metadata_t **restrict mdata) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR("fname", fname); - dyad_rc_t rc = DYAD_RC_OK; - const size_t topic_len = PATH_MAX; - char topic[PATH_MAX + 1] = {'\0'}; - *mdata = NULL; +DYAD_CORE_FUNC_MODS dyad_rc_t dyad_fetch_metadata (const dyad_ctx_t *restrict ctx, + const char *restrict fname, + const char *restrict upath, + dyad_metadata_t **restrict mdata) +{ + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_UPDATE_STR ("fname", fname); + dyad_rc_t rc = DYAD_RC_OK; + const size_t topic_len = PATH_MAX; + char topic[PATH_MAX + 1] = {'\0'}; + *mdata = NULL; #if 0 if (fname == NULL || upath == NULL || strlen (fname) == 0ul || strlen (upath) == 0ul) { rc = DYAD_RC_BADFIO; goto get_metadata_done; } #endif - // Set reenter to false to avoid recursively performing DYAD operations - DYAD_LOG_INFO(ctx, "Obtained file path relative to consumer directory: %s\n", - upath); - DYAD_C_FUNCTION_UPDATE_STR("upath", upath); - // Generate the KVS key from the file path relative to - // the consumer-managed directory - gen_path_key(upath, topic, topic_len, ctx->key_depth, ctx->key_bins); - DYAD_LOG_INFO(ctx, "Generated KVS key for consumer: %s\n", topic); - // Call dyad_kvs_read to retrieve infromation about the file - // from the Flux KVS - rc = dyad_kvs_read(ctx, topic, upath, true, mdata); - // If an error occured in dyad_kvs_read, log it and propagate the return - // code - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, "dyad_kvs_read failed!\n"); - goto fetch_done; - } - // There are two cases where we do not want to perform file transfer: - // 1. if the shared storage feature is enabled - // 2. if the producer and consumer share the same rank - // In either of these cases, skip the creation of the dyad_kvs_response_t - // object, and return DYAD_OK. This will cause the file transfer step to be - // skipped - DYAD_C_FUNCTION_UPDATE_INT("owner_rank", (*mdata)->owner_rank); - DYAD_C_FUNCTION_UPDATE_INT("node_idx", ctx->node_idx); - if (((*mdata)->owner_rank / ctx->service_mux) == ctx->node_idx) { - DYAD_LOG_INFO( - ctx, - "Either shared-storage is indicated or the producer rank (%u) is the" - " same as the consumer rank (%u)", - (*mdata)->owner_rank, ctx->rank); - if (mdata != NULL && *mdata != NULL) { - dyad_free_metadata(mdata); + // Set reenter to false to avoid recursively performing DYAD operations + DYAD_LOG_INFO (ctx, "Obtained file path relative to consumer directory: %s\n", upath); + DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); + // Generate the KVS key from the file path relative to + // the consumer-managed directory + gen_path_key (upath, topic, topic_len, ctx->key_depth, ctx->key_bins); + DYAD_LOG_INFO (ctx, "Generated KVS key for consumer: %s\n", topic); + // Call dyad_kvs_read to retrieve infromation about the file + // from the Flux KVS + rc = dyad_kvs_read (ctx, topic, upath, true, mdata); + // If an error occured in dyad_kvs_read, log it and propagate the return + // code + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, "dyad_kvs_read failed!\n"); + goto fetch_done; + } + // There are two cases where we do not want to perform file transfer: + // 1. if the shared storage feature is enabled + // 2. if the producer and consumer share the same rank + // In either of these cases, skip the creation of the dyad_kvs_response_t + // object, and return DYAD_OK. This will cause the file transfer step to be + // skipped + DYAD_C_FUNCTION_UPDATE_INT ("owner_rank", (*mdata)->owner_rank); + DYAD_C_FUNCTION_UPDATE_INT ("node_idx", ctx->node_idx); + if (((*mdata)->owner_rank / ctx->service_mux) == ctx->node_idx) { + DYAD_LOG_INFO (ctx, + "Either shared-storage is indicated or the producer rank (%u) is the" + " same as the consumer rank (%u)", + (*mdata)->owner_rank, + ctx->rank); + if (mdata != NULL && *mdata != NULL) { + dyad_free_metadata (mdata); + } + rc = DYAD_RC_OK; + DYAD_C_FUNCTION_UPDATE_INT ("is_local", 1); + goto fetch_done; } + DYAD_C_FUNCTION_UPDATE_INT ("is_local", 0); rc = DYAD_RC_OK; - DYAD_C_FUNCTION_UPDATE_INT("is_local", 1); - goto fetch_done; - } - DYAD_C_FUNCTION_UPDATE_INT("is_local", 0); - rc = DYAD_RC_OK; fetch_done:; - DYAD_C_FUNCTION_END(); - return rc; + DYAD_C_FUNCTION_END (); + return rc; } -DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data(const dyad_ctx_t *restrict ctx, - const dyad_metadata_t *restrict mdata, - char **restrict file_data, - size_t *restrict file_len) { - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - flux_future_t *f = NULL; - json_t *rpc_payload = NULL; - DYAD_LOG_INFO(ctx, "Packing payload for RPC to DYAD module"); - DYAD_C_FUNCTION_UPDATE_INT("owner_rank", mdata->owner_rank); - DYAD_C_FUNCTION_UPDATE_STR("fpath", mdata->fpath); - rc = ctx->dtl_handle->rpc_pack(ctx, mdata->fpath, mdata->owner_rank, - &rpc_payload); - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, "Cannot create JSON payload for Flux RPC to " +DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data (const dyad_ctx_t *restrict ctx, + const dyad_metadata_t *restrict mdata, + char **restrict file_data, + size_t *restrict file_len) +{ + DYAD_C_FUNCTION_START (); + dyad_rc_t rc = DYAD_RC_OK; + flux_future_t *f = NULL; + json_t *rpc_payload = NULL; + DYAD_LOG_INFO (ctx, "Packing payload for RPC to DYAD module"); + DYAD_C_FUNCTION_UPDATE_INT ("owner_rank", mdata->owner_rank); + DYAD_C_FUNCTION_UPDATE_STR ("fpath", mdata->fpath); + rc = ctx->dtl_handle->rpc_pack (ctx, mdata->fpath, mdata->owner_rank, &rpc_payload); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, + "Cannot create JSON payload for Flux RPC to " "DYAD module\n"); - goto get_done; - } - DYAD_LOG_INFO(ctx, "Sending payload for RPC to DYAD module"); - f = flux_rpc_pack((flux_t *)ctx->h, DYAD_DTL_RPC_NAME, mdata->owner_rank, - FLUX_RPC_STREAMING, "o", rpc_payload); - if (f == NULL) { - DYAD_LOG_ERROR(ctx, "Cannot send RPC to producer module\n"); - rc = DYAD_RC_BADRPC; - goto get_done; - } - DYAD_LOG_INFO(ctx, "Receive RPC response from DYAD module"); - rc = ctx->dtl_handle->rpc_recv_response(ctx, f); - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, "Cannot receive and/or parse the RPC response\n"); - goto get_done; - } - DYAD_LOG_INFO(ctx, "Establish DTL connection with DYAD module"); - rc = ctx->dtl_handle->establish_connection(ctx); - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, - "Cannot establish connection with DYAD module on broker " - "%u\n", - mdata->owner_rank); - goto get_done; - } - DYAD_LOG_INFO(ctx, "Receive file data via DTL"); - rc = ctx->dtl_handle->recv(ctx, (void **)file_data, file_len); - DYAD_LOG_INFO(ctx, "Close DTL connection with DYAD module"); - ctx->dtl_handle->close_connection(ctx); - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, "Cannot receive data from producer module\n"); - goto get_done; - } - DYAD_C_FUNCTION_UPDATE_INT("file_len", *file_len); - - rc = DYAD_RC_OK; + goto get_done; + } + DYAD_LOG_INFO (ctx, "Sending payload for RPC to DYAD module"); + f = flux_rpc_pack ((flux_t *)ctx->h, + DYAD_DTL_RPC_NAME, + mdata->owner_rank, + FLUX_RPC_STREAMING, + "o", + rpc_payload); + if (f == NULL) { + DYAD_LOG_ERROR (ctx, "Cannot send RPC to producer module\n"); + rc = DYAD_RC_BADRPC; + goto get_done; + } + DYAD_LOG_INFO (ctx, "Receive RPC response from DYAD module"); + rc = ctx->dtl_handle->rpc_recv_response (ctx, f); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, "Cannot receive and/or parse the RPC response\n"); + goto get_done; + } + DYAD_LOG_INFO (ctx, "Establish DTL connection with DYAD module"); + rc = ctx->dtl_handle->establish_connection (ctx); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, + "Cannot establish connection with DYAD module on broker " + "%u\n", + mdata->owner_rank); + goto get_done; + } + DYAD_LOG_INFO (ctx, "Receive file data via DTL"); + rc = ctx->dtl_handle->recv (ctx, (void **)file_data, file_len); + DYAD_LOG_INFO (ctx, "Close DTL connection with DYAD module"); + ctx->dtl_handle->close_connection (ctx); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, "Cannot receive data from producer module\n"); + goto get_done; + } + DYAD_C_FUNCTION_UPDATE_INT ("file_len", *file_len); + + rc = DYAD_RC_OK; get_done:; - // There are two return codes that have special meaning when coming from the - // DTL: - // * DYAD_RC_RPC_FINISHED: occurs when an ENODATA error occurs - // * DYAD_RC_BADRPC: occurs when a previous RPC operation fails - // In either of these cases, we do not need to wait for the end of stream - // because the RPC is already completely messed up. If we do not have either - // of these cases, we will wait for one more RPC message. If everything went - // well in the module, this last message will set errno to ENODATA (i.e., - // end of stream). Otherwise, something went wrong, so we'll return - // DYAD_RC_BADRPC. - DYAD_LOG_INFO( - ctx, "Wait for end-of-stream message from module (current RC = %d)\n", - rc); - if (rc != DYAD_RC_RPC_FINISHED && rc != DYAD_RC_BADRPC) { - if (!(flux_rpc_get(f, NULL) < 0 && errno == ENODATA)) { - DYAD_LOG_ERROR(ctx, - "An error occured at end of getting data! Either the " - "module sent too many responses, or the module " - "failed with a bad error (errno = %d)\n", - errno); - rc = DYAD_RC_BADRPC; - } - } + // There are two return codes that have special meaning when coming from the + // DTL: + // * DYAD_RC_RPC_FINISHED: occurs when an ENODATA error occurs + // * DYAD_RC_BADRPC: occurs when a previous RPC operation fails + // In either of these cases, we do not need to wait for the end of stream + // because the RPC is already completely messed up. If we do not have either + // of these cases, we will wait for one more RPC message. If everything went + // well in the module, this last message will set errno to ENODATA (i.e., + // end of stream). Otherwise, something went wrong, so we'll return + // DYAD_RC_BADRPC. + DYAD_LOG_INFO (ctx, "Wait for end-of-stream message from module (current RC = %d)\n", rc); + if (rc != DYAD_RC_RPC_FINISHED && rc != DYAD_RC_BADRPC) { + if (!(flux_rpc_get (f, NULL) < 0 && errno == ENODATA)) { + DYAD_LOG_ERROR (ctx, + "An error occured at end of getting data! Either the " + "module sent too many responses, or the module " + "failed with a bad error (errno = %d)\n", + errno); + rc = DYAD_RC_BADRPC; + } + } #ifdef DYAD_ENABLE_UCX_RMA - ctx->dtl_handle->get_buffer(ctx, 0, (void **)file_data); - ssize_t read_len = 0l; - memcpy(&read_len, *file_data, sizeof(read_len)); - if (read_len < 0l) { - *file_len = 0ul; - DYAD_LOG_DEBUG(ctx, "Not able to read from %s file", mdata->fpath); - rc = DYAD_RC_BADFIO; - } else { - *file_len = (size_t)read_len; - } - *file_data = ((char *)*file_data) + sizeof(read_len); - DYAD_LOG_INFO(ctx, "Read %zd bytes from %s file", *file_len, mdata->fpath); + ctx->dtl_handle->get_buffer (ctx, 0, (void **)file_data); + ssize_t read_len = 0l; + memcpy (&read_len, *file_data, sizeof (read_len)); + if (read_len < 0l) { + *file_len = 0ul; + DYAD_LOG_DEBUG (ctx, "Not able to read from %s file", mdata->fpath); + rc = DYAD_RC_BADFIO; + } else { + *file_len = (size_t)read_len; + } + *file_data = ((char *)*file_data) + sizeof (read_len); + DYAD_LOG_INFO (ctx, "Read %zd bytes from %s file", *file_len, mdata->fpath); #endif - DYAD_LOG_INFO(ctx, "Destroy the Flux future for the RPC\n"); - flux_future_destroy(f); - DYAD_C_FUNCTION_END(); - return rc; + DYAD_LOG_INFO (ctx, "Destroy the Flux future for the RPC\n"); + flux_future_destroy (f); + DYAD_C_FUNCTION_END (); + return rc; } -DYAD_CORE_FUNC_MODS dyad_rc_t dyad_cons_store( - const dyad_ctx_t *restrict ctx, const dyad_metadata_t *restrict mdata, - int fd, const size_t data_len, char *restrict file_data) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_INT("fd", fd); - dyad_rc_t rc = DYAD_RC_OK; - const char *odir = NULL; - char file_path[PATH_MAX + 1] = {'\0'}; - char file_path_copy[PATH_MAX + 1] = {'\0'}; - mode_t m = (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID); - size_t written_len = 0; - memset(file_path, 0, PATH_MAX + 1); - memset(file_path_copy, 0, PATH_MAX + 1); - - // Build the full path to the file being consumed - strncpy(file_path, ctx->cons_managed_path, PATH_MAX - 1); - concat_str(file_path, mdata->fpath, "/", PATH_MAX); - memcpy(file_path_copy, file_path, PATH_MAX); // dirname modifies the arg - DYAD_C_FUNCTION_UPDATE_STR("cons_managed_path", ctx->cons_managed_path); - DYAD_C_FUNCTION_UPDATE_STR("fpath", mdata->fpath); - DYAD_C_FUNCTION_UPDATE_STR("file_path_copy", file_path_copy); - - DYAD_LOG_INFO(ctx, "Saving retrieved data to %s\n", file_path); - // Create the directory as needed - // TODO: Need to be consistent with the mode at the source - odir = dirname(file_path_copy); - if ((strncmp(odir, ".", strlen(".")) != 0) && - (mkdir_as_needed(odir, m) < 0)) { - DYAD_LOG_ERROR(ctx, "Cannot create needed directories for pulled file\n"); - rc = DYAD_RC_BADFIO; - goto pull_done; - } - - // Write the file contents to the location specified by the user - written_len = write(fd, file_data, data_len); - if (written_len != data_len) { - DYAD_LOG_ERROR(ctx, "cons store write of pulled file failed!\n"); - rc = DYAD_RC_BADFIO; - goto pull_done; - } - DYAD_C_FUNCTION_UPDATE_INT("data_len", data_len); - rc = DYAD_RC_OK; +DYAD_CORE_FUNC_MODS dyad_rc_t dyad_cons_store (const dyad_ctx_t *restrict ctx, + const dyad_metadata_t *restrict mdata, + int fd, + const size_t data_len, + char *restrict file_data) +{ + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_UPDATE_INT ("fd", fd); + dyad_rc_t rc = DYAD_RC_OK; + const char *odir = NULL; + char file_path[PATH_MAX + 1] = {'\0'}; + char file_path_copy[PATH_MAX + 1] = {'\0'}; + mode_t m = (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID); + size_t written_len = 0; + memset (file_path, 0, PATH_MAX + 1); + memset (file_path_copy, 0, PATH_MAX + 1); + + // Build the full path to the file being consumed + strncpy (file_path, ctx->cons_managed_path, PATH_MAX - 1); + concat_str (file_path, mdata->fpath, "/", PATH_MAX); + memcpy (file_path_copy, file_path, PATH_MAX); // dirname modifies the arg + DYAD_C_FUNCTION_UPDATE_STR ("cons_managed_path", ctx->cons_managed_path); + DYAD_C_FUNCTION_UPDATE_STR ("fpath", mdata->fpath); + DYAD_C_FUNCTION_UPDATE_STR ("file_path_copy", file_path_copy); + + DYAD_LOG_INFO (ctx, "Saving retrieved data to %s\n", file_path); + // Create the directory as needed + // TODO: Need to be consistent with the mode at the source + odir = dirname (file_path_copy); + if ((strncmp (odir, ".", strlen (".")) != 0) && (mkdir_as_needed (odir, m) < 0)) { + DYAD_LOG_ERROR (ctx, "Cannot create needed directories for pulled file\n"); + rc = DYAD_RC_BADFIO; + goto pull_done; + } + + // Write the file contents to the location specified by the user + written_len = write (fd, file_data, data_len); + if (written_len != data_len) { + DYAD_LOG_ERROR (ctx, "cons store write of pulled file failed!\n"); + rc = DYAD_RC_BADFIO; + goto pull_done; + } + DYAD_C_FUNCTION_UPDATE_INT ("data_len", data_len); + rc = DYAD_RC_OK; pull_done:; - // If "check" is set and the operation was successful, set the - // DYAD_CHECK_ENV environment variable to "ok" - if (rc == DYAD_RC_OK && (ctx && ctx->check)) - setenv(DYAD_CHECK_ENV, "ok", 1); - DYAD_C_FUNCTION_END(); - return rc; + // If "check" is set and the operation was successful, set the + // DYAD_CHECK_ENV environment variable to "ok" + if (rc == DYAD_RC_OK && (ctx && ctx->check)) + setenv (DYAD_CHECK_ENV, "ok", 1); + DYAD_C_FUNCTION_END (); + return rc; } -dyad_rc_t dyad_produce(dyad_ctx_t *restrict ctx, const char *restrict fname) { - DYAD_C_FUNCTION_START(); - ctx->fname = fname; - DYAD_C_FUNCTION_UPDATE_STR("fname", ctx->fname); - DYAD_LOG_DEBUG(ctx, "Executing dyad_produce"); - dyad_rc_t rc = DYAD_RC_OK; - // If the context is not defined, then it is not valid. - // So, return DYAD_NOCTX - if (!ctx || !ctx->h) { - DYAD_LOG_ERROR(ctx, "No CTX found in dyad_produce"); - rc = DYAD_RC_NOCTX; - goto produce_done; - } - // If the producer-managed path is NULL or empty, then the context is not - // valid for a producer operation. So, return DYAD_BADMANAGEDPATH - if (ctx->prod_managed_path == NULL) { - DYAD_LOG_ERROR(ctx, "No or empty producer managed path was found %s", - ctx->prod_managed_path); - rc = DYAD_RC_BADMANAGEDPATH; - goto produce_done; - } - // If the context is valid, call dyad_commit to perform - // the producer operation - rc = dyad_commit(ctx, fname); +dyad_rc_t dyad_produce (dyad_ctx_t *restrict ctx, const char *restrict fname) +{ + DYAD_C_FUNCTION_START (); + ctx->fname = fname; + DYAD_C_FUNCTION_UPDATE_STR ("fname", ctx->fname); + DYAD_LOG_DEBUG (ctx, "Executing dyad_produce"); + dyad_rc_t rc = DYAD_RC_OK; + // If the context is not defined, then it is not valid. + // So, return DYAD_NOCTX + if (!ctx || !ctx->h) { + DYAD_LOG_ERROR (ctx, "No CTX found in dyad_produce"); + rc = DYAD_RC_NOCTX; + goto produce_done; + } + // If the producer-managed path is NULL or empty, then the context is not + // valid for a producer operation. So, return DYAD_BADMANAGEDPATH + if (ctx->prod_managed_path == NULL) { + DYAD_LOG_ERROR (ctx, + "No or empty producer managed path was found %s", + ctx->prod_managed_path); + rc = DYAD_RC_BADMANAGEDPATH; + goto produce_done; + } + // If the context is valid, call dyad_commit to perform + // the producer operation + rc = dyad_commit (ctx, fname); produce_done:; - DYAD_C_FUNCTION_END(); - return rc; + DYAD_C_FUNCTION_END (); + return rc; } /** This function is coupled with Python API. This populates `mdata' which * is used by `dyad_consume_w_metadata ()' */ -dyad_rc_t dyad_get_metadata(dyad_ctx_t *restrict ctx, - const char *restrict fname, bool should_wait, - dyad_metadata_t **restrict mdata) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR("fname", fname); - DYAD_C_FUNCTION_UPDATE_INT("should_wait", should_wait); - dyad_rc_t rc = DYAD_RC_OK; +dyad_rc_t dyad_get_metadata (dyad_ctx_t *restrict ctx, + const char *restrict fname, + bool should_wait, + dyad_metadata_t **restrict mdata) +{ + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_UPDATE_STR ("fname", fname); + DYAD_C_FUNCTION_UPDATE_INT ("should_wait", should_wait); + dyad_rc_t rc = DYAD_RC_OK; #if 0 if (fname == NULL || strlen (fname) > PATH_MAX) { @@ -565,392 +579,399 @@ dyad_rc_t dyad_get_metadata(dyad_ctx_t *restrict ctx, goto get_metadata_done; } #endif - const size_t fname_len = strlen(fname); - char upath[PATH_MAX + 1] = {'\0'}; - - DYAD_LOG_INFO(ctx, "Obtaining file path relative to consumer directory: %s", - upath); - - if (fname_len == 0ul) { - rc = DYAD_RC_BADFIO; - goto get_metadata_done; - } - if (ctx->relative_to_managed_path && - (strncmp(fname, DYAD_PATH_DELIM, ctx->delim_len) != - 0)) { // fname is a relative path that is relative to the - // cons_managed_path - memcpy(upath, fname, fname_len); - } else if (!cmp_canonical_path_prefix(ctx, false, fname, upath, PATH_MAX)) { - // Extract the path to the file specified by fname relative to the - // producer-managed path - // This relative path will be stored in upath - // DYAD_LOG_TRACE (ctx, "%s is not in the Consumer's managed path\n", - // fname); - // NOTE: This is different from what dyad_fetch/commit returns, - // which is DYAD_RC_OK such that dyad does not interfere accesses on - // non-managed directories. - rc = DYAD_RC_UNTRACKED; - goto get_metadata_done; - } - ctx->reenter = false; - DYAD_C_FUNCTION_UPDATE_STR("upath", upath); - - // check if file exist locally, if so skip kvs - int fd = open(fname, O_RDONLY); - if (fd != -1) { - close(fd); - if (mdata == NULL) { - DYAD_LOG_ERROR(ctx, "Metadata double pointer is NULL. " - "Cannot correctly create metadata object"); - rc = DYAD_RC_NOTFOUND; - goto get_metadata_done; + const size_t fname_len = strlen (fname); + char upath[PATH_MAX + 1] = {'\0'}; + + DYAD_LOG_INFO (ctx, "Obtaining file path relative to consumer directory: %s", upath); + + if (fname_len == 0ul) { + rc = DYAD_RC_BADFIO; + goto get_metadata_done; } - if (*mdata != NULL) { - DYAD_LOG_INFO( - ctx, "Metadata object is already allocated. Skipping allocation"); - } else { - *mdata = (dyad_metadata_t *)malloc(sizeof(struct dyad_metadata)); - if (*mdata == NULL) { - DYAD_LOG_ERROR(ctx, "Cannot allocate memory for metadata object"); - rc = DYAD_RC_SYSFAIL; + if (ctx->relative_to_managed_path + && (strncmp (fname, DYAD_PATH_DELIM, ctx->delim_len) + != 0)) { // fname is a relative path that is relative to the + // cons_managed_path + memcpy (upath, fname, fname_len); + } else if (!cmp_canonical_path_prefix (ctx, false, fname, upath, PATH_MAX)) { + // Extract the path to the file specified by fname relative to the + // producer-managed path + // This relative path will be stored in upath + // DYAD_LOG_TRACE (ctx, "%s is not in the Consumer's managed path\n", + // fname); + // NOTE: This is different from what dyad_fetch/commit returns, + // which is DYAD_RC_OK such that dyad does not interfere accesses on + // non-managed directories. + rc = DYAD_RC_UNTRACKED; + goto get_metadata_done; + } + ctx->reenter = false; + DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); + + // check if file exist locally, if so skip kvs + int fd = open (fname, O_RDONLY); + if (fd != -1) { + close (fd); + if (mdata == NULL) { + DYAD_LOG_ERROR (ctx, + "Metadata double pointer is NULL. " + "Cannot correctly create metadata object"); + rc = DYAD_RC_NOTFOUND; + goto get_metadata_done; + } + if (*mdata != NULL) { + DYAD_LOG_INFO (ctx, "Metadata object is already allocated. Skipping allocation"); + } else { + *mdata = (dyad_metadata_t *)malloc (sizeof (struct dyad_metadata)); + if (*mdata == NULL) { + DYAD_LOG_ERROR (ctx, "Cannot allocate memory for metadata object"); + rc = DYAD_RC_SYSFAIL; + goto get_metadata_done; + } + } + (*mdata)->fpath = (char *)malloc (fname_len + 1); + if ((*mdata)->fpath == NULL) { + DYAD_LOG_ERROR (ctx, "Cannot allocate memory for fpath in metadata object"); + rc = DYAD_RC_SYSFAIL; + goto get_metadata_done; + } + memset ((*mdata)->fpath, '\0', fname_len + 1); + memcpy ((*mdata)->fpath, fname, fname_len); + (*mdata)->owner_rank = ctx->rank; + rc = DYAD_RC_OK; + goto get_metadata_done; + } + + const size_t topic_len = PATH_MAX; + char topic[PATH_MAX + 1] = {'\0'}; + DYAD_LOG_INFO (ctx, "Generating KVS key: %s", topic); + gen_path_key (upath, topic, topic_len, ctx->key_depth, ctx->key_bins); + rc = dyad_kvs_read (ctx, topic, upath, should_wait, mdata); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, "Could not read data from the KVS"); goto get_metadata_done; - } } - (*mdata)->fpath = (char *)malloc(fname_len + 1); - if ((*mdata)->fpath == NULL) { - DYAD_LOG_ERROR(ctx, - "Cannot allocate memory for fpath in metadata object"); - rc = DYAD_RC_SYSFAIL; - goto get_metadata_done; - } - memset((*mdata)->fpath, '\0', fname_len + 1); - memcpy((*mdata)->fpath, fname, fname_len); - (*mdata)->owner_rank = ctx->rank; rc = DYAD_RC_OK; - goto get_metadata_done; - } - - const size_t topic_len = PATH_MAX; - char topic[PATH_MAX + 1] = {'\0'}; - DYAD_LOG_INFO(ctx, "Generating KVS key: %s", topic); - gen_path_key(upath, topic, topic_len, ctx->key_depth, ctx->key_bins); - rc = dyad_kvs_read(ctx, topic, upath, should_wait, mdata); - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, "Could not read data from the KVS"); - goto get_metadata_done; - } - rc = DYAD_RC_OK; get_metadata_done:; - if (DYAD_IS_ERROR(rc) && mdata != NULL && *mdata != NULL) { - dyad_free_metadata(mdata); - } - ctx->reenter = true; - DYAD_C_FUNCTION_END(); - return rc; + if (DYAD_IS_ERROR (rc) && mdata != NULL && *mdata != NULL) { + dyad_free_metadata (mdata); + } + ctx->reenter = true; + DYAD_C_FUNCTION_END (); + return rc; } -dyad_rc_t dyad_free_metadata(dyad_metadata_t **mdata) { - DYAD_C_FUNCTION_START(); - if (mdata == NULL || *mdata == NULL) { +dyad_rc_t dyad_free_metadata (dyad_metadata_t **mdata) +{ + DYAD_C_FUNCTION_START (); + if (mdata == NULL || *mdata == NULL) { + return DYAD_RC_OK; + } + if ((*mdata)->fpath != NULL) + free ((*mdata)->fpath); + free (*mdata); + *mdata = NULL; + DYAD_C_FUNCTION_END (); return DYAD_RC_OK; - } - if ((*mdata)->fpath != NULL) - free((*mdata)->fpath); - free(*mdata); - *mdata = NULL; - DYAD_C_FUNCTION_END(); - return DYAD_RC_OK; } -dyad_rc_t dyad_consume(dyad_ctx_t *restrict ctx, const char *restrict fname) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR("fname", fname); - dyad_rc_t rc = DYAD_RC_OK; - int lock_fd = -1, io_fd = -1; - ssize_t file_size = -1; - char *file_data = NULL; - size_t data_len = 0ul; - dyad_metadata_t *mdata = NULL; - struct flock exclusive_lock; - char upath[PATH_MAX + 1] = {'\0'}; - - // If the context is not defined, then it is not valid. - // So, return DYAD_NOCTX - if (!ctx || !ctx->h) { - rc = DYAD_RC_NOCTX; - goto consume_close; - } - // If the consumer-managed path is NULL or empty, then the context is not - // valid for a consumer operation. So, return DYAD_BADMANAGEDPATH - if (ctx->cons_managed_path == NULL) { - rc = DYAD_RC_BADMANAGEDPATH; - goto consume_close; - } - - if (ctx->relative_to_managed_path && (strlen(fname) > 0ul) && - (strncmp(fname, DYAD_PATH_DELIM, ctx->delim_len) != - 0)) { // fname is a relative path that is relative to the - // cons_managed_path - memcpy(upath, fname, strlen(fname)); - } else if (!cmp_canonical_path_prefix(ctx, false, fname, upath, PATH_MAX)) { - // Extract the path to the file specified by fname relative to the - // consumer-managed path - // This relative path will be stored in upath - // DYAD_LOG_TRACE (ctx, "%s is not in the Consumer's managed path\n", - // fname); - rc = DYAD_RC_OK; - goto consume_close; - } - ctx->reenter = false; - - lock_fd = open(fname, O_RDWR | O_CREAT, 0666); - if (lock_fd == -1) { - // This could be a system file on which users have no write permission - DYAD_LOG_ERROR(ctx, "Cannot create file (%s) for dyad_consume!\n", fname); - rc = DYAD_RC_BADFIO; - goto consume_close; - } - rc = dyad_excl_flock(ctx, lock_fd, &exclusive_lock); - if (DYAD_IS_ERROR(rc)) { - dyad_release_flock(ctx, lock_fd, &exclusive_lock); - goto consume_done; - } - file_size = get_file_size(lock_fd); - if (ctx->shared_storage) { - dyad_release_flock(ctx, lock_fd, &exclusive_lock); - if (!ctx->use_fs_locks || file_size <= 0) { - // as file size was zero that means consumer won the lock first so has to - // wait for kvs. or we cannot use file lock based synchronization as it - // does not work with the files managed by c++ fstream. - rc = dyad_fetch_metadata(ctx, fname, upath, &mdata); - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, - "dyad_fetch_metadata failed fore shared storage!\n"); - goto consume_done; - } - } - } else { - if (file_size <= 0) { - DYAD_LOG_INFO(ctx, - "[node %u rank %u pid %d] File (%s with lock_fd %d) is not " - "fetched yet", - ctx->node_idx, ctx->rank, ctx->pid, fname, lock_fd); - // Call dyad_fetch to get (and possibly wait on) - // data from the Flux KVS - rc = dyad_fetch_metadata(ctx, fname, upath, &mdata); - // If an error occured in dyad_fetch_metadata, log an error - // and return the corresponding DYAD return code - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, "dyad_fetch_metadata failed!\n"); - dyad_release_flock(ctx, lock_fd, &exclusive_lock); - goto consume_done; - } - // If dyad_fetch_metadata was successful, but mdata is still NULL, - // then we need to skip data transfer. - // This is either because producer and consumer share storage - // or because the file is not on the managed directory. - if (mdata == NULL) { - DYAD_LOG_INFO(ctx, "File '%s' is local!\n", fname); +dyad_rc_t dyad_consume (dyad_ctx_t *restrict ctx, const char *restrict fname) +{ + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_UPDATE_STR ("fname", fname); + dyad_rc_t rc = DYAD_RC_OK; + int lock_fd = -1, io_fd = -1; + ssize_t file_size = -1; + char *file_data = NULL; + size_t data_len = 0ul; + dyad_metadata_t *mdata = NULL; + struct flock exclusive_lock; + char upath[PATH_MAX + 1] = {'\0'}; + + // If the context is not defined, then it is not valid. + // So, return DYAD_NOCTX + if (!ctx || !ctx->h) { + rc = DYAD_RC_NOCTX; + goto consume_close; + } + // If the consumer-managed path is NULL or empty, then the context is not + // valid for a consumer operation. So, return DYAD_BADMANAGEDPATH + if (ctx->cons_managed_path == NULL) { + rc = DYAD_RC_BADMANAGEDPATH; + goto consume_close; + } + + if (ctx->relative_to_managed_path && (strlen (fname) > 0ul) + && (strncmp (fname, DYAD_PATH_DELIM, ctx->delim_len) + != 0)) { // fname is a relative path that is relative to the + // cons_managed_path + memcpy (upath, fname, strlen (fname)); + } else if (!cmp_canonical_path_prefix (ctx, false, fname, upath, PATH_MAX)) { + // Extract the path to the file specified by fname relative to the + // consumer-managed path + // This relative path will be stored in upath + // DYAD_LOG_TRACE (ctx, "%s is not in the Consumer's managed path\n", + // fname); rc = DYAD_RC_OK; - dyad_release_flock(ctx, lock_fd, &exclusive_lock); - goto consume_done; - } - - // Call dyad_get_data to dispatch a RPC to the producer's Flux broker - // and retrieve the data associated with the file - rc = dyad_get_data(ctx, mdata, &file_data, &data_len); - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, "dyad_get_data failed!\n"); - dyad_release_flock(ctx, lock_fd, &exclusive_lock); - goto consume_done; - } - DYAD_C_FUNCTION_UPDATE_INT("data_len", data_len); - io_fd = open(fname, O_WRONLY); - DYAD_C_FUNCTION_UPDATE_INT("io_fd", io_fd); - if (io_fd == -1) { - DYAD_LOG_ERROR( - ctx, "Cannot open file (%s) in write mode for dyad_consume!\n", - fname); - rc = DYAD_RC_BADFIO; goto consume_close; - } - // Call dyad_pull to fetch the data from the producer's - // Flux broker - rc = dyad_cons_store(ctx, mdata, io_fd, data_len, file_data); - // Regardless if there was an error in dyad_pull, - // free the KVS response object - if (mdata != NULL) { - dyad_free_metadata(&mdata); - } - if (close(io_fd) != 0) { + } + ctx->reenter = false; + + lock_fd = open (fname, O_RDWR | O_CREAT, 0666); + if (lock_fd == -1) { + // This could be a system file on which users have no write permission + DYAD_LOG_ERROR (ctx, "Cannot create file (%s) for dyad_consume!\n", fname); rc = DYAD_RC_BADFIO; - dyad_release_flock(ctx, lock_fd, &exclusive_lock); - goto consume_done; - } - // If an error occured in dyad_pull, log it - // and return the corresponding DYAD return code - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, "dyad_cons_store failed!\n"); - dyad_release_flock(ctx, lock_fd, &exclusive_lock); + goto consume_close; + } + rc = dyad_excl_flock (ctx, lock_fd, &exclusive_lock); + if (DYAD_IS_ERROR (rc)) { + dyad_release_flock (ctx, lock_fd, &exclusive_lock); goto consume_done; - }; } - dyad_release_flock(ctx, lock_fd, &exclusive_lock); - } - DYAD_C_FUNCTION_UPDATE_INT("file_size", file_size); + file_size = get_file_size (lock_fd); + if (ctx->shared_storage) { + dyad_release_flock (ctx, lock_fd, &exclusive_lock); + if (!ctx->use_fs_locks || file_size <= 0) { + // as file size was zero that means consumer won the lock first so has to + // wait for kvs. or we cannot use file lock based synchronization as it + // does not work with the files managed by c++ fstream. + rc = dyad_fetch_metadata (ctx, fname, upath, &mdata); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, "dyad_fetch_metadata failed fore shared storage!\n"); + goto consume_done; + } + } + } else { + if (file_size <= 0) { + DYAD_LOG_INFO (ctx, + "[node %u rank %u pid %d] File (%s with lock_fd %d) is not " + "fetched yet", + ctx->node_idx, + ctx->rank, + ctx->pid, + fname, + lock_fd); + // Call dyad_fetch to get (and possibly wait on) + // data from the Flux KVS + rc = dyad_fetch_metadata (ctx, fname, upath, &mdata); + // If an error occured in dyad_fetch_metadata, log an error + // and return the corresponding DYAD return code + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, "dyad_fetch_metadata failed!\n"); + dyad_release_flock (ctx, lock_fd, &exclusive_lock); + goto consume_done; + } + // If dyad_fetch_metadata was successful, but mdata is still NULL, + // then we need to skip data transfer. + // This is either because producer and consumer share storage + // or because the file is not on the managed directory. + if (mdata == NULL) { + DYAD_LOG_INFO (ctx, "File '%s' is local!\n", fname); + rc = DYAD_RC_OK; + dyad_release_flock (ctx, lock_fd, &exclusive_lock); + goto consume_done; + } + + // Call dyad_get_data to dispatch a RPC to the producer's Flux broker + // and retrieve the data associated with the file + rc = dyad_get_data (ctx, mdata, &file_data, &data_len); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, "dyad_get_data failed!\n"); + dyad_release_flock (ctx, lock_fd, &exclusive_lock); + goto consume_done; + } + DYAD_C_FUNCTION_UPDATE_INT ("data_len", data_len); + io_fd = open (fname, O_WRONLY); + DYAD_C_FUNCTION_UPDATE_INT ("io_fd", io_fd); + if (io_fd == -1) { + DYAD_LOG_ERROR (ctx, + "Cannot open file (%s) in write mode for dyad_consume!\n", + fname); + rc = DYAD_RC_BADFIO; + goto consume_close; + } + // Call dyad_pull to fetch the data from the producer's + // Flux broker + rc = dyad_cons_store (ctx, mdata, io_fd, data_len, file_data); + // Regardless if there was an error in dyad_pull, + // free the KVS response object + if (mdata != NULL) { + dyad_free_metadata (&mdata); + } + if (close (io_fd) != 0) { + rc = DYAD_RC_BADFIO; + dyad_release_flock (ctx, lock_fd, &exclusive_lock); + goto consume_done; + } + // If an error occured in dyad_pull, log it + // and return the corresponding DYAD return code + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, "dyad_cons_store failed!\n"); + dyad_release_flock (ctx, lock_fd, &exclusive_lock); + goto consume_done; + }; + } + dyad_release_flock (ctx, lock_fd, &exclusive_lock); + } + DYAD_C_FUNCTION_UPDATE_INT ("file_size", file_size); consume_done:; - if (close(lock_fd) != 0) { - rc = DYAD_RC_BADFIO; - } - if (file_data != NULL) { - ctx->dtl_handle->return_buffer(ctx, (void **)&file_data); - } - // Set reenter to true to allow additional intercepting + if (close (lock_fd) != 0) { + rc = DYAD_RC_BADFIO; + } + if (file_data != NULL) { + ctx->dtl_handle->return_buffer (ctx, (void **)&file_data); + } + // Set reenter to true to allow additional intercepting consume_close:; - ctx->reenter = true; - DYAD_C_FUNCTION_END(); - return rc; + ctx->reenter = true; + DYAD_C_FUNCTION_END (); + return rc; } -dyad_rc_t dyad_consume_w_metadata(dyad_ctx_t *restrict ctx, const char *fname, - const dyad_metadata_t *restrict mdata) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR("fname", fname); - dyad_rc_t rc = DYAD_RC_OK; - int lock_fd = -1, io_fd = -1; - ssize_t file_size = -1; - char *file_data = NULL; - size_t data_len = 0ul; - struct flock exclusive_lock; - // If the context is not defined, then it is not valid. - // So, return DYAD_NOCTX - if (!ctx || !ctx->h) { - rc = DYAD_RC_NOCTX; - goto consume_close; - } - // If dyad_get_metadata was successful, but mdata is still NULL, - // then we need to skip data transfer. - if (mdata == NULL) { - DYAD_LOG_INFO(ctx, "File '%s' is local!\n", fname); +dyad_rc_t dyad_consume_w_metadata (dyad_ctx_t *restrict ctx, + const char *fname, + const dyad_metadata_t *restrict mdata) +{ + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_UPDATE_STR ("fname", fname); + dyad_rc_t rc = DYAD_RC_OK; + int lock_fd = -1, io_fd = -1; + ssize_t file_size = -1; + char *file_data = NULL; + size_t data_len = 0ul; + struct flock exclusive_lock; + // If the context is not defined, then it is not valid. + // So, return DYAD_NOCTX + if (!ctx || !ctx->h) { + rc = DYAD_RC_NOCTX; + goto consume_close; + } + // If dyad_get_metadata was successful, but mdata is still NULL, + // then we need to skip data transfer. + if (mdata == NULL) { + DYAD_LOG_INFO (ctx, "File '%s' is local!\n", fname); + rc = DYAD_RC_OK; + goto consume_close; + } + // If the consumer-managed path is NULL or empty, then the context is not + // valid for a consumer operation. So, return DYAD_BADMANAGEDPATH + if (ctx->cons_managed_path == NULL) { + rc = DYAD_RC_BADMANAGEDPATH; + goto consume_close; + } + // Set reenter to false to avoid recursively performing DYAD operations + ctx->reenter = false; + lock_fd = open (fname, O_RDWR | O_CREAT, 0666); + DYAD_C_FUNCTION_UPDATE_INT ("lock_fd", lock_fd); + if (lock_fd == -1) { + DYAD_LOG_ERROR (ctx, "Cannot create file (%s) for dyad_consume_w_metadata!\n", fname); + rc = DYAD_RC_BADFIO; + goto consume_close; + } + rc = dyad_excl_flock (ctx, lock_fd, &exclusive_lock); + if (DYAD_IS_ERROR (rc)) { + dyad_release_flock (ctx, lock_fd, &exclusive_lock); + goto consume_close; + } + if ((file_size = get_file_size (lock_fd)) <= 0) { + DYAD_LOG_INFO (ctx, + "[node %u rank %u pid %d] File (%s with fd %d) is not fetched yet", + ctx->node_idx, + ctx->rank, + ctx->pid, + fname, + lock_fd); + + // Call dyad_get_data to dispatch a RPC to the producer's Flux broker + // and retrieve the data associated with the file + rc = dyad_get_data (ctx, mdata, &file_data, &data_len); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, "dyad_get_data failed!\n"); + dyad_release_flock (ctx, lock_fd, &exclusive_lock); + goto consume_done; + } + DYAD_C_FUNCTION_UPDATE_INT ("data_len", data_len); + io_fd = open (fname, O_WRONLY); + DYAD_C_FUNCTION_UPDATE_INT ("io_fd", io_fd); + if (io_fd == -1) { + DYAD_LOG_ERROR (ctx, "Cannot open file (%s) in write mode for dyad_consume!\n", fname); + rc = DYAD_RC_BADFIO; + goto consume_close; + } + // Call dyad_pull to fetch the data from the producer's + // Flux broker + rc = dyad_cons_store (ctx, mdata, io_fd, data_len, file_data); + + if (close (io_fd) != 0) { + rc = DYAD_RC_BADFIO; + dyad_release_flock (ctx, lock_fd, &exclusive_lock); + goto consume_done; + } + // If an error occured in dyad_pull, log it + // and return the corresponding DYAD return code + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERROR (ctx, "dyad_cons_store failed!\n"); + dyad_release_flock (ctx, io_fd, &exclusive_lock); + goto consume_done; + }; + } + dyad_release_flock (ctx, lock_fd, &exclusive_lock); + DYAD_C_FUNCTION_UPDATE_INT ("file_size", file_size); + + if (close (lock_fd) != 0) { + rc = DYAD_RC_BADFIO; + goto consume_done; + } rc = DYAD_RC_OK; - goto consume_close; - } - // If the consumer-managed path is NULL or empty, then the context is not - // valid for a consumer operation. So, return DYAD_BADMANAGEDPATH - if (ctx->cons_managed_path == NULL) { - rc = DYAD_RC_BADMANAGEDPATH; - goto consume_close; - } - // Set reenter to false to avoid recursively performing DYAD operations - ctx->reenter = false; - lock_fd = open(fname, O_RDWR | O_CREAT, 0666); - DYAD_C_FUNCTION_UPDATE_INT("lock_fd", lock_fd); - if (lock_fd == -1) { - DYAD_LOG_ERROR( - ctx, "Cannot create file (%s) for dyad_consume_w_metadata!\n", fname); - rc = DYAD_RC_BADFIO; - goto consume_close; - } - rc = dyad_excl_flock(ctx, lock_fd, &exclusive_lock); - if (DYAD_IS_ERROR(rc)) { - dyad_release_flock(ctx, lock_fd, &exclusive_lock); - goto consume_close; - } - if ((file_size = get_file_size(lock_fd)) <= 0) { - DYAD_LOG_INFO( - ctx, "[node %u rank %u pid %d] File (%s with fd %d) is not fetched yet", - ctx->node_idx, ctx->rank, ctx->pid, fname, lock_fd); - - // Call dyad_get_data to dispatch a RPC to the producer's Flux broker - // and retrieve the data associated with the file - rc = dyad_get_data(ctx, mdata, &file_data, &data_len); - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, "dyad_get_data failed!\n"); - dyad_release_flock(ctx, lock_fd, &exclusive_lock); - goto consume_done; - } - DYAD_C_FUNCTION_UPDATE_INT("data_len", data_len); - io_fd = open(fname, O_WRONLY); - DYAD_C_FUNCTION_UPDATE_INT("io_fd", io_fd); - if (io_fd == -1) { - DYAD_LOG_ERROR(ctx, - "Cannot open file (%s) in write mode for dyad_consume!\n", - fname); - rc = DYAD_RC_BADFIO; - goto consume_close; - } - // Call dyad_pull to fetch the data from the producer's - // Flux broker - rc = dyad_cons_store(ctx, mdata, io_fd, data_len, file_data); - - if (close(io_fd) != 0) { - rc = DYAD_RC_BADFIO; - dyad_release_flock(ctx, lock_fd, &exclusive_lock); - goto consume_done; - } - // If an error occured in dyad_pull, log it - // and return the corresponding DYAD return code - if (DYAD_IS_ERROR(rc)) { - DYAD_LOG_ERROR(ctx, "dyad_cons_store failed!\n"); - dyad_release_flock(ctx, io_fd, &exclusive_lock); - goto consume_done; - }; - } - dyad_release_flock(ctx, lock_fd, &exclusive_lock); - DYAD_C_FUNCTION_UPDATE_INT("file_size", file_size); - - if (close(lock_fd) != 0) { - rc = DYAD_RC_BADFIO; - goto consume_done; - } - rc = DYAD_RC_OK; consume_done:; - if (file_data != NULL) { - ctx->dtl_handle->return_buffer(ctx, (void **)&file_data); - } + if (file_data != NULL) { + ctx->dtl_handle->return_buffer (ctx, (void **)&file_data); + } consume_close:; - // Set reenter to true to allow additional intercepting - ctx->reenter = true; - DYAD_C_FUNCTION_END(); - return rc; + // Set reenter to true to allow additional intercepting + ctx->reenter = true; + DYAD_C_FUNCTION_END (); + return rc; } #if DYAD_SYNC_DIR -int dyad_sync_directory(dyad_ctx_t *restrict ctx, const char *restrict path) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR("path", path); - // Flush new directory entry https://lwn.net/Articles/457671/ - char path_copy[PATH_MAX + 1] = {'\0'}; - int odir_fd = -1; - char *odir = NULL; - bool reenter = false; - int rc = 0; - memset(path_copy, 0, PATH_MAX + 1); - - strncpy(path_copy, path, PATH_MAX); - odir = dirname(path_copy); - - reenter = ctx->reenter; // backup ctx->reenter - if (ctx != NULL) - ctx->reenter = false; - - if ((odir_fd = open(odir, O_RDONLY)) < 0) { - IPRINTF(ctx, "Cannot open the directory \"%s\"\n", odir); - rc = -1; - } else { - if (fsync(odir_fd) < 0) { - IPRINTF(ctx, "Cannot flush the directory \"%s\"\n", odir); - rc = -1; - } - if (close(odir_fd) < 0) { - IPRINTF(ctx, "Cannot close the directory \"%s\"\n", odir); - rc = -1; - } - } - if (ctx != NULL) - ctx->reenter = reenter; - DYAD_C_FUNCTION_END(); - return rc; +int dyad_sync_directory (dyad_ctx_t *restrict ctx, const char *restrict path) +{ + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_UPDATE_STR ("path", path); + // Flush new directory entry https://lwn.net/Articles/457671/ + char path_copy[PATH_MAX + 1] = {'\0'}; + int odir_fd = -1; + char *odir = NULL; + bool reenter = false; + int rc = 0; + memset (path_copy, 0, PATH_MAX + 1); + + strncpy (path_copy, path, PATH_MAX); + odir = dirname (path_copy); + + reenter = ctx->reenter; // backup ctx->reenter + if (ctx != NULL) + ctx->reenter = false; + + if ((odir_fd = open (odir, O_RDONLY)) < 0) { + IPRINTF (ctx, "Cannot open the directory \"%s\"\n", odir); + rc = -1; + } else { + if (fsync (odir_fd) < 0) { + IPRINTF (ctx, "Cannot flush the directory \"%s\"\n", odir); + rc = -1; + } + if (close (odir_fd) < 0) { + IPRINTF (ctx, "Cannot close the directory \"%s\"\n", odir); + rc = -1; + } + } + if (ctx != NULL) + ctx->reenter = reenter; + DYAD_C_FUNCTION_END (); + return rc; } #endif diff --git a/src/dyad/client/dyad_client_int.h b/src/dyad/client/dyad_client_int.h index c8fc8288..9e73b02b 100644 --- a/src/dyad/client/dyad_client_int.h +++ b/src/dyad/client/dyad_client_int.h @@ -9,24 +9,23 @@ // Debug message #ifndef DPRINTF #if VA_OPT_SUPPORTED -#define DPRINTF(curr_dyad_ctx, fmt, ...) \ - do { \ - if ((curr_dyad_ctx) && (curr_dyad_ctx)->debug) \ - fprintf(stderr, (fmt)__VA_OPT__(, ) __VA_ARGS__); \ - } while (0) +#define DPRINTF(curr_dyad_ctx, fmt, ...) \ + do { \ + if ((curr_dyad_ctx) && (curr_dyad_ctx)->debug) \ + fprintf (stderr, (fmt)__VA_OPT__ (, ) __VA_ARGS__); \ + } while (0) #else -#define DPRINTF(curr_dyad_ctx, fmt, ...) \ - do { \ - if ((curr_dyad_ctx) && (curr_dyad_ctx)->debug) \ - fprintf(stderr, (fmt), ##__VA_ARGS__); \ - } while (0) +#define DPRINTF(curr_dyad_ctx, fmt, ...) \ + do { \ + if ((curr_dyad_ctx) && (curr_dyad_ctx)->debug) \ + fprintf (stderr, (fmt), ##__VA_ARGS__); \ + } while (0) #endif -#endif // DPRINTF +#endif // DPRINTF -#define TIME_DIFF(Tstart, Tend) \ - ((double)(1000000000L * ((Tend).tv_sec - (Tstart).tv_sec) + (Tend).tv_nsec - \ - (Tstart).tv_nsec) / \ - 1000000000L) +#define TIME_DIFF(Tstart, Tend) \ + ((double)(1000000000L * ((Tend).tv_sec - (Tstart).tv_sec) + (Tend).tv_nsec - (Tstart).tv_nsec) \ + / 1000000000L) // Detailed information message that can be omitted #if DYAD_FULL_DEBUG @@ -34,28 +33,31 @@ #define IPRINTF_DEFINED #else #define IPRINTF(curr_dyad_ctx, fmt, ...) -#endif // DYAD_FULL_DEBUG +#endif // DYAD_FULL_DEBUG -DYAD_DLL_EXPORTED int gen_path_key(const char *str, char *path_key, - const size_t len, const uint32_t depth, - const uint32_t width); +DYAD_DLL_EXPORTED int gen_path_key (const char *str, + char *path_key, + const size_t len, + const uint32_t depth, + const uint32_t width); /** * Private Function definitions */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data(const dyad_ctx_t *ctx, - const dyad_metadata_t *mdata, - char **file_data, size_t *file_len); -DYAD_DLL_EXPORTED dyad_rc_t dyad_commit(dyad_ctx_t *ctx, const char *fname); - -DYAD_DLL_EXPORTED dyad_rc_t dyad_kvs_read(const dyad_ctx_t *ctx, - const char *topic, const char *upath, - bool should_wait, - dyad_metadata_t **mdata); +DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data (const dyad_ctx_t *ctx, + const dyad_metadata_t *mdata, + char **file_data, + size_t *file_len); +DYAD_DLL_EXPORTED dyad_rc_t dyad_commit (dyad_ctx_t *ctx, const char *fname); + +DYAD_DLL_EXPORTED dyad_rc_t dyad_kvs_read (const dyad_ctx_t *ctx, + const char *topic, + const char *upath, + bool should_wait, + dyad_metadata_t **mdata); #if DYAD_SYNC_DIR -DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED int dyad_sync_directory(dyad_ctx_t *ctx, - const char *path); +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED int dyad_sync_directory (dyad_ctx_t *ctx, const char *path); #endif -#endif /* DYAD_CORE_DYAD_CORE_INT_H */ \ No newline at end of file +#endif /* DYAD_CORE_DYAD_CORE_INT_H */ diff --git a/src/dyad/common/dyad_structures_int.h b/src/dyad/common/dyad_structures_int.h index d6eb8f73..bbbc1572 100644 --- a/src/dyad/common/dyad_structures_int.h +++ b/src/dyad/common/dyad_structures_int.h @@ -24,41 +24,40 @@ extern "C" { * @struct dyad_ctx */ struct dyad_ctx { - // Internal - void *h; // the Flux handle for DYAD - struct dyad_dtl *dtl_handle; // Opaque handle to DTL info - const char *fname; // Used to track which file is getting processed. - bool use_fs_locks; // Used to track if fs locks should be used. - char *prod_real_path; // producer managed real path - char *cons_real_path; // consumer managed real path - uint32_t prod_managed_len; // length of producer path managed by DYAD - uint32_t cons_managed_len; // length of consumer path managed by DYAD - uint32_t prod_real_len; // length of producer managed real path - uint32_t cons_real_len; // length of consumer managed real path - uint32_t prod_managed_hash; // hash of producer path managed by DYAD - uint32_t cons_managed_hash; // hash of consumer path managed by DYAD - uint32_t prod_real_hash; // hash of producer managed real path - uint32_t cons_real_hash; // hash of consumer managed real path - uint32_t delim_len; // length of path delimiter - // User Facing - bool debug; // if true, perform debug logging - bool check; // if true, perform some check logging - bool reenter; // if false, do not recursively enter DYAD - bool initialized; // if true, DYAD is initialized - bool shared_storage; // if true, the managed path is shared - bool async_publish; // Enable asynchronous publish by producer - bool fsync_write; // Apply fsync after write by producer - unsigned int key_depth; // Depth of bins for the Flux KVS - unsigned int key_bins; // Number of bins for the Flux KVS - uint32_t rank; // Flux rank for DYAD - uint32_t service_mux; // Number of Flux brokers sharing node-local storage - uint32_t node_idx; // Index of the node hosting broker(s) - int pid; // unix process id, obtained by getpid() - char *kvs_namespace; // Flux KVS namespace for DYAD - char *prod_managed_path; // producer path managed by DYAD - char *cons_managed_path; // consumer path managed by DYAD - bool - relative_to_managed_path; // relative path is relative to the managed path + // Internal + void *h; // the Flux handle for DYAD + struct dyad_dtl *dtl_handle; // Opaque handle to DTL info + const char *fname; // Used to track which file is getting processed. + bool use_fs_locks; // Used to track if fs locks should be used. + char *prod_real_path; // producer managed real path + char *cons_real_path; // consumer managed real path + uint32_t prod_managed_len; // length of producer path managed by DYAD + uint32_t cons_managed_len; // length of consumer path managed by DYAD + uint32_t prod_real_len; // length of producer managed real path + uint32_t cons_real_len; // length of consumer managed real path + uint32_t prod_managed_hash; // hash of producer path managed by DYAD + uint32_t cons_managed_hash; // hash of consumer path managed by DYAD + uint32_t prod_real_hash; // hash of producer managed real path + uint32_t cons_real_hash; // hash of consumer managed real path + uint32_t delim_len; // length of path delimiter + // User Facing + bool debug; // if true, perform debug logging + bool check; // if true, perform some check logging + bool reenter; // if false, do not recursively enter DYAD + bool initialized; // if true, DYAD is initialized + bool shared_storage; // if true, the managed path is shared + bool async_publish; // Enable asynchronous publish by producer + bool fsync_write; // Apply fsync after write by producer + unsigned int key_depth; // Depth of bins for the Flux KVS + unsigned int key_bins; // Number of bins for the Flux KVS + uint32_t rank; // Flux rank for DYAD + uint32_t service_mux; // Number of Flux brokers sharing node-local storage + uint32_t node_idx; // Index of the node hosting broker(s) + int pid; // unix process id, obtained by getpid() + char *kvs_namespace; // Flux KVS namespace for DYAD + char *prod_managed_path; // producer path managed by DYAD + char *cons_managed_path; // consumer path managed by DYAD + bool relative_to_managed_path; // relative path is relative to the managed path }; typedef void *ucx_ep_cache_h; @@ -66,4 +65,4 @@ typedef void *ucx_ep_cache_h; } #endif -#endif // DYAD_COMMON_STRUCTURES_INT_H +#endif // DYAD_COMMON_STRUCTURES_INT_H diff --git a/src/dyad/wrapper/wrapper.c b/src/dyad/wrapper/wrapper.c index 035baa4c..254074b3 100644 --- a/src/dyad/wrapper/wrapper.c +++ b/src/dyad/wrapper/wrapper.c @@ -16,7 +16,7 @@ #ifndef _GNU_SOURCE #define _GNU_SOURCE -#endif // _GNU_SOURCE +#endif // _GNU_SOURCE #if defined(__cplusplus) #include @@ -26,7 +26,7 @@ #include #include #include -using namespace std; // std::clock () +using namespace std; // std::clock () // #include // c++11 #else #include @@ -37,7 +37,7 @@ using namespace std; // std::clock () #include #include #include -#endif // defined(__cplusplus) +#endif // defined(__cplusplus) #include #include @@ -47,7 +47,7 @@ using namespace std; // std::clock () #include #include #include -#include // dirname +#include // dirname #include #ifdef __cplusplus @@ -56,12 +56,12 @@ extern "C" { static __thread const dyad_ctx_t *ctx = NULL; static __thread dyad_ctx_t *ctx_mutable = NULL; -static void dyad_wrapper_init(void) __attribute__((constructor)); -static void dyad_wrapper_fini(void) __attribute__((destructor)); +static void dyad_wrapper_init (void) __attribute__ ((constructor)); +static void dyad_wrapper_fini (void) __attribute__ ((destructor)); #if DYAD_SYNC_DIR -int sync_directory(const char *path); -#endif // DYAD_SYNC_DIR +int sync_directory (const char *path); +#endif // DYAD_SYNC_DIR /***************************************************************************** * * @@ -77,13 +77,14 @@ int sync_directory(const char *path); * @return 1 if the file descriptor is write-only, 0 if not, and -1 * if there was an error in fcntl */ -static inline int is_wronly(int fd) { - int rc = fcntl(fd, F_GETFL); - if (rc == -1) - return -1; - if ((rc & O_ACCMODE) == O_WRONLY) - return 1; - return 0; +static inline int is_wronly (int fd) +{ + int rc = fcntl (fd, F_GETFL); + if (rc == -1) + return -1; + if ((rc & O_ACCMODE) == O_WRONLY) + return 1; + return 0; } /***************************************************************************** @@ -92,322 +93,324 @@ static inline int is_wronly(int fd) { * * *****************************************************************************/ -void dyad_wrapper_init(void) { +void dyad_wrapper_init (void) +{ #if DYAD_PROFILER == 3 - DFTRACER_C_FINI(); + DFTRACER_C_FINI (); #endif - DYAD_C_FUNCTION_START(); - dyad_ctx_init(DYAD_COMM_RECV, NULL); - ctx = ctx_mutable = dyad_ctx_get(); - DYAD_LOG_INFO(ctx, "DYAD Wrapper Initialized"); - DYAD_C_FUNCTION_END(); + DYAD_C_FUNCTION_START (); + dyad_ctx_init (DYAD_COMM_RECV, NULL); + ctx = ctx_mutable = dyad_ctx_get (); + DYAD_LOG_INFO (ctx, "DYAD Wrapper Initialized"); + DYAD_C_FUNCTION_END (); } -void dyad_wrapper_fini() { - DYAD_C_FUNCTION_START(); - DYAD_LOG_INFO(ctx, "DYAD Wrapper Finalized"); - dyad_ctx_fini(); - DYAD_C_FUNCTION_END(); +void dyad_wrapper_fini () +{ + DYAD_C_FUNCTION_START (); + DYAD_LOG_INFO (ctx, "DYAD Wrapper Finalized"); + dyad_ctx_fini (); + DYAD_C_FUNCTION_END (); #if DYAD_PROFILER == 3 - DFTRACER_C_FINI(); + DFTRACER_C_FINI (); #endif } -DYAD_DLL_EXPORTED int open(const char *path, int oflag, ...) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR("path", "path"); - char *error = NULL; - typedef int (*open_ptr_t)(const char *, int, mode_t, ...); - open_ptr_t func_ptr = NULL; - int mode = 0; - char upath[PATH_MAX + 1] = {'\0'}; - - if (oflag & O_CREAT) { - va_list arg; - va_start(arg, oflag); - mode = va_arg(arg, int); - va_end(arg); - } - - // https://stackoverflow.com/questions/14134245/iso-c-void-and-function-pointers - // func_ptr = (open_ptr_t)dlsym (RTLD_NEXT, "open"); - *(void **)&func_ptr = dlsym(RTLD_NEXT, "open"); - if ((error = dlerror())) { - DPRINTF(ctx, "DYAD_SYNC: error in dlsym: %s", error); - DYAD_C_FUNCTION_END(); - return -1; - } - - if ((mode != O_RDONLY) || is_path_dir(path)) { - // TODO: make sure if the directory mode is consistent - goto real_call; - } - - if (!(ctx && ctx->h) || (ctx && !ctx->reenter)) { - IPRINTF(ctx, "DYAD_SYNC: open sync not applicable for \"%s\".", path); - goto real_call; - } - - IPRINTF(ctx, "DYAD_SYNC: enters open sync (\"%s\").", path); - if (DYAD_IS_ERROR(dyad_consume(ctx_mutable, path))) { - DPRINTF(ctx, "DYAD_SYNC: failed open sync (\"%s\").", path); - goto real_call; - } - IPRINTF(ctx, "DYAD_SYNC: exists open sync (\"%s\").", path); +DYAD_DLL_EXPORTED int open (const char *path, int oflag, ...) +{ + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_UPDATE_STR ("path", "path"); + char *error = NULL; + typedef int (*open_ptr_t) (const char *, int, mode_t, ...); + open_ptr_t func_ptr = NULL; + int mode = 0; + char upath[PATH_MAX + 1] = {'\0'}; + + if (oflag & O_CREAT) { + va_list arg; + va_start (arg, oflag); + mode = va_arg (arg, int); + va_end (arg); + } + + // https://stackoverflow.com/questions/14134245/iso-c-void-and-function-pointers + // func_ptr = (open_ptr_t)dlsym (RTLD_NEXT, "open"); + *(void **)&func_ptr = dlsym (RTLD_NEXT, "open"); + if ((error = dlerror ())) { + DPRINTF (ctx, "DYAD_SYNC: error in dlsym: %s", error); + DYAD_C_FUNCTION_END (); + return -1; + } + + if ((mode != O_RDONLY) || is_path_dir (path)) { + // TODO: make sure if the directory mode is consistent + goto real_call; + } + + if (!(ctx && ctx->h) || (ctx && !ctx->reenter)) { + IPRINTF (ctx, "DYAD_SYNC: open sync not applicable for \"%s\".", path); + goto real_call; + } + + IPRINTF (ctx, "DYAD_SYNC: enters open sync (\"%s\").", path); + if (DYAD_IS_ERROR (dyad_consume (ctx_mutable, path))) { + DPRINTF (ctx, "DYAD_SYNC: failed open sync (\"%s\").", path); + goto real_call; + } + IPRINTF (ctx, "DYAD_SYNC: exists open sync (\"%s\").", path); real_call:; - int ret = (func_ptr(path, oflag, mode)); - - // This lock is to protect the file being produced by a producer - // from a consumer that has direct access to the file. For example, - // either the file is on a shared storage or the consumer is on - // the same node as where the producer is. - if ((ret > 0) && (mode == O_WRONLY || mode == O_APPEND) && - !is_path_dir(path)) { - if ((ctx->relative_to_managed_path && - (strncmp(path, DYAD_PATH_DELIM, ctx->delim_len) != 0)) || - cmp_canonical_path_prefix(ctx, true, path, upath, PATH_MAX)) { - struct flock exclusive_lock; - dyad_rc_t rc = dyad_excl_flock(ctx, ret, &exclusive_lock); - if (DYAD_IS_ERROR(rc)) { - dyad_release_flock(ctx, ret, &exclusive_lock); - } + int ret = (func_ptr (path, oflag, mode)); + + // This lock is to protect the file being produced by a producer + // from a consumer that has direct access to the file. For example, + // either the file is on a shared storage or the consumer is on + // the same node as where the producer is. + if ((ret > 0) && (mode == O_WRONLY || mode == O_APPEND) && !is_path_dir (path)) { + if ((ctx->relative_to_managed_path + && (strncmp (path, DYAD_PATH_DELIM, ctx->delim_len) != 0)) + || cmp_canonical_path_prefix (ctx, true, path, upath, PATH_MAX)) { + struct flock exclusive_lock; + dyad_rc_t rc = dyad_excl_flock (ctx, ret, &exclusive_lock); + if (DYAD_IS_ERROR (rc)) { + dyad_release_flock (ctx, ret, &exclusive_lock); + } + } } - } - DYAD_C_FUNCTION_END(); - return ret; + DYAD_C_FUNCTION_END (); + return ret; } -DYAD_DLL_EXPORTED FILE *fopen(const char *path, const char *mode) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR("path", "path"); - char *error = NULL; - typedef FILE *(*fopen_ptr_t)(const char *, const char *); - fopen_ptr_t func_ptr = NULL; - char upath[PATH_MAX + 1] = {'\0'}; - - // func_ptr = (fopen_ptr_t)dlsym (RTLD_NEXT, "fopen"); - *(void **)&func_ptr = dlsym(RTLD_NEXT, "fopen"); - if ((error = dlerror())) { - DPRINTF(ctx, "DYAD_SYNC: error in dlsym: %s\n", error); - DYAD_C_FUNCTION_END(); - return NULL; - } - - if ((strcmp(mode, "r") != 0) || is_path_dir(path)) { - // TODO: make sure if the directory mode is consistent - goto real_call; - } - - if (!(ctx && ctx->h) || (ctx && !ctx->reenter) || !path) { - IPRINTF(ctx, "DYAD_SYNC: fopen sync not applicable for \"%s\".\n", - ((path) ? path : "")); - goto real_call; - } - - IPRINTF(ctx, "DYAD_SYNC: enters fopen sync (\"%s\").\n", path); - if (DYAD_IS_ERROR(dyad_consume(ctx_mutable, path))) { - DPRINTF(ctx, "DYAD_SYNC: failed fopen sync (\"%s\").\n", path); - goto real_call; - } - IPRINTF(ctx, "DYAD_SYNC: exits fopen sync (\"%s\").\n", path); +DYAD_DLL_EXPORTED FILE *fopen (const char *path, const char *mode) +{ + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_UPDATE_STR ("path", "path"); + char *error = NULL; + typedef FILE *(*fopen_ptr_t) (const char *, const char *); + fopen_ptr_t func_ptr = NULL; + char upath[PATH_MAX + 1] = {'\0'}; + + // func_ptr = (fopen_ptr_t)dlsym (RTLD_NEXT, "fopen"); + *(void **)&func_ptr = dlsym (RTLD_NEXT, "fopen"); + if ((error = dlerror ())) { + DPRINTF (ctx, "DYAD_SYNC: error in dlsym: %s\n", error); + DYAD_C_FUNCTION_END (); + return NULL; + } + + if ((strcmp (mode, "r") != 0) || is_path_dir (path)) { + // TODO: make sure if the directory mode is consistent + goto real_call; + } + + if (!(ctx && ctx->h) || (ctx && !ctx->reenter) || !path) { + IPRINTF (ctx, "DYAD_SYNC: fopen sync not applicable for \"%s\".\n", ((path) ? path : "")); + goto real_call; + } + + IPRINTF (ctx, "DYAD_SYNC: enters fopen sync (\"%s\").\n", path); + if (DYAD_IS_ERROR (dyad_consume (ctx_mutable, path))) { + DPRINTF (ctx, "DYAD_SYNC: failed fopen sync (\"%s\").\n", path); + goto real_call; + } + IPRINTF (ctx, "DYAD_SYNC: exits fopen sync (\"%s\").\n", path); real_call:; - FILE *fh = (func_ptr(path, mode)); - - // This lock is to protect the file being produced by a producer - // from a consumer that has direct access to the file. For example, - // either the file is on a shared storage or the consumer is on - // the same node as where the producer is. - if ((fh != NULL) && ((strcmp(mode, "w") == 0) || (strcmp(mode, "a") == 0)) && - !is_path_dir(path)) { - if ((ctx->relative_to_managed_path && - (strncmp(path, DYAD_PATH_DELIM, ctx->delim_len) != 0)) || - cmp_canonical_path_prefix(ctx, true, path, upath, PATH_MAX)) { - int fd = fileno(fh); - struct flock exclusive_lock; - dyad_rc_t rc = dyad_excl_flock(ctx, fd, &exclusive_lock); - if (DYAD_IS_ERROR(rc)) { - dyad_release_flock(ctx, fd, &exclusive_lock); - } + FILE *fh = (func_ptr (path, mode)); + + // This lock is to protect the file being produced by a producer + // from a consumer that has direct access to the file. For example, + // either the file is on a shared storage or the consumer is on + // the same node as where the producer is. + if ((fh != NULL) && ((strcmp (mode, "w") == 0) || (strcmp (mode, "a") == 0)) + && !is_path_dir (path)) { + if ((ctx->relative_to_managed_path + && (strncmp (path, DYAD_PATH_DELIM, ctx->delim_len) != 0)) + || cmp_canonical_path_prefix (ctx, true, path, upath, PATH_MAX)) { + int fd = fileno (fh); + struct flock exclusive_lock; + dyad_rc_t rc = dyad_excl_flock (ctx, fd, &exclusive_lock); + if (DYAD_IS_ERROR (rc)) { + dyad_release_flock (ctx, fd, &exclusive_lock); + } + } } - } - DYAD_C_FUNCTION_END(); - return fh; + DYAD_C_FUNCTION_END (); + return fh; } -DYAD_DLL_EXPORTED int close(int fd) { - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_INT("fd", fd); - bool to_sync = false; - char *error = NULL; - typedef int (*close_ptr_t)(int); - close_ptr_t func_ptr = NULL; - char path[PATH_MAX + 1] = {'\0'}; - int rc = 0; - - // func_ptr = (close_ptr_t)dlsym (RTLD_NEXT, "close"); - *(void **)&func_ptr = dlsym(RTLD_NEXT, "close"); - if ((error = dlerror())) { - DPRINTF(ctx, "DYAD_SYNC: error in dlsym: %s\n", error); - DYAD_C_FUNCTION_END(); - return -1; // return the failure code - } - - if ((fd < 0) || (ctx == NULL) || (ctx->h == NULL) || !ctx->reenter) { +DYAD_DLL_EXPORTED int close (int fd) +{ + DYAD_C_FUNCTION_START (); + DYAD_C_FUNCTION_UPDATE_INT ("fd", fd); + bool to_sync = false; + char *error = NULL; + typedef int (*close_ptr_t) (int); + close_ptr_t func_ptr = NULL; + char path[PATH_MAX + 1] = {'\0'}; + int rc = 0; + + // func_ptr = (close_ptr_t)dlsym (RTLD_NEXT, "close"); + *(void **)&func_ptr = dlsym (RTLD_NEXT, "close"); + if ((error = dlerror ())) { + DPRINTF (ctx, "DYAD_SYNC: error in dlsym: %s\n", error); + DYAD_C_FUNCTION_END (); + return -1; // return the failure code + } + + if ((fd < 0) || (ctx == NULL) || (ctx->h == NULL) || !ctx->reenter) { #if defined(IPRINTF_DEFINED) - if (ctx == NULL) { - IPRINTF(ctx, "DYAD_SYNC: close sync not applicable. (no context)\n"); - } else if (ctx->h == NULL) { - IPRINTF(ctx, "DYAD_SYNC: close sync not applicable. (no flux)\n"); - } else if (!ctx->reenter) { - IPRINTF(ctx, "DYAD_SYNC: close sync not applicable. (no reenter)\n"); - } else if (fd >= 0) { - IPRINTF(ctx, "DYAD_SYNC: close sync not applicable. (invalid file " - "descriptor)\n"); + if (ctx == NULL) { + IPRINTF (ctx, "DYAD_SYNC: close sync not applicable. (no context)\n"); + } else if (ctx->h == NULL) { + IPRINTF (ctx, "DYAD_SYNC: close sync not applicable. (no flux)\n"); + } else if (!ctx->reenter) { + IPRINTF (ctx, "DYAD_SYNC: close sync not applicable. (no reenter)\n"); + } else if (fd >= 0) { + IPRINTF (ctx, + "DYAD_SYNC: close sync not applicable. (invalid file " + "descriptor)\n"); + } +#endif // defined(IPRINTF_DEFINED) + to_sync = false; + goto real_call; } -#endif // defined(IPRINTF_DEFINED) - to_sync = false; - goto real_call; - } - if (is_fd_dir(fd)) { - // TODO: make sure if the directory mode is consistent - goto real_call; - } + if (is_fd_dir (fd)) { + // TODO: make sure if the directory mode is consistent + goto real_call; + } - if (get_path(fd, PATH_MAX - 1, path) < 0) { - DYAD_LOG_DEBUG( - ctx, "DYAD_SYNC: unable to obtain file path from a descriptor.\n"); - to_sync = false; - goto real_call; - } + if (get_path (fd, PATH_MAX - 1, path) < 0) { + DYAD_LOG_DEBUG (ctx, "DYAD_SYNC: unable to obtain file path from a descriptor.\n"); + to_sync = false; + goto real_call; + } - to_sync = true; + to_sync = true; -real_call:; // semicolon here to avoid the error - // "a label can only be part of a statement and a declaration is not a - // statement" +real_call:; // semicolon here to avoid the error + // "a label can only be part of a statement and a declaration is not a + // statement" - int wronly = is_wronly(fd); + int wronly = is_wronly (fd); - if (wronly == -1) { - DPRINTF(ctx, "Failed to check the mode of the file with fcntl: %s\n", - strerror(errno)); - } + if (wronly == -1) { + DPRINTF (ctx, "Failed to check the mode of the file with fcntl: %s\n", strerror (errno)); + } - if (to_sync && wronly == 1) { - if (ctx->fsync_write) { - fsync(fd); + if (to_sync && wronly == 1) { + if (ctx->fsync_write) { + fsync (fd); #if DYAD_SYNC_DIR - dyad_sync_directory(ctx, path); -#endif // DYAD_SYNC_DIR + dyad_sync_directory (ctx, path); +#endif // DYAD_SYNC_DIR + } + + struct flock exclusive_lock; + dyad_release_flock (ctx, fd, &exclusive_lock); + rc = func_ptr (fd); + if (rc != 0) { + DPRINTF (ctx, "Failed close (\"%s\").: %s\n", path, strerror (errno)); + } + IPRINTF (ctx, "DYAD_SYNC: enters close sync (\"%s\").\n", path); + if (DYAD_IS_ERROR (dyad_produce (ctx_mutable, path))) { + DPRINTF (ctx, "DYAD_SYNC: failed close sync (\"%s\").\n", path); + } + IPRINTF (ctx, "DYAD_SYNC: exits close sync (\"%s\").\n", path); + } else { + rc = func_ptr (fd); } + DYAD_C_FUNCTION_END (); + return rc; +} - struct flock exclusive_lock; - dyad_release_flock(ctx, fd, &exclusive_lock); - rc = func_ptr(fd); - if (rc != 0) { - DPRINTF(ctx, "Failed close (\"%s\").: %s\n", path, strerror(errno)); - } - IPRINTF(ctx, "DYAD_SYNC: enters close sync (\"%s\").\n", path); - if (DYAD_IS_ERROR(dyad_produce(ctx_mutable, path))) { - DPRINTF(ctx, "DYAD_SYNC: failed close sync (\"%s\").\n", path); +DYAD_DLL_EXPORTED int fclose (FILE *fp) +{ + DYAD_C_FUNCTION_START (); + bool to_sync = false; + char *error = NULL; + typedef int (*fclose_ptr_t) (FILE *); + fclose_ptr_t func_ptr = NULL; + char path[PATH_MAX + 1] = {'\0'}; + int rc = 0; + int fd = 0; + + // func_ptr = (fclose_ptr_t)dlsym (RTLD_NEXT, "fclose"); + *(void **)&func_ptr = dlsym (RTLD_NEXT, "fclose"); + if ((error = dlerror ())) { + DYAD_LOG_DEBUG (ctx, "DYAD_SYNC: error in dlsym: %s\n", error); + DYAD_C_FUNCTION_END (); + return EOF; // return the failure code } - IPRINTF(ctx, "DYAD_SYNC: exits close sync (\"%s\").\n", path); - } else { - rc = func_ptr(fd); - } - DYAD_C_FUNCTION_END(); - return rc; -} -DYAD_DLL_EXPORTED int fclose(FILE *fp) { - DYAD_C_FUNCTION_START(); - bool to_sync = false; - char *error = NULL; - typedef int (*fclose_ptr_t)(FILE *); - fclose_ptr_t func_ptr = NULL; - char path[PATH_MAX + 1] = {'\0'}; - int rc = 0; - int fd = 0; - - // func_ptr = (fclose_ptr_t)dlsym (RTLD_NEXT, "fclose"); - *(void **)&func_ptr = dlsym(RTLD_NEXT, "fclose"); - if ((error = dlerror())) { - DYAD_LOG_DEBUG(ctx, "DYAD_SYNC: error in dlsym: %s\n", error); - DYAD_C_FUNCTION_END(); - return EOF; // return the failure code - } - - if ((fp == NULL) || (ctx == NULL) || (ctx->h == NULL) || !ctx->reenter) { + if ((fp == NULL) || (ctx == NULL) || (ctx->h == NULL) || !ctx->reenter) { #if defined(IPRINTF_DEFINED) - if (ctx == NULL) { - IPRINTF(ctx, "DYAD_SYNC: fclose sync not applicable. (no context)\n"); - } else if (ctx->h == NULL) { - IPRINTF(ctx, "DYAD_SYNC: fclose sync not applicable. (no flux)\n"); - } else if (!ctx->reenter) { - IPRINTF(ctx, "DYAD_SYNC: fclose sync not applicable. (no reenter)\n"); - } else if (fp == NULL) { - IPRINTF(ctx, "DYAD_SYNC: fclose sync not applicable. (invalid file " - "pointer)\n"); + if (ctx == NULL) { + IPRINTF (ctx, "DYAD_SYNC: fclose sync not applicable. (no context)\n"); + } else if (ctx->h == NULL) { + IPRINTF (ctx, "DYAD_SYNC: fclose sync not applicable. (no flux)\n"); + } else if (!ctx->reenter) { + IPRINTF (ctx, "DYAD_SYNC: fclose sync not applicable. (no reenter)\n"); + } else if (fp == NULL) { + IPRINTF (ctx, + "DYAD_SYNC: fclose sync not applicable. (invalid file " + "pointer)\n"); + } +#endif // defined(IPRINTF_DEFINED) + to_sync = false; + goto real_call; } -#endif // defined(IPRINTF_DEFINED) - to_sync = false; - goto real_call; - } - if (is_fd_dir(fileno(fp))) { - // TODO: make sure if the directory mode is consistent - goto real_call; - } + if (is_fd_dir (fileno (fp))) { + // TODO: make sure if the directory mode is consistent + goto real_call; + } - if (get_path(fileno(fp), PATH_MAX - 1, path) < 0) { - DYAD_LOG_DEBUG( - ctx, "DYAD_SYNC: unable to obtain file path from a descriptor.\n"); - to_sync = false; - goto real_call; - } + if (get_path (fileno (fp), PATH_MAX - 1, path) < 0) { + DYAD_LOG_DEBUG (ctx, "DYAD_SYNC: unable to obtain file path from a descriptor.\n"); + to_sync = false; + goto real_call; + } - to_sync = true; + to_sync = true; real_call:; - fd = fileno(fp); - - int wronly = is_wronly(fd); + fd = fileno (fp); - if (wronly == -1) { - DPRINTF(ctx, "Failed to check the mode of the file with fcntl: %s\n", - strerror(errno)); - } + int wronly = is_wronly (fd); - if (to_sync && wronly == 1) { - if (ctx->fsync_write) { - fflush(fp); - fsync(fd); -#if DYAD_SYNC_DIR - dyad_sync_directory(ctx, path); -#endif // DYAD_SYNC_DIR + if (wronly == -1) { + DPRINTF (ctx, "Failed to check the mode of the file with fcntl: %s\n", strerror (errno)); } - struct flock exclusive_lock; - dyad_release_flock(ctx, fd, &exclusive_lock); - rc = func_ptr(fp); - if (rc != 0) { - DPRINTF(ctx, "Failed fclose (\"%s\").\n", path); - } - IPRINTF(ctx, "DYAD_SYNC: enters fclose sync (\"%s\").\n", path); - if (DYAD_IS_ERROR(dyad_produce(ctx_mutable, path))) { - DPRINTF(ctx, "DYAD_SYNC: failed fclose sync (\"%s\").\n", path); + if (to_sync && wronly == 1) { + if (ctx->fsync_write) { + fflush (fp); + fsync (fd); +#if DYAD_SYNC_DIR + dyad_sync_directory (ctx, path); +#endif // DYAD_SYNC_DIR + } + + struct flock exclusive_lock; + dyad_release_flock (ctx, fd, &exclusive_lock); + rc = func_ptr (fp); + if (rc != 0) { + DPRINTF (ctx, "Failed fclose (\"%s\").\n", path); + } + IPRINTF (ctx, "DYAD_SYNC: enters fclose sync (\"%s\").\n", path); + if (DYAD_IS_ERROR (dyad_produce (ctx_mutable, path))) { + DPRINTF (ctx, "DYAD_SYNC: failed fclose sync (\"%s\").\n", path); + } + IPRINTF (ctx, "DYAD_SYNC: exits fclose sync (\"%s\").\n", path); + } else { + rc = func_ptr (fp); } - IPRINTF(ctx, "DYAD_SYNC: exits fclose sync (\"%s\").\n", path); - } else { - rc = func_ptr(fp); - } - DYAD_C_FUNCTION_END(); - return rc; + DYAD_C_FUNCTION_END (); + return rc; } #ifdef __cplusplus From 9fcd6f40c62d2603d76ada00632e31d725f80e62 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Mon, 16 Dec 2024 19:14:57 -0500 Subject: [PATCH 11/11] Fixes rebase issue where an old block of dyad includes were preserved --- src/dyad/core/dyad_ctx.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/dyad/core/dyad_ctx.c b/src/dyad/core/dyad_ctx.c index f64b1de9..26f16be4 100644 --- a/src/dyad/core/dyad_ctx.c +++ b/src/dyad/core/dyad_ctx.c @@ -26,14 +26,6 @@ #include #endif -#include -#include -#include -#include -#include -#include -#include - // Note: // To ensure we don't have multiple initialization, we need the following: // 1) The DYAD context (ctx below) must be static