diff --git a/.clang-format b/.clang-format index 3c2dece7082..de4f8b8e95c 100644 --- a/.clang-format +++ b/.clang-format @@ -58,8 +58,8 @@ IndentCaseLabels: true IndentGotoLabels: false #llvm11: IndentExternBlock: AfterExternBlock #llvm11: InsertTrailingCommas: None -MacroBlockBegin: "^BEGIN_FUNC" -MacroBlockEnd: "^END_FUNC" +MacroBlockBegin: "^H5_BEFORE_USER_CB*|^H5E_PAUSE_ERRORS" +MacroBlockEnd: "^H5_AFTER_USER_CB*|^H5E_RESUME_ERRORS" ObjCBlockIndentWidth: 4 #llvm11: ObjCBreakBeforeNestedBlockParam: true ReflowComments: true @@ -81,9 +81,8 @@ StatementMacros: - FUNC_LEAVE_NOAPI_NAMECHECK_ONLY - FUNC_LEAVE_NOAPI_VOID_NAMECHECK_ONLY - FUNC_LEAVE_NOAPI_NOFS + - H5E_BEGIN_TRY - H5E_END_TRY - - H5E_PRINTF - - H5E_THROW - H5_BEGIN_TAG - H5_END_TAG - H5_GCC_DIAG_OFF diff --git a/.gitattributes b/.gitattributes index 7e37bf00fee..4ae913ea368 100644 --- a/.gitattributes +++ b/.gitattributes @@ -204,7 +204,7 @@ java/src/hdf/hdf5lib/H5.java -text java/src/hdf/hdf5lib/HDF5Constants.java -text java/src/hdf/hdf5lib/HDFArray.java -text java/src/hdf/hdf5lib/HDFNativeData.java -text -java/src/hdf/hdf5lib/callbacks/Callbacks.java -text +java/src/hdf/hdf5lib/callbacks/H5Callbacks.java -text java/src/hdf/hdf5lib/callbacks/H5A_iterate_cb.java -text java/src/hdf/hdf5lib/callbacks/H5A_iterate_t.java -text java/src/hdf/hdf5lib/callbacks/H5D_append_cb.java -text diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index e34496c6a95..4d53d6a7a45 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,3 +1,3 @@ # These are supported funding model platforms -custom: "https://www.hdfgroup.org/donate" +github: hdfgroup diff --git a/.github/workflows/aocc-cmake.yml b/.github/workflows/aocc-cmake.yml index d4cff890e63..15d1e79411b 100644 --- a/.github/workflows/aocc-cmake.yml +++ b/.github/workflows/aocc-cmake.yml @@ -70,6 +70,7 @@ jobs: CC=mpicc cmake -C $GITHUB_WORKSPACE/config/cmake/cacheinit.cmake -G Ninja \ -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=OFF \ -DHDF5_ENABLE_PARALLEL:BOOL=ON \ -DHDF5_ENABLE_SUBFILING_VFD:BOOL=ON \ -DHDF5_BUILD_CPP_LIB:BOOL=OFF \ diff --git a/.github/workflows/autotools.yml b/.github/workflows/autotools.yml index f32c4bce759..74d154c4f29 100644 --- a/.github/workflows/autotools.yml +++ b/.github/workflows/autotools.yml @@ -44,10 +44,27 @@ jobs: with: build_mode: "production" + call-debug-concurrent-autotools: + name: "Autotools Debug Concurrency Workflows" + uses: ./.github/workflows/main-auto.yml + with: + concurrent: enable + thread_safety: disable + build_mode: "debug" + + call-release-concurrent-autotools: + name: "Autotools Release Concurrency Workflows" + uses: ./.github/workflows/main-auto.yml + with: + concurrent: enable + thread_safety: disable + build_mode: "production" + call-debug-thread-autotools: name: "Autotools Debug Thread-Safety Workflows" uses: ./.github/workflows/main-auto.yml with: + concurrent: disable thread_safety: enable build_mode: "debug" @@ -55,6 +72,7 @@ jobs: name: "Autotools Release Thread-Safety Workflows" uses: ./.github/workflows/main-auto.yml with: + concurrent: disable thread_safety: enable build_mode: "production" @@ -62,6 +80,7 @@ jobs: name: "Autotools Debug Workflows" uses: ./.github/workflows/main-auto.yml with: + concurrent: disable thread_safety: disable build_mode: "debug" @@ -69,6 +88,7 @@ jobs: name: "Autotools Release Workflows" uses: ./.github/workflows/main-auto.yml with: + concurrent: disable thread_safety: disable build_mode: "production" diff --git a/.github/workflows/cmake-analysis.yml b/.github/workflows/cmake-analysis.yml index aeddcc245a5..7c5a1d9466d 100644 --- a/.github/workflows/cmake-analysis.yml +++ b/.github/workflows/cmake-analysis.yml @@ -27,12 +27,12 @@ jobs: # Linux (Ubuntu) w/ gcc + coverage # name: "Ubuntu GCC Coverage" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Install CMake Dependencies (Linux_coverage) run: | sudo apt update - sudo apt-get install ninja-build doxygen graphviz curl libncurses5 build-essential + sudo apt-get install ninja-build doxygen graphviz curl build-essential sudo apt install libssl3 libssl-dev libcurl4 libcurl4-openssl-dev sudo apt-get install lcov -q -y @@ -99,6 +99,7 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_PACK_EXAMPLES:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_PACKAGE_EXTLIBS:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_NO_PACKAGES:BOOL=ON") @@ -122,19 +123,19 @@ jobs: # Linux (Ubuntu) w/ clang + LeakSanitizer # name: "Ubuntu Clang LeakSanitizer" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Install CMake Dependencies (Linux_Leak) run: | sudo apt update - sudo apt-get install ninja-build doxygen graphviz curl libncurses5 + sudo apt-get install ninja-build doxygen graphviz curl libtinfo5 - name: add clang to env uses: KyleMayes/install-llvm-action@v2.0.5 id: setup-clang with: env: true - version: '18' + version: '18.1' - name: Set file base name (Linux_Leak) id: set-file-base @@ -202,6 +203,7 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest (Linux_Leak) run: | @@ -222,19 +224,19 @@ jobs: # Linux (Ubuntu) w/ clang + AddressSanitizer # name: "Ubuntu Clang AddressSanitizer" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Install CMake Dependencies (Linux_Address) run: | sudo apt update - sudo apt-get install ninja-build doxygen graphviz curl libncurses5 + sudo apt-get install ninja-build doxygen graphviz curl libtinfo5 - name: add clang to env uses: KyleMayes/install-llvm-action@v2.0.5 id: setup-clang with: env: true - version: '18' + version: '18.1' - name: Set file base name (Linux_Address) id: set-file-base @@ -302,6 +304,7 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest (Linux_Address) run: | @@ -322,19 +325,19 @@ jobs: # Linux (Ubuntu) w/ clang + UndefinedBehaviorSanitizer # name: "Ubuntu Clang UndefinedBehaviorSanitizer" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Install CMake Dependencies (Linux_UndefinedBehavior) run: | sudo apt update - sudo apt-get install ninja-build doxygen graphviz curl libncurses5 + sudo apt-get install ninja-build doxygen graphviz curl libtinfo5 - name: add clang to env uses: KyleMayes/install-llvm-action@v2.0.5 id: setup-clang with: env: true - version: '18' + version: '18.1' - name: Set file base name (Linux_UndefinedBehavior) id: set-file-base @@ -402,6 +405,7 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest (Linux_UndefinedBehavior) run: | diff --git a/.github/workflows/cmake-ctest.yml b/.github/workflows/cmake-ctest.yml index cb67868305c..13963c64340 100644 --- a/.github/workflows/cmake-ctest.yml +++ b/.github/workflows/cmake-ctest.yml @@ -333,6 +333,12 @@ jobs: with: version: "1.9.7" + - name: check clang version + shell: bash + run: | + which clang + clang -v + - name: Install the Apple certificate and provisioning profile shell: bash env: @@ -680,7 +686,7 @@ jobs: env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cxx }} + BINSIGN: ${{ needs.check-secret.outputs.sign-state }} SIGNTOOLDIR: ${{ github.workspace }}/Microsoft.Windows.SDK.BuildTools/bin/10.0.22621.0/x64 run: | @@ -796,7 +802,6 @@ jobs: env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cxx }} run: | cd "${{ runner.workspace }}/hdf5/${{ steps.set-file-base.outputs.SOURCE_BASE }}" cmake --workflow --preset=${{ inputs.preset_name }}-Intel --fresh diff --git a/.github/workflows/cmake-par-script.yml b/.github/workflows/cmake-par-script.yml index c47b89e89d9..61b9dde0e53 100644 --- a/.github/workflows/cmake-par-script.yml +++ b/.github/workflows/cmake-par-script.yml @@ -131,9 +131,10 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_SZIP_ENCODING:BOOL=OFF") - set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest script (${{ matrix.mpi }}) run: | @@ -244,9 +245,10 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_SZIP_ENCODING:BOOL=OFF") - set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest script (${{ matrix.mpi }}) run: | @@ -353,9 +355,10 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_SZIP_ENCODING:BOOL=OFF") - set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=OF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest script (${{ matrix.mpi }}) run: | diff --git a/.github/workflows/cmake-par-source.yml b/.github/workflows/cmake-par-source.yml index 04e2cb1764f..97b8e2e4d4b 100644 --- a/.github/workflows/cmake-par-source.yml +++ b/.github/workflows/cmake-par-source.yml @@ -136,6 +136,7 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest script (OpenMPI) run: | @@ -251,6 +252,7 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest script (MPICH) run: | diff --git a/.github/workflows/cmake-script.yml b/.github/workflows/cmake-script.yml index 4b76dfc0ce8..914298d89e9 100644 --- a/.github/workflows/cmake-script.yml +++ b/.github/workflows/cmake-script.yml @@ -109,6 +109,7 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest script (Windows) run: | @@ -200,6 +201,7 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest (Linux) run: | @@ -230,6 +232,12 @@ jobs: with: version: "1.9.7" + - name: check clang version + shell: bash + run: | + which clang + clang -v + - name: Set up JDK 19 uses: actions/setup-java@v4 with: @@ -303,6 +311,7 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest (MacOS_latest) id: run-ctest @@ -391,6 +400,7 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest (Linux S3) run: | @@ -495,12 +505,12 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest (Windows_intel) with oneapi env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cc }} run: | cd "${{ runner.workspace }}/hdf5" ctest -S HDF5config.cmake,CTEST_SITE_EXT=GH-${{ github.event.repository.full_name }}-Intel,LOCAL_SUBMIT=ON,NINJA=TRUE,BUILD_GENERATOR=VS202264,CTEST_SOURCE_NAME=${{ steps.set-file-base.outputs.SOURCE_BASE }} -C Release -VV -O hdf5.log @@ -592,12 +602,12 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest (Linux_intel) env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cxx }} run: | cd "${{ runner.workspace }}/hdf5" ctest -S HDF5config.cmake,CTEST_SITE_EXT=GH-${{ github.event.repository.full_name }}-Intel,LOCAL_SUBMIT=ON,NINJA=TRUE,BUILD_GENERATOR=Unix,CTEST_SOURCE_NAME=${{ steps.set-file-base.outputs.SOURCE_BASE }} -C Release -VV -O hdf5.log @@ -617,19 +627,25 @@ jobs: # Linux (Ubuntu) w/ clang + CMake # name: "Ubuntu Clang CMake" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Install CMake Dependencies (Linux_clang) run: | sudo apt-get update - sudo apt-get install ninja-build doxygen graphviz curl libncurses5 + sudo apt-get install ninja-build doxygen graphviz curl libtinfo5 - name: add clang to env uses: KyleMayes/install-llvm-action@v2.0.5 id: setup-clang with: env: true - version: '18' + version: '18.1' + + - name: check clang version + shell: bash + run: | + which clang + clang -v - name: Set file base name (Linux_clang) id: set-file-base @@ -692,6 +708,7 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") - name: Run ctest (Linux_clang) run: | diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 0305d10c530..a1f1b0b04be 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -28,10 +28,27 @@ jobs: name: "CMake Special Workflows" uses: ./.github/workflows/main-cmake-spc.yml + call-debug-concurrent-cmake: + name: "CMake Debug Concurrency Workflows" + uses: ./.github/workflows/main-cmake.yml + with: + concurrent: "CC" + thread_safety: "" + build_mode: "Debug" + + call-release-concurrent-cmake: + name: "CMake Release Concurrency Workflows" + uses: ./.github/workflows/main-cmake.yml + with: + concurrent: "CC" + thread_safety: "" + build_mode: "Release" + call-debug-thread-cmake: name: "CMake Debug Thread-Safety Workflows" uses: ./.github/workflows/main-cmake.yml with: + concurrent: "" thread_safety: "TS" build_mode: "Debug" @@ -39,6 +56,7 @@ jobs: name: "CMake Release Thread-Safety Workflows" uses: ./.github/workflows/main-cmake.yml with: + concurrent: "" thread_safety: "TS" build_mode: "Release" @@ -46,6 +64,7 @@ jobs: name: "CMake Debug Workflows" uses: ./.github/workflows/main-cmake.yml with: + concurrent: "" thread_safety: "" build_mode: "Debug" @@ -53,6 +72,7 @@ jobs: name: "CMake Release Workflows" uses: ./.github/workflows/main-cmake.yml with: + concurrent: "" thread_safety: "" build_mode: "Release" diff --git a/.github/workflows/cve.yml b/.github/workflows/cve.yml index 8562bce6d66..94e88bb4f1f 100644 --- a/.github/workflows/cve.yml +++ b/.github/workflows/cve.yml @@ -30,14 +30,11 @@ jobs: steps: - uses: actions/checkout@v4.1.7 - - name: Install Autotools Dependencies (Linux) - run: | - sudo apt update - sudo apt install automake autoconf libtool libtool-bin - name: Install HDF5 run: | - ./autogen.sh - ./configure --prefix=/usr/local --disable-tests + mkdir "${{ runner.workspace }}/build" + cd "${{ runner.workspace }}/build" + cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_TESTING:BOOL=OFF $GITHUB_WORKSPACE make sudo make install - name: Checkout CVE test repository diff --git a/.github/workflows/cygwin-cmake.yml b/.github/workflows/cygwin-cmake.yml index 73f13f30569..f28f985b69e 100644 --- a/.github/workflows/cygwin-cmake.yml +++ b/.github/workflows/cygwin-cmake.yml @@ -103,6 +103,9 @@ jobs: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF") set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DPLUGIN_USE_LOCALCONTENT:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DENABLE_ZFP:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DENABLE_ZSTD:BOOL=OFF") - name: Run ctest (Cygwin) shell: C:\cygwin\bin\bash.exe -eo pipefail -o igncr '{0}' diff --git a/.github/workflows/h5py.yml b/.github/workflows/h5py.yml index dd9d2fb2084..1c0e10c0362 100644 --- a/.github/workflows/h5py.yml +++ b/.github/workflows/h5py.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Spack - uses: actions/checkout@v4.1.7 + uses: actions/checkout@v4.2.2 with: repository: spack/spack path: ./spack @@ -27,10 +27,8 @@ jobs: ./spack/bin/spack spec py-h5py@master+mpi ^hdf5@develop-2.0 ./spack/bin/spack install py-h5py@master+mpi ^hdf5@develop-2.0 ./spack/bin/spack install py-pytest - ./spack/bin/spack install py-ipython ./spack/bin/spack install py-pytest-mpi spack load py-h5py spack load py-pytest - spack load py-ipython spack load py-pytest-mpi python -c "import h5py; h5py.run_tests(); print(h5py.version.info);" diff --git a/.github/workflows/i386-cmake.yml b/.github/workflows/i386-cmake.yml index b7aa4cd3417..4eea1b2f801 100644 --- a/.github/workflows/i386-cmake.yml +++ b/.github/workflows/i386-cmake.yml @@ -38,13 +38,12 @@ jobs: cmake -C ../config/cmake/cacheinit.cmake -G "Unix Makefiles" \ --log-level=VERBOSE \ -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=OFF \ -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ + -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=OFF \ -DHDF5_BUILD_CPP_LIB:BOOL=OFF \ - -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF \ - -DZLIB_USE_LOCALCONTENT:BOOL=OFF \ -DHDF5_BUILD_FORTRAN:BOOL=OFF \ -DHDF5_BUILD_JAVA:BOOL=OFF \ - -DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=OFF \ .. - name: CMake Build diff --git a/.github/workflows/intel-auto.yml b/.github/workflows/intel-auto.yml index 6f80a79eebc..2f0c01847bc 100644 --- a/.github/workflows/intel-auto.yml +++ b/.github/workflows/intel-auto.yml @@ -38,7 +38,6 @@ jobs: env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cxx }} run: | sh ./autogen.sh mkdir "${{ runner.workspace }}/build" @@ -54,7 +53,6 @@ jobs: env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cxx }} run: | make -j3 working-directory: ${{ runner.workspace }}/build @@ -64,7 +62,6 @@ jobs: env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cxx }} run: | make check -j2 working-directory: ${{ runner.workspace }}/build @@ -74,7 +71,6 @@ jobs: env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cxx }} run: | make install working-directory: ${{ runner.workspace }}/build @@ -84,7 +80,6 @@ jobs: env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cxx }} run: | make check-install working-directory: ${{ runner.workspace }}/build diff --git a/.github/workflows/intel-cmake.yml b/.github/workflows/intel-cmake.yml index 8cfdcdc1e86..8e69c40800f 100644 --- a/.github/workflows/intel-cmake.yml +++ b/.github/workflows/intel-cmake.yml @@ -39,7 +39,6 @@ jobs: env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cxx }} run: | mkdir "${{ runner.workspace }}/build" cd "${{ runner.workspace }}/build" @@ -47,8 +46,6 @@ jobs: -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ -DHDF5_BUILD_FORTRAN:BOOL=ON \ -DHDF5_BUILD_CPP_LIB:BOOL=ON \ - -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF \ - -DZLIB_USE_LOCALCONTENT:BOOL=OFF \ ${{ github.workspace }} - name: CMake Build (Linux) @@ -56,7 +53,6 @@ jobs: env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cxx }} run: | cmake --build . --parallel 3 --config ${{ inputs.build_mode }} working-directory: ${{ runner.workspace }}/build @@ -66,7 +62,6 @@ jobs: env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cxx }} run: | ctest . --parallel 2 -C ${{ inputs.build_mode }} -V working-directory: ${{ runner.workspace }}/build @@ -93,7 +88,6 @@ jobs: env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cxx }} run: | mkdir "${{ runner.workspace }}/build" Set-Location -Path "${{ runner.workspace }}\\build" @@ -104,7 +98,6 @@ jobs: env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cxx }} run: | cmake --build . --parallel 3 --config ${{ inputs.build_mode }} working-directory: ${{ runner.workspace }}/build @@ -114,7 +107,6 @@ jobs: env: FC: ${{ steps.setup-fortran.outputs.fc }} CC: ${{ steps.setup-fortran.outputs.cc }} - CXX: ${{ steps.setup-fortran.outputs.cxx }} run: | ctest . --parallel 2 -C ${{ inputs.build_mode }} -V -E tfloatsattrs working-directory: ${{ runner.workspace }}/build diff --git a/.github/workflows/julia-cmake.yml b/.github/workflows/julia-cmake.yml index 1d7791b310d..55524f955ff 100644 --- a/.github/workflows/julia-cmake.yml +++ b/.github/workflows/julia-cmake.yml @@ -34,6 +34,7 @@ jobs: cd "${{ runner.workspace }}/build" cmake -C $GITHUB_WORKSPACE/config/cmake/cacheinit.cmake -G Ninja \ -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=OFF \ -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ -DHDF5_ENABLE_PARALLEL:BOOL=OFF \ -DHDF5_BUILD_CPP_LIB:BOOL=OFF \ diff --git a/.github/workflows/main-auto.yml b/.github/workflows/main-auto.yml index 83bbfd2a0c8..d0b023f4a45 100644 --- a/.github/workflows/main-auto.yml +++ b/.github/workflows/main-auto.yml @@ -8,6 +8,10 @@ on: description: "thread-safety enable/disable" required: true type: string + concurrent: + description: "concurrency enable/disable" + required: true + type: string build_mode: description: "release vs. debug build" required: true @@ -22,7 +26,7 @@ jobs: # Linux (Ubuntu) w/ gcc + Autotools # Autotools_build_and_test: - name: "GCC-${{ inputs.build_mode }}-TS=${{ inputs.thread_safety }}d" + name: "GCC-${{ inputs.build_mode }}-TS=${{ inputs.thread_safety }}d-CC=${{ inputs.concurrent }}d" # Don't run the action if the commit message says to skip CI if: "!contains(github.event.head_commit.message, 'skip-ci')" @@ -60,6 +64,7 @@ jobs: --enable-shared \ --disable-parallel \ --${{ inputs.thread_safety }}-threadsafe \ + --${{ inputs.concurrent }}-concurrency \ --enable-cxx \ --enable-fortran \ --enable-java \ @@ -68,7 +73,7 @@ jobs: --enable-ros3-vfd \ --with-szlib=yes shell: bash - if: ${{ inputs.thread_safety == 'disable' }} + if: ${{ inputs.thread_safety == 'disable' && inputs.concurrent == 'disable'}} - name: Autotools Configure (Thread-Safe) run: | @@ -79,6 +84,7 @@ jobs: --enable-build-mode=${{ inputs.build_mode }} \ --enable-shared \ --${{ inputs.thread_safety }}-threadsafe \ + --${{ inputs.concurrent }}-concurrency \ --disable-hl \ --disable-parallel \ --enable-mirror-vfd \ @@ -86,7 +92,26 @@ jobs: --enable-ros3-vfd \ --with-szlib=yes shell: bash - if: ${{ inputs.thread_safety == 'enable' }} + if: ${{ inputs.thread_safety == 'enable' && inputs.concurrent == 'disable' }} + + - name: Autotools Configure (Concurrency) + run: | + sh ./autogen.sh + mkdir "${{ runner.workspace }}/build" + cd "${{ runner.workspace }}/build" + $GITHUB_WORKSPACE/configure \ + --enable-build-mode=${{ inputs.build_mode }} \ + --enable-shared \ + --${{ inputs.thread_safety }}-threadsafe \ + --${{ inputs.concurrent }}-concurrency \ + --disable-hl \ + --disable-parallel \ + --enable-mirror-vfd \ + --enable-direct-vfd \ + --enable-ros3-vfd \ + --with-szlib=yes + shell: bash + if: ${{ inputs.thread_safety == 'disable' && inputs.concurrent == 'enable' }} - name: Autotools Build run: make -j3 diff --git a/.github/workflows/main-cmake.yml b/.github/workflows/main-cmake.yml index c59167ea6ac..37d441d6c4e 100644 --- a/.github/workflows/main-cmake.yml +++ b/.github/workflows/main-cmake.yml @@ -8,6 +8,10 @@ on: description: "TS or empty" required: true type: string + concurrent: + description: "CC or empty" + required: true + type: string build_mode: description: "release vs. debug build" required: true @@ -62,8 +66,6 @@ jobs: # Linux (Ubuntu) w/ gcc + CMake # - # We might think about adding Clang, but MacOS already tests that - # so it's not critical - name: "Ubuntu gcc" os: ubuntu-latest cpp: ON @@ -83,12 +85,10 @@ jobs: # MacOS w/ Clang + CMake # - # We could also build with the Autotools via brew installing them, - # but that seems unnecessary - name: "MacOS Clang" os: macos-latest cpp: ON - fortran: ON + fortran: OFF java: ON docs: ON libaecfc: ON @@ -98,12 +98,12 @@ jobs: parallel: OFF mirror_vfd: ON direct_vfd: OFF - ros3_vfd: OFF + ros3_vfd: ON generator: "-G Ninja" run_tests: true # Sets the job's name from the properties - name: "${{ matrix.name }}-${{ inputs.build_mode }}-${{ inputs.thread_safety }}" + name: "${{ matrix.name }}-${{ inputs.build_mode }}-${{ inputs.thread_safety }}-${{ inputs.concurrent }}" # Don't run the action if the commit message says to skip CI if: "!contains(github.event.head_commit.message, 'skip-ci')" @@ -131,17 +131,17 @@ jobs: # CMake gets libaec from fetchcontent - name: Install Dependencies (macOS) - run: brew install ninja + run: brew install ninja curl if: ${{ matrix.os == 'macos-latest' }} # symlinks the compiler executables to a common location - - name: Install GNU Fortran (macOS) - uses: fortran-lang/setup-fortran@v1 - id: setup-fortran - with: - compiler: gcc - version: 14 - if: ${{ matrix.os == 'macos-latest' }} +# - name: Install GNU Fortran (macOS) +# uses: fortran-lang/setup-fortran@v1 +# id: setup-fortran +# with: +# compiler: gcc +# version: 14 +# if: ${{ matrix.os == 'macos-latest' }} - name: Install Dependencies uses: ssciwr/doxygen-install@v1 @@ -176,6 +176,8 @@ jobs: -DHDF5_BUILD_FORTRAN:BOOL=${{ matrix.fortran }} \ -DHDF5_BUILD_JAVA:BOOL=${{ matrix.java }} \ -DHDF5_BUILD_DOC:BOOL=${{ matrix.docs }} \ + -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=${{ matrix.zlibfc }} \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=${{ matrix.libaecfc }} \ -DLIBAEC_USE_LOCALCONTENT:BOOL=${{ matrix.localaec }} \ -DZLIB_USE_LOCALCONTENT:BOOL=${{ matrix.localzlib }} \ -DHDF5_ENABLE_MIRROR_VFD:BOOL=${{ matrix.mirror_vfd }} \ @@ -186,7 +188,7 @@ jobs: -DHDF5_PACK_MACOSX_DMG:BOOL=OFF \ $GITHUB_WORKSPACE shell: bash - if: ${{ inputs.thread_safety != 'TS' }} + if: ${{ inputs.thread_safety != 'TS' && inputs.concurrent != 'CC'}} - name: CMake Configure (Thread-Safe) run: | @@ -199,12 +201,15 @@ jobs: -DBUILD_STATIC_LIBS:BOOL=${{ (matrix.os != 'windows-latest') }} \ -DHDF5_ENABLE_ALL_WARNINGS:BOOL=ON \ -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ + -DHDF5_ENABLE_CONCURRENCY:BOOL=OFF \ -DHDF5_ENABLE_PARALLEL:BOOL=${{ matrix.parallel }} \ -DHDF5_BUILD_CPP_LIB:BOOL=OFF \ -DHDF5_BUILD_FORTRAN:BOOL=OFF \ -DHDF5_BUILD_JAVA:BOOL=OFF \ -DHDF5_BUILD_HL_LIB:BOOL=OFF \ -DHDF5_BUILD_DOC:BOOL=OFF \ + -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=${{ matrix.zlibfc }} \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=${{ matrix.libaecfc }} \ -DLIBAEC_USE_LOCALCONTENT:BOOL=${{ matrix.localaec }} \ -DZLIB_USE_LOCALCONTENT:BOOL=${{ matrix.localzlib }} \ -DHDF5_ENABLE_MIRROR_VFD:BOOL=${{ matrix.mirror_vfd }} \ @@ -214,7 +219,35 @@ jobs: -DHDF5_PACK_MACOSX_DMG:BOOL=OFF \ $GITHUB_WORKSPACE shell: bash - if: ${{ inputs.thread_safety == 'TS' }} + if: ${{ inputs.thread_safety == 'TS' && inputs.concurrent != 'CC'}} + + - name: CMake Configure (Concurrency) + run: | + mkdir "${{ runner.workspace }}/build" + cd "${{ runner.workspace }}/build" + cmake -C $GITHUB_WORKSPACE/config/cmake/cacheinit.cmake \ + ${{ matrix.generator }} \ + -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_STATIC_LIBS=${{ (matrix.os != 'windows-latest') }} \ + -DHDF5_ENABLE_ALL_WARNINGS=ON \ + -DHDF5_ENABLE_THREADSAFE:BOOL=OFF \ + -DHDF5_ENABLE_CONCURRENCY:BOOL=ON \ + -DHDF5_ENABLE_PARALLEL:BOOL=${{ matrix.parallel }} \ + -DHDF5_BUILD_CPP_LIB:BOOL=OFF \ + -DHDF5_BUILD_FORTRAN:BOOL=OFF \ + -DHDF5_BUILD_JAVA:BOOL=OFF \ + -DHDF5_BUILD_HL_LIB:BOOL=OFF \ + -DHDF5_BUILD_DOC=OFF \ + -DLIBAEC_USE_LOCALCONTENT=${{ matrix.localaec }} \ + -DZLIB_USE_LOCALCONTENT=${{ matrix.localzlib }} \ + -DHDF5_ENABLE_MIRROR_VFD:BOOL=${{ matrix.mirror_vfd }} \ + -DHDF5_ENABLE_DIRECT_VFD:BOOL=${{ matrix.direct_vfd }} \ + -DHDF5_ENABLE_ROS3_VFD:BOOL=${{ matrix.ros3_vfd }} \ + -DHDF5_PACK_EXAMPLES:BOOL=ON \ + $GITHUB_WORKSPACE + shell: bash + if: ${{ inputs.thread_safety != 'TS' && inputs.concurrent == 'CC'}} # BUILD - name: CMake Build @@ -225,13 +258,7 @@ jobs: - name: CMake Run Tests run: ctest . --parallel 2 -C ${{ inputs.build_mode }} -V working-directory: ${{ runner.workspace }}/build - if: ${{ matrix.run_tests && (inputs.thread_safety != 'TS') }} - - # THREAD-SAFE - - name: CMake Run Thread-Safe Tests - run: ctest . --parallel 2 -C ${{ inputs.build_mode }} -V -R ttsafe - working-directory: ${{ runner.workspace }}/build - if: ${{ matrix.run_tests && (inputs.thread_safety == 'TS') }} + if: ${{ matrix.run_tests }} - name: CMake Run Package run: cpack -C ${{ inputs.build_mode }} -V @@ -253,7 +280,7 @@ jobs: name: zip-vs2022_cl-${{ inputs.build_mode }}-binary path: ${{ runner.workspace }}/build/HDF5-*-win64.zip if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` - if: ${{ (matrix.os == 'windows-latest') && (inputs.thread_safety != 'TS') && ( inputs.build_mode != 'Debug') }} + if: ${{ (matrix.os == 'windows-latest') && (inputs.thread_safety != 'TS') && (inputs.concurrent != 'CC') && ( inputs.build_mode != 'Debug') }} - name: Save published binary (linux) uses: actions/upload-artifact@v4 @@ -261,7 +288,7 @@ jobs: name: tgz-ubuntu-2404_gcc-${{ inputs.build_mode }}-binary path: ${{ runner.workspace }}/build/HDF5-*-Linux.tar.gz if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` - if: ${{ (matrix.os == 'ubuntu-latest') && (inputs.thread_safety != 'TS') && ( inputs.build_mode != 'Debug') }} + if: ${{ (matrix.os == 'ubuntu-latest') && (inputs.thread_safety != 'TS') && (inputs.concurrent != 'CC') && ( inputs.build_mode != 'Debug') }} - name: Save published binary (Mac_latest) uses: actions/upload-artifact@v4 @@ -269,4 +296,4 @@ jobs: name: tgz-macos14_clang-${{ inputs.build_mode }}-binary path: ${{ runner.workspace }}/build/HDF5-*-Darwin.tar.gz if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` - if: ${{ (matrix.os == 'macos-latest') && (inputs.thread_safety != 'TS') && ( inputs.build_mode != 'Debug') }} + if: ${{ (matrix.os == 'macos-latest') && (inputs.thread_safety != 'TS') && (inputs.concurrent != 'CC') && ( inputs.build_mode != 'Debug') }} diff --git a/.github/workflows/msys2-cmake.yml b/.github/workflows/msys2-cmake.yml index 363146c85d5..a1ea7ec044e 100644 --- a/.github/workflows/msys2-cmake.yml +++ b/.github/workflows/msys2-cmake.yml @@ -70,8 +70,6 @@ jobs: -DHDF5_BUILD_FORTRAN:BOOL=OFF \ -DHDF5_BUILD_JAVA:BOOL=OFF \ -DHDF5_BUILD_DOC:BOOL=OFF \ - -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF \ - -DZLIB_USE_LOCALCONTENT:BOOL=OFF \ $GITHUB_WORKSPACE - name: CMake Build diff --git a/.github/workflows/netcdf.yml b/.github/workflows/netcdf.yml index e23e45fea4b..f22f6dce913 100644 --- a/.github/workflows/netcdf.yml +++ b/.github/workflows/netcdf.yml @@ -50,8 +50,9 @@ jobs: -DHDF5_BUILD_FORTRAN:BOOL=OFF \ -DHDF5_BUILD_JAVA:BOOL=OFF \ -DHDF5_BUILD_DOC:BOOL=OFF \ - -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF \ + -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=ON \ -DZLIB_USE_LOCALCONTENT:BOOL=OFF \ + -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=ON \ -DH5_NO_DEPRECATED_SYMBOLS:BOOL=OFF \ -DBUILD_TESTING:BOOL=OFF \ -DCMAKE_INSTALL_PREFIX:PATH=/usr/local \ diff --git a/.github/workflows/nvhpc-cmake.yml b/.github/workflows/nvhpc-cmake.yml index dbd0e36f4b2..7f3b484ae1f 100644 --- a/.github/workflows/nvhpc-cmake.yml +++ b/.github/workflows/nvhpc-cmake.yml @@ -52,13 +52,10 @@ jobs: cd "${{ runner.workspace }}/build" cmake -C $GITHUB_WORKSPACE/config/cmake/cacheinit.cmake -G Ninja \ -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \ - -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ -DHDF5_ENABLE_PARALLEL:BOOL=ON \ -DMPIEXEC_NUMPROC_FLAG:STRING=-np \ -DMPIEXEC_MAX_NUMPROCS:STRING=2 \ -DHDF5_BUILD_CPP_LIB:BOOL=OFF \ - -DLIBAEC_USE_LOCALCONTENT:BOOL=OFF \ - -DZLIB_USE_LOCALCONTENT:BOOL=OFF \ -DHDF5_BUILD_FORTRAN:BOOL=ON \ -DHDF5_BUILD_JAVA:BOOL=OFF \ -DMPIEXEC_MAX_NUMPROCS:STRING="2" \ diff --git a/.github/workflows/release-files.yml b/.github/workflows/release-files.yml index a0e1e367b07..721f10fa102 100644 --- a/.github/workflows/release-files.yml +++ b/.github/workflows/release-files.yml @@ -221,7 +221,7 @@ jobs: - name: PreRelease tag id: create_prerelease if: ${{ (inputs.use_environ == 'snapshots') }} - uses: softprops/action-gh-release@e7a8f85e1c67a31e6ed99a94b41bd0b71bbee6b8 # v2.0.9 + uses: softprops/action-gh-release@01570a1f39cb168c169c802c3bceb9e93fb10974 # v2.1.0 with: tag_name: "${{ inputs.use_tag }}" prerelease: true @@ -249,7 +249,7 @@ jobs: - name: Release tag id: create_release if: ${{ (inputs.use_environ == 'release') }} - uses: softprops/action-gh-release@e7a8f85e1c67a31e6ed99a94b41bd0b71bbee6b8 # v2.0.9 + uses: softprops/action-gh-release@01570a1f39cb168c169c802c3bceb9e93fb10974 # v2.1.0 with: tag_name: "${{ inputs.use_tag }}" prerelease: false diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index a90240dd273..ed87159e5c7 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -67,6 +67,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 + uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5 with: sarif_file: results.sarif diff --git a/.github/workflows/vol_adios2.yml b/.github/workflows/vol_adios2.yml index ad781c10a28..835e7e020fe 100644 --- a/.github/workflows/vol_adios2.yml +++ b/.github/workflows/vol_adios2.yml @@ -43,6 +43,7 @@ jobs: -DHDF5_ENABLE_PARALLEL:BOOL=ON \ -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ -DHDF5_ALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=OFF \ -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ ${{ github.workspace }}/hdf5 cat src/libhdf5.settings diff --git a/.github/workflows/vol_async.yml b/.github/workflows/vol_async.yml index cd631ff4def..a48838ffeb4 100644 --- a/.github/workflows/vol_async.yml +++ b/.github/workflows/vol_async.yml @@ -55,6 +55,7 @@ jobs: -DHDF5_ENABLE_PARALLEL:BOOL=ON \ -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ -DHDF5_ALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=OFF \ -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ -DHDF5_VOL_ALLOW_EXTERNAL:STRING="GIT" \ -DHDF5_VOL_URL01:STRING="https://github.com/HDFGroup/vol-async.git" \ diff --git a/.github/workflows/vol_cache.yml b/.github/workflows/vol_cache.yml index 4f6a55ad04a..58eecfc0d5d 100644 --- a/.github/workflows/vol_cache.yml +++ b/.github/workflows/vol_cache.yml @@ -83,6 +83,7 @@ jobs: -DHDF5_ENABLE_PARALLEL:BOOL=ON \ -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ -DHDF5_ALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=OFF \ -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ -DHDF5_VOL_ALLOW_EXTERNAL:STRING="GIT" \ -DHDF5_VOL_URL01:STRING="https://github.com/HDFGroup/vol-async.git" \ diff --git a/.github/workflows/vol_ext_passthru.yml b/.github/workflows/vol_ext_passthru.yml index b9579063638..629f24cab11 100644 --- a/.github/workflows/vol_ext_passthru.yml +++ b/.github/workflows/vol_ext_passthru.yml @@ -42,6 +42,7 @@ jobs: -DBUILD_STATIC_LIBS=OFF \ -DHDF5_TEST_API:BOOL=ON \ -DHDF5_ENABLE_PARALLEL:BOOL=ON \ + -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=OFF \ -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ -DHDF5_VOL_ALLOW_EXTERNAL:STRING="GIT" \ -DHDF5_VOL_URL01:STRING="https://github.com/hpc-io/vol-external-passthrough.git" \ diff --git a/.github/workflows/vol_log.yml b/.github/workflows/vol_log.yml index 20c4a816ed9..5dadbe6e8c5 100644 --- a/.github/workflows/vol_log.yml +++ b/.github/workflows/vol_log.yml @@ -41,6 +41,7 @@ jobs: -DHDF5_ENABLE_PARALLEL:BOOL=ON \ -DHDF5_ENABLE_THREADSAFE:BOOL=ON \ -DHDF5_ALLOW_UNSUPPORTED:BOOL=ON \ + -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=ON \ -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ ${{ github.workspace }}/hdf5 cat src/libhdf5.settings diff --git a/.github/workflows/vol_rest.yml b/.github/workflows/vol_rest.yml index fe964453a4c..3e5d0713016 100644 --- a/.github/workflows/vol_rest.yml +++ b/.github/workflows/vol_rest.yml @@ -57,8 +57,8 @@ jobs: -DHDF5_BUILD_HL_LIB:BOOL=ON \ -DHDF5_TEST_API:BOOL=ON \ -DHDF5_ALLOW_UNSUPPORTED:BOOL=ON \ - -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=OFF \ + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \ -DHDF5_VOL_ALLOW_EXTERNAL:STRING="GIT" \ -DHDF5_VOL_URL01:STRING="https://github.com/HDFGroup/vol-rest.git" \ -DHDF5_VOL_VOL-REST_BRANCH:STRING="master" \ diff --git a/CMakeFilters.cmake b/CMakeFilters.cmake index 7bf152aa357..595ecb2d863 100644 --- a/CMakeFilters.cmake +++ b/CMakeFilters.cmake @@ -14,6 +14,8 @@ option (HDF5_USE_ZLIB_STATIC "Find static zlib library" OFF) option (HDF5_USE_LIBAEC_STATIC "Find static AEC library" OFF) option (ZLIB_USE_EXTERNAL "Use External Library Building for ZLIB" OFF) option (SZIP_USE_EXTERNAL "Use External Library Building for SZIP" OFF) +option (ZLIB_USE_LOCALCONTENT "Use local file for ZLIB FetchContent" OFF) +option (LIBAEC_USE_LOCALCONTENT "Use local file for LIBAEC FetchContent" OFF) if (NOT ZLIB_USE_LOCALCONTENT) if (HDF5_USE_ZLIB_NG) @@ -77,7 +79,13 @@ endif () # Option for ZLib support #----------------------------------------------------------------------------- set(H5_ZLIB_FOUND FALSE) -option (HDF5_ENABLE_ZLIB_SUPPORT "Enable Zlib Filters" ON) +if(NOT DEFINED ZLIB_PACKAGE_NAME) + set(ZLIB_PACKAGE_NAME "zlib") +endif () +if(NOT DEFINED ZLIBNG_PACKAGE_NAME) + set(ZLIBNG_PACKAGE_NAME "zlib-ng") +endif () +option (HDF5_ENABLE_ZLIB_SUPPORT "Enable Zlib Filters" OFF) if (HDF5_ENABLE_ZLIB_SUPPORT) if (NOT H5_ZLIB_HEADER) if (NOT ZLIB_USE_EXTERNAL) @@ -88,11 +96,12 @@ if (HDF5_ENABLE_ZLIB_SUPPORT) endif () set(ZLIB_FOUND FALSE) if (HDF5_USE_ZLIB_STATIC) - set(ZLIB_SEACH_TYPE static) + set(ZLIB_SEARCH_TYPE static) else () - set(ZLIB_SEACH_TYPE shared) + set(ZLIB_SEARCH_TYPE shared) endif () - find_package (ZLIB NAMES ${PACKAGE_NAME} COMPONENTS ${ZLIB_SEACH_TYPE}) + # Search pure Config mode first + find_package (ZLIB NAMES ${PACKAGE_NAME} OPTIONAL_COMPONENTS ${ZLIB_SEARCH_TYPE}) if (NOT ZLIB_FOUND) if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") set(ZLIB_USE_STATIC_LIBS ${HDF5_USE_ZLIB_STATIC}) @@ -140,16 +149,19 @@ if (HDF5_ENABLE_ZLIB_SUPPORT) message (VERBOSE "Filter HDF5_ZLIB is ON") else () set (HDF5_ENABLE_ZLIB_SUPPORT OFF CACHE BOOL "" FORCE) - message (WARNING " ZLib support in HDF5 was enabled but not found") + message (FATAL_ERROR " ZLib support in HDF5 was enabled but not found") endif () - message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}") + message (VERBOSE "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}") endif () #----------------------------------------------------------------------------- # Option for SzLib support #----------------------------------------------------------------------------- set(H5_SZIP_FOUND FALSE) -option (HDF5_ENABLE_SZIP_SUPPORT "Use SZip Filter" ON) +if(NOT DEFINED LIBAEC_PACKAGE_NAME) + set(LIBAEC_PACKAGE_NAME "libaec") +endif () +option (HDF5_ENABLE_SZIP_SUPPORT "Use SZip Filter" OFF) if (HDF5_ENABLE_SZIP_SUPPORT) option (HDF5_ENABLE_SZIP_ENCODING "Use SZip Encoding" ON) if (NOT SZIP_USE_EXTERNAL) @@ -160,24 +172,27 @@ if (HDF5_ENABLE_SZIP_SUPPORT) endif () set(libaec_USE_STATIC_LIBS ${HDF5_USE_LIBAEC_STATIC}) set(SZIP_FOUND FALSE) - find_package (SZIP NAMES ${LIBAEC_PACKAGE_NAME}${HDF_PACKAGE_EXT} COMPONENTS ${LIBAEC_SEARCH_TYPE}) - if (NOT SZIP_FOUND) - find_package (SZIP) # Legacy find - endif () - set(H5_SZIP_FOUND ${SZIP_FOUND}) + # Search pure Config mode, there is not a FindSZIP module available + find_package (${LIBAEC_PACKAGE_NAME} NAMES ${LIBAEC_PACKAGE_NAME}${HDF_PACKAGE_EXT} OPTIONAL_COMPONENTS ${LIBAEC_SEARCH_TYPE}) + set(H5_SZIP_FOUND ${${LIBAEC_PACKAGE_NAME}_FOUND}) if (H5_SZIP_FOUND) set (H5_SZIP_INCLUDE_DIR_GEN ${SZIP_INCLUDE_DIR}) set (H5_SZIP_INCLUDE_DIRS ${H5_SZIP_INCLUDE_DIRS} ${SZIP_INCLUDE_DIR}) - set (LINK_COMP_LIBS ${LINK_COMP_LIBS} ${SZIP_LIBRARIES}) + if(LIBAEC_PACKAGE_NAME STREQUAL "libaec") + set (LINK_COMP_LIBS ${LINK_COMP_LIBS} libaec::sz libaec::aec) + else () + set (LINK_COMP_LIBS ${LINK_COMP_LIBS} ${SZIP_LIBRARIES}) + endif () endif () + message (VERBOSE "H5_SZIP_FOUND=${SZIP_FOUND} and LINK_COMP_LIBS=${LINK_COMP_LIBS}") else () if (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT" OR HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "TGZ") EXTERNAL_SZIP_LIBRARY (${HDF5_ALLOW_EXTERNAL_SUPPORT} ${HDF5_ENABLE_SZIP_ENCODING}) - message (VERBOSE "Filter SZIP is built") - message (VERBOSE "... with library AEC") + message (VERBOSE "Filter SZIP is built using library AEC") set (LINK_COMP_LIBS ${LINK_COMP_LIBS} ${H5_SZIP_STATIC_LIBRARY}) endif () endif () + message (VERBOSE "LINK_COMP_LIBS=${LINK_COMP_LIBS}") if (H5_SZIP_FOUND) set (H5_HAVE_FILTER_SZIP 1) set (H5_HAVE_SZLIB_H 1) @@ -193,6 +208,6 @@ if (HDF5_ENABLE_SZIP_SUPPORT) endif () else () set (HDF5_ENABLE_SZIP_SUPPORT OFF CACHE BOOL "" FORCE) - message (WARNING "SZIP support in HDF5 was enabled but not found") + message (FATAL_ERROR "SZIP support in HDF5 was enabled but not found") endif () endif () diff --git a/CMakeInstallation.cmake b/CMakeInstallation.cmake index 6211c959e99..7d9199f1fa2 100644 --- a/CMakeInstallation.cmake +++ b/CMakeInstallation.cmake @@ -465,11 +465,12 @@ The HDF5 data model, file format, API, library, and tools are open and distribut endif () endif () if (H5_SZIP_FOUND AND SZIP_USE_EXTERNAL) + set (SZIP_PROJNAME "${LIBAEC_PACKAGE_NAME}") if (WIN32) - set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${H5_SZIP_INCLUDE_DIR_GEN};SZIP;ALL;/") + set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${H5_SZIP_INCLUDE_DIR_GEN};${SZIP_PROJNAME};ALL;/") else () - set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${H5_SZIP_INCLUDE_DIR_GEN};SZIP;libraries;/") - set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${H5_SZIP_INCLUDE_DIR_GEN};SZIP;configinstall;/") + set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${H5_SZIP_INCLUDE_DIR_GEN};${SZIP_PROJNAME};libraries;/") + set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${H5_SZIP_INCLUDE_DIR_GEN};${SZIP_PROJNAME};configinstall;/") endif () endif () if (PLUGIN_FOUND AND PLUGIN_USE_EXTERNAL) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c8366d8393..48c25154dc7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -920,7 +920,11 @@ include (UserMacros.cmake) #----------------------------------------------------------------------------- # Include filter (zlib, szip, etc.) macros #----------------------------------------------------------------------------- -include (CMakeFilters.cmake) +option (HDF5_ENABLE_SZIP_SUPPORT "Use SZip Filter" OFF) +option (HDF5_ENABLE_ZLIB_SUPPORT "Enable Zlib Filters" OFF) +if (HDF5_ENABLE_ZLIB_SUPPORT OR HDF5_ENABLE_SZIP_SUPPORT) + include (CMakeFilters.cmake) +endif () #----------------------------------------------------------------------------- # Include external VOL connectors @@ -985,6 +989,59 @@ if (HDF5_ENABLE_THREADSAFE) set (H5_HAVE_THREADSAFE 1) endif () +#----------------------------------------------------------------------------- +# Option to use multi-threaded concurrency +#----------------------------------------------------------------------------- +option (HDF5_ENABLE_CONCURRENCY "Enable multi-threaded concurrency" OFF) +if (HDF5_ENABLE_CONCURRENCY) + # check for unsupported options + if (WIN32) + if (BUILD_STATIC_LIBS) + message (FATAL_ERROR " **** multi-threaded concurrency option not supported with static library **** ") + endif () + endif () + if (HDF_ENABLE_PARALLEL) + if (NOT ALLOW_UNSUPPORTED) + message (FATAL_ERROR " **** Parallel and multi-threaded concurrency options are not supported, override with ALLOW_UNSUPPORTED option **** ") + else () + message (VERBOSE " **** Allowing unsupported parallel and multi-threaded concurrency options **** ") + endif () + endif () + if (HDF5_BUILD_FORTRAN) + if (NOT ALLOW_UNSUPPORTED) + message (FATAL_ERROR " **** Fortran and multi-threaded concurrency options are not supported, override with ALLOW_UNSUPPORTED option **** ") + else () + message (VERBOSE " **** Allowing unsupported Fortran and multi-threaded concurrency options **** ") + endif () + endif () + if (HDF5_BUILD_CPP_LIB) + if (NOT ALLOW_UNSUPPORTED) + message (FATAL_ERROR " **** C++ and multi-threaded concurrency options are not supported, override with ALLOW_UNSUPPORTED option **** ") + else () + message (VERBOSE " **** Allowing unsupported C++ and multi-threaded concurrency options **** ") + endif () + endif () + if (HDF5_BUILD_HL_LIB) + if (NOT ALLOW_UNSUPPORTED) + message (FATAL_ERROR " **** HL and multi-threaded concurrency options are not supported, override with ALLOW_UNSUPPORTED option **** ") + else () + message (VERBOSE " **** Allowing unsupported HL and multi-threaded concurrency options **** ") + endif () + endif () + + # Check for threading package + if (NOT Threads_FOUND) + message (FATAL_ERROR " **** multi-threaded concurrency option requires a threading package and none was found **** ") + endif () + + # Multi-threaded concurrency and threadsafe options are mutually exclusive + if (HDF5_ENABLE_THREADSAFE) + message (FATAL_ERROR " **** multi-threaded concurrency and threadsafe options are mutually exclusive **** ") + endif () + + set (H5_HAVE_CONCURRENCY 1) +endif () + #----------------------------------------------------------------------------- # Option to build the map API #----------------------------------------------------------------------------- @@ -1026,10 +1083,10 @@ if (HDF5_BUILD_DOC AND EXISTS "${HDF5_DOXYGEN_DIR}" AND IS_DIRECTORY "${HDF5_DOX else () set (HDF5_DOXY_WARNINGS "NO") endif () - message(STATUS "Doxygen version: ${DOXYGEN_VERSION}") + message(VERBOSE "Doxygen version: ${DOXYGEN_VERSION}") add_subdirectory (doxygen) else () - message(STATUS "Doxygen needs to be installed to generate the doxygen documentation") + message(WARNING "Doxygen needs to be installed to generate the doxygen documentation") endif () endif () @@ -1048,22 +1105,22 @@ if (EXISTS "${HDF5_SOURCE_DIR}/utils" AND IS_DIRECTORY "${HDF5_SOURCE_DIR}/utils option (HDF5_BUILD_PARALLEL_TOOLS "Build Parallel HDF5 Tools" OFF) if (HDF5_BUILD_PARALLEL_TOOLS AND HDF5_ENABLE_PARALLEL) set (CMAKE_PREFIX_PATH "$HDF_RESOURCES_DIR") - find_package(MFU REQUIRED) + find_package (MFU REQUIRED) if (MFU_FOUND) - message(STATUS "LL_PATH=${LL_PATH}") + message (VERBOSE "LL_PATH=${LL_PATH}") set (H5_HAVE_LIBMFU 1) set (H5_HAVE_MFU_H 1) set (CMAKE_REQUIRED_INCLUDES "${MFU_INCLUDE_DIR}") set (MFU_LIBRARY_DEBUG "$MFU_LIBRARY") set (MFU_LIBRARY_RELEASE "$MFU_LIBRARY") endif () - find_package(CIRCLE REQUIRED) + find_package (CIRCLE REQUIRED) if (CIRCLE_FOUND) set (H5_HAVE_LIBCIRCLE 1) set (H5_HAVE_CIRCLE_H 1) set (CMAKE_REQUIRED_INCLUDES "${CIRCLE_INCLUDE_DIR}") endif () - find_package(DTCMP REQUIRED) + find_package (DTCMP REQUIRED) if (DTCMP_FOUND) set (H5_HAVE_LIBDTCMP 1) set (H5_HAVE_DTCMP_H 1) @@ -1086,19 +1143,16 @@ endif () #----------------------------------------------------------------------------- # Include filter plugins #----------------------------------------------------------------------------- -if (${H5_LIBVER_DIR} EQUAL 16 OR HDF5_DEFAULT_API_VERSION MATCHES "v16") - set (HDF5_ENABLE_PLUGIN_SUPPORT OFF CACHE BOOL "" FORCE) - message (VERBOSE "Filter PLUGINs cannot be used with 1.6 API") -else () - include (CMakePlugins.cmake) +option (HDF5_ENABLE_PLUGIN_SUPPORT "Enable PLUGIN Filters" OFF) +if (HDF5_ENABLE_PLUGIN_SUPPORT) + if (${H5_LIBVER_DIR} EQUAL 16 OR HDF5_DEFAULT_API_VERSION MATCHES "v16") + set (HDF5_ENABLE_PLUGIN_SUPPORT OFF CACHE BOOL "" FORCE) + message (VERBOSE "Filter PLUGINs cannot be used with 1.6 API") + else () + include (CMakePlugins.cmake) - if (HDF5_PACKAGE_EXTLIBS AND NOT HDF5_NO_PACKAGES) - if (HDF5_ENABLE_PLUGIN_SUPPORT AND PLUGIN_FOUND) + if (HDF5_PACKAGE_EXTLIBS AND NOT HDF5_NO_PACKAGES AND PLUGIN_FOUND) PACKAGE_PLUGIN_LIBRARY (${HDF5_ALLOW_EXTERNAL_SUPPORT}) -# option (HDF5_TEST_PLUGIN "Execute plugin tests" ON) -# mark_as_advanced (HDF5_TEST_PLUGIN) - -# TEST_PLUGIN_LIBRARY () endif () endif () endif () diff --git a/CMakePlugins.cmake b/CMakePlugins.cmake index 5fccb375224..91749c26cfd 100644 --- a/CMakePlugins.cmake +++ b/CMakePlugins.cmake @@ -10,6 +10,7 @@ # help@hdfgroup.org. # option (PLUGIN_USE_EXTERNAL "Use External Library Building for filter PLUGIN else search" OFF) +option (PLUGIN_USE_LOCALCONTENT "Use local file for PLUGIN FetchContent" OFF) if (NOT PLUGIN_USE_LOCALCONTENT) set (PLUGIN_URL ${PLUGIN_TGZ_ORIGPATH}/${PLUGIN_TGZ_NAME}) @@ -19,7 +20,7 @@ else () endif () set (PLUGIN_URL ${H5PL_TGZPATH}/${PLUGIN_TGZ_NAME}) endif () -message (STATUS "Filter PLUGIN file is ${PLUGIN_URL}") +message (VERBOSE "Filter PLUGIN file is ${PLUGIN_URL}") include (ExternalProject) #option (HDF5_ALLOW_EXTERNAL_SUPPORT "Allow External Library Building (NO GIT TGZ)" "NO") @@ -49,21 +50,22 @@ endif () #----------------------------------------------------------------------------- # Option for PLUGIN support #----------------------------------------------------------------------------- -option (HDF5_ENABLE_PLUGIN_SUPPORT "Enable PLUGIN Filters" OFF) if (HDF5_ENABLE_PLUGIN_SUPPORT) if (NOT PLUGIN_USE_EXTERNAL) find_package (PLUGIN NAMES ${PLUGIN_PACKAGE_NAME}${HDF_PACKAGE_EXT}) if (NOT PLUGIN_FOUND) find_package (PLUGIN) # Legacy find endif () - endif () - if (NOT PLUGIN_FOUND) + else () if (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT" OR HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "TGZ") EXTERNAL_PLUGIN_LIBRARY (${HDF5_ALLOW_EXTERNAL_SUPPORT}) message (STATUS "Filter PLUGIN is built") - else () - message (FATAL_ERROR " PLUGIN is Required for PLUGIN support in HDF5") endif () endif () - message (STATUS "Filter PLUGIN is ON") + if (PLUGIN_FOUND) + message (STATUS "Filter PLUGIN is ON") + else () + set (HDF5_ENABLE_PLUGIN_SUPPORT OFF CACHE BOOL "" FORCE) + message (FATAL_ERROR " PLUGIN support in HDF5 was enabled but not found") + endif () endif () diff --git a/CMakeTests.cmake b/CMakeTests.cmake index 6b83d5652c1..2bd2ea19944 100644 --- a/CMakeTests.cmake +++ b/CMakeTests.cmake @@ -26,7 +26,7 @@ option (HDF5_DISABLE_TESTS_REGEX "Regex pattern to set execution of specific tests to DISABLED" "") mark_as_advanced (HDF5_DISABLE_TESTS_REGEX) - option (HDF5_TEST_API "Execute HDF5 API tests" OFF) + option (HDF5_TEST_API "Execute HDF5 API tests" ON) mark_as_advanced (HDF5_TEST_API) if (HDF5_TEST_API) option (HDF5_TEST_API_INSTALL "Install HDF5 API tests" OFF) diff --git a/HDF5Examples/CMakeLists.txt b/HDF5Examples/CMakeLists.txt index 3fd1a7eb0da..6efd8afebad 100644 --- a/HDF5Examples/CMakeLists.txt +++ b/HDF5Examples/CMakeLists.txt @@ -86,7 +86,7 @@ if (H5EX_ENABLE_PARALLEL) CHECK_SYMBOL_EXISTS (MPI_Comm_c2f "${MPI_C_INCLUDE_DIRS}/mpi.h" H5_HAVE_MPI_MULTI_LANG_Comm) CHECK_SYMBOL_EXISTS (MPI_Info_c2f "${MPI_C_INCLUDE_DIRS}/mpi.h" H5_HAVE_MPI_MULTI_LANG_Info) else () - message (STATUS "Parallel libraries not found") + message (WARNING "Parallel libraries not found") endif () endif () diff --git a/HDF5Examples/Using_CMake.txt b/HDF5Examples/Using_CMake.txt index c543300092b..fc13645009e 100644 --- a/HDF5Examples/Using_CMake.txt +++ b/HDF5Examples/Using_CMake.txt @@ -5,7 +5,7 @@ Notes: This short instruction is written for users who want to quickly build HDF5 Examples using the HDF5 binary package using the CMake tools. - More information about using CMake can be found at the KitWare + More information about using CMake can be found at the Kitware site, www.cmake.org. CMake uses the command line; however, the visual CMake tool is diff --git a/LICENSE b/LICENSE index 38061fb7854..a660f0c14ae 100644 --- a/LICENSE +++ b/LICENSE @@ -1,103 +1,103 @@ -Copyright Notice and License Terms for +Copyright Notice and License Terms for HDF5 (Hierarchical Data Format 5) Software Library and Utilities ----------------------------------------------------------------------------- HDF5 (Hierarchical Data Format 5) Software Library and Utilities -Copyright 2006 by The HDF Group. +Copyright 2006 by The HDF Group. NCSA HDF5 (Hierarchical Data Format 5) Software Library and Utilities -Copyright 1998-2006 by The Board of Trustees of the University of Illinois. +Copyright 1998-2006 by The Board of Trustees of the University of Illinois. All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted for any purpose (including commercial purposes) +Redistribution and use in source and binary forms, with or without +modification, are permitted for any purpose (including commercial purposes) provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright notice, +1. Redistributions of source code must retain the above copyright notice, this list of conditions, and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions, and the following disclaimer in the documentation +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation and/or materials provided with the distribution. -3. Neither the name of The HDF Group, the name of the University, nor the - name of any Contributor may be used to endorse or promote products derived - from this software without specific prior written permission from +3. Neither the name of The HDF Group, the name of the University, nor the + name of any Contributor may be used to endorse or promote products derived + from this software without specific prior written permission from The HDF Group, the University, or the Contributor, respectively. -DISCLAIMER: -THIS SOFTWARE IS PROVIDED BY THE HDF GROUP AND THE CONTRIBUTORS -"AS IS" WITH NO WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED. IN NO -EVENT SHALL THE HDF GROUP OR THE CONTRIBUTORS BE LIABLE FOR ANY DAMAGES -SUFFERED BY THE USERS ARISING OUT OF THE USE OF THIS SOFTWARE, EVEN IF +DISCLAIMER: +THIS SOFTWARE IS PROVIDED BY THE HDF GROUP AND THE CONTRIBUTORS +"AS IS" WITH NO WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED. IN NO +EVENT SHALL THE HDF GROUP OR THE CONTRIBUTORS BE LIABLE FOR ANY DAMAGES +SUFFERED BY THE USERS ARISING OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You are under no obligation whatsoever to provide any bug fixes, patches, or -upgrades to the features, functionality or performance of the source code -("Enhancements") to anyone; however, if you choose to make your Enhancements -available either publicly, or directly to The HDF Group, without imposing a -separate written license agreement for such Enhancements, then you hereby -grant the following license: a non-exclusive, royalty-free perpetual license -to install, use, modify, prepare derivative works, incorporate into other -computer software, distribute, and sublicense such enhancements or derivative + +You are under no obligation whatsoever to provide any bug fixes, patches, or +upgrades to the features, functionality or performance of the source code +("Enhancements") to anyone; however, if you choose to make your Enhancements +available either publicly, or directly to The HDF Group, without imposing a +separate written license agreement for such Enhancements, then you hereby +grant the following license: a non-exclusive, royalty-free perpetual license +to install, use, modify, prepare derivative works, incorporate into other +computer software, distribute, and sublicense such enhancements or derivative works thereof, in binary and source code form. ----------------------------------------------------------------------------- ----------------------------------------------------------------------------- -Limited portions of HDF5 1.12.0 were developed by Lawrence Berkeley National +Limited portions of HDF5 1.12.0 were developed by Lawrence Berkeley National Laboratory (LBNL). LBNL's Copyright Notice and Licensing Terms can be found in the LICENSE_LBNL_HDF5 file in this directory. ----------------------------------------------------------------------------- ----------------------------------------------------------------------------- -Contributors: National Center for Supercomputing Applications (NCSA) at -the University of Illinois, Fortner Software, Unidata Program Center -(netCDF), The Independent JPEG Group (JPEG), Jean-loup Gailly and Mark Adler +Contributors: National Center for Supercomputing Applications (NCSA) at +the University of Illinois, Fortner Software, Unidata Program Center +(netCDF), The Independent JPEG Group (JPEG), Jean-loup Gailly and Mark Adler (gzip), and Digital Equipment Corporation (DEC). ----------------------------------------------------------------------------- - -Portions of HDF5 were developed with support from the Lawrence Berkeley -National Laboratory (LBNL) and the United States Department of Energy + +Portions of HDF5 were developed with support from the Lawrence Berkeley +National Laboratory (LBNL) and the United States Department of Energy under Prime Contract No. DE-AC02-05CH11231. ----------------------------------------------------------------------------- -Portions of HDF5 were developed with support from Lawrence Livermore -National Laboratory and the United States Department of Energy under +Portions of HDF5 were developed with support from Lawrence Livermore +National Laboratory and the United States Department of Energy under Prime Contract No. DE-AC52-07NA27344. ----------------------------------------------------------------------------- -Portions of HDF5 were developed with support from the University of -California, Lawrence Livermore National Laboratory (UC LLNL). -The following statement applies to those portions of the product and must -be retained in any redistribution of source code, binaries, documentation, +Portions of HDF5 were developed with support from the University of +California, Lawrence Livermore National Laboratory (UC LLNL). +The following statement applies to those portions of the product and must +be retained in any redistribution of source code, binaries, documentation, and/or accompanying materials: - This work was partially produced at the University of California, - Lawrence Livermore National Laboratory (UC LLNL) under contract - no. W-7405-ENG-48 (Contract 48) between the U.S. Department of Energy - (DOE) and The Regents of the University of California (University) + This work was partially produced at the University of California, + Lawrence Livermore National Laboratory (UC LLNL) under contract + no. W-7405-ENG-48 (Contract 48) between the U.S. Department of Energy + (DOE) and The Regents of the University of California (University) for the operation of UC LLNL. - DISCLAIMER: - THIS WORK WAS PREPARED AS AN ACCOUNT OF WORK SPONSORED BY AN AGENCY OF - THE UNITED STATES GOVERNMENT. NEITHER THE UNITED STATES GOVERNMENT NOR - THE UNIVERSITY OF CALIFORNIA NOR ANY OF THEIR EMPLOYEES, MAKES ANY - WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LIABILITY OR RESPONSIBILITY - FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, - APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE - WOULD NOT INFRINGE PRIVATELY- OWNED RIGHTS. REFERENCE HEREIN TO ANY - SPECIFIC COMMERCIAL PRODUCTS, PROCESS, OR SERVICE BY TRADE NAME, - TRADEMARK, MANUFACTURER, OR OTHERWISE, DOES NOT NECESSARILY CONSTITUTE - OR IMPLY ITS ENDORSEMENT, RECOMMENDATION, OR FAVORING BY THE UNITED - STATES GOVERNMENT OR THE UNIVERSITY OF CALIFORNIA. THE VIEWS AND - OPINIONS OF AUTHORS EXPRESSED HEREIN DO NOT NECESSARILY STATE OR REFLECT - THOSE OF THE UNITED STATES GOVERNMENT OR THE UNIVERSITY OF CALIFORNIA, + DISCLAIMER: + THIS WORK WAS PREPARED AS AN ACCOUNT OF WORK SPONSORED BY AN AGENCY OF + THE UNITED STATES GOVERNMENT. NEITHER THE UNITED STATES GOVERNMENT NOR + THE UNIVERSITY OF CALIFORNIA NOR ANY OF THEIR EMPLOYEES, MAKES ANY + WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LIABILITY OR RESPONSIBILITY + FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, + APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE + WOULD NOT INFRINGE PRIVATELY- OWNED RIGHTS. REFERENCE HEREIN TO ANY + SPECIFIC COMMERCIAL PRODUCTS, PROCESS, OR SERVICE BY TRADE NAME, + TRADEMARK, MANUFACTURER, OR OTHERWISE, DOES NOT NECESSARILY CONSTITUTE + OR IMPLY ITS ENDORSEMENT, RECOMMENDATION, OR FAVORING BY THE UNITED + STATES GOVERNMENT OR THE UNIVERSITY OF CALIFORNIA. THE VIEWS AND + OPINIONS OF AUTHORS EXPRESSED HEREIN DO NOT NECESSARILY STATE OR REFLECT + THOSE OF THE UNITED STATES GOVERNMENT OR THE UNIVERSITY OF CALIFORNIA, AND SHALL NOT BE USED FOR ADVERTISING OR PRODUCT ENDORSEMENT PURPOSES. ----------------------------------------------------------------------------- diff --git a/README.md b/README.md index 1f3c6444a1c..4aa6ff84a10 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ are tentative. | Release | New Features | | ------- | ------------ | -| 2.0.0 | Drop Autotools support, drop the C++ API, drop the HDF5 <--> GIF tools, add complex number support, update library defaults (cache sizes, etc.), use semantic versioning | +| 2.0.0 | Drop Autotools support, drop the HDF5 <--> GIF tools, add complex number support, update library defaults (cache sizes, etc.), use semantic versioning | | FUTURE | Multi-threaded HDF5, crashproofing / metadata journaling, Full (VFD) SWMR, encryption, digital signatures, sparse datasets, improved storage for variable-length datatypes, better Unicode support (especially on Windows) | NOTE: In the March 2025 release we will begin using semantic versioning (https://semver.org/) and the previously announced 1.16.0 version will instead be numbered 2.0.0. diff --git a/bin/make_vers b/bin/make_vers index ba335cae27e..6f93b20510d 100755 --- a/bin/make_vers +++ b/bin/make_vers @@ -3,20 +3,49 @@ require 5.003; use warnings; # Global settings -# (The max_idx parameter is the only thing that needs to be changed when adding -# support for a new major release. If support for a prior major release -# is added (like support for 1.4, etc), the min_sup_idx parameter will -# need to be decremented.) -# Max. library "index" (0 = v1.0, 1 = 1.2, 2 = 1.4, 3 = 1.6, 4 = 1.8, 5 = 1.10, 6 = 1.12, 7 = 1.14, 8 = 2.0, etc) -$max_idx = 8; - -# Min. supported previous library version "index" (0 = v1.0, 1 = 1.2, etc) -$min_sup_idx = 3; +# Supported version strings. "16" = HDF5 1.6.0, "200" = HDF5 2.0.0, etc. +# +# Note that the scheme changed with 2.0.0, when we went to semantic versioning. +# We use 200 instead of 20 so that HDF5 11.0.0 will get 1100 instead of 110, +# which would conflict with HDF5 1.10. +# +# Also, note that this scheme started at the 1.6/1.8 transition, so earlier +# versions of the API aren't versioned. +@versions = ("16", "18", "110", "112", "114", "200"); # Number of spaces to indent preprocessor commands inside ifdefs $indent = 2; +# Global hash of functions ==> # symbol versions parsed from H5vers.txt +$functions = {}; + +# Global hash of functions ==> versioned function params parsed from H5vers.txt +$func_params = {}; + +# Global hash of typedefs ==> # symbol versions parsed from H5vers.txt +$typedefs = {}; + +# Hash of API versions to a hash of (API call name --> symbol version) +# +# There is one hash for each version in @versions and the (API call name +# --> symbol version) hash maps an API name like H5Dopen to the symbol version +# (e.g. H5Dopen --> 1 or 2). +# +# So... +# +# 200 --> {H5Dopen --> 2, H5Iregister_type --> 2, etc.} +my %api_vers_to_function_vers; +foreach my $key (@versions) { + $api_vers_to_function_vers{$key} = {}; +} + +# Hash of API versions to a hash of (typedef name --> symbol version) +my %api_vers_to_type_vers; +foreach my $key (@versions) { + $api_vers_to_type_vers{$key} = {}; +} + # # Copyright by The HDF Group. # All rights reserved. @@ -83,7 +112,6 @@ sub print_startprotect ($$) { # sub print_checkoptions ($) { my $fh = shift; # File handle for output file - my $curr_idx; # Current API version index # Print the option checking print $fh "\n\n/* Issue error if contradicting macros have been defined. */\n"; @@ -91,9 +119,12 @@ sub print_checkoptions ($) { # Print the #ifdef print $fh "#if ("; - for $curr_idx ($min_sup_idx .. ($max_idx - 1)) { - print $fh "defined(H5_USE_1", ($curr_idx * 2), "_API)"; - if($curr_idx < ($max_idx - 1)) { + for my $i (0 .. $#versions - 1) { + print $fh "defined(H5_USE_", $versions[$i], "_API)"; + + # -2 because we're ignoring the last version in the array, and the + # last version we DO write out can't have an || after it + if ($i < @versions - 2) { print $fh " || "; } } @@ -104,9 +135,10 @@ sub print_checkoptions ($) { # Print the #endif print $fh "#endif /* ("; - for $curr_idx ($min_sup_idx .. ($max_idx - 1)) { - print $fh "defined(H5_USE_1", ($curr_idx * 2), "_API)"; - if($curr_idx < ($max_idx - 1)) { + for my $i (0 .. $#versions - 1) { + print $fh "defined(H5_USE_", $versions[$i], "_API)"; + + if ($i < @versions - 2) { print $fh " || "; } } @@ -118,7 +150,6 @@ sub print_checkoptions ($) { # sub print_globalapidefvers ($) { my $fh = shift; # File handle for output file - my $curr_idx; # Current API version index # Print the descriptive comment print $fh "\n\n/* If a particular default \"global\" version of the library's interfaces is\n"; @@ -126,13 +157,13 @@ sub print_globalapidefvers ($) { print $fh " *\n"; print $fh " */\n"; - for $curr_idx ($min_sup_idx .. ($max_idx - 1)) { + for my $api_vers (@versions) { # Print API version ifdef - print $fh "\n#if defined(H5_USE_1", ($curr_idx * 2), "_API_DEFAULT) && !defined(H5_USE_1", ($curr_idx * 2), "_API)\n"; + print $fh "\n#if defined(H5_USE_", $api_vers, "_API_DEFAULT) && !defined(H5_USE_", $api_vers, "_API)\n"; # Print API version definition - print $fh " " x $indent, "#define H5_USE_1", ($curr_idx * 2), "_API 1\n"; + print $fh " " x $indent, "#define H5_USE_", $api_vers, "_API 1\n"; # Print API version endif - print $fh "#endif /* H5_USE_1", ($curr_idx * 2), "_API_DEFAULT && !H5_USE_1", ($curr_idx * 2), "_API */\n"; + print $fh "#endif /* H5_USE_", $api_vers, "_API_DEFAULT && !H5_USE_", $api_vers, "_API */\n"; } } @@ -141,7 +172,6 @@ sub print_globalapidefvers ($) { # sub print_globalapisymbolvers ($) { my $fh = shift; # File handle for output file - my $curr_idx; # Current API version index # Print the descriptive comment print $fh "\n\n/* If a particular \"global\" version of the library's interfaces is chosen,\n"; @@ -152,18 +182,18 @@ sub print_globalapisymbolvers ($) { print $fh " */\n"; # Loop over supported older library APIs and define the appropriate macros - for $curr_idx ($min_sup_idx .. ($max_idx - 1)) { + foreach my $api_vers (@versions) { # Print API version ifdef - print $fh "\n#ifdef H5_USE_1", ($curr_idx * 2), "_API\n"; + print $fh "\n#ifdef H5_USE_", $api_vers, "_API\n"; # Print the version macro info for each function that is defined for # this API version print $fh "\n/*************/\n"; print $fh "/* Functions */\n"; print $fh "/*************/\n"; - for $name (sort keys %{$func_vers[$curr_idx]}) { + for $name (sort keys %{$api_vers_to_function_vers{$api_vers}}) { print $fh "\n#if !defined(", $name, "_vers)\n"; - print $fh " " x $indent, "#define ", $name, "_vers $func_vers[$curr_idx]{$name}\n"; + print $fh " " x $indent, "#define ", $name, "_vers $api_vers_to_function_vers{$api_vers}{$name}\n"; print $fh "#endif /* !defined(", $name, "_vers) */\n"; } @@ -172,14 +202,14 @@ sub print_globalapisymbolvers ($) { print $fh "\n/************/\n"; print $fh "/* Typedefs */\n"; print $fh "/************/\n"; - for $name (sort keys %{$type_vers[$curr_idx]}) { + for $name (sort keys %{$api_vers_to_type_vers{$api_vers}}) { print $fh "\n#if !defined(", $name, "_t_vers)\n"; - print $fh " " x $indent, "#define ", $name, "_t_vers $type_vers[$curr_idx]{$name}\n"; + print $fh " " x $indent, "#define ", $name, "_t_vers $api_vers_to_type_vers{$api_vers}{$name}\n"; print $fh "#endif /* !defined(", $name, "_t_vers) */\n"; } # Print API version endif - print $fh "\n#endif /* H5_USE_1", ($curr_idx * 2), "_API */\n"; + print $fh "\n#endif /* H5_USE_", $api_vers, "_API */\n"; } } @@ -222,7 +252,7 @@ sub print_defaultapivers ($) { print $fh " " x $indent, "#define $curr_name $curr_name$curr_vers\n"; # Print function's dependent parameter types - foreach(sort(@param_list)) { + foreach (sort (@param_list)) { print $fh " " x $indent, "#define ${_}_t $_${curr_vers}_t\n"; } @@ -233,7 +263,7 @@ sub print_defaultapivers ($) { print $fh " " x $indent, "#define $curr_name $curr_name$curr_vers\n"; # Print function's dependent parameter types - foreach(sort(@param_list)) { + foreach (sort (@param_list)) { print $fh " " x $indent, "#define ${_}_t $_${curr_vers}_t\n"; } @@ -267,7 +297,7 @@ sub print_defaultapivers ($) { # Loop to print earlier version name mappings $curr_vers--; - while($curr_vers > 0) { + while ($curr_vers > 0) { print $fh "#elif $curr_vers_name == $curr_vers\n"; print $fh " " x $indent, "#define ${curr_name}_t $curr_name${curr_vers}_t\n"; $curr_vers--; @@ -293,6 +323,57 @@ sub print_endprotect ($$) { print $fh "#endif /* ${file}_H */\n\n"; } +############################################################################## +# Validate a line from H5vers.txt +# +sub validate_line { + my $name = $_[0]; + my $params = $_[1]; + my $vers = $_[2]; + + my @vers_list; # Version strings ("v18", etc.) + my %sym_versions; # Versions already seen (for duplicate checks) + + # Check if the name already exists in the list of symbols + if (exists ($functions{$name}) || exists($typedefs{$name})) { + die "duplicated symbol: $name"; + } + + # Check for no version info given + if ($vers eq "") { + die "no version information: $name"; + } + + # Separate the versions on commas (produces string elements like "v18") + @vers_list = split (/\s*,\s*/, $vers); + + # Check for invalid version data + foreach (@vers_list) { + # Note: v111 is allowed because H5O functions were prematurely versioned + # in HDF5 1.10. Because users were affected by this, the versioning + # was rescinded but the H5O version 2 functions were kept to be + # called directly. Now that the version macros are added in 1.12, + # along with a 3rd version of the H5O functions, the H5O function + # version for default api=v110 should be version 1 to work correctly + # with 1.10 applications that were using unversioned H5O functions, + # and the H5O function version should be version 3 for default api=v112 + # (the default api version for 1.12). Allowing a v111 entry allows + # a version 2 that is never accessed via the H5O function macros. + if (!( $_ =~ /v1[02468]/ || $_ =~ /v11[02468]/ || $_ =~ /v111/ || $_ =~ /v200/ )) { + die "bad version information: $name"; + } + + # Make sure we didn't specify duplicate versions on this line + if (exists($sym_versions{$_})) { + die "duplicate version information: $name"; + } + + # Store the versions for the function in a local hash table, indexed by the version + # (this is only used to check for duplicates) + $sym_versions{$_}=$_; + } +} + ############################################################################## # Parse a meaningful line (not a comment or blank line) into the appropriate # data structure @@ -301,149 +382,67 @@ sub parse_line ($) { my $line = shift; # Get the line to parse # Parse API function lines -#print "line=$line\n"; - if($line =~ /^\s*FUNCTION:/ || $line =~ /^\s*TYPEDEF:/) { +#print "line=$line"; + if ($line =~ /^\s*FUNCTION:/ || $line =~ /^\s*TYPEDEF:/) { my $name; # The name of the function my $params; # Typedefs for function parameters - my $vers; # The version info for the function - my @vers_list; # Version info, as a list - my @vers_nums; # Version info, as a numeric list - my $num_versions; # Number of versions for function - my %sym_versions; # Versions for a symbol - my $last_idx; # The previous version index seen for a function - my $last_vers; # The previous version # seen for a function + my $vers_string; # The version info for the function + my @vers_list; # Version info, as a list (e.g., "112", "200", etc. my $line_type; # Type of line we are parsing # Determine the type of the line to parse - if($line =~ /^\s*FUNCTION:/) { + if ($line =~ /^\s*FUNCTION:/) { $line_type = 1; # Get the function's name & version info - ($name, $params, $vers) = ($line =~ /^\s*FUNCTION:\s*(\w*);\s*(.*?)\s*;\s*(.*?)\s*$/); -#print "parse_line: name='$name', params='$params', vers='$vers'\n"; + ($name, $params, $vers_string) = ($line =~ /^\s*FUNCTION:\s*(\w*);\s*(.*?)\s*;\s*(.*?)\s*$/); +#print "parse_line: name='$name', params='$params', vers_string='$vers_string'\n"; } - elsif($line =~ /^\s*TYPEDEF:/) { + elsif ($line =~ /^\s*TYPEDEF:/) { $line_type = 2; # Get the typedefs's name & version info - ($name, $vers) = ($line =~ /^\s*TYPEDEF:\s*(\w*);\s*(.*?)\s*$/); -#print "parse_line: name='$name', vers='$vers'\n"; + ($name, $vers_string) = ($line =~ /^\s*TYPEDEF:\s*(\w*);\s*(.*?)\s*$/); +#print "parse_line: name='$name', vers_string='$vers_string'\n"; + } else { + die "unknown line type: $line"; } #print "parse_line: line_type='$line_type'\n"; + validate_line($name, $params, $vers_string); - # Check if the name already exists in the list of symbols - if(exists($functions{$name}) || exists($typedefs{$name})) { - die "duplicated symbol: $name"; - } - - # Check for no version info given - if($vers eq "") { - die "no version information: $name"; - } - - # Split up version info - @vers_list = split(/\s*,\s*/, $vers); + # Split the version info and strip off the leading "v" + @vers_list = split(/\s*,\s*/, $vers_string); + @vers_list = map { substr($_, 1) } @vers_list; #print "parse_line: vers_list=(@vers_list)\n"; - # Parse the version list into numbers, checking for invalid input - foreach(@vers_list) { - my $vers_idx; # Index of version in array - - # Do some validation on the input - # Note: v111 is allowed because H5O functions were prematurely versioned - # in HDF5 1.10. Because users were affected by this, the versioning - # was rescinded but the H5O version 2 functions were kept to be - # called directly. Now that the version macros are added in 1.12, - # along with a 3rd version of the H5O functions, the H5O function - # version for default api=v110 should be version 1 to work correctly - # with 1.10 applications that were using unversioned H5O functions, - # and the H5O function version should be version 3 for default api=v112 - # (the default api version for 1.12). Allowing a v111 entry and - # incrementing its index 13 lines below allows a version 2 that is - # never accessed via the H5O function macros. - if(!( $_ =~ /v1[02468]/ || $_ =~ /v11[02468]/ || $_ =~ /v111/ )) { - die "bad version information: $name"; - } - if(exists($sym_versions{$_})) { - die "duplicate version information: $name"; - } - - # Store the versions for the function in a local hash table, indexed by the version - $sym_versions{$_}=$_; - -#print "parse_line: _=$_\n"; - # Get the index of the version - ($vers_idx) = ($_ =~ /v1(\d+)/); - if($vers_idx == 11) { - $vers_idx++; - } - $vers_idx /= 2; -#print "parse_line: vers_idx='$vers_idx'\n"; - push(@vers_nums, $vers_idx); - } -#print "parse_line: vers_nums=(@vers_nums)\n"; - - # Check for invalid version info given - $last_idx = -1; - $last_vers = 1; - foreach(sort(@vers_nums)) { -#print "parse_line: _=$_ last_idx='$last_idx'\n"; - # Update intermediate versions of the library that included the API routine - if($last_idx >= 0) { -#print "parse_line: name='$name'\n"; -#print "parse_line: last_vers='$last_vers'\n"; -#print "parse_line: last_idx='$last_idx'\n"; - - # Add the function to the list of API routines available in - # different versions of the library - while($last_idx <= $_) { - if($line_type == 1) { - $func_vers[$last_idx]{$name} = $last_vers; - } elsif($line_type == 2) { - $type_vers[$last_idx]{$name} = $last_vers; + # Parse the version list into the hashes of version and type info + my $curr_sym_number = 1; + foreach my $vers (@vers_list) { + foreach my $hash_vers (@versions) { + if ($vers > $hash_vers) { + next; + } else { + if ($line_type == 1) { + $api_vers_to_function_vers{$hash_vers}{$name} = $curr_sym_number; } else { - die "unknown line type: $line"; + $api_vers_to_type_vers{$hash_vers}{$name} = $curr_sym_number; } - $last_idx++; } - - # Increment the version # of the function - $last_vers++; } - # Keep track of last version index seen - $last_idx = $_; - } - - # Finish updating versions of the library that included the API routine - if($last_idx >= 0) { -#print "parse_line: max_idx='$max_idx'\n"; - - # Add the function to the list of API routines available in - # different versions of the library - while($last_idx <= $max_idx) { - if($line_type == 1) { - $func_vers[$last_idx]{$name} = $last_vers; - } elsif($line_type == 2) { - $type_vers[$last_idx]{$name} = $last_vers; - } else { - die "unknown line type: $line"; - } - $last_idx++; - } + $curr_sym_number++; } # Store the number of symbol versions in a hash table, indexed by the name - if($line_type == 1) { + if ($line_type == 1) { $functions{$name} = $#vers_list + 1; # Store the function's parameter types for later $func_params{$name} = $params; - } elsif($line_type == 2) { + } elsif ($line_type == 2) { $typedefs{$name} = $#vers_list + 1; - } else { - die "unknown line type: $line"; } +#print "\n"; } # Unknown keyword else { @@ -488,7 +487,7 @@ for $file (@ARGV) { #print "file = '$file'\n"; # Check for directory prefix on input file - if($file =~ /\//) { + if ($file =~ /\//) { ($prefix) = ($file =~ /(^.*\/)/); } else { @@ -497,9 +496,9 @@ for $file (@ARGV) { #print "prefix = '$prefix'\n"; # Read in the entire file open SOURCE, $file or die "$file: $!\n"; - while ( defined ($line=) ) { + while ( defined ($line = ) ) { # Skip blank lines and those lines whose first character is a '#' - if(!($line =~ /(^\s*#.*$)|(^\s*$)/)) { + if (!($line =~ /(^\s*#.*$)|(^\s*$)/)) { # Construct data structures for later printing parse_line($line); } @@ -509,20 +508,5 @@ for $file (@ARGV) { # Create header files print "Generating 'H5version.h'\n"; create_public($prefix); - -#for $name (sort keys %functions) { -# print "functions{$name} = $functions{$name}\n"; -#} - -#for $i (0 .. $#func_vers) { -# my $vers_name; # Name of indexed version -# $vers_name = "v1." . ($i * 2); -# print "$vers_name functions: "; -# for $name (sort keys %{$func_vers[$i]}) { -# print "$name$func_vers[$i]{$name} "; -# } -# print "\n"; -#} - } diff --git a/c++/src/C2Cppfunction_map.htm b/c++/src/C2Cppfunction_map.htm index 4ea67544efe..791892ee006 100644 --- a/c++/src/C2Cppfunction_map.htm +++ b/c++/src/C2Cppfunction_map.htm @@ -9992,7 +9992,7 @@ none;mso-border-top-alt:solid windowtext .5pt;mso-border-alt:solid windowtext .5pt; padding:0in 5.4pt 0in 5.4pt'>

H5Iregister_type

+ normal'>H5Iregister_type2

+ +Name + +Default + +Description + + + +aNoAttributes + + +acYesMeta data cache + + +bYesB-Trees + + +dYesDatasets + + +eYesError handling + + +fYesFiles + + +gYesGroups + + +hgYesGlobal heap + + +hlNoLocal heaps + + +iYesInterface abstraction + + +mfNoFile memory management + + +mmYesLibrary memory management + + +oNoObject headers and messages + + +pYesProperty lists + + +sYesData spaces + + +tYesDatatypes + + +vYesVectors + + +zYesRaw data filters + + + +In addition to including the code at compile time the application must enable each package at +runtime. This is done by listing the package names in the HDF5_DEBUG environment variable. That +variable may also contain file descriptor numbers (the default is '2') which control the output +for all following packages up to the next file number. The word 'all' refers to all packages. Any +word my be preceded by a minus sign to turn debugging off for the package. + +\subsection subsec_adddbg_stats_sample Sample debug specifications + + + + + + + + + + + + + +
all +This causes debugging output from all packages to be sent to the standard error stream. +
all -t -s +Debugging output for all packages except datatypes and data spaces will appear on the standard error stream. +
-all ac 255 t,s +This disables all debugging even if the default was to debug something, then output +from the meta data cache is send to the standard error stream and output from data types +and spaces is sent to file descriptor 255 which should be redirected by the shell. +
+The components of the HDF5_DEBUG value may be separated by any non-lowercase letter. + +*/ diff --git a/doxygen/dox/FileFormatDisc.dox b/doxygen/dox/FileFormatDisc.dox new file mode 100644 index 00000000000..8075840b97e --- /dev/null +++ b/doxygen/dox/FileFormatDisc.dox @@ -0,0 +1,1114 @@ + +/** \page FMTDISC HDF5 File Format Discussion + + +
+
Document's Audience:
+
Current H5 library designers and knowledgeable external developers.
+ +
Background Reading:
+
\ref FMT3
This describes the current HDF5 file format.
+
+ +\section sec_fmtdisc_intro Introduction +What is this document about?
+This document attempts to explain the HDF5 file format specification with a +few examples and describes some potential improvements to the format specification. + +\section sec_fmtdisc_exam File Format Examples +This section has several small programs and describes the format of a file +created with each of them. + +Example program one - Create an empty file: +\code +#include "hdf5.h" +#include + +int main() +{ + hid_t fid; /* File ID */ + herr_t ret; /* Generic return value */ + + /* Create the file */ + fid=H5Fcreate("example1.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + assert(fid>=0); + + /* Close the file */ + ret=H5Fclose(fid); + assert(ret>=0); + + return(0); +} +\endcode + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Super Block
bytebytebytebyte
\211'H''D''F'
\\r\\n\032\\n
0000
0880
416
0x00000003
0

0xffffffffffffffff

?

0xffffffffffffffff

+ + + + + + + + + + + + + + + + +
0

928

H5G_CACHED_STAB (1)
0
+ + + + + + + +

384


96

+
+
+ +\code +%h5debug example1.h5 + +Reading signature at address 0 (rel) +File Super Block... +File name: example1.h5 +File access flags 0x00000000 +File open reference count: 1 +Address of super block: 0 (abs) +Size of user block: 0 bytes +Super block version number: 0 +Free list version number: 0 +Root group symbol table entry version number: 0 +Shared header version number: 0 +Size of file offsets (haddr_t type): 8 bytes +Size of file lengths (hsize_t type): 8 bytes +Symbol table leaf node 1/2 rank: 4 +Symbol table internal node 1/2 rank: 16 +File consistency flags: 0x00000003 +Base address: 0 (abs) +Free list address: UNDEF (rel) +Address of driver information block: UNDEF (rel) +Root group symbol table entry: + Name offset into private heap: 0 + Object header address: 928 + Dirty: Yes + Cache info type: Symbol Table + Cached information: + B-tree address: 384 + Heap address: 96 +\endcode + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Root Group Object Header
bytebytebytebyte
102
1
32
0x001116
0x010
+ + + + + + + +

384


96

+
00
0x000
+ +\code +%h5debug example1.h5 928 + +New address: 928 +Reading signature at address 928 (rel) +Object Header... +Dirty: 0 +Version: 1 +Header size (in bytes): 16 +Number of links: 1 +Number of messages (allocated): 2 (32) +Number of chunks (allocated): 1 (8) +Chunk 0... + Dirty: 0 + Address: 944 + Size in bytes: 32 +Message 0... + Message ID (sequence number): 0x0011 stab(0) + Shared message: No + Constant: Yes + Raw size in obj header: 16 bytes + Chunk number: 0 + Message Information: + B-tree address: 384 + Name heap address: 96 +Message 1... + Message ID (sequence number): 0x0000 null(0) + Shared message: No + Constant: No + Raw size in obj header: 0 bytes + Chunk number: 0 + Message Information: + +\endcode + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Root Group Local Heap
bytebytebytebyte
'H''E''A''P'
0
256
8
128
+ +\code +%h5debug example1.h5 96 + +New address: 96 +Reading signature at address 96 (rel) +Local Heap... +Dirty: 0 +Header size (in bytes): 32 +Address of heap data: 128 +Data bytes allocated on disk: 256 +Data bytes allocated in core: 256 +Free Blocks (offset, size): + Block #0: 8, 248 +Percent of heap used: 3.12% +Data follows (`__' indicates free region)... + 0: 00 00 00 00 00 00 00 00 __ __ __ __ __ __ __ __ ........ + 16: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 32: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 48: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 64: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 80: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 96: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 112: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 128: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 144: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 160: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 176: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 192: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 208: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 224: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 240: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + +\endcode + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Root Group B-tree
bytebytebytebyte
'T''R''E''E'
000

0xffffffffffffffff


0xffffffffffffffff

+ +\code +%h5debug example1.h5 384 96 + +New address: 384 +Reading signature at address 384 (rel) +Tree type ID: H5B_SNODE_ID +Size of node: 544 +Size of raw (disk) key: 8 +Dirty flag: False +Number of initial dirty children: 0 +Level: 0 +Address of left sibling: UNDEF +Address of right sibling: UNDEF +Number of children (max): 0 (32) + +\endcode + +Example program two - Create a file with a single dataset in it: +\code +#include "hdf5.h" +#include + +int main() +{ + hid_t fid; /* File ID */ + hid_t sid; /* Dataspace ID */ + hid_t did; /* Dataset ID */ + herr_t ret; /* Generic return value */ + + /* Create the file */ + fid=H5Fcreate("example2.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + assert(fid>=0); + + /* Create a scalar dataspace for the dataset */ + sid=H5Screate(H5S_SCALAR); + assert(sid>=0); + + /* Create a trivial dataset */ + did=H5Dcreate(fid, "Dataset", H5T_NATIVE_INT, sid, H5P_DEFAULT); + assert(did>=0); + + /* Close the dataset */ + ret=H5Dclose(did); + assert(ret>=0); + + /* Close the dataspace */ + ret=H5Sclose(sid); + assert(ret>=0); + + /* Close the file */ + ret=H5Fclose(fid); + assert(ret>=0); + + return(0); +} +\endcode + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Super Block
bytebytebytebyte
\211'H''D''F'
\\r\\n\032\\n
0000
0880
416
0x00000003
0

0xffffffffffffffff

?

0xffffffffffffffff

+ + + + + + + + + + + + + + + + +
0

928

H5G_CACHED_STAB (1)
0
+ + + + + + + +

384


96

+
+
+ +\code +%h5debug example2.h5 + +Reading signature at address 0 (rel) +File Super Block... +File name: example2.h5 +File access flags 0x00000000 +File open reference count: 1 +Address of super block: 0 (abs) +Size of user block: 0 bytes +Super block version number: 0 +Free list version number: 0 +Root group symbol table entry version number: 0 +Shared header version number: 0 +Size of file offsets (haddr_t type): 8 bytes +Size of file lengths (hsize_t type): 8 bytes +Symbol table leaf node 1/2 rank: 4 +Symbol table internal node 1/2 rank: 16 +File consistency flags: 0x00000003 +Base address: 0 (abs) +Free list address: UNDEF (rel) +Address of driver information block: UNDEF (rel) +Root group symbol table entry: + Name offset into private heap: 0 + Object header address: 928 + Dirty: Yes + Cache info type: Symbol Table + Cached entry information: + B-tree address: 384 + Heap address: 96 +\endcode + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Root Group Object Header
bytebytebytebyte
102
1
32
0x001116
0x010
+ + + + + + + +

384


96

+
00
0x000
+ +\code +%h5debug example2.h5 928 + +New address: 928 +Reading signature at address 928 (rel) +Object Header... +Dirty: 0 +Version: 1 +Header size (in bytes): 16 +Number of links: 1 +Number of messages (allocated): 2 (32) +Number of chunks (allocated): 1 (8) +Chunk 0... + Dirty: 0 + Address: 944 + Size in bytes: 32 +Message 0... + Message ID: 0x0011 stab(0) + Shared message: No + Constant: Yes + Raw size in obj header: 16 bytes + Chunk number: 0 + Message Information: + B-tree address: 384 + Name heap address: 96 +Message 1... + Message ID: 0x0000 null(0) + Shared message: No + Constant: No + Raw size in obj header: 0 bytes + Chunk number: 0 + Message Information: + +\endcode + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Root Group Local Heap
bytebytebytebyte
'H''E''A''P'
0
256
16
128
+ +\code +%h5debug example2.h5 96 + +New address: 96 +Reading signature at address 96 (rel) +Local Heap... +Dirty: 0 +Header size (in bytes): 32 +Address of heap data: 128 +Data bytes allocated on disk: 256 +Data bytes allocated in core: 256 +Free Blocks (offset, size): + Block #0: 16, 240 +Percent of heap used: 6.25% +Data follows (`__' indicates free region)... + 0: 00 00 00 00 00 00 00 00 44 61 74 61 73 65 74 00 ........Dataset. + 16: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 32: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 48: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 64: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 80: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 96: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 112: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 128: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 144: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 160: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 176: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 192: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 208: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 224: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 240: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ +\endcode + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Root Group B-tree
bytebytebytebyte
'T''R''E''E'
001

0xffffffffffffffff


0xffffffffffffffff


0


1248


8

+ +\code +%h5debug example2.h5 384 96 + +New address: 384 +Reading signature at address 384 (rel) +Tree type ID: H5B_SNODE_ID +Size of node: 544 +Size of raw (disk) key: 8 +Dirty flag: False +Number of initial dirty children: 0 +Level: 0 +Address of left sibling: UNDEF +Address of right sibling: UNDEF +Number of children (max): 1 (32) +Child 0... + Address: 1248 + Left Key: + Heap offset: 0 + Name : + Right Key: + Heap offset: 8 + Name : Dataset +\endcode + + + + + + + + + + + + + + + + + + + + + + + +
Root Group B-tree Symbol Table Node
bytebytebytebyte
'S''N''O''D'
101
+ + + + + + + + + + + + + + + + +
8

976

0
0


0


+
+ +\code +%h5debug example2.h5 1248 96 + +New address: 1248 +Reading signature at address 1248 (rel) +Symbol Table Node... +Dirty: No +Size of Node (in bytes): 328 +Number of Symbols: 1 of 8 +Symbol 0: + Name: `Dataset' + Name offset into private heap: 8 + Object header address: 976 + Dirty: No + Cache info type: Nothing Cached +\endcode + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
'/Dataset' Object Header
bytebytebytebyte
Version: 1Reserved: 0Number of Header Messages: 6
Object Reference Count: 1
Total Object Header Size: 256
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fill Value Header Message
Message Type: 0x0005Message Data Size: 8
Flags: 0x01Reserved: 0
Version: 1Space Allocation Time: 2 (Late)Fill Value Writing Time: 0 (At allocation)Fill Value Defined: 0 (Undefined)
Fill Value Datatype Size: 0 (Use dataset's datatype for fill-value datatype)
+
+ + + + + + + + + + + + + + + + + + + Reserved: 0 + + + + + + + + + + + + + + + + + + + + + +
Datatype Header Message
Message Type: 0x0003Message Data Size: 16
Flags: 0x01
+ + + + + +
Version: 0x1Class: 0x0 (Fixed-Point)
+ 		Fixed-Point Bit-Field: 0x08 (Little-endian, No padding, Signed)
Size: 4
Bit Offset: 0Bit Precision: 32
Message Alignment Filler: -
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Dataspace Header Message
Message Type: 0x0001Message Data Size: 8
Flags: 0x00Reserved: 0
Version: 1Rank: 0 (Scalar)Flags: 0x00 (No maximum dimensions, no permutation information)Reserved: 0
Reserved: 0
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout Header Message
Message Type: 0x0008Message Data Size: 24
Flags: 0x00Reserved: 0
Version: 1Rank: 1 (Dataspace rank+1)Class: 1 (Contiguous)Reserved: 0
Reserved: 0

Address: 0xffffffffffffffff (Undefined)

Dimension 0 Size: 4 (Datatype size)
Message Alignment Filler: -
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Modification Date & Time Header Message
Message Type: 0x0012Message Data Size: 8
Flags: 0x00Reserved: 0
Version: 1Reserved: 0
Seconds Since Epoch: 1052401700 (2003-05-08 08:48:20 CDT)
+
+ + + + + + + + + + + + + + + + +
Null Header Message
Message Type: 0x0000Message Data Size: 144
Flags: 0x00Reserved: 0
+
+ +\code +%h5debug example2.h5 976 + +New address: 976 +Reading signature at address 976 (rel) +Object Header... +Dirty: 0 +Version: 1 +Header size (in bytes): 16 +Number of links: 1 +Number of messages (allocated): 6 (32) +Number of chunks (allocated): 1 (8) +Chunk 0... + Dirty: 0 + Address: 992 + Size in bytes: 256 +Message 0... + Message ID (sequence number): 0x0005 `fill_new' (0) + Shared: No + Constant: Yes + Raw size in obj header: 8 bytes + Chunk number: 0 + Message Information: + Version: 1 + Space Allocation Time: Late + Fill Time: On Allocation + Fill Value Defined: Undefined + Size: 0 + Data type: +Message 1... + Message ID (sequence number): 0x0003 data_type(0) + Shared message: No + Constant: Yes + Raw size in obj header: 16 bytes + Chunk number: 0 + Message Information: + Type class: integer + Size: 4 bytes + Byte order: little endian + Precision: 32 bits + Offset: 0 bits + Low pad type: zero + High pad type: zero + Sign scheme: 2's comp +Message 2... + Message ID (sequence number): 0x0001 simple_dspace(0) + Shared message: No + Constant: No + Raw size in obj header: 8 bytes + Chunk number: 0 + Message Information: + Rank: 0 +Message 3... + Message ID (sequence number): 0x0008 layout(0) + Shared message: No + Constant: No + Raw size in obj header: 24 bytes + Chunk number: 0 + Message Information: + Data address: UNDEF + Number of dimensions: 1 + Size: {4} +Message 4... + Message ID (sequence number): 0x0012 mtime_new(0) + Shared message: No + Constant: No + Raw size in obj header: 8 bytes + Chunk number: 0 + Message Information: + Time: 2003-03-05 14:52:00 CST +Message 5... + Message ID (sequence number): 0x0000 null(0) + Shared message: No + Constant: No + Raw size in obj header: 144 bytes + Chunk number: 0 + Message Information: + +\endcode + +*/ + diff --git a/doxygen/dox/FileFormatSpec.dox b/doxygen/dox/FileFormatSpec.dox index fc10574dce2..308237c8d67 100644 --- a/doxygen/dox/FileFormatSpec.dox +++ b/doxygen/dox/FileFormatSpec.dox @@ -1,23 +1,7 @@ -/** \page FMT3 HDF5 File Format Specification Version 3.0 +\ref FMT3 -\htmlinclude H5.format.html +\ref FMT2 -*/ +\ref FMT11 -/** \page FMT2 HDF5 File Format Specification Version 2.0 - -\htmlinclude H5.format.2.0.html - -*/ - -/** \page FMT11 HDF5 File Format Specification Version 1.1 - -\htmlinclude H5.format.1.1.html - -*/ - -/** \page FMT1 HDF5 File Format Specification Version 1.0 - -\htmlinclude H5.format.1.0.html - -*/ \ No newline at end of file +\ref FMT1 diff --git a/doxygen/dox/GettingStarted.dox b/doxygen/dox/GettingStarted.dox index 274598c9537..a37b197afea 100644 --- a/doxygen/dox/GettingStarted.dox +++ b/doxygen/dox/GettingStarted.dox @@ -50,7 +50,7 @@ The high-level HDF5 library includes several sets of convenience and standard-us -\ref IntroParHDF5 +@ref IntroParHDF5 A brief introduction to Parallel HDF5. If you are new to HDF5 please see the @ref LearnBasics topic first. @@ -58,7 +58,7 @@ A brief introduction to Parallel HDF5. If you are new to HDF5 please see the @re -\ref ViewTools +@ref ViewTools \li @ref LearnHDFView @@ -71,8 +71,8 @@ A brief introduction to Parallel HDF5. If you are new to HDF5 please see the @re New Features since HDF5-1.10 -\li \ref VDS -\li \ref SWMR +\li @ref VDSTN +\li @ref SWMRTN @@ -80,7 +80,7 @@ New Features since HDF5-1.10 Example Programs -\ref HDF5Examples +@ref HDF5Examples diff --git a/doxygen/dox/H5.format.1.0.dox b/doxygen/dox/H5.format.1.0.dox new file mode 100644 index 00000000000..a5606b09a13 --- /dev/null +++ b/doxygen/dox/H5.format.1.0.dox @@ -0,0 +1,2469 @@ + +/** \page FMT1 HDF5 File Format Specification Version 1.0 +
    +
  1. @ref sec_fmt1_intro
    2. +
  2. @ref sec_fmt1_boot
    3. +
  3. @ref sec_fmt1_group +
      +
    1. @ref subsec_fmt1_group_btrees
      2. +
    2. @ref subsec_fmt1_group_symboltable
      3. +
    3. @ref subsec_fmt1_group_symboltableentry
      4. +
    4. @ref subsec_fmt1_group_localheap
      5. +
    5. @ref subsec_fmt1_group_globalheap
      6. +
    6. @ref subsec_fmt1_group_freespaceindex
      7. +
    4. +
  4. @ref sec_fmt1_dataobject +
      +
    1. @ref subsec_fmt1_dataobject_hdr +
        +
      1. @ref subsubsec_fmt1_dataobject_hdr_nil
        2. +
      2. @ref subsubsec_fmt1_dataobject_hdr_simple
        3. +
      3. @ref subsubsec_fmt1_dataobject_hdr_dtmessage
        4. +
      4. @ref subsubsec_fmt1_dataobject_hdr_fvmessage
        5. +
      5. @ref subsubsec_fmt1_dataobject_hdr_message_0005
        6. +
      6. @ref subsubsec_fmt1_dataobject_hdr_compact
        7. +
      7. @ref subsubsec_fmt1_dataobject_hdr_external
        8. +
      8. @ref subsubsec_fmt1_dataobject_hdr_layout
        9. +
      9. @ref subsubsec_fmt1_dataobject_hdr_message_0009
        10. +
      10. @ref subsubsec_fmt1_dataobject_hdr_message_000A
        11. +
      11. @ref subsubsec_fmt1_dataobject_hdr_filter
        12. +
      12. @ref subsubsec_fmt1_dataobject_hdr_attribute
        13. +
      13. @ref subsubsec_fmt1_dataobject_hdr_name
        14. +
      14. @ref subsubsec_fmt1_dataobject_hdr_modified
        15. +
      15. @ref subsubsec_fmt1_dataobject_hdr_shared
        16. +
      16. @ref subsubsec_fmt1_dataobject_hdr_continuation
        17. +
      17. @ref subsubsec_fmt1_dataobject_hdr_stmgroup
        18. +
      2. +
    2. @ref subsec_fmt1_dataobject_sharedhdr
      3. +
    3. @ref subsec_fmt1_dataobject_storage
      4. +
    +
    5. +
+ + + +\section sec_fmt1_intro Introduction + + + + + + + + + + + + + + +
Figure 1: Relationships among the HDF5 root group, other groups, and objects
\image html FF-IH_FileGroup.gif
Figure 2: HDF5 objects -- datasets, datatypes, or dataspaces
\image html FF-IH_FileObject.gif
+ +The format of an HDF5 file on disk encompasses several key ideas of the HDF4 and AIO file formats as well +as addressing some shortcomings therein. The new format is more self-describing than the HDF4 format and +is more uniformly applied to data objects in the file. + +An HDF5 file appears to the user as a directed graph. The nodes of this graph are the higher-level HDF5 +objects that are exposed by the HDF5 APIs: +\li Groups +\li Datasets +\li Datatypes +\li Dataspaces + +At the lowest level, as information is actually written to the disk, an HDF5 file is made up of the +following objects: +\li A super block +\li B-tree nodes (containing either symbol nodes or raw data chunks) +\li Object headers +\li Collections +\li Local heaps +\li Free space + +The HDF5 library uses these lower-level objects to represent the higher-level objects that are then +presented to the user or to applications through the APIs. For instance, a group is an object header that +contains a message that points to a local heap and to a B-tree which points to symbol nodes. A dataset is +an object header that contains messages that describe datatype, space, layout, filters, external files, +fill value, etc with the layout message pointing to either a raw data chunk or to a B-tree that points to +raw data chunks. + +\subsection subsec_fmt1_intro_doc This Document +This document describes the lower-level data objects; the higher-level objects and their properties are +described in the \ref UG. + +Three levels of information comprise the file format. Level 0 contains basic information for identifying +and defining information about the file. Level 1 information contains the group information (stored as +a B-tree) and is used as the index for all the objects in the file. Level 2 is the rest of the file and +contains all of the data objects, with each object partitioned into header information, also known as +meta information, and data. + +The sizes of various fields in the following layout tables are determined by looking at the number of +columns the field spans in the table. There are three exceptions: (1) The size may be overridden by +specifying a size in parentheses, (2) the size of addresses is determined by the Size of Offsets +field in the super block, and (3) the size of size fields is determined by the Size of Lengths +field in the super block. + +\section sec_fmt1_boot Disk Format: Level 0 - File Signature and Super Block +The super block may begin at certain predefined offsets within the HDF5 file, allowing a block of +unspecified content for users to place additional information at the beginning (and end) of the HDF5 file +without limiting the HDF5 library's ability to manage the objects within the file itself. This feature was +designed to accommodate wrapping an HDF5 file in another file format or adding descriptive information to +the file without requiring the modification of the actual file's information. The super block is located by +searching for the HDF5 file signature at byte offset 0, byte offset 512 and at successive locations in the +file, each a multiple of two of the previous location, i.e. 0, 512, 1024, 2048, etc. + +The super block is composed of a file signature, followed by super block and group version numbers, +information about the sizes of offset and length values used to describe items within the file, the size of +each group page, and a group entry for the root object in the file. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
HDF5 Super Block Layout
bytebytebytebyte

HDF5 File Signature (8 bytes)

Version \# of Super BlockVersion \# of Global Free-space StorageVersion \# of GroupReserved
Version \# of Shared Header Message FormatSize of OffsetsSize of LengthsReserved (zero)
Group Leaf Node KGroup Internal Node K
File Consistency Flags
Base Address*
Address of Global Free-space Heap*
End of File Address*
Driver Information Block Address*
Root Group Address*
+\li Items marked with an asterisk (*) in the above table are of the size specified in "Size of Offsets." + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
File SignatureThis field contains a constant value and can be used to quickly identify a file as being an HDF5 + file. The constant value is designed to allow easy identification of an HDF5 file and to allow + certain types of data corruption to be detected. The file signature of an HDF5 file always + contains the following values: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
decimal13772687013102610
hexadecimal894844460d0a1a0a
ASCII C Notation\211HDF\\r\\n\032\\n
+
+ This signature both identifies the file as an HDF5 file and provides for immediate detection of common + file-transfer problems. The first two bytes distinguish HDF5 files on systems that expect the first two + bytes to identify the file type uniquely. The first byte is chosen as a non-ASCII value to reduce the + probability that a text file may be misrecognized as an HDF5 file; also, it catches bad file transfers + that clear bit 7. Bytes two through four name the format. The CR-LF sequence catches bad file transfers + that alter newline sequences. The control-Z character stops file display under MS-DOS. The final line + feed checks for the inverse of the CR-LF translation problem. (This is a direct descendent of the PNG + file signature.)
Version Number of the Super BlockThis value is used to determine the format of the information in the super block. When the format of + the information in the super block is changed, the version number is incremented to the next integer + and can be used to determine how the information in the super block is formatted.
Version Number of the Global Free-space HeapThis value is used to determine the format of the information in the Global Free-space Heap.
Version Number of the GroupThis value is used to determine the format of the information in the Group. When the format of the + information in the Group is changed, the version number is incremented to the next integer and can be + used to determine how the information in the Group is formatted.
Version Number of the Shared Header Message FormatThis value is used to determine the format of the information in a shared object header message, which + is stored in the global small-data heap. Since the format of the shared header messages differs from + the private header messages, a version number is used to identify changes in the format.
Size of OffsetsThis value contains the number of bytes used to store addresses in the file. The values for the + addresses of objects in the file are offsets relative to a base address, usually the address of the + super block signature. This allows a wrapper to be added after the file is created without invalidating + the internal offset locations.
Size of LengthsThis value contains the number of bytes used to store the size of an object.
Group Leaf Node KEach leaf node of a group B-tree will have at least this many entries but not more than twice this + many. If a group has a single leaf node then it may have fewer entries.
Group Internal Node KEach internal node of a group B-tree will have at least K pointers to other nodes but not more than 2K + pointers. If the group has only one internal node then it might have fewer than K pointers.
Bytes per B-tree PageThis value contains the number of bytes used for symbol pairs per page of the B-trees used in the file. + All B-tree pages will have the same size per page.
+ For 32-bit file offsets, 340 objects is the maximum per 4KB page; for 64-bit file offset, 254 objects + will fit per 4KB page. In general, the equation is:
+    <number of objects> =
       + FLOOR((<page size> - <offset size>) / +
          + (<Symbol size> + <offset size>)) - 1
File Consistency FlagsThis value contains flags to indicate information about the consistency of the information contained + within the file. Currently, the following bit flags are defined: +
    +
  • Bit 0 set indicates that the file is opened for write-access.
    • +
  • Bit 1 set indicates that the file has been verified for consistency and is guaranteed to be + consistent with the format defined in this document.
    • +
  • Bits 2-31 are reserved for future use.
    • +
+ Bit 0 should be set as the first action when a file is opened for write access and should be cleared + only as the final action when closing a file. Bit 1 should be cleared during normal access to a file + and only set after the file's consistency is guaranteed by the library or a consistency utility.
Base AddressThis is the absolute file address of the first byte of the HDF5 data within the file. The library + currently constrains this value to be the absolute file address of the super block itself when creating + new files; future versions of the library may provide greater flexibility. Unless otherwise noted, all + other file addresses are relative to this base address.
Address of Global Free-space HeapFree-space management is not yet defined in the HDF5 file format and is not handled by the library. + Currently this field always contains the undefined address 0xfff...ff.
End of File AddressThis is the relative file address of the first byte past the end of all HDF5 data. It is used to + determine whether a file has been accidentally truncated and as an address where file data allocation + can occur if the free list is not used.
Driver Information Block AddressThis is the relative file address of the file driver information block which contains driver-specific + information needed to reopen the file. If there is no driver information block then this entry should + be the undefined address (all bits set).
Root Group AddressThis is the address of the root group (described later in this document), which serves as the entry + point into the group graph.
+ +The file driver information block is an optional region of the file which contains information +needed by the file driver in order to reopen a file. The format of the file driver information block is: + + + + + + + + + + + + + + + + + + + + + +
Driver Information Block
bytebytebytebyte
VersionReserved (zero)
Driver Information Size (4 bytes)

Driver Identification (8 bytes)



Driver Information


+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number of the driver information block. The file format documented here is version zero.
Driver Information SizeThe size in bytes of the Driver Information part of this structure.
Driver IdentificationThis is an eight-byte ASCII string without null termination which identifies the driver and version number + of the Driver Information block. The predefined drivers supplied with the HDF5 library are identified by the + letters NCSA followed by the first four characters of the driver name. If the Driver Information + block is not the original version then the last letter(s) of the identification will be replaced by a version + number in ASCII. For example, the various versions of the family driver will be identified by + NCSAfami, NCSAfam0, NCSAfam1, etc. (NCSAfami is simply + NCSAfamily truncated to eight characters. Subsequent identifiers will be created by + substituting sequential numerical values for the final character, starting with zero.)
+ Identification for user-defined drivers is arbitrary but should be unique.
Driver InformationDriver information is stored in a format defined by the file driver and encoded/decoded by the driver + callbacks invoked from the H5FD_sb_encode and H5FD_sb_decode functions.
+ +\section sec_fmt1_group Disk Format: Level 1 - File Infrastructure + +\subsection subsec_fmt1_group_btrees Disk Format: Level 1A - B-link Trees and B-tree Nodes +B-link trees allow flexible storage for objects which tend to grow in ways that cause the object to be stored +discontiguously. B-trees are described in various algorithms books including "Introduction to Algorithms" by +Thomas H. Cormen, Charles E. Leiserson, and Ronald L. Rivest. The B-link tree, in which the sibling nodes at +a particular level in the tree are stored in a doubly-linked list, is described in the "Efficient Locking +for Concurrent Operations on B-trees" paper by Phillip Lehman and S. Bing Yao as published in the ACM +Transactions on Database Systems, Vol. 6, No. 4, December 1981. + +The B-link trees implemented by the file format contain one more key than the number of children. In other +words, each child pointer out of a B-tree node has a left key and a right key. The pointers out of internal +nodes point to sub-trees while the pointers out of leaf nodes point to symbol nodes and raw data chunks. +Aside from that difference, internal nodes and leaf nodes are identical. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
B-tree Nodes
bytebytebytebyte
Node Signature
Node TypeNode LevelEntries Used
Address of Left Sibling
Address of Right Sibling
Key 0 (variable size)
Address of Child 0
Key 1 (variable size)
Address of Child 1
...
Key 2K (variable size)
Address of Child 2K
Key 2K+1 (variable size)
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Node SignatureThe ASCII character string TREE is used to indicate the beginning of a B-link tree node. + This gives file consistency checking utilities a better chance of reconstructing a damaged file.
Node TypeEach B-link tree points to a particular type of data. This field indicates the type of data as well as + implying the maximum degree K of the tree and the size of each Key field.
+
+
0
+
This tree points to group nodes.
+
1
+
This tree points to a new data chunk.
+
+
Node LevelThe node level indicates the level at which this node appears in the tree (leaf nodes are at level + zero). Not only does the level indicate whether child pointers point to sub-trees or to data, but it + can also be used to help file consistency checking utilities reconstruct damaged trees.
Entries UsedThis determines the number of children to which this node points. All nodes of a particular type of + tree have the same maximum degree, but most nodes will point to less than that number of children. The + valid child pointers and keys appear at the beginning of the node and the unused pointers and keys + appear at the end of the node. The unused pointers and keys have undefined values.
Address of Left SiblingThis is the file address of the left sibling of the current node relative to the super block. If the + current node is the left-most node at this level then this field is the undefined address (all bits + set).
Address of Right SiblingThis is the file address of the right sibling of the current node relative to the super block. If the + current node is the right-most node at this level then this field is the undefined address (all bits + set).
Keys and Child PointersEach tree has 2K+1 keys with 2K child pointers interleaved between the keys. The number + of keys and child pointers actually containing valid values is determined by the Entries Used + field. If that field is N then the B-link tree contains N child pointers and + N+1 keys.
KeyThe format and size of the key values is determined by the type of data to which this tree points. The + keys are ordered and are boundaries for the contents of the child pointer; that is, the key values + represented by child N fall between Key N and Key N+1. Whether the interval + is open or closed on each end is determined by the type of data to which the tree points.
+ The format of the key depends on the node type. For nodes of node type 1, the key is formatted as follows: + + + + + + + + + + + + + +
Bytes 1-4Size of chunk in bytes.
Bytes 4-8Filter mask, a 32-bit bitfield indicating which filters have been applied to that chunk.
N fields of 8 bytes eachA 64-bit index indicating the offset of the chunk within the dataset where N is the number + of dimensions of the dataset. For example, if a chunk in a 3-dimensional dataset begins at the + position [5,5,5], there will be three such 8-bit indices, each with the value of + 5.
+
+ For nodes of node type 0, the key is formatted as follows: + + + + + +
A single field of Size of Lengths bytesIndicates the byte offset into the local heap for the first object name in the subtree which + that key describes.
+
Child PointersThe tree node contains file addresses of subtrees or data depending on the node level. Nodes at Level + 0 point to data addresses, either data chunk or group nodes. Nodes at non-zero levels point to other + nodes of the same B-tree.
+ +Each B-tree node looks like this: + + + + + + + + + + + + + +
key[0]  child[0]  key[1]  child[1]  key[2]  ...  ...  key[N-1]  child[N-1]  key[N]
+where child[i] is a pointer to a sub-tree (at a level above Level 0) or to data (at Level 0). +Each key[i] describes an item stored by the B-tree (a chunk or an object of a group node). +The range of values represented by child[i] are indicated by key[i] and key[i+1]. + +The following question must next be answered: "Is the value described by key[i] contained in +child[i-1] or in child[i]?" The answer depends on the type of tree. In trees for groups (node +type 0) the object described by key[i] is the greatest object contained in child[i-1] while +in chunk trees (node type 1) the chunk described by key[i] is the least chunk in child[i]. + +That means that key[0] for group trees is sometimes unused; it points to offset zero in the heap, which is +always the empty string and compares as "less-than" any valid object name. + +And key[N] for chunk trees is sometimes unused; it contains a chunk offset which compares as +"greater-than" any other chunk offset and has a chunk byte size of zero to indicate that it is not actually +allocated. + +\subsection subsec_fmt1_group_symboltable Disk Format: Level 1B - Group and Symbol Nodes +A group is an object internal to the file that allows arbitrary nesting of objects (including other groups). +A group maps a set of names to a set of file address relative to the base address. Certain meta data for an +object to which the group points can be duplicated in the group symbol table in addition to the object header. + +An HDF5 object name space can be stored hierarchically by partitioning the name into components and storing +each component in a group. The group entry for a non-ultimate component points to the group containing the +next component. The group entry for the last component points to the object being named. + +A group is a collection of group nodes pointed to by a B-link tree. Each group node contains entries for one +or more symbols. If an attempt is made to add a symbol to an already full group node containing 2K +entries, then the node is split and one node contains K symbols and the other contains K+1 +symbols. + + + + + + + + + + + + + + + + + + + + +
Group Node (A Leaf of a B-tree)
bytebytebytebyte
Node Signature
Version NumberReserved for Future UseNumber of Symbols


Group Entries


+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Node SignatureThe ASCII character string SNOD is used to indicate the beginning of a group node. This + gives file consistency checking utilities a better chance of reconstructing a damaged file.
Version NumberThe version number for the group node. This document describes version 1.
Number of SymbolsAlthough all group nodes have the same length, most contain fewer than the maximum possible number of + symbol entries. This field indicates how many entries contain valid data. The valid entries are packed + at the beginning of the group node while the remaining entries contain undefined values.
Group EntriesEach symbol has an entry in the group node. The format of the entry is described below.
+ +\subsection subsec_fmt1_group_symboltableentry Disk Format: Level 1C - Group Entry +Each group entry in a group node is designed to allow for very fast browsing of stored objects. Toward that +design goal, the group entries include space for caching certain constant meta data from the object header. + + + + + + + + + + + + + + + + + + + + + + + + +
Group Entry
bytebytebytebyte
Name Offset (<size> bytes)
Object Header Address
Cache Type
Reserved


Scratch-pad Space (16 bytes)


+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Name OffsetThis is the byte offset into the group local heap for the name of the object. The name is null + terminated.
Object Header AddressEvery object has an object header which serves as a permanent location for the object's meta data. + In addition to appearing in the object header, some meta data can be cached in the scratch-pad space.
Cache TypeThe cache type is determined from the object header. It also determines the format for the scratch-pad + space.
+
+
0
+
No data is cached by the group entry. This is guaranteed to be the case when an object header has + a link count greater than one.
+
1
+
Object header meta data is cached in the group entry. This implies that the group entry refers to + another group.
+
2
+
The entry is a symbolic link. The first four bytes of the scratch-pad space are the offset into + the local heap for the link value. The object header address will be undefined.
+
N
+
Other cache values can be defined later and libraries that do not understand the new values will + still work properly.
+
+
ReservedThese four bytes are present so that the scratch-pad space is aligned on an eight-byte boundary. They + are always set to zero.
Scratch-pad SpaceThis space is used for different purposes, depending on the value of the Cache Type field. Any meta-data + about a dataset object represented in the scratch-pad space is duplicated in the object header for + that dataset. This meta data can include the datatype and the size of the dataspace for a dataset whose + datatype is atomic and whose dataspace is fixed and less than four dimensions. Furthermore, no data is + cached in the group entry scratch-pad space if the object header for the group entry has a link count + greater than one.
+ +\subsubsection subsubsec_fmt1_group_symboltableentry_scratch Format of the Scratch-pad Space +The group entry scratch-pad space is formatted according to the value in the Cache Type field. + +If the Cache Type field contains the value zero (0) then no information is stored in the +scratch-pad space. + +If the Cache Type field contains the value one (1), then the scratch-pad space contains +cached meta data for another object header in the following format: + + + + + + + + + + + + + +
Object Header Scratch-pad Format
bytebytebytebyte
Address of B-tree
Address of Name Heap
+ + + + + + + + + + + + + + +
Field NameDescription
Address of B-treeThis is the file address for the root of the group's B-tree.
Address of Name HeapThis is the file address for the group's local heap, in which are stored the symbol names.
+ +If the Cache Type field contains the value two (2), then the scratch-pad space contains +cached meta data for another symbolic link in the following format: + + + + + + + + + + + +
Symbolic Link Scratch-pad Format
bytebytebytebyte
Offset to Link Value
+ + + + + + + + + + +
Field NameDescription
Offset to Link ValueThe value of a symbolic link (that is, the name of the thing to which it points) is stored in the + local heap. This field is the 4-byte offset into the local heap for the start of the link value, which + is null terminated.
+ +\subsection subsec_fmt1_group_localheap Disk Format: Level 1D - Local Heaps +A heap is a collection of small heap objects. Objects can be inserted and removed from the heap at any time. +The address of a heap does not change once the heap is created. References to objects are stored in the group +table; the names of those objects are stored in the local heap. + + + + + + + + + + + + + + + + + + + + + + + +
Local Heaps
bytebytebytebyte
Heap Signature
Reserved (zero)
Data Segment Size
Offset to Head of Free-list (<size> bytes)
Address of Data Segment
+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Heap SignatureThe ASCII character string HEAP is used to indicate the beginning of a heap. This gives + file consistency checking utilities a better chance of reconstructing a damaged file.
Data Segment SizeThe total amount of disk memory allocated for the heap data. This may be larger than the amount of space + required by the object stored in the heap. The extra unused space holds a linked list of free blocks.
Offset to Head of Free-listThis is the offset within the heap data segment of the first free block (or all 0xff bytes if there is + no free block). The free block contains <size> bytes that are the offset of the next free chunk + (or all 0xff bytes if this is the last free chunk) followed by <size> bytes that store the size + of this free chunk.
Address of Data SegmentThe data segment originally starts immediately after the heap header, but if the data segment must grow + as a result of adding more objects, then the data segment may be relocated, in its entirety, to another + part of the file.
+ +Objects within the heap should be aligned on an 8-byte boundary. + +\subsection subsec_fmt1_group_globalheap Disk Format: Level 1E - Global Heap +Each HDF5 file has a global heap which stores various types of information which is typically shared between +datasets. The global heap was designed to satisfy these goals: +
    +
  1. Repeated access to a heap object must be efficient without resulting in repeated file I/O requests. + Since global heap objects will typically be shared among several datasets, it is probable that the + object will be accessed repeatedly.
    2. +
  2. Collections of related global heap objects should result in fewer and larger I/O requests. For + instance, a dataset of void pointers will have a global heap object for each pointer. Reading the + entire set of void pointer objects should result in a few large I/O requests instead of one small + I/O request for each object.
    3. +
  3. It should be possible to remove objects from the global heap and the resulting file hole should be + eligible to be reclaimed for other uses.
    4. +
+ +The implementation of the heap makes use of the memory management already available at the file level and +combines that with a new top-level object called a collection to achieve Goal B. The global heap is +the set of all collections. Each global heap object belongs to exactly one collection and each collection +contains one or more global heap objects. For the purposes of disk I/O and caching, a collection is treated +as an atomic object. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
A Global Heap Collection
bytebytebytebyte
Magic Number
VersionReserved
Collection Size

Global Heap Object 1 (described below)


Global Heap Object 2


...


Global Heap Object N


Global Heap Object 0 (free space)

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Magic NumberThe magic number for global heap collections are the four bytes G, C, + O, and L.
VersionEach collection has its own version number so that new collections can be added to old files. This + document describes version zero of the collections.
Collection Data SizeThis is the size in bytes of the entire collection including this field. The default (and minimum) + collection size is 4096 bytes which is a typical file system block size and which allows for 170 16-byte + heap objects plus their overhead.
Object 1 through NThe objects are stored in any order with no intervening unused space.
Object 0Object 0 (zero), when present, represents the free space in the collection. Free space always appears + at the end of the collection. If the free space is too small to store the header for Object 0 (described + below) then the header is implied and the collection contains no free space.
+ + + + + + + + + + + + + + + + + + + + + + +
Global Heap Object
bytebytebytebyte
Object IDReference Count
Reserved
Object Data Size

Object Data

+ + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Object IDEach object has a unique identification number within a collection. The identification numbers are + chosen so that new objects have the smallest value possible with the exception that the identifier + 0 always refers to the object which represents all free space within the collection.
Reference CountAll heap objects have a reference count field. An object which is referenced from some other part of the + file will have a positive reference count. The reference count for Object 0 is always zero.
ReservedZero padding to align next field on an 8-byte boundary.
Object Size This is the size of the fields above plus the object data stored for the object. + The actual storage size is rounded up to a multiple of eight.
Object DataThe object data is treated as a one-dimensional array of bytes to be interpreted by the caller.
+ +\subsection subsec_fmt1_group_freespaceindex Disk Format: Level 1F - Free-space Heap +The Free-space Index is a collection of blocks of data, dispersed throughout the file, which are currently +not used by any file objects. + +The super block contains a pointer to root of the free-space description; that pointer is currently (i.e., in +HDF5 Release 1.2) required to be the undefined address 0xfff...ff. + +The free-sapce index is not otherwise publicly defined at this time. + +\section sec_fmt1_dataobject Disk Format: Level 2 - Data Objects +Data objects contain the real information in the file. These objects compose the scientific data and other +information which are generally thought of as "data" by the end-user. All the other information in the file +is provided as a framework for these data objects. + +A data object is composed of header information and data information. The header information contains the +information needed to interpret the data information for the data object as well as additional "meta-data" +or pointers to additional "meta-data" used to describe or annotate each data object. + +\subsection subsec_fmt1_dataobject_hdr Disk Format: Level 2a - Data Object Headers +The header information of an object is designed to encompass all the information about an object which would +be desired to be known, except for the data itself. This information includes the dimensionality, number-type, +information about how the data is stored on disk (in external files, compressed, broken up in blocks, etc.), +as well as other information used by the library to speed up access to the data objects or maintain a file's +integrity. The header of each object is not necessarily located immediately prior to the object's data in the +file and in fact may be located in any position in the file. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Object Headers
bytebytebytebyte
Version \# of Object HeaderReservedNumber of Header Messages
Object Reference Count

Total Object Header Size

Header Message Type \#1Size of Header Message Data \#1
FlagsReserved

Header Message Data \#1

.
.
.
Header Message Type \#nSize of Header Message Data \#n
FlagsReserved

Header Message Data \#n

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Version number of the object headerThis value is used to determine the format of the information in the object header. When the format of + the information in the object header is changed, the version number is incremented and can be used to + determine how the information in the object header is formatted.
ReservedAlways set to zero.
Number of header messagesThis value determines the number of messages listed in this object header. This provides a fast way for + software to prepare storage for the messages in the header.
Object Reference CountThis value specifies the number of references to this object within the current file. References to the + data object from external files are not tracked.
Total Object Header SizeThis value specifies the total number of bytes of header message data following this length field for the + current message as well as any continuation data located elsewhere in the file.
Header Message TypeThe header message type specifies the type of information included in the header message data following + the type along with a small amount of other information. Bit 15 of the message type is set if the message + is constant (constant messages cannot be changed since they may be cached in group entries throughout the + file). The header message types for the pre-defined header messages will be included in further discussion + below.
Size of Header Message DataThis value specifies the number of bytes of header message data following the header message type and + length information for the current message. The size includes padding bytes to make the message a multiple + of eight bytes.
FlagsThis is a bit field with the following definition: +
+
0
+
If set, the message data is constant. This is used for messages like the datatype message of a + dataset.
+
1
+
If set, the message is stored in the global heap and the Header Message Data field contains a + Shared Object message and the Size of Header Message Data field contains the size of that Shared + Object message.
+
2-7
+
Reserved
+
+
Header Message DataThe format and length of this field is determined by the header message type and size respectively. + Some header message types do not require any data and this information can be eliminated by setting the + length of the message to zero. The data is padded with enough zeros to make the size a multiple of + eight.
+ +The header message types and the message data associated with them compose the critical "meta-data" about +each object. Some header messages are required for each object while others are optional. Some optional +header messages may also be repeated several times in the header itself, the requirements and number of +times allowed in the header will be noted in each header message description below. + +The following is a list of currently defined header messages: + +\subsubsection subsubsec_fmt1_dataobject_hdr_nil Name: NIL +Type: 0x0000
+Length: varies
+Status: Optional, may be repeated.
+Purpose and Description: The NIL message is used to indicate a message which is to be ignored +when reading the header messages for a data object. [Probably one which has been deleted for some reason.]
+Format of Data: Unspecified.
+ +\subsubsection subsubsec_fmt1_dataobject_hdr_simple Name: Simple Dataspace +Type: 0x0001
+Length: Varies according to the number of dimensions, as described in the following table
+Status: The Simple Dataspace message is required and may not be repeated. This message +is currently used with datasets and named dataspaces.
+ +The Simple Dataspace message describes the number of dimensions and size of each dimension that the +data object has. This message is only used for datasets which have a simple, rectilinear grid layout; datasets +requiring a more complex layout (irregularly structured or unstructured grids, etc.) must use the +Complex Dataspace message for expressing the space the dataset inhabits. (Note: The +Complex Dataspace functionality is not yet implemented (as of HDF5 Release 1.2). It is not described +in this document.) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Simple Dataspace Message
bytebytebytebyte
VersionDimensionalityFlagsReserved
Reserved
Dimension Size \#1 (<size> bytes)
.
.
.
Dimension Size \#n (<size> bytes)
Dimension Maximum \#1 (<size> bytes)
.
.
.
Dimension Maximum \#n (<size> bytes)
Permutation Index \#1
.
.
.
Permutation Index \#n
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Version This value is used to determine the format of the Simple Dataspace Message. When the format of the + information in the message is changed, the version number is incremented and can be used to determine + how the information in the object header is formatted.
DimensionalityThis value is the number of dimensions that the data object has.
FlagsThis field is used to store flags to indicate the presence of parts of this message. Bit 0 (the least + significant bit) is used to indicate that maximum dimensions are present. Bit 1 is used to indicate + that permutation indices are present for each dimension.
Dimension Size \#n (<size> bytes)This value is the current size of the dimension of the data as stored in the file. The first dimension + stored in the list of dimensions is the slowest changing dimension and the last dimension stored is the + fastest changing dimension.
Dimension Maximum \#n (<size> bytes)This value is the maximum size of the dimension of the data as stored in the file. This value may be + the special value <UNLIMITED> (all bits set) which indicates that the data may expand along this + dimension indefinitely. If these values are not stored, the maximum value of each dimension is assumed + to be the same as the current size value.
Permutation Index \#n (4 bytes)This value is the index permutation used to map each dimension from the canonical representation to an + alternate axis for each dimension. If these values are not stored, the first dimension stored in the list + of dimensions is the slowest changing dimension and the last dimension stored is the fastest changing + dimension.
+ +\subsubsection subsubsec_fmt1_dataobject_hdr_dtmessage Name: Datatype +Type: 0x0003
+Length: variable
+Status: One required per dataset or named datatype
+ +The datatype message defines the datatype for each data point of a dataset. A datatype can describe an atomic +type like a fixed- or floating-point type or a compound type like a C struct. A datatype does not, however, +describe how data points are combined to produce a dataset. Datatypes are stored on disk as a datatype message, +which is a list of datatype classes and their associated properties. + + + + + + + + + + + + + + + + + + + +
Datatype Message
bytebytebytebyte
Type Class and VersionClass Bit Field
Size in Bytes (4 bytes)


Properties

/
+ +The Class Bit Field and Properties fields vary depending on the Type Class, which is the low-order four bits +of the Type Class and Version field (the high-order four bits are the version, which should be set to the +value one). The type class is one of 0 (fixed-point number), 1 (floating-point number), 2 (date and time), +3 (text string), 4 (bit field), 5 (opaque), 6 (compound), 7 (reference), 8 (enumeration), or 9 +(variable-length). The Class Bit Field is zero and the size of the Properties field is zero except for the +cases noted here. + + + + + + + + + + + + + + + + + + + + + + + +
+ Bit Field for Fixed-point Numbers (Class 0) +
BitsMeaning
0Byte Order. If zero, byte order is little-endian; otherwise, byte order is big endian.
1, 2Padding type. Bit 1 is the lo_pad type and bit 2 is the hi_pad type. If a datum has + unused bits at either end, then the lo_pad or hi_pad bit is copied to those locations.
3Signed. If this bit is set then the fixed-point number is in 2's complement form.
4-23Reserved (zero).
+ + + + + + + + + + + + + +
+ Properties for Fixed-point Numbers (Class 0) +
ByteByteByteByte
Bit OffsetBit Precision
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Bit Field for Floating-point Numbers (Class 1)
BitsMeaning
0Byte Order. If zero, byte order is little-endian; otherwise, byte order is big endian.
1, 2, 3Padding type. Bit 1 is the low bits pad type, bit 2 is the high bits pad type, and bit + 3 is the internal bits pad type. If a datum has unused bits at either or between the sign bit, exponent, + or mantissa, then the value of bit 1, 2, or 3 is copied to those locations.
4-5Normalization. The value can be 0 if there is no normalization, 1 if the most significant + bit of the mantissa is always set (except for 0.0), and 2 if the most significant bit of the mantissa is not + stored but is implied to be set. The value 3 is reserved and will not appear in this field.
6-7Reserved (zero).
8-15Sign. This is the bit position of the sign bit.
16-23Reserved (zero).
+ + + + + + + + + + + + + + + + + + + + + + +
Properties for Floating-point Numbers (Class 1)
ByteByteByteByte
Bit OffsetBit Precision
Exponent LocationExponent Size in BitsMantissa LocationMantissa Size in Bits
Exponent Bias
+ + + + + + + + + + + + + + + + + + +
Bit Field for Strings (Class 3)
BitsMeaning
0-3Padding type. This four-bit value determines the type of padding to use for the + string. The values are: +
+
0 Null terminate.
+
A zero byte marks the end of the string and is guaranteed to be present after converting a long + string to a short string. When converting a short string to a long string the value is padded with + additional null characters as necessary.
+
1 Null pad.
+
Null characters are added to the end of the value during conversions from short values to long values + but conversion in the opposite direction simply truncates the value.
+
2 Space pad.
+
Space characters are added to the end of the value during conversions from short values to long values + but conversion in the opposite direction simply truncates the value. This is the Fortran + representation of the string.
+
3-15 Reserved.
+
These values are reserved for future use.
+
+
4-7Character Set. The character set to use for encoding the string. The only character set + supported is the 8-bit ASCII (zero) so no translations have been defined yet.
8-23Reserved (zero).
+ + + + + + + + + + + + + + + + + + + +
Bit Field for Bitfield Types (Class 4)
BitsMeaning
0Byte Order. If zero, byte order is little-endian; otherwise, byte order is big endian.
1, 2Padding type. Bit 1 is the lo_pad type and bit 2 is the hi_pad type. If a datum has + unused bits at either end, then the lo_pad or hi_pad bit is copied to those locations.
3-23Reserved (zero).
+ + + + + + + + + + + + + +
Properties for Bitfield Types (Class 4)
ByteByteByteByte
Bit OffsetBit Precision
+ + + + + + + + + + + +
Bit Field for Opaque Types (Class 5)
BitsMeaning
0-23Reserved (zero).
+ + + + + + + + + + + + +
Properties for Opaque Types (Class 5)
ByteByteByteByte

Null-terminated ASCII Tag
(multiple of 8 bytes)

+ + + + + + + + + + + + + + + +
Bit Field for Compound Types (Class 6)
BitsMeaning
0-15Number of Members. This field contains the number of members defined for the + compound datatype. The member definitions are listed in the Properties field of the data type + message.
15-23Reserved (zero).
+ +The Properties field of a compound datatype is a list of the member definitions of the compound datatype. +The member definitions appear one after another with no intervening bytes. The member types are described +with a recursive datatype message. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Properties for Compound Types (Class 6)
ByteByteByteByte


Name (null terminated, multiple of eight bytes)

/
Byte Offset of Member in Compound Instance
Dimensionalityreserved
Dimension Permutation
Reserved
Size of Dimension 0 (required)
Size of Dimension 1 (required)
Size of Dimension 2 (required)
Size of Dimension 3 (required)


Member Type Message


+ + + + + + + + + + + + + + + +
Bit Field for Enumeration Types (Class 8)
BitsMeaning
0-15Number of Members. The number of name/value pairs defined for the enumeration type.
16-23Reserved (zero).
+ + + + + + + + + + + + + + + + + + +
Properties for Enumeration Types (Class 8)
ByteByteByteByte

Parent Type


Names


Values

+ + + + + + + + + + + + + + +
Parent Type:Each enumeration type is based on some parent type, usually an integer. The information + for that parent type is described recursively by this field.
Names:The name for each name/value pair. Each name is stored as a null terminated ASCII string + in a multiple of eight bytes. The names are in no particular order.
Values:The list of values in the same order as the names. The values are packed (no inter-value + padding) and the size of each value is determined by the parent type.
+ + + + + + + + + + + + + + + + + + + + + + + +
Bit Field for Variable-length Types (Class 9)
BitsMeaning
0-3 +
+
Type
+
0 Variable-length sequence
+
This variable-length datatype can be of any sequence of data. Variable-length sequences do not + have padding or character set information.
+
1 Variable-length string
+
This variable-length datatype is composed of a series of characters. Variable-length strings + have padding and character set information.
+
4-7 +
+
Padding type (variable-length string only)
+
This four-bit value determines the type of padding used for variable-length strings. The values + are the same as for the string padding type, as follows:
+
0 Null terminate
+
A zero byte marks the end of a string and is guaranteed to be present after converting a long + string to a short string. When converting a short string to a long string, the value is padded + with additional null characters as necessary.
+
1 Null pad
+
Null characters are added to the end of the value during conversion from a short string to a + longer string. Conversion from a long string to a shorter string simply truncates the value.
+
2 Space pad
+
Space characters are added to the end of the value during conversion from a short string to a + longer string. Conversion from a long string to a shorter string simply truncates the value. + This is the Fortran representation of the string.
+
3-15 Reserved
+
These values are reserved for future use.
+
8-11 +
+
Character set (variable-length string only)
+
This four-bit value specifies the character set to be used for encoding the string.
+
0 8-bit ASCII
+
As of this writing (July 2002, Release 1.4.4), 8-bit ASCII is the only character set supported. + Therefore, no translations have been defined.
+
12-23Reserved (zero).
+ + + + + + + + + + + + +
Properties for Variable-length Types (Class 9)
ByteByteByteByte

Parent Type

+ + + + + + +
Parent Type:Each variable-length type is based on some parent type. The information for that parent + type is described recursively by this field.
+ +\subsubsection subsubsec_fmt1_dataobject_hdr_fvmessage Name: Data Storage - Fill Value +Type: 0x0004
+Length: varies
+Status: Optional, may not be repeated.
+ +The fill value message stores a single data point value which is returned to the application when an +uninitialized data point is read from the dataset. The fill value is interpreted with the same datatype as +the dataset. If no fill value message is present then a fill value of all zero is assumed. + + + + + + + + + + + + + + + +
Fill Value Message
bytebytebytebyte
Size (4 bytes)

Fill Value

+
+ + + + + + + + + + + + + +
Field NameDescription
Size (4 bytes)This is the size of the Fill Value field in bytes.
Fill ValueThe fill value. The bytes of the fill value are interpreted using the same datatype as for the dataset.
+ +\subsubsection subsubsec_fmt1_dataobject_hdr_message_0005 Name: Reserved - Not Assigned Yet +Type: 0x0005
+Length: N/A
+Status: N/A
+ +\subsubsection subsubsec_fmt1_dataobject_hdr_compact Name: Data Storage - Compact +Type: 0x0006
+Length: varies
+Status: Optional, may not be repeated.
+ +This message indicates that the data for the data object is stored within the current HDF file by including +the actual data as the header data for this message. The data is stored internally in the normal +format, i.e. in one chunk, uncompressed, etc. + +Note that one and only one of the Data Storage headers can be stored for each data object. + +Format of Data: The message data is actually composed of dataset data, so the format will +be determined by the dataset format. + +\subsubsection subsubsec_fmt1_dataobject_hdr_external Name: Data Storage - External Data Files +Type: 0x0007
+Length: varies
+Status: Optional, may not be repeated.
+ +

Purpose and Description: The external object message indicates that the data for an +object is stored outside the HDF5 file. The filename of the object is stored as a Universal Resource Location +(URL) of the actual filename containing the data. An external file list record also contains the byte offset +of the start of the data within the file and the amount of space reserved in the file for that data. + + + + + + + + + + + + + + + + + + + + + + + +
External File List Message
bytebytebytebyte
VersionReserved
Allocated SlotsUsed Slots

Heap Address


Slot Definitions...

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThis value is used to determine the format of the External File List Message. When the format of the + information in the message is changed, the version number is incremented and can be used to determine + how the information in the object header is formatted.
ReservedThis field is reserved for future use.
Allocated SlotsThe total number of slots allocated in the message. Its value must be at least as large as the value + contained in the Used Slots field.
Used SlotsThe number of initial slots which contain valid information. The remaining slots are zero filled.
Heap AddressThis is the address of a local name heap which contains the names for the external files. The name at + offset zero in the heap is always the empty string.
Slot DefinitionsThe slot definitions are stored in order according to the array addresses they represent. If more slots + have been allocated than what has been used then the defined slots are all at the beginning of the list.
+ + + + + + + + + + + + + + + + + + +
External File List Slot
bytebytebytebyte

Name Offset (<size> bytes)


File Offset (<size> bytes)


Size

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
Name Offset (<size> bytes)The byte offset within the local name heap for the name of the file. File names are stored as a URL + which has a protocol name, a host name, a port number, and a file name: + protocol:port//host/file. If the protocol is omitted + then "file:" is assumed. If the port number is omitted then a default port for that protocol is used. + If both the protocol and the port number are omitted then the colon can also be omitted. If the double + slash and host name are omitted then "localhost" is assumed. The file name is the only mandatory part, + and if the leading slash is missing then it is relative to the application's current working directory + (the use of relative names is not recommended).
File Offset (<size> bytes)This is the byte offset to the start of the data in the specified file. For files that contain data for + a single dataset this will usually be zero.
SizeThis is the total number of bytes reserved in the specified file for raw data storage. For a file that + contains exactly one complete dataset which is not extendable, the size will usually be the exact size of + the dataset. However, by making the size larger one allows HDF5 to extend the dataset. The size can be set + to a value larger than the entire file since HDF5 will read zeros past the end of the file without failing.
+ +\subsubsection subsubsec_fmt1_dataobject_hdr_layout Name: Data Storage - Layout +Type: 0x0008
+Length: varies
+Status: Required for datasets, may not be repeated. + +Purpose and Description: Data layout describes how the elements of a multi-dimensional array +are arranged in the linear address space of the file. Two types of data layout are supported: +

    +
  1. The array can be stored in one contiguous area of the file. The layout requires that the size of the + array be constant and does not permit chunking, compression, checksums, encryption, etc. The message + stores the total size of the array and the offset of an element from the beginning of the storage area + is computed as in C.
    2. +
  2. The array domain can be regularly decomposed into chunks and each chunk is allocated separately. This + layout supports arbitrary element traversals, compression, encryption, and checksums, and the chunks + can be distributed across external raw data files (these features are described in other messages). + The message stores the size of a chunk instead of the size of the entire array; the size of the entire + array can be calculated by traversing the B-tree that stores the chunk addresses.
    3. +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Data Layout Message
bytebytebytebyte
VersionDimensionalityLayout ClassReserved
Reserved

Address

Dimension 0 (4-bytes)
Dimension 1 (4-bytes)
...
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionA version number for the layout message. This documentation describes version one.
DimensionalityAn array has a fixed dimensionality. This field specifies the number of dimension size fields later + in the message.
Layout ClassThe layout class specifies how the other fields of the layout message are to be interpreted. A value + of one indicates contiguous storage while a value of two indicates chunked storage. Other values will + be defined in the future.
AddressFor contiguous storage, this is the address of the first byte of storage. For chunked storage this is + the address of the B-tree that is used to look up the addresses of the chunks.
DimensionsFor contiguous storage the dimensions define the entire size of the array while for chunked storage + they define the size of a single chunk.
+ +\subsubsection subsubsec_fmt1_dataobject_hdr_message_0009 Name: Reserved - Not Assigned Yet +Type: 0x0009
+Length: N/A
+Status: N/A
+Purpose and Description: N/A
+Format of Data: N/A + +\subsubsection subsubsec_fmt1_dataobject_hdr_message_000A Name: Reserved - Not Assigned Yet +Type: 0x000A
+Length: N/A
+Status: N/A
+Purpose and Description: N/A
+Format of Data: N/A + +\subsubsection subsubsec_fmt1_dataobject_hdr_filter Name: Data Storage - Filter Pipeline +Type: 0x000B
+Length: varies
+Status: Optional, may not be repeated. + +Purpose and Description: This message describes the filter pipeline which should be +applied to the data stream by providing filter identification numbers, flags, a name, an client data. + + + + + + + + + + + + + + + + + + + + +
Filter Pipeline Message
bytebytebytebyte
VersionNumber of FiltersReserved
Reserved

Filter List

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number for this message. This document describes version one.
Number of FiltersThe total number of filters described by this message. The maximum possible number of filters in a + message is 32.
Filter ListA description of each filter. A filter description appears in the next table.
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Filter Pipeline Message
bytebytebytebyte
Filter IdentificationName Length
FlagsClient Data Number of Values

Name


Client Data

Padding
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Filter IdentificationThis is a unique (except in the case of testing) identifier for the filter. Values from zero through + 255 are reserved for filters defined by the NCSA HDF5 library. Values 256 through 511 have been set + aside for use when developing/testing new filters. The remaining values are allocated to specific + filters by contacting the HDF5 development team.
Name LengthEach filter has an optional null-terminated ASCII name and this field holds the length of the name + including the null termination padded with nulls to be a multiple of eight. If the filter has no name + then a value of zero is stored in this field.
FlagsThe flags indicate certain properties for a filter. The bit values defined so far are: +
+
bit 1
+
If set then the filter is an optional filter. During output, if an optional filter fails it will + be silently removed from the pipeline.
+
Client Data Number of ValuesEach filter can store a few integer values to control how the filter operates. The number of entries + in the Client Data array is stored in this field.
NameIf the Name Length field is non-zero then it will contain the size of this field, a multiple of eight. + This field contains a null-terminated, ASCII character string to serve as a comment/name for the filter.
Client DataThis is an array of four-byte integers which will be passed to the filter function. The Client Data + Number of Values determines the number of elements in the array.
PaddingFour bytes of zeros are added to the message at this point if the Client Data Number of Values field + contains an odd number.
+ +\subsubsection subsubsec_fmt1_dataobject_hdr_attribute Name: Attribute +Type: 0x000C
+Length: varies
+Status: Optional, may be repeated.
+ +Purpose and Description: The Attribute message is used to list objects in the HDF +file which are used as attributes, or "meta-data" about the current object. An attribute is a small dataset; +it has a name, a datatype, a data space, and raw data. Since attributes are stored in the object header they +must be relatively small (<64kb) and can be associated with any type of object which has an object header +(groups, datasets, named types and spaces, etc.). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Attribute Message
bytebytebytebyte
VersionReservedName Size
Type SizeSpace Size

Name


Type


Space


Data

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionVersion number for the message. This document describes version 1 of attribute messages.
ReservedThis field is reserved for later use and is set to zero.
Name SizeThe length of the attribute name in bytes including the null terminator. Note that the Name field + below may contain additional padding not represented by this field.
Type SizeThe length of the datatype description in the Type field below. Note that the Type field may contain + additional padding not represented by this field.
Space SizeThe length of the dataspace description in the Space field below. Note that the Space field may contain + additional padding not represented by this field.
NameThe null-terminated attribute name. This field is padded with additional null characters to make it a + multiple of eight bytes.
TypeThe datatype description follows the same format as described for the datatype object header message. + This field is padded with additional zero bytes to make it a multiple of eight bytes.
SpaceThe dataspace description follows the same format as described for the dataspace object header message. + This field is padded with additional zero bytes to make it a multiple of eight bytes.
DataThe raw data for the attribute. The size is determined from the datatype and dataspace descriptions. + This field is not padded with additional zero bytes.
+ +\subsubsection subsubsec_fmt1_dataobject_hdr_name Name: Object Name +Type: 0x000D
+Length: varies
+Status: Optional, may not be repeated. + +Purpose and Description: The object name or comment is designed to be a short description +of an object. An object name is a sequence of non-zero (\0) ASCII characters with no other +formatting included by the library. + + + + + + + + + + + + +
Name Message
bytebytebytebyte

Name
br />
+ + + + + + + + + + +
Field NameDescription
NameA null terminated ASCII character string.
+ +\subsubsection subsubsec_fmt1_dataobject_hdr_modified Name: Object Modification Date & Time +Type: 0x000E
+Length: fixed
+Status: Optional, may not be repeated. + +Purpose and Description: The object modification date and time is a timestamp which +indicates (using ISO-8601 date and time format) the last modification of an object. The time is + updated when any object header message changes according to the system clock where the change was posted. + + + + + + + + + + + + + + + + + + + + + + + + +
Modification Time Message
bytebytebytebyte
Year
MonthDay of Month
HourMinute
SecondReserved
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
YearThe four-digit year as an ASCII string. For example, 1998. All fields of this message + should be interpreted as coordinated universal time (UTC)
MonthThe month number as a two digit ASCII string where January is 01 and December is + 12.
Day of MonthThe day number within the month as a two digit ASCII string. The first day of the month is + 01.
HourThe hour of the day as a two digit ASCII string where midnight is 00 and 11:00pm + is 23.
MinuteThe minute of the hour as a two digit ASCII string where the first minute of the hour is + 00 and the last is 59.
SecondThe second of the minute as a two digit ASCII string where the first second of the minute is + 00 and the last is 59.
ReservedThis field is reserved and should always be zero.
+ +\subsubsection subsubsec_fmt1_dataobject_hdr_shared Name: Shared Object Message +Type: 0x000F
+Length: 4 Bytes
+Status: Optional, may be repeated. + +A constant message can be shared among several object headers by writing that message in the global +heap and having the object headers all point to it. The pointing is accomplished with a Shared Object +message which is understood directly by the object header layer of the library. It is also possible to +have a message of one object header point to a message in some other object header, but care must be +exercised to prevent cycles. + +If a message is shared, then the message appears in the global heap and its message ID appears in the +Header Message Type field of the object header. Also, the Flags field in the object header for that +message will have bit two set (the H5O_FLAG_SHARED bit). The message body in the object +header will be that of a Shared Object message defined here and not that of the pointed-to message. + + + + + + + + + + + + + + + + +
Shared Message Message
byte + byte + byte + byte +
VersionFlagsReserved
Reserved

Pointer

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number for the message. This document describes version one of shared messages.
FlagsThe Shared Message message points to a message which is shared among multiple object headers. The + Flags field describes the type of sharing: +
+
Bit 0
+
If this bit is clear then the actual message is the first message in some other object header; + otherwise the actual message is stored in the global heap.
+
Bits 2-7
+
Reserved (always zero)
+
PointerThis field points to the actual message. The format of the pointer depends on the value of the Flags + field. If the actual message is in the global heap then the pointer is the file address of the global + heap collection that holds the message, and a four-byte index into that collection. Otherwise the + pointer is a group entry that points to some other object header.
+ +\subsubsection subsubsec_fmt1_dataobject_hdr_continuation Name: Object Header Continuation +Type: 0x0010
+Length: fixed
+Status: Optional, may be repeated.
+Purpose and Description: The object header continuation is the location in the file of more +header messages for the current data object. This can be used when header blocks are large, or likely to +change over time.
+Format of Data: The object header continuation is formatted as follows (assuming a 4-byte +length & offset are being used in the current file): + + + + + + + + + + + + + + +
HDF5 Object Header Continuation Message Layout
bytebytebytebyte
Header Continuation Offset
Header Continuation Length
+ +
+
The elements of the Header Continuation Message are described below:
+
+
+
Header Continuation Offset: (<offset> bytes)
+
This value is the offset in bytes from the beginning of the file where the header continuation + information is located.
+
Header Continuation Length: (<length> bytes)
+
This value is the length in bytes of the header continuation information in the file.
+
+
+ +\subsubsection subsubsec_fmt1_dataobject_hdr_stmgroup Name: Group Message +Type: 0x0011
+Length: fixed
+Status: Required for groups, may not be repeated.
+Purpose and Description: Each group has a B-tree and a name heap which are pointed to +by this message.
+Format of data: + +The group message is formatted as follows: + + + + + + + + + + + + + + +
HDF5 Object Header Group Message Layout
bytebytebytebyte
B-tree Address
Heap Address
+ +
+
The elements of the Group Message are described below:
+
+
+
B-tree Address (<offset> bytes)
+
This value is the offset in bytes from the beginning of the file where the B-tree is located.
+
Heap Address (<offset> bytes)
+
This value is the offset in bytes from the beginning of the file where the group name heap + is located.
+
+
+ +\subsection subsec_fmt1_dataobject_sharedhdr Disk Format: Level 2b - Shared Data Object Headers +In order to share header messages between several dataset objects, object header messages may be placed +into the global heap. Since these messages require additional information beyond the basic object header +message information, the format of the shared message is detailed below. + + + + + + + + + + + + + + + +
HDF5 Shared Object Header Message
bytebytebytebyte
Reference Count of Shared Header Message

Shared Object Header Message

+ +
+
The elements of the shared object header message are described below:
+
+
+
Reference Count of Shared Header Message: (32-bit unsigned integer)
+
This value is used to keep a count of the number of dataset objects which refer to this message + from their dataset headers. When this count reaches zero, the shared message header may be + removed from the global heap.
+
Shared Object Header Message: (various lengths)
+
The data stored for the shared object header message is formatted in the same way as the private + object header messages described in the object header description earlier in this document and + begins with the header message Type.
+
+
+ +\subsection subsec_fmt1_dataobject_storage Disk Format: Level 2c - Data Object Data Storage +The data for an object is stored separately from the header information in the file and may not actually +be located in the HDF5 file itself if the header indicates that the data is stored externally. The +information for each record in the object is stored according to the dimensionality of the object +(indicated in the dimensionality header message). Multi-dimensional data is stored in C order [same +as current scheme], i.e. the "last" dimension changes fastest. + +Data whose elements are composed of simple number-types are stored in native-endian IEEE format, unless +they are specifically defined as being stored in a different machine format with the architecture-type +information from the number-type header message. This means that each architecture will need to +[potentially] byte-swap data values into the internal representation for that particular machine. + +Data with a "variable" sized number-type is stored in a data heap internal to the HDF5 file. Global heap +identifiers are stored in the data object storage. + +Data whose elements are composed of pointer number-types are stored in several different ways depending +on the particular pointer type involved. Simple pointers are just stored as the dataset offset of the +object being pointed to with the size of the pointer being the same number of bytes as offsets in the file. +Partial-object pointers are stored as a heap-ID which points to the following information within the +file-heap: an offset of the object pointed to, number-type information (same format as header message), +dimensionality information (same format as header message), sub-set start and end information (i.e. a +coordinate location for each), and field start and end names (i.e. a [pointer to the] string indicating +the first field included and a [pointer to the] string name for the last field). + +Data of a compound datatype is stored as a contiguous stream of the items in the structure, with each +item formatted according to its datatype. + +*/ diff --git a/doxygen/dox/H5.format.1.1.dox b/doxygen/dox/H5.format.1.1.dox new file mode 100644 index 00000000000..e120c35c3ab --- /dev/null +++ b/doxygen/dox/H5.format.1.1.dox @@ -0,0 +1,3887 @@ + +/** \page FMT11 HDF5 File Format Specification Version 1.1 +
    +
  1. @ref sec_fmt11_intro
    2. +
  2. @ref sec_fmt11_meta +
      +
    1. @ref subsec_fmt11_boot_super
      2. +
    2. @ref subsec_fmt11_boot_driver
      3. +
    3. +
  3. @ref sec_fmt11_infra +
      +
    1. @ref subsec_fmt11_infra_btrees
      2. +
    2. @ref subsec_fmt11_infra_symboltable
      3. +
    3. @ref subsec_fmt11_infra_symboltableentry
      4. +
    4. @ref subsec_fmt11_infra_localheap
      5. +
    5. @ref subsec_fmt11_infra_globalheap
      6. +
    6. @ref subsec_fmt11_infra_freespaceindex
      7. +
    4. +
  4. @ref sec_fmt11_dataobject +
      +
    1. @ref subsec_fmt11_dataobject_hdr +
        +
      1. @ref subsubsec_fmt11_dataobject_hdr_nil
        2. +
      2. @ref subsubsec_fmt11_dataobject_hdr_simple
        3. +
      3. @ref subsubsec_fmt11_dataobject_hdr_message_0002
        4. +
      4. @ref subsubsec_fmt11_dataobject_hdr_dtmessage
        5. +
      5. @ref subsubsec_fmt11_dataobject_hdr_ofvmessage
        6. +
      6. @ref subsubsec_fmt11_dataobject_hdr_fvmessage
        7. +
      7. @ref subsubsec_fmt11_dataobject_hdr_message_0006
        8. +
      8. @ref subsubsec_fmt11_dataobject_hdr_external
        9. +
      9. @ref subsubsec_fmt11_dataobject_hdr_layout
        10. +
      10. @ref subsubsec_fmt11_dataobject_hdr_message_0009
        11. +
      11. @ref subsubsec_fmt11_dataobject_hdr_message_000A
        12. +
      12. @ref subsubsec_fmt11_dataobject_hdr_filter
        13. +
      13. @ref subsubsec_fmt11_dataobject_hdr_attribute
        14. +
      14. @ref subsubsec_fmt11_dataobject_hdr_comment
        15. +
      15. @ref subsubsec_fmt11_dataobject_hdr_modified
        16. +
      16. @ref subsubsec_fmt11_dataobject_hdr_shared
        17. +
      17. @ref subsubsec_fmt11_dataobject_hdr_continuation
        18. +
      18. @ref subsubsec_fmt11_dataobject_hdr_stmgroup
        19. +
      19. @ref subsubsec_fmt11_dataobject_hdr_mod
        20. +
      2. +
    2. @ref subsec_fmt11_dataobject_storage
      3. +
    +
    5. +
  5. @ref sec_fmt11_appendix +
+ + + +\section sec_fmt11_intro Introduction + + + + + + + + + + + + + + +
Figure 1: Relationships among the HDF5 root group, other groups, and objects
\image html FF-IH_FileGroup.gif
Figure 2: HDF5 objects -- datasets, datatypes, or dataspaces
\image html FF-IH_FileObject.gif
+ +The format of an HDF5 file on disk encompasses several key ideas of the HDF4 and AIO file formats as well +as addressing some shortcomings therein. The new format is more self-describing than the HDF4 format and +is more uniformly applied to data objects in the file. + +An HDF5 file appears to the user as a directed graph. The nodes of this graph are the higher-level HDF5 +objects that are exposed by the HDF5 APIs: +\li Groups +\li Datasets +\li Named datatypes + +At the lowest level, as information is actually written to the disk, an HDF5 file is made up of the +following objects: +\li A super block +\li B-tree nodes (containing either symbol nodes or raw data chunks) +\li Object headers +\li A global heap +\li Local heaps +\li Free space + +The HDF5 library uses these lower-level objects to represent the higher-level objects that are then +presented to the user or to applications through the APIs. For instance, a group is an object header that +contains a message that points to a local heap and to a B-tree which points to symbol nodes. A dataset is +an object header that contains messages that describe datatype, space, layout, filters, external files, +fill value, etc with the layout message pointing to either a raw data chunk or to a B-tree that points to +raw data chunks. + +\subsection subsec_fmt11_intro_doc This Document +This document describes the lower-level data objects; the higher-level objects and their properties are +described in the \ref UG. + +Three levels of information comprise the file format. Level 0 contains basic information for identifying +and defining information about the file. Level 1 information contains the information about the pieces of a +file shared by many objects in the file (such as a B-trees and heaps). Level 2 is the rest of the file and +contains all of the data objects, with each object partitioned into header information, also known as +metadata, and data. + +The sizes of various fields in the following layout tables are determined by looking at the number of +columns the field spans in the table. There are three exceptions: (1) The size may be overridden by +specifying a size in parentheses, (2) the size of addresses is determined by the Size of Offsets +field in the super block and is indicated in this document with a superscripted 'O', and (3) the size of +length fields is determined by the Size of Lengths field in the super block and is indicated +in this document with a superscripted 'L'. + +Values for all fields in this document should be treated as unsigned integers, unless otherwise noted in +the description of a field. Additionally, all metadata fields are stored in little-endian byte order. + +\section sec_fmt11_meta Disk Format: Level 0 - File Metadata + +\subsection subsec_fmt11_boot_super Disk Format: Level 0A - File Signature and Super Block +The super block may begin at certain predefined offsets within the HDF5 file, allowing a block of +unspecified content for users to place additional information at the beginning (and end) of the HDF5 file +without limiting the HDF5 library's ability to manage the objects within the file itself. This feature was +designed to accommodate wrapping an HDF5 file in another file format or adding descriptive information to +the file without requiring the modification of the actual file's information. The super block is located by +searching for the HDF5 file signature at byte offset 0, byte offset 512 and at successive locations in the +file, each a multiple of two of the previous location, i.e. 0, 512, 1024, 2048, etc. + +The super block is composed of a file signature, followed by super block and group version numbers, +information about the sizes of offset and length values used to describe items within the file, the size of +each group page, and a group entry for the root object in the file. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
HDF5 Super Block Layout
bytebytebytebyte

HDF5 File Signature (8 bytes)

Version \# of Super BlockVersion \# of Global Free-space StorageVersion \# of Root Group Symbol Table EntryReserved (zero)
Version \# of Shared Header Message FormatSize of OffsetsSize of LengthsReserved (zero)
Group Leaf Node KGroup Internal Node K
File Consistency Flags
Indexed Storage Internal Node K1Reserved (zero)1
Base AddressO
Address of Global Free-space HeapO
End of File AddressO
Driver Information Block AddressO
Root Group Symbol Table Entry
+\li Items marked with an 'O' in the above table are of the size specified in "Size of Offsets." +\li Items marked with an '1' in the above table are new in version 1 of the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
HDF5 File SignatureThis field contains a constant value and can be used to quickly identify a file as being an HDF5 + file. The constant value is designed to allow easy identification of an HDF5 file and to allow + certain types of data corruption to be detected. The file signature of an HDF5 file always + contains the following values: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Decimal:13772687013102610
Hexadecimal:894844460d0a1a0a
ASCII C Notation:\211HDF\\r\\n\032\\n
+
+ This signature both identifies the file as an HDF5 file and provides for immediate detection of common + file-transfer problems. The first two bytes distinguish HDF5 files on systems that expect the first two + bytes to identify the file type uniquely. The first byte is chosen as a non-ASCII value to reduce the + probability that a text file may be misrecognized as an HDF5 file; also, it catches bad file transfers + that clear bit 7. Bytes two through four name the format. The CR-LF sequence catches bad file transfers + that alter newline sequences. The control-Z character stops file display under MS-DOS. The final line + feed checks for the inverse of the CR-LF translation problem. (This is a direct descendent of the + PNG + file signature.)
+ This field is present in version 0+ of the superblock.
Version Number of the Super BlockThis value is used to determine the format of the information in the super block. When the format of + the information in the super block is changed, the version number is incremented to the next integer + and can be used to determine how the information in the super block is formatted.
+ Values of 0 and 1 are defined for this field.
+ This field is present in version 0+ of the superblock.
Version Number of the File Free-space HeapThis value is used to determine the format of the information in the File Free-space Heap.
+ The only value currently valid in this field is '0', which indicates that the free space index is + formatted as described in the @ref subsec_fmt11_infra_freespaceindex below.
+ This field is present in version 0+ of the superblock.
Version Number of the Root Group Symbol Table EntryThis value is used to determine the format of the information in the Group. When the format of the + information in the Root Group Symbol Table Entry, When the format of the information in that field + is changed, the version number is incremented to the next integer and can be used to determine how + the information in the field is formatted.
+ The only value currently valid in this field is '0', which indicates that the root group symbol + table entry is formatted as described in the @ref subsec_fmt11_infra_symboltableentry below.
+ This field is present in version 0+ of the superblock.
Version Number of the Shared Header Message FormatThis value is used to determine the format of the information in a shared object header message, which + is stored in the global small-data heap. Since the format of the shared header messages differs from + the other private header messages, a version number is used to identify changes in the format.
+ The only value currently valid in this field is '0', which indicates that shared header messages are + formatted as described in the @ref subsubsec_fmt11_dataobject_hdr_shared below.
+ This field is present in version 0+ of the superblock.
Size of OffsetsThis value contains the number of bytes used to store addresses in the file. The values for the + addresses of objects in the file are offsets relative to a base address, usually the address of the + super block signature. This allows a wrapper to be added after the file is created without invalidating + the internal offset locations.
+ This field is present in version 0+ of the superblock.
Size of LengthsThis value contains the number of bytes used to store the size of an object.
+ This field is present in version 0+ of the superblock.
Group Leaf Node KEach leaf node of a group B-tree will have at least this many entries but not more than twice this + many. If a group has a single leaf node then it may have fewer entries.
+ This value must be greater than zero.
+ See the @ref subsec_fmt11_infra_btrees below.
+ This field is present in version 0+ of the superblock.
Group Internal Node KEach internal node of a group B-tree will have at least this many entries but not more than twice this + many. If the group has only one internal node then it might have fewer entries.
+ This value must be greater than zero.
+ See the @ref subsec_fmt11_infra_btrees below.
+ This field is present in version 0+ of the superblock.
File Consistency FlagsThis value contains flags to indicate information about the consistency of the information contained + within the file. Currently, the following bit flags are defined: +
    +
  • Bit 0 set indicates that the file is opened for write-access.
    • +
  • Bit 1 set indicates that the file has been verified for consistency and is guaranteed to be + consistent with the format defined in this document.
    • +
  • Bits 2-31 are reserved for future use.
    • +
+ Bit 0 should be set as the first action when a file is opened for write access and should be cleared + only as the final action when closing a file. Bit 1 should be cleared during normal access to a file + and only set after the file's consistency is guaranteed by the library or a consistency utility.
+ This field is present in version 0+ of the superblock.
Indexed Storage Internal Node KEach internal node of a indexed storage B-tree will have at least this many entries but not more than + twice this many. If the group has only one internal node then it might have fewer entries.
+ This value must be greater than zero.
+ See the @ref subsec_fmt11_infra_btrees below.
+ This field is present in version 1+ of the superblock.
Base AddressThis is the absolute file address of the first byte of the HDF5 data within the file. The library + currently constrains this value to be the absolute file address of the super block itself when creating + new files; future versions of the library may provide greater flexibility. When opening an existing + file and this address does not match the offset of the superblock, the library assumes that the entire + contents of the HDF5 file have been adjusted in the file and adjusts the base address and end of file + address to reflect their new positions in the file. Unless otherwise noted, all other file addresses + are relative to this base address.
+ This field is present in version 0+ of the superblock.
Address of Global Free-space IndexFree-space management is not yet defined in the HDF5 file format and is not handled by the library. + Currently this field always contains the undefined address.
+ This field is present in version 0+ of the superblock.
End of File AddressThis is the relative file address of the first byte past the end of all HDF5 data. It is used to + determine whether a file has been accidentally truncated and as an address where file data allocation + can occur if the free list is not used.
+ This field is present in version 0+ of the superblock.
Driver Information Block AddressThis is the relative file address of the file driver information block which contains driver-specific + information needed to reopen the file. If there is no driver information block then this entry should + be the undefined address.
+ This field is present in version 0+ of the superblock.
Root Group Symbol Table EntryThis is the @ref subsec_fmt11_infra_symboltableentry of the root group, which serves as the entry-point + into the group graph for the file.
+ This field is present in version 0+ of the superblock.
+ +\subsection subsec_fmt11_boot_driver Disk Format: Level 0B - File Driver Info +The file driver information block is an optional region of the file which contains information +needed by the file driver in order to reopen a file. The format of the file driver information block is: + + + + + + + + + + + + + + + + + + + + + +
Driver Information Block
bytebytebytebyte
VersionReserved (zero)
Driver Information Size (4 bytes)

Driver Identification (8 bytes)



Driver Information (n bytes)


+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number of the driver information block. The file format documented here is version zero.
Driver Information SizeThe size in bytes of the Driver Information part of this structure.
Driver IdentificationThis is an eight-byte ASCII string without null termination which identifies the driver and version number + of the Driver Information block. The predefined drivers supplied with the HDF5 library are identified by the + letters NCSA followed by the first four characters of the driver name. If the Driver Information + block is not the original version then the last letter(s) of the identification will be replaced by a version + number in ASCII.
+ For example, the various versions of the multi driver will be identified by + NCSAmult. (NCSAmult is simply NCSAmulti truncated to eight characters. + Subsequent identifiers will be created by substituting sequential numerical values for the final character, + starting with zero.) multi driver is the only default driver that is encoded in this field.
+ Identification for user-defined drivers is eight-byte long and arbitrary but should be unique and avoid + the four character prefix "NCSA".
Driver InformationDriver information is encoded/decoded in a format defined by the file driver, multi driver is + the only default driver that has driver information stored in this field. Its format is explained in the + following block.
+ +Multi driver has the following format: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Multi Driver Message
bytebytebytebyte
Member MappingMember MappingMember MappingMember Mapping
Member MappingMember MappingReservedReserved

Address of Member File 1


End of Address for Member File 1


Address of Member File 2


End of Address for Member File 2


... ...


Name of Member File 1


Name of Member File 2


... ...

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Member MappingMulti driver enables different types of HDF5 data and metadata to be written to separate + files. These files are viewed by the library as a single virtual HDF5 file with a single file address. + It allows maximal 6 files to be created. In sequence, these Member Mapping fields are for + super block, B-tree, raw data, global heap, local heap, and object header. More than one type of data + can be written to the same file.
+ These Member Mapping fields are integer values from 1 to 6 indicating how the data can be + mapped to or merged with another type of data. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Member MappingDescription
1The super block data.
2The B-tree data.
3The raw data.
4The global heap data.
5The local heap data.
6The object header data.

+ For example, if the third field has the value 3 and all the rest have the + value 1, it means there are two files, one for raw data, one for super block, + B-tree, global heap, local heap, and object header.
ReservedThese fields are reserved and should always be zero.
Address of Member FileSpecifies the virtual address. A normally eight-byte integer with the value from 0 + (zero) to maximal value, at which the member file starts.
End of Address for Member FileThe end of allocated address for the member file. A normally eight-byte integer value.
Name of Member FileThe null-terminated name of member file. Its length should be multiples of 8 bytes. + Additional bytes will be padded with NULLs. The default naming convention is + %%s-X.h5, where X is one of the letters s (for super block), + b (for B-tree), r (for raw data), g (for global heap), + l (for local heap), and o (for object header). The name for the whole + HDF5 file will substitute the %s in the string.
+ +\section sec_fmt11_infra Disk Format: Level 1 - File Infrastructure + +\subsection subsec_fmt11_infra_btrees Disk Format: Level 1A - B-link Trees and B-tree Nodes +B-link trees allow flexible storage for objects which tend to grow in ways that cause the object to be stored +discontiguously. B-trees are described in various algorithms books including "Introduction to Algorithms" by +Thomas H. Cormen, Charles E. Leiserson, and Ronald L. Rivest. The B-link tree, in which the sibling nodes at +a particular level in the tree are stored in a doubly-linked list, is described in the "Efficient Locking +for Concurrent Operations on B-trees" paper by Phillip Lehman and S. Bing Yao as published in the ACM +Transactions on Database Systems, Vol. 6, No. 4, December 1981. + +The B-link trees implemented by the file format contain one more key than the number of children. In other +words, each child pointer out of a B-tree node has a left key and a right key. The pointers out of internal +nodes point to sub-trees while the pointers out of leaf nodes point to symbol nodes and raw data chunks. +Aside from that difference, internal nodes and leaf nodes are identical. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
B-tree Nodes
bytebytebytebyte
Node Signature
Node TypeNode LevelEntries Used
Address of Left SiblingO
Address of Right SiblingO
Key 0 (variable size)
Address of Child 0O
Key 1 (variable size)
Address of Child 1O
...
Key 2K (variable size)
Address of Child 2KO
Key 2K+1 (variable size)
+ + + + + +
(Items marked with an 'O' the above table are of the size specified in "Size of Offsets.")
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string TREE is used to indicate the beginning of a B-link tree node. + This gives file consistency checking utilities a better chance of reconstructing a damaged file.
Node TypeEach B-link tree points to a particular type of data. This field indicates the type of data as well as + implying the maximum degree K of the tree and the size of each Key field.
+ + + + + + + + + + + +
Node TypeDescription
0This tree points to group nodes.1This tree points to a raw data chunk.
+
Node LevelThe node level indicates the level at which this node appears in the tree (leaf nodes are at level + zero). Not only does the level indicate whether child pointers point to sub-trees or to data, but it + can also be used to help file consistency checking utilities reconstruct damaged trees.
Entries UsedThis determines the number of children to which this node points. All nodes of a particular type of + tree have the same maximum degree, but most nodes will point to less than that number of children. The + valid child pointers and keys appear at the beginning of the node and the unused pointers and keys + appear at the end of the node. The unused pointers and keys have undefined values.
Address of Left SiblingThis is the relative file address of the left sibling of the current node. If the current node is the + left-most node at this level then this field is the undefined address.
Address of Right SiblingThis is the relative file address of the right sibling of the current node. If the current node is the + right-most node at this level then this field is the undefined address.
Keys and Child PointersEach tree has 2K+1 keys with 2K child pointers interleaved between the keys. The number + of keys and child pointers actually containing valid values is determined by the node's Entries Used + field. If that field is N then the B-link tree contains N child pointers and + N+1 keys.
KeyThe format and size of the key values is determined by the type of data to which this tree points. The + keys are ordered and are boundaries for the contents of the child pointer; that is, the key values + represented by child N fall between Key N and Key N+1. Whether the interval + is open or closed on each end is determined by the type of data to which the tree points.
+ The format of the key depends on the node type. + For nodes of node type 0, the key is formatted as follows: + + + + + +
A single field of Size of Lengths bytesIndicates the byte offset into the local heap for the first object name in the subtree which + that key describes.
+
+ For nodes of node type 1, the key is formatted as follows: + + + + + + + + + + + + + +
Bytes 1-4Size of chunk in bytes.
Bytes 4-8Filter mask, a 32-bit bitfield indicating which filters have been skipped for this chunk. Each + filter has an index number in the pipeline (starting at 0, with the first filter to apply) and + if that filter is skipped, the bit corresponding to it's index is set.
N 64-bit fieldsA 64-bit index indicating the offset of the chunk within the dataset where N is the number + of dimensions of the dataset. For example, if a chunk in a 3-dimensional dataset begins at the + position [5,5,5], there will be three such 64-bit indices, each with the value of + 5.
+
Child PointerThe tree node contains file addresses of subtrees or data depending on the node level. Nodes at Level + 0 point to data addresses, either raw data chunk or group nodes. Nodes at non-zero levels point to other + nodes of the same B-tree.
+ For raw data chunk nodes, the child pointer is the address of a single raw data chunk. For group nodes, + the child pointer points to a @ref subsec_fmt11_infra_symboltableentry, which contains + information for multiple symbol table entries.
+ +Conceptually, each B-tree node looks like this: + + + + + + + + + + + + + +
key[0] child[0] key[1] child[1] key[2]; ... ... key[N-1] child[N-1] key[N]
+where child[i] is a pointer to a sub-tree (at a level above Level 0) or to data (at Level 0). +Each key[i] describes an item stored by the B-tree (a chunk or an object of a group node). +The range of values represented by child[i] is indicated by key[i] and key[i+1]. + +The following question must next be answered: "Is the value described by key[i] contained in +child[i-1] or in child[i]?" The answer depends on the type of tree. In trees for groups (node +type 0) the object described by key[i] is the greatest object contained in child[i-1] while +in chunk trees (node type 1) the chunk described by key[i] is the least chunk in child[i]. + +That means that key[0] for group trees is sometimes unused; it points to offset zero in the heap, which is +always the empty string and compares as "less-than" any valid object name. + +And key[N] for chunk trees is sometimes unused; it contains a chunk offset which compares as +"greater-than" any other chunk offset and has a chunk byte size of zero to indicate that it is not actually +allocated. + +\subsection subsec_fmt11_infra_symboltable Disk Format: Level 1B - Group and Symbol Nodes +A group is an object internal to the file that allows arbitrary nesting of objects within the file (including +other groups). A group maps a set of names in the group to a set of relative file addresses where objects with +those names are located in the file. Certain metadata for an object to which the group points can be cached in +the group's symbol table in addition to the object's header. + +An HDF5 object name space can be stored hierarchically by partitioning the name into components and storing +each component in a group. The group entry for a non-ultimate component points to the group containing the +next component. The group entry for the last component points to the object being named. + +A group is a collection of group nodes pointed to by a B-link tree. Each group node contains entries for one +or more symbols. If an attempt is made to add a symbol to an already full group node containing 2K +entries, then the node is split and one node contains K symbols and the other contains K+1 +symbols. + + + + + + + + + + + + + + + + + + + + +
Group Node (A Leaf of a B-tree)
bytebytebytebyte
Signature
Version NumberReserved (0)Number of Symbols


Group Entries


+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string SNOD is used to indicate the beginning of a group node. This + gives file consistency checking utilities a better chance of reconstructing a damaged file.
Version NumberThe version number for the group node. This document describes version 1. (There is no version '0' + of the group node)
Number of SymbolsAlthough all group nodes have the same length, most contain fewer than the maximum possible number of + symbol entries. This field indicates how many entries contain valid data. The valid entries are packed + at the beginning of the group node while the remaining entries contain undefined values.
Group EntriesEach symbol has an entry in the group node. The format of the entry is described below. There are + 2K entries in each group node, where K is the "Group Leaf Node K" value from the + @ref subsec_fmt11_boot_super.
+ +\subsection subsec_fmt11_infra_symboltableentry Disk Format: Level 1C - Group Entry +Each group entry in a group node is designed to allow for very fast browsing of stored objects. Toward that +design goal, the group entries include space for caching certain constant metadata from the object header. + + + + + + + + + + + + + + + + + + + + + + + + +
Group Entry
bytebytebytebyte
Name OffsetO
Object Header AddressO
Cache Type
Reserved


Scratch-pad Space (16 bytes)


+ + + + + +
(Items marked with an 'O' the above table are of the size specified in "Size of Offsets.")
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Name OffsetThis is the byte offset into the group local heap for the name of the object. The name is null + terminated.
Object Header AddressEvery object has an object header which serves as a permanent location for the object's metadata. + In addition to appearing in the object header, some metadata can be cached in the scratch-pad space.
Cache TypeThe cache type is determined from the object header. It also determines the format for the scratch-pad + space.
+ + + + + + + + + + + + + + + + + + + + + +
Type:Description:
0No data is cached by the group entry. This is guaranteed to be the case when an object header has + a link count greater than one.
1Object header metadata is cached in the group entry. This implies that the group entry refers to + another group.
2The entry is a symbolic link. The first four bytes of the scratch-pad space are the offset into + the local heap for the link value. The object header address will be undefined.
NOther cache values can be defined later and libraries that do not understand the new values will + still work properly.
+
ReservedThese four bytes are present so that the scratch-pad space is aligned on an eight-byte boundary. They + are always set to zero.
Scratch-pad SpaceThis space is used for different purposes, depending on the value of the Cache Type field. Any meta-data + about a dataset object represented in the scratch-pad space is duplicated in the object header for + that dataset. This metadata can include the datatype and the size of the dataspace for a dataset whose + datatype is atomic and whose dataspace is fixed and less than four dimensions.
+ Furthermore, no data is cached in the group entry scratch-pad space if the object header for the group + entry has a link count greater than one.
+ +\subsubsection subsubsec_fmt11_infra_symboltableentry_scratch Format of the Scratch-pad Space +The group entry scratch-pad space is formatted according to the value in the Cache Type field. + +If the Cache Type field contains the value zero ((0)) then no information is stored in the +scratch-pad space. + +If the Cache Type field contains the value one (1), then the scratch-pad space contains +cached metadata for another object header in the following format: + + + + + + + + + + + + + +
Object Header Scratch-pad Format
bytebytebytebyte
Address of B-treeO
Address of Name HeapO
+ + + + + +
(Items marked with an 'O' the above table are of the size specified in "Size of Offsets.")
+ + + + + + + + + + + + + + +
Field NameDescription
Address of B-treeThis is the file address for the root of the group's B-tree.
Address of Name HeapThis is the file address for the group's local heap, in which are stored the symbol names.
+ +If the Cache Type field contains the value two ((2)), then the scratch-pad space contains +cached metadata for another symbolic link in the following format: + + + + + + + + + + + +
Symbolic Link Scratch-pad Format
bytebytebytebyte
Offset to Link Value
+ + + + + + + + + + +
Field NameDescription
Offset to Link ValueThe value of a symbolic link (that is, the name of the thing to which it points) is stored in the + local heap. This field is the 4-byte offset into the local heap for the start of the link value, which + is null terminated.
+ +\subsection subsec_fmt11_infra_localheap Disk Format: Level 1D - Local Heaps +A heap is a collection of small heap objects. Objects can be inserted and removed from the heap at any time. +The address of a heap does not change once the heap is created. References to objects are stored in the group +table; the names of those objects are stored in the local heap. + + + + + + + + + + + + + + + + + + + + + + + + +
Local Heap
bytebytebytebyte
Signature
VersionReserved (zero)
Data Segment SizeL
Offset to Head of Free-listL
Address of Data SegmentO
+ + + + + + + + +
(Items marked with an 'L' the above table are of the size specified in "Size of Lengths.")
(Items marked with an 'O' the above table are of the size specified in "Size of Offsets.")
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string HEAP is used to indicate the beginning of a heap. This gives + file consistency checking utilities a better chance of reconstructing a damaged file.
VersionEach local heap has its own version number so that new heaps can be added to old files. This document + describes version zero (0) of the local heap.
Data Segment SizeThe total amount of disk memory allocated for the heap data. This may be larger than the amount of space + required by the object stored in the heap. The extra unused space holds a linked list of free blocks.
Offset to Head of Free-listThis is the offset within the heap data segment of the first free block (or the + undefined address if there is no no free block). The free block + contains "Size of Lengths" bytes that are the offset of the next free block (or the value '1' if this is + the last free block) followed by "Size of Lengths" bytes that store the size of this free block. The size + of the free block includes the space used to store the offset of the next free block and the of the current + block, making the minimum size of a free block 2 * "Size of Lengths".
Address of Data SegmentThe data segment originally starts immediately after the heap header, but if the data segment must grow + as a result of adding more objects, then the data segment may be relocated, in its entirety, to another + part of the file.
+ +Objects within the heap should be aligned on an 8-byte boundary. + +\subsection subsec_fmt11_infra_globalheap Disk Format: Level 1E - Global Heap +Each HDF5 file has a global heap which stores various types of information which is typically shared between +datasets. The global heap was designed to satisfy these goals: +
    +
  1. Repeated access to a heap object must be efficient without resulting in repeated file I/O requests. + Since global heap objects will typically be shared among several datasets, it is probable that the + object will be accessed repeatedly.
    2. +
  2. Collections of related global heap objects should result in fewer and larger I/O requests. For + instance, a dataset of object references will have a global heap object for each reference. Reading + the entire set of object references should result in a few large I/O requests instead of one small + I/O request for each reference.
    3. +
  3. It should be possible to remove objects from the global heap and the resulting file hole should be + eligible to be reclaimed for other uses.
    4. +
+ +The implementation of the heap makes use of the memory management already available at the file level and +combines that with a new top-level object called a collection to achieve Goal B. The global heap is +the set of all collections. Each global heap object belongs to exactly one collection and each collection +contains one or more global heap objects. For the purposes of disk I/O and caching, a collection is treated +as an atomic object. + +The HDF5 library creates global heap collections as needed, so there may be multiple collections throughout +the file. The set of all of them is abstractly called the "global heap", although they don't actually link +to each other, and there is no global place in the file where you can discover all of the collections. The +collections are found simply by finding a reference to one through another object in the file (eg. +variable-length datatype elements, etc). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
A Global Heap Collection
bytebytebytebyte
Signature
VersionReserved
Collection SizeL

Global Heap Object 1


Global Heap Object 2


...


Global Heap Object N


Global Heap Object 0 (free space)

+ + + + + +
(Items marked with an 'L' the above table are of the size specified in "Size of Lengths.")
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string "GCOL" is used to indicate the beginning of a collection. + This gives file consistency checking utilities a better chance of reconstructing a damaged file.
VersionEach collection has its own version number so that new collections can be added to old files. This + document describes version one (1) of the collections (there is no version zero (0)).
Collection SizeThis is the size in bytes of the entire collection including this field. The default (and minimum) + collection size is 4096 bytes which is a typical file system block size. This allows for 127 16-byte + heap objects plus their overhead (the collection header of 16 bytes and the 16 bytes of information + about each heap object).
Global Heap Object 1 through NThe objects are stored in any order with no intervening unused space.
Global Heap Object 0Global Heap Object 0 (zero), when present, represents the free space in the collection. Free space always + appears at the end of the collection. If the free space is too small to store the header for Object 0 + (described below) then the header is implied and the collection contains no free space.
+ + + + + + + + + + + + + + + + + + + + + + +
Global Heap Object
bytebytebytebyte
Heap Object IDReference Count
Reserved
Object SizeL

Object Data

+ + + + + +
(Items marked with an 'L' the above table are of the size specified in "Size of Lengths.")
+ + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Heap Object IDEach object has a unique identification number within a collection. The identification numbers are + chosen so that new objects have the smallest value possible with the exception that the identifier + 0 always refers to the object which represents all free space within the collection.
Reference CountAll heap objects have a reference count field. An object which is referenced from some other part of the + file will have a positive reference count. The reference count for Object 0 is always zero.
ReservedZero padding to align next field on an 8-byte boundary.
Object Size This is the size of the object data stored for the object. The actual storage space + allocated for the object data is rounded up to a multiple of eight.
Object DataThe object data is treated as a one-dimensional array of bytes to be interpreted by the caller.
+ +\subsection subsec_fmt11_infra_freespaceindex Disk Format: Level 1F - Free-space Index +The free-space index is a collection of blocks of data, dispersed throughout the file, which are currently +not used by any file objects. + +The super block contains a pointer to root of the free-space description; that pointer is currently required +to be the undefined address. + +The format of the free-space index is not defined at this time. + +\section sec_fmt11_dataobject Disk Format: Level 2 - Data Objects +Data objects contain the real information in the file. These objects compose the scientific data and other +information which are generally thought of as "data" by the end-user. All the other information in the file +is provided as a framework for these data objects. + +A data object is composed of header information and data information. The header information contains the +information needed to interpret the data information for the data object as well as additional "meta-data" +or pointers to additional "meta-data" used to describe or annotate each data object. + +\subsection subsec_fmt11_dataobject_hdr Disk Format: Level 2A - Data Object Headers +The header information of an object is designed to encompass all the information about an object, except for +the data itself. This information includes the dataspace, datatype, information about how the data is stored +on disk (in external files, compressed, broken up in blocks, etc.), as well as other information used by the +library to speed up access to the data objects or maintain a file's integrity. Information stored by user +applications as attributes is also stored in the object's header. The header of each object is not necessarily +located immediately prior to the object's data in the file and in fact may be located in any position in the +file. The order of the messages in an object header is not significant. + +Header messages are aligned on 8-byte boundaries. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Object Headers
bytebytebytebyte
VersionReserved (zero)Number of Header Messages
Object Reference Count
Object Header Size
Header Message Type \#1Size of Header Message Data \#1
Header Message \#1 FlagsReserved (zero)

Header Message Data \#1

.
.
.
Header Message Type \#nSize of Header Message Data \#n
Header Message \#n FlagsReserved (zero)

Header Message Data \#n

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThis value is used to determine the format of the information in the object header. When the format of + the information in the object header is changed, the version number is incremented and can be used to + determine how the information in the object header is formatted. This document describes version one + (1) (there was no version zero (0)).
Number of Header MessagesThis value determines the number of messages listed in object headers for this object. This value + includes the messages in continuation messages for this object.
Object Reference CountThis value specifies the number of "hard links" to this object within the current file. References to the + object from external files, "soft links" in this file and object references in this file are not tracked.
Object Header SizeThis value specifies the total number of bytes of header message data following this length field that + contain object header messages for this object header. This value does not include the size of object header + continuation blocks for this object elsewhere in the file.
Header Message TypeThis value specifies the type of information included in the following header message data. The header + message types for the pre-defined header messages are included in sections below.
Size of Header Message DataThis value specifies the number of bytes of header message data following the header message type and + length information for the current message. The size includes padding bytes to make the message a multiple + of eight bytes.
Header Message FlagsThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
0If set, the message data is constant. This is used for messages like the datatype message of + a dataset.
1If set, the message is stored in the global heap. The Header Message Data field contains a + Shared Object message and the Size of Header Message Data field contains the size of that + Shared Object message.
2-7Reserved
+
Header Message DataThe format and length of this field is determined by the header message type and size respectively. + Some header message types do not require any data and this information can be eliminated by setting the + length of the message to zero. The data is padded with enough zeros to make the size a multiple of + eight.
+ +The header message types and the message data associated with them compose the critical "meta-data" about +each object. Some header messages are required for each object while others are optional. Some optional +header messages may also be repeated several times in the header itself, the requirements and number of +times allowed in the header will be noted in each header message description below. + +The following is a list of currently defined header messages: + +\subsubsection subsubsec_fmt11_dataobject_hdr_nil Name: NIL +Header Message Type: 0x0000 + +Length: varies + +Status: Optional, may be repeated. + +Purpose and Description: The NIL message is used to indicate a message which is to be ignored +when reading the header messages for a data object. [Possibly one which has been deleted for some reason.] + +Format of Data: Unspecified.
+ +\subsubsection subsubsec_fmt11_dataobject_hdr_simple Name: Simple Dataspace +Header Message Type: 0x0001 + +Length: Varies according to the number of dimensions, as described in the following table + +Status: Required for dataset objects, may not be repeated. + +Description:The simple dataspace message describes the number of dimensions (i.e. "rank") +and size of each dimension that the data object has. This message is only used for datasets which have a +simple, rectilinear grid layout; datasets requiring a more complex layout (irregularly structured or +unstructured grids, etc.) must use the Complex Dataspace message for expressing the space the +dataset inhabits. (Note: The Complex Dataspace functionality is not yet implemented (and it is +not described in this document.) + +Format of Data:
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Simple Dataspace Message
bytebytebytebyte
VersionDimensionalityFlagsReserved
Reserved
Dimension \#1 SizeL
.
.
.
Dimension \#n SizeL
Dimension \#1 Maximum SizeL
.
.
.
Dimension \#n Maximum SizeL
Permutation Index \#1L
.
.
.
Permutation Index \#nL
+ + + + + +
(Items marked with an 'L' the above table are of the size specified in "Size of Lengths.")
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Version This value is used to determine the format of the Simple Dataspace Message. When the format of the + information in the message is changed, the version number is incremented and can be used to determine + how the information in the object header is formatted. This document describes version one (1) (there + was no version zero (0)).
DimensionalityThis value is the number of dimensions that the data object has.
FlagsThis field is used to store flags to indicate the presence of parts of this message. Bit 0 (the least + significant bit) is used to indicate that maximum dimensions are present. Bit 1 is used to indicate + that permutation indices are present.
Dimension \#n SizeThis value is the current size of the dimension of the data as stored in the file. The first dimension + stored in the list of dimensions is the slowest changing dimension and the last dimension stored is the + fastest changing dimension.
Dimension \#n Maximum SizeThis value is the maximum size of the dimension of the data as stored in the file. This value may be + the special unlimited size which indicates that the data may expand along + this dimension indefinitely. If these values are not stored, the maximum size of each dimension is assumed + to be the dimension's current size.
Permutation Index \#nThis value is the index permutation used to map each dimension from the canonical representation to an + alternate axis for each dimension. If these values are not stored, the first dimension stored in the list + of dimensions is the slowest changing dimension and the last dimension stored is the fastest changing + dimension.
+ +\subsubsection subsubsec_fmt11_dataobject_hdr_message_0002 Name: Reserved - Not Assigned Yet +Header Message Type: 0x0002
+Length: N/A
+Status: N/A
+Format of Data: N/A
+Purpose and Description: This message type was skipped during the initial specification +of the file format and may be used in a future expansion to the format. + +\subsubsection subsubsec_fmt11_dataobject_hdr_dtmessage Name: Datatype +Header Message Type: 0x0003
+Length: variable
+Status: Required for dataset or named datatype objects, may not be repeated.
+ +Description: The datatype message defines the datatype for each element of a dataset. A +datatype can describe an atomic type like a fixed- or floating-point type or a compound type like a C +struct. Datatypes are stored as a list of datatype classes and their associated properties. + +Datatype messages that are part of a dataset object, do not describe how elements are related to one +another, the dataspace message is used for that purpose. Datatype messages that are part of a named +datatype message describe an "abstract" datatype that can be used by other objects in the file. + +Format of Data:
+ + + + + + + + + + + + + + + + + + + + +
Datatype Message
bytebytebytebyte
Type Class and VersionClass Bit Field, Bits 0-7Class Bit Field, Bits 8-15Class Bit Field, Bits 16-23
Size


Properties

/
+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Class and VersionThe version of the datatype message and the datatype's class information are packed together in + this field. The version number is packed in the top 4 bits of the field and the class is contained + in the bottom 4 bits.
+ The version number information is used for changes in the format of the datatype message and is + described here: + + + + + + + + + + + + + + + + + +
VersionDescription
0Never used
1Used by early versions of the library to encode compound datatypes with explicit array fields. + See the compound datatype description below for further details.
2The current version used by the library.

+ The class of the datatype determines the format for the class bit field and properties portion of the + datatype message, which are described below. The following classes are currently defined: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0Fixed-Point
1Floating-Point
2Time
3String
4Bitfield
5Opaque
6Compound
7Reference
8Enumerated
9Variable-Length
10Array
+
Class Bit FieldsThe information in these bit fields is specific to each datatype class and is described below. + All bits not defined for a datatype class are set to zero.
SizeThe size of the datatype in bytes.
PropertiesThis variable-sized field encodes information specific to each datatype class and is described + below. If there is no property information specified for a datatype class, the size of this field + is zero.
+ +Class specific information for Fixed-Point Numbers (Class 0): + + + + + + + + + + + + + + + + + + + + + + +
Bit Field Description
BitsMeaning
0Byte Order. If zero, byte order is little-endian; otherwise, byte order is big endian.
1, 2Padding type. Bit 1 is the lo_pad type and bit 2 is the hi_pad type. If a datum has + unused bits at either end, then the lo_pad or hi_pad bit is copied to those locations.
3Signed. If this bit is set then the fixed-point number is in 2's complement form.
4-23Reserved (zero).
+ + + + + + + + + + + + + +
+ Property Descriptions +
ByteByteByteByte
Bit OffsetBit Precision
+ + + + + + + + + + + + + + +
Field NameDescription
Bit OffsetThe bit offset of the first significant bit of the fixed-point value within the datatype. The + bit offset specifies the number of bits "to the right of" the value.
Bit PrecisionThe number of bits of precision of the fixed-point value within the datatype.
+ +Class specific information for Floating-Point Numbers (Class 1): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Bit Field Description
BitsMeaning
0Byte Order. If zero, byte order is little-endian; otherwise, byte order is big endian.
1, 2, 3Padding type. Bit 1 is the low bits pad type, bit 2 is the high bits pad type, and bit + 3 is the internal bits pad type. If a datum has unused bits at either end or between the sign bit, exponent, + or mantissa, then the value of bit 1, 2, or 3 is copied to those locations.
4-5Normalization. The value can be 0 if there is no normalization, 1 if the most significant + bit of the mantissa is always set (except for 0.0), and 2 if the most significant bit of the mantissa is not + stored but is implied to be set. The value 3 is reserved and will not appear in this field.
6-7Reserved (zero).
8-15Sign Location. This is the bit position of the sign bit. Bits are numbered with the least + significant bit zero.
16-23Reserved (zero).
+ + + + + + + + + + + + + + + + + + + + + + +
Property Descriptions
ByteByteByteByte
Bit OffsetBit Precision
Exponent LocationExponent SizeMantissa LocationMantissa Size
Exponent Bias
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameProperty Description
Bit OffsetThe bit offset of the first significant bit of the floating-point value within the datatype. The + bit offset specifies the number of bits "to the right of" the value.
Bit PrecisionThe number of bits of precision of the floating-point value within the datatype.
Exponent LocationThe bit position of the exponent field. Bits are numbered with the least significant + bit number zero.
Exponent SizeThe size of the exponent field in bits.
Mantissa LocationThe bit position of the mantissa field. Bits are numbered with the least significant bit number + zero.
Mantissa SizeThe size of the mantissa field in bits.
Exponent BiasThe bias of the exponent field.
+ +Class specific information for Time (Class 2): + + + + + + + + + + + + + + +
Bit Field Description
BitsMeaning
0Byte Order. If zero, byte order is little-endian; otherwise, byte order is big endian.
1-23Reserved (zero).
+ + + + + + + + + + +
Property Descriptions
ByteByte
Bit Precision
+ + + + + + + + + + +
Field NameDescription
Bit PrecisionThe number of bits of precision of the time value.
+ +Class specific information for Strings (Class 3): + + + + + + + + + + + + + + + + + +
Bit Field Description
BitsMeaning
0-3Padding type. This four-bit value determines the type of padding to use for the + string. The values are: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0Null Terminate: A zero byte marks the end of the string and is guaranteed to be present after + converting a long string to a short string. When converting a short string to a long string the + value is padded with additional null characters as necessary.
1Null Pad: Null characters are added to the end of the value during conversions from short values + to long values but conversion in the opposite direction simply truncates the value.
2Space Pad: Space characters are added to the end of the value during conversions from short values + to long values but conversion in the opposite direction simply truncates the value. This is the + Fortran representation of the string.
3-15Reserved.
+
4-7Character Set. The character set to use for encoding the string. The only character set + supported is the 8-bit ASCII (zero) so no translations have been defined yet.
8-23Reserved (zero).
+ +There are no properties defined for the string class. + +Class specific information for Bitfields (Class 4): + + + + + + + + + + + + + + + + + + +
Bit Field Description
BitsMeaning
0Byte Order. If zero, byte order is little-endian; otherwise, byte order is big endian.
1, 2Padding type. Bit 1 is the lo_pad type and bit 2 is the hi_pad type. If a datum has + unused bits at either end, then the lo_pad or hi_pad bit is copied to those locations.
3-23Reserved (zero).
+ + + + + + + + + + + + + +
Property Description
ByteByteByteByte
Bit OffsetBit Precision
+ + + + + + + + + + + + + + +
Field NameDescription
Bit OffsetThe bit offset of the first significant bit of the bitfield within the datatype. The bit + offset specifies the number of bits "to the right of" the value.
Bit PrecisionThe number of bits of precision of the bitfield within the datatype.
+ +Class specific information for Opaque (Class 5): + + + + + + + + + + + + + + +
Bit Field Description
BitsMeaning
0-7Length of ASCII tag in bytes.
8-23Reserved (zero).
+ + + + + + + + + + + + +
Property Description
ByteByteByteByte

ASCII Tag
+ + + + + + + + + + +
Field NameDescription
ASCII TagThis NUL-terminated string provides a description for the opaque type. It is NUL-padded to a + multiple of 8 bytes.
+ +Class specific information for Compound Types (Class 6): + + + + + + + + + + + + + + +
Bit Field Description
BitsMeaning
0-15Number of Members. This field contains the number of members defined for the + compound datatype. The member definitions are listed in the Properties field of the data type + message.
15-23Reserved (zero).
+ +The Properties field of a compound datatype is a list of the member definitions of the compound datatype. +The member definitions appear one after another with no intervening bytes. The member types are described +with a recursive datatype message. + +Note that the property descriptions are different for different versions of the datatype version. Additionally +note that the version 0 properties are deprecated and have been replaced with the version 1 properties in +versions of the HDF5 library from the 1.4 release onward. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Properties Description for Datatype Version 1
ByteByteByteByte


Name

/
Byte Offset of Member
DimensionalityReserved (zero)
Dimension Permutation
Reserved (zero)
Dimension \#1 Size (required)
Dimension \#2 Size (required)
Dimension \#3 Size (required)
Dimension \#4 Size (required)

Member Type Message

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
NameThis NUL-terminated string provides a description for the opaque type. It is NUL-padded to a + multiple of 8 bytes.
Byte Offset of MemberThis is the byte offset of the member within the datatype.
DimensionalityIf set to zero, this field indicates a scalar member. If set to a value greater than zero, + this field indicates that the member is an array of values. For array members, the size of + the array is indicated by the 'Size of Dimension n' field in this message.
Dimension PermutationThis field was intended to allow an array field to have it's dimensions permuted, but this was + never implemented. This field should always be set to zero.
Dimension \#n SizeThis field is the size of a dimension of the array field as stored in the file. The first + dimension stored in the list of dimensions is the slowest changing dimension and the last + dimension stored is the fastest changing dimension.
Member Type MessageThis field is a datatype message describing the datatype of the member.
+ + + + + + + + + + + + + + + + + + +
Properties Description for Datatype Version 2
ByteByteByteByte

Name

Byte Offset of Member

Member Type Message

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
NameThis NUL-terminated string provides a description for the opaque type. It is NUL-padded to a + multiple of 8 bytes.
Byte Offset of MemberThis is the byte offset of the member within the datatype.
Member Type MessageThis field is a datatype message describing the datatype of + the member.
+ +Class specific information for Reference (Class 7): + + + + + + + + + + + + + + +
Bit Field Description
BitsMeaning
0-3Type. This four-bit value contains the type of reference described. The values defined are: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0Object Reference: A reference to another object in this HDF5 file.
1Dataset Region Reference: A reference to a region within a dataset in this HDF5 file.
2Internal Reference: A reference to a region within the current dataset. (Not currently implemented)
3-15Reserved
+
15-23Reserved (zero).
+ +There are no properties defined for the reference class. + +Class specific information for Enumeration (Class 8): + + + + + + + + + + + + + + +
Bit Field Description
BitsMeaning
0-15Number of Members. The number of name/value pairs defined for the enumeration type.
16-23Reserved (zero).
+ + + + + + + + + + + + + + + + + + +
Property Description
ByteByteByteByte

Base Type
/

Names


Values

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
Base Type:Each enumeration type is based on some parent type, usually an integer. The information + for that parent type is described recursively by this field.
Names:The name for each name/value pair. Each name is stored as a null terminated ASCII string + in a multiple of eight bytes. The names are in no particular order.
Values:The list of values in the same order as the names. The values are packed (no inter-value + padding) and the size of each value is determined by the parent type.
+ +Class specific information for Variable-Length (Class 9): + + + + + + + + + + + + + + + + + + + + + + +
Bit Field Description
BitsMeaning
0-3Type. This four-bit value contains the type of variable-length datatype described. + The values defined are: + + + + + + + + + + + + + + + + + +
ValueDescription
0Sequence: A variable-length sequence of any sequence of data. Variable-length sequences do not + have padding or character set information.
1String: A variable-length sequence of characters. Variable-length strings have padding and + character set information.
2-15Reserved
4-7Padding type. (variable-length string only). This four-bit value determines the + type of padding used for variable-length strings. The values are the same as for the string padding + type, as follows: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
00 Null terminate: A zero byte marks the end of a string and is guaranteed to be present after + converting a long string to a short string. When converting a short string to a long string, + the value is padded with additional null characters as necessary.
1Null pad: Null characters are added to the end of the value during conversion from a short string to + a longer string. Conversion from a long string to a shorter string simply truncates the value.
2Space pad: Space characters are added to the end of the value during conversion from a short string + to a longer string. Conversion from a long string to a shorter string simply truncates the value. + This is the Fortran representation of the string.
3-15Reserved
+ This value is set to zero for variable-length sequences.
8-11Character Set. (variable-length string only) This four-bit value specifies the + character set to be used for encoding the string: + + + + + + + + + + + + + +
ValueDescription
0ASCII: As of this writing (July 2003, Release 1.6), 8-bit ASCII is the only character set supported. + Therefore, no translations have been defined.
1-15Reserved
+ This value is set to zero for variable-length sequences.
12-23Reserved (zero).
+ + + + + + + + + + + + +
Property Description
ByteByteByteByte

Base Type

+ + + + + + + + + + +
Field NameDescription
Base TypeEach variable-length type is based on some parent type. The information for that parent + type is described recursively by this field.
+ +Class specific information for Array (Class 10): + +There are no bit fields defined for the array class. + +Note that the dimension information defined in the property for this datatype class is independent of +dataspace information for a dataset. The dimension information here describes the dimensionality of the +information within a data element (or a component of an element, if the array datatype is nested within +another datatype) and the dataspace for a dataset describes the location of the elements in a dataset. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Property Description
ByteByteByteByte
DimensionalityReserved (zero)
Dimension \#1 Size
.
.
.
Dimension \#n Size
Permutation Index \#1
.
.
.
Permutation Index \#n

Base Type

+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
DimensionalityThis value is the number of dimensions that the array has.
Dimension \#n SizeThis value is the size of the dimension of the array as stored in the file. The first dimension + stored in the list of dimensions is the slowest changing dimension and the last dimension stored + is the fastest changing dimension.
Permutation Index \#nThis value is the index permutation used to map each dimension from the canonical representation + to an alternate axis for each dimension. Currently, dimension permutations are not supported and + these indices should be set to the index position minus one (i.e. the first dimension should be + set to 0, the second dimension should be set to 1, etc.)
Base TypeEach array type is based on some parent type. The information for that parent type is described + recursively by this field.
+ +\subsubsection subsubsec_fmt11_dataobject_hdr_ofvmessage Name: Data Storage - Fill Value (Old) +Header Message Type: 0x0004
+Length: varies
+Status: Optional, may not be repeated.
+Description: The fill value message stores a single data point value which is returned +to the application when an uninitialized data point is read from the dataset. The fill value is interpreted +with the same datatype as the dataset. If no fill value message is present then a fill value of all zero is assumed. + +This fill value message is deprecated in favor of the "new" fill value message (Message Type 0x0005) and +is only written to the file for forward compatibility with versions of the HDF5 library before the 1.6.0 +version. Additionally, it only appears for datasets with a user defined fill value (as opposed to the library +default fill value or an explicitly set "undefined" fill value). + +Format of Data: + + + + + + + + + + + + + + +
Fill Value Message (Old)
bytebytebytebyte
Size (4 bytes)

Fill Value

+ + + + + + + + + + + + + + +
Field NameDescription
SizeThis is the size of the Fill Value field in bytes.
Fill ValueThe fill value. The bytes of the fill value are interpreted using the same datatype as for the dataset.
+ +\subsubsection subsubsec_fmt11_dataobject_hdr_fvmessage Name: Data Storage - Fill Value +Header Message Type: 0x0005
+Length: varies
+Status: Required for dataset objects, may not be repeated.
+Description: The fill value message stores a single data value which is returned to the +application when an uninitialized data element is read from a dataset. The fill value is interpreted +with the same datatype as the dataset. + +Format of Data: + + + + + + + + + + + + + + + + + + + + +
Fill Value Message
bytebytebytebyte
VersionSpace Allocation TimeFill Value Write TimeFill Value Defined
Size

Fill Value

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number information is used for changes in the format of the fill value message and + is described here: + + + + + + + + + + + + + + + + + +
VersionDescription
0Never used
1Used by version 1.6.x of the library to encode fill values. In this version, the Size field + is always present. +
2The current version used by the library (version 1.7.3 or later). In this version, the Size + and Fill Value fields are only present if the Fill Value Defined field is set to 1.
+
Space Allocation TimeWhen the storage space for the dataset's raw data will be allocated. The allowed values are: + + + + + + + + + + + + + + + + + +
ValueDescription
1Early allocation. Storage space for the entire dataset should be allocated in the file + when the dataset is created.
2Late allocation. Storage space for the entire dataset should not be allocated until the + dataset is written to.
3Incremental allocation. Storage space for the dataset should not be allocated until the + portion of the dataset is written to. This is currently used in conjunction with chunked + data storage for datasets.
Fill Value Write TimeAt the time that storage space for the dataset's raw data is allocated, this value indicates + whether the fill value should be written to the raw data storage elements. The allowed values are: + + + + + + + + + + + + + + + + + +
ValueDescription
0On allocation. The fill value is always written to the raw data storage when the storage + space is allocated.
1Never. The fill value should never be written to the raw data storage.
2Fill value written if set by user. The fill value will be written to the raw data storage + when the storage space is allocated only if the user explicitly set the fill value. If the + fill value is the library default or is undefined, it will not be written to the raw data storage.
Fill Value DefinedThis value indicates if a fill value is defined for this dataset. If this value is 0, the fill + value is undefined. If this value is 1, a fill value is defined for this dataset. For version 2 + or later of the fill value message, this value controls the presence of the Size field.
SizeThis is the size of the Fill Value field in bytes. This field is not present if the Version + field is >1 and the Fill Value Defined field is set to 0.
Fill ValueThe fill value. The bytes of the fill value are interpreted using the same datatype as for the + dataset. This field is not present if the Version field is >1 and the Fill Value Defined field + is set to 0.
+ +\subsubsection subsubsec_fmt11_dataobject_hdr_message_0006 Name: Reserved - Not Assigned Yet +Header Message Type: 0x0006
+Length: N/A
+Status: N/A
+Purpose and Description: N/A
+Format of Data: N/A
+Purpose and Description: This message type was skipped during the initial + specification of the file format and may be used in a future expansion to the format. + +\subsubsection subsubsec_fmt11_dataobject_hdr_external Name: Data Storage - External Data Files +Header Message Type: 0x0007
+Length: varies
+Status: Optional, may not be repeated.
+Purpose and Description: The external object message indicates that the data for an +object is stored outside the HDF5 file. The filename of the object is stored as a Universal Resource Location +(URL) of the actual filename containing the data. An external file list record also contains the byte offset +of the start of the data within the file and the amount of space reserved in the file for that data. + + + + + + + + + + + + + + + + + + + + + + + +
External File List Message
bytebytebytebyte
VersionReserved
Allocated SlotsUsed Slots

Heap Address


Slot Definitions...

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number information is used for changes in the format of External File List Message + and is described here: + + + + + + + + + + + + + +
VersionDescription
0Never used.
1The current version used by the library.
ReservedThis field is reserved for future use.
Allocated SlotsThe total number of slots allocated in the message. Its value must be at least as large as the value + contained in the Used Slots field. (The current library simply uses the number of Used Slots for this + message)
Used SlotsThe number of initial slots which contain valid information.
Heap AddressThis is the address of a local name heap which contains the names for the external files. (The local + heap information can be found in Disk Format Level 1D in this document). The name at offset zero in + the heap is always the empty string.
Slot DefinitionsThe slot definitions are stored in order according to the array addresses they represent.
+ + + + + + + + + + + + + + + + + + +
External File List Slot
bytebytebytebyte

Name Offset (<size> bytes)


File Offset (<size> bytes)


Size

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
Name Offset (<size> bytes)The byte offset within the local name heap for the name of the file. File names are stored as a URL + which has a protocol name, a host name, a port number, and a file name: + protocol:port//host/file. If the protocol is omitted + then "file:" is assumed. If the port number is omitted then a default port for that protocol is used. + If both the protocol and the port number are omitted then the colon can also be omitted. If the double + slash and host name are omitted then "localhost" is assumed. The file name is the only mandatory part, + and if the leading slash is missing then it is relative to the application's current working directory + (the use of relative names is not recommended).
File Offset (<size> bytes)This is the byte offset to the start of the data in the specified file. For files that contain data for + a single dataset this will usually be zero.
SizeThis is the total number of bytes reserved in the specified file for raw data storage. For a file that + contains exactly one complete dataset which is not extendable, the size will usually be the exact size of + the dataset. However, by making the size larger one allows HDF5 to extend the dataset. The size can be set + to a value larger than the entire file since HDF5 will read zeros past the end of the file without failing.
+ +\subsubsection subsubsec_fmt11_dataobject_hdr_layout Name: Data Storage - Layout +Header Message Type: 0x0008
+Length: varies
+Status: Required for datasets, may not be repeated. + +Purpose and Description: Data layout describes how the elements of a multi-dimensional array +are arranged in the linear address space of the file. Three types of data layout are supported: +
    +
  1. Contiguous: The array can be stored in one contiguous area of the file. The layout requires that the + size of the array be constant and does not permit chunking, compression, checksums, encryption, etc. + The message stores the total size of the array and the offset of an element from the beginning of the + storage area is computed as in C.
    2. +
  2. Chunked: The array domain can be regularly decomposed into chunks and each chunk is allocated separately. + This layout supports arbitrary element traversals, compression, encryption, and checksums, and the chunks + can be distributed across external raw data files (these features are described in other messages). + The message stores the size of a chunk instead of the size of the entire array; the size of the entire + array can be calculated by traversing the B-tree that stores the chunk addresses.
    3. +
  3. Compact: The array can be stored in one contiguous block, as part of this object header message (this is + called "compact" storage below).
    4. +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Data Layout Message (Versions 1 and 2)
bytebytebytebyte
VersionDimensionalityLayout ClassReserved
Reserved

Address

Dimension 0 (4-bytes)
Dimension 1 (4-bytes)
...
Dataset Element Size (optional)
Compact Data Size (4-bytes)

Compact Data...

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number information is used for changes in the format of the data layout message and + is described here: + + + + + + + + + + + + + + + + + +
VersionDescription
0Never used.
1Used by version 1.4 and before of the library to encode layout information. Data space is + always allocated when the data set is created.
2Used by version 1.6.x of the library to encode layout information. Data space is allocated + only when it is necessary.
DimensionalityAn array has a fixed dimensionality. This field specifies the number of dimension size fields later + in the message.
Layout ClassThe layout class specifies how the other fields of the layout message are to be interpreted. A value + of one indicates contiguous storage, a value of two indicates chunked storage, while a value of zero + indicates compact storage. Other values will be defined in the future.
AddressFor contiguous storage, this is the address of the first byte of storage. For chunked storage this is + the address of the B-tree that is used to look up the addresses of the chunks. This field is not present + for compact storage. If the version for this message is set to 2, the address may have the "undefined + address" value, to indicate that storage has not yet been allocated for this array.
DimensionsFor contiguous and compact storage the dimensions define the entire size of the array while for chunked storage + they define the size of a single chunk. In all cases, they are in units of array elements (not bytes). The + first dimension stored in the list of dimensions is the slowest changing dimension and the last dimension + stored is the fastest changing dimension.
Dataset Element SizeThe size of a dataset element, in bytes. This field is only present for chunked storage.
Compact Data SizeThis field is only present for compact data storage. It contains the size of the raw data for the + dataset array.
Compact DataThis field is only present for compact data storage. It contains the raw data for the dataset + array.
+ +Version 3 of this message re-structured the format into specific properties that are required for each layout class. + + + + + + + + + + + + + + + + +
Data Layout Message (Version 3)
bytebytebytebyte
VersionLayout Class 

Properties

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number information is used for changes in the format of layout message and is + described here:

+ + + + + + + + + +
VersionDescription
3Used by the version 1.6.3 and later of the library to store properties for each layout class.
Layout ClassThe layout class specifies how the other fields of the layout message are to be interpreted. A value of one + indicates contiguous storage, a value of two indicates chunked storage, while a value of zero indicates + compact storage.
PropertiesThis variable-sized field encodes information specific to each layout class and is described below. If + there is no property information specified for a layout class, the size of this field is zero bytes.
+ +Class-specific information for compact layout (Class 0): (Note: The dimensionality information is in +the Dataspace message) + + + + + + + + + + + + + + + + +
Property Descriptions
bytebytebytebyte
Size 

Raw Data...

+ + + + + + + + + + + + + + +
Field NameDescription
SizeThis field contains the size of the raw data for the dataset array.
Raw DataThis field contains the raw data for the dataset array.
+ +Class-specific information for contiguous layout (Class 1): (Note: The dimensionality information is +in the Dataspace message) + + + + + + + + + + + + + + + +
Property Descriptions
bytebytebytebyte

Address


Size

+ + + + + + + + + + + + + + +
Field NameDescription
AddressThis is the address of the first byte of raw data storage. The address may have the "undefined + address" value, to indicate that storage has not yet been allocated for this array.
SizeThis field contains the size allocated to store the raw data.
+ +Class-specific information for chunked layout (Class 2): + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Property Descriptions
bytebytebytebyte
Dimensionality 

Address

Dimension 0 (4-bytes)
Dimension 1 (4-bytes)
...
Dataset Element Size
+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Dimensionality>A chunk has a fixed dimensionality. This field specifies the number of dimension size fields + later in the message.
AddressThis is the address of the B-tree that is used to look up the addresses of the chunks. The + address may have the undefined address value, to indicate + that storage has not yet been allocated for this array.
DimensionsThese values define the dimension size of a single chunk, in units of array elements (not bytes). + The first dimension stored in the list of dimensions is the slowest changing dimension and the + last dimension stored is the fastest changing dimension.
Dataset Element SizeThe size of a dataset element, in bytes.
+ +\subsubsection subsubsec_fmt11_dataobject_hdr_message_0009 Name: Reserved - Not Assigned Yet +Header Message Type: 0x0009
+Length: N/A
+Status: N/A
+Format of Data: N/A
+Purpose and Description: This message type was skipped during the initial +specification of the file format and may be used in a future expansion to the format. + +\subsubsection subsubsec_fmt11_dataobject_hdr_message_000A Name: Reserved - Not Assigned Yet +Header Message Type: 0x000A
+Length: N/A
+Status: N/A
+Format of Data: N/A +Purpose and Description: This message type was skipped during the initial +specification of the file format and may be used in a future expansion to the format. + +\subsubsection subsubsec_fmt11_dataobject_hdr_filter Name: Data Storage - Filter Pipeline +Header Message Type: 0x000B
+Length: varies
+Status: Optional, may not be repeated.
+Purpose and Description: This message describes the filter pipeline which should be +applied to the data stream by providing filter identification numbers, flags, a name, an client data. + +Format of Data: + + + + + + + + + + + + + + + + + + + +
Filter Pipeline Message
bytebytebytebyte
VersionNumber of FiltersReserved
Reserved

Filter List

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number for this message. This document describes version 1.
Number of FiltersThe total number of filters described by this message. The maximum possible number of filters in a + message is 32.
Filter ListA description of each filter. A filter description appears in the next table.
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Filter Description
bytebytebytebyte
Filter IdentificationName Length
FlagsNumber of Values for Client Data

Name


Client Data

Padding
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Filter IdentificationThis value, often referred to as a filter identifier, is designed to be a unique identifier for + the filter. Values from zero through 32,767 are reserved for filters supported by The HDF Group + in the HDF5 library and for filters requested and supported by third parties. Filters supported + by The HDF Group are documented immediately below. Information on 3rd-party filters can be found + at + https://github.com/HDFGroup/hdf5_plugins/blob/master/docs/RegisteredFilterPlugins.md. + 1
To request a filter identifier, + please contact The HDF Group’s Help Desk at HDF Help Desk. + You will be asked to provide the following information: +
    +
  1. Contact information for the developer requesting the new identifier +
  2. A short description of the new filter +
  3. Links to any relevant information, including licensing information +

+ Values from 32768 to 65535 are reserved for non-distributed uses (for example, internal company usage) + or for application usage when testing a feature. The HDF Group does not track or document the use of + the filters with identifiers from this range.
+ The filters currently in library version 1.6.5 are listed below: + + + + + + + + + + + + + + + + + + + + + + + + + + +
IdentificationNameDescription
1deflateGZIP deflate compression
2shuffleData element shuffling
3fletcher32Fletcher32 checksum
4szipSZIP compression
Name LengthEach filter has an optional null-terminated ASCII name and this field holds the length of the name + including the null termination padded with nulls to be a multiple of eight. If the filter has no name + then a value of zero is stored in this field.
FlagsThe flags indicate certain properties for a filter. The bit values defined so far are: + + + + + + + + + +
ValueDescription
bit 1If set then the filter is an optional filter. During output, if an optional filter fails it will be + silently removed from the pipeline.
Client Data Number of ValuesEach filter can store a few integer values to control how the filter operates. The number of entries + in the Client Data array is stored in this field.
NameIf the Name Length field is non-zero then it will contain the size of this field, a multiple of eight. + This field contains a null-terminated, ASCII character string to serve as a comment/name for the filter.
Client DataThis is an array of four-byte integers which will be passed to the filter function. The Client Data + Number of Values determines the number of elements in the array.
PaddingFour bytes of zeros are added to the message at this point if the Client Data Number of Values field + contains an odd number.
+\anchor FMT11Footnote1Change 1 If you are reading an earlier version of this document, this +link may have changed. If the link does not work, use the latest version of this document on +The HDF Group’s github website, +HDF5 File Format Specification; the link there will always be correct. + +\subsubsection subsubsec_fmt11_dataobject_hdr_attribute Name: Attribute +Header Message Type: 0x000C
+Length: varies
+Status: Optional, may be repeated.
+Description: The Attribute message is used to list objects in the HDF +file which are used as attributes, or "meta-data" about the current object. An attribute is a small dataset; +it has a name, a datatype, a data space, and raw data. Since attributes are stored in the object header they +must be relatively small (<64KB) and can be associated with any type of object which has an object header +(groups, datasets, named types and spaces, etc.). + +Note: Attributes on an object must have unique names. (The HDF5 library currently enforces this by causing +the creation of an attribute with a duplicate name to fail). Attributes on different objects may have the +same name, however. + +Format of Data: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Attribute Message
bytebytebytebyte
VersionReservedName Size
Datatype SizeDataspace Size

Name


Datatype


Dataspace


Data

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number information is used for changes in the format of the attribute message and is + described here: + + + + + + + + + + + + + +
VersionDescription
0Never used.
1Used by the library before version 1.6 to encode attribute message. This version does not + support shared data type.
ReservedThis field is reserved for later use and is set to zero.
Name SizeThe length of the attribute name in bytes including the null terminator. Note that the Name field + below may contain additional padding not represented by this field.
Datatype SizeThe length of the datatype description in the Datatype field below. Note that the Datatype field may contain + additional padding not represented by this field.
Dataspace SizeThe length of the dataspace description in the Dataspace field below. Note that the Dataspace field may contain + additional padding not represented by this field.
NameThe null-terminated attribute name. This field is padded with additional null characters to make it a + multiple of eight bytes.
TypeThe datatype description follows the same format as described for the datatype object header message. + This field is padded with additional zero bytes to make it a multiple of eight bytes.
SpaceThe dataspace description follows the same format as described for the dataspace object header message. + This field is padded with additional zero bytes to make it a multiple of eight bytes.
DataThe raw data for the attribute. The size is determined from the datatype and dataspace descriptions. + This field is not padded with additional bytes.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Attribute Message (Version 2)
bytebytebytebyte
VersionFlagName Size
Type SizeSpace Size

Name


Type


Space


Data

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number information is used for changes in the format of the attribute message and is + described here: + + + + + + + + + +
VersionDescription
2Used by the library of version 1.6.x and after to encode attribute message. This version + supports shared data type. The fields of name, type, and space are not padded with additional + bytes of zero.
Flag>This field indicates whether the data type of this attribute is shared: + + + + + + + + + + + + + +
ValueDescription
0Datatype is not shared.
1Datatype is shared.
Name Size>The length of the attribute name in bytes including the null terminator.
Datatype SizeThe length of the datatype description in the Datatype field below.
Dataspace SizeThe length of the dataspace description in the Dataspace field below.
NameThe null-terminated attribute name. This field is not padded with additional bytes.
DatatypeThe datatype description follows the same format as described for the datatype object + header message. This field is not padded with additional bytes.
DataspaceThe dataspace description follows the same format as described for the dataspace object + header message. This field is not padded with additional bytes.
DataThe raw data for the attribute. The size is determined from the datatype and dataspace + descriptions. This field is not padded with additional zero bytes.
+ +\subsubsection subsubsec_fmt11_dataobject_hdr_comment Name: Object Comment +Header Message Type: 0x000D
+Length: varies
+Status: Optional, may not be repeated.
+Description: The object comment is designed to be a short description of an +object. An object comment is a sequence of non-zero (\0) ASCII characters with +no other formatting included by the library.
+Format of Data: + + + + + + + + + + + +
Name Message
bytebytebytebyte

Comment

+
+ + + + + + + + + +
Field NameDescription
NameA null terminated ASCII character string.
+ +\subsubsection subsubsec_fmt11_dataobject_hdr_modified Name: Object Modification Date & Time (Old) +Header Message Type: 0x000E
+Length: fixed
+Status: Optional, may not be repeated.
+Description: The object modification date and time is a timestamp which +indicates (using ISO-8601 date and time format) the last modification of an object. The time is +updated when any object header message changes according to the system clock where the change was posted. +
+This modification time message is deprecated in favor of the "new" modification time message +(Message Type 0x0012) and is no longer written to the file in versions of the HDF5 library after +the 1.6.0 version.
+Format of Data: + + + + + + + + + + + + + + + + + + + + + + + +
Modification Time Message
bytebytebytebyte
Year
MonthDay of Month
HourMinute
SecondReserved
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
YearThe four-digit year as an ASCII string. For example, 1998. All fields of this message + should be interpreted as coordinated universal time (UTC)
MonthThe month number as a two digit ASCII string where January is 01 and December is + 12.
Day of MonthThe day number within the month as a two digit ASCII string. The first day of the month is + 01.
HourThe hour of the day as a two digit ASCII string where midnight is 00 and 11:00pm + is 23.
MinuteThe minute of the hour as a two digit ASCII string where the first minute of the hour is + 00 and the last is 59.
SecondThe second of the minute as a two digit ASCII string where the first second of the minute is + 00 and the last is 59.
ReservedThis field is reserved and should always be zero.
+ +\subsubsection subsubsec_fmt11_dataobject_hdr_shared Name: Shared Object Message +Header Message Type: 0x000F
+Length: 4 Bytes
+Status: Optional, may be repeated.
+Description: A constant message can be shared among several object headers. A +Shared Object Message contains the address of the object message to be shared. Care must +be exercised to prevent cycles when a message of one object header points to a message in some other +object header. Starting from Version 2 of the Shared Object Message, the Flags field becomes unused. + +Format of Data: + + + + + + + + + + + + + + + +
Shared Object Message (Version 1)
byte + byte + byte + byte +
VersionFlagsReserved
Reserved

Pointer

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number is used when there are changes in the format of a shared object message and + is described here: + + + + + + + + + + + + + +
VersionDescription
0Never used.
1Used by the library before version 1.6.1. In this version, the Flags field is used to + indicate whether the actual message is stored in the global heap (never implemented). + The Pointer field either contains the header message address in the global heap + (never implemented) or the address of the shared object header.
FlagsThe Shared Message message points to a message which is shared among multiple object headers. The + Flags field describes the type of sharing: + + + + + + + + + + + + + +
BitDescription
0If this bit is clear then the actual message is the first message in some other object header; + otherwise the actual message is stored in the global heap. (never implemented).
2-7Reserved (always zero)
PointerThe address of the object header containing the message to be shared.
+ + + + + + + + + + + + + +
Shared Object Message (Version 2)
byte + byte + byte + byte +
VersionFlags 

Pointer

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number is used when there are changes in the format of a shared object message and + is described here: + + + + + + + + + +
VersionDescription
2Used by the library of version 1.6.1 and after. In this version, The Flags field is not used + and the Pointer field contains the address of the object header containing the message to be + shared.
FlagsUnused.
PointerThe address of the object header containing the message to be shared.
+ +\subsubsection subsubsec_fmt11_dataobject_hdr_continuation Name: Object Header Continuation +Header Message Type: 0x0010
+Length: fixed
+Status: Optional, may be repeated.
+Description: The object header continuation is the location in the file of more +header messages for the current data object. This can be used when header blocks become too large, +or are likely to change over time.
+Format of Data: + + + + + + + + + + + + + + +
Object Header Continuation Message
bytebytebytebyte
Offset
Length
+ + + + + + + + + + + + + + +
Field NameDescription
OffsetThis value is the offset in bytes from the beginning of the file where the header continuation + information is located.
LengthThis value is the length in bytes of the header continuation information in the file.
+ +\subsubsection subsubsec_fmt11_dataobject_hdr_stmgroup Name: Group Message +Header Message Type: 0x0011
+Length: fixed
+Status: Required for groups, may not be repeated.
+Description: Each group has a B-tree and a name heap which are pointed to +by this message.
+Format of data: + + + + + + + + + + + + + + +
HGroup Message Layout
bytebytebytebyte

B-tree Address


Heap Address

+ + + + + + + + + + + + + + +
Field NameDescription
B-tree AddressThis value is the offset in bytes from the beginning of the file where the B-tree is located.
Heap AddressThis value is the offset in bytes from the beginning of the file where the group name heap + is located.
+ +\subsubsection subsubsec_fmt11_dataobject_hdr_mod Name: Object Modification Date and Time +Header Message Type: 0x0012
+Length: fixed
+Status: Optional, may not be repeated.
+Description: The object modification date and time is a timestamp which indicates +the last modification of an object. The time is updated when any object header message changes according +to the system clock where the change was posted.
+Format of data: + + + + + + + + + + + + + + + +
Modification Time Message
bytebytebytebyte
VersionReserved
Seconds After Epoch
+ + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number is used for changes in the format of Object Modification Time and is described + here: + + + + + + + + + + + + + +
VersionDescription
0Never used.
1Used by Version 1.6.1 and after of the library to encode time. In this version, the time is + the seconds after Epoch.
ReservedThis field is reserved and should always be zero.
Seconds After EpochThe number of seconds since 0 hours, 0 minutes, 0 seconds, January 1, 1970, Coordinated + Universal Time.
+ +\subsection subsec_fmt11_dataobject_storage Disk Format: Level 2b - Data Object Data Storage +The data for an object is stored separately from the header information in the file and may not actually +be located in the HDF5 file itself if the header indicates that the data is stored externally. The +information for each record in the object is stored according to the dimensionality of the object +(indicated in the dimensionality header message). Multi-dimensional data is stored in C order [same +as current scheme], i.e. the "last" dimension changes fastest. + +Data whose elements are composed of simple number-types are stored in native-endian IEEE format, unless +they are specifically defined as being stored in a different machine format with the architecture-type +information from the number-type header message. This means that each architecture will need to +[potentially] byte-swap data values into the internal representation for that particular machine. + +Data with a variable-length datatype is stored in the global heap of the HDF5 file. Global heap +identifiers are stored in the data object storage. + +Data whose elements are composed of pointer number-types are stored in several different ways depending +on the particular pointer type involved. Simple pointers are just stored as the dataset offset of the +object being pointed to with the size of the pointer being the same number of bytes as offsets in the file. +Dataset region references are stored as a heap-ID which points to the following information within the +file-heap: an offset of the object pointed to, number-type information (same format as header message), +dimensionality information (same format as header message), sub-set start and end information (i.e. a +coordinate location for each), and field start and end names (i.e. a [pointer to the] string indicating +the first field included and a [pointer to the] string name for the last field). + +Data of a compound datatype is stored as a contiguous stream of the items in the structure, with each +item formatted according to its datatype. + +\section sec_fmt11_appendix Appendix +Definitions of various terms used in this document. + +\anchor FMT11UndefinedAddress The "undefined address" for a file is a file address with all bits set, +i.e. 0xffff...ff. + +\anchor FMT11UnlimitedDim The "unlimited size" for a size is a value with all bits set, +i.e. 0xffff...ff. + +*/ diff --git a/doxygen/dox/H5.format.2.0.dox b/doxygen/dox/H5.format.2.0.dox new file mode 100644 index 00000000000..53d84df8885 --- /dev/null +++ b/doxygen/dox/H5.format.2.0.dox @@ -0,0 +1,9285 @@ + +/** \page FMT2 HDF5 File Format Specification Version 2.0 +
    +
  1. @ref sec_fmt2_intro +
      +
    1. @ref subsec_fmt2_intro_doc
      2. +
    2. @ref subsec_fmt2_intro_110
      3. +
    2. +
  2. @ref sec_fmt2_meta +
      +
    1. @ref subsec_fmt2_boot_super
      2. +
    2. @ref subsec_fmt2_boot_driver
      3. +
    3. @ref subsec_fmt2_boot_supext
      4. +
    3. +
  3. @ref sec_fmt2_infra +
      +
    1. @ref subsec_fmt2_infra_btrees +
        +
      1. @ref subsubsec_fmt2_infra_btrees_v1
        2. +
      2. @ref subsubsec_fmt2_infra_btrees_v2
        3. +
      2. +
    2. @ref subsec_fmt2_infra_symboltable
      3. +
    3. @ref subsec_fmt2_infra_symboltableentry
      4. +
    4. @ref subsec_fmt2_infra_localheap
      5. +
    5. @ref subsec_fmt2_infra_globalheap
      6. +
    6. @ref subsec_fmt2_infra_fractalheap
      7. +
    7. @ref subsec_fmt2_infra_freespaceindex
      8. +
    8. @ref subsec_fmt2_infra_sohm
      9. +
    4. +
  4. @ref sec_fmt2_dataobject +
      +
    1. @ref subsec_fmt2_dataobject_hdr +
        +
      1. @ref subsec_fmt2_dataobject_hdr_prefix
        2. +
          +
        1. @ref subsubsec_fmt2_dataobject_hdr_prefix_one
          2. +
        2. @ref subsubsec_fmt2_dataobject_hdr_prefix_two
          3. +
        +
      2. @ref subsec_fmt2_dataobject_hdr_msg
        3. +
          +
        1. @ref subsubsec_fmt2_dataobject_hdr_msg_nil
          2. +
        2. @ref subsubsec_fmt2_dataobject_hdr_msg_simple
          3. +
        3. @ref subsubsec_fmt2_dataobject_hdr_msg_linkinfo
          4. +
        4. @ref subsubsec_fmt2_dataobject_hdr_msg_dtmessage
          5. +
        5. @ref subsubsec_fmt2_dataobject_hdr_msg_ofvmessage
          6. +
        6. @ref subsubsec_fmt2_dataobject_hdr_msg_fvmessage
          7. +
        7. @ref subsubsec_fmt2_dataobject_hdr_msg_link
          8. +
        8. @ref subsubsec_fmt2_dataobject_hdr_msg_external
          9. +
        9. @ref subsubsec_fmt2_dataobject_hdr_msg_layout
          10. +
        10. @ref subsubsec_fmt2_dataobject_hdr_msg_bogus
          11. +
        11. @ref subsubsec_fmt2_dataobject_hdr_msg_groupinfo
          12. +
        12. @ref subsubsec_fmt2_dataobject_hdr_msg_filter
          13. +
        13. @ref subsubsec_fmt2_dataobject_hdr_msg_attribute
          14. +
        14. @ref subsubsec_fmt2_dataobject_hdr_msg_comment
          15. +
        15. @ref subsubsec_fmt2_dataobject_hdr_msg_omodified
          16. +
        16. @ref subsubsec_fmt2_dataobject_hdr_msg_shared
          17. +
        17. @ref subsubsec_fmt2_dataobject_hdr_msg_continuation
          18. +
        18. @ref subsubsec_fmt2_dataobject_hdr_msg_stmgroup
          19. +
        19. @ref subsubsec_fmt2_dataobject_hdr_msg_mod
          20. +
        20. @ref subsubsec_fmt2_dataobject_hdr_msg_btreek
          21. +
        21. @ref subsubsec_fmt2_dataobject_hdr_msg_drvinfo
          22. +
        22. @ref subsubsec_fmt2_dataobject_hdr_msg_attrinfo
          23. +
        23. @ref subsubsec_fmt2_dataobject_hdr_msg_refcount
          24. +
        24. @ref subsubsec_fmt2_dataobject_hdr_msg_fsinfo
          25. +
        +
      2. +
    2. @ref subsec_fmt2_dataobject_storage
      3. +
    +
    5. +
  5. @ref sec_fmt2_appendixa +
  6. @ref sec_fmt2_appendixb +
+ + + +\section sec_fmt2_intro I. Introduction + + + + + + + + + + + + + + +
Figure 1: Relationships among the HDF5 root group, other groups, and objects
\image html FF-IH_FileGroup.gif
Figure 2: HDF5 objects -- datasets, datatypes, or dataspaces
\image html FF-IH_FileObject.gif
+ +The format of an HDF5 file on disk encompasses several key ideas of the HDF4 and AIO file formats as well +as addressing some shortcomings therein. The new format is more self-describing than the HDF4 format and +is more uniformly applied to data objects in the file. + +An HDF5 file appears to the user as a directed graph. The nodes of this graph are the higher-level HDF5 +objects that are exposed by the HDF5 APIs: +\li Groups +\li Datasets +\li Committed (formerly Named) datatypes + +At the lowest level, as information is actually written to the disk, an HDF5 file is made up of the +following objects: +\li A superblock +\li B-tree nodes +\li Heap blocks +\li Object headers +\li Object data +\li Free space + +The HDF5 library uses these lower-level objects to represent the higher-level objects that are then +presented to the user or to applications through the APIs. For instance, a group is an object header that +contains a message that points to a local heap (for storing the links to objects in the group) and to a +B-tree (which indexes the links). A dataset is an object header that contains messages that describe +datatype, dataspace, layout, filters, external files, fill value, and other elements with the layout message +pointing to either a raw data chunk or to a B-tree that points to raw data chunks. + +\subsection subsec_fmt2_intro_doc I.A. This Document +This document describes the lower-level data objects; the higher-level objects and their properties are +described in the \ref UG. + +Three levels of information comprise the file format. Level 0 contains basic information for identifying +and defining information about the file. Level 1 information contains the information about the pieces of a +file shared by many objects in the file (such as a B-trees and heaps). Level 2 is the rest of the file and +contains all of the data objects, with each object partitioned into header information, also known as +metadata, and data. + +The sizes of various fields in the following layout tables are determined by looking at the number of +columns the field spans in the table. There are three exceptions: (1) The size may be overridden by +specifying a size in parentheses, (2) the size of addresses is determined by the Size of Offsets +field in the superblock and is indicated in this document with a superscripted ‘O’, and (3) +the size of length fields is determined by the Size of Lengths field in the superblock and is +indicated in this document with a superscripted ‘L’. + +Values for all fields in this document should be treated as unsigned integers, unless otherwise noted in +the description of a field. Additionally, all metadata fields are stored in little-endian byte order. + +All checksums used in the format are computed with the +Jenkins’ lookup3 algorithm. + +Whenever a bit flag or field is mentioned for an entry, bits are numbered from the lowest bit position +in the entry. + +Various tables in this document aligned with “This space inserted only to align table nicely”. +These entries in the table are just to make the table presentation nicer and do not represent any values +or padding in the file. + +\subsection subsec_fmt2_intro_110 I.B. Changes for HDF5 1.10 +As of October 2015, changes in the file format for HDF5 1.10 have not yet been finalized. + +\section sec_fmt2_meta II. Disk Format: Level 0 - File Metadata + +\subsection subsec_fmt2_boot_super II.A. Disk Format: Level 0A - Format Signature and Superblock +The superblock may begin at certain predefined offsets within the HDF5 file, allowing a block of +unspecified content for users to place additional information at the beginning (and end) of the HDF5 file +without limiting the HDF5 library’s ability to manage the objects within the file itself. This feature +was designed to accommodate wrapping an HDF5 file in another file format or adding descriptive information +to an HDF5 file without requiring the modification of the actual file’s information. The superblock +is located by searching for the HDF5 file signature at byte offset 0, byte offset 512 and at successive +locations in the file, each a multiple of two of the previous location, in other words, at these byte +offsets: 0, 512, 1024, 2048, and so on. + +The superblock is composed of the format signature, followed by a superblock version number and information +that is specific to each version of the superblock. Currently, there are three versions of the superblock +format. Version 0 is the default format, while version 1 is basically the same as version 0 with additional +information when a non-default B-tree ‘K’ value is stored. Version 2 is the latest format, with +some fields eliminated or compressed and with superblock extension and checksum support. + +Version 0 and 1 of the superblock are described below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Superblock (Versions 0 and 1)
bytebytebytebyte

Format Signature (8 bytes)

Version \# of SuperblockVersion \# of File’s Free Space StorageVersion \# of Root Group Symbol Table EntryReserved (zero)
Version \# of Shared Header Message FormatSize of OffsetsSize of LengthsReserved (zero)
Group Leaf Node KGroup Internal Node K
File Consistency Flags
Indexed Storage Internal Node K1Reserved (zero)1
Base AddressO
Address of File Free Space InfoO
End of File AddressO
Driver Information Block AddressO
Root Group Symbol Table Entry
+\li Items marked with an ‘1’ in the above table are new in version 1 of the superblock. +\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Format SignatureThis field contains a constant value and can be used to quickly identify a file as being an HDF5 + file. The constant value is designed to allow easy identification of an HDF5 file and to allow + certain types of data corruption to be detected. The file signature of an HDF5 file always + contains the following values: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Decimal:13772687013102610
Hexadecimal:894844460d0a1a0a
ASCII C Notation:\211HDF\\r\\n\032\\n
+
+ This signature both identifies the file as an HDF5 file and provides for immediate detection of common + file-transfer problems. The first two bytes distinguish HDF5 files on systems that expect the first two + bytes to identify the file type uniquely. The first byte is chosen as a non-ASCII value to reduce the + probability that a text file may be misrecognized as an HDF5 file; also, it catches bad file transfers + that clear bit 7. Bytes two through four name the format. The CR-LF sequence catches bad file transfers + that alter newline sequences. The control-Z character stops file display under MS-DOS. The final line + feed checks for the inverse of the CR-LF translation problem. (This is a direct descendent of the + PNG + file signature.)
+ This field is present in version 0+ of the superblock.
Version Number of the SuperblockThis value is used to determine the format of the information in the superblock. When the format of + the information in the superblock is changed, the version number is incremented to the next integer + and can be used to determine how the information in the superblock is formatted.
+ Values of 0, 1 and 2 are defined for this field. (The format of version 2 is described below, not + here)
This field is present in version 0+ of the superblock.
Version Number of the File’s Free Space InformationThis value is used to determine the format of the file’s free space information.
+ The only value currently valid in this field is ‘0’, which indicates that the file’s + free space index is as described in @ref subsec_fmt2_infra_freespaceindex below.
+ This field is present in version 0 and 1 of the superblock.
Version Number of the Root Group Symbol Table EntryThis value is used to determine the format of the information in the Root Group Symbol Table Entry. + When the format of the information in that field is changed, the version number is incremented to the + next integer and can be used to determine how the information in the field is formatted.
+ The only value currently valid in this field is ‘0’, which indicates that the root group + symbol table entry is formatted as described in @ref subsec_fmt2_infra_symboltableentry below.
+ This field is present in version 0 and 1 of the superblock.
Version Number of the Shared Header Message FormatThis value is used to determine the format of the information in a shared object header message. + Since the format of the shared header messages differs from the other private header messages, a + version number is used to identify changes in the format.
+ The only value currently valid in this field is ‘0’, which indicates that shared + header messages are formatted as described in @ref subsubsec_fmt2_dataobject_hdr_msg_shared below.
+ This field is present in version 0 and 1 of the superblock.
Size of OffsetsThis value contains the number of bytes used to store addresses in the file. The values for the + addresses of objects in the file are offsets relative to a base address, usually the address of the + superblock signature. This allows a wrapper to be added after the file is created without invalidating + the internal offset locations.
+ This field is present in version 0+ of the superblock.
Size of LengthsThis value contains the number of bytes used to store the size of an object.
+ This field is present in version 0+ of the superblock.
Group Leaf Node KEach leaf node of a group B-tree will have at least this many entries but not more than twice this + many. If a group has a single leaf node then it may have fewer entries.
+ This value must be greater than zero.
+ See the @ref subsec_fmt2_infra_btrees below.
+ This field is present in version 0 and 1 of the superblock.
Group Internal Node KEach internal node of a group B-tree will have at least this many entries but not more than twice this + many. If the group has only one internal node then it might have fewer entries.
+ This value must be greater than zero.
+ See the @ref subsec_fmt2_infra_btrees below.
+ This field is present in version 0 and 1 of the superblock.
File Consistency FlagsThis value contains flags to indicate information about the consistency of the information contained + within the file. Currently, the following bit flags are defined: +
    +
  • Bit 0 set indicates that the file is opened for write-access.
    • +
  • Bit 1 set indicates that the file has been verified for consistency and is guaranteed to be + consistent with the format defined in this document.
    • +
  • Bits 2-31 are reserved for future use.
    • +
+ Bit 0 should be set as the first action when a file is opened for write access and should be cleared + only as the final action when closing a file. Bit 1 should be cleared during normal access to a file + and only set after the file’s consistency is guaranteed by the library or a consistency utility.
+ This field is present in version 0+ of the superblock.
Indexed Storage Internal Node KEach internal node of a indexed storage B-tree will have at least this many entries but not more than + twice this many. If the ndex storage B-tree has only one internal node then it might have fewer + entries.
+ This value must be greater than zero.
+ See the @ref subsec_fmt2_infra_btrees below.
+ This field is present in version 1 of the superblock.
Base AddressThis is the absolute file address of the first byte of the HDF5 data within the file. The library + currently constrains this value to be the absolute file address of the superblock itself when creating + new files; future versions of the library may provide greater flexibility. When opening an existing + file and this address does not match the offset of the superblock, the library assumes that the entire + contents of the HDF5 file have been adjusted in the file and adjusts the base address and end of file + address to reflect their new positions in the file. Unless otherwise noted, all other file addresses + are relative to this base address.
+ This field is present in version 0+ of the superblock.
Address of Global Free Space IndexThe file’s free space management is not persistent for version 0 and 1 of the superblock. + Currently this field always contains the @ref FMT2UndefinedAddress "undefined address".
+ This field is present in version 0 and 1 of the superblock.
End of File AddressThis is the absolute file address of the first byte past the end of all HDF5 data. It is used to + determine whether a file has been accidentally truncated and as an address where file data allocation + can occur if space from the free list is not used.
+ This field is present in version 0+ of the superblock.
Driver Information Block AddressThis is the relative file address of the file driver information block which contains driver-specific + information needed to reopen the file. If there is no driver information block then this entry should + be the @ref FMT2UndefinedAddress "undefined address".
+ This field is present in version 0 and 1 of the superblock.
Root Group Symbol Table EntryThis is the @ref subsec_fmt2_infra_symboltableentry of the root group, which serves as the entry-point + into the group graph for the file.
+ This field is present in version 0 and 1 of the superblock.
+ +Version 2 of the superblock is described below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Superblock (Version 2)
bytebytebytebyte

Format Signature (8 bytes)

Version \# of SuperblockSize of OffsetsSize of LengthsFile Consistency Flags

Base AddressO


Superblock Extension AddressO


End of File AddressO


Root Group Object Header AddressO

Superblock Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Format SignatureThis field is the same as described for versions 0 and 1 of the superblock.
Version Number of the SuperblockThis field has a value of 2 and has the same meaning as for versions 0 and 1.
Size of OffsetsThis field is the same as described for versions 0 and 1 of the superblock.
Size of LengthsThis field is the same as described for versions 0 and 1 of the superblock.
File Consistency FlagsThis field is the same as described for versions 0 and 1 except that it is smaller (the number + of reserved bits has been reduced from 30 to 6).
Base AddressThis field is the same as described for versions 0 and 1 of the superblock.
Superblock Extension AddressThe field is the address of the object header for the @ref subsec_fmt2_boot_supext. If there is no + extension then this entry should be the @ref FMT2UndefinedAddress "undefined address".
End of File AddressThis field is the same as described for versions 0 and 1 of the superblock.
Root Group Object Header AddressThis is the address of the @ref sec_fmt2_dataobject, which serves as the entry point into the group + graph for the file.
Superblock ChecksumThe checksum for the superblock.
+ +\subsection subsec_fmt2_boot_driver II.B. Disk Format: Level 0B - File Driver Info +The driver information block is an optional region of the file which contains information +needed by the file driver in order to reopen a file. The format is described below: + + + + + + + + + + + + + + + + + + + + + +
Driver Information Block
bytebytebytebyte
VersionReserved
Driver Information Size

Driver Identification (8 bytes)



Driver Information (variable size bytes)


+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number of the Driver Information Block. This document describes version 0.
Driver Information SizeThe size in bytes of the Driver Information field.
Driver IdentificationThis is an eight-byte ASCII string without null termination which identifies the driver and/or version number + of the Driver Information block. The predefined driver encoded in this field by the HDF5 library is identified + by the letters NCSA followed by the first four characters of the driver name. If the Driver Information + Block is not the original version then the last letter(s) of the identification will be replaced by a version + number in ASCII, starting with 0.
+ Identification for user-defined drivers is also eight-byte long. It can be arbitrary but should be unique to + avoid the four character prefix “NCSA”.
Driver InformationDriver information is stored in a format defined by the file driver (see description below).
+ +The two drivers encoded in the Driver Identification field are as follows: +\li Multi driver:
The identifier for this driver is “NCSAmulti”. This driver provides + a mechanism for segregating raw data and different types of metadata into multiple files. These files + are viewed by the library as a single virtual HDF5 file with a single file address. A maximum of 6 + files will be created for the following data: superblock, B-tree, raw data, global heap, local heap, + and object header. More than one type of data can be written to the same file. +\li Family driver:
The identifier for this driver is “NCSAfami” and is encoded in this + field for library version 1.8 and after. This driver is designed for systems that do not support files + larger than 2 gigabytes by splitting the HDF5 file address space across several smaller files. It does + nothing to segregate metadata and raw data; they are mixed in the address space just as they would be + in a single contiguous file. + +The format of the Driver Information field for the above two drivers are described below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Multi Driver Information
bytebytebytebyte
Member MappingMember MappingMember MappingMember Mapping
Member MappingMember MappingReservedReserved

Address of Member File 1


End of Address for Member File 1


Address of Member File 2


End of Address for Member File 2


... ...


Address of Member File N


End of Address for Member File N


Name of Member File 1 (variable size)


Name of Member File 2 (variable size)


... ...


Name of Member File N (variable size)

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Member MappingThese fields are integer values from 1 to 6 indicating how the data can be mapped to or + merged with another type of data. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Member MappingDescription
1The superblock data.
2The B-tree data.
3The raw data.
4The global heap data.
5The local heap data.
6The object header data.

+ For example, if the third field has the value 3 and all the rest have the + value 1, it means there are two files, one for raw data, and one for superblock, + B-tree, global heap, local heap, and object header.
ReservedThese fields are reserved and should always be zero.
Address of Member File NThis field specifies the virtual address at which the member file starts.
+ N is the number of member files.
End of Address for Member File NThis field is the end of allocated address for the member file.
Name of Member File NThis field is the null-terminated name of member file. And its length should be multiples + of 8 bytes. Additional bytes will be padded with NULLs. The default naming convention is + %%s-X.h5, where X is one of the letters s (for superblock), + b (for B-tree), r (for raw data), g (for global heap), + l (for local heap), and o (for object header). The name for the whole + HDF5 file will substitute the %s in the string.
+ + + + + + + + + + + + +
Family Driver Information
bytebytebytebyte

Size of Member File

+ + + + + + + + + + +
Field NameDescription
Size of Member FileThis field is the size of the member file in the family of files.
+ +\subsection subsec_fmt2_boot_supext II.C. Disk Format: Level 0C - Superblock Extension +The superblock extension is used to store superblock metadata which is either optional, or added +after the version of the superblock was defined. Superblock extensions may only exist when version 2+ of +superblock is used. A superblock extension is an object header which may hold the following messages: +\li \ref subsec_fmt2_infra_sohm containing information to locate the master table of shared object + header message indices. +\li \ref subsubsec_fmt2_dataobject_hdr_msg_btreek containing non-default B-tree ‘K’ values. +\li \ref subsubsec_fmt2_dataobject_hdr_msg_drvinfo containing information needed by the file driver in + order to reopen a file. See also the \ref subsec_fmt2_boot_driver section above. +\li \ref subsubsec_fmt2_dataobject_hdr_msg_fsinfo containing information about file space handling in the file. + +\section sec_fmt2_infra III. Disk Format: Level 1 - File Infrastructure + +\subsection subsec_fmt2_infra_btrees III.A. Disk Format: Level 1A - B-trees and B-tree Nodes +B-trees allow flexible storage for objects which tend to grow in ways that cause the object to be stored +discontiguously. B-trees are described in various algorithms books including "Introduction to Algorithms" by +Thomas H. Cormen, Charles E. Leiserson, and Ronald L. Rivest. The B-trees are used in several places in +the HDF5 file format, when an index is needed for another data structure. + +The version 1 B-tree structure described below is the original index structure, but are limited by some +bugs in our implementation (mainly in how they handle deleting records). The version 1 B-trees are +being phased out in favor of the version 2 B-trees described below, although both types of structures may +be found in the same file, depending on application settings when creating the file. + +\subsubsection subsubsec_fmt2_infra_btrees_v1 III.A.1. Disk Format: Level 1A1 - Version 1 B-trees (B-link Trees) +Version 1 B-trees in HDF5 files an implementation of the B-link tree, in which the sibling nodes at a +particular level in the tree are stored in a doubly-linked list, is described in the “Efficient Locking +for Concurrent Operations on B-trees” paper by Phillip Lehman and S. Bing Yao as published in the + ACM Transactions on Database Systems, Vol. 6, No. 4, December 1981. + +The B-link trees implemented by the file format contain one more key than the number of children. In other +words, each child pointer out of a B-tree node has a left key and a right key. The pointers out of internal +nodes point to sub-trees while the pointers out of leaf nodes point to symbol nodes and raw data chunks. +Aside from that difference, internal nodes and leaf nodes are identical. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
B-link Tree Nodes
bytebytebytebyte
Signature
Node TypeNode LevelEntries Used

Address of Left SiblingO


Address of Right SiblingO

Key 0 (variable size)

Address of Child 0O

Key 1 (variable size)

Address of Child 1O

...
Key 2K (variable size)

Address of Child 2KO

Key 2K+1 (variable size)
+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “TREE” is used to indicate the beginning of a + B-link tree node. This gives file consistency checking utilities a better chance of reconstructing + a damaged file.
Node TypeEach B-link tree points to a particular type of data. This field indicates the type of data as well as + implying the maximum degree K of the tree and the size of each Key field.
+ + + + + + + + + + + + + +
Node TypeDescription
0This tree points to group nodes.
1This tree points to a raw data chunk.
+
Node LevelThe node level indicates the level at which this node appears in the tree (leaf nodes are at level + zero). Not only does the level indicate whether child pointers point to sub-trees or to data, but it + can also be used to help file consistency checking utilities reconstruct damaged trees.
Entries UsedThis determines the number of children to which this node points. All nodes of a particular type of + tree have the same maximum degree, but most nodes will point to less than that number of children. The + valid child pointers and keys appear at the beginning of the node and the unused pointers and keys + appear at the end of the node. The unused pointers and keys have undefined values.
Address of Left SiblingThis is the relative file address of the left sibling of the current node. If the current node is the + left-most node at this level then this field is the @ref FMT2UndefinedAddress "undefined address".
Address of Right SiblingThis is the relative file address of the right sibling of the current node. If the current node is the + right-most node at this level then this field is the @ref FMT2UndefinedAddress "undefined address".
Keys and Child PointersEach tree has 2K+1 keys with 2K child pointers interleaved between the keys. The number + of keys and child pointers actually containing valid values is determined by the node’s + Entries Used field. If that field is N then the B-link tree contains N child + pointers and N+1 keys.
KeyThe format and size of the key values is determined by the type of data to which this tree points. The + keys are ordered and are boundaries for the contents of the child pointer; that is, the key values + represented by child N fall between Key N and Key N+1. Whether the interval + is open or closed on each end is determined by the type of data to which the tree points.
+ The format of the key depends on the node type. For nodes of node type 0 (group nodes), the key is + formatted as follows: + + + + + +
A single field of Size of Lengths bytes.Indicates the byte offset into the local heap for the first object name in the subtree which + that key describes.
+
+ For nodes of node type 1 (chunked raw data nodes), the key is formatted as follows: + + + + + + + + + + + + + +
Bytes 1-4Size of chunk in bytes.
Bytes 4-8Filter mask, a 32-bit bit field indicating which filters have been skipped for this chunk. Each + filter has an index number in the pipeline (starting at 0, with the first filter to apply) and + if that filter is skipped, the bit corresponding to its index is set.
(D + 1) 64-bit fieldsThe offset of the chunk within the dataset where D is the number + of dimensions of the dataset, and the last value is the offset within the dataset’s + datatype and should always be zero. For example, if a chunk in a 3-dimensional dataset begins at the + position [5,5,5], there will be three such 64-bit indices, each with the value of + 5, followed by a 0 value.
+
Child PointerThe tree node contains file addresses of subtrees or data depending on the node level. Nodes at Level + 0 point to data addresses, either raw data chunks or group nodes. Nodes at non-zero levels point to other + nodes of the same B-tree.
+ For raw data chunk nodes, the child pointer is the address of a single raw data chunk. For group nodes, + the child pointer points to a @ref subsec_fmt2_infra_symboltableentry, which contains + information for multiple symbol table entries.
+ +Conceptually, each B-tree node looks like this: + + + + + + + + + + + + + + + + + + + + + + +
key[0] child[0] key[1] child[1] key[2]; ... ... key[N-1] child[N-1] key[N]
+where child[i] is a pointer to a sub-tree (at a level above Level 0) or to data (at Level 0). +Each key[i] describes an item stored by the B-tree (a chunk or an object of a group node). +The range of values represented by child[i] is indicated by key[i] and key[i+1]. + +The following question must next be answered: "Is the value described by key[i] contained in +child[i-1] or in child[i]?" The answer depends on the type of tree. In trees for groups (node +type 0) the object described by key[i] is the greatest object contained in child[i-1] while +in chunk trees (node type 1) the chunk described by key[i] is the least chunk in child[i]. + +That means that key[0] for group trees is sometimes unused; it points to offset zero in the heap, which is +always the empty string and compares as "less-than" any valid object name. + +And key[N] for chunk trees is sometimes unused; it contains a chunk offset which compares as +"greater-than" any other chunk offset and has a chunk byte size of zero to indicate that it is not actually +allocated. + +\subsubsection subsubsec_fmt2_infra_btrees_v2 III.A.2. Disk Format: Level 1A2 - Version 2 B-trees +Version 2 B-trees are “traditional” B-trees, with one major difference. Instead of just using +a simple pointer (or address in the file) to a child of an internal node, the pointer to the child node +contains two additional pieces of information: the number of records in the child node itself, and the +total number of records in the child node and all its descendants. Storing this additional information +allows fast array-like indexing to locate the nth record in the B-tree. + +The entry into a version 2 B-tree is a header which contains global information about the structure of +the B-tree. The root node address field in the header points to the B-tree root node, which is +either an internal or leaf node, depending on the value in the header’s depth field. An +internal node consists of records plus pointers to further leaf or internal nodes in the tree. A leaf +node consists of solely of records. The format of the records depends on the B-tree type (stored in +the header). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Version 2 B-tree Header
bytebytebytebyte
Signature
VersionTypeThis space inserted only to align table nicely
Node Size
Record SizeDepth
Split PercentMerge PercentThis space inserted only to align table nicely

Root Node AddressO

Number of Records in Root NodeThis space inserted only to align table nicely

Total Number of Records in B-treeL

Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “BTHD” is used to indicate the header of a + version 2 B-link tree node.
VersionThe version number for this B-tree header. This document describes version 0.
TypeThis field indicates the type of B-tree: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0A “testing” B-tree, this value should not be used for storing + records in actual HDF5 files.
1This B-tree is used for indexing indirectly accessed, non-filtered ‘huge’ + fractal heap objects.
2This B-tree is used for indexing indirectly accessed, filtered ‘huge’ + fractal heap objects.
3This B-tree is used for indexing directly accessed, non-filtered ‘huge’ + fractal heap objects.
4This B-tree is used for indexing directly accessed, filtered ‘huge’ + fractal heap objects.
5This B-tree is used for indexing the ‘name’ field for links in indexed + groups.
6This B-tree is used for indexing the ‘creation order’ field for links + in indexed groups.
7This B-tree is used for indexing shared object header messages.
8This B-tree is used for indexing the ‘name’ field for indexed + attributes.
9This B-tree is used for indexing the ‘creation order’ field for + indexed attributes.
+ The format of records for each type is described below.
Node SizeThis is the size in bytes of all B-tree nodes.
Record SizeThis field is the size in bytes of the B-tree record.
DepthThis is the depth of the B-tree.
Split PercentThe percent full that a node needs to increase above before it is split.
Merge PercentThe percent full that a node needs to be decrease below before it is split.
Root Node AddressThis is the address of the root B-tree node. A B-tree with no records will have the + @ref FMT2UndefinedAddress "undefined address" in this field.
Number of Records in Root NodeThis is the number of records in the root node.
Total Number of Records in B-treeThis is the total number of records in the entire B-tree.
ChecksumThis is the checksum for the B-tree header.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Version 2 B-tree Internal Node
bytebytebytebyte
Signature
VersionTypeRecords 0, 1, 2...N-1 (variable size)

Child Node Pointer 0O


Number of Records N0 for Child Node 0 (variable size)

Total Number of Records for Child Node 0 (optional, variable size)

Child Node Pointer 1O


Number of Records N1 for Child Node 1 (variable size)

Total Number of Records for Child Node 1 (optional, variable size)
...

Child Node Pointer NO


Number of Records Nn for Child Node N (variable size)

Total Number of Records for Child Node N (optional, variable size)
Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “ BTIN ” is used to indicate the internal node + of a B-link tree.
VersionThe version number for this B-tree internal node. This document describes version 0.
TypeThis field is the type of the B-tree node. It should always be the same as the B-tree type in + the header.
RecordsThe size of this field is determined by the number of records for this node and the record size + (from the header). The format of records depends on the type of B-tree.
Child Node PointerThis field is the address of the child node pointed to by the internal node.
Number of Records in Child NodeThis is the number of records in the child node pointed to by the corresponding Node Pointer.
+ The number of bytes used to store this field is determined by the maximum possible number of records able + to be stored in the child node.
+ The maximum number of records in a child node is computed in the following way: +
    +
  • Subtract the fixed size overhead for the child node (for example, its signature, version, + checksum, and so on and one pointer triplet of information for the child node + (because there is one more pointer triplet than records in each internal node)) from the size + of nodes for the B-tree.
    • +
  • Divide that result by the size of a record plus the pointer triplet of information stored to + reach each child node from this node.
    • +

+ Note that leaf nodes do not encode any child pointer triplets, so the maximum number of records in a + leaf node is just the node size minus the leaf node overhead, divided by the record size.
+ Also note that the first level of internal nodes above the leaf nodes do not encode the Total + Number of Records in Child Node value in the child pointer triplets (since it is the same as + the Number of Records in Child Node), so the maximum number of records in these nodes is + computed with the equation above, but using (Child Pointer, Number of Records in Child + Node) pairs instead of triplets.
+ The number of bytes used to encode this field is the least number of bytes required to encode the + maximum number of records in a child node value for the child nodes below this level in the B-tree.
+ For example, if the maximum number of child records is 123, one byte will be used to encode these + values in this node; if the maximum number of child records is 20000, two bytes will be used to + encode these values in this node; and so on. The maximum number of bytes used to encode these values + is 8 (in other words, an unsigned 64-bit integer).
Total Number of Records in Child NodeThis is the total number of records for the node pointed to by the corresponding Node Pointer + and all its children. This field exists only in nodes whose depth in the B-tree node is greater than 1 + (in other words, the “twig” internal nodes, just above leaf nodes, do not store this field + in their child node pointers).
+ The number of bytes used to store this field is determined by the maximum possible number of records + able to be stored in the child node and its descendants.
+ The maximum possible number of records able to be stored in a child node and its descendants is + computed iteratively, in the following way: The maximum number of records in a leaf node is + computed, then that value is used to compute the maximum possible number of records in the first + level of internal nodes above the leaf nodes. Multiplying these two values together determines the + maximum possible number of records in child node pointers for the level of nodes two levels above + leaf nodes. This process is continued up to any level in the B-tree.
+ The number of bytes used to encode this value is computed in the same way as for the Number + of Records in Child Node field.
ChecksumThis is the checksum for this node.
+ + + + + + + + + + + + + + + + + + + + +
Version 2 B-tree Leaf Node
bytebytebytebyte
Signature
VersionTypeRecord 0, 1, 2...N-1 (variable size)
Checksum
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “ BTLF “ is used to indicate the leaf node + of a version 2 B-link tree.
VersionThe version number for this B-tree leaf node. This document describes version 0.
TypeThis field is the type of the B-tree node. It should always be the same as the B-tree type in + the header.
RecordsThe size of this field is determined by the number of records for this node and the record size + (from the header). The format of records depends on the type of B-tree.
ChecksumThis is the checksum for this node.
+ +The record layout for each stored (in other words, non-testing) B-tree type is as follows: + + + + + + + + + + + + + + + + + +
Version 2 B-tree, Type 1 Record Layout - Indirectly Accessed, Non-Filtered, + ‘Huge’ Fractal Heap Objects
bytebytebytebyte

Huge Object AddressO


Huge Object LengthL


Huge Object IDL

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + +
Field NameDescription
Huge Object AddressThe address of the huge object in the file.
Huge Object LengthThe length of the huge object in the file.
Huge Object IDThe heap ID for the huge object.
+ + + + + + + + + + + + + + + + + + + + + + + + +
Version 2 B-tree, Type 2 Record Layout - Indirectly Accessed, Filtered, ‘Huge’ + Fractal Heap Objects
bytebytebytebyte

Filtered Huge Object AddressO


Filtered Huge Object LengthL

Filter Mask

Filtered Huge Object Memory SizeL


Huge Object IDL

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Filtered Huge Object AddressThe address of the filtered huge object in the file.
Filtered Huge Object LengthThe length of the filtered huge object in the file.
Filter MaskA 32-bit bit field indicating which filters have been skipped for this chunk. Each filter has + an index number in the pipeline (starting at 0, with the first filter to apply) and if that filter + is skipped, the bit corresponding to its index is set.
Filtered Huge Object Memory SizeThe size of the de-filtered huge object in memory.
Huge Object IDThe heap ID for the huge object.
+ + + + + + + + + + + + + + + +
Version 2 B-tree, Type 3 Record Layout - Directly Accessed, Non-Filtered, ‘Huge’ + Fractal Heap Objects
bytebytebytebyte

Huge Object AddressO


Huge Object LengthL

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + +
Field NameDescription
Huge Object AddressThe address of the huge object in the file.
Huge Object LengthThe length of the huge object in the file.
+ + + + + + + + + + + + + + + + + + + + + +
Version 2 B-tree, Type 4 Record Layout - Directly Accessed, Filtered, ‘Huge’ + Fractal Heap Objects
bytebytebytebyte

Filtered Huge Object AddressO


Filtered Huge Object LengthL

Filter Mask

Filtered Huge Object Memory SizeL

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Filtered Huge Object AddressThe address of the filtered huge object in the file.
Filtered Huge Object LengthThe length of the filtered huge object in the file.
Filter MaskA 32-bit bit field indicating which filters have been skipped for this chunk. Each filter has an + index number in the pipeline (starting at 0, with the first filter to apply) and if that filter + is skipped, the bit corresponding to its index is set.
Filtered Huge Object Memory SizeThe size of the de-filtered huge object in memory.
+ + + + + + + + + + + + + + + + + + +
Version 2 B-tree, Type 5 Record Layout - Link Name for Indexed Group
bytebytebytebyte
Hash of Name
ID (bytes 1-4)
ID (bytes 5-7)
+ + + + + + + + + + + + + + +
Field NameDescription
HashThis field is hash value of the name for the link. The hash value is the Jenkins’ lookup3 + checksum algorithm applied to the link’s name.
IDThis is a 7-byte sequence of bytes and is the heap ID for the link record in the group’s + fractal heap.
+ + + + + + + + + + + + + + + + + + + +
Version 2 B-tree, Type 6 Record Layout - Creation Order for Indexed Group
bytebytebytebyte

Creation Order (8 bytes)

ID (bytes 1-4)
ID (bytes 5-7)
+ + + + + + + + + + + + + + +
Field NameDescription
Creation OrderThis field is the creation order value for the link.
IDThis is a 7-byte sequence of bytes and is the heap ID for the link record in the group’s + fractal heap.
+ + + + + + + + + + + + + + + + + + + + + + +
Version 2 B-tree, Type 7 Record Layout - Shared Object Header Messages + (Sub-Type 0 - Message in Heap)
bytebytebytebyte
Message LocationThis space inserted only to align table nicely
Hash
Reference Count

Heap ID (8 bytes)

+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Message LocationThis field Indicates the location where the message is stored: + + + + + + + + + + + + + +
ValueDescription
0Shared message is stored in shared message index heap.
1Shared message is stored in object header.
+
HashThis field is hash value of the shared message. The hash value is the Jenkins’ lookup3 + checksum algorithm applied to the shared message.
Reference CountThe number of objects which reference this message.
Heap IDThis is an 8-byte sequence of bytes and is the heap ID for the shared message in the shared + message index’s fractal heap.
+ + + + + + + + + + + + + + + + + + + + + + + + +
Version 2 B-tree, Type 7 Record Layout - Shared Object Header Messages + (Sub-Type 1 - Message in Object Header)
bytebytebytebyte
Message LocationThis space inserted only to align table nicely
Hash
Reserved (zero)Message TypeObject Header Index

Object Header AddressO

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Message LocationThis field Indicates the location where the message is stored: + + + + + + + + + + + + + +
ValueDescription
0Shared message is stored in shared message index heap.
1Shared message is stored in object header.
+
HashThis field is hash value of the shared message. The hash value is the Jenkins’ lookup3 + checksum algorithm applied to the shared message.
Message TypeThe object header message type of the shared message.
Object Header IndexThis field indicates that the shared message is the nth message of its type in the + specified object header.
Object Header AddressThe address of the object header containing the shared message.
+ + + + + + + + + + + + + + + + + + + + + + +
Version 2 B-tree, Type 8 Record Layout - Attribute Name for Indexed Attributes
bytebytebytebyte

Heap ID (8 bytes)

Message FlagsThis space inserted only to align table nicely
Creation Order
Hash of Name
+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Heap IDThis is an 8-byte sequence of bytes and is the heap ID for the attribute in the object’s + attribute fractal heap.
Message FlagsThe object header message flags for the attribute message.
Creation OrderThis field is the creation order value for the attribute.
HashThis field is hash value of the name for the attribute. The hash value is the Jenkins’ + lookup3 checksum algorithm applied to the attribute’s name.
+ + + + + + + + + + + + + + + + + + + +
Version 2 B-tree, Type 9 Record Layout- Creation Order for Indexed Attributes
bytebytebytebyte

Heap ID (8 bytes)

Message FlagsThis space inserted only to align table nicely
Creation Order
+ + + + + + + + + + + + + + + + + + + +
Field NameDescription
Heap IDThis is an 8-byte sequence of bytes and is the heap ID for the attribute in the object’s + attribute fractal heap.
Message FlagsThe object header message flags for the attribute message.
Creation OrderThis field is the creation order value for the attribute.
+ +\subsection subsec_fmt2_infra_symboltable III.B. Disk Format: Level 1B - Group Symbol Table Nodes +A group is an object internal to the file that allows arbitrary nesting of objects within the file (including +other groups). A group maps a set of link names in the group to a set of relative file addresses of objects +in the file. Certain metadata for an object to which the group points can be cached in +object’s header. + +An HDF5 object name space can be stored hierarchically by partitioning the name into components and storing +each component as a link in a group. The link for a non-ultimate component points to the group containing the +next component. The link for the last component points to the object being named. + +One implementation a group is a collection of symbol table nodes indexed by a B-link tree. Each symbol table +node contains entries for one or more links. If an attempt is made to add a link to an already full +symbol table node containing 2K entries, then the node is split and one node contains K +symbols and the other contains K+1 symbols. + + + + + + + + + + + + + + + + + + + + +
Symbol Table Node (A Leaf of a B-link tree)
bytebytebytebyte
Signature
Version NumberReserved (zero)Number of Symbols


Group Entries


+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string SNOD is used to indicate the beginning of a symbol table node. This + gives file consistency checking utilities a better chance of reconstructing a damaged file.
Version NumberThe version number for the symbol table node. This document describes version 1. (There is no version + ‘0’ of the symbol table node)
Number of SymbolsAlthough all symbol table nodes have the same length, most contain fewer than the maximum possible number of + link entries. This field indicates how many entries contain valid data. The valid entries are packed + at the beginning of the symbol table node while the remaining entries contain undefined values.
Group EntriesEach link has an entry in the symbol table node. The format of the entry is described below. There are + 2K entries in each group node, where K is the “Group Leaf Node K” value + from the @ref subsec_fmt2_boot_super.
+ +\subsection subsec_fmt2_infra_symboltableentry III.C. Disk Format: Level 1C - Symbol Table Entry +Each symbol table entry in a symbol table node is designed to allow for very fast browsing of stored objects. +Toward that design goal, the symbol table entries include space for caching certain constant metadata from the +object header. + + + + + + + + + + + + + + + + + + + + + + + + +
Symbol Table Entry
bytebytebytebyte
Link Name OffsetO
Object Header AddressO
Cache Type
Reserved (zero)


Scratch-pad Space (16 bytes)


+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Link Name OffsetThis is the byte offset into the group’s local heap for the name of the link. The name is null + terminated.
Object Header AddressEvery object has an object header which serves as a permanent location for the object’s metadata. + In addition to appearing in the object header, some of the object’s metadata can be cached in the + scratch-pad space.
Cache TypeThe cache type is determined from the object header. It also determines the format for the scratch-pad + space.
+ + + + + + + + + + + + + + + + + +
Type:Description:
0No data is cached by the group entry. This is guaranteed to be the case when an object header has + a link count greater than one.
1Group object header metadata is cached in the scratch-pad space. This implies that the symbol table + entry refers to another group.
2The entry is a symbolic link. The first four bytes of the scratch-pad space are the offset into + the local heap for the link value. The object header address will be undefined.
+
ReservedThese four bytes are present so that the scratch-pad space is aligned on an eight-byte boundary. They + are always set to zero.
Scratch-pad SpaceThis space is used for different purposes, depending on the value of the Cache Type field. Any meta-data + about an object represented in the scratch-pad space is duplicated in the object header for + that object.
+ Furthermore, no data is cached in the group entry scratch-pad space if the object header for the object + has a link count greater than one.
+ +\subsubsection subsubsec_fmt2_infra_symboltableentry_scratch Format of the Scratch-pad Space +The symbol table entry scratch-pad space is formatted according to the value in the Cache Type field. + +If the Cache Type field contains the value zero ((0)) then no information is stored in the +scratch-pad space. + +If the Cache Type field contains the value one (1), then the scratch-pad space contains +cached metadata for another object header in the following format: + + + + + + + + + + + + + +
Object Header Scratch-pad Format
bytebytebytebyte
Address of B-treeO
Address of Name HeapO
+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + +
Field NameDescription
Address of B-treeThis is the file address for the root of the group’s B-tree.
Address of Name HeapThis is the file address for the group’s local heap, in which are stored the group’s + symbol names.
+ +If the Cache Type field contains the value two ((2)), then the scratch-pad space contains +cached metadata for a symbolic link in the following format: + + + + + + + + + + + +
Symbolic Link Scratch-pad Format
bytebytebytebyte
Offset to Link Value
+ + + + + + + + + + +
Field NameDescription
Offset to Link ValueThe value of a symbolic link (that is, the name of the thing to which it points) is stored in the + local heap. This field is the 4-byte offset into the local heap for the start of the link value, which + is null terminated.
+ +\subsection subsec_fmt2_infra_localheap III.D. Disk Format: Level 1D - Local Heaps +A local heap is a collection of small pieces of data that are particular to a single object in the HDF5 file. +Objects can be inserted and removed from the heap at any time. The address of a heap does not change once +the heap is created. For example, a group stores addresses of objects in symbol table nodes with the names +of links stored in the group’s local heap. + + + + + + + + + + + + + + + + + + + + + + + + +
Local Heap
bytebytebytebyte
Signature
VersionReserved (zero)
Data Segment SizeL
Offset to Head of Free-listL
Address of Data SegmentO
+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “HEAP ” is used to indicate the beginning of a heap. + This gives file consistency checking utilities a better chance of reconstructing a damaged file.
VersionEach local heap has its own version number so that new heaps can be added to old files. This document + describes version zero (0) of the local heap.
Data Segment SizeThe total amount of disk memory allocated for the heap data. This may be larger than the amount of space + required by the objects stored in the heap. The extra unused space in the heap holds a linked list of + free blocks.
Offset to Head of Free-listThis is the offset within the heap data segment of the first free block (or the + @ref FMT2UndefinedAddress "undefined address" if there is no no free block). The free block + contains “Size of Lengths” bytes that are the offset of the next free block (or the value + ‘1’ if this is the last free block) followed by “Size of Lengths” bytes that + store the size of this free block. The size of the free block includes the space used to store the + offset of the next free block and the size of the current block, making the minimum size of a free + block 2 * “Size of Lengths”.
Address of Data SegmentThe data segment originally starts immediately after the heap header, but if the data segment must grow + as a result of adding more objects, then the data segment may be relocated, in its entirety, to another + part of the file.
+ +Objects within a local heap should be aligned on an 8-byte boundary. + +\subsection subsec_fmt2_infra_globalheap III.E. Disk Format: Level 1E - Global Heap +Each HDF5 file has a global heap which stores various types of information which is typically shared between +datasets. The global heap was designed to satisfy these goals: +
    +
  1. Repeated access to a heap object must be efficient without resulting in repeated file I/O requests. + Since global heap objects will typically be shared among several datasets, it is probable that the + object will be accessed repeatedly.
    2. +
  2. Collections of related global heap objects should result in fewer and larger I/O requests. For + instance, a dataset of object references will have a global heap object for each reference. Reading + the entire set of object references should result in a few large I/O requests instead of one small + I/O request for each reference.
    3. +
  3. It should be possible to remove objects from the global heap and the resulting file hole should be + eligible to be reclaimed for other uses.
    4. +
+ +The implementation of the heap makes use of the memory management already available at the file level and +combines that with a new object called a collection to achieve goal B. The global heap is +the set of all collections. Each global heap object belongs to exactly one collection and each collection +contains one or more global heap objects. For the purposes of disk I/O and caching, a collection is treated +as an atomic object, addressing goal A. + +When a global heap object is deleted from a collection (which occurs when its reference count falls to zero), +objects located after the deleted object in the collection are packed down toward the beginning of the +collection and the collection’s global heap object 0 is created (if possible) or its size is increased +to account for the recently freed space. There are no gaps between objects in each collection, with the possible +exception of the final space in the collection, if it is not large enough to hold the header for the +collection’s global heap object 0. These features address goal C. + +The HDF5 library creates global heap collections as needed, so there may be multiple collections throughout +the file. The set of all of them is abstractly called the “global heap”, although they do not +actually link to each other, and there is no global place in the file where you can discover all of the +collections. The collections are found simply by finding a reference to one through another object in the file. +For example, data of variable-length datatype elements is stored in the global heap and is accessed via a +global heap ID. The format for global heap IDs is described at the end of this section. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
A Global Heap Collection
bytebytebytebyte
Signature
VersionReserved (zero)

Collection SizeL


Global Heap Object 1


Global Heap Object 2


...


Global Heap Object N


Global Heap Object 0 (free space)

+\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “GCOL” is used to indicate the beginning of a collection. + This gives file consistency checking utilities a better chance of reconstructing a damaged file.
VersionEach collection has its own version number so that new collections can be added to old files. This + document describes version one (1) of the collections (there is no version zero (0)).
Collection SizeThis is the size in bytes of the entire collection including this field. The default (and minimum) + collection size is 4096 bytes which is a typical file system block size. This allows for 127 16-byte + heap objects plus their overhead (the collection header of 16 bytes and the 16 bytes of information + about each heap object).
Global Heap Object 1 through NThe objects are stored in any order with no intervening unused space.
Global Heap Object 0Global Heap Object 0 (zero), when present, represents the free space in the collection. Free space always + appears at the end of the collection. If the free space is too small to store the header for Object 0 + (described below) then the header is implied and the collection contains no free space.
+ + + + + + + + + + + + + + + + + + + + + + +
Global Heap Object
bytebytebytebyte
Heap Object IndexReference Count
Reserved (zero)

Object SizeL


Object Data

+\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Heap Object IndexEach object has a unique identification number within a collection. The identification numbers are + chosen so that new objects have the smallest value possible with the exception that the identifier + 0 always refers to the object which represents all free space within the collection.
Reference CountAll heap objects have a reference count field. An object which is referenced from some other part of the + file will have a positive reference count. The reference count for Object 0 is always zero.
ReservedZero padding to align next field on an 8-byte boundary.
Object Size This is the size of the object data stored for the object. The actual storage space + allocated for the object data is rounded up to a multiple of eight.
Object DataThe object data is treated as a one-dimensional array of bytes to be interpreted by the caller.
+ +
+The format for the ID used to locate an object in the global heap is described here: + + + + + + + + + + + + + + +
Global Heap ID
bytebytebytebyte

Collection AddressO

Object Index
+\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + +
Field NameDescription
Collection AddressThis field is the address of the global heap collection where the data object is stored.
IDThis field is the index of the data object within the global heap collection.
+ +\subsection subsec_fmt2_infra_fractalheap III.F. Disk Format: Level 1F - Fractal Heap +Each fractal heap consists of a header and zero or more direct and indirect blocks (described below). +The header contains general information as well as initialization parameters for the doubling +table. The Root Block Address in the header points to the first direct or indirect block in +the heap. + +Fractal heaps are based on a data structure called a doubling table. A doubling table provides +a mechanism for quickly extending an array-like data structure that minimizes the number of empty blocks +in the heap, while retaining very fast lookup of any element within the array. More information on +fractal heaps and doubling tables can be found in the RFC +“\ref_rfc20070115 .” + +The fractal heap implements the doubling table structure with indirect and direct blocks. Indirect +blocks in the heap do not actually contain data for objects in the heap, their “size” is +abstract - they represent the indexing structure for locating the direct blocks in the doubling table. +Direct blocks contain the actual data for objects stored in the heap. + +All indirect blocks have a constant number of block entries in each row, called the width +of the doubling table (stored in the heap header). The number of rows for each indirect block in the +heap is determined by the size of the block that the indirect block represents in the doubling table +(calculation of this is shown below) and is constant, except for the “root” indirect block, +which expands and shrinks its number of rows as needed. + +Blocks in the first two rows of an indirect block are Starting Block Size number of +bytes in size, and the blocks in each subsequent row are twice the size of the blocks in the previous +row. In other words, blocks in the third row are twice the Starting Block Size, blocks in the +fourth row are four times the Starting Block Size, and so on. Entries for blocks up to the +Maximum Direct Block Size point to direct blocks, and entries for blocks greater than that size +point to further indirect blocks (which have their own entries for direct and indirect blocks). + +The number of rows of blocks, nrows, in an indirect block of size iblock_size is given +by the following expression:

+nrows = (log2(iblock_size) - log2(<Starting Block Size> +* <Width>)) + 1 + +The maximum number of rows of direct blocks, max_dblock_rows, in any indirect block of a fractal +heap is given by the following expression:

+max_dblock_rows = (log2(<Max. Direct Block Size>) - +log2(<Starting Block Size>)) + 2 + +Using the computed values for nrows and max_dblock_rows, along with the Width +of the doubling table, the number of direct and indirect block entries (K and N in the +indirect block description, below) in an indirect block can be computed:

+K = MIN(nrows, max_dblock_rows) * Width

+If nrows is less than or equal to max_dblock_rows, N is 0. Otherwise, N +is simply computed:

+N = K - (max_dblock_rows * Width) + +The size indirect blocks on disk is determined by the number of rows in the indirect block (computed above). +The size of direct blocks on disk is exactly the size of the block in the doubling table. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fractal Heap Header
bytebytebytebyte
Signature
VersionThis space inserted only to align table nicely
Heap ID LengthI/O Filters’ Encoded Length
FlagsThis space inserted only to align table nicely
Maximum Size of Managed Objects

Next Huge Object IDL


v2 B-tree Address of Huge ObjectsO


Amount of Free Space in Managed BlocksL


Address of Managed Block Free Space ManagerO


Amount of Managed Space in HeapL


Amount of Allocated Managed Space in HeapL


Offset of Direct Block Allocation Iterator in Managed SpaceL


Number of Managed Objects in HeapL


Size of Huge Objects in HeapL


Number of Huge Objects in HeapL


Size of Tiny Objects in HeapL


Number of Tiny Objects in HeapL

Table WidthThis space insertedonly to align table nicely

Starting Block SizeL


Maximum Direct Block SizeL

Maximum Heap SizeStarting \# of Rows in Root Indirect Block

Address of Root BlockO

Current \# of Rows in Root Indirect BlockThis space inserted only to align table nicely

Size of Filtered Root Direct Block (optional)L

I/O Filter Mask (optional)
I/O Filter Information (optional, variable size)
Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock."Size of Offset field in the superblocks." +\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “FRHP” is used to indicate the beginning of a + fractal heap header. This gives file consistency checking utilities a better chance of reconstructing + a damaged file.
VersionThis document describes version 0.
Heap ID LengthThis is the length in bytes of heap object IDs for this heap.
I/O Filters’ Encoded LengthThis is the size in bytes of the encoded I/O Filter Information.
FlagsThis field is the heap status flag and is a bit field indicating additional information about + the fractal heap. + + + + + + + + + + + + + + + + + +
Bit(s)Description
0If set, the ID value to use for huge object has wrapped around. If the value for the + Next Huge Object ID has wrapped around, each new huge object inserted into the + heap will require a search for an ID value. +
1If set, the direct blocks in the heap are checksummed.
2-7Reserved
Maximum Size of Managed ObjectsThis is the maximum size of managed objects allowed in the heap. Objects greater than this this + are ‘huge’ objects and will be stored in the file directly, rather than in a direct + block for the heap.
Next Huge Object IDThis is the next ID value to use for a huge object in the heap.
v2 B-tree Address of Huge ObjectsThis is the address of the @ref subsubsec_fmt2_infra_btrees_v2 used to track huge objects in the heap. + The type of records stored in the v2 B-tree will be determined by whether the address & + length of a huge object can fit into a heap ID (if yes, it is a “directly” accessed huge + object) and whether there is a filter used on objects in the heap.
Amount of Free Space in Managed BlocksThis is the total amount of free space in managed direct blocks (in bytes).
Address of Managed Block Free Space ManagerThis is the address of the @ref subsec_fmt2_infra_freespaceindex + for managed blocks.
Amount of Managed Space in HeapThis is the total amount of managed space in the heap (in bytes), essentially the + upper bound of the heap’s linear address space.
Amount of Allocated Managed Space in HeapThis is the total amount of managed space (in bytes) actually allocated in the heap. + This can be less than the Amount of Managed Space in Heap field, if some direct + blocks in the heap’s linear address space are not allocated.
Offset of Direct Block Allocation Iterator in Managed SpaceThis is the linear heap offset where the next direct block should be allocated at (in bytes). + This may be less than the Amount of Managed Space in Heap value because the heap’s + address space is increased by a “row” of direct blocks at a time, rather than by single + direct block increments.
Number of Managed Objects in HeapThis is the number of managed objects in the heap.
Size of Huge Objects in HeapThis is the total size of huge objects in the heap (in bytes).
Number of Huge Objects in HeapThis is the number of huge objects in the heap.
Size of Tiny Objects in HeapThis is the total size of tiny objects that are packed in heap IDs (in bytes).
Number of Tiny Objects in HeapThis is the number of tiny objects that are packed in heap IDs.
Table WidthThis is the number of columns in the doubling table for managed blocks. This value + must be a power of two.
Starting Block SizeThis is the starting block size to use in the doubling table for managed blocks (in bytes). + This value must be a power of two.
Maximum Direct Block SizeThis is the maximum size allowed for a managed direct block. Objects inserted into the heap that + are larger than this value (less the \# of bytes of direct block prefix/suffix) are stored as + ‘huge’ objects. This value must be a power of two.
Maximum Heap SizeThis is the maximum size of the heap’s linear address space for managed objects (in bytes). + The value stored is the log2 of the actual value, that is: the \# of bits of the address space. + ‘Huge’ and ‘tiny’ objects are not counted in this value, since they do not + store objects in the linear address space of the heap.
Starting \# of Rows in Root Indirect BlockThis is the starting number of rows for the root indirect block. A value of 0 indicates that the + root indirect block will have the maximum number of rows needed to address the heap’s + Maximum Heap Size.
Address of Root BlockThis is the address of the root block for the heap. It can be the + @ref FMT2UndefinedAddress "undefined address" if there is no data in the heap. It either + points to a direct block (if the Current \# of Rows in the Root Indirect Block value is 0), + or an indirect block.
Current \# of Rows in Root Indirect BlockThis is the current number of rows in the root indirect block. A value of 0 indicates that + Address of Root Block points to direct block instead of indirect block.
Size of Filtered Root Direct BlockThis is the size of the root direct block, if filters are applied to heap objects (in bytes). + This field is only stored in the header if the I/O Filters’ Encoded Length is + greater than 0.
I/O Filter MaskThis is the filter mask for the root direct block, if filters are applied to heap objects. This + mask has the same format as that used for the filter mask in chunked raw data records in a + @ref subsubsec_fmt2_infra_btrees_v1. This field is only stored in the header if the I/O Filters’ + Encoded Length is greater than 0.
I/O Filter InformationThis is the I/O filter information encoding direct blocks and huge objects, if filters are applied to + heap objects. This field is encoded as a @ref subsubsec_fmt2_dataobject_hdr_msg_filter message. The size + of this field is determined by I/O Filters’ Encoded Length.
ChecksumThis is the checksum for the header.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fractal Heap Direct Block
bytebytebytebyte
Signature
VersionThis space inserted only to align table nicely

Heap Header AddressO

Block Offset (variable size)
Checksum (optional)

Object Data (variable size)

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “FHDB” is used to indicate the beginning of + a fractal heap direct block. This gives file consistency checking utilities a better chance of + reconstructing a damaged file.
VersionThis document describes version 0.
Heap Header AddressThis is the address for the fractal heap header that this block belongs to. This field is + principally used for file integrity checking.
Block OffsetThis is the offset of the block within the fractal heap’s address space (in bytes). The + number of bytes used to encode this field is the Maximum Heap Size (in the heap’s + header) divided by 8 and rounded up to the next highest integer, for values that are not a multiple + of 8. This value is principally used for file integrity checking.
ChecksumThis is the checksum for the direct block. This field is only present if bit 1 of Flags + in the heap’s header is set.
Object DataThis section of the direct block stores the actual data for objects in the heap. The size of this + section is determined by the direct block’s size minus the size of the other fields stored in the + direct block (for example, the Signature, Version, and others including the + Checksum if it is present).
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fractal Heap Indirect Block
bytebytebytebyte
Signature
VersionThis space inserted only to align table nicely

Heap Header AddressO

Block Offset (variable size)

Child Direct Block \#0 AddressO


Size of Filtered Direct Block \#0 (optional) L

Filter Mask for Direct Block \#0 (optional)

Child Direct Block \#1 AddressO


Size of Filtered Direct Block \#1 (optional)L

Filter Mask for Direct Block \#1 (optional)
...

Child Direct Block \#K-1 AddressO


Size of Filtered Direct Block \#K-1 (optional)L

Filter Mask for Direct Block \#K-1 (optional)

Child Indirect Block \#0 AddressO


Child Indirect Block \#1 AddressO

...

Child Indirect Block \#N-1 AddressO

Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “FHIB” is used to indicate the beginning of a + fractal heap indirect block. This gives file consistency checking utilities a better chance of + reconstructing a damaged file.
VersionThis document describes version 0.
Heap Header AddressThis is the address for the fractal heap header that this block belongs to. This field is principally + used for file integrity checking.
Block OffsetThis is the offset of the block within the fractal heap’s address space (in bytes). The number + of bytes used to encode this field is the Maximum Heap Size (in the heap’s header) + divided by 8 and rounded up to the next highest integer, for values that are not a multiple of 8. This + value is principally used for file integrity checking.
Child Direct Block \#K AddressThis field is the address of the child direct block. The size of the [uncompressed] direct block can + be computed by its offset in the heap’s linear address space.
Size of Filtered Direct Block \#KThis is the size of the child direct block after passing through the I/O filters defined for this heap + (in bytes). If no I/O filters are present for this heap, this field is not present.
Filter Mask for Direct Block \#KThis is the I/O filter mask for the filtered direct block. This mask has the same format as that + used for the filter mask in chunked raw data records in a @ref subsubsec_fmt2_infra_btrees_v1. If + no I/O filters are present for this heap, this field is not present.
Child Indirect Block \#N AddressThis field is the address of the child indirect block. The size of the indirect block can be computed + by its offset in the heap’s linear address space.
ChecksumThis is the checksum for the indirect block.
+ +An object in the fractal heap is identified by means of a fractal heap ID, which encodes information to +locate the object in the heap. Currently, the fractal heap stores an object in one of three ways, +depending on the object’s size: + + + + + + + + + + + + + + + + + +
TypeDescription
TinyWhen an object is small enough to be encoded in the heap ID, the object’s data is embedded + in the fractal heap ID itself. There are 2 sub-types for this type of object: normal and extended. + The sub-type for tiny heap IDs depends on whether the heap ID is large enough to store objects + greater than 16 bytes or not. If the heap ID length is 18 bytes or smaller, the ‘normal’ + tiny heap ID form is used. If the heap ID length is greater than 18 bytes in length, the + “extended” form is used. See format description below for both sub-types.
HugeWhen the size of an object is larger than Maximum Size of Managed Objects in the + Fractal Heap Header, the object’s data is stored on its own in the file and the object + is tracked/indexed via a version 2 B-tree. All huge objects for a particular fractal heap use the same + v2 B-tree. All huge objects for a particular fractal heap use the same format for their huge object IDs. +
Depending on whether the IDs for a heap are large enough to hold the object’s retrieval + information and whether I/O pipeline filters are applied to the heap’s objects, 4 sub-types are + derived for huge object IDs for this heap: + + + + + + + + + + + + + + + + + + + + + +
Sub-typeDescription
Directly accessed, non-filteredThe object’s address and length are embedded in the fractal heap ID itself and the + object is directly accessed from them. This allows the object to be accessed without resorting + to the B-tree.
Directly accessed, filteredThe filtered object’s address, length, filter mask and de-filtered size are embedded + in the fractal heap ID itself and the object is accessed directly with them. This allows the + object to be accessed without resorting to the B-tree.
Indirectly accessed, non-filteredThe object is located by using a B-tree key embedded in the fractal heap ID to retrieve the + address and length from the version 2 B-tree for huge objects. Then, the address and length + are used to access the object.
Indirectly accessed, filteredThe object is located by using a B-tree key embedded in the fractal heap ID to retrieve the + filtered object’s address, length, filter mask and de-filtered size from the version + 2 B-tree for huge objects. Then, this information is used to access the object.
ManagedWhen the size of an object does not meet the above two conditions, the object is stored and managed + via the direct and indirect blocks based on the doubling table.
+ +The specific format for each type of heap ID is described below: + + + + + + + + + + + + + + + +
Fractal Heap ID for Tiny Objects (sub-type 1 - ‘Normal’)
bytebytebytebyte
Version, Type & LengthThis space inserted only to align table nicely

Data (variable size)
+ + + + + + + + + + + + + + +
Field NameDescription
Version, Type & LengthThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
6-7The current version of ID format. This document describes version 0.
4-5The ID type. Tiny objects have a value of 2. +
0-3The length of the tiny object. The value stored is one less than the actual length (since + zero-length objects are not allowed to be stored in the heap). For example, an object of + actual length 1 has an encoded length of 0, an object of actual length 2 has an encoded + length of 1, and so on.
DataThis is the data for the object.
+ + + + + + + + + + + + + + + + + +
Fractal Heap ID for Tiny Objects (sub-type 2 - ‘Extended’)
bytebytebytebyte
Version, Type & LengthExtended LengthThis space inserted only to align table nicely
Data (variable size)
+ + + + + + + + + + + + + + + + + + +
Field NameDescription
Version, Type & LengthThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
6-7The current version of ID format. This document describes version 0.
4-5The ID type. Tiny objects have a value of 2.
0-3These 4 bits, together with the next byte, form an unsigned 12-bit integer for holding the + length of the object. These 4-bits are bits 8-11 of the 12-bit integer. See description + for the Extended Length field below.
Extended LengthThis byte, together with the 4 bits in the previous byte, forms an unsigned 12-bit integer for + holding the length of the tiny object. These 8 bits are bits 0-7 of the 12-bit integer formed. The + value stored is one less than the actual length (since zero-length objects are not allowed to be + stored in the heap). For example, an object of actual length 1 has an encoded length of 0, an object of + actual length 2 has an encoded length of 1, and so on.
DataThis is the data for the object.
+ + + + + + + + + + + + + + + + +
Fractal Heap ID for Huge Objects (sub-type 1 & 2): indirectly accessed, + non-filtered/filtered
bytebytebytebyte
Version & TypeThis space inserted only to align table nicely

v2 B-tree KeyL (variable size)

+\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + +
Field NameDescription
Version & TypeThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
6-7The current version of ID format. This document describes version 0.
4-5The ID type. Huge objects have a value of 1.
0-3Reserved.
v2 B-tree KeyThis field is the B-tree key for retrieving the information from the version 2 B-tree for huge + objects needed to access the object. See the description of @ref subsubsec_fmt2_infra_btrees_v2 + records sub-type 1 & 2 for a description of the fields. New key values are derived from Next + Huge Object ID in the Fractal Heap Header.
+ + + + + + + + + + + + + + + + + + + +
Fractal Heap ID for Huge Objects (sub-type 3): directly accessed, non-filtered
bytebytebytebyte
Version & TypeThis space inserted only to align table nicely

Address O


Length L

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + +
Field NameDescription
Version & TypeThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
6-7The current version of ID format. This document describes version 0.
4-5The ID type. Huge objects have a value of 1.
0-3Reserved.
AddressThis field is the address of the object in the file.
LengthThis field is the length of the object in the file.
+ + + + + + + + + + + + + + + + + + + + + + + + + +
Fractal Heap ID for Huge Objects (sub-type 4): directly accessed, filtered
bytebytebytebyte
Version & TypeThis space inserted only to align table nicely

Address O


Length L

Filter Mask

De-filtered Size L

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Version & TypeThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
6-7The current version of ID format. This document describes version 0.
4-5The ID type. Huge objects have a value of 1.
0-3Reserved.
AddressThis field is the address of the filtered object in the file.
LengthThis field is the length of the filtered object in the file.
Filter MaskThis field is the I/O pipeline filter mask for the filtered object in the file.
Filtered SizeThis field is the size of the de-filtered object in the file.
+ + + + + + + + + + + + + + + + + + + +
Fractal Heap ID for Managed Objects
bytebytebytebyte
Version & TypeThis space inserted only to align table nicely
Offset (variable size)
Length (variable size)
+ + + + + + + + + + + + + + + + + + +
Field NameDescription
Version & TypeThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
6-7The current version of ID format. This document describes version 0.
4-5The ID type. Managed objects have a value of 0.
0-3Reserved.
OffsetThis field is the offset of the object in the heap. This field’s size is the minimum number of + bytes necessary to encode the Maximum Heap Size value (from the Fractal Heap Header). + For example, if the value of the Maximum Heap Size is less than 256 bytes, this field is 1 + byte in length, a Maximum Heap Size of 256-65535 bytes uses a 2 byte length, and so on.
LengthThis field is the length of the object in the heap. It is determined by taking the minimum value + of Maximum Direct Block Size and Maximum Size of Managed Objects in the Fractal + Heap Header. Again, the minimum number of bytes needed to encode that value is used for the size + of this field.
+ +\subsection subsec_fmt2_infra_freespaceindex III.G. Disk Format: Level 1G - Free-space Index +Free-space managers are used to describe space within a heap or the entire HDF5 file that is not currently +used for that heap or file. + +The free-space manager header contains metadata information about the space being tracked, along +with the address of the list of free space sections which actually describes the free space. The +header records information about free-space sections being tracked, creation parameters for handling +free-space sections of a client, and section information used to locate the collection of free-space sections. + +The free-space section list stores a collection of free-space sections that is specific to each +client of the free-space manager. For example, the fractal heap is a client of the free space +manager and uses it to track unused space within the heap. There are 4 types of section records for the +fractal heap, each of which has its own format, listed below. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Free-space Manager Header
bytebytebytebyte
Signature
VersionClient IDThis space inserted only to align table nicely

Total Space TrackedL


Total Number of SectionsL


Number of Serialized SectionsL


Number of Un-Serialized SectionsL

Number of Section ClassesThis space inserted only to align table nicely
Shrink PercentExpand Percent
Size of Address SpaceThis space inserted only to align table nicely

Maximum Section Size L


Address of Serialized Section ListO


Size of Serialized Section List UsedL


Allocated Size of Serialized Section ListL

Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “FSHD” is used to indicate the beginning of the + Free-space Manager Header. This gives file consistency checking utilities a better chance of + reconstructing a damaged file.
VersionThis is the version number for the Free-space Manager Header and this document describes version 0.
Client IDThis is the client ID for identifying the user of this free-space manager: + + + + + + + + + + + + + + + + + +
IDDescription
0Fractal heap
1File
2+Reserved.
Total Space TrackedThis is the total amount of free space being tracked, in bytes.
Total Number of SectionsThis is the total number of free-space sections being tracked.
Number of Serialized SectionsThis is the number of serialized free-space sections being tracked.
Number of Un-Serialized SectionsThis is the number of un-serialized free-space sections being managed. Un-serialized sections are + created by the free-space client when the list of sections is read in.
Number of Section ClassesThis is the number of section classes handled by this free space manager for the free-space client.
Shrink PercentThis is the percent of current size to shrink the allocated serialized free-space section list.
Expand PercentThis is the percent of current size to expand the allocated serialized free-space section list.
Size of Address SpaceThis is the size of the address space that free-space sections are within. This is stored as the + log2 of the actual value (in other words, the number of bits required to store values + within that address space).
Maximum Section SizeThis is the maximum size of a section to be tracked.
Address of Serialized Section ListThis is the address where the serialized free-space section list is stored.
Size of Serialized Section List UsedThis is the size of the serialized free-space section list used (in bytes). This value must be + less than or equal to the allocated size of serialized section list, below.
Allocated Size of Serialized Section ListThis is the size of serialized free-space section list actually allocated (in bytes).
ChecksumThis is the checksum for the free-space manager header.
+ +The free-space sections being managed are stored in a free-space section list, described below. +The sections in the free-space section list are stored in the following way: a count of the number of sections +describing a particular size of free space and the size of the free-space described (in bytes), followed +by a list of section description records; then another section count and size, followed by the list of +section descriptions for that size; and so on. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Free-space Section List
bytebytebytebyte
Signature
VersionThis space inserted only to align table nicely

Free-space Manager Header AddressO

Number of Section Records in Set \#0 (variable size)
Size of Free-space Section Described in Record Set \#0 (variable size)
Record Set \#0 Section Record \#0 Offset (variable size)
Record Set \#0 Section Record #0 TypeThis space inserted only to align table nicely
Record Set \#0 Section Record \#0 Data (variable size)
...
Record Set \#0 Section Record \#K-1 Offset (variable size)
Record Set \#0 Section Record \#K-1 TypeThis space inserted only to align table nicely
Record Set \#0 Section Record \#K-1 Data (variable size)
Number of Section Records in Set \#1 (variable size)
Size of Free-space Section Described in Record Set \#1 (variable size)
Record Set \#1 Section Record \#0 Offset (variable size)
Record Set \#1 Section Record \#0 TypeThis space inserted only to align table nicely
Record Set \#1 Section Record \#0 Data (variable size)
...
Record Set \#1 Section Record \#K-1 Offset (variable size)
Record Set \#1 Section Record \#K-1 TypeThis space inserted only to align table nicely
Record Set \#1 Section Record \#K-1 Data (variable size)
...
...
Number of Section Records in Set \#N-1 (variable size)
Size of Free-space Section Described in Record Set \#N-1 (variable size)
Record Set \#N-1 Section Record \#0 Offset (variable size)
Record Set \#N-1 Section Record \#0 TypeThis space inserted only to align table nicely
Record Set \#N-1 Section Record \#0 Data (variable size)
...
Record Set \#N-1 Section Record \#K-1 Offset (variable size)
Record Set \#N-1 Section Record \#K-1 TypeThis space inserted only to align table nicely
Record Set \#N-1 Section Record \#K-1 Data (variable size)
Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “FSSE” is used to indicate the beginning of the + Free-space Section Information. This gives file consistency checking utilities a better chance of + reconstructing a damaged file.
VersionThis is the version number for the Free-space Section List and this document describes version 0.
Free-space Manager Header AddressThis is the address of the Free-space Manager Header. This field is principally used for file + integrity checking.
Number of Section Records for Set \#NThis is the number of free-space section records for set \#N. The length of this field is the minimum + number of bytes needed to store the number of serialized sections (from the free-space + manager header).
+ The number of sets of free-space section records is determined by the size of serialized section + list in the free-space manager header.
Section Size for Record Set \#NThis is the size (in bytes) of the free-space section described for all the section records + in set \#N.
+ The length of this field is the minimum number of bytes needed to store the maximum section + size (from the free-space manager header).
Record Set \#N Section \#K OffsetThis is the offset (in bytes) of the free-space section within the client for the free-space manager. +
The length of this field is the minimum number of bytes needed to store the size of address + space (from the free-space manager header).
Record Set \#N Section \#K TypeThis is the type of the section record, used to decode the record set \#N section \#K data + information. The defined record type for file client is: + + + + + + + + + + + + + +
TypeDescription
0File’s section (a range of actual bytes in file)
1+Reserved.
+
The defined record types for a fractal heap client are: + + + + + + + + + + + + + + + + + + + + + + + + + +
TypeDescription
0Fractal heap “single” section
1Fractal heap “first row” section
2Fractal heap “normal row” section
3Fractal heap “indirect” section
4+Reserved.
Record Set \#N Section \#K DataThis is the section-type specific information for each record in the record set, described below.
ChecksumThis is the checksum for the Free-space Section List.
+ +The section-type specific data for each free-space section record is described below: + + + + + +
File’s Section Data Record
No additional record data stored
+
+ + + + + +
Fractal Heap “Single” Section Data Record
No additional record data stored
+
+ + + + + +
Fractal Heap “First Row” Section Data Record
Same format as “indirect” section data
+
+ + + + + +
Fractal Heap “Normal Row” Section Data Record
No additional record data stored
+
+ + + + + + + + + + + + + + + + + + + +
Fractal Heap “Indirect” Section Data Record
bytebytebytebyte
Fractal Heap Indirect Block Offset (variable size)
Block Start RowBlock Start Column
Number of BlocksThis space inserted only to align table nicely
+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Fractal Heap Block OffsetThe offset of the indirect block in the fractal heap’s address space containing the empty + blocks.
+ The number of bytes used to encode this field is the minimum number of bytes needed to encode + values for the Maximum Heap Size (in the fractal heap’s header).
Block Start RowThis is the row that the empty blocks start in.
Block Start ColumnThis is the column that the empty blocks start in.
Number of BlocksThis is the number of empty blocks covered by the section.
+ +\subsection subsec_fmt2_infra_sohm III.H. Disk Format: Level 1H - Shared Object Header Message Table +The shared object header message table is used to locate object header messages that are shared +between two or more object headers in the file. Shared object header messages are stored and indexed in +the file in one of two ways: indexed sequentially in a shared header message list or indexed +with a v2 B-tree. The shared messages themselves are either stored in a fractal heap (when two or more +objects share the message), or remain in an object’s header (when only one object uses the message +currently, but the message can be shared in the future). + +The shared object header message table contains a list of shared message index headers. Each +index header records information about the version of the index format, the index storage type, flags +for the message types indexed, the number of messages in the index, the address where the index resides, +and the fractal heap address if shared messages are stored there. + +Each index can be either a list or a v2 B-tree and may transition between those two forms as the number +of messages in the index varies. Each shared message record contains information used to locate the +shared message from either a fractal heap or an object header. The types of messages that can be shared +are: Dataspace, Datatype, Fill Value, Filter Pipeline and Attribute. + +The shared object header message table is pointed to from a +@ref subsubsec_fmt2_dataobject_hdr_msg_shared message in the superblock extension for a file. This +message stores the version of the table format, along with the number of index headers in the table. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Shared Object Header Message Table
bytebytebytebyte
Signature
Version for index \#0Index Type for index #0Message Type Flags for index \#0
Minimum Message Size for index \#0
List Cutoff for index \#0v2 B-tree Cutoff for index \#0
Number of Messages for index \#0This space inserted only to align table nicely

Index AddressO for index \#0


Fractal Heap AddressO for index \#0

...
...
Version for index \#N-1Index Type for index \#N-1Message Type Flags for index \#N-1
Minimum Message Size for index \#N-1
List Cutoff for index \#N-1v2 B-tree Cutoff for index \#N-1
Number of Messages for index \#N-1This space inserted only to align table nicely

Index AddressO for index \#N-1


Fractal Heap AddressO for index \#N-1

Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “SMTB” is used to indicate the beginning of the + Shared Object Header Message table. This gives file consistency checking utilities a better chance + of reconstructing a damaged file.
Version for index \#NThis is the version number for the list of shared object header message indexes and this document + describes version 0.
Index Type for index \#NThe type of index can be an unsorted list or a v2 B-tree.
Message Type Flags for index \#NThis field indicates the type of messages tracked in the index, as follows: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
BitsDescription
0If set, the index tracks Dataspace Messages.
1If set, the message tracks Datatype Messages.
2If set, the message tracks Fill Value Messages.
3If set, the message tracks Filter Pipeline Messages.
4If set, the message tracks Attribute Messages.
5-15Reserved (zero).
+ An index can track more than one type of message, but each type of message can only by in one index.
Minimum Message Size for index \#NThis is the message size sharing threshold for the index. If the encoded size of the message is + less than this value, the message is not shared.
List Cutoff for index \#NThis is the cutoff value for the indexing of messages to switch from a list to a v2 B-tree. If the + number of messages is greater than this value, the index should be a v2 B-tree.
v2 B-tree Cutoff for index \#NThis is the cutoff value for the indexing of messages to switch from a v2 B-tree back to a list. + If the number of messages is less than this value, the index should be a list.
Number of Messages for index \#NThe number of shared messages being tracked for the index.
Index Address for index \#NThis field is the address of the list or v2 B-tree where the index nodes reside.
Fractal Heap Address for index \#NThis field is the address of the fractal heap if shared messages are stored there.
ChecksumThis is the checksum for the table.
+ +Shared messages are indexed either with a shared message record list, described below, +or using a v2 B-tree (using record type 7). The number of records in the shared message record +list is determined in the index’s entry in the shared object header message table. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Shared Message Record List
bytebytebytebyte
Signature
Shared Message Record \#0
Shared Message Record \#1
...
Shared Message Record \#N-1
Checksum
+ + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “SMLI” is used to indicate the beginning of a + list of index nodes. This gives file consistency checking utilities a better chance of + reconstructing a damaged file.
Shared Message Record \#NThe record for locating the shared message, either in the fractal heap for the index, or an object + header (see format for index nodes below).
ChecksumThis is the checksum for the list.
+ +The record for each shared message in an index is stored in one of the following forms: + + + + + + + + + + + + + + + + + + + + + +
Shared Message Record, for messages stored in a fractal heap
bytebytebytebyte
Message LocationThis space inserted only to align table nicely
Hash Value
Reference Count

Fractal Heap ID

+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Message LocationThis has a value of 0 indicating that the message is stored in the heap.
Hash ValueThis is the hash value for the message.
Reference CountThis is the number of times the message is used in the file.
Fractal Heap IDThis is an 8-byte fractal heap ID for the message as stored in the fractal heap for the index.
+ + + + + + + + + + + + + + + + + + + + + + + + +
Shared Message Record, for messages stored in an object header
bytebytebytebyte
Message LocationThis space inserted only to align table nicely
Hash Value
ReservedMessage TypeCreation Index

Object Header AddressO

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Message LocationThis has a value of 1 indicating that the message is stored in an object header.
Hash ValueThis is the hash value for the message.
Message TypeThis is the message type in the object header.
Creation IndexThis is the creation index of the message within the object header.
Object Header AddressThis is the address of the object header where the message is located.
+ +\section sec_fmt2_dataobject IV. Disk Format: Level 2 - Data Objects +Data objects contain the “real” user-visible information in the file. These objects compose +the scientific data and other information which are generally thought of as “data” by the +end-user. All the other information in the file is provided as a framework for storing and accessing +these data objects. + +A data object is composed of header and data information. The header information contains the +information needed to interpret the data information for the object as well as additional “metadata” +or pointers to additional “metadata” used to describe or annotate each object. + +\subsection subsec_fmt2_dataobject_hdr IV.A. Disk Format: Level 2A - Data Object Headers +The header information of an object is designed to encompass all the information about an object, except for +the data itself. This information includes the dataspace, datatype, information about how the data is stored +on disk (in external files, compressed, broken up in blocks, etc.), as well as other information used by the +library to speed up access to the data objects or maintain a file’s integrity. Information stored by user +applications as attributes is also stored in the object’s header. The header of each object is not necessarily +located immediately prior to the object’s data in the file and in fact may be located in any position in the +file. The order of the messages in an object header is not significant. + +Object headers are composed of a prefix and a set of messages. The prefix contains the information needed to +interpret the messages and a small amount of metadata about the object, and the messages contain the majority +of the metadata about the object. + +\subsection subsec_fmt2_dataobject_hdr_prefix IV.A.1 Disk Format: Level 2A1 - Data Object Headers + +\subsubsection subsubsec_fmt2_dataobject_hdr_prefix_one IV.A.1.a Version 1 Data Object Header Prefix +Header messages are aligned on 8-byte boundaries for version 1 object headers. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Version 1 Object Header
bytebytebytebyte
VersionReserved (zero)Total Number of Header Messages
Object Reference Count
Object Header Size
Header Message Type \#1Size of Header Message Data \#1
Header Message \#1 FlagsReserved (zero)

Header Message Data \#1

.
.
.
Header Message Type \#nSize of Header Message Data \#n
Header Message \#n FlagsReserved (zero)

Header Message Data \#n

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThis value is used to determine the format of the information in the object header. When the format of + the object header is changed, the version number is incremented and can be used to determine how the + information in the object header is formatted. This is version one (1) (there was no version zero (0)) + of the object header.
Total Number of Header MessagesThis value determines the total number of messages listed in object headers for this object. This value + includes the messages in continuation messages for this object.
Object Reference CountThis value specifies the number of “hard links” to this object within the current file. + References to the object from external files, “soft links” in this file and object + references in this file are not tracked.
Object Header SizeThis value specifies the number of bytes of header message data following this length field that + contain object header messages for this object header. This value does not include the size of object header + continuation blocks for this object elsewhere in the file.
Header Message \#n TypeThis value specifies the type of information included in the following header message data. The + message types for header messages are defined in sections below.
Size of Header Message \#n DataThis value specifies the number of bytes of header message data following the header message type and + length information for the current message. The size includes padding bytes to make the message a multiple + of eight bytes.
Header Message \#n FlagsThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
BitDescription
0If set, the message data is constant. This is used for messages like the datatype message of + a dataset.
1If set, the message is shared and stored in another location than the object header. + The Header Message Data field contains a Shared Message (described in the @ref + subsec_fmt2_dataobject_hdr_msg section below) and the Size of Header Message Data field contains + the size of that Shared Message.
2If set, the message should not be shared.
3If set, the HDF5 decoder should fail to open this object if it does not understand the + message’s type and the file is open with permissions allowing write access to the file. + (Normally, unknown messages can just be ignored by HDF5 decoders)
4If set, the HDF5 decoder should set bit 5 of this message’s flags (in other words, this + bit field) if it does not understand the message’s type and the object is modified in any + way. (Normally, unknown messages can just be ignored by HDF5 decoders)
5If set, this object was modified by software that did not understand this message. (Normally, + unknown messages should just be ignored by HDF5 decoders) (Can be used to invalidate an index + or a similar feature)
6If set, this message is shareable.
7If set, the HDF5 decoder should always fail to open this object if it does not understand the + message’s type (whether it is open for read-only or read-write access). (Normally, unknown + messages can just be ignored by HDF5 decoders)
+
Header Message \#n DataThe format and length of this field is determined by the header message type and size respectively. + Some header message types do not require any data and this information can be eliminated by setting the + length of the message to zero. The data is padded with enough zeros to make the size a multiple of + eight.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_prefix_two IV.A.1.b Version 2 Data Object Header Prefix +Note that the “total number of messages” field has been dropped from the data object header +prefix in this version. The number of messages in the data object header is just determined by the +messages encountered in all the object header blocks. + +Note also that the fields and messages in this version of data object headers have no alignment +or padding bytes inserted - they are stored packed together. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Version 2 Object Header
bytebytebytebyte
Signature
VersionFlagsThis space inserted only to align table nicely
Access time (optional)
Modification Time (optional)
Change Time (optional)
Birth Time (optional)
Maximum \# of compact attributes (optional)Minimum \# of dense attributes (optional)
Size of Chunk \#0 (variable size)This space inserted only to align table nicely
Header Message Type \#1Size of Header Message Data \#1Header Message \#1 Flags
Header Message \#1 Creation Order (optional)This space inserted only to align table nicely

Header Message Data \#1

.
.
.
Header Message Type \#nSize of Header Message Data \#nHeader Message \#n Flags
Header Message \#n Creation Order (optional)This space inserted only to align table nicely

Header Message Data \#n

Gap (optional, variable size)
Checksum
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “OHDR” is used to indicate the beginning of + an object header. This gives file consistency checking utilities a better chance of reconstructing + a damaged file.
VersionThis field has a value of 2 indicating version 2 of the object header.
FlagsThis field is a bit field indicating additional information about the object header. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Bit(s)Description
0-1This two bit field determines the size of the Size of Chunk \#0 field. The values are: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0The Size of Chunk \#0 field is 1 byte.
1The Size of Chunk \#0 field is 2 bytes.
2The Size of Chunk \#0 field is 4 bytes.
3The Size of Chunk \#0 field is 8 bytes.
2If set, attribute creation order is tracked.
3If set, attribute creation order is indexed.
4If set, non-default attribute storage phase change values are stored.
5If set, access, modification, change and birth times are stored.
6-7Reserved
Access TimeThis 32-bit value represents the number of seconds after the UNIX epoch when the object’s + raw data was last accessed (in other words, read or written). This field is present if bit 5 of + flags is set.
Modification TimeThis 32-bit value represents the number of seconds after the UNIX epoch when the object’s + raw data was last modified (in other words, written). This field is present if bit 5 of + flags is set.
Change TimeThis 32-bit value represents the number of seconds after the UNIX epoch when the object’s + metadata was last changed. This field is present if bit 5 of flags is set.
Birth TimeThis 32-bit value represents the number of seconds after the UNIX epoch when the object was + created. This field is present if bit 5 of flags is set.
Maximum \# of compact attributesThis is the maximum number of attributes to store in the compact format before switching to the + indexed format. This field is present if bit 4 of flags is set.
Minimum \# of dense attributesThis is the minimum number of attributes to store in the indexed format before switching to the + compact format. This field is present if bit 4 of flags is set.
Size of Chunk \#0This unsigned value specifies the number of bytes of header message data following this field + that contain object header information. This value does not include the size of object header + continuation blocks for this object elsewhere in the file. The length of this field varies + depending on bits 0 and 1 of the flags field.
Header Message \#n TypeSame format as version 1 of the object header, described above.
Size of Header Message \#n DataThis value specifies the number of bytes of header message data following the header message + type and length information for the current message. The size of messages in this version does + not include any padding bytes.
Header Message \#n FlagsSame format as version 1 of the object header, described above.
Header Message \#n Creation OrderThis field stores the order that a message of a given type was created in.
+ This field is present if bit 2 of flags is set.
Header Message \#n DataSame format as version 1 of the object header, described above.
GapA gap in an object header chunk is inferred by the end of the messages for the chunk before the + beginning of the chunk’s checksum. Gaps are always smaller than the size of an object header + message prefix (message type + message size + message flags).
+ Gaps are formed when a message (typically an attribute message) in an earlier chunk is deleted + and a message from a later chunk that does not quite fit into the free space is moved into the + earlier chunk.
ChecksumThis is the checksum for the object header chunk.
+ +The header message types and the message data associated with them compose the critical "meta-data" about +each object. Some header messages are required for each object while others are optional. Some optional +header messages may also be repeated several times in the header itself, the requirements and number of +times allowed in the header will be noted in each header message description below. + +\subsection subsec_fmt2_dataobject_hdr_msg IV.A.2 Disk Format: Level 2A2 - Data Object Header Messages +Data object header messages are small pieces of metadata that are stored in the data object header for +each object in an HDF5 file. Data object header messages provide the metadata required to describe an +object and its contents, as well as optional pieces of metadata that annotate the meaning or purpose of +the object. + +Data object header messages are either stored directly in the data object header for the object or are +shared between multiple objects in the file. When a message is shared, a flag in the Message Flags +indicates that the actual Message Data portion of that message is stored in another location +(such as another data object header, or a heap in the file) and the Message Data field +contains the information needed to locate the actual information for the message. + +The format of shared message data is described here: + + + + + + + + + + + + + + + + + + + +
Shared Message (Version 1)
bytebytebytebyte
VersionTypeReserved (zero)
Reserved (zero)

AddressO

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number is used when there are changes in the format of a shared object message and is + described here: + + + + + + + + + + + + + +
VersionDescription
0Never used.
1Used by the library before version 1.6.1.
TypeThe type of shared message location: + + + + + + + + + +
ValueDescription
0Message stored in another object’s header (a committed message).
AddressThe address of the object header containing the message to be shared.
+ + + + + + + + + + + + + + + + + +
Shared Message (Version 2)
bytebytebytebyte
VersionTypeThis space inserted only to align table nicely

AddressO

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number is used when there are changes in the format of a shared object message and is + described here: + + + + + + + + + +
VersionDescription
2Used by the library of version 1.6.1 and after.
TypeThe type of shared message location: + + + + + + + + + +
ValueDescription
0Message stored in another object’s header (a committed message).
AddressThe address of the object header containing the message to be shared.
+ + + + + + + + + + + + + + + + + +
Shared Message (Version 3)
bytebytebytebyte
VersionTypeThis space inserted only to align table nicely
Location (variable size)
+ + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number indicates changes in the format of shared object message and is described here: + + + + + + + + + +
VersionDescription
3Used by the library of version 1.8 and after. In this version, the Type field can + indicate that the message is stored in the fractal heap.
TypeThe type of shared message location: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0Message is not shared and is not shareable.
1Message stored in file’s shared object header message heap + (a shared message).
2Message stored in another object’s header (a committed message).
3Message stored is not shared, but is shareable.
LocationThis field contains either a Size of Offsets-bytes address of the object header + containing the message to be shared, or an 8-byte fractal heap ID for the message in the + file’s shared object header message heap.
+ +The following is a list of currently defined header messages: + +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_nil IV.A.2.a. The NIL Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: NIL
Header Message Type: 0x0000
Length: Varies
Status: Optional; may be repeated.
Description:The NIL message is used to indicate a message which is to be ignored when reading the header messages + for a data object. [Possibly one which has been deleted for some reason.]
Format of Data: Unspecified
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_simple IV.A.2.b. The Dataspace Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Dataspace
Header Message Type: 0x0001
Length: Varies according to the number of dimensions, as described in the following + table.
Status: Required for dataset objects; may not be repeated.
Description:The dataspace message describes the number of dimensions (in other words, “rank”) and size + of each dimension that the data object has. This message is only used for datasets which have a + simple, rectilinear, array-like layout; datasets requiring a more complex layout are not yet supported.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Dataspace Message - Version 1
bytebytebytebyte
VersionDimensionalityFlagsReserved
Reserved
Dimension \#1 SizeL

.
.
.

Dimension \#n SizeL


Dimension \#1 Maximum SizeL

.
.
.

Dimension \#n Maximum SizeL


Permutation Index \#1L

.
.
.

Permutation Index \#nL

+\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Version This value is used to determine the format of the Dataspace Message. When the format of the + information in the message is changed, the version number is incremented and can be used to determine + how the information in the object header is formatted. This document describes version one (1) (there + was no version zero (0)).
DimensionalityThis value is the number of dimensions that the data object has.
FlagsThis field is used to store flags to indicate the presence of parts of this message. Bit 0 (the least + significant bit) is used to indicate that maximum dimensions are present. Bit 1 is used to indicate + that permutation indices are present.
Dimension \#n SizeThis value is the current size of the dimension of the data as stored in the file. The first dimension + stored in the list of dimensions is the slowest changing dimension and the last dimension stored is the + fastest changing dimension.
Dimension \#n Maximum SizeThis value is the maximum size of the dimension of the data as stored in the file. This value may be + the special “@ref FMT2UnlimitedDim "unlimited size"” which indicates that the data + may expand along this dimension indefinitely. If these values are not stored, the maximum size of each + dimension is assumed to be the dimension’s current size.
Permutation Index \#nThis value is the index permutation used to map each dimension from the canonical representation to an + alternate axis for each dimension. If these values are not stored, the first dimension stored in the list + of dimensions is the slowest changing dimension and the last dimension stored is the fastest changing + dimension.
+ +Version 2 of the dataspace message dropped the optional permutation index value support, as it was never +implemented in the HDF5 Library: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Dataspace Message - Version 2
bytebytebytebyte
VersionDimensionalityFlagsType
Dimension \#1 SizeL

.
.
.

Dimension \#n SizeL


Dimension \#1 Maximum SizeL

.
.
.

Dimension \#n Maximum SizeL

+\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Version This value is used to determine the format of the Dataspace Message. This field should be ‘2’ + for version 2 format messages.
DimensionalityThis value is the number of dimensions that the data object has.
FlagsThis field is used to store flags to indicate the presence of parts of this message. Bit 0 (the least + significant bit) is used to indicate that maximum dimensions are present.
Typeindicates the type of the dataspace: + + + + + + + + + + + + + + + + + +
ValueDescription
0A scalar dataspace; in other words, a dataspace with a single, dimensionless element.
1A simple dataspace; in other words, a dataspace with a rank > 0 and an appropriate \# + of dimensions.
2A null dataspace; in other words, a dataspace with no elements.
Dimension \#n SizeThis value is the current size of the dimension of the data as stored in the file. The first dimension + stored in the list of dimensions is the slowest changing dimension and the last dimension stored is the + fastest changing dimension.
Dimension \#n Maximum SizeThis value is the maximum size of the dimension of the data as stored in the file. This value may be + the special “@ref FMT2UnlimitedDim "unlimited size"” which indicates that the data + may expand along this dimension indefinitely. If these values are not stored, the maximum size of each + dimension is assumed to be the dimension’s current size.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_linkinfo IV.A.2.c. The Link Info Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Link Info
Header Message Type: 0x002
Length: Varies
Status: Optional; may not be repeated.
Description:The link info message tracks variable information about the current state of the links for a + “new style” group’s behavior. Variable information will be stored in this + message and constant information will be stored in the @ref + subsubsec_fmt2_dataobject_hdr_msg_groupinfo message.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Link Info
bytebytebytebyte
VersionFlagsThis space inserted only to align table nicely

Maximum Creation Index (8 bytes, optional)


Fractal Heap AddressO


Address of v2 B-tree for Name IndexO


Address of v2 B-tree for Creation Order IndexO (optional)

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number for this message. This document describes version 0.
FlagsThis field determines various optional aspects of the link info message: + + + + + + + + + + + + + + + + + +
BitDescription
0If set, creation order for the links is tracked.
1If set, creation order for the links is indexed.
2-7Reserved
Maximum Creation IndexThis 64-bit value is the maximum creation order index value stored for a link in this group.
+ This field is present if bit 0 of flags is set.
Fractal Heap AddressThis is the address of the fractal heap to store dense links. Each link stored in the fractal heap + is stored as a @ref subsubsec_fmt2_dataobject_hdr_msg_link.
+ If there are no links in the group, or the group’s links are stored “compactly” + (as object header messages), this value will be the @ref FMT2UndefinedAddress "undefined address".
Address of v2 B-tree for Name IndexThis is the address of the version 2 B-tree to index names of links.
+ If there are no links in the group, or the group’s links are stored “compactly” + (as object header messages), this value will be the @ref FMT2UndefinedAddress "undefined address".
Address of v2 B-tree for Creation Order IndexThis is the address of the version 2 B-tree to index creation order of links.
+ If there are no links in the group, or the group’s links are stored “compactly” + (as object header messages), this value will be the @ref FMT2UndefinedAddress "undefined address".
+ This field exists if bit 1 of flags is set.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_dtmessage IV.A.2.d. The Datatype Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Datatype
Header Message Type: 0x0003
Length: Variable
Status: Required for dataset or committed datatype (formerly named datatype) + objects; may not be repeated.
Description:The datatype message defines the datatype for each element of a dataset or a common datatype for + sharing between multiple datasets. A datatype can describe an atomic type like a fixed- or + floating-point type or more complex types like a C struct (compound datatype), array (array datatype) + or C++ vector (variable-length datatype).
+ Datatype messages that are part of a dataset object do not describe how elements are related to one + another; the dataspace message is used for that purpose. Datatype messages that are part of a committed + datatype (formerly named datatype) message describe a common datatype that can be shared by multiple + datasets in the file.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + +
Datatype Message
bytebytebytebyte
Class and VersionClass Bit Field, Bits 0-7Class Bit Field, Bits 8-15Class Bit Field, Bits 16-23
Size


Properties

/
+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Class and VersionThe version of the datatype message and the datatype’s class information are packed together in + this field. The version number is packed in the top 4 bits of the field and the class is contained + in the bottom 4 bits.
+ The version number information is used for changes in the format of the datatype message and is + described here: + + + + + + + + + + + + + + + + + + + + + +
VersionDescription
0Never used
1Used by early versions of the library to encode compound datatypes with explicit array fields. + See the compound datatype description below for further details.
2Used when an array datatype needs to be encoded.
3Used when a VAX byte-ordered type needs to be encoded. Packs various other datatype classes more + efficiently also.

+ The class of the datatype determines the format for the class bit field and properties portion of the + datatype message, which are described below. The following classes are currently defined: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0Fixed-Point
1Floating-Point
2Time
3String
4Bit field
5Opaque
6Compound
7Reference
8Enumerated
9Variable-Length
10Array
+
Class Bit FieldsThe information in these bit fields is specific to each datatype class and is described below. + All bits not defined for a datatype class are set to zero.
SizeThe size of a datatype element in bytes.
PropertiesThis variable-sized sequence of bytes encodes information specific to each datatype class and is + described for each class below. If there is no property information specified for a datatype class, + the size of this field is zero bytes.
+ +Class specific information for Fixed-Point Numbers (Class 0): + + + + + + + + + + + + + + + + + + + + + + +
Fixed-point Bit Field Description
BitsMeaning
0Byte Order. If zero, byte order is little-endian; otherwise, byte order is big endian.
1, 2Padding type. Bit 1 is the lo_pad type and bit 2 is the hi_pad type. If a datum has + unused bits at either end, then the lo_pad or hi_pad bit is copied to those locations.
3Signed. If this bit is set then the fixed-point number is in 2’s complement form.
4-23Reserved (zero).
+ + + + + + + + + + + + + +
+ Fixed-Point Property Descriptions +
ByteByteByteByte
Bit OffsetBit Precision
+ + + + + + + + + + + + + + +
Field NameDescription
Bit OffsetThe bit offset of the first significant bit of the fixed-point value within the datatype. The + bit offset specifies the number of bits “to the right of” the value (which are + set to the lo_pad bit value).
Bit PrecisionThe number of bits of precision of the fixed-point value within the datatype. This value, + combined with the datatype element’s size and the Bit Offset field specifies the number + of bits “to the left of” the value (which are set to the hi_pad bit value).
+ +Class specific information for Floating-Point Numbers (Class 1): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Floating-Point Bit Field Description
BitsMeaning
0Byte Order. These two non-contiguous bits specify the “endianness” of + the bytes in the datatype element. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Bit 6Bit 0Description
00Byte order is little-endian
01Byte order is big-endian
10Reserved
11Byte order is VAX-endian
1, 2, 3Padding type. Bit 1 is the low bits pad type, bit 2 is the high bits pad type, and bit + 3 is the internal bits pad type. If a datum has unused bits at either end or between the sign bit, exponent, + or mantissa, then the value of bit 1, 2, or 3 is copied to those locations.
4-5Mantissa Normalization. This 2-bit bit field specifies how the most significant bit of + the mantissa is managed. + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0No normalization
1The most significant bit of the mantissa is always set (except for 0.0).
2The most significant bit of the mantissa is not stored, but is implied to be set.
3Reserved.
6-7Reserved (zero).
8-15Sign Location. This is the bit position of the sign bit. Bits are numbered with the least + significant bit zero.
16-23Reserved (zero).
+ + + + + + + + + + + + + + + + + + + + + + +
Floating-Point Property Description
ByteByteByteByte
Bit OffsetBit Precision
Exponent LocationExponent SizeMantissa LocationMantissa Size
Exponent Bias
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameProperty Description
Bit OffsetThe bit offset of the first significant bit of the floating-point value within the datatype. The + bit offset specifies the number of bits “to the right of” the value.
Bit PrecisionThe number of bits of precision of the floating-point value within the datatype.
Exponent LocationThe bit position of the exponent field. Bits are numbered with the least significant + bit number zero.
Exponent SizeThe size of the exponent field in bits.
Mantissa LocationThe bit position of the mantissa field. Bits are numbered with the least significant bit number + zero.
Mantissa SizeThe size of the mantissa field in bits.
Exponent BiasThe bias of the exponent field.
+ +Class specific information for Time (Class 2): + + + + + + + + + + + + + + +
Time Bit Field Description
BitsMeaning
0Byte Order. If zero, byte order is little-endian; otherwise, byte order is big endian.
1-23Reserved (zero).
+
+ + + + + + + + + + +
Time Property Description
ByteByte
Bit Precision
+
+ + + + + + + + + + +
Field NameDescription
Bit PrecisionThe number of bits of precision of the time value.
+
+ +Class specific information for Strings (Class 3): + + + + + + + + + + + + + + + + + +
String Bit Field Description
BitsMeaning
0-3Padding type. This four-bit value determines the type of padding to use for the + string. The values are: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0Null Terminate: A zero byte marks the end of the string and is guaranteed to be present after + converting a long string to a short string. When converting a short string to a long string the + value is padded with additional null characters as necessary.
1Null Pad: Null characters are added to the end of the value during conversions from short values + to long values but conversion in the opposite direction simply truncates the value.
2Space Pad: Space characters are added to the end of the value during conversions from short values + to long values but conversion in the opposite direction simply truncates the value. This is the + Fortran representation of the string.
3-15Reserved.
+
4-7Character Set. The character set used to encode the string. + + + + + + + + + + + + + + + + + +
ValueDescription
0ASCII character set encoding
1UTF-8 character set encoding
2-15Reserved
8-23Reserved (zero).
+ +There are no properties defined for the string class. + +Class specific information for Bitfields (Class 4): + + + + + + + + + + + + + + + + + + +
Bitfield Bit Field Description
BitsMeaning
0Byte Order. If zero, byte order is little-endian; otherwise, byte order is big endian.
1, 2Padding type. Bit 1 is the lo_pad type and bit 2 is the hi_pad type. If a datum has + unused bits at either end, then the lo_pad or hi_pad bit is copied to those locations.
3-23Reserved (zero).
+ + + + + + + + + + + + + +
Bit Field Property Description
ByteByteByteByte
Bit OffsetBit Precision
+ + + + + + + + + + + + + + +
Field NameDescription
Bit OffsetThe bit offset of the first significant bit of the bitfield within the datatype. The bit + offset specifies the number of bits “to the right of” the value.
Bit PrecisionThe number of bits of precision of the bit field within the datatype.
+ +Class specific information for Opaque (Class 5): + + + + + + + + + + + + + + +
Opaque Bit Field Description
BitsMeaning
0-7Length of ASCII tag in bytes.
8-23Reserved (zero).
+ + + + + + + + + + + + +
Opaque Property Description
ByteByteByteByte

ASCII Tag

+ + + + + + + + + + +
Field NameDescription
ASCII TagThis NUL-terminated string provides a description for the opaque type. It is NUL-padded to a + multiple of 8 bytes.
+ +Class specific information for Compound Types (Class 6): + + + + + + + + + + + + + + +
Compound Bit Field Description
BitsMeaning
0-15Number of Members. This field contains the number of members defined for the + compound datatype. The member definitions are listed in the Properties field of the data type + message.
16-23Reserved (zero).
+ +The Properties field of a compound datatype is a list of the member definitions of the compound datatype. +The member definitions appear one after another with no intervening bytes. The member types are described +with a (recursively) encoded datatype message. + +Note that the property descriptions are different for different versions of the datatype version. Additionally +note that the version 0 properties are deprecated and has been replaced with later encodings in +versions of the HDF5 library from the 1.4 release onward. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Compound Properties Description for Datatype Version 1
ByteByteByteByte


Name

/
Byte Offset of Member
DimensionalityReserved (zero)
Dimension Permutation
Reserved (zero)
Dimension \#1 Size (required)
Dimension \#2 Size (required)
Dimension \#3 Size (required)
Dimension \#4 Size (required)

Member Type Message

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
NameThis NUL-terminated string provides a description for the opaque type. It is NUL-padded to a + multiple of 8 bytes.
Byte Offset of MemberThis is the byte offset of the member within the datatype.
DimensionalityIf set to zero, this field indicates a scalar member. If set to a value greater than zero, + this field indicates that the member is an array of values. For array members, the size of + the array is indicated by the ‘Size of Dimension n’ field in this message.
Dimension PermutationThis field was intended to allow an array field to have its dimensions permuted, but this was + never implemented. This field should always be set to zero.
Dimension \#n SizeThis field is the size of a dimension of the array field as stored in the file. The first + dimension stored in the list of dimensions is the slowest changing dimension and the last + dimension stored is the fastest changing dimension.
Member Type MessageThis field is a datatype message describing the datatype of the member.
+ + + + + + + + + + + + + + + + + + +
Compound Properties Description for Datatype Version 2
ByteByteByteByte

Name

Byte Offset of Member

Member Type Message

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
NameThis NUL-terminated string provides a description for the opaque type. It is NUL-padded to a multiple + of 8 bytes.
Byte Offset of MemberThis is the byte offset of the member within the datatype.
Member Type MessageThis field is a datatype message describing the datatype of the member.
+ + + + + + + + + + + + + + + + + + +
Compound Properties Description for Datatype Version 3
ByteByteByteByte

Name

Byte Offset of Member

Member Type Message

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
NameThis NUL-terminated string provides a description for the opaque type. It is not NUL-padded to a + multiple of 8 bytes.
Byte Offset of MemberThis is the byte offset of the member within the datatype. The field size is the minimum number of bytes + necessary, based on the size of the datatype element. For example, a datatype element size of less than + 256 bytes uses a 1 byte length, a datatype element size of 256-65535 bytes uses a 2 byte length, and + so on.
Member Type MessageThis field is a datatype message describing the datatype of the member.
+ +Class specific information for Reference (Class 7): + + + + + + + + + + + + + + +
Reference Bit Field Description
BitsMeaning
0-3Type. This four-bit value contains the type of reference described. The values defined are: + + + + + + + + + + + + + + + + + +
ValueDescription
0Object Reference: A reference to another object in this HDF5 file.
1Dataset Region Reference: A reference to a region within a dataset in this HDF5 file.
2-15Reserved
+
4-23Reserved (zero).
+ +There are no properties defined for the reference class. + +Class specific information for Enumeration (Class 8): + + + + + + + + + + + + + + +
Enumeration Bit Field Description
BitsMeaning
0-15Number of Members. The number of name/value pairs defined for the enumeration type.
16-23Reserved (zero).
+ + + + + + + + + + + + + + + + + + +
Enumeration Property Description for Datatype Versions 1 & 2
ByteByteByteByte

Base Type
/

Names


Values

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
Base TypeEach enumeration type is based on some parent type, usually an integer. The information + for that parent type is described recursively by this field.
NamesThe name for each name/value pair. Each name is stored as a null terminated ASCII string + in a multiple of eight bytes. The names are in no particular order.
ValuesThe list of values in the same order as the names. The values are packed (no inter-value + padding) and the size of each value is determined by the parent type.
+ + + + + + + + + + + + + + + + + + +
Enumeration Property Description for Datatype Versions 3
ByteByteByteByte

Base Type
/

Names


Values

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
Base TypeEach enumeration type is based on some parent type, usually an integer. The information + for that parent type is described recursively by this field.
NamesThe name for each name/value pair. Each name is stored as a null terminated ASCII string, + not padded to a multiple of eight bytes. The names are in no particular order.
ValuesThe list of values in the same order as the names. The values are packed (no inter-value + padding) and the size of each value is determined by the parent type.
+ +Class specific information for Variable-Length (Class 9): + + + + + + + + + + + + + + + + + + + + + + +
Variable-Length Bit Field Description
BitsMeaning
0-3Type. This four-bit value contains the type of variable-length datatype described. + The values defined are: + + + + + + + + + + + + + + + + + +
ValueDescription
0Sequence: A variable-length sequence of any datatype. Variable-length sequences do not + have padding or character set information.
1String: A variable-length sequence of characters. Variable-length strings have padding and + character set information.
2-15Reserved
4-7Padding type. (variable-length string only). This four-bit value determines the + type of padding used for variable-length strings. The values are the same as for the string padding + type, as follows: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
00 Null terminate: A zero byte marks the end of a string and is guaranteed to be present after + converting a long string to a short string. When converting a short string to a long string, + the value is padded with additional null characters as necessary.
1Null pad: Null characters are added to the end of the value during conversion from a short string to + a longer string. Conversion from a long string to a shorter string simply truncates the value.
2Space pad: Space characters are added to the end of the value during conversion from a short string + to a longer string. Conversion from a long string to a shorter string simply truncates the value. + This is the Fortran representation of the string.
3-15Reserved
+ This value is set to zero for variable-length sequences.
8-11Character Set. (variable-length string only) This four-bit value specifies the + character set to be used for encoding the string: + + + + + + + + + + + + + + + + + +
ValueDescription
0ASCII character set encoding.
1UTF-8 character set encoding.
2-15Reserved
+ This value is set to zero for variable-length sequences.
12-23Reserved (zero).
+ + + + + + + + + + + + +
Variable-Length Property Description
ByteByteByteByte

Base Type

+ + + + + + + + + + +
Field NameDescription
Base TypeEach variable-length type is based on some parent type. The information for that parent + type is described recursively by this field.
+ +Class specific information for Array (Class 10): + +There are no bit fields defined for the array class. + +Note that the dimension information defined in the property for this datatype class is independent of +dataspace information for a dataset. The dimension information here describes the dimensionality of the +information within a data element (or a component of an element, if the array datatype is nested within +another datatype) and the dataspace for a dataset describes the location of the elements in a dataset. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Array Property Description for Datatype Version 2
ByteByteByteByte
DimensionalityReserved (zero)
Dimension \#1 Size
.
.
.
Dimension \#n Size
Permutation Index \#1
.
.
.
Permutation Index \#n

Base Type

+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
DimensionalityThis value is the number of dimensions that the array has.
Dimension \#n SizeThis value is the size of the dimension of the array as stored in the file. The first dimension + stored in the list of dimensions is the slowest changing dimension and the last dimension stored + is the fastest changing dimension.
Permutation Index \#nThis value is the index permutation used to map each dimension from the canonical representation + to an alternate axis for each dimension. Currently, dimension permutations are not supported and + these indices should be set to the index position minus one (i.e. the first dimension should be + set to 0, the second dimension should be set to 1, and so on.)
Base TypeEach array type is based on some parent type. The information for that parent type is described + recursively by this field.
+ + + + + + + + + + + + + + + + + + + + + + + + + +
Array Property Description for Datatype Version 3
ByteByteByteByte
DimensionalityThis space inserted only to align table nicely
Dimension \#1 Size
.
.
.
Dimension \#n Size

Base Type

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
DimensionalityThis value is the number of dimensions that the array has.
Dimension \#n SizeThis value is the size of the dimension of the array as stored in the file. The first dimension + stored in the list of dimensions is the slowest changing dimension and the last dimension stored + is the fastest changing dimension.
Base TypeEach array type is based on some parent type. The information for that parent type is described + recursively by this field.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_ofvmessage IV.A.2.e. Data Storage - Fill Value (Old) Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Fill Value (old)
Header Message Type: 0x0004
Length: Varies
Status: Optional; may not be repeated.
Description:The fill value message stores a single data value which is returned to the application when an + uninitialized data element is read from a dataset. The fill value is interpreted with the same + datatype as the dataset. If no fill value message is present then a fill value of all zero bytes + is assumed.
+ This fill value message is deprecated in favor of the “new” fill value message (Message + Type 0x0005) and is only written to the file for forward compatibility with versions of the HDF5 + Library before the 1.6.0 version. Additionally, it only appears for datasets with a user-defined + fill value (as opposed to the library default fill value or an explicitly set “undefined” + fill value).
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + +
Fill Value Message (Old)
bytebytebytebyte
Size (4 bytes)

Fill Value (optional, variable size)

+
+ + + + + + + + + + + + + +
Field NameDescription
SizeThis is the size of the Fill Value field in bytes.
Fill ValueThe fill value. The bytes of the fill value are interpreted using the same datatype as for the dataset.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_fvmessage IV.A.2.f. The Data Storage - Fill Value Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Fill Value
Header Message Type: 0x0005
Length: Varies
Status: Required for dataset objects; may not be repeated.
Description:The fill value message stores a single data value which is returned to the application when an + uninitialized data element is read from a dataset. The fill value is interpreted with the same + datatype as the dataset.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + +
Fill Value Message - Versions 1 & 2
bytebytebytebyte
VersionSpace Allocation TimeFill Value Write TimeFill Value Defined
Size (optional)

Fill Value (optional, variable size)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number information is used for changes in the format of the fill value message and + is described here: + + + + + + + + + + + + + + + + + + + + + +
VersionDescription
0Never used
1Initial version of this message.
2In this version, the Size and Fill Value fields are only present if the Fill Value Defined + field is set to 1.
3This version packs the other fields in the message more efficiently than version 2.
+
Space Allocation TimeWhen the storage space for the dataset’s raw data will be allocated. The allowed values are: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0Not used
1Early allocation. Storage space for the entire dataset should be allocated in the file + when the dataset is created.
2Late allocation. Storage space for the entire dataset should not be allocated until the + dataset is written to.
3Incremental allocation. Storage space for the dataset should not be allocated until the + portion of the dataset is written to. This is currently used in conjunction with chunked + data storage for datasets.
Fill Value Write TimeAt the time that storage space for the dataset’s raw data is allocated, this value indicates + whether the fill value should be written to the raw data storage elements. The allowed values are: + + + + + + + + + + + + + + + + + +
ValueDescription
0On allocation. The fill value is always written to the raw data storage when the storage + space is allocated.
1Never. The fill value should never be written to the raw data storage.
2Fill value written if set by user. The fill value will be written to the raw data storage + when the storage space is allocated only if the user explicitly set the fill value. If the + fill value is the library default or is undefined, it will not be written to the raw data storage.
Fill Value DefinedThis value indicates if a fill value is defined for this dataset. If this value is 0, the fill + value is undefined. If this value is 1, a fill value is defined for this dataset. For version 2 + or later of the fill value message, this value controls the presence of the Size and Fill field.
SizeThis is the size of the Fill Value field in bytes. This field is not present if the Version + field is greater than 1 and the Fill Value Defined field is set to 0.
Fill ValueThe fill value. The bytes of the fill value are interpreted using the same datatype as for the + dataset. This field is not present if the Version field is greater than 1 and the Fill Value + Defined field is set to 0.
+ + + + + + + + + + + + + + + + + + + + + +
Fill Value Message - Versions 3
bytebytebytebyte
VersionFlagsFill Value Write TimeThis space inserted only to align table nicely
Size (optional)

Fill Value (optional, variable size)

+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number information is used for changes in the format of the fill value message and + is described here: + + + + + + + + + + + + + + + + + + + + + +
VersionDescription
0Never used
1Initial version of this message.
2In this version, the Size and Fill Value fields are only present if the Fill Value Defined + field is set to 1.
3This version packs the other fields in the message more efficiently than version 2.
+
FlagsWhen the storage space for the dataset’s raw data will be allocated. The allowed values are: + + + + + + + + + + + + + + + + + + + + + + + + + +
BitsDescription
0-1Space Allocation Time, with the same values as versions 1 and 2 of the message.
2-3Fill Value Write Time, with the same values as versions 1 and 2 of the message.
4Fill Value Undefined, indicating that the fill value has been marked as “undefined” + for this dataset. Bits 4 and 5 cannot both be set.
5Fill Value Defined, with the same values as versions 1 and 2 of the message. Bits 4 and 5 + cannot both be set.
6-7Reserved (zero).
SizeThis is the size of the Fill Value field in bytes. This field is not present if the Version + field is greater than 1 and the Fill Value Defined flag is set to 0.
Fill ValueThe fill value. The bytes of the fill value are interpreted using the same datatype as for the + dataset. This field is not present if the Version field is greater than 1 and the Fill Value + Defined flag is set to 0.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_link IV.A.2.g. The Link Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Link
Header Message Type: 0x0006
Length: Varies
Status: Optional; may be repeated.
Description:This message encodes the information for a link in a group’s object header, when the group is + storing its links “compactly”, or in the group’s fractal heap, when the group is + storing its links “densely”.
+ A group is storing its links compactly when the fractal heap address in the + @ref subsubsec_fmt2_dataobject_hdr_msg_linkinfo is set to the + @ref FMT2UndefinedAddress "undefined address" value.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Link Message
bytebytebytebyte
VersionFlagsLink type (optional)This space inserted only to align table nicely

Creation Order (8 bytes, optional)

Link Name Character Set (optional)Length of Link Name (variable size)This space inserted only to align table nicely
Link Name (variable size)

Link Information (variable size)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number for this message. This document describes version 1.
FlagsThis field contains information about the link and controls the presence of other fields below. + + + + + + + + + + + + + + + + + + + + + + + + + +
BitsDescription
0-1Determines the size of the Length of Link Name field. + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0The size of the Length of Link Name field is 1 byte.
1The size of the Length of Link Name field is 2 bytes.
2The size of the Length of Link Name field is 4 bytes.
3The size of the Length of Link Name field is 8 bytes.
2Creation Order Field Present: if set, the Creation Order field is present. If + not set, creation order information is not stored for links in this group.
3Link Type Field Present: if set, the link is not a hard link and the Link Type + field is present. If not set, the link is a hard link.
4Link Name Character Set Field Present: if set, the link name is not represented with the + ASCII character set and the Link Name Character Set field is present. If not set, + the link name is represented with the ASCII character set.
5-7Reserved (zero).
Link typeThis is the link class type and can be one of the following values: + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0A hard link (should never be stored in the file)
1A soft link.
2-63Reserved for future HDF5 internal use.
64An external link.
65-255Reserved, but available for user-defined link types.
+ This field is present if bit 3 of Flags is set.
Creation OrderThis 64-bit value is an index of the link’s creation time within the group. Values start at + 0 when the group is created an increment by one for each link added to the group. Removing a link + from a group does not change existing links’ creation order field.
+ This field is present if bit 2 of Flags is set.
Link Name Character SetThis is the character set for encoding the link’s name: + + + + + + + + + + + + + +
ValueDescription
0ASCII character set encoding (this should never be stored in the file)
1UTF-8 character set encoding
+ This field is present if bit 4 of Flags is set.
Length of link nameThis is the length of the link’s name. The size of this field depends on bits 0 and 1 of Flags.
Link nameThis is the name of the link, non-NULL terminated.
Link informationThe format of this field depends on the link type.
+ For hard links, the field is formatted as follows: + + + + + +
Size of Offsets bytes:The address of the object header for the object that the link points to.
+
+ For soft links, the field is formatted as follows: + + + + + + + + + +
Bytes 1-2:Length of soft link value.
Length of soft link value bytes:A non-NULL-terminated string storing the value of the soft link.
+
+ For external links, the field is formatted as follows: + + + + + + + + + +
Bytes 1-2:Length of external link value.
Length of external link value bytes:The first byte contains the version number in the upper 4 bits and flags in the lower 4 bits + for the external link. Both version and flags are defined to be zero in this document. The + remaining bytes consist of two NULL-terminated strings, with no padding between them. The first + string is the name of the HDF5 file containing the object linked to and the second string is the + full path to the object linked to, within the HDF5 file’s group hierarchy.
+
+ For user-defined links, the field is formatted as follows: + + + + + + + + + +
Bytes 1-2:Length of user-defined data.
Length of user-defined link value bytes:The data supplied for the user-defined link type.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_external IV.A.2.h. The Data Storage - External Data Files Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: External Data Files
Header Message Type: 0x0007
Length: Varies
Status: Optional; may not be repeated.
Description:The external data storage message indicates that the data for an object is stored outside the HDF5 + file. The filename of the object is stored as a Universal Resource Location (URL) of the actual + filename containing the data. An external file list record also contains the byte offset of the + start of the data within the file and the amount of space reserved in the file for that data.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + +
External File List Message
bytebytebytebyte
VersionReserved (zero)
Allocated SlotsUsed Slots

Heap AddressO


Slot Definitions...

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number information is used for changes in the format of External Data Storage Message + and is described here: + + + + + + + + + + + + + +
VersionDescription
0Never used.
1The current version used by the library.
Allocated SlotsThe total number of slots allocated in the message. Its value must be at least as large as the value + contained in the Used Slots field. (The current library simply uses the number of Used Slots for this + message)
Used SlotsThe number of initial slots which contain valid information.
Heap AddressThis is the address of a local name heap which contains the names for the external files. (The local + heap information can be found in Disk Format Level 1D in this document). The name at offset zero in + the heap is always the empty string.
Slot DefinitionsThe slot definitions are stored in order according to the array addresses they represent.
+ + + + + + + + + + + + + + + + + + +
External File List Slot
bytebytebytebyte

Name Offset in Local HeapL


File Offset in External Data FileL


Data Size in External FileL

+\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + +
Field NameDescription
Name Offset in Local HeapThe byte offset within the local name heap for the name of the file. File names are stored as a URL + which has a protocol name, a host name, a port number, and a file name: + protocol:port//host/file. If the protocol is omitted + then “file:” is assumed. If the port number is omitted then a default port for that protocol + is used. If both the protocol and the port number are omitted then the colon can also be omitted. If the double + slash and host name are omitted then “localhost” is assumed. The file name is the only mandatory part, + and if the leading slash is missing then it is relative to the application’s current working directory + (the use of relative names is not recommended).
Offset in External Data FileThis is the byte offset to the start of the data in the specified file. For files that contain data for + a single dataset this will usually be zero.
Data Size in External FileThis is the total number of bytes reserved in the specified file for raw data storage. For a file that + contains exactly one complete dataset which is not extendable, the size will usually be the exact size of + the dataset. However, by making the size larger one allows HDF5 to extend the dataset. The size can be set + to a value larger than the entire file since HDF5 will read zeros past the end of the file without failing.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_layout IV.A.2.i. The Data Storage - Layout Message + + + + + + + + + + + + + + + + + + + +
Header Message Name: Data Storage - Layout
Header Message Type: 0x0008
Length: Varies
Status: Required for datasets; may not be repeated.
Description: Data layout describes how the elements of a multi-dimensional array + are stored in the HDF5 file. Three types of data layout are supported: +
    +
  1. Contiguous: The array is stored in one contiguous area of the file. This layout requires that the + size of the array be constant: data manipulations such as chunking, compression, checksums or encryption + are not permitted. The message stores the total storage size of the array. The offset of an element from + the beginning of the storage area is computed as in a C array.
    2. +
  2. Chunked: The array domain is regularly decomposed into chunks, and each chunk is allocated and stored + separately. This layout supports arbitrary element traversals, compression, encryption, and checksums + (these features are described in other messages). The message stores the size of a chunk instead of the + size of the entire array; the size of the entire array can be calculated by traversing the B-tree that + stores the chunk addresses.
    3. +
  3. Compact: The array is stored in one contiguous block, as part of this object header messagei.
    4. +
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Data Layout Message (Versions 1 and 2)
bytebytebytebyte
VersionDimensionalityLayout ClassReserved (zero)
Reserved (zero)

Data AddressO (optional)

Dimension 0 Size
Dimension 1 Size
...
Dataset Element Size (optional)
Compact Data Size (optional)

Compact Data...(variable size, optional)

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number information is used for changes in the format of the data layout message and + is described here: + + + + + + + + + + + + + + + + + +
VersionDescription
0Never used.
1Used by version 1.4 and before of the library to encode layout information. Data space is + always allocated when the data set is created.
2Used by version 1.6.x of the library to encode layout information. Data space is allocated + only when it is necessary.
DimensionalityAn array has a fixed dimensionality. This field specifies the number of dimension size fields later + in the message. The value stored for chunked storage is 1 greater than the number of dimensions in + the dataset’s dataspace. For example, 2 is stored for a 1 dimensional dataset.
Layout ClassThe layout class specifies the type of storage for the data and how the other fields of the layout + message are to be interpreted. + + + + + + + + + + + + + + + + + +
ValueDescription
0Compact Storage
1Contiguous Storage
2Chunked Storage
Data AddressFor contiguous storage, this is the address of the raw data in the file. For chunked storage this is + the address of the v1 B-tree that is used to look up the addresses of the chunks. This field is not present + for compact storage. If the version for this message is greater than 1, the address may have the + @ref FMT2UndefinedAddress "undefined address" value, to indicate that storage has not yet been + allocated for this array.
Dimension \#n SizeFor contiguous and compact storage the dimensions define the entire size of the array while for chunked storage + they define the size of a single chunk. In all cases, they are in units of array elements (not bytes). The + first dimension stored in the list of dimensions is the slowest changing dimension and the last dimension + stored is the fastest changing dimension.
Dataset Element SizeThe size of a dataset element, in bytes. This field is only present for chunked storage.
Compact Data SizeThis field is only present for compact data storage. It contains the size of the raw data for the + dataset array, in bytes.
Compact DataThis field is only present for compact data storage. It contains the raw data for the dataset + array.
+ +Version 3 of this message re-structured the format into specific properties that are required for each layout class. + + + + + + + + + + + + + + + + +
Data Layout Message (Version 3)
bytebytebytebyte
VersionLayout ClassThis space inserted only to align table nicely

Properties (variable size)

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number information is used for changes in the format of layout message and is + described here:

+ + + + + + + + + +
VersionDescription
3Used by the version 1.6.3 and later of the library to store properties for each layout class.
Layout ClassThe layout class specifies how the other fields of the layout message are to be interpreted. + + + + + + + + + + + + + + + + + +
ValueDescription
0Compact Storage
1Contiguous Storage
2Chunked Storage
PropertiesThis variable-sized field encodes information specific to each layout class and is described below. If + there is no property information specified for a layout class, the size of this field is zero bytes.
+ +Class-specific information for compact layout (Class 0): (Note: The dimensionality information is in +the Dataspace message) + + + + + + + + + + + + + + + + +
Compact Storage Property Description
bytebytebytebyte
SizeThis space inserted only to align table nicely

Raw Data...(variable size)

+ + + + + + + + + + + + + + +
Field NameDescription
SizeThis field contains the size of the raw data for the dataset array, in bytes.
Raw DataThis field contains the raw data for the dataset array.
+ +Class-specific information for contiguous layout (Class 1): (Note: The dimensionality information is +in the Dataspace message) + + + + + + + + + + + + + + + +
Contiguous Storage Property Description
bytebytebytebyte

AddressO


SizeL

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + +
Field NameDescription
AddressThis is the address of the raw data in the file. The address may have the + @ref FMT2UndefinedAddress "undefined address" value, to indicate that storage has not + yet been allocated for this array.
SizeThis field contains the size allocated to store the raw data, in bytes.
+ +Class-specific information for chunked layout (Class 2): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Chunked Storage Property Description
bytebytebytebyte
DimensionalityThis space inserted only to align table nicely

AddressO

Dimension 0 Size
Dimension 1 Size
...
Dimension \#n Size
Dataset Element Size
+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Dimensionality>A chunk has a fixed dimensionality. This field specifies the number of dimension size fields + later in the message.
AddressThis is the address of the v1 B-tree that is used to look up the addresses of the chunks that + actually store portions of the array data. The address may have the + @ref FMT2UndefinedAddress "undefined address" value, to indicate + that storage has not yet been allocated for this array.
Dimension \#n SizeThese values define the dimension size of a single chunk, in units of array elements (not bytes). + The first dimension stored in the list of dimensions is the slowest changing dimension and the + last dimension stored is the fastest changing dimension.
Dataset Element SizeThe size of a dataset element, in bytes.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_bogus IV.A.2.j. The Bogus Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Bogus
Header Message Type: 0x0009
Length: 4 bytes
Status: For testing only; should never be stored in a valid file.
Description:This message is used for testing the HDF5 Library’s response to an “unknown” + message type and should never be encountered in a valid HDF5 file.
Format of Data: See the tables below.
+ + + + + + + + + + + + +
Bogus Message
bytebytebytebyte
Bogus Value
+ + + + + + + + + + +
Field NameDescription
Bogus ValueThis value should always be: 0xdeadbeef.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_groupinfo IV.A.2.k. The Group Info Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Group Info
Header Message Type: 0x000A
Length: Varies
Status: Optional; may not be repeated.
Description:This message stores information for the constants defining a “new style” group’s + behavior. Constant information will be stored in this message and variable information will be stored + in the @ref subsubsec_fmt2_dataobject_hdr_msg_linkinfo message.
+ Note: the “estimated entry” information below is used when determining the size of the + object header for the group when it is created.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + +
Group Info Message
bytebytebytebyte
VersionFlagsLink Phase Change: Maximum Compact Value (optional)
Link Phase Change: Minimum Dense Value (optional)Estimated Number of Entries (optional)
Estimated Link Name Length of Entries (optional)This space inserted only to align table nicely
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number for this message. This document describes version 0.
FlagsThis is the group information flag with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
0If set, link phase change values are stored.
1If set, the estimated entry information is non-default + and is stored.
2-7Reserved
Link Phase Change: Maximum Compact ValueThe is the maximum number of links to store “compactly” (in the group’s object header).
+ This field is present if bit 0 of Flags is set.
Link Phase Change: Minimum Dense ValueThis is the minimum number of links to store “densely” (in the group’s fractal + heap). The fractal heap’s address is located in the @ref subsubsec_fmt2_dataobject_hdr_msg_linkinfo + message.
+ This field is present if bit 0 of Flags is set.
Estimated Number of EntriesThis is the estimated number of entries in groups. If this field is not present, the default value of + 4 will be used for the estimated number of group entries.
+ This field is present if bit 1 of Flags is set.
Estimated Link Name Length of EntriesThis is the estimated length of entry name. If this field is not present, the default value of + 8 will be used for the estimated link name length of group entries.
+ This field is present if bit 1 of Flags is set.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_filter IV.A.2.l. The Data Storage - Filter Pipeline Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Data Storage - Filter Pipeline
Header Message Type: 0x000B
Length: Varies
Status: Optional; may not be repeated.
Description:This message describes the filter pipeline which should be applied to the data stream by providing filter + identification numbers, flags, a name, and client data.
+ This message may be present in the object headers of both dataset and group objects. For datasets, it + specifies the filters to apply to raw data. For groups, it specifies the filters to apply to the + group’s fractal heap. Currently, only datasets using chunked data storage use the filter pipeline + on their raw data.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + +
Filter Pipeline Message - Version 1
bytebytebytebyte
VersionNumber of FiltersReserved (zero)
Reserved (zero)

Filter Description List (variable size)

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number for this message. This table describes version 1.
Number of FiltersThe total number of filters described in this message. The maximum possible number of filters in a + message is 32.
Filter Description ListA description of each filter. A filter description appears in the next table.
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Filter Description
bytebytebytebyte
Filter IdentificationName Length
FlagsNumber of Values for Client Data

Name (variable size, optional)


Client Data (variable size, optional)

Padding (variable size, optional)
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Filter Identification ValueThis value, often referred to as a filter identifier, is designed to be a unique identifier for + the filter. Values from zero through 32,767 are reserved for filters supported by The HDF Group + in the HDF5 library and for filters requested and supported by third parties. Filters supported + by The HDF Group are documented immediately below. Information on 3rd-party filters can be found + at The HDF Group’s + Registered Filters page.
+ 1
To request a filter identifier, + please contact The HDF Group’s Help Desk at HDF Help Desk. + You will be asked to provide the following information: +
    +
  1. Contact information for the developer requesting the new identifier
    2. +
  2. A short description of the new filter
    3. +
  3. Links to any relevant information, including licensing information
    4. +

+ Values from 32768 to 65535 are reserved for non-distributed uses (for example, internal company usage) + or for application usage when testing a feature. The HDF Group does not track or document the use of + the filters with identifiers from this range.
+ The filters currently in library version 1.8.0 are listed below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IdentificationNameDescription
0N/AReserved
1deflateGZIP deflate compression
2shuffleData element shuffling
3fletcher32Fletcher32 checksum
4szipSZIP compression
5nbitN-bit packing
6scaleoffsetScale and offset encoded values
Name LengthEach filter has an optional null-terminated ASCII name and this field holds the length of the name + including the null termination padded with nulls to be a multiple of eight. If the filter has no name + then a value of zero is stored in this field.
FlagsThe flags indicate certain properties for a filter. The bit values defined so far are: + + + + + + + + + + + + + +
ValueDescription
0If set then the filter is an optional filter. During output, if an optional filter fails it will be + silently skipped in the pipeline.
1-15Reserved (zero)
Number of Client Data ValuesEach filter can store integer values to control how the filter operates. The number of entries + in the Client Data array is stored in this field.
NameIf the Name Length field is non-zero then it will contain the size of this field, padded + to a multiple of eight. This field contains a null-terminated, ASCII character string to serve as + a comment/name for the filter.
Client DataThis is an array of four-byte integers which will be passed to the filter function. The Client Data + Number of Values determines the number of elements in the array.
PaddingFour bytes of zeroes are added to the message at this point if the Client Data Number of Values field + contains an odd number.
+\anchor FMT2Footnote1Change 1 If you are reading an earlier version of this document, this +link may have changed. If the link does not work, use the latest version of this document on +The HDF Group’s github website, +HDF5 File Format Specification; the link there will always be correct. + + + + + + + + + + + + + + + + + +
Filter Pipeline Message - Version 2
bytebytebytebyte
VersionNumber of FiltersThis space inserted only to align table nicely

Filter Description List (variable size)

+ + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number for this message. This table describes version 2.
Number of FiltersThe total number of filters described in this message. The maximum possible number of filters in a + message is 32.
Filter Description ListA description of each filter. A filter description appears in the next table.
+ + + + + + + + + + + + + + + + + + + + + + + +
Filter Description
bytebytebytebyte
Filter IdentificationName Length (optional)
FlagsNumber Client Data Values

Name (variable size, optional)


Client Data (variable size, optional)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
Filter Identification ValueThis value, often referred to as a filter identifier, is designed to be a unique identifier for + the filter. Values from zero through 32,767 are reserved for filters supported by The HDF Group + in the HDF5 library and for filters requested and supported by third parties. Filters supported + by The HDF Group are documented immediately below. Information on 3rd-party filters can be found + at The HDF Group’s + Registered Filters page.
+ 1
To request a filter identifier, + please contact The HDF Group’s Help Desk at HDF Help Desk. + You will be asked to provide the following information: +
    +
  1. Contact information for the developer requesting the new identifier
    2. +
  2. A short description of the new filter
    3. +
  3. Links to any relevant information, including licensing information
    4. +

+ Values from 32768 to 65535 are reserved for non-distributed uses (for example, internal company usage) + or for application usage when testing a feature. The HDF Group does not track or document the use of + the filters with identifiers from this range.
+ The filters currently in library version 1.8.0 are listed below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IdentificationNameDescription
0N/AReserved
1deflateGZIP deflate compression
2shuffleData element shuffling
3fletcher32Fletcher32 checksum
4szipSZIP compression
5nbitN-bit packing
6scaleoffsetScale and offset encoded values
Name LengthEach filter has an optional null-terminated ASCII name and this field holds the length of the name + including the null termination padded with nulls to be a multiple of eight. If the filter has no name + then a value of zero is stored in this field.
+ Filters with IDs less than 256 (in other words, filters that are defined in this format documentation) + do not store the Name Length or Name fields.
FlagsThe flags indicate certain properties for a filter. The bit values defined so far are: + + + + + + + + + + + + + +
ValueDescription
0If set then the filter is an optional filter. During output, if an optional filter fails it will be + silently skipped in the pipeline.
1-15Reserved (zero)
Number of Client Data ValuesEach filter can store integer values to control how the filter operates. The number of entries + in the Client Data array is stored in this field.
NameIf the Name Length field is non-zero then it will contain the size of this field, padded + to a multiple of eight. This field contains a null-terminated, ASCII character string to serve as + a comment/name for the filter.
Client DataThis is an array of four-byte integers which will be passed to the filter function. The Client Data + Number of Values determines the number of elements in the array.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_attribute IV.A.2.m. The Attribute Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Attribute
Header Message Type: 0x000C
Length: Varies
Status: Optional; may be repeated.
Description:The Attribute message is used to store objects in the HDF5 file which are used as attributes, + or “metadata” about the current object. An attribute is a small dataset; it has a name, + a datatype, a dataspace, and raw data. Since attributes are stored in the object header, they should + be relatively small (in other words, less than 64KB). They can be associated with any type of object + which has an object header (groups, datasets, or committed (named) datatypes).
+ In 1.8.x versions of the library, attributes can be larger than 64KB. See the + “ @ref subsec_attribute_special ” section of the Attributes chapter in + the @ref UG for more information.
+ Note: Attributes on an object must have unique names: the HDF5 Library currently enforces this by + causing the creation of an attribute with a duplicate name to fail. Attributes on different + objects may have the same name, however.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Attribute Message (Version 1)
bytebytebytebyte
VersionReserved (zero)Name Size
Datatype SizeDataspace Size

Name (variable size)


Datatype (variable size)


Dataspace (variable size)


Data (variable size)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number information is used for changes in the format of the attribute message and is + described here: + + + + + + + + + + + + + +
VersionDescription
0Never used.
1Used by the library before version 1.6 to encode attribute message. This version does not + support shared datatypes.
Name SizeThe length of the attribute name in bytes including the null terminator. Note that the + Name field below may contain additional padding not represented by this field.
Datatype SizeThe length of the datatype description in the Datatype field below. Note that the + Datatype field may contain additional padding not represented by this field.
Dataspace SizeThe length of the dataspace description in the Dataspace field below. Note that the + Dataspace field may contain additional padding not represented by this field.
NameThe null-terminated attribute name. This field is padded with additional null characters to make it a + multiple of eight bytes.
TypeThe datatype description follows the same format as described for the datatype object header message. + This field is padded with additional zero bytes to make it a multiple of eight bytes.
SpaceThe dataspace description follows the same format as described for the dataspace object header message. + This field is padded with additional zero bytes to make it a multiple of eight bytes.
DataThe raw data for the attribute. The size is determined from the datatype and dataspace descriptions. + This field is not padded with additional bytes.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Attribute Message (Version 2)
bytebytebytebyte
VersionFlagsName Size
Datatype SizeDataspace Size

Name (variable size)


Datatype (variable size)


Dataspace (variable size)


Data (variable size)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number information is used for changes in the format of the attribute message and is + described here: + + + + + + + + + +
VersionDescription
2Used by the library of version 1.6.x and after to encode attribute messages. This version + supports shared datatypes. The fields of name, datatype, and dataspace are not padded with + additional bytes of zero.
Flags>This bit field contains extra information about interpreting the attribute message: + + + + + + + + + + + + + +
BitDescription
0If set, datatype is shared.
1If set, dataspace is shared.
Name Size>The length of the attribute name in bytes including the null terminator.
Datatype SizeThe length of the datatype description in the Datatype field below.
Dataspace SizeThe length of the dataspace description in the Dataspace field below.
NameThe null-terminated attribute name. This field is not padded with additional bytes.
DatatypeThe datatype description follows the same format as described for the datatype object + header message.
+ If the Flag field indicates this attribute’s datatype is shared, this field will + contain a “shared message” encoding instead of the datatype encoding.
+ This field is not padded with additional bytes.
DataspaceThe dataspace description follows the same format as described for the dataspace object + header message.
+ If the Flag field indicates this attribute’s dataspace is shared, this field will + contain a “shared message” encoding instead of the dataspace encoding.
+ This field is not padded with additional bytes.
DataThe raw data for the attribute. The size is determined from the datatype and dataspace + descriptions.
+ This field is not padded with additional zero bytes.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Attribute Message (Version 3)
bytebytebytebyte
VersionFlagsName Size
Datatype SizeDataspace Size
Name Character Set EncodingThis space inserted only to align table nicely

Name (variable size)


Datatype (variable size)


Dataspace (variable size)


Data (variable size)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number information is used for changes in the format of the attribute message and is + described here: + + + + + + + + + +
VersionDescription
2Used by the library of version 1.8.x and after to encode attribute messages. This version + supports attributes with non-ASCII names.
Flags>This bit field contains extra information about interpreting the attribute message: + + + + + + + + + + + + + +
BitDescription
0If set, datatype is shared.
1If set, dataspace is shared.
Name Size>The length of the attribute name in bytes including the null terminator.
Datatype SizeThe length of the datatype description in the Datatype field below.
Dataspace SizeThe length of the dataspace description in the Dataspace field below.
Name Character Set EncodingThe character set encoding for the attribute’s name: + + + + + + + + + + + + + +
ValueDescription
0ASCII character set encoding
1UTF-8 character set encoding
NameThe null-terminated attribute name. This field is not padded with additional bytes.
DatatypeThe datatype description follows the same format as described for the datatype object + header message.
+ If the Flag field indicates this attribute’s datatype is shared, this field will + contain a “shared message” encoding instead of the datatype encoding.
+ This field is not padded with additional bytes.
DataspaceThe dataspace description follows the same format as described for the dataspace object + header message.
+ If the Flag field indicates this attribute’s dataspace is shared, this field will + contain a “shared message” encoding instead of the dataspace encoding.
+ This field is not padded with additional bytes.
DataThe raw data for the attribute. The size is determined from the datatype and dataspace + descriptions.
+ This field is not padded with additional zero bytes.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_comment IV.A.2.n. The Object Comment Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Object Comment
Header Message Type: 0x000D
Length: Varies
Status: Optional; may not be repeated.
Description:The object comment is designed to be a short description of an object. An object comment is a sequence + of non-zero (\0) ASCII characters with no other formatting included by the library.
Format of Data: See the tables below.
+ + + + + + + + + + + + +
Name Message
bytebytebytebyte

Comment (variable size)

+
+ + + + + + + + + + +
Field NameDescription
NameA null terminated ASCII character string.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_omodified IV.A.2.o. The Object Modification Time (Old) Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Object Modification Time (Old)
Header Message Type: 0x000E
Length: Fixed
Status: Optional; may not be repeated.
Description:The object modification date and time is a timestamp which indicates (using ISO-8601 date and + time format) the last modification of an object. The time is updated when any object header + message changes according to the system clock where the change was posted. All fields of this + message should be interpreted as coordinated universal time (UTC).
+ This modification time message is deprecated in favor of the “new” + @ref subsubsec_fmt2_dataobject_hdr_msg_mod message and is no longer written to the file in + versions of the HDF5 Library after the 1.6.0 version.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + + +
Modification Time Message
bytebytebytebyte
Year
MonthDay of Month
HourMinute
SecondReserved
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
YearThe four-digit year as an ASCII string. For example, 1998.
MonthThe month number as a two digit ASCII string where January is 01 and December is + 12.
Day of MonthThe day number within the month as a two digit ASCII string. The first day of the month is + 01.
HourThe hour of the day as a two digit ASCII string where midnight is 00 and 11:00pm + is 23.
MinuteThe minute of the hour as a two digit ASCII string where the first minute of the hour is + 00 and the last is 59.
SecondThe second of the minute as a two digit ASCII string where the first second of the minute is + 00 and the last is 59.
ReservedThis field is reserved and should always be zero.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_shared IV.A.2.p. The Shared Message Table Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Shared Message Table
Header Message Type: 0x000F
Length: Fixed
Status: Optional; may not be repeated.
Description:This message is used to locate the table of shared object header message (SOHM) indexes. Each + index consists of information to find the shared messages from either the heap or object header. + This message is only found in the superblock extension.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + +
Shared Message Table Message
bytebytebytebyte
VersionThis space inserted only to align table nicely

Shared Object Header Message Table AddressO

Number of IndicesThis space inserted only to align table nicely
+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number for this message. This document describes version 0.
Shared Object Header Message Table AddressThis field is the address of the master table for shared object header message indexes.
Number of IndicesThis field is the number of indices in the master table.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_continuation IV.A.2.q. The Object Header Continuation Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Object Header Continuation
Header Message Type: 0x0010
Length: Fixed
Status: Optional; may be repeated.
Description:The object header continuation is the location in the file of a block containing more header messages + for the current data object. This can be used when header blocks become too large or are likely to + change over time.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + +
Object Header Continuation Message
bytebytebytebyte

OffsetO


LengthL

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + +
Field NameDescription
OffsetThis value is the address in the file where the header continuation block is located.
LengthThis value is the length in bytes of the header continuation block in the file.
+ +The format of the header continuation block that this message points to depends on the version of the +object header that the message is contained within. + +Continuation blocks for version 1 object headers have no special formatting information; they are +merely a list of object header message info sequences (type, size, flags, reserved bytes and data for +each message sequence). See the description of @ref subsubsec_fmt2_dataobject_hdr_prefix_one. + +Continuation blocks for version 2 object headers do have special formatting information as +described here (see also the description of @ref subsubsec_fmt2_dataobject_hdr_prefix_two): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Version 2 Object Header Continuation Block
bytebytebytebyte
Signature
Header Message Type \#1Size of Header Message Data \#1Header Message \#1 Flags
Header Message \#1 Creation Order (optional)This space inserted only to align table nicely

Header Message Data \#1

.
.
.
Header Message Type \#nSize of Header Message Data \#nHeader Message \#n Flags
Header Message \#n Creation Order (optional)This space inserted only to align table nicely

Header Message Data \#n

Gap (optional, variable size)
Checksum
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
SignatureThe ASCII character string “OCHK” is used to indicate the + beginning of an object header continuation block. This gives file consistency checking + utilities a better chance of reconstructing a damaged file.
Header Message \#n TypeSame format as version 1 of the object header, described above.
Size of Header Message \#n DataSame format as version 1 of the object header, described above.
Header Message \#n FlagsSame format as version 1 of the object header, described above.
Header Message \#n Creation OrderThis field stores the order that a message of a given type was created in.
+ This field is present if bit 2 of flags is set.
Header Message \#n DataSame format as version 1 of the object header, described above.
GapA gap in an object header chunk is inferred by the end of the messages for the chunk before the + beginning of the chunk’s checksum. Gaps are always smaller than the size of an object header + message prefix (message type + message size + message flags).
+ Gaps are formed when a message (typically an attribute message) in an earlier chunk is deleted + and a message from a later chunk that does not quite fit into the free space is moved into the + earlier chunk.
ChecksumThis is the checksum for the object header chunk.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_stmgroup IV.A.2.r. The Symbol Table Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Symbol Table Message
Header Message Type: 0x0011
Length: Fixed
Status: Required for “old style” groups; may not be repeated.
Description:Each “old style” group has a v1 B-tree and a local heap for storing symbol table entries, + which are located with this message.
Format of data: See the tables below.
+ + + + + + + + + + + + + + + +
Symbol Table Message
bytebytebytebyte

v1 B-tree AddressO


Local Heap AddressO

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + +
Field NameDescription
v1 B-tree AddressThis value is the address of the v1 B-tree containing the symbol table entries for the group.
Local Heap AddressThis value is the address of the local heap containing the link names for the symbol table + entries for the group.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_mod IV.A.2.s. The Object Modification Time Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Object Modification Time
Header Message Type: 0x0012
Length: Fixed
Status: Optional; may not be repeated.
Description:The object modification time is a timestamp which indicates the time of the last modification of + an object. The time is updated when any object header message changes according to the system clock + where the change was posted.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + +
Modification Time Message
bytebytebytebyte
VersionReserved (zero)
Seconds After UNIX Epoch
+ + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number is used for changes in the format of Object Modification Time and is described + here: + + + + + + + + + + + + + +
VersionDescription
0Never used.
1Used by Version 1.6.1 and after of the library to encode time. In this version, the time is + the seconds after Epoch.
Seconds After UNIX EpochA 32-bit unsigned integer value that stores the number of seconds since 0 hours, 0 minutes, + 0 seconds, January 1, 1970, Coordinated Universal Time.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_btreek IV.A.2.t. The B-tree ‘K’ Values Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: B-tree ‘K’ Values
Header Message Type: 0x0013
Length: Fixed
Status: Optional; may not be repeated.
Description:This message retrieves non-default ‘K’ values for internal and leaf nodes of a group + or indexed storage v1 B-trees. This message is only found in the superblock extension.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + +
B-tree ‘K’ Values Message
bytebytebytebyte
VersionIndexed Storage Internal Node KThis space inserted only to align table nicely
Group Internal Node KGroup Leaf Node K
+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number for this message. This document describes version 0.
Indexed Storage Internal Node KThis is the node ‘K’ value for each internal node of an indexed storage v1 B-tree. + See the description of this field in version 0 and 1 of the superblock as well the section on + v1 B-trees.
Group Internal Node KThis is the node ‘K’ value for each internal node of a group v1 B-tree. See the + description of this field in version 0 and 1 of the superblock as well as the section + on v1 B-trees.
Group Leaf Node KThis is the node ‘K’ value for each leaf node of a group v1 B-tree. See the + description of this field in version 0 and 1 of the superblock as well as the section on v1 + B-trees.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_drvinfo IV.A.2.u. The Driver Info Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Driver Info
Header Message Type: 0x0014
Length: Varies
Status: Optional; may not be repeated.
Description:This message contains information needed by the file driver to reopen a file. This message is + only found in the superblock extension: see the @ref subsec_fmt2_boot_supext section + for more information. For more information on the fields in the driver info message, see the + @ref subsec_fmt2_boot_driver section; those who use the multi and family file drivers will find + this section particularly helpful.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + +
Driver Info Message
bytebytebytebyte
VersionThis space inserted only to align table nicely

Driver Identification
Driver Information SizeThis space inserted only to align table nicely


Driver Information (variable size)


+ + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number for this message. This document describes version 0.
Driver IdentificationThis is an eight-byte ASCII string without null termination which identifies the driver.
Driver Information SizeThe size in bytes of the Driver Information field of this message.
Driver InformationDriver information is stored in a format defined by the file driver.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_attrinfo IV.A.2.v. The Attribute Info Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Attribute Info
Header Message Type: 0x0015
Length: Varies
Status: Optional; may not be repeated.
Description:This message stores information about the attributes on an object, such as the maximum creation + index for the attributes created and the location of the attribute storage when the attributes + are stored “densely”.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + +
Attribute Info Message
bytebytebytebyte
VersionFlagsMaximum Creation Index (optional)

Fractal Heap AddressO


Attribute Name v2 B-tree AddressO


Attribute Creation Order v2 B-tree AddressO (optional)

+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number for this message. This document + describes version 0.
FlagsThis is the attribute index information flag with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
0If set, creation order for attributes is tracked.
1If set, creation order for attributes is indexed.
2-7Reserved
Maximum Creation IndexThe is the maximum creation order index value for the attributes on the object.
+ This field is present if bit 0 of Flags is set.
Fractal Heap AddressThis is the address of the fractal heap to store dense attributes.
Attribute Name v2 B-tree AddressThis is the address of the version 2 B-tree to index the names of densely stored attributes.
Attribute Creation Order v2 B-tree AddressThis is the address of the version 2 B-tree to index the creation order of densely stored + attributes.
+ This field is present if bit 1 of Flags is set.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_refcount IV.A.2.w. The Object Reference Count Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Object Reference Count
Header Message Type: 0x0016
Length: Fixed
Status: Optional; may not be repeated.
Description:This message stores the number of hard links (in groups or objects) pointing to an object: in + other words, its reference count.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + +
Object Reference Count
bytebytebytebyte
VersionThis space inserted only to align table nicely
Reference count
+ + + + + + + + + + + + + + +
Field NameDescription
VersionThe version number for this message. This document describes version 0.
Reference CountThe unsigned 32-bit integer is the reference count for the object. This message is only present + in “version 2” (or later) object headers, and if not present those object header versions, + the reference count for the object is assumed to be 1.
+ +\subsubsection subsubsec_fmt2_dataobject_hdr_msg_fsinfo IV.A.2.x. The File Space Info Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: File Space Info
Header Message Type: 0x0018
Length: Fixed
Status: Optional; may not be repeated.
Description:This message stores the file space management strategy (see description below) that the library + uses in handling file space request for the file. It also contains the free-space section + threshold used by the library’s free-space managers for the file. If the strategy is 1, + this message also contains the addresses of the file’s free-space managers which track free + space for each type of file space allocation. There are six basic types of file space allocation: + superblock, B-tree, raw data, global heap, local heap, and object header. See the description of + @ref subsec_fmt2_infra_freespaceindex as well the description of allocation types in + @ref sec_fmt2_appendixb.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
File Space Info
bytebytebytebyte
VersionStrategyThresholdL
Super-block Free-space Manager AddressO
B-tree Free-space Manager AddressO
Raw Data Free-space Manager AddressO
Global Heap Free-space Manager AddressO
Local Heap Free-space Manager AddressO
Object Header Free-space Manager AddressO
+\li Items marked with an ‘O’ in the above table are of the size specified in “Size of + Offsets” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in “Size of + Lengths” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
VersionThis is the version number of this message. This document describes version 0.
StrategyThis is the file space management strategy for the file. There are four types of strategies: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
1With this strategy, the HDF5 Library’s free-space managers track the free space + that results from the manipulation of HDF5 objects in the HDF5 file. The free space + information is saved when the file is closed, and reloaded when the file is reopened.
+ When space is needed for file metadata or raw data, the HDF5 Library first requests space + from the library’s free-space managers. If the request is not satisfied, the library + requests space from the aggregators. If the request is still not satisfied, the library + requests space from the virtual file driver. That is, the library will use all of the + mechanisms for allocating space.
2This is the HDF5 Library’s default file space management strategy. With this strategy, + the library’s free-space managers track the free space that results from the + manipulation of HDF5 objects in the HDF5 file. The free space information is NOT saved when + the file is closed and the free space that exists upon file closing becomes unaccounted space + in the file.
+ As with strategy #1, the library will try all of the mechanisms for allocating space. When + space is needed for file metadata or raw data, the library first requests space from the + free-space managers. If the request is not satisfied, the library requests space from the + aggregators. If the request is still not satisfied, the library requests space from the virtual + file driver.
3With this strategy, the HDF5 Library does not track free space that results from the + manipulation of HDF5 objects in the HDF5 file and the free space becomes unaccounted + space in the file.
+ When space is needed for file metadata or raw data, the library first requests space + from the aggregators. If the request is not satisfied, the library requests space from + the virtual file driver.
4With this strategy, the HDF5 Library does not track free space that results from the + manipulation of HDF5 objects in the HDF5 file and the free space becomes unaccounted + space in the file.
+ When space is needed for file metadata or raw data, the library requests space from + the virtual file driver.
ThresholdThis is the free-space section threshold. The library’s free-space managers will track + only free-space sections with size greater than or equal to threshold. The default is to + track free-space sections of all sizes.
Superblock Free-space Manager AddressThis is the address of the free-space manager for #H5FD_MEM_SUPER allocation type.
B-tree Free-space Manager AddressThis is the address of the free-space manager for #H5FD_MEM_BTREE allocation type.
Raw Data Free-space Manager AddressThis is the address of the free-space manager for #H5FD_MEM_DRAW allocation type.
Global Heap Free-space Manager AddressThis is the address of the free-space manager for #H5FD_MEM_GHEAP allocation type.
Local Heap Free-space Manager AddressThis is the address of the free-space manager for #H5FD_MEM_LHEAP allocation type.
Object Header Free-space Manager AddressThis is the address of the free-space manager for #H5FD_MEM_OHDR allocation type.
+ +\subsection subsec_fmt2_dataobject_storage IV.B. Disk Format: Level 2B - Data Object Data Storage +The data for an object is stored separately from the header information in the file and may not actually +be located in the HDF5 file itself if the header indicates that the data is stored externally. The +information for each record in the object is stored according to the dimensionality of the object +(indicated in the dataspace header message). Multi-dimensional array data is stored in C order; in other +words, the “last” dimension changes fastest. + +Data whose elements are composed of atomic datatypes are stored in IEEE format, unless +they are specifically defined as being stored in a different machine format with the architecture-type +information from the datatype header message. This means that each architecture will need to +[potentially] byte-swap data values into the internal representation for that particular machine. + +Data with a variable-length datatype is stored in the global heap of the HDF5 file. Global heap +identifiers are stored in the data object storage. + +Data whose elements are composed of reference datatypes are stored in several different ways depending +on the particular reference type involved. Object pointers are just stored as the offset of the +object header being pointed to with the size of the pointer being the same number of bytes as offsets +in the file. + +Dataset region references are stored as a heap-ID which points to the following information within the +file-heap: an offset of the object pointed to, number-type information (same format as header message), +dimensionality information (same format as header message), sub-set start and end information (in other +words, a coordinate location for each), and field start and end names (in other words, a [pointer to the] +string indicating the first field included and a [pointer to the] string name for the last field). + +Data of a compound datatype is stored as a contiguous stream of the items in the structure, with each +item formatted according to its datatype. + +\section sec_fmt2_appendixa V. Appendix A: Definitions +Definitions of various terms used in this document are included in this section. + + + + + + + + + + + + + +
TermDefinition
Undefined Address\anchor FMT2UndefinedAddress The "undefined address" for a file is a file address with all bits + set: in other words, 0xffff...ff.
Unlimited Size\anchor FMT2UnlimitedDim The "unlimited size" for a size is a value with all bits set: in other words, + 0xffff...ff.
+ +\section sec_fmt2_appendixb VI. Appendix B: File Memory Allocation Types +There are six basic types of file memory allocation as follows: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Basic Allocation TypeDescription
#H5FD_MEM_SUPERFile memory allocated for Superblock.
#H5FD_MEM_BTREEFile memory allocated for B-tree.
#H5FD_MEM_DRAWFile memory allocated for raw data.
#H5FD_MEM_GHEAPFile memory allocated for Global Heap.
#H5FD_MEM_LHEAPFile memory allocated for Local Heap.
#H5FD_MEM_OHDRFile memory allocated for Object Header.
+ +There are other file memory allocation types that are mapped to the above six basic allocation types +because they are similar in nature. The mapping is listed in the following table: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Basic Allocation TypeMapping of Allocation Types to Basic Allocation Types
#H5FD_MEM_SUPERnone
#H5FD_MEM_BTREE#H5FD_MEM_SOHM_INDEX
#H5FD_MEM_DRAW#H5FD_MEM_FHEAP_HUGE_OBJ
#H5FD_MEM_GHEAPnone
#H5FD_MEM_LHEAP#H5FD_MEM_FHEAP_DBLOCK, #H5FD_MEM_FSPACE_SINFO
#H5FD_MEM_OHDR#H5FD_MEM_FHEAP_HDR, #H5FD_MEM_FHEAP_IBLOCK, #H5FD_MEM_FSPACE_HDR, #H5FD_MEM_SOHM_TABLE
+ +Allocation types that are mapped to basic allocation types are described below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Allocation TypeDescription
#H5FD_MEM_FHEAP_HDRFile memory allocated for Fractal Heap Header.
#H5FD_MEM_FHEAP_DBLOCKFile memory allocated for Fractal Heap Direct Blocks.
#H5FD_MEM_FHEAP_IBLOCKFile memory allocated for Fractal Heap Indirect Blocks.
#H5FD_MEM_FHEAP_HUGE_OBJFile memory allocated for huge objects in the fractal heap.
#H5FD_MEM_FSPACE_HDRFile memory allocated for Free-space Manager Header.
#H5FD_MEM_FSPACE_SINFOFile memory allocated for Free-space Section List of the free-space manager.
#H5FD_MEM_SOHM_TABLEFile memory allocated for Shared Object Header Message Table.
#H5FD_MEM_SOHM_INDEXFile memory allocated for Shared Message Record List.
+ +*/ diff --git a/doxygen/dox/H5.format.3.0.dox b/doxygen/dox/H5.format.3.0.dox new file mode 100644 index 00000000000..e7ba0ef0e19 --- /dev/null +++ b/doxygen/dox/H5.format.3.0.dox @@ -0,0 +1,12591 @@ + +/** \page FMT3 HDF5 File Format Specification Version 3.0 +
    +
  1. @ref sec_fmt3_intro +
      +
    1. @ref subsec_fmt3_intro_doc
      2. +
    2. @ref subsec_fmt3_intro_20
      3. +
    3. @ref subsec_fmt3_intro_112
      4. +
    4. @ref subsec_fmt3_intro_110
      5. +
    2. +
  2. @ref sec_fmt3_meta +
      +
    1. @ref subsec_fmt3_boot_super
      2. +
    2. @ref subsec_fmt3_boot_driver
      3. +
    3. @ref subsec_fmt3_boot_supext
      4. +
    3. +
  3. @ref sec_fmt3_infra +
      +
    1. @ref subsec_fmt3_infra_btrees +
        +
      1. @ref subsubsec_fmt3_infra_btrees_v1
        2. +
      2. @ref subsubsec_fmt3_infra_btrees_v2
        3. +
      2. +
    2. @ref subsec_fmt3_infra_symboltable
      3. +
    3. @ref subsec_fmt3_infra_symboltableentry
      4. +
    4. @ref subsec_fmt3_infra_localheap
      5. +
    5. @ref subsec_fmt3_infra_globalheap
      6. +
    6. @ref subsec_fmt3_infra_globalheapvds
      7. +
    7. @ref subsec_fmt3_infra_fractalheap
      8. +
    8. @ref subsec_fmt3_infra_freespaceindex
      9. +
    9. @ref subsec_fmt3_infra_sohm
      10. +
    4. +
  4. @ref sec_fmt3_dataobject +
      +
    1. @ref subsec_fmt3_dataobject_hdr +
        +
      1. @ref subsec_fmt3_dataobject_hdr_prefix
        2. +
          +
        1. @ref subsubsec_fmt3_dataobject_hdr_prefix_one
          2. +
        2. @ref subsubsec_fmt3_dataobject_hdr_prefix_two
          3. +
        +
      2. @ref subsec_fmt3_dataobject_hdr_msg
        3. +
          +
        1. @ref subsubsec_fmt3_dataobject_hdr_msg_nil
          2. +
        2. @ref subsubsec_fmt3_dataobject_hdr_msg_simple
          3. +
        3. @ref subsubsec_fmt3_dataobject_hdr_msg_linkinfo
          4. +
        4. @ref subsubsec_fmt3_dataobject_hdr_msg_dtmessage
          5. +
        5. @ref subsubsec_fmt3_dataobject_hdr_msg_ofvmessage
          6. +
        6. @ref subsubsec_fmt3_dataobject_hdr_msg_fvmessage
          7. +
        7. @ref subsubsec_fmt3_dataobject_hdr_msg_link
          8. +
        8. @ref subsubsec_fmt3_dataobject_hdr_msg_external
          9. +
        9. @ref subsubsec_fmt3_dataobject_hdr_msg_layout
          10. +
        10. @ref subsubsec_fmt3_dataobject_hdr_msg_bogus
          11. +
        11. @ref subsubsec_fmt3_dataobject_hdr_msg_groupinfo
          12. +
        12. @ref subsubsec_fmt3_dataobject_hdr_msg_filter
          13. +
        13. @ref subsubsec_fmt3_dataobject_hdr_msg_attribute
          14. +
        14. @ref subsubsec_fmt3_dataobject_hdr_msg_comment
          15. +
        15. @ref subsubsec_fmt3_dataobject_hdr_msg_omodified
          16. +
        16. @ref subsubsec_fmt3_dataobject_hdr_msg_shared
          17. +
        17. @ref subsubsec_fmt3_dataobject_hdr_msg_continuation
          18. +
        18. @ref subsubsec_fmt3_dataobject_hdr_msg_stmgroup
          19. +
        19. @ref subsubsec_fmt3_dataobject_hdr_msg_mod
          20. +
        20. @ref subsubsec_fmt3_dataobject_hdr_msg_btreek
          21. +
        21. @ref subsubsec_fmt3_dataobject_hdr_msg_drvinfo
          22. +
        22. @ref subsubsec_fmt3_dataobject_hdr_msg_attrinfo
          23. +
        23. @ref subsubsec_fmt3_dataobject_hdr_msg_refcount
          24. +
        24. @ref subsubsec_fmt3_dataobject_hdr_msg_fsinfo
          25. +
        +
      2. +
    2. @ref subsec_fmt3_dataobject_storage
      3. +
    +
    5. +
  5. @ref sec_fmt3_appendixa +
  6. @ref sec_fmt3_appendixb +
  7. @ref sec_fmt3_appendixc +
      +
    1. @ref subsec_fmt3_appendixc_chunk +
    2. @ref subsec_fmt3_appendixc_implicit +
    3. @ref subsec_fmt3_appendixc_fixedarr +
    4. @ref subsec_fmt3_appendixc_extarr +
    5. @ref subsec_fmt3_appendixc_appv2btree +
    +
    8. +
  8. @ref sec_fmt3_appendixd +
      +
    1. @ref subsec_fmt3_appendixd_encode +
    2. @ref subsec_fmt3_appendixd_encoderv +
    3. @ref subsec_fmt3_appendixd_encodedp +
    +
    9. +
+ + + +\section sec_fmt3_intro I. Introduction + + + + + + + + + + + + + + +
Figure 1: Relationships among the HDF5 root group, other groups, and objects
\image html FF-IH_FileGroup.gif
Figure 2: HDF5 objects -- datasets, datatypes, or dataspaces
\image html FF-IH_FileObject.gif
+ +The format of an HDF5 file on disk encompasses several key ideas of the HDF4 and AIO file formats as well +as addressing some shortcomings therein. The new format is more self-describing than the HDF4 format and +is more uniformly applied to data objects in the file. + +An HDF5 file appears to the user as a directed graph. The nodes of this graph are the higher-level HDF5 +objects that are exposed by the HDF5 APIs: +\li Groups +\li Datasets +\li Committed (formerly Named) datatypes + +At the lowest level, as information is actually written to the disk, an HDF5 file is made up of the +following objects: +\li A superblock +\li B-tree nodes +\li Heap blocks +\li Object headers +\li Object data +\li Free space + +The HDF5 library uses these lower-level objects to represent the higher-level objects that are then +presented to the user or to applications through the APIs. For instance, a group is an object header that +contains a message that points to a local heap (for storing the links to objects in the group) and to a +B-tree (which indexes the links). A dataset is an object header that contains messages that describe +datatype, dataspace, layout, filters, external files, fill value, and other elements with the layout message +pointing to either a raw data chunk or to a B-tree that points to raw data chunks. + +\subsection subsec_fmt3_intro_doc I.A. This Document +This document describes the lower-level data objects; the higher-level objects and their properties are +described in the \ref UG. + +Three levels of information comprise the file format. Level 0 contains basic information for identifying +and defining information about the file. Level 1 information contains the information about the pieces of a +file shared by many objects in the file (such as a B-trees and heaps). Level 2 is the rest of the file and +contains all of the data objects with each object partitioned into header information, also known as +metadata, and data. + +The various components of the lower-level data objects are described in pairs of tables. The first table +shows the format layout, and the second table describes the fields. The titles of format layout tables +begin with “Layout”. The titles of the tables where the fields are described begin with +“Fields”. For example, the table that describes the format of the +@ref subsubsec_fmt3_infra_btrees_v2 has a title of “Layout: Version 2 B-tree Header”, and the +fields in the version 2 B-tree header are described in the table titled “Fields: Version 2 B-tree Header”. + +The sizes of various fields in the following layout tables are determined by looking at the number of +columns the field spans in the table. There are exceptions: +\li The size may be overridden by specifying a size in parentheses +\li The size of addresses is determined by the @ref FMT3SizeOfOffsetsV0 "Size of Offsets" field +in the superblock and is indicated in this document with a superscripted ‘O’ +\li The size of length fields is determined by the @ref FMT3SizeOfLengthsV0 "Size of Lengths" field +in the superblock and is indicated in this document with a superscripted ‘L’. + +Values for all fields in this document should be treated as unsigned integers, unless otherwise noted in +the description of a field. Additionally, all metadata fields are stored in little-endian byte order. + +All checksums used in the format are computed with the +Jenkins’ lookup3 algorithm. + +Whenever a bit flag or field is mentioned for an entry, bits are numbered from the lowest bit position +in the entry. + +Various format tables in this document have cells with “This space inserted only to align table nicely”. +These entries in the table are just to make the table presentation nicer and do not represent any values +or padding in the file. + +\subsection subsec_fmt3_intro_20 I.B. Changes for HDF5 2.0 +The following sections have been changed or added for the 2.0 release: +\li Under @ref subsubsec_fmt3_dataobject_hdr_msg_dtmessage, in the Description for + “Fields:Datatype Message”, version 5 was added, as well as the new Complex class (11). + +\subsection subsec_fmt3_intro_112 I.C. Changes for HDF5 1.12 +The following sections have been changed or added for the 1.12 release: +\li Under @ref subsubsec_fmt3_dataobject_hdr_msg_dtmessage, in the Description for + “Fields:Datatype Message”, version 4 was added and Reference class (7) of the + datatype was updated to describe version 4. +\li @ref sec_fmt3_appendixd was added. + +\subsection subsec_fmt3_intro_110 I.D. Changes for HDF5 1.10 +The following sections have been changed or added for the 1.10 release: +\li In the @ref subsec_fmt3_boot_super section, version 3 of the superblock was added. +\li In the @ref subsec_fmt3_boot_supext section, a link to the Data Storage message was added. +\li In the @ref subsubsec_fmt3_infra_btrees_v2 section, additional B-tree types were added. + Tables that describe the @ref FMT3V2BtType10"type 10" and @ref FMT3V2BtType11"11" record + layouts were added at the end of the section. +\li The @ref subsec_fmt3_infra_globalheapvds was added. +\li @ref subsubsec_fmt3_dataobject_hdr_msg_layout section was changed. The name was changed, + and @ref FMT3DataLayoutV4"version 4" of the data layout message was added for the virtual type. +\li The @ref subsubsec_fmt3_dataobject_hdr_msg_fsinfo header message type was added. +\li @ref sec_fmt3_appendixc was added. Five indexing types were added. + +\section sec_fmt3_meta II. Disk Format: Level 0 - File Metadata + +\subsection subsec_fmt3_boot_super II.A. Disk Format: Level 0A - Format Signature and Superblock +The superblock may begin at certain predefined offsets within the HDF5 file, allowing a block of +unspecified content for users to place additional information at the beginning (and end) of the HDF5 file +without limiting the HDF5 library’s ability to manage the objects within the file itself. This feature +was designed to accommodate wrapping an HDF5 file in another file format or adding descriptive information +to an HDF5 file without requiring the modification of the actual file’s information. The superblock +is located by searching for the HDF5 file signature at byte offset 0, byte offset 512 and at successive +locations in the file, each a multiple of two of the previous location, in other words, at these byte +offsets: 0, 512, 1024, 2048, and so on. + +The superblock is composed of the format signature, followed by a superblock version number and information +that is specific to each version of the superblock. + +Currently, there are four versions of the superblock format: +\li Version 0 is the default format. +\li Version 1 is the same as version 0 but with the “Indexed Storage Internal Node K” + field for storing non-default B-tree ‘K’ value. +\li Version 2 has some fields eliminated and compressed from superblock format versions 0 and 1. It has + added checksum support and superblock extension to store additional superblock metadata. +\li Version 3 is the same as version 2 except that the field “File Consistency Flags” + is used for file locking. This format version will enable support for the latest version. + +Version 0 and 1 of the superblock are described below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Superblock (Versions 0 and 1)
bytebytebytebyte

Format Signature (8 bytes)

Version \# of SuperblockVersion \# of File’s Free Space StorageVersion \# of Root Group Symbol Table EntryReserved (zero)
Version \# of Shared Header Message FormatSize of OffsetsSize of LengthsReserved (zero)
Group Leaf Node KGroup Internal Node K
File Consistency Flags
Indexed Storage Internal Node K1Reserved (zero)1
Base AddressO
Address of File Free Space InfoO
End of File AddressO
Driver Information Block AddressO
Root Group Symbol Table Entry
+\li Items marked with an ‘1’ in the above table are new in version 1 of the superblock. +\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Superblock (Versions 0 and 1)
Field NameDescription
Format SignatureThis field contains a constant value and can be used to quickly identify a file as being an HDF5 + file. The constant value is designed to allow easy identification of an HDF5 file and to allow + certain types of data corruption to be detected. The file signature of an HDF5 file always + contains the following values: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Decimal:13772687013102610
Hexadecimal:894844460d0a1a0a
ASCII C Notation:\211HDF\\r\\n\032\\n
+
+ This signature both identifies the file as an HDF5 file and provides for immediate detection of common + file-transfer problems. The first two bytes distinguish HDF5 files on systems that expect the first two + bytes to identify the file type uniquely. The first byte is chosen as a non-ASCII value to reduce the + probability that a text file may be misrecognized as an HDF5 file; also, it catches bad file transfers + that clear bit 7. Bytes two through four name the format. The CR-LF sequence catches bad file transfers + that alter newline sequences. The control-Z character stops file display under MS-DOS. The final line + feed checks for the inverse of the CR-LF translation problem. (This is a direct descendent of the + PNG + file signature.)
+ This field is present in version 0+ of the superblock.
Version Number of the SuperblockThis value is used to determine the format of the information in the superblock. When the format of + the information in the superblock is changed, the version number is incremented to the next integer + and can be used to determine how the information in the superblock is formatted.
+ Values of 0, 1 and 2 are defined for this field (the format of version 2 is described below, not + here).
+ This field is present in version 0+ of the superblock.
Version Number of the File’s Free Space InformationThis value is used to determine the format of the file’s free space information.
+ The only value currently valid in this field is ‘0’, which indicates that the file’s + free space index is as described in @ref subsec_fmt3_infra_freespaceindex below.
+ This field is present in version 0 and 1 of the superblock.
Version Number of the Root Group Symbol Table EntryThis value is used to determine the format of the information in the Root Group Symbol Table Entry. + When the format of the information in that field is changed, the version number is incremented to the + next integer and can be used to determine how the information in the field is formatted.
+ The only value currently valid in this field is ‘0’, which indicates that the root group + symbol table entry is formatted as described in @ref subsec_fmt3_infra_symboltableentry below.
+ This field is present in version 0 and 1 of the superblock.
Version Number of the Shared Header Message FormatThis value is used to determine the format of the information in a shared object header message. + Since the format of the shared header messages differs from the other private header messages, a + version number is used to identify changes in the format.
+ The only value currently valid in this field is ‘0’, which indicates that shared + header messages are formatted as described in @ref subsubsec_fmt3_dataobject_hdr_msg_shared below.
+ This field is present in version 0 and 1 of the superblock.
\anchor FMT3SizeOfOffsetsV0 Size of OffsetsThis value contains the number of bytes used to store addresses in the file. The values for the + addresses of objects in the file are offsets relative to a base address, usually the address of the + superblock signature. This allows a wrapper to be added after the file is created without invalidating + the internal offset locations.
+ This field is present in version 0+ of the superblock.
\anchor FMT3SizeOfLengthsV0 Size of LengthsThis value contains the number of bytes used to store the size of an object.
+ This field is present in version 0+ of the superblock.
Group Leaf Node KEach leaf node of a group B-tree will have at least this many entries but not more than twice this + many. If a group has a single leaf node then it may have fewer entries.
+ This value must be greater than zero.
+ See the @ref subsec_fmt3_infra_btrees below.
+ This field is present in version 0 and 1 of the superblock.
Group Internal Node KEach internal node of a group B-tree will have at least this many entries but not more than twice this + many. If the group has only one internal node then it might have fewer entries.
+ This value must be greater than zero.
+ See the @ref subsec_fmt3_infra_btrees below.
+ This field is present in version 0 and 1 of the superblock.
File Consistency FlagsThis field is unused and should be ignored.
+ This field is present in version 0+ of the superblock.
Indexed Storage Internal Node KEach internal node of a indexed storage B-tree will have at least this many entries but not more than + twice this many. If the ndex storage B-tree has only one internal node then it might have fewer + entries.
+ This value must be greater than zero.
+ See the @ref subsec_fmt3_infra_btrees below.
+ This field is present in version 1 of the superblock.
Base AddressThis is the absolute file address of the first byte of the HDF5 data within the file. The library + currently constrains this value to be the absolute file address of the superblock itself when creating + new files; future versions of the library may provide greater flexibility. When opening an existing + file and this address does not match the offset of the superblock, the library assumes that the entire + contents of the HDF5 file have been adjusted in the file and adjusts the base address and end of file + address to reflect their new positions in the file. Unless otherwise noted, all other file addresses + are relative to this base address.
+ This field is present in version 0+ of the superblock.
Address of Global Free Space IndexThe file’s free space management is not persistent for version 0 and 1 of the superblock. + Currently this field always contains the @ref FMT3UndefinedAddress "undefined address".
+ This field is present in version 0 and 1 of the superblock.
End of File AddressThis is the absolute file address of the first byte past the end of all HDF5 data. It is used to + determine whether a file has been accidentally truncated and as an address where file data allocation + can occur if space from the free list is not used.
+ This field is present in version 0+ of the superblock.
Driver Information Block AddressThis is the relative file address of the file driver information block which contains driver-specific + information needed to reopen the file. If there is no driver information block then this entry should + be the @ref FMT3UndefinedAddress "undefined address".
+ This field is present in version 0 and 1 of the superblock.
Root Group Symbol Table EntryThis is the @ref subsec_fmt3_infra_symboltableentry of the root group, which serves as the entry-point + into the group graph for the file.
+ This field is present in version 0 and 1 of the superblock.
+ +Versions 2 and 3 of the superblock is described below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Superblock (Versions 2 and 3)
bytebytebytebyte

Format Signature (8 bytes)

Version \# of SuperblockSize of OffsetsSize of LengthsFile Consistency Flags

Base AddressO


Superblock Extension AddressO


End of File AddressO


Root Group Object Header AddressO

Superblock Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Superblock (Versions 2 and 3)
Field NameDescription
Format SignatureThis field is the same as described for versions 0 and 1 of the superblock.
Version Number of the SuperblockThis field has a value of 2 and has the same meaning as for versions 0 and 1.
Size of OffsetsThis field is the same as described for @ref FMT3SizeOfOffsetsV0 "versions 0 and 1" of the superblock.
Size of LengthsThis field is the same as described for @ref FMT3SizeOfLengthsV0 "versions 0 and 1" of the superblock.
File Consistency FlagsFor superblock version 2: This field is unused and should be ignored.
+ For superblock version 3: This value contains flags to ensure file consistency for file locking. + Currently, the following bit flags are defined: +
    +
  • Bit 0 if set indicates that the file has been opened for write access.
    • +
  • Bit 1 is reserved for future use.
    • +
  • Bit 2 if set indicates that the file has been opened for single-writer/multiple-reader + (SWMR) write access.
    • +
  • Bits 3-7 are reserved for future use.
    • +

+ Bit 0 should be set as the first action when a file has been opened for write access. Bit 2 should + be set when a file has been opened for SWMR write access. These two bits should be cleared only as + the final action when closing a file.
+ This field is present in version 0+ of the superblock.
+ The size of this field has been reduced from 4 bytes in superblock format versions 0 and 1 to + 1 byte.
Base AddressThis field is the same as described for versions 0 and 1 of the superblock.
Superblock Extension AddressThe field is the address of the object header for the @ref subsec_fmt3_boot_supext. If there is no + extension then this entry should be the @ref FMT3UndefinedAddress "undefined address".
End of File AddressThis field is the same as described for versions 0 and 1 of the superblock.
Root Group Object Header AddressThis is the address of the @ref sec_fmt3_dataobject, which serves as the entry point into the group + graph for the file.
Superblock ChecksumThe checksum for the superblock.
+ +\subsection subsec_fmt3_boot_driver II.B. Disk Format: Level 0B - File Driver Info +The driver information block is an optional region of the file which contains information +needed by the file driver in order to reopen a file. The format is described below: + + + + + + + + + + + + + + + + + + + + + +
Layout: Driver Information Block
bytebytebytebyte
VersionReserved
Driver Information Size

Driver Identification (8 bytes)



Driver Information (variable size)


+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: Driver Information Block
Field NameDescription
VersionThe version number of the Driver Information Block. This document describes version 0.
Driver Information SizeThe size in bytes of the Driver Information field.
Driver IdentificationThis is an eight-byte ASCII string without null termination which identifies the driver and/or version number + of the Driver Information block. The predefined driver encoded in this field by the HDF5 library is identified + by the letters NCSA followed by the first four characters of the driver name. If the Driver Information + Block is not the original version then the last letter(s) of the identification will be replaced by a version + number in ASCII, starting with 0.
+ Identification for user-defined drivers is also eight-byte long. It can be arbitrary but should be unique to + avoid the four character prefix “NCSA”.
Driver InformationDriver information is stored in a format defined by the file driver (see description below).
+ +The two drivers encoded in the Driver Identification field are as follows: +\li Multi driver:
The identifier for this driver is “NCSAmulti”. This driver provides + a mechanism for segregating raw data and different types of metadata into multiple files. These files + are viewed by the library as a single virtual HDF5 file with a single file address. A maximum of 6 + files will be created for the following data: superblock, B-tree, raw data, global heap, local heap, + and object header. More than one type of data can be written to the same file. +\li Family driver:
The identifier for this driver is “NCSAfami” and is encoded in this + field for library version 1.8 and after. This driver is designed for systems that do not support files + larger than 2 gigabytes by splitting the HDF5 file address space across several smaller files. It does + nothing to segregate metadata and raw data; they are mixed in the address space just as they would be + in a single contiguous file. + +The format of the Driver Information field for the above two drivers are described below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Multi Driver Information
bytebytebytebyte
Member MappingMember MappingMember MappingMember Mapping
Member MappingMember MappingReservedReserved

Address of Member File 1


End of Address for Member File 1


Address of Member File 2


End of Address for Member File 2


... ...


Address of Member File N


End of Address for Member File N


Name of Member File 1 (variable size)


Name of Member File 2 (variable size)


... ...


Name of Member File N (variable size)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Multi Driver Information
Field NameDescription
Member MappingThese fields are integer values from 1 to 6 indicating how the data can be mapped to or + merged with another type of data. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Member MappingDescription
1The superblock data.
2The B-tree data.
3The raw data.
4The global heap data.
5The local heap data.
6The object header data.

+ For example, if the third field has the value 3 and all the rest have the + value 1, it means there are two files, one for raw data, and one for superblock, + B-tree, global heap, local heap, and object header.
ReservedThese fields are reserved and should always be zero.
Address of Member File NThis field specifies the virtual address at which the member file starts.
+ N is the number of member files.
End of Address for Member File NThis field is the end of allocated address for the member file.
Name of Member File NThis field is the null-terminated name of member file. And its length should be multiples + of 8 bytes. Additional bytes will be padded with NULLs. The default naming convention is + %%s-X.h5, where X is one of the letters s (for superblock), + b (for B-tree), r (for raw data), g (for global heap), + l (for local heap), and o (for object header). The name for the whole + HDF5 file will substitute the %s in the string.
+
+ + + + + + + + + + + +
Layout: Family Driver Information
bytebytebytebyte

Size of Member File

+
+ + + + + + + + + + +
Fields: Family Driver Information
Field NameDescription
Size of Member FileThis field is the size of the member file in the family of files.
+ +\subsection subsec_fmt3_boot_supext II.C. Disk Format: Level 0C - Superblock Extension +The superblock extension is used to store superblock metadata which is either optional, or added +after the version of the superblock was defined. Superblock extensions may only exist when version 2+ or +later of the superblock is used. A superblock extension is an object header which may hold the following messages: +\li \ref subsec_fmt3_infra_sohm containing information to locate the master table of shared object + header message indices. +\li \ref subsubsec_fmt3_dataobject_hdr_msg_btreek containing non-default B-tree ‘K’ values. +\li \ref subsubsec_fmt3_dataobject_hdr_msg_drvinfo containing information needed by the file driver in + order to reopen a file. See also the \ref subsec_fmt3_boot_driver section above. +\li \ref subsubsec_fmt3_dataobject_hdr_msg_fsinfo containing information about file space handling in the file. + +\section sec_fmt3_infra III. Disk Format: Level 1 - File Infrastructure + +\subsection subsec_fmt3_infra_btrees III.A. Disk Format: Level 1A - B-trees and B-tree Nodes +B-trees allow flexible storage for objects which tend to grow in ways that cause the object to be stored +discontiguously. B-trees are described in various algorithms books including "Introduction to Algorithms" by +Thomas H. Cormen, Charles E. Leiserson, and Ronald L. Rivest. The B-trees are used in several places in +the HDF5 file format, when an index is needed for another data structure. + +The version 1 B-tree structure described below is the original index structure. The version 1 B-trees are +being phased out in favor of the version 2 B-trees described below. Note that both types of structures may +be found in the same file depending on the application settings when creating the file. + +\subsubsection subsubsec_fmt3_infra_btrees_v1 III.A.1. Disk Format: Level 1A1 - Version 1 B-trees +Version 1 B-trees in HDF5 files an implementation of the B-tree. The sibling nodes at a +particular level in the tree are stored in a doubly-linked list. See the “Efficient Locking +for Concurrent Operations on B-trees” paper by Phillip Lehman and S. Bing Yao as published in the + ACM Transactions on Database Systems, Vol. 6, No. 4, December 1981. + +The B-trees implemented by the file format contain one more key than the number of children. In other +words, each child pointer out of a B-tree node has a left key and a right key. The pointers out of internal +nodes point to sub-trees while the pointers out of leaf nodes point to symbol nodes and raw data chunks. +Aside from that difference, internal nodes and leaf nodes are identical. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: B-tree Nodes
bytebytebytebyte
Signature
Node TypeNode LevelEntries Used

Address of Left SiblingO


Address of Right SiblingO

Key 1 (variable size)

Address of Child 1O

Key 2 (variable size)

Address of Child 2O

...
Key 2K (variable size)

Address of Child 2KO

Key 2K+1 (variable size)
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: B-tree Nodes
Field NameDescription
SignatureThe ASCII character string “TREE” is used to indicate the beginning of a + B-tree node. This gives file consistency checking utilities a better chance of reconstructing + a damaged file.
Node TypeEach B-tree points to a particular type of data. This field indicates the type of data as well as + implying the maximum degree K of the tree and the size of each Key field.
+ + + + + + + + + + + + + +
Node TypeDescription
0This tree points to group nodes.
1This tree points to a raw data chunk.
+
Node LevelThe node level indicates the level at which this node appears in the tree (leaf nodes are at level + zero). Not only does the level indicate whether child pointers point to sub-trees or to data, but it + can also be used to help file consistency checking utilities reconstruct damaged trees.
Entries UsedThis determines the number of children to which this node points. All nodes of a particular type of + tree have the same maximum degree, but most nodes will point to less than that number of children. The + valid child pointers and keys appear at the beginning of the node and the unused pointers and keys + appear at the end of the node. The unused pointers and keys have undefined values.
Address of Left SiblingThis is the relative file address of the left sibling of the current node. If the current node is the + left-most node at this level then this field is the @ref FMT3UndefinedAddress "undefined address".
Address of Right SiblingThis is the relative file address of the right sibling of the current node. If the current node is the + right-most node at this level then this field is the @ref FMT3UndefinedAddress "undefined address".
Keys and Child PointersEach tree has 2K+1 keys with 2K child pointers interleaved between the keys. The number + of keys and child pointers actually containing valid values is determined by the node’s + Entries Used field. If that field is N then the B-tree contains N child + pointers and N+1 keys.
KeyThe format and size of the key values is determined by the type of data to which this tree points. The + keys are ordered and are boundaries for the contents of the child pointer; that is, the key values + represented by child N fall between Key N and Key N+1. Whether the interval + is open or closed on each end is determined by the type of data to which the tree points.
+ The format of the key depends on the node type. For nodes of node type 0 (group nodes), the key is + formatted as follows: + + + + + +
A single field of @ref FMT3SizeOfLengthsV0 "Size of Lengths" bytes.Indicates the byte offset into the local heap for the first object name in the subtree which + that key describes.
+
+ For nodes of node type 1 (chunked raw data nodes), the key is formatted as follows: + + + + + + + + + + + + + +
Bytes 1-4Size of chunk in bytes.
Bytes 4-8Filter mask, a 32-bit bit field indicating which filters have been skipped for this chunk. Each + filter has an index number in the pipeline (starting at 0, with the first filter to apply) and + if that filter is skipped, the bit corresponding to its index is set.
(D + 1) 64-bit fieldsThe offset of the chunk within the dataset where D is the number + of dimensions of the dataset, and the last value is the offset within the dataset’s + datatype and should always be zero. For example, if a chunk in a 3-dimensional dataset begins at the + position [5,5,5], there will be three such 64-bit indices, each with the value of + 5, followed by a 0 value.
+
Child PointerThe tree node contains file addresses of subtrees or data depending on the node level. Nodes at Level + 0 point to data addresses, either raw data chunks or group nodes. Nodes at non-zero levels point to other + nodes of the same B-tree.
+ For raw data chunk nodes, the child pointer is the address of a single raw data chunk. For group nodes, + the child pointer points to a @ref subsec_fmt3_infra_symboltableentry, which contains + information for multiple symbol table entries.
+ +Conceptually, each B-tree node looks like this: + + + + + + + + + + + + + + + + + + + + + + +
key[0] child[0] key[1] child[1] key[2]; ... ... key[N-1] child[N-1] key[N]
+where child[i] is a pointer to a sub-tree (at a level above Level 0) or to data (at Level 0). +Each key[i] describes an item stored by the B-tree (a chunk or an object of a group node). +The range of values represented by child[i] is indicated by key[i] and key[i+1]. + +The following question must next be answered: "Is the value described by key[i] contained in +child[i-1] or in child[i]?" The answer depends on the type of tree. In trees for groups (node +type 0) the object described by key[i] is the greatest object contained in child[i-1] while +in chunk trees (node type 1) the chunk described by key[i] is the least chunk in child[i]. + +That means that key[0] for group trees is sometimes unused; it points to offset zero in the heap, which is +always the empty string and compares as "less-than" any valid object name. + +And key[N] for chunk trees is sometimes unused; it contains a chunk offset which compares as +"greater-than" any other chunk offset and has a chunk byte size of zero to indicate that it is not actually +allocated. + +\subsubsection subsubsec_fmt3_infra_btrees_v2 III.A.2. Disk Format: Level 1A2 - Version 2 B-trees +Version 2 (v2) B-trees are “traditional” B-trees, with one major difference. Instead of just using +a simple pointer (or address in the file) to a child of an internal node, the pointer to the child node +contains two additional pieces of information: the number of records in the child node itself, and the +total number of records in the child node and all its descendants. Storing this additional information +allows fast array-like indexing to locate the nth record in the B-tree. + +The entry into a version 2 B-tree is a header which contains global information about the structure of +the B-tree. The root node address field in the header points to the B-tree root node, which is +either an internal or leaf node, depending on the value in the header’s depth field. An +internal node consists of records plus pointers to further leaf or internal nodes in the tree. A leaf +node consists of solely of records. The format of the records depends on the B-tree type (stored in +the header). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 2 B-tree Header
bytebytebytebyte
Signature
VersionTypeThis space inserted only to align table nicely
Node Size
Record SizeDepth
Split PercentMerge PercentThis space inserted only to align table nicely

Root Node AddressO

Number of Records in Root NodeThis space inserted only to align table nicely

Total Number of Records in B-treeL

Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 B-tree Header
Field NameDescription
SignatureThe ASCII character string “BTHD” is used to indicate the header of a + version 2 (v2) B-tree node.
VersionThe version number for this B-tree header. This document describes version 0.
TypeThis field indicates the type of B-tree: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0This B-tree is used for testing only. This value should not be used for storing + records in actual HDF5 files.
1This B-tree is used for indexing indirectly accessed, non-filtered ‘huge’ + fractal heap objects.
2This B-tree is used for indexing indirectly accessed, filtered ‘huge’ + fractal heap objects.
3This B-tree is used for indexing directly accessed, non-filtered ‘huge’ + fractal heap objects.
4This B-tree is used for indexing directly accessed, filtered ‘huge’ + fractal heap objects.
5This B-tree is used for indexing the ‘name’ field for links in indexed + groups.
6This B-tree is used for indexing the ‘creation order’ field for links + in indexed groups.
7This B-tree is used for indexing shared object header messages.
8This B-tree is used for indexing the ‘name’ field for indexed + attributes.
9This B-tree is used for indexing the ‘creation order’ field for + indexed attributes.
10This B-tree is used for indexing chunks of datasets with no filters and with more + than one dimension of unlimited extent.
11This B-tree is used for indexing chunks of datasets with filters and more than one + dimension of unlimited extent.
+ The format of records for each type is described below.
Node SizeThis is the size in bytes of all B-tree nodes.
Record SizeThis field is the size in bytes of the B-tree record.
DepthThis is the depth of the B-tree.
Split PercentThe percent full that a node needs to increase above before it is split.
Merge PercentThe percent full that a node needs to be decrease below before it is split.
Root Node AddressThis is the address of the root B-tree node. A B-tree with no records will have the + @ref FMT3UndefinedAddress "undefined address" in this field.
Number of Records in Root NodeThis is the number of records in the root node.
Total Number of Records in B-treeThis is the total number of records in the entire B-tree.
ChecksumThis is the checksum for the B-tree header.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 2 B-tree Internal Node
bytebytebytebyte
Signature
VersionTypeRecords 0, 1, 2...N-1 (variable size)

Child Node Pointer 0O


Number of Records N0 for Child Node 0 (variable size)

Total Number of Records for Child Node 0 (optional, variable size)

Child Node Pointer 1O


Number of Records N1 for Child Node 1 (variable size)

Total Number of Records for Child Node 1 (optional, variable size)
...

Child Node Pointer NO


Number of Records Nn for Child Node N (variable size)

Total Number of Records for Child Node N (optional, variable size)
Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 B-tree Internal Node
Field NameDescription
SignatureThe ASCII character string “ BTIN ” is used to indicate the internal node + of a B-tree.
VersionThe version number for this B-tree internal node. This document describes version 0.
TypeThis field is the type of the B-tree node. It should always be the same as the B-tree type in + the header.
RecordsThe size of this field is determined by the number of records for this node and the record size + (from the header). The format of records depends on the type of B-tree.
Child Node PointerThis field is the address of the child node pointed to by the internal node.
Number of Records in Child NodeThis is the number of records in the child node pointed to by the corresponding Node Pointer.
+ The number of bytes used to store this field is determined by the maximum possible number of records able + to be stored in the child node.
+ The maximum number of records in a child node is computed in the following way: +
    +
  • Subtract the fixed size overhead for the child node (for example, its signature, version, + checksum, and so on and one pointer triplet of information for the child node + (because there is one more pointer triplet than records in each internal node)) from the size + of nodes for the B-tree.
    • +
  • Divide that result by the size of a record plus the pointer triplet of information stored to + reach each child node from this node.
    • +

+ Note that leaf nodes do not encode any child pointer triplets, so the maximum number of records in a + leaf node is just the node size minus the leaf node overhead, divided by the record size.
+ Also note that the first level of internal nodes above the leaf nodes do not encode the Total + Number of Records in Child Node value in the child pointer triplets (since it is the same as + the Number of Records in Child Node), so the maximum number of records in these nodes is + computed with the equation above, but using (Child Pointer, Number of Records in Child + Node) pairs instead of triplets.
+ The number of bytes used to encode this field is the least number of bytes required to encode the + maximum number of records in a child node value for the child nodes below this level in the B-tree.
+ For example, if the maximum number of child records is 123, one byte will be used to encode these + values in this node; if the maximum number of child records is 20000, two bytes will be used to + encode these values in this node; and so on. The maximum number of bytes used to encode these values + is 8 (in other words, an unsigned 64-bit integer).
Total Number of Records in Child NodeThis is the total number of records for the node pointed to by the corresponding Node Pointer + and all its children. This field exists only in nodes whose depth in the B-tree node is greater than 1 + (in other words, the “twig” internal nodes, just above leaf nodes, do not store this field + in their child node pointers).
+ The number of bytes used to store this field is determined by the maximum possible number of records + able to be stored in the child node and its descendants.
+ The maximum possible number of records able to be stored in a child node and its descendants is + computed iteratively, in the following way: The maximum number of records in a leaf node is + computed, then that value is used to compute the maximum possible number of records in the first + level of internal nodes above the leaf nodes. Multiplying these two values together determines the + maximum possible number of records in child node pointers for the level of nodes two levels above + leaf nodes. This process is continued up to any level in the B-tree.
+ The number of bytes used to encode this value is computed in the same way as for the Number + of Records in Child Node field.
ChecksumThis is the checksum for this node.
+ + + + + + + + + + + + + + + + + + + + +
Layout: Version 2 B-tree Leaf Node
bytebytebytebyte
Signature
VersionTypeRecord 0, 1, 2...N-1 (variable size)
Checksum
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 B-tree Leaf Node
Field NameDescription
SignatureThe ASCII character string “ BTLF “ is used to indicate the leaf node + of a version 2 (v2) B-tree.
VersionThe version number for this B-tree leaf node. This document describes version 0.
TypeThis field is the type of the B-tree node. It should always be the same as the B-tree type in + the header.
RecordsThe size of this field is determined by the number of records for this node and the record size + (from the header). The format of records depends on the type of B-tree.
ChecksumThis is the checksum for this node.
+ +The record layout for each stored (in other words, non-testing) B-tree type is as follows: + + + + + + + + + + + + + + + + + +
Layout: Version 2 B-tree, Type 1 Record Layout - Indirectly Accessed, Non-Filtered, + ‘Huge’ Fractal Heap Objects
bytebytebytebyte

Huge Object AddressO


Huge Object LengthL


Huge Object IDL

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 B-tree, Type 1 Record Layout - Indirectly Accessed, Non-Filtered, + ‘Huge’ Fractal Heap Objects
Field NameDescription
Huge Object AddressThe address of the huge object in the file.
Huge Object LengthThe length of the huge object in the file.
Huge Object IDThe heap ID for the huge object.
+ + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 2 B-tree, Type 2 Record Layout - Indirectly Accessed, Filtered, ‘Huge’ + Fractal Heap Objects
bytebytebytebyte

Filtered Huge Object AddressO


Filtered Huge Object LengthL

Filter Mask

Filtered Huge Object Memory SizeL


Huge Object IDL

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 B-tree, Type 2 Record Layout - Indirectly Accessed, Filtered, ‘Huge’ + Fractal Heap Objects
Field NameDescription
Filtered Huge Object AddressThe address of the filtered huge object in the file.
Filtered Huge Object LengthThe length of the filtered huge object in the file.
Filter MaskA 32-bit bit field indicating which filters have been skipped for this chunk. Each filter has + an index number in the pipeline (starting at 0, with the first filter to apply) and if that filter + is skipped, the bit corresponding to its index is set.
Filtered Huge Object Memory SizeThe size of the de-filtered huge object in memory.
Huge Object IDThe heap ID for the huge object.
+ + + + + + + + + + + + + + + +
Layout: Version 2 B-tree, Type 3 Record Layout - Directly Accessed, Non-Filtered, ‘Huge’ + Fractal Heap Objects
bytebytebytebyte

Huge Object AddressO


Huge Object LengthL

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + +
Fields: Version 2 B-tree, Type 3 Record Layout - Directly Accessed, Non-Filtered, ‘Huge’ + Fractal Heap Objects
Field NameDescription
Huge Object AddressThe address of the huge object in the file.
Huge Object LengthThe length of the huge object in the file.
+ + + + + + + + + + + + + + + + + + + + + +
Layout: Version 2 B-tree, Type 4 Record Layout - Directly Accessed, Filtered, ‘Huge’ + Fractal Heap Objects
bytebytebytebyte

Filtered Huge Object AddressO


Filtered Huge Object LengthL

Filter Mask

Filtered Huge Object Memory SizeL

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 B-tree, Type 4 Record Layout - Directly Accessed, Filtered, ‘Huge’ + Fractal Heap Objects
Field NameDescription
Filtered Huge Object AddressThe address of the filtered huge object in the file.
Filtered Huge Object LengthThe length of the filtered huge object in the file.
Filter MaskA 32-bit bit field indicating which filters have been skipped for this chunk. Each filter has an + index number in the pipeline (starting at 0, with the first filter to apply) and if that filter + is skipped, the bit corresponding to its index is set.
Filtered Huge Object Memory SizeThe size of the de-filtered huge object in memory.
+ + + + + + + + + + + + + + + + + + +
Layout: Version 2 B-tree, Type 5 Record Layout - Link Name for Indexed Group
bytebytebytebyte
Hash of Name
ID (bytes 1-4)
ID (bytes 5-7)
+ + + + + + + + + + + + + + + +
Fields: Version 2 B-tree, Type 5 Record Layout - Link Name for Indexed Group
Field NameDescription
HashThis field is hash value of the name for the link. The hash value is the Jenkins’ lookup3 + checksum algorithm applied to the link’s name.
IDThis is a 7-byte sequence of bytes and is the heap ID for the link record in the group’s + fractal heap.
+ + + + + + + + + + + + + + + + + + + +
Layout: Version 2 B-tree, Type 6 Record Layout - Creation Order for Indexed Group
bytebytebytebyte

Creation Order (8 bytes)

ID (bytes 1-4)
ID (bytes 5-7)
+ + + + + + + + + + + + + + + +
Fields: Version 2 B-tree, Type 6 Record Layout - Creation Order for Indexed Group
Field NameDescription
Creation OrderThis field is the creation order value for the link.
IDThis is a 7-byte sequence of bytes and is the heap ID for the link record in the group’s + fractal heap.
+ + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 2 B-tree, Type 7 Record Layout - Shared Object Header Messages + (Sub-Type 0 - Message in Heap)
bytebytebytebyte
Message LocationThis space inserted only to align table nicely
Hash
Reference Count

Heap ID (8 bytes)

+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 B-tree, Type 7 Record Layout - Shared Object Header Messages + (Sub-Type 0 - Message in Heap)
Field NameDescription
Message LocationThis field Indicates the location where the message is stored: + + + + + + + + + + + + + +
ValueDescription
0Shared message is stored in shared message index heap.
1Shared message is stored in object header.
+
HashThis field is hash value of the shared message. The hash value is the Jenkins’ lookup3 + checksum algorithm applied to the shared message.
Reference CountThe number of objects which reference this message.
Heap IDThis is an 8-byte sequence of bytes and is the heap ID for the shared message in the shared + message index’s fractal heap.
+ + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 2 B-tree, Type 7 Record Layout - Shared Object Header Messages + (Sub-Type 1 - Message in Object Header)
bytebytebytebyte
Message LocationThis space inserted only to align table nicely
Hash
Reserved (zero)Message TypeObject Header Index

Object Header AddressO

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 B-tree, Type 7 Record Layout - Shared Object Header Messages + (Sub-Type 1 - Message in Object Header)
Field NameDescription
Message LocationThis field Indicates the location where the message is stored: + + + + + + + + + + + + + +
ValueDescription
0Shared message is stored in shared message index heap.
1Shared message is stored in object header.
+
HashThis field is hash value of the shared message. The hash value is the Jenkins’ lookup3 + checksum algorithm applied to the shared message.
Message TypeThe object header message type of the shared message.
Object Header IndexThis field indicates that the shared message is the nth message of its type in the + specified object header.
Object Header AddressThe address of the object header containing the shared message.
+ + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 2 B-tree, Type 8 Record Layout - Attribute Name for Indexed Attributes
bytebytebytebyte

Heap ID (8 bytes)

Message FlagsThis space inserted only to align table nicely
Creation Order
Hash of Name
+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 B-tree, Type 8 Record Layout - Attribute Name for Indexed Attributes
Field NameDescription
Heap IDThis is an 8-byte sequence of bytes and is the heap ID for the attribute in the object’s + attribute fractal heap.
Message FlagsThe object header message flags for the attribute message.
Creation OrderThis field is the creation order value for the attribute.
HashThis field is hash value of the name for the attribute. The hash value is the Jenkins’ + lookup3 checksum algorithm applied to the attribute’s name.
+ + + + + + + + + + + + + + + + + + + +
Layout: Version 2 B-tree, Type 9 Record Layout- Creation Order for Indexed Attributes
bytebytebytebyte

Heap ID (8 bytes)

Message FlagsThis space inserted only to align table nicely
Creation Order
+ + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 B-tree, Type 9 Record Layout- Creation Order for Indexed Attributes
Field NameDescription
Heap IDThis is an 8-byte sequence of bytes and is the heap ID for the attribute in the object’s + attribute fractal heap.
Message FlagsThe object header message flags for the attribute message.
Creation OrderThis field is the creation order value for the attribute.
+ + + + + + + + + + + + + + + + + + + + + + + + +
\anchor FMT3V2BtType10 Layout: Version 2 B-tree, Type 10 Record Layout - Non-filtered Dataset Chunks
bytebytebytebyte

AddressO


Dimension 0 Scaled Offset (8 bytes)


Dimension 1 Scaled Offset (8 bytes)


...


Dimension \#n Scaled Offset (8 bytes)

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + +
Fields: Version 2 B-tree, Type 11 Record Layout - Filtered Dataset Chunks
Field NameDescription
AddressThis field is the address of the dataset chunk in the file.
Dimension \#n Scaled OffsetThis field is the scaled offset of the chunk within the dataset. n is the number of + dimensions for the dataset. The first scaled offset stored in the list is for the slowest + changing dimension, and the last scaled offset stored is for the fastest changing dimension. + Scaled offset is calculated by dividing the chunk dimension sizes into the chunk offsets.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
\anchor FMT3V2BtType11 Layout: Version 2 B-tree, Type 11 Record Layout - Filtered Dataset Chunks
bytebytebytebyte

AddressO


Chunk Size (variable size; at most 8 bytes)

Filter Mask

Dimension 0 Scaled Offset (8 bytes)


Dimension 1 Scaled Offset (8 bytes)


...


Dimension \#n Scaled Offset (8 bytes)

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 B-tree, Type 5 Record Layout - Non-filtered Dataset Chunks
Field NameDescription
AddressThis field is the address of the dataset chunk in the file.
Chunk SizeThis field is the size of the dataset chunk in bytes.
Filter MaskThis field is the filter mask which indicates the filter + to skip for the dataset chunk. Each filter has an index + number in the pipeline and if that filter is skipped, + the bit corresponding to its index is set.
Dimension \#n Scaled OffsetThis field is the scaled offset of the chunk within the dataset. n is the number of + dimensions for the dataset. The first scaled offset stored in the list is for the slowest + changing dimension, and the last scaled offset stored is for the fastest changing dimension.
+ +\subsection subsec_fmt3_infra_symboltable III.B. Disk Format: Level 1B - Group Symbol Table Nodes +A group is an object internal to the file that allows arbitrary nesting of objects within the file (including +other groups). A group maps a set of link names in the group to a set of relative file addresses of objects +in the file. Certain metadata for an object to which the group points can be cached in +object’s header. + +An HDF5 object name space can be stored hierarchically by partitioning the name into components and storing +each component as a link in a group. The link for a non-ultimate component points to the group containing the +next component. The link for the last component points to the object being named. + +One implementation a group is a collection of symbol table nodes indexed by a B-tree. Each symbol table +node contains entries for one or more links. If an attempt is made to add a link to an already full +symbol table node containing 2K entries, then the node is split and one node contains K +symbols and the other contains K+1 symbols. + + + + + + + + + + + + + + + + + + + + +
Layout: Symbol Table Node (A Leaf of a B-tree)
bytebytebytebyte
Signature
Version NumberReserved (zero)Number of Symbols


Group Entries


+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: Symbol Table Node (A Leaf of a B-tree)
Field NameDescription
SignatureThe ASCII character string SNOD is used to indicate the beginning of a symbol table node. This + gives file consistency checking utilities a better chance of reconstructing a damaged file.
Version NumberThe version number for the symbol table node. This document describes version 1. (There is no version + ‘0’ of the symbol table node)
Number of SymbolsAlthough all symbol table nodes have the same length, most contain fewer than the maximum possible number of + link entries. This field indicates how many entries contain valid data. The valid entries are packed + at the beginning of the symbol table node while the remaining entries contain undefined values.
Group EntriesEach link has an entry in the symbol table node. The format of the entry is described below. There are + 2K entries in each group node, where K is the “Group Leaf Node K” value + from the @ref subsec_fmt3_boot_super.
+ +\subsection subsec_fmt3_infra_symboltableentry III.C. Disk Format: Level 1C - Symbol Table Entry +Each symbol table entry in a symbol table node is designed to allow for very fast browsing of stored objects. +Toward that design goal, the symbol table entries include space for caching certain constant metadata from the +object header. + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Symbol Table Entry
bytebytebytebyte
Link Name OffsetO
Object Header AddressO
Cache Type
Reserved (zero)


Scratch-pad Space (16 bytes)


+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Symbol Table Entry
Field NameDescription
Link Name OffsetThis is the byte offset into the group’s local heap for the name of the link. The name is null + terminated.
Object Header AddressEvery object has an object header which serves as a permanent location for the object’s metadata. + In addition to appearing in the object header, some of the object’s metadata can be cached in the + scratch-pad space.
Cache TypeThe cache type is determined from the object header. It also determines the format for the scratch-pad + space.
+ + + + + + + + + + + + + + + + + +
Type:Description:
0No data is cached by the group entry. This is guaranteed to be the case when an object header has + a link count greater than one.
1Group object header metadata is cached in the scratch-pad space. This implies that the symbol table + entry refers to another group.
2The entry is a symbolic link. The first four bytes of the scratch-pad space are the offset into + the local heap for the link value. The object header address will be undefined.
+
ReservedThese four bytes are present so that the scratch-pad space is aligned on an eight-byte boundary. They + are always set to zero.
Scratch-pad SpaceThis space is used for different purposes, depending on the value of the Cache Type field. Any meta-data + about an object represented in the scratch-pad space is duplicated in the object header for + that object.
+ Furthermore, no data is cached in the group entry scratch-pad space if the object header for the object + has a link count greater than one.
+ +\subsubsection subsubsec_fmt3_infra_symboltableentry_scratch Format of the Scratch-pad Space +The symbol table entry scratch-pad space is formatted according to the value in the Cache Type field. + +If the Cache Type field contains the value zero ((0)) then no information is stored in the +scratch-pad space. + +If the Cache Type field contains the value one (1), then the scratch-pad space contains +cached metadata for another object header in the following format: + + + + + + + + + + + + + +
Layout: Object Header Scratch-pad Format
bytebytebytebyte
Address of B-treeO
Address of Name HeapO
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + +
Fields: Object Header Scratch-pad Format
Field NameDescription
Address of B-treeThis is the file address for the root of the group’s B-tree.
Address of Name HeapThis is the file address for the group’s local heap, in which are stored the group’s + symbol names.
+ +If the Cache Type field contains the value two ((2)), then the scratch-pad space contains +cached metadata for a symbolic link in the following format: + + + + + + + + + + + +
Layout: Symbolic Link Scratch-pad Format
bytebytebytebyte
Offset to Link Value
+ + + + + + + + + + + +
Fields: Symbolic Link Scratch-pad Format
Field NameDescription
Offset to Link ValueThe value of a symbolic link (that is, the name of the thing to which it points) is stored in the + local heap. This field is the 4-byte offset into the local heap for the start of the link value, which + is null terminated.
+ +\subsection subsec_fmt3_infra_localheap III.D. Disk Format: Level 1D - Local Heaps +A local heap is a collection of small pieces of data that are particular to a single object in the HDF5 file. +Objects can be inserted and removed from the heap at any time. The address of a heap does not change once +the heap is created. For example, a group stores addresses of objects in symbol table nodes with the names +of links stored in the group’s local heap. + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Local Heap
bytebytebytebyte
Signature
VersionReserved (zero)
Data Segment SizeL
Offset to Head of Free-listL
Address of Data SegmentO
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Local Heap
Field NameDescription
SignatureThe ASCII character string “HEAP ” is used to indicate the beginning of a heap. + This gives file consistency checking utilities a better chance of reconstructing a damaged file.
VersionEach local heap has its own version number so that new heaps can be added to old files. This document + describes version zero (0) of the local heap.
Data Segment SizeThe total amount of disk memory allocated for the heap data. This may be larger than the amount of space + required by the objects stored in the heap. The extra unused space in the heap holds a linked list of + free blocks.
Offset to Head of Free-listThis is the offset within the heap data segment of the first free block (or the + @ref FMT3UndefinedAddress "undefined address" if there is no no free block). The free block + contains “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” bytes that are the offset of the next free block (or the value + ‘1’ if this is the last free block) followed by “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” bytes that + store the size of this free block. The size of the free block includes the space used to store the + offset of the next free block and the size of the current block, making the minimum size of a free + block 2 * “@ref FMT3SizeOfLengthsV0 "Size of Lengths"”.
Address of Data SegmentThe data segment originally starts immediately after the heap header, but if the data segment must grow + as a result of adding more objects, then the data segment may be relocated, in its entirety, to another + part of the file.
+ +Objects within a local heap should be aligned on an 8-byte boundary. + +\subsection subsec_fmt3_infra_globalheap III.E. Disk Format: Level 1E - Global Heap +Each HDF5 file has a global heap which stores various types of information which is typically shared between +datasets. The global heap was designed to satisfy these goals: +
    +
  1. Repeated access to a heap object must be efficient without resulting in repeated file I/O requests. + Since global heap objects will typically be shared among several datasets, it is probable that the + object will be accessed repeatedly.
    2. +
  2. Collections of related global heap objects should result in fewer and larger I/O requests. For + instance, a dataset of object references will have a global heap object for each reference. Reading + the entire set of object references should result in a few large I/O requests instead of one small + I/O request for each reference.
    3. +
  3. It should be possible to remove objects from the global heap and the resulting file hole should be + eligible to be reclaimed for other uses.
    4. +
+ +The implementation of the heap makes use of the memory management already available at the file level and +combines that with a new object called a collection to achieve goal B. The global heap is +the set of all collections. Each global heap object belongs to exactly one collection and each collection +contains one or more global heap objects. For the purposes of disk I/O and caching, a collection is treated +as an atomic object, addressing goal A. + +When a global heap object is deleted from a collection (which occurs when its reference count falls to zero), +objects located after the deleted object in the collection are packed down toward the beginning of the +collection and the collection’s global heap object 0 is created (if possible) or its size is increased +to account for the recently freed space. There are no gaps between objects in each collection, with the possible +exception of the final space in the collection, if it is not large enough to hold the header for the +collection’s global heap object 0. These features address goal C. + +The HDF5 library creates global heap collections as needed, so there may be multiple collections throughout +the file. The set of all of them is abstractly called the “global heap”, although they do not +actually link to each other, and there is no global place in the file where you can discover all of the +collections. The collections are found simply by finding a reference to one through another object in the file. +For example, data of variable-length datatype elements is stored in the global heap and is accessed via a +global heap ID. The format for global heap IDs is described at the end of this section. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: A Global Heap Collection
bytebytebytebyte
Signature
VersionReserved (zero)

Collection SizeL


Global Heap Object 1


Global Heap Object 2


...


Global Heap Object N


Global Heap Object 0 (free space)

+\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: A Global Heap Collection
Field NameDescription
SignatureThe ASCII character string “GCOL” is used to indicate the beginning of a collection. + This gives file consistency checking utilities a better chance of reconstructing a damaged file.
VersionEach collection has its own version number so that new collections can be added to old files. This + document describes version one (1) of the collections (there is no version zero (0)).
Collection SizeThis is the size in bytes of the entire collection including this field. The default (and minimum) + collection size is 4096 bytes which is a typical file system block size. This allows for 127 16-byte + heap objects plus their overhead (the collection header of 16 bytes and the 16 bytes of information + about each heap object).
Global Heap Object 1 through NThe objects are stored in any order with no intervening unused space.
Global Heap Object 0Global Heap Object 0 (zero), when present, represents the free space in the collection. Free space always + appears at the end of the collection. If the free space is too small to store the header for Object 0 + (described below) then the header is implied and is not written.
+ The field Object Size for Object 0 indicates the amount of possible free space in the collection + including the 16-byte header size of Object 0.
+ + + + + + + + + + + + + + + + + + + + + + +
Layout: Global Heap Object
bytebytebytebyte
Heap Object IndexReference Count
Reserved (zero)

Object SizeL


Object Data

+\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Global Heap Object
Field NameDescription
Heap Object IndexEach object has a unique identification number within a collection. The identification numbers are + chosen so that new objects have the smallest value possible with the exception that the identifier + 0 always refers to the object which represents all free space within the collection.
Reference CountAll heap objects have a reference count field. An object which is referenced from some other part of the + file will have a positive reference count. The reference count for Object 0 is always zero.
ReservedZero padding to align next field on an 8-byte boundary.
Object Size This is the size of the object data stored for the object. The actual storage space + allocated for the object data is rounded up to a multiple of eight.
Object DataThe object data is treated as a one-dimensional array of bytes to be interpreted by the caller.
+ +
+\anchor FMT3GlobalHeapID

The format for the ID used to locate an object in the global heap is described here:

+ + + + + + + + + + + + + + +
Layout: Global Heap ID
bytebytebytebyte

Collection AddressO

Object Index
+\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + +
Fields: Global Heap ID
Field NameDescription
Collection AddressThis field is the address of the global heap collection where the data object is stored.
IDThis field is the index of the data object within the global heap collection.
+ +\subsection subsec_fmt3_infra_globalheapvds III.F. Disk Format: Level 1F - Global Heap Block for Virtual Datasets +The layout for the global heap block used with virtual datasets is described below. For more information +on global heaps, see “ @ref subsec_fmt3_infra_globalheap ” + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Global Heap Block for Virtual Dataset
bytebytebytebyte
VersionThis space inserted only to align table nicely

Num EntriesL


Source Filename \#1 (variable size)


Source Dataset \#1 (variable size)


Source Selection \#1 (variable size)


Virtual Selection \#1 (variable size)

.
.
.

Source Filename \#n (variable size)


Source Dataset \#n (variable size)


Source Selection \#n (variable size)


Virtual Selection \#n (variable size)

Checksum
+\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Global Heap Block for Virtual Dataset
Field NameDescription
VersionThe version number for the block; the value is 0.

Num EntriesL

The number of entries in the block.

Source Filename \#n (variable size)

The source file name where the source dataset is located.

Source Dataset \#n (variable size)

The source dataset name that is mapped to the virtual dataset.

Source Selection \#n (variable size)

The @ref FMT3DataspaceSEL "dataspace selection" in the source dataset that is mapped + to the virtual selection.

Virtual Selection \#n (variable size)

This is the @ref FMT3DataspaceSEL "dataspace selection" in the virtual dataset that + is mapped to the source selection.
ChecksumThis is the checksum for the block.
+ +\subsection subsec_fmt3_infra_fractalheap III.G. Disk Format: Level 1G - Fractal Heap +Each fractal heap consists of a header and zero or more direct and indirect blocks (described below). +The header contains general information as well as initialization parameters for the doubling +table. The Address of Root Block field in the header points to the first direct or indirect block in +the heap. + +Fractal heaps are based on a data structure called a doubling table. A doubling table provides +a mechanism for quickly extending an array-like data structure that minimizes the number of empty blocks +in the heap, while retaining very fast lookup of any element within the array. More information on +fractal heaps and doubling tables can be found in the RFC +“\ref_rfc20070115 .” + +The fractal heap implements the doubling table structure with indirect and direct blocks. Indirect +blocks in the heap do not actually contain data for objects in the heap, their “size” is +abstract - they represent the indexing structure for locating the direct blocks in the doubling table. +Direct blocks contain the actual data for objects stored in the heap. + +All indirect blocks have a constant number of block entries in each row, called the width +of the doubling table (see Table Width field in the header). The number of rows for each indirect +block in the heap is determined by the size of the block that the indirect block represents in the doubling table +(calculation of this is shown below) and is constant, except for the “root” indirect block, +which expands and shrinks its number of rows as needed. + +Blocks in the first two rows of an indirect block are Starting Block Size number of +bytes in size. For example, if the row width of the doubling table is 4, then the first eight block +entries in the indirect block are Starting Block Size number of bytes in size. The blocks in each +subsequent row are twice the size of the blocks in the previous +row. In other words, blocks in the third row are twice the Starting Block Size, blocks in the +fourth row are four times the Starting Block Size, and so on. Entries for blocks up to the +Maximum Direct Block Size point to direct blocks, and entries for blocks greater than that size +point to further indirect blocks (which have their own entries for direct and indirect blocks). Starting +Block Size and Maximum Direct Block Size are fields stored in the header. + +The number of rows of blocks, nrows, in an indirect block is calculated +by the following expression:

+nrows = (log2(iblock_size) - log2(<Starting Block Size>)) + 1 +where block_size is the size of the block that the indirect block +represents in the doubling table. For example, to represent a block with block_size equals to 1024, +and Starting Block Size equals to 256, three rows are needed. + +The maximum number of rows of direct blocks, max_dblock_rows, in any indirect block of a fractal +heap is given by the following expression:

+max_dblock_rows = (log2(<Maximum Direct Block Size>) - +log2(<Starting Block Size>)) + 2 + +Using the computed values for nrows and max_dblock_rows, along with the Width +of the doubling table, the number of direct and indirect block entries (K and N in the +indirect block description, below) in an indirect block can be computed:

+K = MIN(nrows, max_dblock_rows) * Table Width

+If nrows is less than or equal to max_dblock_rows, N is 0. Otherwise, N +is simply computed:

+N = K - (max_dblock_rows * Table Width) + +The size of indirect blocks on disk is determined by the number of rows in the indirect block +(computed above). The size of direct blocks on disk is exactly the size of the block in the doubling table. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Fractal Heap Header
bytebytebytebyte
Signature
VersionThis space inserted only to align table nicely
Heap ID LengthI/O Filters’ Encoded Length
FlagsThis space inserted only to align table nicely
Maximum Size of Managed Objects

Next Huge Object IDL


v2 B-tree Address of Huge ObjectsO


Amount of Free Space in Managed BlocksL


Address of Managed Block Free Space ManagerO


Amount of Managed Space in HeapL


Amount of Allocated Managed Space in HeapL


Offset of Direct Block Allocation Iterator in Managed SpaceL


Number of Managed Objects in HeapL


Size of Huge Objects in HeapL


Number of Huge Objects in HeapL


Size of Tiny Objects in HeapL


Number of Tiny Objects in HeapL

Table WidthThis space insertedonly to align table nicely

Starting Block SizeL


Maximum Direct Block SizeL

Maximum Heap SizeStarting \# of Rows in Root Indirect Block

Address of Root BlockO

Current \# of Rows in Root Indirect BlockThis space inserted only to align table nicely

Size of Filtered Root Direct Block (optional)L

I/O Filter Mask (optional)
I/O Filter Information (optional, variable size)
Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Fractal Heap Header
Field NameDescription
SignatureThe ASCII character string “FRHP” is used to indicate the beginning of a + fractal heap header. This gives file consistency checking utilities a better chance of reconstructing + a damaged file.
VersionThis document describes version 0.
Heap ID LengthThis is the length in bytes of heap object IDs for this heap.
I/O Filters’ Encoded LengthThis is the size in bytes of the encoded I/O Filter Information.
FlagsThis field is the heap status flag and is a bit field indicating additional information about + the fractal heap. + + + + + + + + + + + + + + + + + +
Bit(s)Description
0If set, the ID value to use for huge object has wrapped around. If the value for the + Next Huge Object ID has wrapped around, each new huge object inserted into the + heap will require a search for an ID value. +
1If set, the direct blocks in the heap are checksummed.
2-7Reserved
Maximum Size of Managed ObjectsThis is the maximum size of managed objects allowed in the heap. Objects greater than this this + are ‘huge’ objects and will be stored in the file directly, rather than in a direct + block for the heap.
Next Huge Object IDThis is the next ID value to use for a huge object in the heap.
v2 B-tree Address of Huge ObjectsThis is the address of the @ref subsubsec_fmt3_infra_btrees_v2 used to track huge objects in the heap. + The type of records stored in the v2 B-tree will be determined by whether the address and + length of a huge object can fit into a heap ID (if yes, it is a “directly” accessed huge + object) and whether there is a filter used on objects in the heap.
Amount of Free Space in Managed BlocksThis is the total amount of free space in managed direct blocks (in bytes).
Address of Managed Block Free Space ManagerThis is the address of the @ref subsec_fmt3_infra_freespaceindex + for managed blocks.
Amount of Managed Space in HeapThis is the total amount of managed space in the heap (in bytes), essentially the + upper bound of the heap’s linear address space.
Amount of Allocated Managed Space in HeapThis is the total amount of managed space (in bytes) actually allocated in the heap. + This can be less than the Amount of Managed Space in Heap field, if some direct + blocks in the heap’s linear address space are not allocated.
Offset of Direct Block Allocation Iterator in Managed SpaceThis is the linear heap offset where the next direct block should be allocated at (in bytes). + This may be less than the Amount of Managed Space in Heap value because the heap’s + address space is increased by a “row” of direct blocks at a time, rather than by single + direct block increments.
Number of Managed Objects in HeapThis is the number of managed objects in the heap.
Size of Huge Objects in HeapThis is the total size of huge objects in the heap (in bytes).
Number of Huge Objects in HeapThis is the number of huge objects in the heap.
Size of Tiny Objects in HeapThis is the total size of tiny objects that are packed in heap IDs (in bytes).
Number of Tiny Objects in HeapThis is the number of tiny objects that are packed in heap IDs.
Table WidthThis is the number of columns in the doubling table for managed blocks. This value + must be a power of two.
Starting Block SizeThis is the starting block size to use in the doubling table for managed blocks (in bytes). + This value must be a power of two.
Maximum Direct Block SizeThis is the maximum size allowed for a managed direct block. Objects inserted into the heap that + are larger than this value (less the number of bytes of direct block prefix/suffix) are stored as + ‘huge’ objects. This value must be a power of two.
Maximum Heap SizeThis is the maximum size of the heap’s linear address space for managed objects (in bytes). + The value stored is the log2 of the actual value, that is: the number of bits of the address space. + ‘Huge’ and ‘tiny’ objects are not counted in this value, since they do not + store objects in the linear address space of the heap.
Starting \# of Rows in Root Indirect BlockThis is the starting number of rows for the root indirect block. A value of 0 indicates that the + root indirect block will have the maximum number of rows needed to address the heap’s + Maximum Heap Size.
Address of Root BlockThis is the address of the root block for the heap. It can be the + @ref FMT3UndefinedAddress "undefined address" if there is no data in the heap. It either + points to a direct block (if the Current \# of Rows in the Root Indirect Block value is 0), + or an indirect block.
Current \# of Rows in Root Indirect BlockThis is the current number of rows in the root indirect block. A value of 0 indicates that + Address of Root Block points to direct block instead of indirect block.
Size of Filtered Root Direct BlockThis is the size of the root direct block, if filters are applied to heap objects (in bytes). + This field is only stored in the header if the I/O Filters’ Encoded Length is + greater than 0.
I/O Filter MaskThis is the filter mask for the root direct block, if filters are applied to heap objects. This + mask has the same format as that used for the filter mask in chunked raw data records in a + @ref subsubsec_fmt3_infra_btrees_v1. This field is only stored in the header if the I/O Filters’ + Encoded Length is greater than 0.
I/O Filter InformationThis is the I/O filter information encoding direct blocks and huge objects, if filters are applied to + heap objects. This field is encoded as a @ref subsubsec_fmt3_dataobject_hdr_msg_filter message. The size + of this field is determined by I/O Filters’ Encoded Length.
ChecksumThis is the checksum for the header.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Fractal Heap Direct Block
bytebytebytebyte
Signature
VersionThis space inserted only to align table nicely

Heap Header AddressO

Block Offset (variable size)
Checksum (optional)

Object Data (variable size)

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Fractal Heap Direct Block
Field NameDescription
SignatureThe ASCII character string “FHDB” is used to indicate the beginning of + a fractal heap direct block. This gives file consistency checking utilities a better chance of + reconstructing a damaged file.
VersionThis document describes version 0.
Heap Header AddressThis is the address for the fractal heap header that this block belongs to. This field is + principally used for file integrity checking.
Block OffsetThis is the offset of the block within the fractal heap’s address space (in bytes). The + number of bytes used to encode this field is the Maximum Heap Size (in the heap’s + header) divided by 8 and rounded up to the next highest integer, for values that are not a multiple + of 8. This value is principally used for file integrity checking.
ChecksumThis is the checksum for the direct block. This field is only present if bit 1 of Flags + in the heap’s header is set.
Object DataThis section of the direct block stores the actual data for objects in the heap. The size of this + section is determined by the direct block’s size minus the size of the other fields stored in the + direct block (for example, the Signature, Version, and others including the + Checksum if it is present).
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Fractal Heap Indirect Block
bytebytebytebyte
Signature
VersionThis space inserted only to align table nicely

Heap Header AddressO

Block Offset (variable size)

Child Direct Block \#0 AddressO


Size of Filtered Direct Block \#0 (optional) L

Filter Mask for Direct Block \#0 (optional)

Child Direct Block \#1 AddressO


Size of Filtered Direct Block \#1 (optional)L

Filter Mask for Direct Block \#1 (optional)
...

Child Direct Block \#K-1 AddressO


Size of Filtered Direct Block \#K-1 (optional)L

Filter Mask for Direct Block \#K-1 (optional)

Child Indirect Block \#0 AddressO


Child Indirect Block \#1 AddressO

...

Child Indirect Block \#N-1 AddressO

Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Fractal Heap Indirect Block
Field NameDescription
SignatureThe ASCII character string “FHIB” is used to indicate the beginning of a + fractal heap indirect block. This gives file consistency checking utilities a better chance of + reconstructing a damaged file.
VersionThis document describes version 0.
Heap Header AddressThis is the address for the fractal heap header that this block belongs to. This field is principally + used for file integrity checking.
Block OffsetThis is the offset of the block within the fractal heap’s address space (in bytes). The number + of bytes used to encode this field is the Maximum Heap Size (in the heap’s header) + divided by 8 and rounded up to the next highest integer, for values that are not a multiple of 8. This + value is principally used for file integrity checking.
Child Direct Block \#K AddressThis field is the address of the child direct block. The size of the [uncompressed] direct block can + be computed by its offset in the heap’s linear address space.
Size of Filtered Direct Block \#KThis is the size of the child direct block after passing through the I/O filters defined for this heap + (in bytes). If no I/O filters are present for this heap, this field is not present.
Filter Mask for Direct Block \#KThis is the I/O filter mask for the filtered direct block. This mask has the same format as that + used for the filter mask in chunked raw data records in a @ref subsubsec_fmt3_infra_btrees_v1. If + no I/O filters are present for this heap, this field is not present.
Child Indirect Block \#N AddressThis field is the address of the child indirect block. The size of the indirect block can be computed + by its offset in the heap’s linear address space.
ChecksumThis is the checksum for the indirect block.
+ +An object in the fractal heap is identified by means of a fractal heap ID, which encodes information to +locate the object in the heap. Currently, the fractal heap stores an object in one of three ways, +depending on the object’s size: + + + + + + + + + + + + + + + + + +
TypeDescription
TinyWhen an object is small enough to be encoded in the heap ID, the object’s data is embedded + in the fractal heap ID itself. There are two sub-types for this type of object: normal and extended. + The sub-type for tiny heap IDs depends on whether the heap ID is large enough to store objects + greater than 16 bytes or not. If the heap ID length is 18 bytes or smaller, the ‘normal’ + tiny heap ID form is used. If the heap ID length is greater than 18 bytes in length, the + “extended” form is used. See format description below for both sub-types.
HugeWhen the size of an object is larger than Maximum Size of Managed Objects in the + Fractal Heap Header, the object’s data is stored on its own in the file and the object + is tracked/indexed via a version 2 B-tree. All huge objects for a particular fractal heap use the same + v2 B-tree. All huge objects for a particular fractal heap use the same format for their huge object IDs. +
Depending on whether the IDs for a heap are large enough to hold the object’s retrieval + information and whether I/O pipeline filters are applied to the heap’s objects, 4 sub-types are + derived for huge object IDs for this heap: + + + + + + + + + + + + + + + + + + + + + +
Sub-typeDescription
Directly accessed, non-filteredThe object’s address and length are embedded in the fractal heap ID itself and the + object is directly accessed from them. This allows the object to be accessed without resorting + to the B-tree.
Directly accessed, filteredThe filtered object’s address, length, filter mask and de-filtered size are embedded + in the fractal heap ID itself and the object is accessed directly with them. This allows the + object to be accessed without resorting to the B-tree.
Indirectly accessed, non-filteredThe object is located by using a B-tree key embedded in the fractal heap ID to retrieve the + address and length from the version 2 B-tree for huge objects. Then, the address and length + are used to access the object.
Indirectly accessed, filteredThe object is located by using a B-tree key embedded in the fractal heap ID to retrieve the + filtered object’s address, length, filter mask and de-filtered size from the version + 2 B-tree for huge objects. Then, this information is used to access the object.
ManagedWhen the size of an object does not meet the above two conditions, the object is stored and managed + via the direct and indirect blocks based on the doubling table.
+ +The specific format for each type of heap ID is described below: + + + + + + + + + + + + + + + +
Layout: Fractal Heap ID for Tiny Objects (sub-type 1 - ‘Normal’)
bytebytebytebyte
Version, Type & LengthThis space inserted only to align table nicely

Data (variable size)
+ + + + + + + + + + + + + + + +
Fields: Fractal Heap ID for Tiny Objects (sub-type 1 - ‘Normal’)
Field NameDescription
Version, Type, and LengthThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
6-7The current version of ID format. This document describes version 0.
4-5The ID type. Tiny objects have a value of 2. +
0-3The length of the tiny object. The value stored is one less than the actual length (since + zero-length objects are not allowed to be stored in the heap). For example, an object of + actual length 1 has an encoded length of 0, an object of actual length 2 has an encoded + length of 1, and so on.
DataThis is the data for the object.
+ + + + + + + + + + + + + + + + + +
Layout: Fractal Heap ID for Tiny Objects (sub-type 2 - ‘Extended’)
bytebytebytebyte
Version, Type, and LengthExtended LengthThis space inserted only to align table nicely
Data (variable size)
+ + + + + + + + + + + + + + + + + + + +
Fields: Fractal Heap ID for Tiny Objects (sub-type 2 - ‘Extended’)
Field NameDescription
Version, Type, and LengthThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
6-7The current version of ID format. This document describes version 0.
4-5The ID type. Tiny objects have a value of 2.
0-3These 4 bits, together with the next byte, form an unsigned 12-bit integer for holding the + length of the object. These 4-bits are bits 8-11 of the 12-bit integer. See description + for the Extended Length field below.
Extended LengthThis byte, together with the 4 bits in the previous byte, forms an unsigned 12-bit integer for + holding the length of the tiny object. These 8 bits are bits 0-7 of the 12-bit integer formed. The + value stored is one less than the actual length (since zero-length objects are not allowed to be + stored in the heap). For example, an object of actual length 1 has an encoded length of 0, an object of + actual length 2 has an encoded length of 1, and so on.
DataThis is the data for the object.
+ + + + + + + + + + + + + + + + +
Layout: Fractal Heap ID for Huge Objects (sub-type 1 & 2): indirectly accessed, + non-filtered/filtered
bytebytebytebyte
Version and TypeThis space inserted only to align table nicely

v2 B-tree KeyL (variable size)

+\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + +
Fields: Fractal Heap ID for Huge Objects (sub-type 1 & 2): indirectly accessed, + non-filtered/filtered
Field NameDescription
Version and TypeThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
6-7The current version of ID format. This document describes version 0.
4-5The ID type. Huge objects have a value of 1.
0-3Reserved.
v2 B-tree KeyThis field is the B-tree key for retrieving the information from the version 2 B-tree for huge + objects needed to access the object. See the description of @ref subsubsec_fmt3_infra_btrees_v2 + records sub-type 1 & 2 for a description of the fields. New key values are derived from Next + Huge Object ID in the Fractal Heap Header.
+ + + + + + + + + + + + + + + + + + + +
Layout: Fractal Heap ID for Huge Objects (sub-type 3): directly accessed, non-filtered
bytebytebytebyte
Version and TypeThis space inserted only to align table nicely

Address O


Length L

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + +
Fields: Fractal Heap ID for Huge Objects (sub-type 3): directly accessed, non-filtered
Field NameDescription
Version and TypeThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
6-7The current version of ID format. This document describes version 0.
4-5The ID type. Huge objects have a value of 1.
0-3Reserved.
AddressThis field is the address of the object in the file.
LengthThis field is the length of the object in the file.
+ + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Fractal Heap ID for Huge Objects (sub-type 4): directly accessed, filtered
bytebytebytebyte
Version and TypeThis space inserted only to align table nicely

Address O


Length L

Filter Mask

De-filtered Size L

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Fractal Heap ID for Huge Objects (sub-type 4): directly accessed, filtered
Field NameDescription
Version and TypeThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
6-7The current version of ID format. This document describes version 0.
4-5The ID type. Huge objects have a value of 1.
0-3Reserved.
AddressThis field is the address of the filtered object in the file.
LengthThis field is the length of the filtered object in the file.
Filter MaskThis field is the I/O pipeline filter mask for the filtered object in the file.
Filtered SizeThis field is the size of the de-filtered object in the file.
+ + + + + + + + + + + + + + + + + + + +
Layout: Fractal Heap ID for Managed Objects
bytebytebytebyte
Version and TypeThis space inserted only to align table nicely
Offset (variable size)
Length (variable size)
+ + + + + + + + + + + + + + + + + + + +
Fields: Fractal Heap ID for Managed Objects
Field NameDescription
Version and TypeThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
6-7The current version of ID format. This document describes version 0.
4-5The ID type. Managed objects have a value of 0.
0-3Reserved.
OffsetThis field is the offset of the object in the heap. This field’s size is the minimum number of + bytes necessary to encode the Maximum Heap Size value (from the Fractal Heap Header). + For example, if the value of the Maximum Heap Size is less than 256 bytes, this field is 1 + byte in length, a Maximum Heap Size of 256-65535 bytes uses a 2 byte length, and so on.
LengthThis field is the length of the object in the heap. It is determined by taking the minimum value + of Maximum Direct Block Size and Maximum Size of Managed Objects in the Fractal + Heap Header. Again, the minimum number of bytes needed to encode that value is used for the size + of this field.
+ +\subsection subsec_fmt3_infra_freespaceindex III.H. Disk Format: Level 1H - Free-space Index +Free-space managers are used to describe space within a heap or the entire HDF5 file that is not currently +used for that heap or file. + +The free-space manager header contains metadata information about the space being tracked, along +with the address of the list of free space sections which actually describes the free space. The +header records information about free-space sections being tracked, creation parameters for handling +free-space sections of a client, and section information used to locate the collection of free-space sections. + +The free-space section list stores a collection of free-space sections that is specific to each +client of the free-space manager. For example, the fractal heap is a client of the free space +manager and uses it to track unused space within the heap. There are 4 types of section records for the +fractal heap, each of which has its own format, listed below. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Free-space Manager Header
bytebytebytebyte
Signature
VersionClient IDThis space inserted only to align table nicely

Total Space TrackedL


Total Number of SectionsL


Number of Serialized SectionsL


Number of Un-Serialized SectionsL

Number of Section ClassesThis space inserted only to align table nicely
Shrink PercentExpand Percent
Size of Address SpaceThis space inserted only to align table nicely

Maximum Section Size L


Address of Serialized Section ListO


Size of Serialized Section List UsedL


Allocated Size of Serialized Section ListL

Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Free-space Manager Header
Field NameDescription
SignatureThe ASCII character string “FSHD” is used to indicate the beginning of the + Free-space Manager Header. This gives file consistency checking utilities a better chance of + reconstructing a damaged file.
VersionThis is the version number for the Free-space Manager Header and this document describes version 0.
Client IDThis is the client ID for identifying the user of this free-space manager: + + + + + + + + + + + + + + + + + +
IDDescription
0Fractal heap
1File
2+Reserved.
Total Space TrackedThis is the total amount of free space being tracked, in bytes.
Total Number of SectionsThis is the total number of free-space sections being tracked.
Number of Serialized SectionsThis is the number of serialized free-space sections being tracked.
Number of Un-Serialized SectionsThis is the number of un-serialized free-space sections being managed. Un-serialized sections are + created by the free-space client when the list of sections is read in.
Number of Section ClassesThis is the number of section classes handled by this free space manager for the free-space client.
Shrink PercentThis is the percent of current size to shrink the allocated serialized free-space section list.
Expand PercentThis is the percent of current size to expand the allocated serialized free-space section list.
Size of Address SpaceThis is the size of the address space that free-space sections are within. This is stored as the + log2 of the actual value (in other words, the number of bits required to store values + within that address space).
Maximum Section SizeThis is the maximum size of a section to be tracked.
Address of Serialized Section ListThis is the address where the serialized free-space section list is stored.
Size of Serialized Section List UsedThis is the size of the serialized free-space section list used (in bytes). This value must be + less than or equal to the allocated size of serialized section list, below.
Allocated Size of Serialized Section ListThis is the size of serialized free-space section list actually allocated (in bytes).
ChecksumThis is the checksum for the free-space manager header.
+ +The free-space sections being managed are stored in a free-space section list, described below. +The sections in the free-space section list are stored in the following way: a count of the number of sections +describing a particular size of free space and the size of the free-space described (in bytes), followed +by a list of section description records; then another section count and size, followed by the list of +section descriptions for that size; and so on. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Free-space Section List
bytebytebytebyte
Signature
VersionThis space inserted only to align table nicely

Free-space Manager Header AddressO

Number of Section Records in Set \#0 (variable size)
Size of Free-space Section Described in Record Set \#0 (variable size)
Record Set \#0 Section Record \#0 Offset (variable size)
Record Set \#0 Section Record #0 TypeThis space inserted only to align table nicely
Record Set \#0 Section Record \#0 Data (variable size)
...
Record Set \#0 Section Record \#K-1 Offset (variable size)
Record Set \#0 Section Record \#K-1 TypeThis space inserted only to align table nicely
Record Set \#0 Section Record \#K-1 Data (variable size)
Number of Section Records in Set \#1 (variable size)
Size of Free-space Section Described in Record Set \#1 (variable size)
Record Set \#1 Section Record \#0 Offset (variable size)
Record Set \#1 Section Record \#0 TypeThis space inserted only to align table nicely
Record Set \#1 Section Record \#0 Data (variable size)
...
Record Set \#1 Section Record \#K-1 Offset (variable size)
Record Set \#1 Section Record \#K-1 TypeThis space inserted only to align table nicely
Record Set \#1 Section Record \#K-1 Data (variable size)
...
...
Number of Section Records in Set \#N-1 (variable size)
Size of Free-space Section Described in Record Set \#N-1 (variable size)
Record Set \#N-1 Section Record \#0 Offset (variable size)
Record Set \#N-1 Section Record \#0 TypeThis space inserted only to align table nicely
Record Set \#N-1 Section Record \#0 Data (variable size)
...
Record Set \#N-1 Section Record \#K-1 Offset (variable size)
Record Set \#N-1 Section Record \#K-1 TypeThis space inserted only to align table nicely
Record Set \#N-1 Section Record \#K-1 Data (variable size)
Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Free-space Section List
Field NameDescription
SignatureThe ASCII character string “FSSE” is used to indicate the beginning of the + Free-space Section Information. This gives file consistency checking utilities a better chance of + reconstructing a damaged file.
VersionThis is the version number for the Free-space Section List and this document describes version 0.
Free-space Manager Header AddressThis is the address of the Free-space Manager Header. This field is principally used for file + integrity checking.
Number of Section Records for Set \#NThis is the number of free-space section records for set \#N. The length of this field is the minimum + number of bytes needed to store the number of serialized sections (from the free-space + manager header).
+ The number of sets of free-space section records is determined by the size of serialized section + list in the free-space manager header.
Section Size for Record Set \#NThis is the size (in bytes) of the free-space section described for all the section records + in set \#N.
+ The length of this field is the minimum number of bytes needed to store the maximum section + size (from the free-space manager header).
Record Set \#N Section \#K OffsetThis is the offset (in bytes) of the free-space section within the client for the free-space manager. +
The length of this field is the minimum number of bytes needed to store the size of address + space (from the free-space manager header).
Record Set \#N Section \#K TypeThis is the type of the section record, used to decode the record set \#N section \#K data + information. The defined record type for file client is: + + + + + + + + + + + + + +
TypeDescription
0File’s section (a range of actual bytes in file)
1+Reserved.
+
The defined record types for a fractal heap client are: + + + + + + + + + + + + + + + + + + + + + + + + + +
TypeDescription
0Fractal heap “single” section
1Fractal heap “first row” section
2Fractal heap “normal row” section
3Fractal heap “indirect” section
4+Reserved.
Record Set \#N Section \#K DataThis is the section-type specific information for each record in the record set, described below.
ChecksumThis is the checksum for the Free-space Section List.
+ +The section-type specific data for each free-space section record is described below: + + + + + +
Layout: File’s Section Data Record
No additional record data stored
+
+ + + + + +
Layout: Fractal Heap “Single” Section Data Record
No additional record data stored
+
+ + + + + +
Layout: Fractal Heap “First Row” Section Data Record
Same format as “indirect” section data
+
+ + + + + +
Layout: Fractal Heap “Normal Row” Section Data Record
No additional record data stored
+
+ + + + + + + + + + + + + + + + + + + +
Layout: Fractal Heap “Indirect” Section Data Record
bytebytebytebyte
Fractal Heap Indirect Block Offset (variable size)
Block Start RowBlock Start Column
Number of BlocksThis space inserted only to align table nicely
+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: Fractal Heap “Indirect” Section Data Record
Field NameDescription
Fractal Heap Block OffsetThe offset of the indirect block in the fractal heap’s address space containing the empty + blocks.
+ The number of bytes used to encode this field is the minimum number of bytes needed to encode + values for the Maximum Heap Size (in the fractal heap’s header).
Block Start RowThis is the row that the empty blocks start in.
Block Start ColumnThis is the column that the empty blocks start in.
Number of BlocksThis is the number of empty blocks covered by the section.
+ +\subsection subsec_fmt3_infra_sohm III.I. Disk Format: Level 1I - Shared Object Header Message Table +The shared object header message table is used to locate object header messages that are shared +between two or more object headers in the file. Shared object header messages are stored and indexed in +the file in one of two ways: indexed sequentially in a shared header message list or indexed +with a v2 B-tree. The shared messages themselves are either stored in a fractal heap (when two or more +objects share the message), or remain in an object’s header (when only one object uses the message +currently, but the message can be shared in the future). + +The shared object header message table contains a list of shared message index headers. Each +index header records information about the version of the index format, the index storage type, flags +for the message types indexed, the number of messages in the index, the address where the index resides, +and the fractal heap address if shared messages are stored there. + +Each index can be either a list or a v2 B-tree and may transition between those two forms as the number +of messages in the index varies. Each shared message record contains information used to locate the +shared message from either a fractal heap or an object header. The types of messages that can be shared +are: Dataspace, Datatype, Fill Value, Filter Pipeline and Attribute. + +The shared object header message table is pointed to from a +@ref subsubsec_fmt3_dataobject_hdr_msg_shared message in the superblock extension for a file. This +message stores the version of the table format, along with the number of index headers in the table. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Shared Object Header Message Table
bytebytebytebyte
Signature
Version for index \#0Index Type for index #0Message Type Flags for index \#0
Minimum Message Size for index \#0
List Cutoff for index \#0v2 B-tree Cutoff for index \#0
Number of Messages for index \#0This space inserted only to align table nicely

Index AddressO for index \#0


Fractal Heap AddressO for index \#0

...
...
Version for index \#N-1Index Type for index \#N-1Message Type Flags for index \#N-1
Minimum Message Size for index \#N-1
List Cutoff for index \#N-1v2 B-tree Cutoff for index \#N-1
Number of Messages for index \#N-1This space inserted only to align table nicely

Index AddressO for index \#N-1


Fractal Heap AddressO for index \#N-1

Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Shared Object Header Message Table
Field NameDescription
SignatureThe ASCII character string “SMTB” is used to indicate the beginning of the + Shared Object Header Message table. This gives file consistency checking utilities a better chance + of reconstructing a damaged file.
Version for index \#NThis is the version number for the list of shared object header message indexes and this document + describes version 0.
Index Type for index \#NThe type of index can be an unsorted list or a v2 B-tree.
Message Type Flags for index \#NThis field indicates the type of messages tracked in the index, as follows: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
BitsDescription
0If set, the index tracks Dataspace Messages.
1If set, the message tracks Datatype Messages.
2If set, the message tracks Fill Value Messages.
3If set, the message tracks Filter Pipeline Messages.
4If set, the message tracks Attribute Messages.
5-15Reserved (zero).
+ An index can track more than one type of message, but each type of message can only by in one index.
Minimum Message Size for index \#NThis is the message size sharing threshold for the index. If the encoded size of the message is + less than this value, the message is not shared.
List Cutoff for index \#NThis is the cutoff value for the indexing of messages to switch from a list to a v2 B-tree. If the + number of messages is greater than this value, the index should be a v2 B-tree.
v2 B-tree Cutoff for index \#NThis is the cutoff value for the indexing of messages to switch from a v2 B-tree back to a list. + If the number of messages is less than this value, the index should be a list.
Number of Messages for index \#NThe number of shared messages being tracked for the index.
Index Address for index \#NThis field is the address of the list or v2 B-tree where the index nodes reside.
Fractal Heap Address for index \#NThis field is the address of the fractal heap if shared messages are stored there.
ChecksumThis is the checksum for the table.
+ +Shared messages are indexed either with a shared message record list, described below, +or using a v2 B-tree (using record type 7). The number of records in the shared message record +list is determined in the index’s entry in the shared object header message table. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Shared Message Record List
bytebytebytebyte
Signature
Shared Message Record \#0
Shared Message Record \#1
...
Shared Message Record \#N-1
Checksum
+ + + + + + + + + + + + + + + + + + + +
Fields: Shared Message Record List
Field NameDescription
SignatureThe ASCII character string “SMLI” is used to indicate the beginning of a + list of index nodes. This gives file consistency checking utilities a better chance of + reconstructing a damaged file.
Shared Message Record \#NThe record for locating the shared message, either in the fractal heap for the index, or an object + header (see format for index nodes below).
ChecksumThis is the checksum for the list.
+ +The record for each shared message in an index is stored in one of the following forms: + + + + + + + + + + + + + + + + + + + + + +
Layout: Shared Message Record, for Messages Stored in a Fractal Heap
bytebytebytebyte
Message LocationThis space inserted only to align table nicely
Hash Value
Reference Count

Fractal Heap ID

+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: Shared Message Record, for Messages Stored in a Fractal Heap
Field NameDescription
Message LocationThis has a value of 0 indicating that the message is stored in the heap.
Hash ValueThis is the hash value for the message.
Reference CountThis is the number of times the message is used in the file.
Fractal Heap IDThis is an 8-byte fractal heap ID for the message as stored in the fractal heap for the index.
+ + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Shared Message Record, for Messages Stored in an Object Header
bytebytebytebyte
Message LocationThis space inserted only to align table nicely
Hash Value
ReservedMessage TypeCreation Index

Object Header AddressO

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Shared Message Record, for Messages Stored in an Object Header
Field NameDescription
Message LocationThis has a value of 1 indicating that the message is stored in an object header.
Hash ValueThis is the hash value for the message.
Message TypeThis is the message type in the object header.
Creation IndexThis is the creation index of the message within the object header.
Object Header AddressThis is the address of the object header where the message is located.
+ +\section sec_fmt3_dataobject IV. Disk Format: Level 2 - Data Objects +Data objects contain the “real” user-visible information in the file. These objects compose +the scientific data and other information which are generally thought of as “data” by the +end-user. All the other information in the file is provided as a framework for storing and accessing +these data objects. + +A data object is composed of header and data information. The header information contains the +information needed to interpret the data information for the object as well as additional “metadata” +or pointers to additional “metadata” used to describe or annotate each object. + +\subsection subsec_fmt3_dataobject_hdr IV.A. Disk Format: Level 2A - Data Object Headers +The header information of an object is designed to encompass all the information about an object, except for +the data itself. This information includes the dataspace, datatype, information about how the data is stored +on disk (in external files, compressed, broken up in blocks, and so on), as well as other information used by the +library to speed up access to the data objects or maintain a file’s integrity. Information stored by user +applications as attributes is also stored in the object’s header. The header of each object is not necessarily +located immediately prior to the object’s data in the file and in fact may be located in any position in the +file. The order of the messages in an object header is not significant. + +Object headers are composed of a prefix and a set of messages. The prefix contains the information needed to +interpret the messages and a small amount of metadata about the object, and the messages contain the majority +of the metadata about the object. + +\subsection subsec_fmt3_dataobject_hdr_prefix IV.A.1 Disk Format: Level 2A1 - Data Object Header Prefix + +\subsubsection subsubsec_fmt3_dataobject_hdr_prefix_one IV.A.1.a Version 1 Data Object Header Prefix +Header messages are aligned on 8-byte boundaries for version 1 object headers. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 1 Object Header
bytebytebytebyte
VersionReserved (zero)Total Number of Header Messages
Object Reference Count
Object Header Size
Reserved (zero)
Header Message Type \#1Size of Header Message Data \#1
Header Message \#1 FlagsReserved (zero)

Header Message Data \#1

.
.
.
Header Message Type \#nSize of Header Message Data \#n
Header Message \#n FlagsReserved (zero)

Header Message Data \#n

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 1 Object Header
Field NameDescription
VersionThis value is used to determine the format of the information in the object header. When the format of + the object header is changed, the version number is incremented and can be used to determine how the + information in the object header is formatted. This is version one (1) (there was no version zero (0)) + of the object header.
Total Number of Header MessagesThis value determines the total number of messages listed in object headers for this object. This value + includes the messages in continuation messages for this object.
Object Reference CountThis value specifies the number of “hard links” to this object within the current file. + References to the object from external files, “soft links” in this file and object + references in this file are not tracked.
Object Header SizeThis value specifies the number of bytes of header message data following this length field that + contain object header messages for this object header. This value does not include the size of object header + continuation blocks for this object elsewhere in the file.
Header Message \#n TypeThis value specifies the type of information included in the following header message data. The + message types for header messages are defined in sections below.
Size of Header Message \#n DataThis value specifies the number of bytes of header message data following the header message type and + length information for the current message. The size includes padding bytes to make the message a multiple + of eight bytes.
Header Message \#n FlagsThis is a bit field with the following definition: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
BitDescription
0If set, the message data is constant. This is used for messages like the datatype message of + a dataset.
1If set, the message is shared and stored in another location than the object header. + The Header Message Data field contains a Shared Message (described in the @ref + subsec_fmt3_dataobject_hdr_msg section below) and the Size of Header Message Data field contains + the size of that Shared Message.
2If set, the message should not be shared.
3If set, the HDF5 decoder should fail to open this object if it does not understand the + message’s type and the file is open with permissions allowing write access to the file. + (Normally, unknown messages can just be ignored by HDF5 decoders)
4If set, the HDF5 decoder should set bit 5 of this message’s flags (in other words, this + bit field) if it does not understand the message’s type and the object is modified in any + way. (Normally, unknown messages can just be ignored by HDF5 decoders)
5If set, this object was modified by software that did not understand this message. (Normally, + unknown messages should just be ignored by HDF5 decoders) (Can be used to invalidate an index + or a similar feature)
6If set, this message is shareable.
7If set, the HDF5 decoder should always fail to open this object if it does not understand the + message’s type (whether it is open for read-only or read-write access). (Normally, unknown + messages can just be ignored by HDF5 decoders)
+
Header Message \#n DataThe format and length of this field is determined by the header message type and size respectively. + Some header message types do not require any data and this information can be eliminated by setting the + length of the message to zero. The data is padded with enough zeros to make the size a multiple of + eight.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_prefix_two IV.A.1.b Version 2 Data Object Header Prefix +Note that the “total number of messages” field has been dropped from the data object header +prefix in this version. The number of messages in the data object header is just determined by the +messages encountered in all the object header blocks. + +Note also that the fields and messages in this version of data object headers have no alignment +or padding bytes inserted - they are stored packed together. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 2 Object Header
bytebytebytebyte
Signature
VersionFlagsThis space inserted only to align table nicely
Access time (optional)
Modification Time (optional)
Change Time (optional)
Birth Time (optional)
Maximum \# of compact attributes (optional)Minimum \# of dense attributes (optional)
Size of Chunk \#0 (variable size)This space inserted only to align table nicely
Header Message Type \#1Size of Header Message Data \#1Header Message \#1 Flags
Header Message \#1 Creation Order (optional)This space inserted only to align table nicely

Header Message Data \#1

.
.
.
Header Message Type \#nSize of Header Message Data \#nHeader Message \#n Flags
Header Message \#n Creation Order (optional)This space inserted only to align table nicely

Header Message Data \#n

Gap (optional, variable size)
Checksum
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 Object Header
Field NameDescription
SignatureThe ASCII character string “OHDR” is used to indicate the beginning of + an object header. This gives file consistency checking utilities a better chance of reconstructing + a damaged file.
VersionThis field has a value of 2 indicating version 2 of the object header.
FlagsThis field is a bit field indicating additional information about the object header. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Bit(s)Description
0-1This two bit field determines the size of the Size of Chunk \#0 field. The values are: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0The Size of Chunk \#0 field is 1 byte.
1The Size of Chunk \#0 field is 2 bytes.
2The Size of Chunk \#0 field is 4 bytes.
3The Size of Chunk \#0 field is 8 bytes.
2If set, attribute creation order is tracked.
3If set, attribute creation order is indexed.
4If set, non-default attribute storage phase change values are stored.
5If set, access, modification, change and birth times are stored.
6-7Reserved
Access TimeThis 32-bit value represents the number of seconds after the UNIX epoch when the object’s + raw data was last accessed (in other words, read or written). This field is present if bit 5 of + flags is set.
Modification TimeThis 32-bit value represents the number of seconds after the UNIX epoch when the object’s + raw data was last modified (in other words, written). This field is present if bit 5 of + flags is set.
Change TimeThis 32-bit value represents the number of seconds after the UNIX epoch when the object’s + metadata was last changed. This field is present if bit 5 of flags is set.
Birth TimeThis 32-bit value represents the number of seconds after the UNIX epoch when the object was + created. This field is present if bit 5 of flags is set.
Maximum \# of compact attributesThis is the maximum number of attributes to store in the compact format before switching to the + indexed format. This field is present if bit 4 of flags is set.
Minimum \# of dense attributesThis is the minimum number of attributes to store in the indexed format before switching to the + compact format. This field is present if bit 4 of flags is set.
Size of Chunk \#0This unsigned value specifies the number of bytes of header message data following this field + that contain object header information. This value does not include the size of object header + continuation blocks for this object elsewhere in the file. The length of this field varies + depending on bits 0 and 1 of the flags field.
Header Message \#n TypeSame format as version 1 of the object header, described above.
Size of Header Message \#n DataThis value specifies the number of bytes of header message data following the header message + type and length information for the current message. The size of messages in this version does + not include any padding bytes.
Header Message \#n FlagsSame format as version 1 of the object header, described above.
Header Message \#n Creation OrderThis field stores the order that a message of a given type was created in.
+ This field is present if bit 2 of flags is set.
Header Message \#n DataSame format as version 1 of the object header, described above.
GapA gap in an object header chunk is inferred by the end of the messages for the chunk before the + beginning of the chunk’s checksum. Gaps are always smaller than the size of an object header + message prefix (message type + message size + message flags).
+ Gaps are formed when a message (typically an attribute message) in an earlier chunk is deleted + and a message from a later chunk that does not quite fit into the free space is moved into the + earlier chunk.
ChecksumThis is the checksum for the object header chunk.
+ +The header message types and the message data associated with them compose the critical "meta-data" about +each object. Some header messages are required for each object while others are optional. Some optional +header messages may also be repeated several times in the header itself, the requirements and number of +times allowed in the header will be noted in each header message description below. + +\subsection subsec_fmt3_dataobject_hdr_msg IV.A.2 Disk Format: Level 2A2 - Data Object Header Messages +Data object header messages are small pieces of metadata that are stored in the data object header for +each object in an HDF5 file. Data object header messages provide the metadata required to describe an +object and its contents, as well as optional pieces of metadata that annotate the meaning or purpose of +the object. + +Data object header messages are either stored directly in the data object header for the object or are +shared between multiple objects in the file. When a message is shared, a flag in the Message Flags +indicates that the actual Message Data portion of that message is stored in another location +(such as another data object header, or a heap in the file) and the Message Data field +contains the information needed to locate the actual information for the message. + +The format of shared message data is described here: + + + + + + + + + + + + + + + + + + + +
Layout: Shared Message (Version 1)
bytebytebytebyte
VersionTypeReserved (zero)
Reserved (zero)

AddressO

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + +
Fields: Shared Message (Version 1)
Field NameDescription
VersionThe version number is used when there are changes in the format of a shared object message and is + described here: + + + + + + + + + + + + + +
VersionDescription
0Never used.
1Used by the library before version 1.6.1.
TypeThe type of shared message location: + + + + + + + + + +
ValueDescription
0Message stored in another object’s header (a committed message).
AddressThe address of the object header containing the message to be shared.
+ + + + + + + + + + + + + + + + + +
Layout: Shared Message (Version 2)
bytebytebytebyte
VersionTypeThis space inserted only to align table nicely

AddressO

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + +
Fields: Shared Message (Version 2)
Field NameDescription
VersionThe version number is used when there are changes in the format of a shared object message and is + described here: + + + + + + + + + +
VersionDescription
2Used by the library of version 1.6.1 and after.
TypeThe type of shared message location: + + + + + + + + + +
ValueDescription
0Message stored in another object’s header (a committed message).
AddressThe address of the object header containing the message to be shared.
+ + + + + + + + + + + + + + + + + +
Layout: Shared Message (Version 3)
bytebytebytebyte
VersionTypeThis space inserted only to align table nicely
Location (variable size)
+ + + + + + + + + + + + + + + + + + + +
Fields: Shared Message (Version 3)
Field NameDescription
VersionThe version number indicates changes in the format of shared object message and is described here: + + + + + + + + + +
VersionDescription
3Used by the library of version 1.8 and after. In this version, the Type field can + indicate that the message is stored in the fractal heap.
TypeThe type of shared message location: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0Message is not shared and is not shareable.
1Message stored in file’s shared object header message heap + (a shared message).
2Message stored in another object’s header (a committed message).
3Message stored is not shared, but is shareable.
LocationThis field contains either a @ref FMT3SizeOfOffsetsV0 "Size of Offsets"-bytes address of the object header + containing the message to be shared, or an 8-byte fractal heap ID for the message in the + file’s shared object header message heap.
+ +The following is a list of currently defined header messages: + +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_nil IV.A.2.a. The NIL Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: NIL
Header Message Type: 0x0000
Length: Varies
Status: Optional; may be repeated.
Description:The NIL message is used to indicate a message which is to be ignored when reading the header messages + for a data object. [Possibly one which has been deleted for some reason.]
Format of Data: Unspecified
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_simple IV.A.2.b. The Dataspace Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Dataspace
Header Message Type: 0x0001
Length: Varies according to the number of dimensions, as described in the following + table.
Status: Required for dataset objects; may not be repeated.
Description:The dataspace message describes the number of dimensions (in other words, “rank”) and size + of each dimension that the data object has. This message is only used for datasets which have a + simple, rectilinear, array-like layout; datasets requiring a more complex layout are not yet supported.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Dataspace Message - Version 1
bytebytebytebyte
VersionDimensionalityFlagsReserved
Reserved
Dimension \#1 SizeL

.
.
.

Dimension \#n SizeL


Dimension \#1 Maximum SizeL

.
.
.

Dimension \#n Maximum SizeL


Permutation Index \#1L

.
.
.

Permutation Index \#nL

+\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Dataspace Message - Version 1
Field NameDescription
Version This value is used to determine the format of the Dataspace Message. When the format of the + information in the message is changed, the version number is incremented and can be used to determine + how the information in the object header is formatted. This document describes version one (1) (there + was no version zero (0)).
DimensionalityThis value is the number of dimensions that the data object has.
FlagsThis field is used to store flags to indicate the presence of parts of this message. Bit 0 (the least + significant bit) is used to indicate that maximum dimensions are present. Bit 1 is used to indicate + that permutation indices are present.
Dimension \#n SizeThis value is the current size of the dimension of the data as stored in the file. The first dimension + stored in the list of dimensions is the slowest changing dimension and the last dimension stored is the + fastest changing dimension.
Dimension \#n Maximum SizeThis value is the maximum size of the dimension of the data as stored in the file. This value may be + the special “@ref FMT3UnlimitedDim "unlimited size"” which indicates that the data + may expand along this dimension indefinitely. If these values are not stored, the maximum size of each + dimension is assumed to be the dimension’s current size.
Permutation Index \#nThis value is the index permutation used to map each dimension from the canonical representation to an + alternate axis for each dimension. If these values are not stored, the first dimension stored in the list + of dimensions is the slowest changing dimension and the last dimension stored is the fastest changing + dimension.
+ +Version 2 of the dataspace message dropped the optional permutation index value support, as it was never +implemented in the HDF5 Library: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Dataspace Message - Version 2
bytebytebytebyte
VersionDimensionalityFlagsType
Dimension \#1 SizeL

.
.
.

Dimension \#n SizeL


Dimension \#1 Maximum SizeL

.
.
.

Dimension \#n Maximum SizeL

+\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Dataspace Message - Version 2
Field NameDescription
Version This value is used to determine the format of the Dataspace Message. This field should be ‘2’ + for version 2 format messages.
DimensionalityThis value is the number of dimensions that the data object has.
FlagsThis field is used to store flags to indicate the presence of parts of this message. Bit 0 (the least + significant bit) is used to indicate that maximum dimensions are present.
Typeindicates the type of the dataspace: + + + + + + + + + + + + + + + + + +
ValueDescription
0A scalar dataspace; in other words, a dataspace with a single, dimensionless element.
1A simple dataspace; in other words, a dataspace with a rank > 0 and an appropriate number + of dimensions.
2A null dataspace; in other words, a dataspace with no elements.
Dimension \#n SizeThis value is the current size of the dimension of the data as stored in the file. The first dimension + stored in the list of dimensions is the slowest changing dimension and the last dimension stored is the + fastest changing dimension.
Dimension \#n Maximum SizeThis value is the maximum size of the dimension of the data as stored in the file. This value may be + the special “@ref FMT3UnlimitedDim "unlimited size"” which indicates that the data + may expand along this dimension indefinitely. If these values are not stored, the maximum size of each + dimension is assumed to be the dimension’s current size.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_linkinfo IV.A.2.c. The Link Info Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Link Info
Header Message Type: 0x002
Length: Varies
Status: Optional; may not be repeated.
Description:The link info message tracks variable information about the current state of the links for a + “new style” group’s behavior. Variable information will be stored in this + message and constant information will be stored in the @ref + subsubsec_fmt3_dataobject_hdr_msg_groupinfo message.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Link Info
bytebytebytebyte
VersionFlagsThis space inserted only to align table nicely

Maximum Creation Index (8 bytes, optional)


Fractal Heap AddressO


Address of v2 B-tree for Name IndexO


Address of v2 B-tree for Creation Order IndexO (optional)

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Link Info
Field NameDescription
VersionThe version number for this message. This document describes version 0.
FlagsThis field determines various optional aspects of the link info message: + + + + + + + + + + + + + + + + + +
BitDescription
0If set, creation order for the links is tracked.
1If set, creation order for the links is indexed.
2-7Reserved
Maximum Creation IndexThis 64-bit value is the maximum creation order index value stored for a link in this group.
+ This field is present if bit 0 of flags is set.
Fractal Heap AddressThis is the address of the fractal heap to store dense links. Each link stored in the fractal heap + is stored as a @ref subsubsec_fmt3_dataobject_hdr_msg_link.
+ If there are no links in the group, or the group’s links are stored “compactly” + (as object header messages), this value will be the @ref FMT3UndefinedAddress "undefined address".
Address of v2 B-tree for Name IndexThis is the address of the version 2 B-tree to index names of links.
+ If there are no links in the group, or the group’s links are stored “compactly” + (as object header messages), this value will be the @ref FMT3UndefinedAddress "undefined address".
Address of v2 B-tree for Creation Order IndexThis is the address of the version 2 B-tree to index creation order of links.
+ If there are no links in the group, or the group’s links are stored “compactly” + (as object header messages), this value will be the @ref FMT3UndefinedAddress "undefined address".
+ This field exists if bit 1 of flags is set.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_dtmessage IV.A.2.d. The Datatype Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Datatype
Header Message Type: 0x0003
Length: Variable
Status: Required for dataset or committed datatype (formerly named datatype) + objects; may not be repeated.
Description:The datatype message defines the datatype for each element of a dataset or a common datatype for + sharing between multiple datasets. A datatype can describe an atomic type like a fixed- or + floating-point type or more complex types like a C struct (compound datatype), array (array datatype) + or C++ vector (variable-length datatype).
+ Datatype messages that are part of a dataset object do not describe how elements are related to one + another; the dataspace message is used for that purpose. Datatype messages that are part of a committed + datatype (formerly named datatype) message describe a common datatype that can be shared by multiple + datasets in the file.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + +
Layout: Datatype Message
bytebytebytebyte
Class and VersionClass Bit Field, Bits 0-7Class Bit Field, Bits 8-15Class Bit Field, Bits 16-23
Size


Properties

/
+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: Datatype Message
Field NameDescription
Class and VersionThe version of the datatype message and the datatype’s class information are packed together in + this field. The version number is packed in the top 4 bits of the field and the class is contained + in the bottom 4 bits.
+ The version number information is used for changes in the format of the datatype message and is + described here: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
VersionDescription
0Never used
1Used by early versions of the library to encode compound datatypes with explicit array fields. + See the compound datatype description below for further details.
2Used when an array datatype needs to be encoded.
3Used when a VAX byte-ordered type needs to be encoded. Packs various other datatype classes more + efficiently also.
4Used to encode the revised reference datatype.
5Used when a complex number datatype needs to be encoded.

+ The class of the datatype determines the format for the class bit field and properties portion of the + datatype message, which are described below. The following classes are currently defined: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0\ref FMT3ClassFixedPoint "Fixed-Point"
1\ref FMT3ClassFloatingPoint "Floating-Point"
2\ref FMT3ClassTime "Time"
3\ref FMT3ClassString "String"
4\ref FMT3ClassBitField "Bit field"
5\ref FMT3ClassOpaque "Opaque"
6\ref FMT3ClassCompound "Compound"
7\ref FMT3ClassReference "Reference"
8\ref FMT3ClassEnum "Enumerated"
9\ref FMT3ClassVarLen "Variable-Length"
10\ref FMT3ClassArray "Array"
11\ref FMT3ClassComplex "Complex"
Class Bit FieldsThe information in these bit fields is specific to each datatype class and is described below. + All bits not defined for a datatype class are set to zero.
SizeThe size of a datatype element in bytes.
PropertiesThis variable-sized sequence of bytes encodes information specific to each datatype class and is + described for each class below. If there is no property information specified for a datatype class, + the size of this field is zero bytes.
+ +\anchor FMT3ClassFixedPoint

Class specific information for Fixed-Point Numbers (Class 0):

+ + + + + + + + + + + + + + + + + + + + + + +
Bits: Fixed-point Bit Field Description
BitsMeaning
0Byte Order. If zero, byte order is little-endian; otherwise, byte order is big endian.
1, 2Padding type. Bit 1 is the lo_pad type and bit 2 is the hi_pad type. If a datum has + unused bits at either end, then the lo_pad or hi_pad bit is copied to those locations.
3Signed. If this bit is set then the fixed-point number is in 2’s complement form.
4-23Reserved (zero).
+ + + + + + + + + + + + + +
Layout: Fixed-Point Property Descriptions
ByteByteByteByte
Bit OffsetBit Precision
+ + + + + + + + + + + + + + + +
Fields: Fixed-Point Property Descriptions
Field NameDescription
Bit OffsetThe bit offset of the first significant bit of the fixed-point value within the datatype. The + bit offset specifies the number of bits “to the right of” the value (which are + set to the lo_pad bit value).
Bit PrecisionThe number of bits of precision of the fixed-point value within the datatype. This value, + combined with the datatype element’s size and the Bit Offset field specifies the number + of bits “to the left of” the value (which are set to the hi_pad bit value).
+ +\anchor FMT3ClassFloatingPoint

Class specific information for Floating-Point Numbers (Class 1):

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Bits: Floating-Point Bit Field Description
BitsMeaning
0Byte Order. These two non-contiguous bits specify the “endianness” of + the bytes in the datatype element. + + + + + + + + + + + + + + + + + + + + + + + + + + +
Bit 6Bit 0Description
00Byte order is little-endian
01Byte order is big-endian
10Reserved
11Byte order is VAX-endian
1, 2, 3Padding type. Bit 1 is the low bits pad type, bit 2 is the high bits pad type, and bit + 3 is the internal bits pad type. If a datum has unused bits at either end or between the sign bit, exponent, + or mantissa, then the value of bit 1, 2, or 3 is copied to those locations.
4-5Mantissa Normalization. This 2-bit bit field specifies how the most significant bit of + the mantissa is managed. + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0No normalization
1The most significant bit of the mantissa is always set (except for 0.0).
2The most significant bit of the mantissa is not stored, but is implied to be set.
3Reserved.
6-7Reserved (zero).
8-15Sign Location. This is the bit position of the sign bit. Bits are numbered with the least + significant bit zero.
16-23Reserved (zero).
+ + + + + + + + + + + + + + + + + + + + + + +
Layout: Floating-Point Property Description
ByteByteByteByte
Bit OffsetBit Precision
Exponent LocationExponent SizeMantissa LocationMantissa Size
Exponent Bias
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Floating-Point Property Description
Field NameProperty Description
Bit OffsetThe bit offset of the first significant bit of the floating-point value within the datatype. The + bit offset specifies the number of bits “to the right of” the value.
Bit PrecisionThe number of bits of precision of the floating-point value within the datatype.
Exponent LocationThe bit position of the exponent field. Bits are numbered with the least significant + bit number zero.
Exponent SizeThe size of the exponent field in bits.
Mantissa LocationThe bit position of the mantissa field. Bits are numbered with the least significant bit number + zero.
Mantissa SizeThe size of the mantissa field in bits.
Exponent BiasThe bias of the exponent field.
+ +\anchor FMT3ClassTime

Class specific information for Time (Class 2):

+ + + + + + + + + + + + + + +
Bits: Time Bit Field Description
BitsMeaning
0Byte Order. If zero, byte order is little-endian; otherwise, byte order is big endian.
1-23Reserved (zero).
+
+ + + + + + + + + + +
Layout: Time Property Description
ByteByte
Bit Precision
+
+ + + + + + + + + + + +
Fields: Time Property Description
Field NameDescription
Bit PrecisionThe number of bits of precision of the time value.
+
+ +\anchor FMT3ClassString

Class specific information for Strings (Class 3):

+ + + + + + + + + + + + + + + + + +
Bits: String Bit Field Description
BitsMeaning
0-3Padding type. This four-bit value determines the type of padding to use for the + string. The values are: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0Null Terminate: A zero byte marks the end of the string and is guaranteed to be present after + converting a long string to a short string. When converting a short string to a long string the + value is padded with additional null characters as necessary.
1Null Pad: Null characters are added to the end of the value during conversions from short values + to long values but conversion in the opposite direction simply truncates the value.
2Space Pad: Space characters are added to the end of the value during conversions from short values + to long values but conversion in the opposite direction simply truncates the value. This is the + Fortran representation of the string.
3-15Reserved.
+
4-7Character Set. The character set used to encode the string. + + + + + + + + + + + + + + + + + +
ValueDescription
0ASCII character set encoding
1UTF-8 character set encoding
2-15Reserved
8-23Reserved (zero).
+ +There are no properties defined for the string class. + +\anchor FMT3ClassBitField

Class specific information for Bitfields (Class 4):

+ + + + + + + + + + + + + + + + + + +
Bits: Bitfield Bit Field Description
BitsMeaning
0Byte Order. If zero, byte order is little-endian; otherwise, byte order is big endian.
1, 2Padding type. Bit 1 is the lo_pad type and bit 2 is the hi_pad type. If a datum has + unused bits at either end, then the lo_pad or hi_pad bit is copied to those locations.
3-23Reserved (zero).
+ + + + + + + + + + + + + +
Layout: Bit Field Property Description
ByteByteByteByte
Bit OffsetBit Precision
+ + + + + + + + + + + + + + + +
Fields: Bit Field Property Description
Field NameDescription
Bit OffsetThe bit offset of the first significant bit of the bitfield within the datatype. The bit + offset specifies the number of bits “to the right of” the value.
Bit PrecisionThe number of bits of precision of the bit field within the datatype.
+ +\anchor FMT3ClassOpaque

Class specific information for Opaque (Class 5):

+ + + + + + + + + + + + + + +
Bits: Opaque Bit Field Description
BitsMeaning
0-7Length of ASCII tag in bytes.
8-23Reserved (zero).
+ + + + + + + + + + + + +
Layout: Opaque Property Description
ByteByteByteByte

ASCII Tag

+ + + + + + + + + + + +
Fields: Opaque Property Description
Field NameDescription
ASCII TagThis NUL-terminated string provides a description for the opaque type. It is NUL-padded to a + multiple of 8 bytes.
+ +\anchor FMT3ClassCompound

Class specific information for Compound Types (Class 6):

+ + + + + + + + + + + + + + +
Bits: Compound Bit Field Description
BitsMeaning
0-15Number of Members. This field contains the number of members defined for the + compound datatype. The member definitions are listed in the Properties field of the data type + message.
16-23Reserved (zero).
+ +The Properties field of a compound datatype is a list of the member definitions of the compound datatype. +The member definitions appear one after another with no intervening bytes. The member types are described +with a (recursively) encoded datatype message. + +Note that the property descriptions are different for different versions of the datatype version. Additionally +note that the version 0 properties are deprecated and has been replaced with later encodings in +versions of the HDF5 library from the 1.4 release onward. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Compound Properties Description for Datatype Version 1
ByteByteByteByte


Name

/
Byte Offset of Member
DimensionalityReserved (zero)
Dimension Permutation
Reserved (zero)
Dimension \#1 Size (required)
Dimension \#2 Size (required)
Dimension \#3 Size (required)
Dimension \#4 Size (required)

Member Type Message

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Compound Properties Description for Datatype Version 1
Field NameDescription
NameThis NUL-terminated string provides a description for the opaque type. It is NUL-padded to a + multiple of 8 bytes.
Byte Offset of MemberThis is the byte offset of the member within the datatype.
DimensionalityIf set to zero, this field indicates a scalar member. If set to a value greater than zero, + this field indicates that the member is an array of values. For array members, the size of + the array is indicated by the ‘Size of Dimension n’ field in this message.
Dimension PermutationThis field was intended to allow an array field to have its dimensions permuted, but this was + never implemented. This field should always be set to zero.
Dimension \#n SizeThis field is the size of a dimension of the array field as stored in the file. The first + dimension stored in the list of dimensions is the slowest changing dimension and the last + dimension stored is the fastest changing dimension.
Member Type MessageThis field is a datatype message describing the datatype of the member.
+ + + + + + + + + + + + + + + + + + +
Layout: Compound Properties Description for Datatype Version 2
ByteByteByteByte

Name

Byte Offset of Member

Member Type Message

+ + + + + + + + + + + + + + + + + + + +
Fields: Compound Properties Description for Datatype Version 2
Field NameDescription
NameThis NUL-terminated string provides a description for the opaque type. It is NUL-padded to a multiple + of 8 bytes.
Byte Offset of MemberThis is the byte offset of the member within the datatype.
Member Type MessageThis field is a datatype message describing the datatype of the member.
+ + + + + + + + + + + + + + + + + + +
Layout: Compound Properties Description for Datatype Version 3
ByteByteByteByte

Name

Byte Offset of Member

Member Type Message

+ + + + + + + + + + + + + + + + + + + +
Fields: Compound Properties Description for Datatype Version 3
Field NameDescription
NameThis NUL-terminated string provides a description for the opaque type. It is not NUL-padded to a + multiple of 8 bytes.
Byte Offset of MemberThis is the byte offset of the member within the datatype. The field size is the minimum number of bytes + necessary, based on the size of the datatype element. For example, a datatype element size of less than + 256 bytes uses a 1 byte length, a datatype element size of 256-65535 bytes uses a 2 byte length, and + so on.
Member Type MessageThis field is a datatype message describing the datatype of the member.
+ +\anchor FMT3ClassReference

Class specific information for Reference (Class 7):


+Note that for region references, the stored data is a \ref FMT3GlobalHeapID "Global Heap ID" pointing to information +about the region stored in the global heap. + + + + + + + + + + + + + + + +
Bits: Reference Bit Field Description for Datatype Version < 4
BitsMeaning
0-3Type. This four-bit value contains the reference types which are supported + for backward compatibility. The values defined are: + + + + + + + + + + + + + + + + + +
ValueDescription
0Object Reference (#H5R_OBJECT1): A reference to another object in this HDF5 file.
1Dataset Region Reference (#H5R_DATASET_REGION1): A reference to a region within a dataset + in this HDF5 file.
2-15Reserved
4-23Reserved (zero).
+ + + + + + + + + + + + + + + + + + + +
Bits: Reference Bit Field Description for Datatype Version 4
BitsMeaning
0-3Type. This four-bit value contains the revised reference types. The values + defined are: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
2Object Reference (#H5R_OBJECT2): A reference to another object in this file or an + external file.
3Dataset Region Reference (#H5R_DATASET_REGION2): A reference to a region within a + dataset in this file or an external file.
4Attribute Reference (#H5R_ATTR): A reference to an attribute attached to an object + in this file or an external file.
5-15Reserved
+
4-7Version. This four-bit value contains the version for encoding the revised + reference types. The values defined are: + + + + + + + + + + + + + + + + + +
ValueDescription
0Unused
1The version for encoding the revised reference types: Object Reference (2), + Dataset Region Reference (3) and Attribute Reference (4).
2-15Reserved
8-23Reserved (zero).
+ +There are no properties defined for the reference class. + +\anchor FMT3ClassEnum

Class specific information for Enumeration (Class 8):

+ + + + + + + + + + + + + + +
Bits: Enumeration Bit Field Description
BitsMeaning
0-15Number of Members. The number of name/value pairs defined for the enumeration type.
16-23Reserved (zero).
+ + + + + + + + + + + + + + + + + + +
Layout: Enumeration Property Description for Datatype Versions 1 and 2
ByteByteByteByte

Base Type
/

Names


Values

+ + + + + + + + + + + + + + + + + + + +
Fields: Enumeration Property Description for Datatype Versions 1 and 2
Field NameDescription
Base TypeEach enumeration type is based on some parent type, usually an integer. The information + for that parent type is described recursively by this field.
NamesThe name for each name/value pair. Each name is stored as a null terminated ASCII string + in a multiple of eight bytes. The names are in no particular order.
ValuesThe list of values in the same order as the names. The values are packed (no inter-value + padding) and the size of each value is determined by the parent type.
+ + + + + + + + + + + + + + + + + + +
Layout: Enumeration Property Description for Datatype Versions 3
ByteByteByteByte

Base Type
/

Names


Values

+ + + + + + + + + + + + + + + + + + + +
Fields: Enumeration Property Description for Datatype Versions 3
Field NameDescription
Base TypeEach enumeration type is based on some parent type, usually an integer. The information + for that parent type is described recursively by this field.
NamesThe name for each name/value pair. Each name is stored as a null terminated ASCII string, + not padded to a multiple of eight bytes. The names are in no particular order.
ValuesThe list of values in the same order as the names. The values are packed (no inter-value + padding) and the size of each value is determined by the parent type.
+ +\anchor FMT3ClassVarLen

Class specific information for Variable-Length (Class 9):

+ + + + + + + + + + + + + + + + + + + + + + +
Bits: Variable-Length Bit Field Description
BitsMeaning
0-3Type. This four-bit value contains the type of variable-length datatype described. + The values defined are: + + + + + + + + + + + + + + + + + +
ValueDescription
0Sequence: A variable-length sequence of any datatype. Variable-length sequences do not + have padding or character set information.
1String: A variable-length sequence of characters. Variable-length strings have padding and + character set information.
2-15Reserved
4-7Padding type. (variable-length string only). This four-bit value determines the + type of padding used for variable-length strings. The values are the same as for the string padding + type, as follows: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
00 Null terminate: A zero byte marks the end of a string and is guaranteed to be present after + converting a long string to a short string. When converting a short string to a long string, + the value is padded with additional null characters as necessary.
1Null pad: Null characters are added to the end of the value during conversion from a short string to + a longer string. Conversion from a long string to a shorter string simply truncates the value.
2Space pad: Space characters are added to the end of the value during conversion from a short string + to a longer string. Conversion from a long string to a shorter string simply truncates the value. + This is the Fortran representation of the string.
3-15Reserved
+ This value is set to zero for variable-length sequences.
8-11Character Set. (variable-length string only) This four-bit value specifies the + character set to be used for encoding the string: + + + + + + + + + + + + + + + + + +
ValueDescription
0ASCII character set encoding.
1UTF-8 character set encoding.
2-15Reserved
+ This value is set to zero for variable-length sequences.
12-23Reserved (zero).
+ + + + + + + + + + + + +
Layout: Variable-Length Property Description
ByteByteByteByte

Parent Type Message

+ + + + + + + + + + + +
Fields: Variable-Length Property Description
Field NameDescription
Parent TypeEach variable-length type is based on some parent type. This field contains the datatype + message describing that parent type. In the case of nested variable-length types, this parent + datatype message will recursively contain all parent datatype messages. Variable-length strings + are considered to have the parent type #H5T_NATIVE_UCHAR.
+ +\anchor FMT3ClassArray

Class specific information for Array (Class 10):

+ +There are no bit fields defined for the array class. + +Note that the dimension information defined in the property for this datatype class is independent of +dataspace information for a dataset. The dimension information here describes the dimensionality of the +information within a data element (or a component of an element, if the array datatype is nested within +another datatype) and the dataspace for a dataset describes the location of the elements in a dataset. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Array Property Description for Datatype Version 2
ByteByteByteByte
DimensionalityReserved (zero)
Dimension \#1 Size
.
.
.
Dimension \#n Size
Permutation Index \#1
.
.
.
Permutation Index \#n

Base Type

+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: Array Property Description for Datatype Version 2
Field NameDescription
DimensionalityThis value is the number of dimensions that the array has.
Dimension \#n SizeThis value is the size of the dimension of the array as stored in the file. The first dimension + stored in the list of dimensions is the slowest changing dimension and the last dimension stored + is the fastest changing dimension.
Permutation Index \#nThis value is the index permutation used to map each dimension from the canonical representation + to an alternate axis for each dimension. Currently, dimension permutations are not supported and + these indices should be set to the index position minus one (i.e. the first dimension should be + set to 0, the second dimension should be set to 1, and so on.)
Base TypeEach array type is based on some parent type. The information for that parent type is described + recursively by this field.
+ + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Array Property Description for Datatype Version 3
ByteByteByteByte
DimensionalityThis space inserted only to align table nicely
Dimension \#1 Size
.
.
.
Dimension \#n Size

Base Type

+ + + + + + + + + + + + + + + + + + + +
Fields: Array Property Description for Datatype Version 3
Field NameDescription
DimensionalityThis value is the number of dimensions that the array has.
Dimension \#n SizeThis value is the size of the dimension of the array as stored in the file. The first dimension + stored in the list of dimensions is the slowest changing dimension and the last dimension stored + is the fastest changing dimension.
Base TypeEach array type is based on some parent type. The information for that parent type is described + recursively by this field.
+ +\anchor FMT3ClassComplex

Class specific information for the Complex class (Class 11):

+ + + + + + + + + + + + + + + + + + +
Bits: Complex Bit Field Description
BitsMeaning
0Homogeneous. If zero, each part of the complex number + datatype is a different floating point datatype (heterogeneous). + Otherwise, each part of the complex number datatype is the same + floating point datatype (homogeneous). Currently, only homogeneous + complex number datatypes are supported.
1,2Complex number form. This two-bit value contains the type of complex number datatype + described. The values defined are: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0Rectangular
1Polar
2Exponential
3Reserved
+ Currently, only rectangular complex number datatypes are supported.
3-23Reserved (zero).
+ + + + + + + + + + + + +
Layout: Complex Property Description
ByteByteByteByte

Parent Type Message

+ + + + + + + + + + + +
Fields: Complex Property Description
Field NameDescription
Parent Type MessageEach complex number type is based on a parent floating point type. This field contains the + datatype message describing that parent type.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_ofvmessage IV.A.2.e. Data Storage - Fill Value (Old) Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Fill Value (old)
Header Message Type: 0x0004
Length: Varies
Status: Optional; may not be repeated.
Description:The fill value message stores a single data value which is returned to the application when an + uninitialized data element is read from a dataset. The fill value is interpreted with the same + datatype as the dataset. If no fill value message is present then a fill value of all zero bytes + is assumed.
+ This fill value message is deprecated in favor of the “new” fill value message (Message + Type 0x0005) and is only written to the file for forward compatibility with versions of the HDF5 + Library before the 1.6.0 version. Additionally, it only appears for datasets with a user-defined + fill value (as opposed to the library default fill value or an explicitly set “undefined” + fill value).
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + +
Layout: Fill Value Message (Old)
bytebytebytebyte
Size (4 bytes)

Fill Value (optional, variable size)

+
+ + + + + + + + + + + + + + +
Fields: Fill Value Message (Old)
Field NameDescription
SizeThis is the size of the Fill Value field in bytes.
Fill ValueThe fill value. The bytes of the fill value are interpreted using the same datatype as for the dataset.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_fvmessage IV.A.2.f. The Data Storage - Fill Value Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Fill Value
Header Message Type: 0x0005
Length: Varies
Status: Required for dataset objects; may not be repeated.
Description:The fill value message stores a single data value which is returned to the application when an + uninitialized data element is read from a dataset. The fill value is interpreted with the same + datatype as the dataset.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + +
Layout: Fill Value Message - Versions 1 & 2
bytebytebytebyte
VersionSpace Allocation TimeFill Value Write TimeFill Value Defined
Size (optional)

Fill Value (optional, variable size)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Fill Value Message - Versions 1 & 2
Field NameDescription
VersionThe version number information is used for changes in the format of the fill value message and + is described here: + + + + + + + + + + + + + + + + + + + + + +
VersionDescription
0Never used
1Initial version of this message.
2In this version, the Size and Fill Value fields are only present if the Fill Value Defined + field is set to 1.
3This version packs the other fields in the message more efficiently than version 2.
+
Space Allocation TimeWhen the storage space for the dataset’s raw data will be allocated. The allowed values are: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0Not used
1Early allocation. Storage space for the entire dataset should be allocated in the file + when the dataset is created.
2Late allocation. Storage space for the entire dataset should not be allocated until the + dataset is written to.
3Incremental allocation. Storage space for the dataset should not be allocated until the + portion of the dataset is written to. This is currently used in conjunction with chunked + data storage for datasets.
Fill Value Write TimeAt the time that storage space for the dataset’s raw data is allocated, this value indicates + whether the fill value should be written to the raw data storage elements. The allowed values are: + + + + + + + + + + + + + + + + + +
ValueDescription
0On allocation. The fill value is always written to the raw data storage when the storage + space is allocated.
1Never. The fill value should never be written to the raw data storage.
2Fill value written if set by user. The fill value will be written to the raw data storage + when the storage space is allocated only if the user explicitly set the fill value. If the + fill value is the library default or is undefined, it will not be written to the raw data storage.
Fill Value DefinedThis value indicates if a fill value is defined for this dataset. If this value is 0, the fill + value is undefined. If this value is 1, a fill value is defined for this dataset. For version 2 + or later of the fill value message, this value controls the presence of the Size and Fill field.
SizeThis is the size of the Fill Value field in bytes. This field is not present if the Version + field is greater than 1 and the Fill Value Defined field is set to 0.
Fill ValueThe fill value. The bytes of the fill value are interpreted using the same datatype as for the + dataset. This field is not present if the Version field is greater than 1 and the Fill Value + Defined field is set to 0.
+ + + + + + + + + + + + + + + + + + + + + +
Layout: Fill Value Message - Versions 3
bytebytebytebyte
VersionFlagsFill Value Write TimeThis space inserted only to align table nicely
Size (optional)

Fill Value (optional, variable size)

+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: Fill Value Message - Versions 3
Field NameDescription
VersionThe version number information is used for changes in the format of the fill value message and + is described here: + + + + + + + + + + + + + + + + + + + + + +
VersionDescription
0Never used
1Initial version of this message.
2In this version, the Size and Fill Value fields are only present if the Fill Value Defined + field is set to 1.
3This version packs the other fields in the message more efficiently than version 2.
+
FlagsWhen the storage space for the dataset’s raw data will be allocated. The allowed values are: + + + + + + + + + + + + + + + + + + + + + + + + + +
BitsDescription
0-1Space Allocation Time, with the same values as versions 1 and 2 of the message.
2-3Fill Value Write Time, with the same values as versions 1 and 2 of the message.
4Fill Value Undefined, indicating that the fill value has been marked as “undefined” + for this dataset. Bits 4 and 5 cannot both be set.
5Fill Value Defined, with the same values as versions 1 and 2 of the message. Bits 4 and 5 + cannot both be set.
6-7Reserved (zero).
SizeThis is the size of the Fill Value field in bytes. This field is not present if the Version + field is greater than 1 and the Fill Value Defined flag is set to 0.
Fill ValueThe fill value. The bytes of the fill value are interpreted using the same datatype as for the + dataset. This field is not present if the Version field is greater than 1 and the Fill Value + Defined flag is set to 0.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_link IV.A.2.g. The Link Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Link
Header Message Type: 0x0006
Length: Varies
Status: Optional; may be repeated.
Description:This message encodes the information for a link in a group’s object header, when the group is + storing its links “compactly”, or in the group’s fractal heap, when the group is + storing its links “densely”.
+ A group is storing its links compactly when the fractal heap address in the + @ref subsubsec_fmt3_dataobject_hdr_msg_linkinfo is set to the + @ref FMT3UndefinedAddress "undefined address" value.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Link Message
bytebytebytebyte
VersionFlagsLink type (optional)This space inserted only to align table nicely

Creation Order (8 bytes, optional)

Link Name Character Set (optional)Length of Link Name (variable size)This space inserted only to align table nicely
Link Name (variable size)

Link Information (variable size)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Link Message
Field NameDescription
VersionThe version number for this message. This document describes version 1.
FlagsThis field contains information about the link and controls the presence of other fields below. + + + + + + + + + + + + + + + + + + + + + + + + + +
BitsDescription
0-1Determines the size of the Length of Link Name field. + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0The size of the Length of Link Name field is 1 byte.
1The size of the Length of Link Name field is 2 bytes.
2The size of the Length of Link Name field is 4 bytes.
3The size of the Length of Link Name field is 8 bytes.
2Creation Order Field Present: if set, the Creation Order field is present. If + not set, creation order information is not stored for links in this group.
3Link Type Field Present: if set, the link is not a hard link and the Link Type + field is present. If not set, the link is a hard link.
4Link Name Character Set Field Present: if set, the link name is not represented with the + ASCII character set and the Link Name Character Set field is present. If not set, + the link name is represented with the ASCII character set.
5-7Reserved (zero).
Link typeThis is the link class type and can be one of the following values: + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0A hard link (should never be stored in the file)
1A soft link.
2-63Reserved for future HDF5 internal use.
64An external link.
65-255Reserved, but available for user-defined link types.
+ This field is present if bit 3 of Flags is set.
Creation OrderThis 64-bit value is an index of the link’s creation time within the group. Values start at + 0 when the group is created an increment by one for each link added to the group. Removing a link + from a group does not change existing links’ creation order field.
+ This field is present if bit 2 of Flags is set.
Link Name Character SetThis is the character set for encoding the link’s name: + + + + + + + + + + + + + +
ValueDescription
0ASCII character set encoding (this should never be stored in the file)
1UTF-8 character set encoding
+ This field is present if bit 4 of Flags is set.
Length of link nameThis is the length of the link’s name. The size of this field depends on bits 0 and 1 of Flags.
Link nameThis is the name of the link, non-NULL terminated.
Link informationThe format of this field depends on the link type.
+ For hard links, the field is formatted as follows: + + + + + +
@ref FMT3SizeOfOffsetsV0 "Size of Offsets" bytes:The address of the object header for the object that the link points to.
+
+ For soft links, the field is formatted as follows: + + + + + + + + + +
Bytes 1-2:Length of soft link value.
Length of soft link value bytes:A non-NULL-terminated string storing the value of the soft link.
+
+ For external links, the field is formatted as follows: + + + + + + + + + +
Bytes 1-2:Length of external link value.
Length of external link value bytes:The first byte contains the version number in the upper 4 bits and flags in the lower 4 bits + for the external link. Both version and flags are defined to be zero in this document. The + remaining bytes consist of two NULL-terminated strings, with no padding between them. The first + string is the name of the HDF5 file containing the object linked to and the second string is the + full path to the object linked to, within the HDF5 file’s group hierarchy.
+
+ For user-defined links, the field is formatted as follows: + + + + + + + + + +
Bytes 1-2:Length of user-defined data.
Length of user-defined link value bytes:The data supplied for the user-defined link type.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_external IV.A.2.h. The Data Storage - External Data Files Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: External Data Files
Header Message Type: 0x0007
Length: Varies
Status: Optional; may not be repeated.
Description:The external data storage message indicates that the data for an object is stored outside the HDF5 + file. The filename of the object is stored as a Universal Resource Location (URL) of the actual + filename containing the data. An external file list record also contains the byte offset of the + start of the data within the file and the amount of space reserved in the file for that data.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + +
Layout: External File List Message
bytebytebytebyte
VersionReserved (zero)
Allocated SlotsUsed Slots

Heap AddressO


Slot Definitions...

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: External File List Message
Field NameDescription
VersionThe version number information is used for changes in the format of External Data Storage Message + and is described here: + + + + + + + + + + + + + +
VersionDescription
0Never used.
1The current version used by the library.
Allocated SlotsThe total number of slots allocated in the message. Its value must be at least as large as the value + contained in the Used Slots field. (The current library simply uses the number of Used Slots for this + message)
Used SlotsThe number of initial slots which contain valid information.
Heap AddressThis is the address of a local name heap which contains the names for the external files. (The local + heap information can be found in @ref subsec_fmt3_infra_localheap in this document). The name at offset + zero in the heap is always the empty string.
Slot DefinitionsThe slot definitions are stored in order according to the array addresses they represent.
+ + + + + + + + + + + + + + + + + + +
Layout: External File List Slot
bytebytebytebyte

Name Offset in Local HeapL


File Offset in External Data FileL


Data Size in External FileL

+\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + +
Fields: External File List Slot
Field NameDescription
Name Offset in Local HeapThe byte offset within the local name heap for the name of the file. File names are stored as a URL + which has a protocol name, a host name, a port number, and a file name: + protocol:port//host/file. If the protocol is omitted + then “file:” is assumed. If the port number is omitted then a default port for that protocol + is used. If both the protocol and the port number are omitted then the colon can also be omitted. If the double + slash and host name are omitted then “localhost” is assumed. The file name is the only mandatory part, + and if the leading slash is missing then it is relative to the application’s current working directory + (the use of relative names is not recommended).
Offset in External Data FileThis is the byte offset to the start of the data in the specified file. For files that contain data for + a single dataset this will usually be zero.
Data Size in External FileThis is the total number of bytes reserved in the specified file for raw data storage. For a file that + contains exactly one complete dataset which is not extendable, the size will usually be the exact size of + the dataset. However, by making the size larger one allows HDF5 to extend the dataset. The size can be set + to a value larger than the entire file since HDF5 will read zeros past the end of the file without failing.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_layout IV.A.2.i. The Data Layout Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Data Storage - Layout
Header Message Type: 0x0008
Length: Varies
Status: Required for datasets; may not be repeated.
Description:The Data Layout message describes how the elements of a multi-dimensional array + are stored in the HDF5 file. Four types of data layout are supported: +
    +
  1. Contiguous: The array is stored in one contiguous area of the file. This layout requires that the + size of the array be constant: data manipulations such as chunking, compression, checksums or encryption + are not permitted. The message stores the total storage size of the array. The offset of an element from + the beginning of the storage area is computed as in a C array.
    2. +
  2. Chunked: The array domain is regularly decomposed into chunks, and each chunk is allocated and stored + separately. This layout supports arbitrary element traversals, compression, encryption, and checksums + (these features are described in other messages). The message stores the size of a chunk instead of the + size of the entire array; the size of the entire array can be calculated by traversing the B-tree that + stores the chunk addresses.
    3. +
  3. Compact: The array is stored in one contiguous block, as part of this object header messagei.
    4. +
  4. Virtual: This is only supported for version 4 of the Data Layout message. The message stores information + that is used to locate the global heap collection containing the Virtual Dataset (VDS) mapping information. + The mapping associates the VDS to the source dataset elements that are stored across a collection + of HDF5 files.
    5. +
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Data Layout Message (Versions 1 and 2)
bytebytebytebyte
VersionDimensionalityLayout ClassReserved (zero)
Reserved (zero)

Data AddressO (optional)

Dimension 0 Size
Dimension 1 Size
...
Dataset Element Size (optional)
Compact Data Size (optional)

Compact Data...(variable size, optional)

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Data Layout Message (Versions 1 and 2)
Field NameDescription
VersionThe version number information is used for changes in the format of the data layout message and + is described here: + + + + + + + + + + + + + + + + + +
VersionDescription
0Never used.
1Used by version 1.4 and before of the library to encode layout information. Data space is + always allocated when the data set is created.
2Used by version 1.6.x of the library to encode layout information. Data space is allocated + only when it is necessary.
DimensionalityAn array has a fixed dimensionality. This field specifies the number of dimension size fields later + in the message. The value stored for chunked storage is 1 greater than the number of dimensions in + the dataset’s dataspace. For example, 2 is stored for a 1 dimensional dataset.
Layout ClassThe layout class specifies the type of storage for the data and how the other fields of the layout + message are to be interpreted. + + + + + + + + + + + + + + + + + +
ValueDescription
0Compact Storage
1Contiguous Storage
2Chunked Storage
Data AddressFor contiguous storage, this is the address of the raw data in the file. For chunked storage this is + the address of the @ref subsubsec_fmt3_infra_btrees_v1 that is used to look up the addresses of the + chunks. This field is not present for compact storage. If the version for this message is greater than 1, + the address may have the @ref FMT3UndefinedAddress "undefined address" value, to indicate that storage + has not yet been allocated for this array.
Dimension \#n SizeFor contiguous and compact storage the dimensions define the entire size of the array while for chunked storage + they define the size of a single chunk. In all cases, they are in units of array elements (not bytes). The + first dimension stored in the list of dimensions is the slowest changing dimension and the last dimension + stored is the fastest changing dimension.
Dataset Element SizeThe size of a dataset element, in bytes. This field is only present for chunked storage.
Compact Data SizeThis field is only present for compact data storage. It contains the size of the raw data for the + dataset array, in bytes.
Compact DataThis field is only present for compact data storage. It contains the raw data for the dataset + array.
+ +Version 3 of this message re-structured the format into specific properties that are required for each layout class. + + + + + + + + + + + + + + + + +
Layout: Data Layout Message (Version 3)
bytebytebytebyte
VersionLayout ClassThis space inserted only to align table nicely

Properties (variable size)

+ + + + + + + + + + + + + + + + + + + +
Fields: Data Layout Message (Version 3)
Field NameDescription
VersionThe version number information is used for changes in the format of layout message and is + described here: + + + + + + + + + +
VersionDescription
3Used by the version 1.6.3 and later of the library to store properties for each layout class.
Layout ClassThe layout class specifies how the other fields of the layout message are to be interpreted. + + + + + + + + + + + + + + + + + +
ValueDescription
0Compact Storage
1Contiguous Storage
2Chunked Storage
PropertiesThis variable-sized field encodes information specific to each layout class and is described below. If + there is no property information specified for a layout class, the size of this field is zero bytes.
+ +\anchor FMT3CompactStorage

Class-specific information for compact layout (Class 0):


+(Note: The dimensionality information is in the Dataspace message) + + + + + + + + + + + + + + + +
Layout: Compact Storage Property Description
bytebytebytebyte
SizeThis space inserted only to align table nicely

Raw Data...(variable size)

+ + + + + + + + + + + + + + + +
Fields: Compact Storage Property Description
Field NameDescription
SizeThis field contains the size of the raw data for the dataset array, in bytes.
Raw DataThis field contains the raw data for the dataset array.
+ +\anchor FMT3ContiguousStorage

Class-specific information for contiguous storage (layout class 1):


+(Note: The dimensionality information is in the Dataspace message) + + + + + + + + + + + + + + + +
Layout: Contiguous Storage Property Description
bytebytebytebyte

AddressO


SizeL

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + +
Fields: Contiguous Storage Property Description
Field NameDescription
AddressThis is the address of the raw data in the file. The address may have the + @ref FMT3UndefinedAddress "undefined address" value, to indicate that storage has not + yet been allocated for this array.
SizeThis field contains the size allocated to store the raw data, in bytes.
+ +Class-specific information for chunked storage (layout class 2): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Chunked Storage Property Description
bytebytebytebyte
DimensionalityThis space inserted only to align table nicely

AddressO

Dimension 0 Size
Dimension 1 Size
...
Dimension \#n Size
Dataset Element Size
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Chunked Storage Property Description
Field NameDescription
Dimensionality>A chunk has a fixed dimensionality. This field specifies the number of dimension size fields + later in the message.
AddressThis is the address of the @ref subsubsec_fmt3_infra_btrees_v1 that is used to look up the + addresses of the chunks that actually store portions of the array data. The address may have the + @ref FMT3UndefinedAddress "undefined address" value, to indicate + that storage has not yet been allocated for this array.
Dimension \#n SizeThese values define the dimension size of a single chunk, in units of array elements (not bytes). + The first dimension stored in the list of dimensions is the slowest changing dimension and the + last dimension stored is the fastest changing dimension.
Dataset Element SizeThe size of a dataset element, in bytes.
+ +\anchor FMT3DataLayoutV4

Version 4 of this message is similar to version 3 but has additional +information for the virtual layout class as well as indexing information for the chunked layout class.

+ + + + + + + + + + + + + + + + +
Layout: Data Layout Message (Version 4)
bytebytebytebyte
VersionLayout ClassThis space inserted only to align table nicely

Properties (variable size)

+ + + + + + + + + + + + + + + + + + + +
Fields: Data Layout Message (Version 4)
Field NameDescription
VersionThe value for this field is 4 and is used by version 1.10.0 and later of the library to + store properties for each layout class and indexing information for the chunked layout.
Layout ClassThe layout class specifies specifies the type of storage for the data and how the other fields + of the layout message are to be interpreted. + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0Compact Storage
1Contiguous Storage
2Chunked Storage
3Virtual Storage
PropertiesThis variable-sized field encodes information specific to each layout class as follows: + + + + + + + + + + + + + + + + + + + + + +
Layout ClassDescription
Compact StorageSee @ref FMT3CompactStorage "Compact Storage Property Description" for the version 3 + Data Layout message.
Contiguous StorageSee @ref FMT3ContiguousStorage "Contiguous Storage Property Description" for the version 3 + Data Layout message.
Chunked StorageSee @ref FMT3ChunkedStorage "Chunked Storage Property Description" below.
Virtual StorageSee @ref FMT3VirtualStorage "Virtual Storage Property Description" below.
+ +\anchor FMT3ChunkedStorage

Class-specific information for chunked storage (layout class 2):

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Chunked Storage Property Description
bytebytebytebyte
FlagsDimensionalityDimension Size Encoded LengthThis space inserted only to align table nicely

Dimension 0 Size (variable size)


Dimension 1 Size (variable size)


...


Dimension \#n Size (variable size)

Chunk Indexing TypeThis space inserted only to align table nicely

Indexing Type Information (variable size)


AddressO

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Chunked Storage Property Description
Field NameDescription
FlagsThis is the chunked layout feature flag: + + + + + + + + + + + + + +
ValueDescription
DONT_FILTER_PARTIAL_BOUND_CHUNKS (bit 0)Do not apply filter to a partial edge chunk.
SINGLE_INDEX_WITH_FILTER (bit 1)A filtered chunk for Single Chunk indexing.
DimensionalityA chunk has fixed dimension. This field specifies the number of Dimension Size fields + later in the message.
Dimension Size Encoded LengthThis is the size in bytes used to encode Dimension Size.
Dimension \#n SizeThese values define the dimension size of a single chunk, in units of array elements (not bytes). + The first dimension stored in the list of dimensions is the slowest changing dimension and the + last dimension stored is the fastest changing dimension.
Chunk Indexing TypeThere are five indexing types used to look up addresses of the chunks. For more information on each + type, see @ref sec_fmt3_appendixc
+ + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
1@ref subsec_fmt3_appendixc_chunk indexing type.
2@ref subsec_fmt3_appendixc_implicit indexing type.
3@ref subsec_fmt3_appendixc_fixedarr indexing type.
4@ref subsec_fmt3_appendixc_extarr indexing type.
5@ref subsec_fmt3_appendixc_appv2btree indexing type.
Indexing Type InformationThis variable-sized field encodes information specific to an indexing type. More information on + what is encoded with each type can be found below this table. +
    +
  • See @ref FMT3IndexInfoSingle "Single Chunk" below.
    • +
  • See @ref FMT3IndexInfoImplicit "Implicit" below.
    • +
  • See @ref FMT3IndexInfoFixed "Fixed Array" below.
    • +
  • See @ref FMT3IndexInfoExtensible "Extensible Array" below.
    • +
  • See @ref FMT3IndexInfoV2Btrees "Version 2 B-tree" below.
    • +
AddressThis is the address specific to an indexing type. The address may be undefined if the chunk or + index storage is not allocated yet.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
Single Chunk indexAddress of the single chunk.
Implicit indexAddress of the array of dataset chunks.
Fixed Array indexAddress of the index.
Extensible Array indexAddress of the index.
Version 2 B-tree indexAddress of the index.
+ +
    +
  1. \anchor FMT3IndexInfoSingle

    Index-specific information for Single Chunk:

    +The following information exists only when the chunk is filtered. In other words, when +DONT_FILTER_PARTIAL_BOUND_CHUNKS (bit 0) is enabled in the field flags. + + + + + + + + + + + + + + +
    Layout: Single Chunk Indexing Information
    bytebytebytebyte

    Size of filtered chunkL

    Filters for chunk
    +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + +
    Fields: Single Chunk Indexing Information
    Field NameDescription
    Size of filtered chunkThis field is the size of a filtered chunk.
    Filters for chunkThis field contains filters for the chunk.
    +
    2. + +
  2. \anchor FMT3IndexInfoImplicit

    Index-specific information for Implicit:

    + + + + + + + + + + + +
    Layout: Implicit Indexing Information
    bytebytebytebyte
    No specific indexing information
    +
    3. + +
  3. \anchor FMT3IndexInfoFixed

    Index-specific information for Fixed Array:

    + + + + + + + + + + + + +
    Layout: Fixed Array Indexing Information
    bytebytebytebyte
    Page BitsThis space inserted only to align table nicely
    + + + + + + + + + + +
    Fields: Fixed Array Indexing Information
    Field NameDescription
    Page BitsThis field contains the number of bits needed to store the maximum number of elements in a + data block page.
    +
    4. + +
  4. \anchor FMT3IndexInfoExtensible

    Index-specific information for Extensible Array:

    + + + + + + + + + + + + + + + + + +
    Layout: Extensible Array Indexing Information
    bytebytebytebyte
    Max BitsIndex ElementsMin PointersMin Elements
    Page BitsThis space inserted only to align table nicely
    + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Fields: Extensible Array Indexing Information
    Field NameDescription
    Max BitsThis field contains the number of bits needed to store the maximum number of elements + in the array.
    Index ElementsThis field contains the number of elements to store in the index block.
    Min PointersThis field contains the minimum number of data block pointers for a superblock.
    Min ElementsThis field contains the minimum number of elements per data block.
    Page BitsThis field contains the number of bits needed to store the maximum number of elements in + a data block page.
    +
    5. + +
  5. \anchor FMT3IndexInfoV2Btrees

    Index-specific information for Version 2 B-tree:

    + + + + + + + + + + + + + + + + +
    Layout: Version 2 B-tree Indexing Information
    bytebytebytebyte
    Node Size
    Split PercentMerge PercentThis space inserted only to align table nicely
    + + + + + + + + + + + + + + + + + + +
    Fields: Version 2 B-tree Indexing Information
    Field NameDescription
    Node SizeThis field is the size in bytes of a B-tree node.
    Split PercentThis field is the percentage full of a B-tree node at which to split the node.
    Merge PercentThis field is the percentage full of a B-tree node at which to merge the node.
    +
    6. +
+ +\anchor FMT3VirtualStorage

Class-specific information for virtual storage (layout class 3):

+ + + + + + + <>byte + + + + + + + +
Layout: Virtual Storage Property Description
bytebytebyte

AddressO

Index
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + +
Fields: Virtual Storage Property Description
Field NameDescription
AddressThis is the address of the global heap collection where the VDS mapping entries are stored. + See @ref subsec_fmt3_infra_globalheapvds
IndexThis is the index of the data object within the global heap collection.
+ + +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_bogus IV.A.2.j. The Bogus Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Bogus
Header Message Type:0x0009
Length: 4 bytes
Status: For testing only; should never be stored in a valid file.
Description:This message is used for testing the HDF5 Library’s response to an “unknown” + message type and should never be encountered in a valid HDF5 file.
Format of Data: See the tables below.
+ + + + + + + + + + + + +
Layout: Bogus Message
bytebytebytebyte
Bogus Value
+ + + + + + + + + + + +
Fields: Bogus Message
Field NameDescription
Bogus ValueThis value should always be: 0xdeadbeef.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_groupinfo IV.A.2.k. The Group Info Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Group Info
Header Message Type: 0x000A
Length: Varies
Status: Optional; may not be repeated.
Description:This message stores information for the constants defining a “new style” group’s + behavior. Constant information will be stored in this message and variable information will be stored + in the @ref subsubsec_fmt3_dataobject_hdr_msg_linkinfo message.
+ Note: the “estimated entry” information below is used when determining the size of the + object header for the group when it is created.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + +
Layout: Group Info Message
bytebytebytebyte
VersionFlagsLink Phase Change: Maximum Compact Value (optional)
Link Phase Change: Minimum Dense Value (optional)Estimated Number of Entries (optional)
Estimated Link Name Length of Entries (optional)This space inserted only to align table nicely
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Group Info Message
Field NameDescription
VersionThe version number for this message. This document describes version 0.
FlagsThis is the group information flag with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
0If set, link phase change values are stored.
1If set, the estimated entry information is non-default + and is stored.
2-7Reserved
Link Phase Change: Maximum Compact ValueThe is the maximum number of links to store “compactly” (in the group’s object header).
+ This field is present if bit 0 of Flags is set.
Link Phase Change: Minimum Dense ValueThis is the minimum number of links to store “densely” (in the group’s fractal + heap). The fractal heap’s address is located in the @ref subsubsec_fmt3_dataobject_hdr_msg_linkinfo + message.
+ This field is present if bit 0 of Flags is set.
Estimated Number of EntriesThis is the estimated number of entries in groups. If this field is not present, the default value of + 4 will be used for the estimated number of group entries.
+ This field is present if bit 1 of Flags is set.
Estimated Link Name Length of EntriesThis is the estimated length of entry name. If this field is not present, the default value of + 8 will be used for the estimated link name length of group entries.
+ This field is present if bit 1 of Flags is set.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_filter IV.A.2.l. The Data Storage - Filter Pipeline Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Data Storage - Filter Pipeline
Header Message Type: 0x000B
Length: Varies
Status: Optional; may not be repeated.
Description:This message describes the filter pipeline which should be applied to the data stream by providing filter + identification numbers, flags, a name, and client data.
+ This message may be present in the object headers of both dataset and group objects. For datasets, it + specifies the filters to apply to raw data. For groups, it specifies the filters to apply to the + group’s fractal heap. Currently, only datasets using chunked data storage use the filter pipeline + on their raw data.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + +
Layout: Filter Pipeline Message - Version 1
bytebytebytebyte
VersionNumber of FiltersReserved (zero)
Reserved (zero)

Filter Description List (variable size)

+ + + + + + + + + + + + + + + + + + + +
Fields: Filter Pipeline Message - Version 1
Field NameDescription
VersionThe version number for this message. This table describes version 1.
Number of FiltersThe total number of filters described in this message. The maximum possible number of filters in a + message is 32.
Filter Description ListA description of each filter. A filter description appears in the next table.
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Filter Description
bytebytebytebyte
Filter IdentificationName Length
FlagsNumber of Values for Client Data

Name (variable size, optional)


Client Data (variable size, optional)

Padding (variable size, optional)
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Filter Description
Field NameDescription
Filter Identification ValueThis value, often referred to as a filter identifier, is designed to be a unique identifier for + the filter. Values from zero through 32,767 are reserved for filters supported by The HDF Group + in the HDF5 library and for filters requested and supported by third parties. Filters supported + by The HDF Group are documented immediately below. Information on 3rd-party filters can be found + at The HDF Group’s + Registered Filters page.
+ 1
To request a filter identifier, + please contact The HDF Group’s Help Desk at HDF Help Desk. + You will be asked to provide the following information: +
    +
  1. Contact information for the developer requesting the new identifier
    2. +
  2. A short description of the new filter
    3. +
  3. Links to any relevant information, including licensing information
    4. +

+ Values from 32768 to 65535 are reserved for non-distributed uses (for example, internal company usage) + or for application usage when testing a feature. The HDF Group does not track or document the use of + the filters with identifiers from this range.
+ The filters currently in library version 1.8.0 are listed below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IdentificationNameDescription
0N/AReserved
1deflateGZIP deflate compression
2shuffleData element shuffling
3fletcher32Fletcher32 checksum
4szipSZIP compression
5nbitN-bit packing
6scaleoffsetScale and offset encoded values
Name LengthEach filter has an optional null-terminated ASCII name and this field holds the length of the name + including the null termination padded with nulls to be a multiple of eight. If the filter has no name + then a value of zero is stored in this field.
FlagsThe flags indicate certain properties for a filter. The bit values defined so far are: + + + + + + + + + + + + + +
ValueDescription
0If set then the filter is an optional filter. During output, if an optional filter fails it will be + silently skipped in the pipeline.
1-15Reserved (zero)
Number of Client Data ValuesEach filter can store integer values to control how the filter operates. The number of entries + in the Client Data array is stored in this field.
NameIf the Name Length field is non-zero then it will contain the size of this field, padded + to a multiple of eight. This field contains a null-terminated, ASCII character string to serve as + a comment/name for the filter.
Client DataThis is an array of four-byte integers which will be passed to the filter function. The Client Data + Number of Values determines the number of elements in the array.
PaddingFour bytes of zeroes are added to the message at this point if the Client Data Number of Values field + contains an odd number.
+\anchor FMT3Footnote1Change 1 If you are reading an earlier version of this document, this +link may have changed. If the link does not work, use the latest version of this document on +The HDF Group’s github website, +HDF5 File Format Specification; the link there will always be correct. + + + + + + + + + + + + + + + + + +
Layout: Filter Pipeline Message - Version 2
bytebytebytebyte
VersionNumber of FiltersThis space inserted only to align table nicely

Filter Description List (variable size)

+ + + + + + + + + + + + + + + + + + + +
Fields: Filter Pipeline Message - Version 2
Field NameDescription
VersionThe version number for this message. This table describes version 2.
Number of FiltersThe total number of filters described in this message. The maximum possible number of filters in a + message is 32.
Filter Description ListA description of each filter. A filter description appears in the next table.
+ + + + + + + + + + + + + + + + + + + + + + + +
Layout: Filter Description
bytebytebytebyte
Filter IdentificationName Length (optional)
FlagsNumber Client Data Values

Name (variable size, optional)


Client Data (variable size, optional)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Filter Description
Field NameDescription
Filter Identification ValueThis value, often referred to as a filter identifier, is designed to be a unique identifier for + the filter. Values from zero through 32,767 are reserved for filters supported by The HDF Group + in the HDF5 library and for filters requested and supported by third parties. Filters supported + by The HDF Group are documented immediately below. Information on 3rd-party filters can be found + at The HDF Group’s + Registered Filters page.
+ 1
To request a filter identifier, + please contact The HDF Group’s Help Desk at HDF Help Desk. + You will be asked to provide the following information: +
    +
  1. Contact information for the developer requesting the new identifier
    2. +
  2. A short description of the new filter
    3. +
  3. Links to any relevant information, including licensing information
    4. +

+ Values from 32768 to 65535 are reserved for non-distributed uses (for example, internal company usage) + or for application usage when testing a feature. The HDF Group does not track or document the use of + the filters with identifiers from this range.
+ The filters currently in library version 1.8.0 are listed below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IdentificationNameDescription
0N/AReserved
1deflateGZIP deflate compression
2shuffleData element shuffling
3fletcher32Fletcher32 checksum
4szipSZIP compression
5nbitN-bit packing
6scaleoffsetScale and offset encoded values
Name LengthEach filter has an optional null-terminated ASCII name and this field holds the length of the name + including the null termination padded with nulls to be a multiple of eight. If the filter has no name + then a value of zero is stored in this field.
+ Filters with IDs less than 256 (in other words, filters that are defined in this format documentation) + do not store the Name Length or Name fields.
FlagsThe flags indicate certain properties for a filter. The bit values defined so far are: + + + + + + + + + + + + + +
ValueDescription
0If set then the filter is an optional filter. During output, if an optional filter fails it will be + silently skipped in the pipeline.
1-15Reserved (zero)
Number of Client Data ValuesEach filter can store integer values to control how the filter operates. The number of entries + in the Client Data array is stored in this field.
NameIf the Name Length field is non-zero then it will contain the size of this field, padded + to a multiple of eight. This field contains a null-terminated, ASCII character string to serve as + a comment/name for the filter.
Client DataThis is an array of four-byte integers which will be passed to the filter function. The Client Data + Number of Values determines the number of elements in the array.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_attribute IV.A.2.m. The Attribute Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Attribute
Header Message Type: 0x000C
Length: Varies
Status: Optional; may be repeated.
Description:The Attribute message is used to store objects in the HDF5 file which are used as attributes, + or “metadata” about the current object. An attribute is a small dataset; it has a name, + a datatype, a dataspace, and raw data. Since attributes are stored in the object header, they should + be relatively small (in other words, less than 64KB). They can be associated with any type of object + which has an object header (groups, datasets, or committed (named) datatypes).
+ In 1.8.x versions of the library, attributes can be larger than 64KB. See the + “ @ref subsec_attribute_special ” section of the Attributes chapter in + the @ref UG for more information.
+ Note: Attributes on an object must have unique names: the HDF5 Library currently enforces this by + causing the creation of an attribute with a duplicate name to fail. Attributes on different + objects may have the same name, however.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Attribute Message (Version 1)
bytebytebytebyte
VersionReserved (zero)Name Size
Datatype SizeDataspace Size

Name (variable size)


Datatype (variable size)


Dataspace (variable size)


Data (variable size)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Attribute Message (Version 1)
Field NameDescription
VersionThe version number information is used for changes in the format of the attribute message and is + described here: + + + + + + + + + + + + + +
VersionDescription
0Never used.
1Used by the library before version 1.6 to encode attribute message. This version does not + support shared datatypes.
Name SizeThe length of the attribute name in bytes including the null terminator. Note that the + Name field below may contain additional padding not represented by this field.
Datatype SizeThe length of the datatype description in the Datatype field below. Note that the + Datatype field may contain additional padding not represented by this field.
Dataspace SizeThe length of the dataspace description in the Dataspace field below. Note that the + Dataspace field may contain additional padding not represented by this field.
NameThe null-terminated attribute name. This field is padded with additional null characters to make it a + multiple of eight bytes.
TypeThe datatype description follows the same format as described for the datatype object header message. + This field is padded with additional zero bytes to make it a multiple of eight bytes.
SpaceThe dataspace description follows the same format as described for the dataspace object header message. + This field is padded with additional zero bytes to make it a multiple of eight bytes.
DataThe raw data for the attribute. The size is determined from the datatype and dataspace descriptions. + This field is not padded with additional bytes.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Attribute Message (Version 2)
bytebytebytebyte
VersionFlagsName Size
Datatype SizeDataspace Size

Name (variable size)


Datatype (variable size)


Dataspace (variable size)


Data (variable size)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Attribute Message (Version 2)
Field NameDescription
VersionThe version number information is used for changes in the format of the attribute message and is + described here: + + + + + + + + + +
VersionDescription
2Used by the library of version 1.6.x and after to encode attribute messages. This version + supports shared datatypes. The fields of name, datatype, and dataspace are not padded with + additional bytes of zero.
Flags>This bit field contains extra information about interpreting the attribute message: + + + + + + + + + + + + + +
BitDescription
0If set, datatype is shared.
1If set, dataspace is shared.
Name SizeThe length of the attribute name in bytes including the null terminator.
Datatype SizeThe length of the datatype description in the Datatype field below.
Dataspace SizeThe length of the dataspace description in the Dataspace field below.
NameThe null-terminated attribute name. This field is not padded with additional bytes.
DatatypeThe datatype description follows the same format as described for the datatype object + header message.
+ If the Flag field indicates this attribute’s datatype is shared, this field will + contain a “shared message” encoding instead of the datatype encoding.
+ This field is not padded with additional bytes.
DataspaceThe dataspace description follows the same format as described for the dataspace object + header message.
+ If the Flag field indicates this attribute’s dataspace is shared, this field will + contain a “shared message” encoding instead of the dataspace encoding.
+ This field is not padded with additional bytes.
DataThe raw data for the attribute. The size is determined from the datatype and dataspace + descriptions.
+ This field is not padded with additional zero bytes.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Attribute Message (Version 3)
bytebytebytebyte
VersionFlagsName Size
Datatype SizeDataspace Size
Name Character Set EncodingThis space inserted only to align table nicely

Name (variable size)


Datatype (variable size)


Dataspace (variable size)


Data (variable size)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Attribute Message (Version 3)
Field NameDescription
VersionThe version number information is used for changes in the format of the attribute message and is + described here: + + + + + + + + + +
VersionDescription
2Used by the library of version 1.8.x and after to encode attribute messages. This version + supports attributes with non-ASCII names.
Flags>This bit field contains extra information about interpreting the attribute message: + + + + + + + + + + + + + +
BitDescription
0If set, datatype is shared.
1If set, dataspace is shared.
Name SizeThe length of the attribute name in bytes including the null terminator.
Datatype SizeThe length of the datatype description in the Datatype field below.
Dataspace SizeThe length of the dataspace description in the Dataspace field below.
Name Character Set EncodingThe character set encoding for the attribute’s name: + + + + + + + + + + + + + +
ValueDescription
0ASCII character set encoding
1UTF-8 character set encoding
NameThe null-terminated attribute name. This field is not padded with additional bytes.
DatatypeThe datatype description follows the same format as described for the datatype object + header message.
+ If the Flag field indicates this attribute’s datatype is shared, this field will + contain a “shared message” encoding instead of the datatype encoding.
+ This field is not padded with additional bytes.
DataspaceThe dataspace description follows the same format as described for the dataspace object + header message.
+ If the Flag field indicates this attribute’s dataspace is shared, this field will + contain a “shared message” encoding instead of the dataspace encoding.
+ This field is not padded with additional bytes.
DataThe raw data for the attribute. The size is determined from the datatype and dataspace + descriptions.
+ This field is not padded with additional zero bytes.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_comment IV.A.2.n. The Object Comment Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Object Comment
Header Message Type: 0x000D
Length: Varies
Status: Optional; may not be repeated.
Description:The object comment is designed to be a short description of an object. An object comment is a sequence + of non-zero (\0) ASCII characters with no other formatting included by the library.
Format of Data: See the tables below.
+ + + + + + + + + + + + +
Layout: Object Comment Message
bytebytebytebyte

Comment (variable size)

+
+ + + + + + + + + + + +
Fields: Object Comment Message
Field NameDescription
NameA null terminated ASCII character string.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_omodified IV.A.2.o. The Object Modification Time (Old) Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Object Modification Time (Old)
Header Message Type: 0x000E
Length: Fixed
Status: Optional; may not be repeated.
Description:The object modification date and time is a timestamp which indicates (using ISO-8601 date and + time format) the last modification of an object. The time is updated when any object header + message changes according to the system clock where the change was posted. All fields of this + message should be interpreted as coordinated universal time (UTC).
+ This modification time message is deprecated in favor of the “new” + @ref subsubsec_fmt3_dataobject_hdr_msg_mod message and is no longer written to the file in + versions of the HDF5 Library after the 1.6.0 version.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Modification Time Message (Old)
bytebytebytebyte
Year
MonthDay of Month
HourMinute
SecondReserved
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Modification Time Message (Old)
Field NameDescription
YearThe four-digit year as an ASCII string. For example, 1998.
MonthThe month number as a two digit ASCII string where January is 01 and December is + 12.
Day of MonthThe day number within the month as a two digit ASCII string. The first day of the month is + 01.
HourThe hour of the day as a two digit ASCII string where midnight is 00 and 11:00pm + is 23.
MinuteThe minute of the hour as a two digit ASCII string where the first minute of the hour is + 00 and the last is 59.
SecondThe second of the minute as a two digit ASCII string where the first second of the minute is + 00 and the last is 59.
ReservedThis field is reserved and should always be zero.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_shared IV.A.2.p. The Shared Message Table Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Shared Message Table
Header Message Type: 0x000F
Length: Fixed
Status: Optional; may not be repeated.
Description:This message is used to locate the table of shared object header message (SOHM) indexes. Each + index consists of information to find the shared messages from either the heap or object header. + This message is only found in the superblock extension.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + +
Layout: Shared Message Table Message
bytebytebytebyte
VersionThis space inserted only to align table nicely

Shared Object Header Message Table AddressO

Number of IndicesThis space inserted only to align table nicely
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + +
Fields: Shared Message Table Message
Field NameDescription
VersionThe version number for this message. This document describes version 0.
Shared Object Header Message Table AddressThis field is the address of the master table for shared object header message indexes.
Number of IndicesThis field is the number of indices in the master table.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_continuation IV.A.2.q. The Object Header Continuation Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Object Header Continuation
Header Message Type: 0x0010
Length: Fixed
Status: Optional; may be repeated.
Description:The object header continuation is the location in the file of a block containing more header messages + for the current data object. This can be used when header blocks become too large or are likely to + change over time.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + +
Layout: Object Header Continuation Message
bytebytebytebyte

OffsetO


LengthL

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + +
Fields: Object Header Continuation Message
Field NameDescription
OffsetThis value is the address in the file where the header continuation block is located.
LengthThis value is the length in bytes of the header continuation block in the file.
+ +The format of the header continuation block that this message points to depends on the version of the +object header that the message is contained within. + +Continuation blocks for version 1 object headers have no special formatting information; they are +merely a list of object header message info sequences (type, size, flags, reserved bytes and data for +each message sequence). See the description of @ref subsubsec_fmt3_dataobject_hdr_prefix_one. + +Continuation blocks for version 2 object headers do have special formatting information as +described here (see also the description of @ref subsubsec_fmt3_dataobject_hdr_prefix_two): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 2 Object Header Continuation Block
bytebytebytebyte
Signature
Header Message Type \#1Size of Header Message Data \#1Header Message \#1 Flags
Header Message \#1 Creation Order (optional)This space inserted only to align table nicely

Header Message Data \#1

.
.
.
Header Message Type \#nSize of Header Message Data \#nHeader Message \#n Flags
Header Message \#n Creation Order (optional)This space inserted only to align table nicely

Header Message Data \#n

Gap (optional, variable size)
Checksum
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 Object Header Continuation Block
Field NameDescription
SignatureThe ASCII character string “OCHK” is used to indicate the + beginning of an object header continuation block. This gives file consistency checking + utilities a better chance of reconstructing a damaged file.
Header Message \#n TypeSame format as version 1 of the object header, described above.
Size of Header Message \#n DataSame format as version 1 of the object header, described above.
Header Message \#n FlagsSame format as version 1 of the object header, described above.
Header Message \#n Creation OrderThis field stores the order that a message of a given type was created in.
+ This field is present if bit 2 of flags is set.
Header Message \#n DataSame format as version 1 of the object header, described above.
GapA gap in an object header chunk is inferred by the end of the messages for the chunk before the + beginning of the chunk’s checksum. Gaps are always smaller than the size of an object header + message prefix (message type + message size + message flags).
+ Gaps are formed when a message (typically an attribute message) in an earlier chunk is deleted + and a message from a later chunk that does not quite fit into the free space is moved into the + earlier chunk.
ChecksumThis is the checksum for the object header chunk.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_stmgroup IV.A.2.r. The Symbol Table Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Symbol Table Message
Header Message Type: 0x0011
Length: Fixed
Status: Required for “old style” groups; may not be repeated.
Description:Each “old style” group has a v1 B-tree and a local heap for storing symbol table entries, + which are located with this message.
Format of data: See the tables below.
+ + + + + + + + + + + + + + + +
Layout: Symbol Table Message
bytebytebytebyte

v1 B-tree AddressO


Local Heap AddressO

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + +
Fields: Symbol Table Message
Field NameDescription
v1 B-tree AddressThis value is the address of the v1 B-tree containing the symbol table entries for the group.
Local Heap AddressThis value is the address of the local heap containing the link names for the symbol table + entries for the group.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_mod IV.A.2.s. The Object Modification Time Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Object Modification Time
Header Message Type: 0x0012
Length: Fixed
Status: Optional; may not be repeated.
Description:The object modification time is a timestamp which indicates the time of the last modification of + an object. The time is updated when any object header message changes according to the system clock + where the change was posted.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + +
Layout: Modification Time Message
bytebytebytebyte
VersionReserved (zero)
Seconds After UNIX Epoch
+ + + + + + + + + + + + + + + +
Fields: Modification Time Message
Field NameDescription
VersionThe version number is used for changes in the format of Object Modification Time and is described + here: + + + + + + + + + + + + + +
VersionDescription
0Never used.
1Used by Version 1.6.1 and after of the library to encode time. In this version, the time is + the seconds after Epoch.
Seconds After UNIX EpochA 32-bit unsigned integer value that stores the number of seconds since 0 hours, 0 minutes, + 0 seconds, January 1, 1970, Coordinated Universal Time.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_btreek IV.A.2.t. The B-tree ‘K’ Values Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: B-tree ‘K’ Values
Header Message Type: 0x0013
Length: Fixed
Status: Optional; may not be repeated.
Description:This message retrieves non-default ‘K’ values for internal and leaf nodes of a group + or indexed storage v1 B-trees. This message is only found in the superblock extension.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + +
Layout: B-tree ‘K’ Values Message
bytebytebytebyte
VersionIndexed Storage Internal Node KThis space inserted only to align table nicely
Group Internal Node KGroup Leaf Node K
+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: B-tree ‘K’ Values Message
Field NameDescription
VersionThe version number for this message. This document describes version 0.
Indexed Storage Internal Node KThis is the node ‘K’ value for each internal node of an indexed storage v1 B-tree. + See the description of this field in version 0 and 1 of the superblock as well the section on + v1 B-trees.
Group Internal Node KThis is the node ‘K’ value for each internal node of a group v1 B-tree. See the + description of this field in version 0 and 1 of the superblock as well as the section + on v1 B-trees.
Group Leaf Node KThis is the node ‘K’ value for each leaf node of a group v1 B-tree. See the + description of this field in version 0 and 1 of the superblock as well as the section on v1 + B-trees.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_drvinfo IV.A.2.u. The Driver Info Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Driver Info
Header Message Type: 0x0014
Length: Varies
Status: Optional; may not be repeated.
Description:This message contains information needed by the file driver to reopen a file. This message is + only found in the superblock extension: see the @ref subsec_fmt3_boot_supext section + for more information. For more information on the fields in the driver info message, see the + @ref subsec_fmt3_boot_driver section; those who use the multi and family file drivers will find + this section particularly helpful.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + +
Layout: Driver Info Message
bytebytebytebyte
VersionThis space inserted only to align table nicely

Driver Identification
Driver Information SizeThis space inserted only to align table nicely


Driver Information (variable size)


+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: Driver Info Message
Field NameDescription
VersionThe version number for this message. This document describes version 0.
Driver IdentificationThis is an eight-byte ASCII string without null termination which identifies the driver.
Driver Information SizeThe size in bytes of the Driver Information field of this message.
Driver InformationDriver information is stored in a format defined by the file driver.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_attrinfo IV.A.2.v. The Attribute Info Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Attribute Info
Header Message Type: 0x0015
Length: Varies
Status: Optional; may not be repeated.
Description:This message stores information about the attributes on an object, such as the maximum creation + index for the attributes created and the location of the attribute storage when the attributes + are stored “densely”.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + +
Layout: Attribute Info Message
bytebytebytebyte
VersionFlagsMaximum Creation Index (optional)

Fractal Heap AddressO


Attribute Name v2 B-tree AddressO


Attribute Creation Order v2 B-tree AddressO (optional)

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Attribute Info Message
Field NameDescription
VersionThe version number for this message. This document + describes version 0.
FlagsThis is the attribute index information flag with the following definition: + + + + + + + + + + + + + + + + + +
BitDescription
0If set, creation order for attributes is tracked.
1If set, creation order for attributes is indexed.
2-7Reserved
Maximum Creation IndexThe is the maximum creation order index value for the attributes on the object.
+ This field is present if bit 0 of Flags is set.
Fractal Heap AddressThis is the address of the fractal heap to store dense attributes. Each attribute stored in the + fractal heap is described by the @ref subsubsec_fmt3_dataobject_hdr_msg_attribute.
Attribute Name v2 B-tree AddressThis is the address of the version 2 B-tree to index the names of densely stored attributes.
Attribute Creation Order v2 B-tree AddressThis is the address of the version 2 B-tree to index the creation order of densely stored + attributes.
+ This field is present if bit 1 of Flags is set.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_refcount IV.A.2.w. The Object Reference Count Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: Object Reference Count
Header Message Type: 0x0016
Length: Fixed
Status: Optional; may not be repeated.
Description:This message stores the number of hard links (in groups or objects) pointing to an object: in + other words, its reference count.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + +
Layout: Object Reference Count
bytebytebytebyte
VersionThis space inserted only to align table nicely
Reference count
+ + + + + + + + + + + + + + + +
Fields: Object Reference Count
Field NameDescription
VersionThe version number for this message. This document describes version 0.
Reference CountThe unsigned 32-bit integer is the reference count for the object. This message is only present + in “version 2” (or later) object headers, and if not present in those object header versions, + the reference count for the object is assumed to be 1.
+ +\subsubsection subsubsec_fmt3_dataobject_hdr_msg_fsinfo IV.A.2.x. The File Space Info Message + + + + + + + + + + + + + + + + + + + + +
Header Message Name: File Space Info
Header Message Type: 0x0017
Length: Fixed
Status: Optional; may not be repeated.
Description:This message stores the file space management information that the library uses in handling file + space requests for the file. Version 0 of the message is used for release 1.10.0 only. Version 1 + of the message is used for release 1.10.1+. There is no File Space Info message before release + 1.10 as the library does not track file space across multiple file opens.
+ Note that version 0 is deprecated starting in release 1.10.1. That means when the 1.10.1+ library + opens an HDF5 file with a version 0 message, the library will decode and map the message to + version 1. On file close, it will encode the message as a version 1 message.
+ The library uses the following three mechanisms to manage file space in an HDF5 file: +
    +
  • Free-space managers
    + They track free-space sections of various sizes in the file that are not currently + allocated. Each free-space manager corresponds to a file space type. There are two main + groups of file space types: metadata and raw data. Metadata is further divided into five + types: superblock, B-tree, global heap, local heap, and object header. See the description + of @ref subsec_fmt3_infra_freespaceindex as well the description of file space allocation + types in @ref sec_fmt3_appendixb.
    • +
  • Aggregators
    + The library manages two aggregators, one for metadata and one for raw data. Aggregator is + a contiguous block of free-space in the file. The size of each aggregator is tunable via + public routines #H5Pset_meta_block_size and #H5Pset_small_data_block_size respectively.
    • +
  • Virtual file drivers
    + The library's virtual file driver interface dispatches requests for additional space to the + allocation routine of the file driver associated with the file. For example, if the sec2 + file driver is being used, its allocation routine will increase the size of the file to + service the requests.
    • +
+ For release 1.10.0, the library derives the following four file space strategies based on the mechanisms: +
    +
  • #H5F_FILE_SPACE_ALL +
      +
    • Mechanisms used: free-space managers, aggregators, and virtual file drivers
      • +
    • Does not persist free-space across file opens
      • +
    • This strategy is the library default
      • +
    +
    • +
  • #H5F_FILE_SPACE_ALL_PERSIST
    • +
      +
    • Mechanisms used: free-space managers, aggregators, and virtual file drivers
      • +
    • Persist free-space across file opens
      • +
    +
  • #H5F_FILE_SPACE_AGGR_VFD
    • +
      +
    • Mechanisms used: aggregators and virtual file drivers
      • +
    • Does not persist free-space across file opens
      • +
    +
  • #H5F_FILE_SPACE_VFD
    • +
      +
    • Mechanisms used: virtual file drivers
      • +
    • Does not persist free-space across file opens
      • +
    +
+ For release 1.10.1+, the free-space manager mechanism is modified to handle paged aggregation + which aggregates small metadata and raw data allocations into constant-sized well-aligned pages + to allow efficient I/O accesses. With the support of this feature, the library derives the + following four file space strategies: +
    +
  • #H5F_FSPACE_STRATEGY_FSM_AGGR
    • +
      +
    • Mechanisms used: free-space managers, aggregators, and virtual file drivers
      • +
    • This strategy is the library default
      • +
    +
  • #H5F_FSPACE_STRATEGY_PAGE
    • +
      +
    • Mechanisms used: free-space managers with embedded paged aggregation and virtual file drivers
      • +
    +
  • #H5F_FSPACE_STRATEGY_AGGR
    • +
      +
    • Mechanisms used: aggregators and virtual file drivers
      • +
    +
  • #H5F_FSPACE_STRATEGY_NONE
    • +
      +
    • Mechanisms used: virtual file drivers
      • +
    +
+ The default is not persisting free-space across file opens for the above four strategies. User can use + the public routine #H5Pset_file_space_strategy to request persisting free-space.
Format of Data: See the tables below.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: File Space Info
bytebytebytebyte
VersionStrategyThresholdL

Free-space manager addressO for #H5FD_MEM_SUPER


Free-space manager addressO for #H5FD_MEM_BTREE


Free-space manager addressO for #H5FD_MEM_DRAW


Free-space manager addressO for #H5FD_MEM_GHEAP


Free-space manager addressO for #H5FD_MEM_LHEAP


Free-space manager addressO for #H5FD_MEM_OHDR

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + +
Fields: File Space Info
Field NameDescription
VersionThis is the version 0 of this message.
StrategyThis is the file space strategy used to manage file space. There are four types: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
1#H5F_FILE_SPACE_ALL_PERSIST
2#H5F_FILE_SPACE_ALL
3#H5F_FILE_SPACE_AGGR_VFD
4#H5F_FILE_SPACE_VFD
ThresholdThis is the smallest free-space section size that the free-space manager will track.
Free-space manager addressesThese are the six free-space manager addresses for the six file space allocation types: +
    +
  • #H5FD_MEM_SUPER
    • +
  • #H5FD_MEM_BTREE
    • +
  • #H5FD_MEM_DRAW
    • +
  • #H5FD_MEM_GHEAP
    • +
  • #H5FD_MEM_LHEAP
    • +
  • #H5FD_MEM_OHDR
    • +
+ Note that these six fields exist only if the value for the field “Strategy” + is #H5F_FILE_SPACE_ALL_PERSIST.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: File Space Info - Version 1
bytebytebytebyte
VersionStrategyPersisting free-spaceThis space inserted only to align table nicely
Free-space Section ThresholdL
File Space Page Size
Page-end Metadata thresholdThis space inserted only to align table nicely
EOAO

AddressO of small-sized free-space manager for #H5FD_MEM_SUPER


AddressO of small-sized free-space manager for #H5FD_MEM_BTREE


AddressO of small-sized free-space manager for #H5FD_MEM_DRAW


AddressO of small-sized free-space manager for #H5FD_MEM_GHEAP


AddressO of small-sized free-space manager for #H5FD_MEM_LHEAP


AddressO of small-sized free-space manager for #H5FD_MEM_OHDR


AddressO of large-sized free-space manager for #H5FD_MEM_SUPER


AddressO of large-sized free-space manager for #H5FD_MEM_BTREE


AddressO of large-sized free-space manager for #H5FD_MEM_DRAW


AddressO of large-sized free-space manager for #H5FD_MEM_GHEAP


AddressO of large-sized free-space manager for #H5FD_MEM_LHEAP


AddressO of large-sized free-space manager for #H5FD_MEM_OHDR

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: File Space Info - Version 1
Field NameDescription
VersionThis is the version 1 of this message.
StrategyThis is the file space strategy used to manage file space. There are four types: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0#H5F_FSPACE_STRATEGY_FSM_AGGR
1#H5F_FSPACE_STRATEGY_PAGE
2#H5F_FSPACE_STRATEGY_AGGR
3#H5F_FSPACE_STRATEGY_NONE
Persisting free-spaceTrue or false in persisting free-space.
Free-space Section ThresholdThis is the smallest free-space section size that the free-space manager will track.
File space page sizeThis is the file space page size, which is used when the paged aggregation feature is enabled.
Page-end metadata thresholdThis is the smallest free-space section size at the end of a page that the free-space manager will + track. This is used when the paged aggregation feature is enabled.
EOAThe EOA before the allocation of free-space manager header and section info for the self-referential + free-space managers when persisting free-space.
+ Note that self-referential free-space managers are managers that involve file space allocation for + the managers' free-space header and section info.
Addresses of small-sized free-space managersThese are the addresses of the six small-sized free-space manager addresses for the six file space + allocation types: +
    +
  • #H5FD_MEM_SUPER
    • +
  • #H5FD_MEM_BTREE
    • +
  • #H5FD_MEM_DRAW
    • +
  • #H5FD_MEM_GHEAP
    • +
  • #H5FD_MEM_LHEAP
    • +
  • #H5FD_MEM_OHDR
    • +
+ Note that these six fields exist only if the value for the field + “Persisting free-space” is true.
Addresses of large-sized free-space managersThese are the addresses of the six large-sized free-space manager addresses for the six file space + allocation types: +
    +
  • #H5FD_MEM_SUPER
    • +
  • #H5FD_MEM_BTREE
    • +
  • #H5FD_MEM_DRAW
    • +
  • #H5FD_MEM_GHEAP
    • +
  • #H5FD_MEM_LHEAP
    • +
  • #H5FD_MEM_OHDR
    • +
+ Note that these six fields exist only if the value for the field + “Persisting free-space” is true.
+ +\subsection subsec_fmt3_dataobject_storage IV.B. Disk Format: Level 2B - Data Object Data Storage +The data for an object is stored separately from the header information in the file and may not actually +be located in the HDF5 file itself if the header indicates that the data is stored externally. The +information for each record in the object is stored according to the dimensionality of the object +(indicated in the dataspace header message). Multi-dimensional array data is stored in C order; in other +words, the “last” dimension changes fastest. + +Data whose elements are composed of atomic datatypes are stored in IEEE format, unless +they are specifically defined as being stored in a different machine format with the architecture-type +information from the datatype header message. This means that each architecture will need to +[potentially] byte-swap data values into the internal representation for that particular machine. + +Data with a variable-length datatype is stored in the global heap of the HDF5 file. Global heap +identifiers are stored in the data object storage. + +Data whose elements are composed of reference datatypes are stored in several different ways depending +on the particular reference type involved. Object pointers are just stored as the offset of the +object header being pointed to with the size of the pointer being the same number of bytes as offsets +in the file. + +Dataset region references are stored as a heap-ID which points to the following information within the +file-heap: an offset of the object pointed to, number-type information (same format as header message), +dimensionality information (same format as header message), sub-set start and end information (in other +words, a coordinate location for each), and field start and end names (in other words, a [pointer to the] +string indicating the first field included and a [pointer to the] string name for the last field). + +Data of a compound datatype is stored as a contiguous stream of the items in the structure, with each +item formatted according to its datatype.
+Description of datatypes for variable-length, references and compound classes can be found in +@ref subsubsec_fmt3_dataobject_hdr_msg_dtmessage.
+Information about global heap and heap ID can be found in @ref subsec_fmt3_infra_globalheap..
+For reference datatype, see also the encoding description for @ref subsec_fmt3_appendixd_encoderv and +@ref subsec_fmt3_appendixd_encodedp in Appendix D. + + +\section sec_fmt3_appendixa V. Appendix A: Definitions +Definitions of various terms used in this document are included in this section. + + + + + + + + + + + + + +
TermDefinition
Undefined Address\anchor FMT3UndefinedAddress The "undefined address" for a file is a file address with all bits + set: in other words, 0xffff...ff.
Unlimited Size\anchor FMT3UnlimitedDim The "unlimited size" for a size is a value with all bits set: in other words, + 0xffff...ff.
+ +\section sec_fmt3_appendixb VI. Appendix B: File Space Allocation Types +There are six basic types of file memory allocation as follows: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Basic Allocation TypeDescription
#H5FD_MEM_SUPERFile space allocated for Superblock.
#H5FD_MEM_BTREEFile space allocated for B-tree.
#H5FD_MEM_DRAWFile space allocated for raw data.
#H5FD_MEM_GHEAPFile space allocated for Global Heap.
#H5FD_MEM_LHEAPFile space allocated for Local Heap.
#H5FD_MEM_OHDRFile space allocated for Object Header.
+ +There are other file space allocation types that are mapped to the above six basic types +because they are similar in nature. The mapping and the corresponding description are +listed in the following two tables: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Basic Allocation TypeMapping of Allocation Types to Basic Allocation Types
#H5FD_MEM_SUPERnone
#H5FD_MEM_BTREE#H5FD_MEM_SOHM_INDEX
#H5FD_MEM_DRAW#H5FD_MEM_FHEAP_HUGE_OBJ
#H5FD_MEM_GHEAPnone
#H5FD_MEM_LHEAP#H5FD_MEM_FHEAP_DBLOCK, #H5FD_MEM_FSPACE_SINFO
#H5FD_MEM_OHDR#H5FD_MEM_FHEAP_HDR, #H5FD_MEM_FHEAP_IBLOCK, #H5FD_MEM_FSPACE_HDR, #H5FD_MEM_SOHM_TABLE
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Allocation TypeDescription
#H5FD_MEM_FHEAP_HDRFile space allocated for Fractal Heap Header.
#H5FD_MEM_FHEAP_DBLOCKFile space allocated for Fractal Heap Direct Blocks.
#H5FD_MEM_FHEAP_IBLOCKFile space allocated for Fractal Heap Indirect Blocks.
#H5FD_MEM_FHEAP_HUGE_OBJFile space allocated for huge objects in the fractal heap.
#H5FD_MEM_FSPACE_HDRFile space allocated for Free-space Manager Header.
#H5FD_MEM_FSPACE_SINFOFile space allocated for Free-space Section List of the free-space manager.
#H5FD_MEM_SOHM_TABLEFile space allocated for Shared Object Header Message Table.
#H5FD_MEM_SOHM_INDEXFile space allocated for Shared Message Record List.
+ +\section sec_fmt3_appendixc VII. Appendix C: Types of Indexes for Dataset Chunks +For an HDF5 file without the latest format enabled, the library uses the +@ref subsubsec_fmt3_infra_btrees_v1 to index dataset chunks.
+For an HDF5 file with the latest format enabled, the library uses one of the following five +indexing types depending on a chunked dataset’s dimension specification and the way it +is extended. + +\subsection subsec_fmt3_appendixc_chunk VII.A. The Single Chunk Index +The Single Chunk index can be used when the dataset fulfills the following condition: +
    +
  • the current, maximum, and chunk dimension sizes are all the same
    • +
+ +The dataset has only one chunk, and the address of the single chunk is stored in the version 4 +Data Layout message. See the @ref subsec_fmt3_appendixc_chunk layout and field description tables. + +\subsection subsec_fmt3_appendixc_implicit VII.B. The Implicit Index +The Implicit index can be used when the dataset fulfills the following conditions: +
    +
  • fixed maximum dimension sizes
    • +
  • no filter applied to the dataset
    • +
  • the timing for the space allocation of the dataset chunks is + #H5D_ALLOC_TIME_EARLY
    • +
+ +Since the dataset’s dimension sizes are known and storage space is to be allocated early, an +array of dataset chunks are allocated based on the maximum dimension sizes when the dataset is created. +The base address of the array is stored in the version 4 Data Layout message. See the +@ref subsec_fmt3_appendixc_chunk layout layout and field description tables.
+When accessing a dataset chunk with a specified offset, the address of the chunk in the array is computed +as below: +\code +base address + (size of a chunk in bytes * chunk index associated with the offset) +\endcode + +\anchor FMT3ChunkIndex A chunk index starts at 0 and increases according to the fastest changing +dimension, then the next fastest, and so on. The chunk index for a dataset chunk offset is computed as below: +
    +
  1. Calculate the scaled offset for each dimension in scaled_offset:
    + scaled_offset = chunk_offset/chunk_dims
    2. +
  2. Calculate the # of chunks for each dimension in nchunks:
    + nchunks = (curr_dims + chunk_dims - 1)/chunk_dims
    3. +
  3. Calculate the down chunks for each dimension in down_chunks:
    + + // n is the # of dimensions + for(i = (int)(n-1), acc = 1; i >= 0; i--) { + down_chunks[i] = acc; + acc *= nchunks[i]; + } +
    4. +
  4. Calculate the chunk index in chunk_index:
    + + // n is the # of dimensions + for(u = 0, chunk_index = 0; u < n; u++) + chunk_index += down_chunks[u] * scaled_offset[u]; +
    5. +
+ +For example, for a 2-dimensional dataset with curr_dims[4,5] and +chunk_dims[3,2], there will be a total of 6 chunks, with 3 chunks in the fastest +changing dimension and 2 chunks in the slowest changing dimension. See the figure below. +The chunk index for the chunk offset [3,4] is computed as below: +
    +
  1. scaled_offset[0] = 1, scaled_offset[1] = 2
    2. +
  2. nchunks[0] = 2, nchunks[1] = 3
    3. +
  3. down_chunks[0] = 3, down_chunks[1] = 1
    4. +
  4. chunk_index = 5
    5. +
+ + + + + + + + +
Figure 3: Implicit index chunk diagram
\image html FileFormatSpecChunkDiagram.jpg
+ +\subsection subsec_fmt3_appendixc_fixedarr VII.C. The Fixed Array Index +The Fixed Array index can be used when the dataset fulfills the following condition: +
    +
  • fixed maximum dimension sizes
    • +
+ +Since the maximum number of chunks is known, an array of in-file-on-disk addresses based on the +maximum number of chunks is allocated when data is written to the dataset. To access a dataset +chunk with a specified offset, the @ref FMT3ChunkIndex "chunk index" associated with the offset +is calculated. The index is mapped into the array to locate the disk address for the chunk.
+The Fixed Array (FA) index structure provides space and speed improvements in locating chunks over +index structures that handle more dynamic data accesses like a +@ref subsec_fmt3_appendixc_appv2btree index.The entry into the Fixed Array is the Fixed Array +header which contains metadata about the entries stored in the array. The header contains a +pointer to a data block which stores the array of entries that describe the dataset chunks. +For greater efficiency, the array will be divided into multiple pages if the number of entries +exceeds a threshold value. The space for the data block and possibly data block pages are allocated +as a single contiguous block of space.
+The content of the data block depends on whether paging is activated or not. When paging is not +used, elements that describe the chunks are stored in the data block. If paging is turned on, +the data block contains a bitmap indicating which pages are initialized. Then subsequent data +block pages will contain the entries that describe the chunks.
+An entry describes either a filtered or non-filtered dataset chunk. The formats for both element +types are described below.
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Fixed Array Header
bytebytebytebyte
Signature
VersionClient IDEntry SizePage Bits

Max Num EntriesL


Data Block AddressO

Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Fixed Array Header
Field NameDescription
SignatureThe ASCII character string “FAHD” + is used to indicate the beginning of a Fixed Array header. + This gives file consistency checking utilities a better + chance of reconstructing a damaged file.
VersionThis document describes version 0.
Client IDThe ID for identifying the client of the Fixed Array: + + + + + + + + + + + + + + + + + +
IDDescription
0Non-filtered dataset chunks
1Filtered dataset chunks
2+Reserved
Entry SizeThe size in bytes of an entry in the Fixed Array.
Page BitsThe number of bits needed to store the maximum number of entries in a + @ref FMT3FADataBlockPage "data block page"
Max Num EntriesThe maximum number of entries in the Fixed Array.
Data Block AddressThe address of the data block in the Fixed Array.
ChecksumThe checksum for the header.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Fixed Array Data Block
bytebytebytebyte
Signature
VersionClient IDThis space inserted only to align table nicely

Header AddressO


Page Bitmap (variable size and optional)


Elements (variable size and optional)

Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Fixed Array Data Block
Field NameDescription
SignatureThe ASCII character string “FADB” is used to indicate the + beginning of a Fixed Array data block. This gives file consistency checking utilities a + better chance of reconstructing a damaged file.
VersionThis document describes version 0.
Client IDThe ID for identifying the client of the Fixed Array: + + + + + + + + + + + + + + + + + +
IDDescription
0Non-filtered dataset chunks
1Filtered dataset chunks
2+Reserved.
Header AddressThe address of the Fixed Array header. Principally used for file integrity checking.
Page BitmapA bitmap indicating which data block pages are initialized.
+ Exists only if the data block is paged.
ElementsContains the elements stored in the data block and exists only if the data block is not paged. + There are two element types: + + + + + + + + + + + + + +
IDDescription
0@ref FMT3FaNonFilterChunk "Non-filtered dataset chunks"
1@ref FMT3FaFilterChunk "Filtered dataset chunks"
ChecksumThe checksum for the Fixed Array data block.
+ + + + + + + + + + + + + + + +
\anchor FMT3FADataBlockPage Layout: Fixed Array Data Block Page
bytebytebytebyte

Elements (variable size)

Checksum
+ + + + + + + + + + + + + + + +
Fields: Fixed Array Data Block Page
Field NameDescription
ElementsContains the elements stored in the data block page. + There are two element types: + + + + + + + + + + + + + +
IDDescription
0@ref FMT3FaNonFilterChunk "Non-filtered dataset chunks"
1@ref FMT3FaFilterChunk "Filtered dataset chunks"
ChecksumThe checksum for a Fixed Array data block page.
+ +\anchor FMT3FaNonFilterChunk + + + + + + + + + + + +
Layout: Data Block Element for Non-filtered Dataset Chunk
bytebytebytebyte

AddressO

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + +
Fields: Data Block Element for Non-filtered Dataset Chunk
Field NameDescription
AddressThe address of the dataset chunk in the file.
+ +\anchor FMT3FaFilterChunk + + + + + + + + + + + + + + + + + +
Layout: Data Block Element for Filtered Dataset Chunk
bytebytebytebyte

AddressO


Chunk Size (variable size; at most 8 bytes)

Filter Mask
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + +
Fields: Data Block Element for Filtered Dataset Chunk
Field NameDescription
AddressThe address of the dataset chunk in the file.
Chunk SizeThe size of the dataset chunk in bytes.
Filter MaskIndicates the filter to skip for the dataset chunk. Each + filter has an index number in the pipeline; if that filter is + skipped, the bit corresponding to its index is set.
+ +\section subsec_fmt3_appendixc_extarr VII.D. The Extensible Array Index +The Extensible Array index can be used when the dataset fulfills the following condition: +
    +
  • only one dimension of unlimited extent
    • +
+ +The Extensible Array (EA) is a data structure that is used as a chunk index in datasets where the +dataspace has a single unlimited dimension. In other words, one dimension is set to +H5S_UNLIMITED, and the other dimensions are any number of fixed-size dimensions. The +idea behind the extensible array is that a particular data object can be located via a lightweight +indexing structure of fixed depth for a given address space. This indexing structure requires only +a few (2-3) file operations per element lookup and gives good cache performance. Unlike the B-tree +structure, the extensible array is optimized for appends. Where a B-tree would always add at the +rightmost node under these circumstances, either creating a deep tree (version 1) or requiring +expensive rebalances to correct (version 2), the extensible array has already mapped out a pre-balanced +internal structure. This optimized internal structure is instantiated as needed when chunk +records are inserted into the structure.
+An Extensible Array consists of a header, an index block, secondary blocks, data blocks, and +(optional) data block pages. The general scheme is that the index block is used to reference a +secondary block, which is, in turn, used to reference the data block page where the chunk information +is stored. The data blocks will be paged for efficiency when their size passes a threshold value. +These pages are laid out contiguously on the disk after the data block, are initialized as needed, +and are tracked via bitmaps stored in the secondary block. The number of secondary and data +blocks/pages in a chunk index varies as they are allocated as needed and the first few are +(conceptually) stored in parent elements as an optimization. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Layout: Extensible Array Header +
bytebytebytebyte
Signature
VersionClient IDElement SizeMax Nelmts Bits
Index Blk ElmtsData Blk Min ElmtsSecondary Blk Min Data PtrsMax Data Blk Page Nelmts Bits

Num Secondary BlksL


Secondary Blk SizeL


Num Data BlksL


Data Blk SizeL


Max Index SetL


Num ElementsL


Index Block AddressO

Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. +\li Items marked with an ‘L’ in the above table are of the size specified in + “@ref FMT3SizeOfLengthsV0 "Size of Lengths"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Extensible Array Header
Field NameDescription
SignatureThe ASCII character string “EAHD” is used to indicate the beginning + of an Extensible Array header. This gives file consistency checking utilities a better chance + of reconstructing a damaged file.
VersionThis document describes version 0.
Client IDThe ID for identifying the client of the Fixed Array: + + + + + + + + + + + + + + + + + +
IDDescription
0Non-filtered dataset chunks
1Filtered dataset chunks
2+Reserved.
Element SizeThe size in bytes of an element in the Extensible Array.
Max Nelmts BitsThe number of bits needed to store the maximum number of elements in the Extensible Array.
Index Blk ElmtsThe number of elements to store in the index block.
Data Blk Min ElmtsThe minimum number of elements per data block.
Secondary Blk Min Data PtrsThe minimum number of data block pointers for a secondary block.
Max Dblk Page Nelmts BitsThe number of bits needed to store the maximum number of elements in a data block page.
Num Secondary BlksThe number of secondary blocks created.
Secondary Blk SizeThe size of the secondary blocks created.
Num Data BlksThe number of data blocks created.
Data Blk SizeThe size of the data blocks created.
Max Index SetThe maximum index set.
Num ElmtsThe number of elements realized.
Index Block AddressThe address of the index block.
ChecksumThe checksum for the header.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Extensible Array Index Block
bytebytebytebyte
Signature
VersionClient IDThis space inserted only to align table nicely

Header AddressO


Elements (variable size and optional)


Data Block Addresses (variable size and optional)


Secondary Block Addresses (variable size and optional)

Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Extensible Array Index Block
Field NameDescription
SignatureThe ASCII character string “EAIB” is used to indicate the beginning + of an Extensible Array Index Block. This gives file consistency checking utilities a better + chance of reconstructing a damaged file.
VersionThis document describes version 0.
Client IDThe client ID for identifying the user of the Extensible Array: + + + + + + + + + + + + + + + + + +
IDDescription
0Non-filtered dataset chunks
1Filtered dataset chunks
2+Reserved.
Header AddressThe address of the Extensible Array header. Principally used for file integrity checking.
ElementsContains the elements that are stored directly in the index block. An optimization to avoid unnecessary + secondary blocks.
There are two element types: + + + + + + + + + + + + + +
IDDescription
0@ref FMT3EaNonFilterChunk "Non-filtered dataset chunks"
1@ref FMT3EaFilterChunk "Filtered dataset chunks"
Data Block AddressesContains the addresses of the data blocks that are stored directly in the Index Block. An + optimization to avoid unnecessary secondary blocks.
Secondary Block AddressesContains the addresses of the secondary blocks.
ChecksumThe checksum for the Extensible Array Index Block.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Extensible Array Secondary Block
bytebytebytebyte
Signature
VersionClient IDThis space inserted only to align table nicely

Header AddressO


Block Offset (variable size)


Page Bitmap (variable size and optional)


Data Block Addresses (variable size and optional)

Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Extensible Array Secondary Block
Field NameDescription
SignatureThe ASCII character string “EASB” is used to indicate the beginning + of an Extensible Array Secondary Block. This gives file consistency checking utilities + a better chance of reconstructing a damaged file.
VersionThis document describes version 0.
Client IDThe ID for identifying the client of the Extensible Array: + + + + + + + + + + + + + + + + + +
IDDescription
0Non-filtered dataset chunks
1Filtered dataset chunks
2+Reserved.
Header AddressThe address of the Extensible Array header. Principally used for file integrity checking.
Block OffsetStores the offset of the block in the array.
Page BitmapA bitmap indicating which data block pages are initialized.
+ Exists only if the data block is paged.
Data Block AddressesContains the addresses of the data blocks referenced by this secondary block.
ChecksumThe checksum for the Extensible Array Secondary Block.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Extensible Array Data Block
bytebytebytebyte
Signature
VersionClient IDThis space inserted only to align table nicely

Header AddressO


Block Offset (variable size)


Elements (variable size and optional)

Checksum
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Extensible Array Data Block
Field NameDescription
SignatureThe ASCII character string “EADB” is used to indicate the beginning + of an Extensible Array data block. This gives file consistency checking utilities a better + chance of reconstructing a damaged file.
VersionThis document describes version 0.
Client IDThe ID for identifying the client of the Extensible Array: + + + + + + + + + + + + + + + + + +
IDDescription
0Non-filtered dataset chunks
1Filtered dataset chunks
2+Reserved.
Header AddressThe address of the Extensible Array header. Principally used for file integrity checking.
Block OffsetThe offset of the block in the array.
ElementsContains the elements stored in the data block and exists only if the data block is not paged. +
There are two element types: + + + + + + + + + + + + + +
IDDescription
0@ref FMT3EaNonFilterChunk "Non-filtered dataset chunks"
1@ref FMT3EaFilterChunk "Filtered dataset chunks"
ChecksumThe checksum for the Extensible Array data block.
+ + + + + + + + + + + + + + + +
Layout: Extensible Array Data Block Page
bytebytebytebyte

Elements (variable size)

Checksum
+
+ + + + + + + + + + + + + + +
Fields: Extensible Array Data Block Page
Field NameDescription
ElementsContains the elements stored in the data block page.
+ There are two element types: + + + + + + + + + + + + + +
IDDescription
0@ref FMT3EaNonFilterChunk "Non-filtered dataset chunks"
1@ref FMT3EaFilterChunk "Filtered dataset chunks"
ChecksumThe checksum for an Extensible Array data block page.
+ +\anchor FMT3EaNonFilterChunk + + + + + + + + + + + +
Layout: Data Block Element for Non-filtered Dataset Chunk
bytebytebytebyte

AddressO

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + +
Fields: Data Block Element for Non-filtered Dataset Chunk
Field NameDescription
AddressThe address of the dataset chunk in the file.
+ +\anchor FMT3EaFilterChunk + + + + + + + + + + + + + + + + + +
Layout: Data Block Element for Filtered Dataset Chunk
bytebytebytebyte

AddressO


Chunk Size (variable size; at most 8 bytes)

Filter Mask
+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + + + + + +
Fields: Data Block Element for Filtered Dataset Chunk
Field NameDescription
AddressThe address of the dataset chunk in the file.
Chunk SizeThe size of the dataset chunk in bytes.
Filter MaskIndicates the filter to skip for the dataset chunk. + Each filter has an index number in the pipeline; if that + filter is skipped, the bit corresponding to its index is set.
+ +\section subsec_fmt3_appendixc_appv2btree VII.E. The Version 2 B-trees Index +The Version 2 B-trees index can be used when the dataset fulfills the following condition: +
    +
  • more than one dimension of unlimited extent
    • +
+ +Version 2 B-trees can be used to index various objects in the library. See +@ref subsubsec_fmt3_infra_btrees_v2 for more information. The B-tree types +@ref FMT3V2BtType10 "10" and @ref FMT3V2BtType11 "11" record layouts are for +indexing dataset chunks. + +\section sec_fmt3_appendixd VIII. Appendix D: Encoding for Dataspace and Reference + +\section subsec_fmt3_appendixd_encode VIII.A. Dataspace Encoding +#H5Sencode is a public routine that encodes a dataspace description into a buffer while +#H5Sdecode is the corresponding routine that decodes the description encoded in the buffer. +See the reference manual description for these two public routines. + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Dataspace Description for #H5Sencode/#H5Sdecode
bytebytebytebyte
Dataspace IDEncode VersionSize of SizeThis space inserted only to align table nicely

Size of Extent



Dataspace Message (variable size)



Dataspace Selection (variable size)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Dataspace Description for #H5Sencode/#H5Sdecode
Field NameDescription
Dataspace IDThe datspace message ID which is 1.
Encode VersionH5S_ENCODE_VERSION which is 0.
Size of SizeThe number of bytes used to store the size of an object.
Size of ExtentSize of the dataspace message.
Dataspace MessageThe dataspace message information. See @ref subsubsec_fmt3_dataobject_hdr_msg_simple
Dataspace SelectionThe dataspace selection information. See @ref FMT3DataspaceSEL"Dataspace Selection".
+ +\anchor FMT3DataspaceSEL + + + + + + + + + + + + + + +
Layout: Dataspace Selection
bytebytebytebyte
Selection Type

Selection Info (variable size)

+
+ + + + + + + + + + + + + + +
Fields: Dataspace Selection
Field NameDescription
Selection TypeThere are 4 types of selection: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0#H5S_SEL_NONE: Nothing selected
1#H5S_SEL_POINTS: Sequence of points selected
2#H5S_SEL_HYPERSLABS: Hyperslab selected
3#H5S_SEL_ALL: Entire extent selected
Selection InfoThere are 4 types of selection info: + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
0Selection info for #H5S_SEL_NONE Layout and Fields @ref FMT3SelNONE "tables"
1Selection info for #H5S_SEL_POINTS Layout and Fields @ref FMT3SelPOINTS "tables"
2Selection info for #H5S_SEL_HYPERSLABS Layout and Fields @ref FMT3SelHYPER "tables"
3Selection for #H5S_SEL_ALL Layout and Fields @ref FMT3SelALL "tables"
+ +\anchor FMT3SelNONE + + + + + + + + + + + + + + +
Layout: Selection Info for #H5S_SEL_NONE
bytebytebytebyte
Version

Reserved (zero, 8 bytes)

+
+ + + + + + + + + + +
Fields: Selection Info for #H5S_SEL_NONE
Field NameDescription
VersionThe version number for the #H5S_SEL_NONE Selection Info. The value is 1.
+ +\anchor FMT3SelPOINTS + + + + + + + + + + + + + + +
Layout: Selection Info for #H5S_SEL_POINTS
bytebytebytebyte
Version


Points Selection Info (variable size)

+ + + + + + + + + + + + + + + +
Fields: Selection Info for #H5S_SEL_POINTS
Field NameDescription
VersionThe version number for the #H5S_SEL_POINTS Selection Info. The value is either 1 or 2.
Points Selection InfoDepending on version: + + + + + + + + + + + + + +
VersionDescription
1See @ref FMT3SelPOINTSV1 "Version 1 Points Selection Info"
2See @ref FMT3SelPOINTSV2 "Version 2 Points Selection Info"
+ +\anchor FMT3SelPOINTSV1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 1 Points Selection Info
bytebytebytebyte
Reserved (zero)
Length
Rank
Num Points
Point \#1: coordinate \#1
.
.
.
Point \#1: coordinate \#u
.
.
.
Point \#n: coordinate \#1
.
.
.
Point \#n: coordinate \#u
+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 1 Points Selection Info
Field NameDescription
LengthThe size in bytes from Length to the end of the selection info.
RankThe number of dimensions.
Num PointsThe number of points in the selection.
Point \#n: coordinate \#uThe array of points in the selection. The points selected are \#1 to \#n where n is + Num Points. The list of coordinates for each point are \#1 to \#u where u is + Rank.
+ +\anchor FMT3SelPOINTSV2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 2 Points Selection Info
bytebytebytebyte
Encode SizeThis space inserted only to align table nicely
Rank
Num Points(2, 4 or 8 bytes)
Point \#1: coordinate \#1(2, 4 or 8 bytes)
.
.
.
Point \#1: coordinate \#u(2, 4 or 8 bytes)
.
.
.
Point \#n: coordinate \#1 (2, 4 or 8 bytes)
.
.
.
Point \#n: coordinate \#u(2, 4 or 8 bytes)
+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 Points Selection Info
Field NameDescription
Encode SizeThe size for encoding the points selection info which can be 2, 4 or 8 bytes.
RankThe number of dimensions.
Num PointsThe number of points in the selection. The field Encode Size indicates the size + of this field
Point \#n: coordinate \#uThe array of points in the selection. The points selected are \#1 to \#n where n is + Num Points. The list of coordinates for each point are \#1 to \#u where u is + Rank. The field Encode Size indicates the size of this field
+ +\anchor FMT3SelHYPER + + + + + + + + + + + + + + +
Layout: Selection Info for #H5S_SEL_HYPERSLABS
bytebytebytebyte
Version

Hyperslab Selection Info (variable size)

+ + + + + + + + + + + + + + + +
Fields: Selection Info for #H5S_SEL_HYPERSLABS
Field NameDescription
VersionThe version number for the #H5S_SEL_HYPERSLABS selection info. The value is 1, 2 or 3.
Hyperslab Selection InfoDepending on version: + + + + + + + + + + + + + + + + + +
VersionDescription
1See @ref FMT3SelHYPERV1 "Version 1 Hyperslab Selection Info".
2See @ref FMT3SelHYPERV2 "Version 2 Hyperslab Selection Info"
3See @ref FMT3SelHYPERV3 "Version 3 Hyperslab Selection Info"
+ +\anchor FMT3SelHYPERV1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 1 Hyperslab Selection Info
bytebytebytebyte
Reserved
Length
Rank
Num Blocks
Starting Offset \#1 for Block \#1
.
.
.
Starting Offset \#n for Block \#1
Ending Offset \#1 for Block \#1
.
.
.
Ending Offset \#n for Block \#1
.
.
.
.
.
.
.
.
.
Starting Offset \#1 for Block \#u
.
.
.
Starting Offset \#n for Block \#u
Ending Offset \#1 for Block \#u
.
.
.
Ending Offset \#n for Block \#u
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 1 Hyperslab Selection Info
Field NameDescription
LengthThe size in bytes from the field Rank to the end of the Selection Info.
RankThe number of dimensions in the dataspace.
Num BlocksThe number of blocks in the selection.
Starting Offset \#n for Block \#uThe offset \#n of the starting element in block \#u. \#n is from 1 to Rank. + \#u is from 1 to Num Blocks moving from the fastest changing dimension to + the slowest changing dimension.
Ending Offset \#n for Block \#uThe offset \#n of the ending element in block \#u. \#n is from 1 to Rank. + \#u is from 1 to Num Blocks moving from the fastest changing dimension to + the slowest changing dimension.
+ +\anchor FMT3SelHYPERV2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 2 Hyperslab Selection Info
bytebytebytebyte
FlagsThis space inserted only to align table nicely
Length
Rank
Start \#1 (8 bytes)
Stride \#1 (8 bytes)
Count \#1 (8 bytes)
Block \#1 (8 bytes)
.
.
.
Start \#n (8 bytes)
Stride \#n (8 bytes)
Count \#n (8 bytes)
Block \#n (8 bytes)
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 2 Hyperslab Selection Info
Field NameDescription
FlagsThis is a bit field with the following definition. Currently, this is always set to 0x1. + + + + + + + + + +
BitDescription
0If set, it is a regular hyperslab, otherwise, irregular.
LengthThe size in bytes from the field Rank to the end of the Selection Info.
RankThe number of dimensions in the dataspace.
Start \#nThe offset of the starting element in the block. \#n is from 1 to Rank.
Stride \#nThe number of elements to move in each dimension. \#n is from 1 to Rank.
Count \#nThe number of blocks to select in each dimension. \#n is from 1 to Rank.
Block \#nThe size (in elements) of each block in each dimension. \#n is from 1 to Rank.
+ +\anchor FMT3SelHYPERV3 + + + + + + + + + + + + + + + + + + + +
Layout: Version 3 Hyperslab Selection Info
bytebytebytebyte
FlagsEncode SizeThis space inserted only to align table nicely
Rank

Regular/Irregular Hyperslab Selection Info (variable size)

+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 3 Hyperslab Selection Info
Field NameDescription
FlagsThis is a bit field with the following definition: + + + + + + + + + +
BitDescription
0If set, it is a regular hyperslab, otherwise, irregular.
Encode SizeThe size for encoding hyperslab selection info, which can 2, 4 or 8 bytes.
RankThe number of dimensions in the dataspace.
Regular/Irregular Hyperslab Selection InfoThis is the selection info for version 3 hyperslab which can be regular or irregular. + If bit 0 of the field Flags is set, see + @ref FMT3SelHYPERV3REG "Version 3 Regular Hyperslab Selection Info" + Otherwise, see @ref FMT3SelHYPERV3IRREG "Version 3 Irregular Hyperslab Selection Info"
+ +\anchor FMT3SelHYPERV3REG + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 3 Regular Hyperslab Selection Info
bytebytebytebyte
Start \#1 (2, 4 or 8 bytes)
Stride \#1 (2, 4 or 8 bytes)
Count \#1 (2, 4 or 8 bytes)
Block \#1 (2, 4 or 8 bytes)
.
.
.
Start \#n (2, 4 or 8 bytes)
Stride \#n (2, 4 or 8 bytes)
Count \#n (2, 4 or 8 bytes)
Block \#n (2, 4 or 8 bytes)
+ + + + + + + + + + + + + + + + + + + + + + + +
Fields: Version 3 Regular Hyperslab Selection Info
Field NameDescription
Start \#nThe offset of the starting element in the block. \#n is from 1 to Rank. + The field Encode Size indicates the size of this field.
Stride \#nThe number of elements to move in each dimension. \#n is from 1 to Rank. + The field Encode Size indicates the size of this field.
Count \#nThe number of blocks to select in each dimension. \#n is from 1 to Rank. + The field Encode Size indicates the size of this field.
Block \#nThe size (in elements) of each block in each dimension. \#n is from 1 to Rank. + The field Encode Size indicates the size of this field.
+ +\anchor FMT3SelHYPERV3IRREG + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Version 3 Irregular Hyperslab Selection Info
bytebytebytebyte
Num Blocks (2, 4 or 8 bytes)
Starting Offset \#1 for Block \#1 (2, 4 or 8 bytes)
.
.
.
Starting Offset \#n for Block \#1 (2, 4 or 8 bytes)
Ending Offset \#1 for Block \#1 (2, 4 or 8 bytes)
.
.
.
Ending Offset \#n for Block \#1 (2, 4 or 8 bytes)
.
.
.
.
.
.
.
.
.
Starting Offset \#1 for Block \#u (2, 4 or 8 bytes)
.
.
.
Starting Offset \#n for Block \#u (2, 4 or 8 bytes)
Ending Offset \#1 for Block \#u (2, 4 or 8 bytes)
.
.
.
Ending Offset \#n for Block \#u (2, 4 or 8 bytes)
+ + + + + + + + + + + + + + + +
Fields: Version 3 Irregular Hyperslab Selection Info
Num BlocksThe number of blocks in the selection. The field Encode Size indicates the size of + this field
Starting Offset \#n for Block \#uThe offset \#n of the starting element in block \#u. \#n is from 1 to Rank. + \#u is from 1 to Num Blocks moving from the fastest changing dimension to the slowest + changing dimension. The field Encode Size indicates the size of this field
Ending Offset \#n for Block \#uThe offset \#n of the ending element in block \#u. \#n is from 1 to Rank. \#u is from + 1 to Num Blocks moving from the fastest changing dimension to the slowest changing + dimension. The field Encode Size indicates the size of this field
+ +\anchor FMT3SelALL + + + + + + + + + + + + + + +
Layout: Selection Info for #H5S_SEL_ALL
bytebytebytebyte
Version

Reserved (zero, 8 bytes)

+
+ + + + + + + + + + +
Fields: Selection Info for #H5S_SEL_ALL
Field NameDescription
VersionThe version number for the #H5S_SEL_ALL Selection Info; the value is 1.
+ +\section subsec_fmt3_appendixd_encoderv VIII.B. Reference Encoding (Revised) +For the following reference type, the Reference Header and Reference Block are stored together +as the dataset's raw data: +
    +
  • Object Reference (#H5R_OBJECT2) (without reference to an external file)
    • +
+ +For the following reference types, the Reference Header plus the @ref FMT3GlobalHeapID "Global Heap ID" +are stored as the dataset's raw data in the file. The global heap ID is used to locate the +Reference Block stored in the global heap: +
    +
  • Object Reference (#H5R_OBJECT2) (with reference to an external file)
    • +
  • Dataset Region Reference (#H5R_DATASET_REGION2) (with/without reference to an external file)
    • +
  • Attribute Reference (#H5R_ATTR) (with/without reference to an external file)
    • +
+ + + + + + + + + + + + + + +
Layout: Reference Header
bytebytebytebyte
Reference TypeFlagsThis space inserted only to align table nicely
+ + + + + + + + + + + + + + + +
Fields: Reference Header
Field NameDescription
Reference TypeThere are 3 types of references: + + + + + + + + + + + + + + + + + +
ValueDescription
2#H5R_OBJECT2: Object Reference
3#H5R_DATASET_REGION2: Dataset Region Reference
4#H5R_ATTR: Attribute Reference
FlagsThis field describes the reference: + + + + + + + + + + + + + +
BitDescription
0If set, the reference is to an external file.
1-7Reserved
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Layout: Reference Block
bytebytebytebyte
Token SizeThis space inserted only to align table nicely

Token (variable size)

Length of External File NameThis space inserted only to align table nicely

External File Name (variable size)

Size of Dataspace Selection
Rank of Dataspace Selection

Dataspace Selection Information (variable size)

Length of Attribute Name This space inserted only to align table nicely

Attribute Name (variable size)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Fields: Reference Block
Field NameDescription
Token sizeThis is the size of the token for the object.
TokenThis is the token for the object.
Length of External File NameThis is the length for the external file name.
+ This field exists if bit 0 of flags is set.
External File NameThis is the name of the external file being referenced.
+ This field exists if bit 0 of flags is set. +
Dataspace Selection InformationSee @ref FMT3DataspaceSEL "Dataspace Selection".
+ This field exists if the Reference Type is #H5R_DATASET_REGION2.
Length of Attribute NameThis is the length of the attribute name.
+ This field exists if the Reference Type is #H5R_ATTR.
Attribute NameThis is the name of the attribute being referenced.
+ This field exists if the Reference Type is #H5R_ATTR.
+ +\section subsec_fmt3_appendixd_encodedp VIII.C. Reference Encoding (Backward Compatibility) +The two references described below are maintained to preserve compatibility with previous versions +of the library.
+For the following reference type, the reference encoding is stored as the dataset's raw data in the file: +
    +
  • Object Reference (#H5R_OBJECT1)
    • +
+ +For the following reference type, the @ref FMT3GlobalHeapID "Global Heap ID" is stored as the dataset's +raw data in the file. The global heap ID is used to locate the reference encoding stored in the global heap: +
    +
  • Dataset Region Reference (#H5R_DATASET_REGION1)
    • +
+ + + + + + + + + + + + +
Layout: Reference for #H5R_OBJECT1
bytebytebytebyte

Object AddressO

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + +
Fields: Reference for #H5R_OBJECT1
Field NameDescription
Object AddressAddress of the object being referenced
+
+ + + + + + + + + + + + + + +
Layout: Reference for #H5R_DATASET_REGION1
bytebytebytebyte

Object AddressO


Dataspace Selection Information (variable size)

+\li Items marked with an ‘O’ in the above table are of the size specified in + “@ref FMT3SizeOfOffsetsV0 "Size of Offsets"” field in the superblock. + + + + + + + + + + + + + + + +
Fields: Reference for #H5R_DATASET_REGION1
Field NameDescription
Object AddressThis is the address of the object being referenced.
Dataspace Selection InformationThis is the dataspace selection for the object being referenced. See + @ref FMT3DataspaceSEL "Dataspace Selection".
+ +*/ diff --git a/doxygen/examples/IOFlow.html b/doxygen/dox/IOFlow.dox similarity index 69% rename from doxygen/examples/IOFlow.html rename to doxygen/dox/IOFlow.dox index b33196d502a..0a2771005fc 100644 --- a/doxygen/examples/IOFlow.html +++ b/doxygen/dox/IOFlow.dox @@ -1,55 +1,20 @@ - - - HDF5 Raw I/O Flow Notes - - - - - - - - -

HDF5 Raw I/O Flow Notes

-

Quincey Koziol
- koziol@ncsa.uiuc.edu
- August 20, 2003 -

- -
    - -

  1. Document's Audience:

    - -
      -
    • Current H5 library designers and knowledgeable external developers.
      • -
    - -

  2. Background Reading:

    - -

  3. Introduction:

    - -
    -
    What is this document about?
    -
    This document attempts to supplement the flow charts describing - the flow of control for raw data I/O in the library. -

    -
    - -

  4. Figures:

    -

    The following figures provide the main information:

    - - - - -
    High-Level View of Writing Raw Data
    Perform Serial or Parallel I/O
    Gather/Convert/Scatter
    - -

  5. Notes From Accompanying Figures:

    - -

    This section provides notes to augment the information in the accompanying - figures. -

    +/** \page IOFLOW HDF5 Raw I/O Flow Notes +\li Created by Quincey Koziol, August 20, 2003 +\li Document's Audience: Current H5 library designers and knowledgeable external developers. + +\section sec_ioflow_intro Introduction +What is this document about?
    +This document attempts to supplement the flow charts describing the flow of control for raw data +I/O in the library. +The following figures provide the main information: + + + + +
    High-Level View of Writing Raw Data
    Perform Serial or Parallel I/O
    Gather/Convert/Scatter
    + +\section sec_ioflow_notes Notes From Accompanying Figures +This section provides notes to augment the information in the accompanying figures.
    1. Validate Parameters - Resolve any H5S_ALL parameters @@ -129,8 +94,5 @@
    -
- - +*/ - diff --git a/doxygen/dox/ImageSpec.dox b/doxygen/dox/ImageSpec.dox new file mode 100644 index 00000000000..18edb62b355 --- /dev/null +++ b/doxygen/dox/ImageSpec.dox @@ -0,0 +1,710 @@ + +/** \page IMG HDF5 Image and Palette Specification Version 1.2 + + +The HDF5 specification defines the standard objects and storage for the +standard HDF5 objects. (For information about the HDF5 library, model and +specification, see the HDF documentation.) This document is an additional +specification to define a standard profile for how to store image data +in HDF5. Image data in HDF5 is stored as HDF5 datasets with standard attributes +to define the properties of the image. + +This specification is primarily concerned with two dimensional raster +data similar to HDF4 Raster Images. Specifications for storing other +types of imagery will be covered in other documents. +This specification defines: +/li Standard storage and attributes for an Image dataset (\ref subsec_image_spec_spec_over) +/li Standard storage and attributes for Palettes (\ref subsec_image_spec_spec_attr) +/li Standard for associating Palettes with Images. (\ref sec_tab_spec_sect3) + +\section sec_image_spec_spec HDF5 Image Specification + +\subsection subsec_image_spec_spec_over Overview +Image data is stored as an HDF5 dataset with values of HDF5 class Integer +or Float. A common example would be a two dimensional dataset, with +elements of class Integer, e.g., a two dimensional array of unsigned 8 +bit integers. However, this specification does not limit the dimensions +or number type that may be used for an Image. + +The dataset for an image is distinguished from other datasets by giving +it an attribute "CLASS=IMAGE". In addition, the Image dataset may +have an optional attribute "PALETTE" that is an array of object references +for zero or more palettes. The Image dataset may have additional attributes +to describe the image data, as defined in \ref subsec_image_spec_spec_attr. + +A Palette is an HDF5 dataset which contains color map information. +A Palette dataset has an attribute "CLASS=PALETTE" and other attributes +indicating the type and size of the palette, as defined in \ref sec_tab_spec_sect2. +A Palette is an independent object, which can be shared +among several Image datasets. + +\subsection subsec_image_spec_spec_attr Image Attributes +The attributes for the Image are scalars unless otherwise noted. +The length of String valued attributes should be at least the number of +characters. Optionally, String valued attributes may be stored in a String +longer than the minimum, in which case it must be zero terminated or null +padded. "Required" attributes must always be used. "Optional" attributes +must be used when required. + +\subsection subsec_image_spec_spec_attr_attr Attributes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1. Attributes of an Image Dataset
Attribute NameRequired or OptionalTypeString SizeValueDescription
CLASSRequiredString5"IMAGE"This attribute is type #H5T_C_S1, with size 5. For all Images, the value of +this attribute is "IMAGE". This attribute identifies this data set as intended +to be interpreted as an image that conforms to the specifications on this page.
PALETTEOptionalArray Object References<references to Palette datasets>1A Image dataset within an HDF5 file may optionally specify an array of +palettes to be viewed with. The dataset will have an attribute field called +"PALETTE" which contains a one-dimensional array of object reference +pointers (HDF5 datatype #H5T_STD_REF_OBJ) which refer to palettes in the +file. The palette datasets must conform to the Palette specification in +\ref sec_tab_spec_sect2. The first palette in this array will be the default palette +that the data may be viewed with.
IMAGE_SUBCLASSOptional2String15,
12,
15,
13		"IMAGE_GRAYSCALE",
"IMAGE_BITMAP",
"IMAGE_TRUECOLOR",
"IMAGE_INDEXED"		If present, the value of this attribute indicates the type of Palette that +should be used with the Image. This attribute is a scalar of type +#H5T_C_S1, with size according to the string plus one. The values +are:
+
  • IMAGE_GRAYSCALE (length 15) A grayscale image
    • +
  • IMAGE_BITMAP (length 12) A bit map image
    • +
  • IMAGE_TRUECOLOR (length 15) A truecolor image
    • +
  • IMAGE_INDEXED (length 13) An indexed image
INTERLACE_MODEOptional3,6String15The layout of components if more than one component per pixel.For images with more than one component for each pixel, this optional attribute +specifies the layout of the data. The values are type #H5T_C_S1 of length +15. See \ref subsec_image_spec_spec_store for information about the +storage layout for data.
+
  • INTERLACE_PIXEL (default): the component values for a pixel are contiguous.
    • +
  • INTERLACE_PLANE: each component is stored as a plane.
DISPLAY_ORIGINOptionalString2If set, indicates the intended location of the pixel (0,0).This optional attribute indicates the intended orientation of the data +on a two-dimensional raster display. The value indicates which corner +the pixel at (0, 0) should be viewed. The values are type #H5T_C_S1 +of length 2. If DISPLAY_ORIGIN is not set, the orientation is undefined.
+
  • UL: (0,0) is at the upper left.
    • +
  • LL: (0,0) is at the lower left.
    • +
  • UR: (0,0) is at the upper right.
    • +
  • LR: (0,0) is at the lower right.
IMAGE_WHITE_IS_ZEROOptional3,4Unsigned Integer0 = false, 1 = trueThis attribute is of type #H5T_NATIVE_UCHAR. 0 = false, 1 = true . +This is used for images with IMAGE_SUBCLASS="IMAGE_GRAYSCALE" or "IMAGE_BITMAP".
IMAGE_MINMAXRANGEOptional3,5Array [2] <same datatype as data values>The (<minimum>, <maximum>) value of the data.If present, this attribute is an array of two numbers, of the same HDF5 +datatype as the data. The first element is the minimum value of the +data, and the second is the maximum. This is used for images with +IMAGE_SUBCLASS="IMAGE_GRAYSCALE", "IMAGE_BITMAP" or "IMAGE_INDEXED".
IMAGE_BACKGROUNDINDEXOptional3Unsigned IntegerThe index of the background color.If set, this attribute indicates the index value that should be interpreted +as the "background color". This attribute is HDF5 type #H5T_NATIVE_UINT.
IMAGE_TRANSPARENCYOptional3,5Unsigned IntegerThe index of the transparent color.If set, this attribute indicates the index value that should be interpreted +as the "transparent color". This attribute is HDF5 type #H5T_NATIVE_UINT. +This attribute may not be used for IMAGE_SUBCLASS="IMAGE_TRUECOLOR".
IMAGE_ASPECTRATIOOptional3,4Unsigned IntegerThe aspect ratio.If set, this attribute indicates the aspect ratio.
IMAGE_COLORMODELOptional3,6String3, 4, or 5The color model, as defined below in the Palette specification for +attribute PAL_COLORMODEL.If set, this attribute indicates the color model of Palette that should +be used with the Image. This attribute is of type #H5T_C_S1, with +size 3, 4, or 5. The value is one of the color models described in +the Palette specification in \ref subsec_tab_spec_sect2_22. +This attribute may be used only for IMAGE_SUBCLASS="IMAGE_TRUECOLOR" or +"IMAGE_INDEXED".
IMAGE_GAMMACORRECTIONOptional3,6FloatThe gamma correction.If set, this attribute gives the Gamma correction. The attribute +is type #H5T_NATIVE_FLOAT. This attribute may be used only for IMAGE_SUBCLASS="IMAGE_TRUECOLOR" +or "IMAGE_INDEXED".
IMAGE_VERSIONRequiredString3"1.2"This attribute is of type #H5T_C_S1, with size corresponding to the length +of the version string. This attribute identifies the version number +of this specification to which it conforms. The current version number +is "1.2".
+ +

Notes

+\li 1. The first element of the array is the defaultPalette. +\li 2. This attribute is required for images +that use one of the standard color map types listed. +\li 3. This attribute is required if set for the source +image, in the case that the image is translated from another file into +HDF5. +\li 4. This applies to: IMAGE_SUBCLASS="IMAGE_GRAYSCALE" or "IMAGE_BITMAP". +\li 5. This applies to: IMAGE_SUBCLASS="IMAGE_GRAYSCALE", "IMAGE_BITMAP", or "IMAGE_INDEXED". +\li 6. This applies to: IMAGE_SUBCLASS="IMAGE_TRUECOLOR", or "IMAGE_INDEXED". + +Table 2 summarizes the standard attributes for an Image dataset(s) using +the common sub-classes. Required means that the attribute listed on the leftmost +column is required for the image subclass on the first row, Optional means that +the attribute is optional for that subclass and NA that the attribute cannot +be applied to that subclass. The two first rows show the only required +attributes for all subclasses. +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 2a. Applicability of Attributes to IMAGE sub-classes
IMAGE_SUBCLASS1IMAGE_GRAYSCALEIMAGE_BITMAP
CLASSRequiredRequired
IMAGE_VERSIONRequiredRequired
INTERLACE_MODENANA
IMAGE_WHITE_IS_ZERORequiredRequired
IMAGE_MINMAXRANGEOptionalOptional
IMAGE_BACKGROUNDINDEXOptionalOptional
IMAGE_TRANSPARENCYOptionalOptional
IMAGE_ASPECTRATIOOptionalOptional
IMAGE_COLORMODELNANA
IMAGE_GAMMACORRECTIONNANA
PALETTEOptionalOptional
DISPLAY_ORIGINOptionalOptional
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 2b. Applicability of Attributes to IMAGE sub-classes
IMAGE_SUBCLASSIMAGE_TRUECOLORIMAGE_INDEXED +
CLASSRequiredRequired
IMAGE_VERSIONRequiredRequired
INTERLACE_MODERequiredNA
IMAGE_WHITE_IS_ZERONANA
IMAGE_MINMAXRANGENAOptional
IMAGE_BACKGROUNDINDEXNAOptional
IMAGE_TRANSPARENCYNAOptional
IMAGE_ASPECTRATIOOptionalOptional
IMAGE_COLORMODELOptionalOptional
IMAGE_GAMMACORRECTIONOptionalOptional
PALETTEOptionalOptional
DISPLAY_ORIGINOptionalOptional
+ +\subsection subsec_image_spec_spec_store Storage Layout and Properties for Images +In the case of an image with more than one component per pixel (e.g., Red, +Green, and Blue), the data may be arranged in one of two ways. Following +HDF4 terminology, the data may be interlaced by pixel or by plane, which +should be indicated by the INTERLACE_MODE attribute. In both +cases, the dataset will have a dataspace with three dimensions, height, +width, and components. The interlace modes specify different orders +for the dimensions. + + + + + + + + + + + + + + +
Table 3. Storage of multiple component image data.
Interlace ModeDimensions in the Dataspace
INTERLACE_PIXEL[height][width][pixel components]
INTERLACE_PLANE[pixel components][height][width]
+ +For example, consider a 5 (rows) by 10 (column) image, with Red, Green, +and Blue components. Each component is an unsigned byte. In HDF5, +the datatype would be declared as an unsigned 8 bit integer. For +pixel interlace, the dataspace would be a three dimensional array, with +dimensions: [10][5][3]. For plane interleave, the dataspace would +be three dimensions: [3][10][5]. + +In the case of images with only one component, the dataspace may be +either a two dimensional array, or a three dimensional array with the third +dimension of size 1. For example, a 5 by 10 image with 8 bit color +indexes would be an HDF5 dataset with type unsigned 8 bit integer. +The dataspace could be either a two dimensional array, with dimensions +[10][5], or three dimensions, with dimensions either [10][5][1] or [1][10][5]. +

Image datasets may be stored with any chunking or compression properties +supported by HDF5. + +A note concerning compatibility with HDF5 GR interface: An Image +dataset is stored as an HDF5 dataset. It is important to note that +the order of the dimensions is the same as for any other HDF5 dataset. +For a two dimensional image that is to be stored as a series of horizontal +scan lines, with the scan lines contiguous (i.e., the fastest changing +dimension is 'width'), the image will have a dataspace with dim[0] = +height and dim[1] = width. This is completely consistent +with all other HDF5 datasets. + +Users familiar with HDF4 should be cautioned that this is not the +same as HDF4, and specifically is not consistent with what the HDF4 +GR interface does. + +\section sec_tab_spec_sect2 HDF5 Palette Specification + +\subsection subsec_tab_spec_sect2_21 Overview +A palette is the means by which color is applied to an image and is also +referred to as a color lookup table. It is a table in which every row contains +the numerical representation of a particular color. In the example of an +8 bit standard RGB color model palette, this numerical representation of +a color is presented as a triplet specifying the intensity of red, green, +and blue components that make up each color. + + + + +
+\image html Palettes.fm.anc.gif +
+ +In this example, the color component numeric type is an 8 bit unsigned +integer. While this is most common and recommended for general use, other +component color numeric datatypes, such as a 16 bit unsigned integer , +may be used. This type is specified as the type attribute of the palette +dataset. + +The minimum and maximum values of the component color numeric are specified +as attribute of the palette dataset. See below (attribute PAL_MINMAXNUMERIC). +If these attributes do not exist, it is assumed that the range of values +will fill the space of the color numeric type. i.e. with an 8 bit unsigned +integer, the valid range would be 0 to 255 for each color component. + +The HDF5 palette specification additionally allows for color models +beyond RGB. YUV, HSV, CMY, CMYK, YCbCr color models are supported, and +may be specified as a color model attribute of the palette dataset. (see +\ref subsec_tab_spec_sect2_22 for details). + +In HDF 4 and earlier, palettes were limited to 256 colors. The HDF5 +palette specification allows for palettes of varying length. The length +is specified as the number of rows of the palette dataset. + + + + + + + +
Important Note: The specification of the Indexed +Palette will change substantially in the next version. The Palette +described here is deprecated and is not supported.
DEPRECATED
+In a standard palette, the color entries are indexed directly. HDF5 +supports the notion of a range index table. Such a table defines an ascending +ordered list of ranges that map dataset values to the palette. If a range +index table exists for the palette, the PAL_TYPE attribute will be set +to "RANGEINDEX", and the PAL_RANGEINDEX attribute will contain an object +reference to a range index table array. If not, the PAL_TYPE attribute +either does not exist, or will be set to "STANDARD". + +The range index table array consists of a one dimensional array with +the same length as the palette dataset - 1. Ideally, the range index would +be of the same type as the dataset it refers to, however this is not a +requirement. + +Example 2: A range index array of type floating point +
+\image html PaletteExample1.gif +
+The range index array attribute defines the "to" of the range. +Notice that the range index array attribute is one less entry in size than +the palette. The first entry of 0.1259, specifies that all values below +and up to 0.1259 inclusive, will map to the first palette entry. The second +entry signifies that all values greater than 0.1259 up to 0.3278 inclusive, +will map to the second palette entry, etc. All value greater than the last +range index array attribute (100000) map to the last entry in the palette.
+ +\subsection subsec_tab_spec_sect2_22 Palette Attributes +A palette exists in an HDF file as an independent data set with accompanying +attributes. The Palette attributes are scalars except where noted +otherwise. String values should have size the length of the string +value plus one. "Required" attributes must be used. "Optional" +attributes must be used when required. + +These attributes are defined as follows: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4. Attributes of a Palette Dataset
Attribute NameRequired or OptionalTypeString SizeValueDescription
CLASSRequiredString7"PALETTE"This attribute is of type H5T_C_S1, with size 7.For all palettes, the value of this attribute is "PALETTE". This attribute +identifies this palette data set as a palette that conforms to the specifications +on this page.
PAL_COLORMODELRequiredString3, 4, or 5Color Model: "RGB", "YUV", "CMY", "CMYK", "YCbCr", or "HSV"This attribute is of type H5T_C_S1, with size 3, 4, or 5. +Possible values for this are "RGB", "YUV", "CMY", "CMYK", "YCbCr", "HSV".
+This defines the color model that the entries in the palette data set represent. +
+
"RGB"
+
Each color index contains a triplet where the first value defines the + red component, second defines the green component, and the third the blue + component.
+ +
"CMY"
+
Each color index contains a triplet where the first value defines the + cyan component, second defines the magenta component, and the third the + yellow component.
+ +
"CMYK"
+
Each color index contains a quadruplet where the first value defines + the cyan component, second defines the magenta component, the third the + yellow component, and the forth the black component.
+ +
"YCbCr"
+
Class Y encoding model. Each color index contains a triplet where the + first value defines the luminance, second defines the Cb Chromonance, and + the third the Cr Chromonance.
+ +
"YUV"
+
Composite encoding color model. Each color index contains a triplet where + the first value defines the luminance component, second defines the + chromonance component, and the third the value component.
+ +
"HSV"
+
Each color index contains a triplet where the first value defines the + hue component, second defines the saturation component, and the third the + value component. The hue component defines the hue spectrum with a low + value representing magenta/red progressing to a high value which would + represent blue/magenta, passing through yellow, green, cyan. A low value + for the saturation component means less color saturation than a high value. + A low value for value will be darker than a high value.
+
PAL_TYPERequiredString9
or 10
"STANDARD8"
or "RANGEINDEX" (Deprecated)
This attribute is of type H5T_C_S1, with size 9 or 10. +The current supported values for this attribute are: "STANDARD8" or "RANGEINDEX".
+A PAL_TYPE of "STANDARD8" defines a palette dataset such that the first +entry defines index 0, the second entry defines index 1, etc. up until +the length of the palette - 1. This assumes an image dataset with direct +indexes into the palette.
+ +
Deprecated If the PAL_TYPE is set to "RANGEINDEX", there will +be an additional attribute with a name of PAL_RANGEINDEX, (See example 2 +for more details)

+ +
+
+
Attribute name="PAL_RANGEINDEX" (Deprecated)
+
The PAL_RANGEINDEX attribute contains an HDF object reference (HDF5 + datatype H5T_STD_REF_OBJ) pointer which specifies a range index array in + the file to be used for color lookups for the palette. (Only for + PAL_TYPE="RANGEINDEX")
+
+
Deprecated
RANGE_INDEX
Deprecated
Object Reference
Deprecated
<Object Reference to Dataset of range index values>
Deprecated
PAL_MINMAXNUMERICOptionalArray[2] of <same datatype as palette>The first value is the <Minimum value for color values>, the second +value is <Maximum value for color values>2If present, this attribute is an array of two numbers, of the same HDF5 +datatype as the palette elements or color numerics. +They specify the minimum and maximum values of the color numeric components. +For example, if the palette was an RGB of type Float, the color numeric +range for Red, Green, and Blue could be set to be between 0.0 and 1.0. +The intensity of the color guns would then be scaled accordingly to be +between this minimum and maximum attribute.
PAL_VERSIONRequiredString4"1.2"This attribute is of type H5T_C_S1, with size corresponding to the +length of the version string. This attribute identifies the version +number of this specification to which it conforms. The current version +is "1.2".
+ +

Notes

+\li 1. The RANGE_INDEX attribute is required if the + PAL_TYPE is "RANGEINDEX". Otherwise, the RANGE_INDEX attribute should + be omitted. (Range index is deprecated.) +\li 2. The minimum and maximum are optional. If not + set, the range is assumed to the maximum range of the number type. + If one of these attributes is set, then both should be set. The value + of the minimum must be less than or equal to the value of the maximum. + +Table 5 summarized the uses of the standard attributes for a palette dataset. +Required means that the attribute listed on the leftmost column is required for +the palette type on the first row, Optional means that the attribute is optional +for that type and NA that the attribute cannot be applied to that type. +The four first rows show the attributes that are always required +for the two palette types. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 5. Applicability of Attributes
PAL_TYPESTANDARD8RANGEINDEX
CLASSRequiredRequired
PAL_VERSIONRequiredRequired
PAL_COLORMODELRequiredRequired
RANGE_INDEXNARequired
PAL_MINMAXNUMERICOptionalOptional
+ +\subsection subsec_tab_spec_sect2_23 Storage Layout for Palettes +The values of the Palette are stored as a dataset. The datatype can +be any HDF5 atomic numeric type. The dataset will have dimensions +(nentries by ncomponents), where 'nentries' +is the number of colors (usually 256) and 'ncomponents' is the +number of values per color (3 for RGB, 4 for CMYK, etc.) + +\section sec_tab_spec_sect3 Consistency and Correlation of Image and Palette Attributes +The objects in this specification are an extension to the base HDF5 specification +and library. They are accessible with the standard HDF5 library, +but the semantics of the objects are not enforced by the base library. +For example, it is perfectly possible to add an attribute called IMAGE +to any dataset, or to include an object reference to any +HDF5 dataset in a PALETTE attribute. This would be a valid +HDF5 file, but not conformant to this specification. The rules defined +in this specification must be implemented with appropriate software, and +applications must use conforming software to assure correctness. + +The Image and Palette specifications include several redundant standard +attributes, such as the IMAGE_COLORMODEL and the PAL_COLORMODEL. +These attributes are informative not normative, in that it is acceptable +to attach a Palette to an Image dataset even if their attributes do not +match. Software is not required to enforce consistency, and files +may contain mismatched associations of Images and Palettes. In all +cases, it is up to applications to determine what kinds of images and color +models can be supported. + +For example, an Image that was created from a file with an "RGB" may +have a "YUV" Palette in its PALETTE attribute array. This +would be a legal HDF5 file and also conforms to this specification, although +it may or may not be correct for a given application.

+ +*/ diff --git a/doxygen/dox/Overview.dox b/doxygen/dox/Overview.dox index 6caffd147aa..bbdb6c6466c 100644 --- a/doxygen/dox/Overview.dox +++ b/doxygen/dox/Overview.dox @@ -21,7 +21,7 @@ documents cover a mix of tasks, concepts, and reference, to help a specific audience succeed. \par Offline reading - You can download it as an archive for offline reading. + You can download it as an archive for offline reading. \par ToDo List There is plenty of unfinished business. diff --git a/doxygen/dox/RelVersion.dox b/doxygen/dox/RelVersion.dox new file mode 100644 index 00000000000..aaace327b90 --- /dev/null +++ b/doxygen/dox/RelVersion.dox @@ -0,0 +1,263 @@ + +/** \page RELVERSION HDF5 Library Release Version Numbers + +\section sec_relver_intro Introduction +HDF5 software is updated on a regular basis. These updates, known +as releases, range in scope and size from small to large. Some updates +may only fix bugs, and some updates may require a change in the format +of the data file. The version numbers that are applied to updates give +information about the kinds of changes made in the updates. This Tech +Note describes what the version numbers mean. + +Note that this document describes release version numbers for the +HDF5 Library. For more information, see the +\ref sec_relver_share section at the end of this document. + +\section sec_relver_def Definitions +Each software release of the HDF5 Library is labeled with a version +number. The version number is a set of three integers written as HDF5-1.2.3, +HDF5 version 1.2 release 3, or HDF5 Release 1.2.3. The version number +might also include an additional number. A patch version might be labeled HDF5-1.2.3.1. +The '5' in "HDF5" is part of the product name and will not change during +the life of the project + +The key components in HDF5 Library version numbers are the major version +number, the minor version number, the release number, and an optional patch number. + +\subsection subsec_relver_def_first First Integer Definitions +The first integer in a version number is the major version +number. This integer increments when there is an extensive change +to the file format or library API. Such a change may require files to +be translated and will likely require applications to be modified. + +\subsection subsec_relver_def_second Second Integer Definitions +The second integer, 2 in the examples above, is the minor version +number. + +\subsubsection subsubsec_relver_def_second_one Pre-2.0 Versions +This number is incremented when there are new features that +require a change in the file format. For example, a change in file format +was required during the change from version 1.6 to version 1.8. Stable +released versions of the library are given even minor version +numbers such as 1.6 and 1.8 while odd minor version numbers such +as 1.7 and 1.9 are used on the trunk for major development. See the +section below for more information. + +\subsubsection subsubsec_relver_def_second_two Post-2.0 Versions +This number is incremented when there are new features that change the +APIs but do not require a change in the file format. For example, a new +functionality that adds extended arguments to the library might +require a change in the API but not in the file format. + +\subsection subsec_relver_def_third Third Integer Definitions +The third integer, 3 in the examples above, is the release +number. A change in this number indicates that the library has +been updated. The updates might include bug fixes, performance +improvements, and new features that do not require a file format +change. + +A version number might also include another number. A patch version might +be made to a released version to make available a feature or a bug +fix. In the figure below, a patch to the 1.8.5 release is labeled +1.8.5.1. A snapshot is an intermediate posting of the software +in a branch or in the trunk. Snapshots are made available so that users +may begin to test changes in the software that affect their software. +The changes may range from bug fixes to new features. Snapshots are made +and released regularly. How regularly depends on whether the software +passes the tests done on each build. The +snapshots are available at +https://github.com/HDFGroup/hdf5/releases/tag/snapshot. + +\section sec_relver_branch The Trunk, Release Branches, and Feature Branches +The HDF Group uses a version control system to manage the HDF5 +project. Within the system, a trunk and branches are used to track +changes. The version numbers described above identify where a given +piece of software was produced in the system. The figure below shows +the general scheme. + + + + +
+\image html trunk_branches.jpg "Figure 1. The trunk, release branches, and feature branches" +
+ +The trunk is the center of the system. New features are +implemented in feature branches and aggregated in the trunk. +Release branches are then created from the trunk. + +Before the 2.0 version, the minor version number of the trunk is always an odd number. From +the time of Release 1.8.0 to the first 1.10 release, the trunk was +version 1.9. The trunk was version 1.7 from the time of release 1.6.0 +until the first 1.8 release. +Since the 2.0 version, the minor version number of the trunk simply increments. + +Projects that add new features, bug fixes, and performance improvements +are developed on feature branches. When a project is completed, +its feature branch is merged into the trunk. In the figure above, the +merging of a feature branch is represented by a dashed arrow from the +feature branch to the trunk. If a feature requires a file format change, +then the feature will stay in the trunk until the next significant +release. This would mean in the figure above that the new feature would +be released in a future 1.10 release branch. If a feature does not +require a file format change, then it might be merged into one or more +release branches. This would mean in the figure above that the new +feature could be merged into the 1.8 branch and could be included in +the 1.8.6 release. If the feature was added to the 1.8.5 branch, then a +patch version might be released. + +Release branches hold software that is distributed to general +users. In the figure above, a few release branches are shown below the +trunk. Work is done in release branches for a period of time. Branches +further from the trunk have less work done in them. For example, a patch +branch such as 1.8.5.1 may contain only one or two changes. A release +branch such as 1.8.5 may contain a number of bug fixes and new functions, +but these changes are small in number compared to the number of changes in +the 1.8 branch. + +We aim to make available to the public two maintenance releases a year. +The releases occur usually in the spring near May 15 and in the fall near +November 15. If two release branches are being maintained, then +maintenance releases may be made for each release branch. For example, +there was a time when both the 1.6 and 1.8 branches were actively +maintained. In one maintenance release, the 1.6.10 and 1.8.4 versions were +released at the same time. The 1.6 and 1.8 branches were both actively +maintained to give early adopters access to new features and to give most +users plenty of time to make the change to 1.8 software from 1.6. + +As we improve any branch, we consider the effect of any change on the +readability of objects. Applications built, for example, with version +1.8.5 will be able to read data files written with any prior version +of the library. So, a 1.8.5 application will be able to read a dataset +written with 1.4.5. A 1.8.5 application may be able to read a dataset +written under the 1.8.7 library if no new features, features not known +to 1.8.5, were used. + +\section sec_relver_supp Version Support from the Library +The library provides macros and functions to query and check +version numbers. + +The following constants are defined in the file H5public.h +and determine the version of the include files. +\li H5_VERS_MAJOR - The major version number +\li H5_VERS_MINOR - The minor version number +\li H5_VERS_RELEASE - The release number +\li H5_VERS_SUBRELEASE - The subrelease number +\li H5_VERS_INFO - A string that contains the version number + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1. Version function calls and macros
Function Call or MacroComments
#H5get_libversionThis function returns through its arguments the version +numbers for the library to which the application is linked.
#H5checkThis macro uses the #H5check_version function +to verify that the version number of the HDF5 include file used +to compile the application matches the version number of the +library to which the application is linked. This check occurs +automatically when the first HDF5 file is created or opened and +is important because a mismatch between the include files and the +library is likely to result in corrupted data and/or segmentation +faults. If a mismatch is detected, the library issues an error +message on the standard error stream and aborts.
#H5check_versionThis function is called by the #H5check macro +with the include file version constants. The function compares +its arguments to the result returned by #H5get_libversion. +If a mismatch is detected, it prints an error message on the standard +error stream and aborts.
The behavior of this function can be modified by the +HDF5_DISABLE_VERSION_CHECK environment variable. Setting +the environment variable to a value of "1" will issue a +warning but continue without aborting. Setting the environment +variable to a value of "2" will suppress the warning +and continue silently without aborting.
#H5_VERSION_GE and #H5_VERSION_LEThese macros compare the version of the HDF5 library being used +against the version number specified in the parameters. At compile +time, they can be used to conditionally include or exclude code +based on the library's version.
#H5Pset_libver_boundsThis function can be used to control the versions of the object +formats that will be used when creating objects in a file.
+ +For more information on these and other function calls and macros, +see the \ref RM. + +\section sec_relver_use Use Cases +The purpose of this section is to describe how some of the version +functions, macros, and constants might be used. + +\subsection subsec_relver_use_app Application Version Checking +Suppose first that a developer builds an application that will read +from and write to an HDF5 file. When the application is compiled, a +version of the HDF5 Library such as 1.8.6 will be used. The version +constants (#H5_VERS_MAJOR, #H5_VERS_MINOR, and #H5_VERS_RELEASE) are +included in the application when it is compiled. + +Suppose next that a user gets a copy of the application and starts it +up on a workstation. The executable is put into memory along with the +HDF5 Library. However, an application may only work successfully with +the version of the library with which the application was compiled. In +other words, the version of the library that is loaded when the application +is started must be the same version as the version of the library with +which the application was compiled. This is verified by the library when +the first HDF5 API routine is called. If an application wants to confirm +early in its startup procedure that the version of the library that will +be loaded into memory at the workstation will work with the application, +then it can use the #H5get_libversion and +#H5check_version function calls. + +\subsection subsec_relver_use_cond Conditional Inclusions or Exclusions Based on the Version +The #H5_VERSION_GE and #H5_VERSION_LE version +macros compare the version of the HDF5 Library being used against the +version number specified in the parameters. At compile time, they can be +used to conditionally include or exclude code based on the library's +version. For example, the link functions, H5Lxxx, are +new in version 1.8, and some group functions, H5Gxxx, +are deprecated in 1.8. With the #H5_VERSION_GE macro, an +application could use #H5Ldelete if the library version is +1.8.0 or greater, or it could use #H5Gunlink if the library +version is less than 1.8.0. + +\subsection subsec_relver_use_spec Specifying a Format +Suppose a data file has three datasets. It is possible that the three +datasets were added to the data file with applications using different +versions of HDF5. The different versions could be 1.4.5, 1.6.10, and +1.8.6. If another dataset is written to the data file, then it will be +written by default in the oldest format possible that has all of the +features needed to successfully write the dataset. If a newer feature +such as compact storage, a new parameter for a function, or a partially +compressed dataset is used, then a newer format will be used. +#H5Pset_libver_bounds could be used to specify the oldest +format used. In the situation above, the owners of the data file might +want all data written to the file in the future to be in a 1.8 format +rather than 1.6 or 1.4. + +\section sec_relver_share Shared Library Version Numbers +HDF5 shared libraries utilize the + +libtool versioning system in order to indicate interface +compatibility between maintenance releases of HDF5. While we always +attempt to maintain interface compatibility between minor maintenance +release versions of HDF5, if we are forced to break interface +compatibility in order to resolve a critical defect within the +library, then the library interface version attached to the shared +libraries for a given release will be incremented accordingly. + +Please note that this libtool version number for interface +compatibility is unrelated to the HDF5 release version for a given +release. + +*/ diff --git a/doxygen/dox/SWMRTechNote.dox b/doxygen/dox/SWMRTechNote.dox new file mode 100644 index 00000000000..7041ac046bc --- /dev/null +++ b/doxygen/dox/SWMRTechNote.dox @@ -0,0 +1,146 @@ + +/** \page SWMRTN Introduction to Single-Writer/Multiple-Reader (SWMR) + +\section sec_swmr_intro Introduction to SWMR +The Single-Writer / Multiple-Reader (SWMR) feature enables multiple processes to read an HDF5 file +while it is being written to (by a single process) without using locks or requiring communication between processes. +tutr-swmr1.png + +All communication between processes must be performed via the HDF5 file. The HDF5 file under SWMR access must +reside on a system that complies with POSIX write() semantics. + +The basic engineering challenge for this to work was to ensure that the readers of an HDF5 file always +see a coherent (though possibly not up to date) HDF5 file. + +The issue is that when writing data there is information in the metadata cache in addition to the physical file on disk: +tutr-swmr2.png + +However, the readers can only see the state contained in the physical file: +tutr-swmr3.png + +The SWMR solution implements dependencies on when the metadata can be flushed to the file. This ensures that metadata cache +flush operations occur in the proper order, so that there will never be internal file pointers in the physical file +that point to invalid (unflushed) file addresses. + +A beneficial side effect of using SWMR access is better fault tolerance. It is more difficult to corrupt a file when using SWMR. + +\subsection subsec_swmr_doc Documentation +\subsubsection subsubsec_swmr_doc_guide User Guide +SWMR User Guide + +\subsubsection subsubsec_swmr_doc_apis HDF5 Library APIs +
    +
  • #H5Fstart_swmr_write — Enables SWMR writing mode for a file
    • +
  • #H5DOappend — Appends data to a dataset along a specified dimension
    • +
  • #H5Pset_object_flush_cb — Sets a callback function to invoke when an object flush occurs in the file
    • +
  • #H5Pget_object_flush_cb — Retrieves the object flush property values from the file access property list
    • +
  • #H5Odisable_mdc_flushes — Prevents metadata entries for an HDF5 object from being flushed from the metadata cache to storage
    • +
  • #H5Oenable_mdc_flushes — Enables flushing of dirty metadata entries from a file's metadata cache
    • +
  • #H5Oare_mdc_flushes_disabled — Determines if an HDF5 object has had flushes of metadata entries disabled
    • +
+ +\subsubsection subsubsec_swmr_doc_tools Tools +\li h5watch — Outputs new records appended to a dataset as the dataset grows +\li h5format_convert — Converts the layout format version and chunked indexing types of datasets created with +HDF5-1.10 so that applications built with HDF5-1.8 can access them +\li h5clear — Clears superblock status_flags field, removes metadata cache image, prints EOA and EOF, or sets EOA of a file + +\subsubsection subsubsec_swmr_doc_design Design Documents + +\subsection subsec_swmr_model Programming Model +Please be aware that the SWMR feature requires that an HDF5 file be created with the latest file format. See +#H5Pset_libver_bounds for more information. + +To use SWMR follow the the general programming model for creating and accessing HDF5 files and objects along with the steps described below. + +\subsubsection subsubsec_swmr_model_writer SWMR Writer +The SWMR writer either opens an existing file and objects or creates them as follows. + +Open an existing file: +Call #H5Fopen using the #H5F_ACC_SWMR_WRITE flag. +Begin writing datasets. +Periodically flush data. + +Create a new file: +Call #H5Fcreate using the latest file format. +Create groups, datasets and attributes, and then close the attributes. +Call #H5Fstart_swmr_write to start SWMR access to the file. +Periodically flush data. + +

Example Code:

+Create the file using the latest file format property: +\code + fapl = H5Pcreate (H5P_FILE_ACCESS); + status = H5Pset_libver_bounds (fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + fid = H5Fcreate (filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + // Create objects (files, datasets, ...). + // Close any attributes and named datatype objects. + // Groups and datasets may remain open before starting SWMR access to them. + + // Start SWMR access to the file: + status = H5Fstart_swmr_write (fid); + + // Reopen the datasets and then start writing, periodically flushing data: + status = H5Dwrite (dset_id, ...); + status = H5Dflush (dset_id); +\endcode + +\subsubsection subsubsec_swmr_model_reader SWMR Reader +The SWMR reader must continually poll for new data: + +Call #H5Fopen using the #H5F_ACC_SWMR_READ flag. +Poll, checking the size of the dataset to see if there is new data available for reading. +Read new data, if any. + +

Example Code:

+\code + // Open the file using the SWMR read flag: + fid = H5Fopen (filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, H5P_DEFAULT); + // Open the dataset and then repeatedly poll the dataset, by getting the dimensions, reading new data, and refreshing: + dset_id = H5Dopen (...); + space_id = H5Dget_space (...); + while (...) { + status = H5Dread (dset_id, ...); + status = H5Drefresh (dset_id); + space_id = H5Dget_space (...); + } +\endcode + +\subsection subsec_swmr_scope Limitations and Scope +An HDF5 file under SWMR access must reside on a system that complies with POSIX write() +semantics. It is also limited in scope as follows. + +The writer process is only allowed to modify raw data of existing datasets by; +Appending data along any unlimited dimension. +Modifying existing data +The following operations are not allowed (and the corresponding HDF5 files will fail) +\li The writer cannot add new objects to the file. +\li The writer cannot delete objects in the file. +\li The writer cannot modify or append data with variable length, string or region reference datatypes. +\li File space recycling is not allowed. As a result the size of a file modified by a SWMR writer may be larger than a file modified by a non-SWMR writer.

+ +\subsection subsec_swmr_tools Tools for Working with SWMR +Two new tools, h5watch and h5clear, are available for use with SWMR. The other HDF5 utilities have also been modified to recognize SWMR +\li The h5watch tool allows a user to monitor the growth of a dataset. +\li The h5clear tool clears the status flags in the superblock of an HDF5 file. +\li The rest of the HDF5 tools will exit gracefully but not work with SWMR otherwise. + +\subsection subsec_swmr_example Programming Example +A good example of using SWMR is included with the HDF5 tests in the source code. You can run it while reading +the file it creates. If you then interrupt the application and reader and look at the resulting file, you will +see that the file is still valid. Follow these steps: +\li Download the HDF5 source code to a local directory on a filesystem (that complies with POSIX write() semantics). +Build the software. No special configuration options are needed to use SWMR. +\li Invoke two command terminal windows. In one window go into the bin directory of the built binaries. +In the other window go into the test directory of the HDF5-1.10 source code that was just built. +\li In the window in the test directory compile and run use_append_chunk.c. The example writes a three +dimensional dataset by planes (with chunks of size 1 x 256 x 256). +\li In the other window (in the bin directory) run h5watch on the file created by +use_append_chunk.c (use_append_chunk.h5). It should be run while use_append_chunk is executing and you +will see valid data displayed with h5watch. +\li Interrupt use_append_chunk while it is running, and stop h5watch. +\li Use h5clear to clear the status flags in the superblock of the HDF5 file (use_append_chunk.h5). +\li View the file with h5dump. You will see that it is a valid file even though the application did not +close properly. It will contain data up to the point that it was interrupted. + +*/ diff --git a/doxygen/dox/Specifications.dox b/doxygen/dox/Specifications.dox index de9c23d80aa..f59d0721cf7 100644 --- a/doxygen/dox/Specifications.dox +++ b/doxygen/dox/Specifications.dox @@ -17,138 +17,7 @@ \section Other \li \ref IMG -\li \ref TBL -\li - HDF5 Dimension Scale Specification - -*/ - -/** \page FMT3 HDF5 File Format Specification Version 3.0 - -\htmlinclude H5.format.html - -*/ - -/** \page FMT2 HDF5 File Format Specification Version 2.0 - -\htmlinclude H5.format.2.0.html - -*/ - -/** \page FMT11 HDF5 File Format Specification Version 1.1 - -\htmlinclude H5.format.1.1.html - -*/ - -/** \page FMT1 HDF5 File Format Specification Version 1.0 - -\htmlinclude H5.format.1.0.html - -*/ - -/** \page IMG HDF5 Image and Palette Specification Version 1.2 - -\htmlinclude ImageSpec.html - -*/ - -/** \page TBL HDF5 Table Specification Version 1.0 -The HDF5 specification defines the standard objects and storage for the standard HDF5 -objects. (For information about the HDF5 library, model and specification, see the HDF -documentation.) This document is an additional specification do define a standard profile -for how to store tables in HDF5. Table data in HDF5 is stored as HDF5 datasets with standard -attributes to define the properties of the tables. - -\section sec_tab_spec_intro Introduction -A generic table is a sequence of records, each record has a name and a type. Table data -is stored as an HDF5 one dimensional compound dataset. A table is defined as a collection -of records whose values are stored in fixed-length fields. All records have the same structure -and all values in each field have the same data type. - -The dataset for a table is distinguished from other datasets by giving it an attribute -"CLASS=TABLE". Optional attributes allow the storage of a title for the Table and for -each column, and a fill value for each column. - -\section sec_tab_spec_attr Table Attributes -The attributes for the Table are strings. They are written with the #H5LTset_attribute_string -Lite API function. "Required" attributes must always be used. "Optional" attributes must be -used when required. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Table 1. Attributes of an Image Dataset
Attribute NameRequired
Optional		TypeString SizeValueDescription
CLASSRequiredString5"TABLE"This attribute is type #H5T_C_S1, with size 5. For all Tables, the value of this attribute is -TABLE. This attribute identifies this data set as intended to be interpreted as Table that -conforms to the specifications on this page.
VERSIONRequiredString3"0.2"This attribute is of type #H5T_C_S1, with size corresponding to the length of the version string. -This attribute identifies the version number of this specification to which it conforms. The current -version number is "0.2".
TITLEOptionalString  The TITLE is an optional String that is to be used as the informative title of the whole table. -The TITLE is set with the parameter table_title of the function #H5TBmake_table.
FIELD_(n)_NAMERequiredString  The FIELD_(n)_NAME is an optional String that is to be used as the informative title of column n -of the table. For each of the fields the word FIELD_ is concatenated with the zero based field (n) -index together with the name of the field.
FIELD_(n)_FILLOptionalString  The FIELD_(n)_FILL is an optional String that is the fill value for column n of the table. -For each of the fields the word FIELD_ is concatenated with the zero based field (n) index -together with the fill value, if present. This value is written only when a fill value is defined -for the table.
- -The following section of code shows the calls necessary to the creation of a table. -\code -// Create a new HDF5 file using default properties. -file_id = H5Fcreate("my_table.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - -// Call the make table function -H5TBmake_table("Table Title", file_id, "Table1", NFIELDS, NRECORDS, dst_size, field_names, dst_offset, field_type, chunk_size, fill_data, compress, p_data); - -// Close the file. -status = H5Fclose(file_id); -\endcode - -For more information see the @ref H5TB reference manual page and the @ref H5TB_UG, which includes examples. - +\li \ref TBLSPEC +\li \ref sec_dim_scales_spec */ diff --git a/doxygen/dox/TableSpec.dox b/doxygen/dox/TableSpec.dox new file mode 100644 index 00000000000..dcc8a342a13 --- /dev/null +++ b/doxygen/dox/TableSpec.dox @@ -0,0 +1,100 @@ + +/** \page TBLSPEC HDF5 Table Specification Version 1.0 +The HDF5 specification defines the standard objects and storage for the standard HDF5 +objects. (For information about the HDF5 library, model and specification, see the HDF +documentation.) This document is an additional specification to define a standard profile +for how to store tables in HDF5. Table data in HDF5 is stored as HDF5 datasets with standard +attributes to define the properties of the tables. + +\section sec_tab_spec_intro Introduction +A generic table is a sequence of records, each record has a name and a type. Table data +is stored as an HDF5 one dimensional compound dataset. A table is defined as a collection +of records whose values are stored in fixed-length fields. All records have the same structure +and all values in each field have the same data type. + +The dataset for a table is distinguished from other datasets by giving it an attribute +"CLASS=TABLE". Optional attributes allow the storage of a title for the Table and for +each column, and a fill value for each column. + +\section sec_tab_spec_attr Table Attributes +The attributes for the Table are strings. They are written with the #H5LTset_attribute_string +Lite API function. "Required" attributes must always be used. "Optional" attributes must be +used when required. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1. Attributes of an Image Dataset
Attribute NameRequired
Optional		TypeString SizeValueDescription
CLASSRequiredString5"TABLE"This attribute is type #H5T_C_S1, with size 5. For all Tables, the value of this attribute is +TABLE. This attribute identifies this data set as intended to be interpreted as Table that +conforms to the specifications on this page.
VERSIONRequiredString3"0.2"This attribute is of type #H5T_C_S1, with size corresponding to the length of the version string. +This attribute identifies the version number of this specification to which it conforms. The current +version number is "0.2".
TITLEOptionalString  The TITLE is an optional String that is to be used as the informative title of the whole table. +The TITLE is set with the parameter table_title of the function #H5TBmake_table.
FIELD_(n)_NAMERequiredString  The FIELD_(n)_NAME is an optional String that is to be used as the informative title of column n +of the table. For each of the fields the word FIELD_ is concatenated with the zero based field (n) +index together with the name of the field.
FIELD_(n)_FILLOptionalString  The FIELD_(n)_FILL is an optional String that is the fill value for column n of the table. +For each of the fields the word FIELD_ is concatenated with the zero based field (n) index +together with the fill value, if present. This value is written only when a fill value is defined +for the table.
+ +The following section of code shows the calls necessary to the creation of a table. +\code +// Create a new HDF5 file using default properties. +file_id = H5Fcreate("my_table.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + +// Call the make table function +H5TBmake_table("Table Title", file_id, "Table1", NFIELDS, NRECORDS, dst_size, field_names, dst_offset, field_type, chunk_size, fill_data, compress, p_data); + +// Close the file. +status = H5Fclose(file_id); +\endcode + +For more information see the @ref H5TB reference manual page and the @ref H5TB_UG, which includes examples. + + +*/ diff --git a/doxygen/dox/TechnicalNotes.dox b/doxygen/dox/TechnicalNotes.dox index 8bea54bfb25..3cb477561af 100644 --- a/doxygen/dox/TechnicalNotes.dox +++ b/doxygen/dox/TechnicalNotes.dox @@ -8,1621 +8,13 @@ \li \ref IOFLOW \li \ref TNMDC \li \ref thread-safe-lib -\li \ref SWMR -\li \ref VDS +\li \ref SWMRTN +\li \ref VDSTN \li \ref RELVERSION \li \ref UNICODE -\li \ref VFL +\li \ref VFLTN \li HDF5 Library Architecture Overview \li \ref VOL_Connector */ -/** \page IOFLOW HDF5 Raw I/O Flow Notes - -\htmlinclude IOFlow.html - -*/ - -/** \page RELVERSION HDF5 Library Release Version Numbers - -\htmlinclude LibraryReleaseVersionNumbers.html - -*/ - -/** \page VFL HDF5 Virtual File Layer - -\section sec_vfl_intro Introduction -The HDF5 file format describes how HDF5 data structures and dataset raw data are mapped -to a linear format address space and the HDF5 library implements that bidirectional mapping -in terms of an API. However, the HDF5 format specifications do not indicate how the format -address space is mapped onto storage and HDF (version 5 and earlier) simply mapped the format -address space directly onto a single file by convention. - -Since early versions of HDF5 it became apparent that users want the ability to map the -format address space onto different types of storage (a single file, multiple files, local -memory, global memory, network distributed global memory, a network protocol, etc.) with -various types of maps. For instance, some users want to be able to handle very large format -address spaces on operating systems that support only 2GB files by partitioning the format -address space into equal-sized parts each served by a separate file. Other users want the -same multi-file storage capability but want to partition the address space according to -purpose (raw data in one file, object headers in another, global heap in a third, etc.) -in order to improve I/O speeds. - -In fact, the number of storage variations is probably larger than the number of methods -that the HDF5 team is capable of implementing and supporting. Therefore, a Virtual File -Layer API is being implemented which will allow application teams or departments to design -and implement their own mapping between the HDF5 format address space and storage, with each -mapping being a separate file driver (possibly written in terms of other file drivers). The -HDF5 team will provide a small set of useful file drivers which will also serve as examples -for those who which to write their own: - - - - - - - - - - - - - - - - -
#H5FD_SEC2This is the default driver which uses Posix file-system functions -like read and write to perform I/O to a single file. All I/O requests are unbuffered -although the driver does optimize file seeking operations to some extent. -
#H5FD_STDIOThis driver uses functions from 'stdio.h' to perform buffered I/O to a single file. -
#H5FD_COREThis driver performs I/O directly to memory and can be -used to create small temporary files that never exist on permanent storage. This -type of storage is generally very fast since the I/O consists only of memory-to-memory copy operations. -
#H5FD_MPIOThis is the driver of choice for accessing files in parallel -using MPI and MPI-IO. It is only predefined if the library is compiled with parallel I/O support. -
#H5FD_FAMILYLarge format address spaces are partitioned into more -manageable pieces and sent to separate storage locations using an underlying driver -of the user's choice. \ref H5TOOL_RT_UG can be used to change the sizes of the family -members when stored as files or to convert a family of files to a single file or vice versa. -
- -\section sec_vfl_use Using a File Driver -Most application writers will use a driver defined by the HDF5 library or contributed by another -programming team. This chapter describes how existing drivers are used. - -\subsection subsec_vfl_use_hdr Driver Header Files -Each file driver is defined in its own public header file which should be included by any -application which plans to use that driver. The predefined drivers are in header files whose -names begin with 'H5FD' followed by the driver name and '.h'. The 'hdf5.h' header file includes -all the predefined driver header files. - -Once the appropriate header file is included a symbol of the form 'H5FD_' followed by the -upper-case driver name will be the driver identification number.(The driver name is by convention -and might not apply to drivers which are not distributed with HDF5.) However, the value may -change if the library is closed (e.g., by calling #H5close) and the symbol is referenced again. - -\subsection subsec_vfl_use_create Creating and Opening Files -In order to create or open a file one must define the method by which the storage is -accessed(The access method also indicates how to translate the storage name to a storage server -such as a file, network protocol, or memory.) and does so by creating a file access property -list(The term "file access property list" is a misnomer since storage isn't required to be a file.) -which is passed to the #H5Fcreate or #H5Fopen function. A default file access property list is created -by calling #H5Pcreate and then the file driver information is inserted by calling a driver initialization -function such as #H5Pset_fapl_family: -\code -hid_t fapl = H5Pcreate(H5P_FILE_ACCESS); -size_t member_size = 100*1024*1024; /*100MB*/ -H5Pset_fapl_family(fapl, member_size, H5P_DEFAULT); -hid_t file = H5Fcreate("foo%05d.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl); -H5Pclose(fapl); -\endcode - -Each file driver will have its own initialization function whose name is H5Pset_fapl_ followed by -the driver name and which takes a file access property list as the first argument followed by additional -driver-dependent arguments. - -An alternative to using the driver initialization function is to set the driver directly using the -#H5Pset_driver function.(This function is overloaded to operate on data transfer property lists also, as described below.) -Its second argument is the file driver identifier, which may have a different numeric value from run to run -depending on the order in which the file drivers are registered with the library. The third argument encapsulates -the additional arguments of the driver initialization function. This method only works if the file driver -writer has made the driver-specific property list structure a public datatype, which is often not the case. -\code -hid_t fapl = H5Pcreate(H5P_FILE_ACCESS); -static H5FD_family_fapl_t fa = {100*1024*1024, H5P_DEFAULT}; -H5Pset_driver(fapl, H5FD_FAMILY, &fa); -hid_t file = H5Fcreate("foo.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl); -H5Pclose(fapl); -\endcode - -It is also possible to query the file driver information from a file access property list by -calling #H5Pget_driver to determine the driver and then calling a driver-defined query function -to obtain the driver information: -\code -hid_t driver = H5Pget_driver(fapl); -if (H5FD_SEC2==driver) { - /*nothing further to get*/ -} else if (H5FD_FAMILY==driver) { - hid_t member_fapl; - haddr_t member_size; - H5Pget_fapl_family(fapl, &member_size, &member_fapl); -} else if (....) { - .... -} -\endcode - -\subsection subsec_vfl_use_per Performing I/O -The #H5Dread and #H5Dwrite functions transfer data between application memory and the file. They both take -an optional data transfer property list which has some general driver-independent properties and optional -driver-defined properties. An application will typically perform I/O in one of three styles via the -#H5Dread or #H5Dwrite function: - -Like file access properties in the previous section, data transfer properties can be set using a driver -initialization function or a general purpose function. For example, to set the MPI-IO driver to use -independent access for I/O operations one would say: -\code -hid_t dxpl = H5Pcreate(H5P_DATA_XFER); -H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); -H5Dread(dataset, type, mspace, fspace, buffer, dxpl); -H5Pclose(dxpl); -\endcode - -The alternative is to initialize a driver defined C struct and pass it to the #H5Pset_driver function: -\code -hid_t dxpl = H5Pcreate(H5P_DATA_XFER); -static H5FD_mpio_dxpl_t dx = {H5FD_MPIO_INDEPENDENT}; -H5Pset_driver(dxpl, H5FD_MPIO, &dx); -H5Dread(dataset, type, mspace, fspace, buffer, dxpl); -\endcode - -The transfer property list can be queried in a manner similar to the file access property list: the driver -provides a function (or functions) to return various information about the transfer property list: -\code -hid_t driver = H5Pget_driver(dxpl); -if (H5FD_MPIO==driver) { - H5FD_mpio_xfer_t xfer_mode; - H5Pget_dxpl_mpio(dxpl, &xfer_mode); -} else { - .... -} -\endcode - -\subsection subsec_vfl_use_inter File Driver Interchangeability -The HDF5 specifications describe two things: the mapping of data onto a linear format address -space and the C API which performs the mapping. However, the mapping of the format address space -onto storage intentionally falls outside the scope of the HDF5 specs. This is a direct result of the -fact that it is not generally possible to store information about how to access storage inside the -storage itself. For instance, given only the file name '/arborea/1225/work/f%03d' the HDF5 library -is unable to tell whether the name refers to a file on the local file system, a family of files on -the local file system, a file on host 'arborea' port 1225, a family of files on a remote system, etc. - -Two ways which library could figure out where the storage is located are: storage access information -can be provided by the user, or the library can try all known file access methods. This implementation -uses the former method. - -In general, if a file was created with one driver then it isn't possible to open it with another driver. -There are of course exceptions: a file created with MPIO could probably be opened with the sec2 driver, -any file created by the sec2 driver could be opened as a family of files with one member, etc. In fact, -sometimes a file must not only be opened with the same driver but also with the same driver properties. -The predefined drivers are written in such a way that specifying the correct driver is sufficient for -opening a file. - -\section sec_vfl_imp Implementation of a Driver -A driver is simply a collection of functions and data structures which are registered with the HDF5 -library at runtime. The functions fall into these categories: -\li Functions which operate on modes -\li Functions which operate on files -\li Functions which operate on the address space -\li Functions which operate on data -\li Functions for driver initialization -\li Optimization functions - -\subsection subsec_vfl_imp_mode Mode Functions -Some drivers need information about file access and data transfers which are very specific to the driver. -The information is usually implemented as a pair of pointers to C structs which are allocated and -initialized as part of an HDF5 property list and passed down to various driver functions. There are two -classes of settings: file access modes that describe how to access the file through the driver, and -data transfer modes which are settings that control I/O operations. Each file opened by a particular -driver may have a different access mode; each dataset I/O request for a particular file may have a -different data transfer mode. - -Since each driver has its own particular requirements for various settings, each driver is responsible -for defining the mode structures that it needs. Higher layers of the library treat the structures as -opaque but must be able to copy and free them. Thus, the driver provides either the size of the -structure or a pair of function pointers for each of the mode types. - -Example: The family driver needs to know how the format address space is partitioned and the file -access property list to use for the family members. -\code -// Driver-specific file access properties -typedef struct H5FD_family_fapl_t { - hsize_t memb_size; // size of each family member - hid_t memb_fapl; // file access property list for each family member -} H5FD_family_fapl_t; - -// Driver specific data transfer properties -typedef struct H5FD_family_dxpl_t { - hid_t memb_dxpl_id; //data xfer property list of each member -} H5FD_family_dxpl_t; -\endcode -n order to copy or free one of these structures the member file access or data transfer properties must -also be copied or freed. This is done by providing a copy and close function for each structure: - -Example: The file access property list copy and close functions for the family driver: -\code -static void * -H5FD_family_fapl_copy(const void *_old_fa) -{ - const H5FD_family_fapl_t *old_fa = (const H5FD_family_fapl_t*)_old_fa; - H5FD_family_fapl_t *new_fa = malloc(sizeof(H5FD_family_fapl_t)); - assert(new_fa); - - memcpy(new_fa, old_fa, sizeof(H5FD_family_fapl_t)); - new_fa->memb_fapl_id = H5Pcopy(old_fa->memb_fapl_id); - return new_fa; -} - -static herr_t -H5FD_family_fapl_free(void *_fa) -{ - H5FD_family_fapl_t *fa = (H5FD_family_fapl_t*)_fa; - H5Pclose(fa->memb_fapl_id); - free(fa); - return 0; -} -\endcode - -Generally when a file is created or opened the file access properties for the driver are copied into the -file pointer which is returned and they may be modified from their original value (for instance, the file -family driver modifies the member size property when opening an existing family). In order to support the -#H5Fget_access_plist function the driver must provide a fapl_get callback which creates a copy of the -driver-specific properties based on a particular file. - -Example: The file family driver copies the member size file access property list into the return value: -\code -static void * -H5FD_family_fapl_get(H5FD_t *_file) -{ - H5FD_family_t *file = (H5FD_family_t*)_file; - H5FD_family_fapl_t *fa = calloc(1, sizeof(H5FD_family_fapl_t*)); - - fa->memb_size = file->memb_size; - fa->memb_fapl_id = H5Pcopy(file->memb_fapl_id); - return fa; -} -\endcode - -\subsection subsec_vfl_imp_file File Functions -The higher layers of the library expect files to have a name and allow the file to be accessed in various modes. -The driver must be able to create a new file, replace an existing file, or open an existing file. Opening or -creating a file should return a handle, a pointer to a specialization of the H5FD_t struct, which allows read-only -or read-write access and which will be passed to the other driver functions as they are called.(Read-only access is -only appropriate when opening an existing file.) -\code -typedef struct { - // Public fields - H5FD_class_t *cls; //class data defined below - - // Private fields -- driver-defined - -} H5FD_t; -\endcode - -Example: The family driver requires handles to the underlying storage, the size of the members for this -particular file (which might be different than the member size specified in the file access property list -if an existing file family is being opened), the name used to open the file in case additional members -must be created, and the flags to use for creating those additional members. The eoa member caches the -size of the format address space so the family members don't have to be queried in order to find it. -\code -// The description of a file belonging to this driver. -typedef struct H5FD_family_t { - H5FD_t pub; // public stuff, must be first - hid_t memb_fapl_id; // file access property list for members - hsize_t memb_size; // maximum size of each member file - int nmembs; // number of family members - int amembs; // number of member slots allocated - H5FD_t **memb; // dynamic array of member pointers - haddr_t eoa; // end of allocated addresses - char *name; // name generator printf format - unsigned flags; // flags for opening additional members -} H5FD_family_t; -\endcode - -Example: The sec2 driver needs to keep track of the underlying Unix file descriptor and also the -end of format address space and current Unix file size. It also keeps track of the current file -position and last operation (read, write, or unknown) in order to optimize calls to lseek. The -device and inode fields are defined on Unix in order to uniquely identify the file and will be -discussed below. -\code -typedef struct H5FD_sec2_t { - H5FD_t pub; // public stuff, must be first - int fd; // the unix file - haddr_t eoa; // end of allocated region - haddr_t eof; // end of file; current file size - haddr_t pos; // current file I/O position - int op; // last operation - dev_t device; // file device number - ino_t inode; // file i-node number -} H5FD_sec2_t; -\endcode - -\subsection subsec_vfl_imp_open Open Files -All drivers must define a function for opening/creating a file. This function should have a prototype which is: - - - - - -
static H5FD_t * open (const char *name, unsigned flags, hid_t fapl, haddr_t maxaddr)The file name name and file access property list fapl are the same as were specified in the #H5Fcreate -or #H5Fopen call. The flags are the same as in those calls also except the flag #H5F_ACC_CREAT is also -present if the call was to H5Fcreate and they are documented in the 'H5Fpublic.h' file. The maxaddr -argument is the maximum format address that the driver should be prepared to handle (the minimum address is always zero).
- -Example: The sec2 driver opens a Unix file with the requested name and saves information which -uniquely identifies the file (the Unix device number and inode). -\code -static H5FD_t * -H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id/*unused*/, - haddr_t maxaddr) -{ - unsigned o_flags; - int fd; - struct stat sb; - H5FD_sec2_t *file=NULL; - - // Check arguments - if (!name || !*name) return NULL; - if (0==maxaddr || HADDR_UNDEF==maxaddr) return NULL; - if (ADDR_OVERFLOW(maxaddr)) return NULL; - - // Build the open flags - o_flags = (H5F_ACC_RDWR & flags) ? O_RDWR : O_RDONLY; - if (H5F_ACC_TRUNC & flags) o_flags |= O_TRUNC; - if (H5F_ACC_CREAT & flags) o_flags |= O_CREAT; - if (H5F_ACC_EXCL & flags) o_flags |= O_EXCL; - - // Open the file - if ((fd=open(name, o_flags, 0666))<0) return NULL; - if (fstat(fd, &sb)<0) { - close(fd); - return NULL; - } - - // Create the new file struct - file = calloc(1, sizeof(H5FD_sec2_t)); - file->fd = fd; - file->eof = sb.st_size; - file->pos = HADDR_UNDEF; - file->op = OP_UNKNOWN; - file->device = sb.st_dev; - file->inode = sb.st_ino; - - return (H5FD_t*)file; -} -\endcode - -\subsection subsec_vfl_imp_close Closing Files -Closing a file simply means that all cached data should be flushed to the next lower layer, the -file should be closed at the next lower layer, and all file-related data structures should be -freed. All information needed by the close function is already present in the file handle. - - - - - -
static herr_t close (H5FD_t *file)The file argument is the handle which was returned by the open function, and the close should -free only memory associated with the driver-specific part of the handle (the public parts will -have already been released by HDF5's virtual file layer).
- -Example: The sec2 driver just closes the underlying Unix file, making sure that the actual -file size is the same as that known to the library by writing a zero to the last file position -it hasn't been written by some previous operation (which happens in the same code which flushes -the file contents and is shown below). -\code -static herr_t -H5FD_sec2_close(H5FD_t *_file) -{ - H5FD_sec2_t *file = (H5FD_sec2_t*)_file; - - if (H5FD_sec2_flush(_file)<0) return -1; - if (close(file->fd)<0) return -1; - free(file); - return 0; -} -\endcode - -\subsection subsec_vfl_imp_key File Keys -Occasionally an application will attempt to open a single file more than one time in order -to obtain multiple handles to the file. HDF5 allows the files to share information(For instance, -writing data to one handle will cause the data to be immediately visible on the other handle.) -but in order to accomplish this HDF5 must be able to tell when two names refer to the same file. -It does this by associating a driver-defined key with each file opened by a driver and comparing -the key for an open request with the keys for all other files currently open by the same driver. - - - - - -
const int cmp (const H5FD_t *f1, const H5FD_t *f2)The driver may provide a function which compares two files f1 and f2 belonging to the same -driver and returns a negative, positive, or zero value a la the strcmp function.(The ordering -is arbitrary as long as it's consistent within a particular file driver.) If this function is -not provided then HDF5 assumes that all calls to the open callback return unique files regardless -of the arguments and it is up to the application to avoid doing this if that assumption is incorrect.
- -Each time a file is opened the library calls the cmp function to compare that file with all other files -currently open by the same driver and if one of them matches (at most one can match) then the file -which was just opened is closed and the previously opened file is used instead. - -Opening a file twice with incompatible flags will result in failure. For instance, opening a file with -the truncate flag is a two step process which first opens the file without truncation so keys can be -compared, and if no matching file is found already open then the file is closed and immediately reopened -with the truncation flag set (if a matching file is already open then the truncating open will fail). - -Example: The sec2 driver uses the Unix device and i-node as the key. They were initialized when -the file was opened. -\code -static int -H5FD_sec2_cmp(const H5FD_t *_f1, const H5FD_t *_f2) -{ - const H5FD_sec2_t *f1 = (const H5FD_sec2_t*)_f1; - const H5FD_sec2_t *f2 = (const H5FD_sec2_t*)_f2; - - if (f1->device < f2->device) return -1; - if (f1->device > f2->device) return 1; - - if (f1->inode < f2->inode) return -1; - if (f1->inode > f2->inode) return 1; - - return 0; -} -\endcode - -\subsection subsec_vfl_imp_save Saving Modes Across Opens -Some drivers may also need to store certain information in the file superblock in order -to be able to reliably open the file at a later date. This is done by three functions: -one to determine how much space will be necessary to store the information in the superblock, -one to encode the information, -and one to decode the information. These functions are optional, but if any one is defined -then the other two must also be defined. - - - - - - - - - - - - - - - - - -
FunctionDescription
static hsize_t sb_size (H5FD_t *file)The sb_size function returns the number of bytes necessary to encode -information needed later if the file is reopened.
static herr_t sb_encode (H5FD_t *file, char *name, unsigned char *buf)The sb_encode function encodes information from the file into buffer buf -allocated by the caller. It also writes an 8-character (plus null termination) into -the name argument, which should be a unique identification for the driver.
static herr_t sb_decode (H5FD_t *file, const char *name, const unsigned char *buf)The sb_decode function looks at the name decodes data from the buffer buf and -updates the file argument with the new information, advancing *p in the process.
-The part of this which is somewhat tricky is that the file must be readable before the -superblock information is decoded. File access modes fall outside the scope of the HDF5 -file format, but they are placed inside the boot block for convenience.(File access modes -do not describe data, but rather describe how the HDF5 format address space is mapped to -the underlying file(s). Thus, in general the mapping must be known before the file -superblock can be read. However, the user usually knows enough about the mapping for -the superblock to be readable and once the superblock is read the library can fill -in the missing parts of the mapping.) - -\section sec_vfl_address Address Space Functions -HDF5 does not assume that a file is a linear address space of bytes. Instead, the library -will call functions to allocate and free portions of the HDF5 format address space, which -in turn map onto functions in the file driver to allocate and free portions of file address -space. The library tells the file driver how much format address space it wants to allocate -and the driver decides what format address to use and how that format address is mapped -onto the file address space. Usually the format address is chosen so that the file address -can be calculated in constant time for data I/O operations (which are always specified by format addresses). - -\subsection subsec_vfl_address_blk Userblock and Superblock -The HDF5 format allows an optional userblock to appear before the actual HDF5 data in such -a way that if the userblock is sucked out of the file and everything remaining is -shifted downward in the file address space, then the file is still a valid HDF5 file. -The userblock size can be zero or any multiple of two greater than or equal to 512 and -the file superblock begins immediately after the userblock. - -HDF5 allocates space for the userblock and superblock by calling an allocation function -defined below, which must return a chunk of memory at format address zero on the first call. - -\subsection subsec_vfl_address_alloc Allocatiion of Format Regions -The library makes many types of allocation requests: - - - - - - - - - - - - - - - - - - - - -
#H5FD_MEM_SUPERuserblock
#H5FD_MEM_BTREEAn allocation request for a node of a B-tree. -
#H5FD_MEM_DRAWAn allocation request for the raw data of a dataset. -
#H5FD_MEM_GHEAPAn allocation request for a global heap collection. Global -heaps are used to store certain types of references such as dataset region references. -The set of all global heap collections can become quite large. -
#H5FD_MEM_LHEAPAn allocation request for a local heap. Local heaps are used -to store the names which are members of a group. The combined size of all local heaps is -a function of the number of object names in the file. -
#H5FD_MEM_OHDRAn allocation request for (part of) an object header. Object -headers are relatively small and include meta information about objects (like the data -space and type of a dataset) and attributes. -
- -When a chunk of memory is freed the library adds it to a free list and allocation requests -are satisfied from the free list before requesting memory from the file driver. Each type of -allocation request enumerated above has its own free list, but the file driver can specify that -certain object types can share a free list. It does so by providing an array which maps a -request type to a free list. If any value of the map is H5MF_DEFAULT (zero) then the object's -own free list is used. The special value H5MF_NOLIST indicates that the library should not -attempt to maintain a free list for that particular object type, instead calling the file driver -each time an object of that type is freed. - -Mappings predefined in the 'H5FDpublic.h' file are: - - - - - - - - - - -
#H5FD_FLMAP_SINGLEAll memory usage types are mapped to a single free list. -
#H5FD_FLMAP_DICHOTOMYMemory usage is segregated into meta data and raw data -for the purposes of memory management. -
#H5FD_FLMAP_DEFAULTEach memory usage type has its own free list. -
- -Example: To make a map that manages object headers on one free list and everything else on -another free list one might initialize the map with the following code: (the use of #H5FD_MEM_SUPER is arbitrary) -\code -H5FD_mem_t mt, map[H5FD_MEM_NTYPES]; - -for (mt = 0; mt < H5FD_MEM_NTYPES; mt++) { - map[mt] = (H5FD_MEM_OHDR== mt) ? mt : H5FD_MEM_SUPER; -} -\endcode - -If an allocation request cannot be satisfied from the free list then one of two things happen. -If the driver defines an allocation callback then it is used to allocate space; otherwise new -memory is allocated from the end of the format address space by incrementing the end-of-address marker. - - - - - -
static haddr_t alloc (H5FD_t *file, H5MF_type_t type, hsize_t size)The file argument is the file from which space is to be allocated, type is the type of -memory being requested (from the list above) without being mapped according to the freelist -map and size is the number of bytes being requested. The library is allowed to allocate large -chunks of storage and manage them in a layer above the file driver (although the current library -doesn't do that). The allocation function should return a format address for the first byte -allocated. The allocated region extends from that address for size bytes. If the request cannot -be honored then the undefined address value is returned (#HADDR_UNDEF). The first call to this -function for a file which has never had memory allocated must return a format address of zero -or #HADDR_UNDEF since this is how the library allocates space for the userblock and/or superblock.
- -\subsection subsec_vfl_address_free Freeing Format Regions -When the library is finished using a certain region of the format address space it will return the -space to the free list according to the type of memory being freed and the free list map described above. -If the free list has been disabled for a particular memory usage type (according to the free list map) -and the driver defines a free callback then it will be invoked. The free callback is also invoked for -all entries on the free list when the file is closed. - - - - - - -
static herr_t free (H5FD_t *file, H5MF_type_t type, haddr_t addr, hsize_t size)The file argument is the file for which space is being freed; type is the type of object being -freed (from the list above) without being mapped according to the freelist map; addr is the first -format address to free; and size is the size in bytes of the region being freed. The region being -freed may refer to just part of the region originally allocated and/or may cross allocation boundaries -provided all regions being freed have the same usage type. However, the library will never attempt -to free regions which have already been freed or which have never been allocated.
-A driver may choose to not define the free function, in which case format addresses will be leaked. -This isn't normally a huge problem since the library contains a simple free list of its own and freeing -parts of the format address space is not a common occurrence. - -\subsection subsec_vfl_address_query Querying the Address Range -Each file driver must have some mechanism for setting and querying the end of address, or -EOA, marker. The EOA marker is the first format address after the last format address ever allocated. -If the last part of the allocated address range is freed then the driver may optionally decrease the eoa marker. - - - - - -
static haddr_t get_eoa (H5FD_t *file)This function returns the current value of the EOA marker for the specified file.
- -Example: The sec2 driver just returns the current eoa marker value which is cached in the file structure: -\code -static haddr_t -H5FD_sec2_get_eoa(H5FD_t *_file) -{ - H5FD_sec2_t *file = (H5FD_sec2_t*)_file; - return file->eoa; -} -\endcode - -The eoa marker is initially zero when a file is opened and the library may set it to some other value -shortly after the file is opened (after the superblock is read and the saved eoa marker is determined) -or when allocating additional memory in the absence of an alloc callback (described above). - -Example: The sec2 driver simply caches the eoa marker in the file structure and does not extend the -underlying Unix file. When the file is flushed or closed then the Unix file size is extended to match -the eoa marker. -\code -static herr_t -H5FD_sec2_set_eoa(H5FD_t *_file, haddr_t addr) -{ - H5FD_sec2_t *file = (H5FD_sec2_t*)_file; - file->eoa = addr; - return 0; -} -\endcode - -\section sec_vfl_data Data Functions -These functions operate on data, transferring a region of the format address space between memory and files. - -\subsection subsec_vfl_data_cont Contiguous I/O Functions -A driver must specify two functions to transfer data from the library to the file and vice versa. - - - - - - - - - -
static herr_t read (H5FD_t *file, H5FD_mem_t type, hid_t dxpl, haddr_t addr, hsize_t size, void *buf)The read function reads data from file file beginning at address addr and continuing -for size bytes into the buffer buf supplied by the caller.
static herr_t write (H5FD_t *file, H5FD_mem_t type, hid_t dxpl, haddr_t addr, hsize_t size, const void *buf)The write function transfers data -in the opposite direction.
-\li Both functions take a data transfer property list dxpl which -indicates the fine points of how the data is to be transferred and which comes directly -from the #H5Dread or #H5Dwrite function. -\li Both functions receive type of data being written, -which may allow a driver to tune it's behavior for different kinds of data. -\li Both functions should return -a negative value if they fail to transfer the requested data, or non-negative if they -succeed. The library will never attempt to read from unallocated regions of the format address space. - -Example: The sec2 driver just makes system calls. It tries not to call lseek if the current operation -is the same as the previous operation and the file position is correct. It also fills the output buffer -with zeros when reading between the current EOF and EOA markers and restarts system calls which were interrupted. -\code -static herr_t -H5FD_sec2_read(H5FD_t *_file, H5FD_mem_t type/*unused*/, hid_t dxpl_id/*unused*/, - haddr_t addr, hsize_t size, void *buf/*out*/) -{ - H5FD_sec2_t *file = (H5FD_sec2_t*)_file; - ssize_t nbytes; - - assert(file && file->pub.cls); - assert(buf); - - /* Check for overflow conditions */ - if (REGION_OVERFLOW(addr, size)) return -1; - if (addr+size>file->eoa) return -1; - - /* Seek to the correct location */ - if ((addr!=file->pos || OP_READ!=file->op) && - file_seek(file->fd, (file_offset_t)addr, SEEK_SET)<0) { - file->pos = HADDR_UNDEF; - file->op = OP_UNKNOWN; - return -1; - } - - /* - * Read data, being careful of interrupted system calls, partial results, - * and the end of the file. - */ - while (size>0) { - do nbytes = read(file->fd, buf, size); - while (-1==nbytes && EINTR==errno); - if (-1==nbytes) { - /* error */ - file->pos = HADDR_UNDEF; - file->op = OP_UNKNOWN; - return -1; - } - if (0==nbytes) { - /* end of file but not end of format address space */ - memset(buf, 0, size); - size = 0; - } - assert(nbytes>=0); - assert((hsize_t)nbytes<=size); - size -= (hsize_t)nbytes; - addr += (haddr_t)nbytes; - buf = (char*)buf + nbytes; - } - - /* Update current position */ - file->pos = addr; - file->op = OP_READ; - return 0; -} -\endcode -Example: The sec2 write callback is similar except it updates the file EOF marker when extending the file. - -\subsection subsec_vfl_data_flush Flushing Cached Data -Some drivers may desire to cache data in memory in order to make larger I/O requests to the -underlying file and thus improving bandwidth. Such drivers should register a cache flushing -function so that the library can insure that data has been flushed out of the drivers in -response to the application calling #H5Fflush. - - - - - -
static herr_t flush (H5FD_t *file)Flush all data for file file to storage.
- -Example: The sec2 driver doesn't cache any data but it also doesn't extend the Unix file as -aggressively as it should. Therefore, when finalizing a file it should write a zero to the last -byte of the allocated region so that when reopening the file later the EOF marker will be at -least as large as the EOA marker saved in the superblock (otherwise HDF5 will refuse to open -the file, claiming that the data appears to be truncated). -\code -static herr_t -H5FD_sec2_flush(H5FD_t *_file) -{ - H5FD_sec2_t *file = (H5FD_sec2_t*)_file; - - if (file->eoa>file->eof) { - if (-1==file_seek(file->fd, file->eoa-1, SEEK_SET)) return -1; - if (write(file->fd, "", 1)!=1) return -1; - file->eof = file->eoa; - file->pos = file->eoa; - file->op = OP_WRITE; - } - - return 0; -} -\endcode - -\section sec_vfl_opt Optimization Functions -The library is capable of performing several generic optimizations on I/O, but these types of -optimizations may not be appropriate for a given VFL driver. - -Each driver may provide a query function to allow the library to query whether to enable these -optimizations. If a driver lacks a query function, the library will disable all types of -optimizations which can be queried. - - - - - - -
static herr_t query (const H5FD_t *file, unsigned long *flags)This function is called by the library to query which optimizations to enable for I/O to this driver.
- -These are the flags which are currently defined: - - - - - - - - - - - - - -
H5FD_FEAT_AGGREGATE_METADATA (0x00000001)Defining the H5FD_FEAT_AGGREGATE_METADATA for a VFL driver means that the library will attempt to allocate -a larger block for metadata and then sub-allocate each metadata request from that larger block.
H5FD_FEAT_ACCUMULATE_METADATA (0x00000002)Defining the H5FD_FEAT_ACCUMULATE_METADATA for a VFL driver means that the library will attempt to cache -metadata as it is written to the file and build up a larger block of metadata to eventually pass to the -VFL 'write' routine.
H5FD_FEAT_DATA_SIEVE (0x00000004)Defining the H5FD_FEAT_DATA_SIEVE for a VFL driver means that the library will attempt to cache raw data - as it is read from/written to a file in a "data sieve" buffer.
- -See Rajeev Thakur's papers: -http://www.mcs.anl.gov/~thakur/papers/romio-coll.ps.gz -http://www.mcs.anl.gov/~thakur/papers/mpio-high-perf.ps.gz - -\section sec_vfl_reg Registration of a Driver -Before a driver can be used the HDF5 library needs to be told of its existence. This is done by -registering the driver, which results in a driver identification number. Instead of passing many -arguments to the registration function, the driver information is entered into a structure and the -address of the structure is passed to the registration function where it is copied. This allows -the HDF5 API to be extended while providing backward compatibility at the source level. - - - - - - -
hid_t H5FDregister (H5FD_class_t *cls)The driver described by struct cls is registered with the library and an ID number for the driver is returned.
- -The H5FD_class_t type is a struct with the following fields: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
const char *nameA pointer to a constant, null-terminated driver name to be used for debugging purposes.
size_t fapl_sizeThe size in bytes of the file access mode structure or zero if the driver supplies a copy function -or doesn't define the structure.
void *(*fapl_copy)(const void *fapl)An optional function which copies a driver-defined file access mode structure. This field takes -precedence over fm_size when both are defined.
void (*fapl_free)(void *fapl)An optional function to free the driver-defined file access mode structure. If null, then the -library calls the C free function to free the structure.
size_t dxpl_sizeThe size in bytes of the data transfer mode structure or zero if the driver supplies a copy -function or doesn't define the structure.
void *(*dxpl_copy)(const void *dxpl)An optional function which copies a driver-defined data transfer mode structure. This field -takes precedence over xm_size when both are defined.
void (*dxpl_free)(void *dxpl)An optional function to free the driver-defined data transfer mode structure. If null, then -the library calls the C free function to free the structure.
H5FD_t *(*open)(const char *name, unsigned flags, hid_t fapl, haddr_t maxaddr)The function which opens or creates a new file.
herr_t (*close)(H5FD_t *file)The function which ends access to a file.
int (*cmp)(const H5FD_t *f1, const H5FD_t *f2)An optional function to determine whether two open files have the same key. If this function -is not present then the library assumes that two files will never be the same.
int (*query)(const H5FD_t *f, unsigned long *flags)An optional function to determine which library optimizations a driver can support.
haddr_t (*alloc)(H5FD_t *file, H5FD_mem_t type, hsize_t size)An optional function to allocate space in the file.
herr_t (*free)(H5FD_t *file, H5FD_mem_t type, haddr_t addr, hsize_t size)An optional function to free space in the file.
haddr_t (*get_eoa)(H5FD_t *file)A function to query how much of the format address space has been allocated.
herr_t (*set_eoa)(H5FD_t *file, haddr_t)A function to set the end of address space.
haddr_t (*get_eof)(H5FD_t *file)A function to return the current end-of-file marker value.
herr_t (*read)(H5FD_t *file, H5FD_mem_t type, hid_t dxpl, haddr_t addr, hsize_t size, void *buffer)A function to read data from a file.
herr_t (*write)(H5FD_t *file, H5FD_mem_t type, hid_t dxpl, haddr_t addr, hsize_t size, const void *buffer)A function to write data to a file.
herr_t (*flush)(H5FD_t *file)A function which flushes cached data to the file.
H5FD_mem_t fl_map[H5FD_MEM_NTYPES]An array which maps a file allocation request type to a free list.
- -Example: The sec2 driver would be registered as: -\code -static const H5FD_class_t H5FD_sec2_g = { - "sec2", /*name */ - MAXADDR, /*maxaddr */ - NULL, /*sb_size */ - NULL, /*sb_encode */ - NULL, /*sb_decode */ - 0, /*fapl_size */ - NULL, /*fapl_get */ - NULL, /*fapl_copy */ - NULL, /*fapl_free */ - 0, /*dxpl_size */ - NULL, /*dxpl_copy */ - NULL, /*dxpl_free */ - H5FD_sec2_open, /*open */ - H5FD_sec2_close, /*close */ - H5FD_sec2_cmp, /*cmp */ - H5FD_sec2_query, /*query */ - NULL, /*alloc */ - NULL, /*free */ - H5FD_sec2_get_eoa, /*get_eoa */ - H5FD_sec2_set_eoa, /*set_eoa */ - H5FD_sec2_get_eof, /*get_eof */ - H5FD_sec2_read, /*read */ - H5FD_sec2_write, /*write */ - H5FD_sec2_flush, /*flush */ - H5FD_FLMAP_SINGLE, /*fl_map */ -}; - -hid_t -H5FD_sec2_init(void) -{ - if (!H5FD_SEC2_g) { - H5FD_SEC2_g = H5FDregister(&H5FD_sec2_g); - } - return H5FD_SEC2_g; -} -\endcode - -A driver can be removed from the library by unregistering it - - - - - -
herr_t H5Dunregister (hid_t driver)Where driver is the ID number returned when the driver was registered.
-Unregistering a driver makes it unusable for creating new file access or data transfer property -lists but doesn't affect any property lists or files that already use that driver. - -\subsection subsec_vfl_reg_prog Programming Note for C++ Developers Using C Functions -If a C routine that takes a function pointer as an argument is called from within C++ code, -the C routine should be returned from normally. - -Examples of this kind of routine include callbacks such as #H5Pset_elink_cb -and #H5Pset_type_conv_cb and functions such as #H5Tconvert and #H5Ewalk2. - -Exiting the routine in its normal fashion allows the HDF5 C Library to clean up -its work properly. In other words, if the C++ application jumps out of the routine -back to the C++ “catch” statement, the library is not given the opportunity to close -any temporary data structures that were set up when the routine was called. The C++ -application should save some state as the routine is started so that any problem that -occurs might be diagnosed. - -\section sec_vfl_query Querying Driver Information - - - - - -
void * H5Pget_driver_data (hid_t fapl)
void * H5Pget_driver_data (hid_t fxpl)		This function is intended to be used by driver functions, not applications. It returns a pointer -directly into the file access property list fapl which is a copy of the driver's file access mode -originally provided to the H5Pset_driver function. If its argument is a data transfer property list -fxpl then it returns a pointer to the driver-specific data transfer information instead. -
- -\section sec_vfl_misc Miscellaneous -The various private H5F_low_* functions will be replaced by public H5FD* functions so they -can be called from drivers. - -All private functions H5F_addr_* which operate on addresses will be renamed as public functions -by removing the first underscore so they can be called by drivers. - -The haddr_t address data type will be passed by value throughout the library. The original -intent was that this type would eventually be a union of file address types for the various -drivers and may become quite large, but that was back when drivers were part of HDF5. It will -become an alias for an unsigned integer type (32 or 64 bits depending on how the library was configured). - -The various H5F*.c driver files will be renamed H5FD*.c and each will have a corresponding header -file. All driver functions except the initializer and API will be declared static. - -This documentation didn't cover optimization functions which would be useful to drivers like MPI-IO. -Some drivers may be able to perform data pipeline operations more efficiently than HDF5 and need to -be given a chance to override those parts of the pipeline. The pipeline would be designed to call -various H5FD optimization functions at various points which return one of three values: the operation -is not implemented by the driver, the operation is implemented but failed in a non-recoverable manner, -the operation is implemented and succeeded. - -Various parts of HDF5 check the only the top-level file driver and do something special if it is -the MPI-IO driver. However, we might want to be able to put the MPI-IO driver under other drivers -such as the raw part of a split driver or under a debug driver whose sole purpose is to accumulate -statistics as it passes all requests through to the MPI-IO driver. Therefore we will probably need -a function which takes a format address and or object type and returns the driver which would have -been used at the lowest level to process the request. - -*/ - -/** \page FMTDISC HDF5 File Format Discussion - -\htmlinclude FileFormat.html - -*/ - -/** \page APPDBG Debugging HDF5 Applications - -\section sec_adddbg_intro Introduction -The HDF5 library contains a number of debugging features to make programmers' lives -easier including the ability to print detailed error messages, check invariant -conditions, display timings and other statistics. - -\subsection subsec_adddbg_intro_err Error Messages -Error messages are normally displayed automatically on the standard error stream and -include a stack trace of the library including file names, line numbers, and function -names. The application has complete control over how error messages are displayed and -can disable the display on a permanent or temporary basis. Refer to the documentation - for the H5E error handling package. - -\subsection subsec_adddbg_intro_invar Invariant Conditions -Unless NDEBUG is defined during compiling, the library will include code to verify that -invariant conditions have the expected values. When a problem is detected the library will -display the file and line number within the library and the invariant condition that -failed. A core dump may be generated for post mortem debugging. The code to perform these -checks can be included on a per-package bases. - -\subsection subsec_adddbg_intro_stats Timings and Statistics -The library can be configured to accumulate certain statistics about things like cache -performance, datatype conversion, data space conversion, and data filters. The code is -included on a per-package basis and enabled at runtime by an environment variable. - -\subsection subsec_adddbg_intro_trace API Tracing -All API calls made by an application can be displayed and include formal argument names -and actual values and the function return value. This code is also conditionally included -at compile time and enabled at runtime. - -The statistics and tracing can be displayed on any output stream (including streams opened by -the shell) with output from different packages even going to different streams. - -\section sec_adddbg_msg Error Messages -By default any API function that fails will print an error stack to the standard error stream. -\code -HDF5-DIAG: Error detected in thread 0. Back trace follows. - #000: H5F.c line 1245 in H5Fopen(): unable to open file - major(04): File interface - minor(10): Unable to open file - #001: H5F.c line 846 in H5F_open(): file does not exist - major(04): File interface - minor(10): Unable to open file -\endcode -The error handling package (H5E) is described elsewhere. - -\section sec_adddbg_invars Invariant Conditions -To include checks for invariant conditions the library should be configured -with --disable-production, the default for versions before 1.2. The library -designers have made every attempt to handle error conditions gracefully but -an invariant condition assertion may fail in certain cases. The output from -a failure usually looks something like this: -\code -Assertion failed: H5.c:123: i - -Name - -Default - -Description - - - -aNoAttributes - - -acYesMeta data cache - - -bYesB-Trees - - -dYesDatasets - - -eYesError handling - - -fYesFiles - - -gYesGroups - - -hgYesGlobal heap - - -hlNoLocal heaps - - -iYesInterface abstraction - - -mfNoFile memory management - - -mmYesLibrary memory management - - -oNoObject headers and messages - - -pYesProperty lists - - -sYesData spaces - - -tYesDatatypes - - -vYesVectors - - -zYesRaw data filters - - - -In addition to including the code at compile time the application must enable each package at -runtime. This is done by listing the package names in the HDF5_DEBUG environment variable. That -variable may also contain file descriptor numbers (the default is '2') which control the output -for all following packages up to the next file number. The word 'all' refers to all packages. Any -word my be preceded by a minus sign to turn debugging off for the package. - -\subsection subsec_adddbg_stats_sample Sample debug specifications - - - - - - - - - - - - - -
all -This causes debugging output from all packages to be sent to the standard error stream. -
all -t -s -Debugging output for all packages except datatypes and data spaces will appear on the standard error stream. -
-all ac 255 t,s -This disables all debugging even if the default was to debug something, then output -from the meta data cache is send to the standard error stream and output from data types -and spaces is sent to file descriptor 255 which should be redirected by the shell. -
-The components of the HDF5_DEBUG value may be separated by any non-lowercase letter. - -*/ - -/** \page SWMR Introduction to Single-Writer/Multiple-Reader (SWMR) - -\section sec_swmr_intro Introduction to SWMR -The Single-Writer / Multiple-Reader (SWMR) feature enables multiple processes to read an HDF5 file -while it is being written to (by a single process) without using locks or requiring communication between processes. -tutr-swmr1.png - -All communication between processes must be performed via the HDF5 file. The HDF5 file under SWMR access must -reside on a system that complies with POSIX write() semantics. - -The basic engineering challenge for this to work was to ensure that the readers of an HDF5 file always -see a coherent (though possibly not up to date) HDF5 file. - -The issue is that when writing data there is information in the metadata cache in addition to the physical file on disk: -tutr-swmr2.png - -However, the readers can only see the state contained in the physical file: -tutr-swmr3.png - -The SWMR solution implements dependencies on when the metadata can be flushed to the file. This ensures that metadata cache -flush operations occur in the proper order, so that there will never be internal file pointers in the physical file -that point to invalid (unflushed) file addresses. - -A beneficial side effect of using SWMR access is better fault tolerance. It is more difficult to corrupt a file when using SWMR. - -\subsection subsec_swmr_doc Documentation -\subsubsection subsubsec_swmr_doc_guide User Guide -SWMR User Guide - -\subsubsection subsubsec_swmr_doc_apis HDF5 Library APIs -
    -
  • #H5Fstart_swmr_write — Enables SWMR writing mode for a file
    • -
  • #H5DOappend — Appends data to a dataset along a specified dimension
    • -
  • #H5Pset_object_flush_cb — Sets a callback function to invoke when an object flush occurs in the file
    • -
  • #H5Pget_object_flush_cb — Retrieves the object flush property values from the file access property list
    • -
  • #H5Odisable_mdc_flushes — Prevents metadata entries for an HDF5 object from being flushed from the metadata cache to storage
    • -
  • #H5Oenable_mdc_flushes — Enables flushing of dirty metadata entries from a file's metadata cache
    • -
  • #H5Oare_mdc_flushes_disabled — Determines if an HDF5 object has had flushes of metadata entries disabled
    • -
- -\subsubsection subsubsec_swmr_doc_tools Tools -\li h5watch — Outputs new records appended to a dataset as the dataset grows -\li h5format_convert — Converts the layout format version and chunked indexing types of datasets created with -HDF5-1.10 so that applications built with HDF5-1.8 can access them -\li h5clear — Clears superblock status_flags field, removes metadata cache image, prints EOA and EOF, or sets EOA of a file - -\subsubsection subsubsec_swmr_doc_design Design Documents - -\subsection subsec_swmr_model Programming Model -Please be aware that the SWMR feature requires that an HDF5 file be created with the latest file format. See -#H5Pset_libver_bounds for more information. - -To use SWMR follow the the general programming model for creating and accessing HDF5 files and objects along with the steps described below. - -\subsubsection subsubsec_swmr_model_writer SWMR Writer -The SWMR writer either opens an existing file and objects or creates them as follows. - -Open an existing file: -Call #H5Fopen using the #H5F_ACC_SWMR_WRITE flag. -Begin writing datasets. -Periodically flush data. - -Create a new file: -Call #H5Fcreate using the latest file format. -Create groups, datasets and attributes, and then close the attributes. -Call #H5Fstart_swmr_write to start SWMR access to the file. -Periodically flush data. - -

Example Code:

-Create the file using the latest file format property: -\code - fapl = H5Pcreate (H5P_FILE_ACCESS); - status = H5Pset_libver_bounds (fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); - fid = H5Fcreate (filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); - // Create objects (files, datasets, ...). - // Close any attributes and named datatype objects. - // Groups and datasets may remain open before starting SWMR access to them. - - // Start SWMR access to the file: - status = H5Fstart_swmr_write (fid); - - // Reopen the datasets and then start writing, periodically flushing data: - status = H5Dwrite (dset_id, ...); - status = H5Dflush (dset_id); -\endcode - -\subsubsection subsubsec_swmr_model_reader SWMR Reader -The SWMR reader must continually poll for new data: - -Call #H5Fopen using the #H5F_ACC_SWMR_READ flag. -Poll, checking the size of the dataset to see if there is new data available for reading. -Read new data, if any. - -

Example Code:

-\code - // Open the file using the SWMR read flag: - fid = H5Fopen (filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, H5P_DEFAULT); - // Open the dataset and then repeatedly poll the dataset, by getting the dimensions, reading new data, and refreshing: - dset_id = H5Dopen (...); - space_id = H5Dget_space (...); - while (...) { - status = H5Dread (dset_id, ...); - status = H5Drefresh (dset_id); - space_id = H5Dget_space (...); - } -\endcode - -\subsection subsec_swmr_scope Limitations and Scope -An HDF5 file under SWMR access must reside on a system that complies with POSIX write() -semantics. It is also limited in scope as follows. - -The writer process is only allowed to modify raw data of existing datasets by; -Appending data along any unlimited dimension. -Modifying existing data -The following operations are not allowed (and the corresponding HDF5 files will fail) -\li The writer cannot add new objects to the file. -\li The writer cannot delete objects in the file. -\li The writer cannot modify or append data with variable length, string or region reference datatypes. -\li File space recycling is not allowed. As a result the size of a file modified by a SWMR writer may be larger than a file modified by a non-SWMR writer.

- -\subsection subsec_swmr_tools Tools for Working with SWMR -Two new tools, h5watch and h5clear, are available for use with SWMR. The other HDF5 utilities have also been modified to recognize SWMR -\li The h5watch tool allows a user to monitor the growth of a dataset. -\li The h5clear tool clears the status flags in the superblock of an HDF5 file. -\li The rest of the HDF5 tools will exit gracefully but not work with SWMR otherwise. - -\subsection subsec_swmr_example Programming Example -A good example of using SWMR is included with the HDF5 tests in the source code. You can run it while reading -the file it creates. If you then interrupt the application and reader and look at the resulting file, you will -see that the file is still valid. Follow these steps: -\li Download the HDF5 source code to a local directory on a filesystem (that complies with POSIX write() semantics). -Build the software. No special configuration options are needed to use SWMR. -\li Invoke two command terminal windows. In one window go into the bin directory of the built binaries. -In the other window go into the test directory of the HDF5-1.10 source code that was just built. -\li In the window in the test directory compile and run use_append_chunk.c. The example writes a three -dimensional dataset by planes (with chunks of size 1 x 256 x 256). -\li In the other window (in the bin directory) run h5watch on the file created by -use_append_chunk.c (use_append_chunk.h5). It should be run while use_append_chunk is executing and you -will see valid data displayed with h5watch. -\li Interrupt use_append_chunk while it is running, and stop h5watch. -\li Use h5clear to clear the status flags in the superblock of the HDF5 file (use_append_chunk.h5). -\li View the file with h5dump. You will see that it is a valid file even though the application did not -close properly. It will contain data up to the point that it was interrupted. - -*/ - -/** \page UNICODE Using UTF-8 Encoding in HDF5 Applications - -\section sec_unicode_intro Introduction -Text and character data are often discussed as though text means ASCII text. We even go so far as -to call a file containing only ASCII text a plain text file. This works reasonably well for English -(though better for American English than British English), but what if that plain text file is in -French, German, Chinese, or any of several hundred other languages? This document introduces the -use of UTF-8 encoding (see note 1), enabling the use of a much more extensive and flexible character -set that can faithfully represent any of those languages. - -This document assumes a working familiarity with UTF-8 and Unicode. Any reader who is unfamiliar -with UTF-8 encoding should read the [Wikipedia UTF-8 article](https://en.wikipedia.org/wiki/UTF-8) -before proceeding; it provides an excellent primer. - -For our context, the most important UTF-8 concepts are: -\li Multi-byte and variable-size character encodings -\li Limitations of the ASCII character set -\li Risks associated with the use of the term plain text -\li Representation of multiple language alphabets or characters in a single document - -More specific technical details will only become important if they affect the specifics of -your application design or implementation. - -\section sec_unicode_support How and Where Is UTF-8 Supported in HDF5? -HDF5 uses characters in object names (which are actually link names, but that's a story for a -different article), dataset raw data, attribute names, and attribute raw data. Though the -mechanisms differ, you can use either ASCII or UTF-8 character sets in all of these situations. - -\subsection subsec_unicode_support_names Object and Attribute Names -By default, HDF5 creates object and attribute names with ASCII character encoding. An object or -attribute creation property list setting is required to create object names with UTF-8 characters. -This uses the function #H5Pset_char_encoding, which sets the character encoding used for object and attribute names. - -For example, the following call sequence could be used to create a dataset with its name encoded with the UTF-8 character set: - -\code - lcpl_id = H5Pcreate(H5P_LINK_CREATE) ; - error = H5Pset_char_encoding(lcpl_id, H5T_CSET_UTF8) ; - dataset_id = H5Dcreate2(group_id, "datos_ñ", datatype_id, dataspace_id, - lcpl_id, H5P_DEFAULT, H5P_DEFAULT) ; -\endcode - -If the character encoding of an attribute name is unknown, the combination of an -#H5Aget_create_plist call and an #H5Pget_char_encoding call will reveal that information. -If the character encoding of an object name is unknown, the information can be accessed -through the object's H5L_info_t structure which can be obtained using #H5Lvisit or #H5Lget_info_by_idx calls. - -\subsection subsec_unicode_support_char Character Datatypes in Datasets and Attributes -Like object names, HDF5 character data in datasets and attributes is encoded as ASCII by -default. Setting up attribute or dataset character data to be UTF-8-encoded is accomplished -while defining the attribute or dataset datatype. This makes use of the function #H5Tset_cset, -which sets the character encoding to be used in building a character datatype. - -For example, the following commands could be used to create an 8-character, UTF-8 encoded, -string datatype for use in either an attribute or dataset: - -\code - datatype_id = H5Tcopy(H5T_C_S1); - error = H5Tset_cset(datatype_id, H5T_CSET_UTF8); - error = H5Tset_size(datatype_id, "8"); -\endcode - -If a character or string datatype's character encoding is unknown, the combination of an -#H5Aget_type or #H5Dget_type call and an #H5Tget_cset call can be used to determine that. - -\section sec_unicode_warn Caveats, Pitfalls, and Things to Watch For -Programmers who are accustomed to using ASCII text without accommodating other text -encodings will have to be aware of certain common issues as they begin using UTF-8 encodings. - -\subsection subsec_unicode_warn_port Cross-platform Portability -Since the HDF5 Library handles datatypes directly, UTF-8 encoded text in dataset and -attribute datatypes in a well-designed HDF5 application and file should work transparently -across platforms. The same should be true of handling names of groups, datasets, committed -datatypes, and attributes within a file. - -Be aware, however, of system or application limitations once data or other information -has been extracted from an HDF5 file. The application or system must be designed to -accommodate UTF-8 encodings if the information is then used elsewhere in the application or system environment. - -Data from a UTF-8 encoded HDF5 datatype, in either a dataset or an attribute, -that has been established within an HDF5 application should "just work" within the HDF5 portions of the application. - -\subsection subsec_unicode_warn_names Filenames -Since file access is a system issue, filenames do not fall within the scope -of HDF5's UTF-8 capabilities; filenames are encoded at the system level. - -Linux and Mac OS systems normally handle UTF-8 encoded filenames correctly -while Windows systems generally do not. - -\section sec_unicode_text The *Plain Text* Illusion -Beware the use of the term *plain text*. *Plain text* is at best ambiguous, but often -misleading. Many will assume that *plain text* means ASCII, but plain text German or -French, for example, cannot be represented in ASCII. Plain text is only unambiguous -in the context of English (and even then can be problematic!). - -\subsection subsec_unicode_warn_store Storage Size -Programmers and data users accustomed to working strictly with ASCII data generally make -the reasonable assumption that 1 character, be it in an object name or in data, requires -1 byte of storage. This equation does not work when using UTF-8 or any other Unicode encoding. -With Unicode encoding, number of characters is not synonymous with number of bytes. One must -get used to thinking in terms of number of characters when talking about content, reserving -number of bytes for discussions of storage size. - -When working with Unicode text, one can no longer assume a 1:1 correspondence between the -number of characters and the data storage requirement. - -\subsection subsec_unicode_warn_sys System Dependencies -Linux, Unix, and similar systems generally handle UTF-8 encodings in correct and -predictable ways. There is an apparent consensus in the Linux community that "UTF-8 is just the right way to go." - -Mac OS systems generally handle UTF-8 encodings correctly. - -Windows systems use a different Unicode encoding, UCS-2 (discussed in this UTF-16 article) at -the system level. Within an HDF5 file and application on a Windows system, UTF-8 encoding should -work correctly and as expected. Problems may arise, however, when that UTF-8 encoding is exposed -directly to the Windows system. For example: -\li File open and close calls on files with UTF-8 encoded names are likely to fail as the HDF5 -open and close operations interact directly with the Windows file system interface. -\li Anytime an HDF5 command-line utility (\ref H5TOOL_LS_UG or \ref H5TOOL_DP_UG, for example) emits text output, the -Windows system must interpret the character encodings. If that output is UTF-8 encoded, Windows -will correctly interpret only those characters in the ASCII subset of UTF-8. - -\section sec_unicode_common Common Characters in UTF-8 and ASCII -One interesting feature of UTF-8 and ASCII is that the ASCII character set is a discrete subset of -the UTF-8 character set. And where they overlap, the encodings are identical. This means that a -character string consisting entirely of members of the ASCII character set can be encoded in either -ASCII or UTF-8, the two encodings will be indistinguishable, and the encodings will require exactly the same storage space. - - -\section sec_unicode_also See Also - -- For object and attribute names: - * #H5Pset_char_encoding - * #H5Pget_char_encoding -- For dataset and attribute datatypes: - * #H5Tset_cset - * #H5Tget_cset -- [UTF-8 article on Wikipedia](https://en.wikipedia.org/wiki/UTF-8) - -

NOTES

-1. UTF-8 is the only Unicode standard encoding supported in HDF5. - -*/ - -/** \page VDS Introduction to the Virtual Dataset - VDS - -\section sec_vds_intro Introduction to VDS -The HDF5 Virtual Dataset (VDS) feature enables users to access data in a collection of HDF5 files as a -single HDF5 dataset and to use the HDF5 APIs to work with that dataset. - -For example, your data may be collected into four files: -tutrvds-multimgs.png - -You can map the datasets in the four files into a single VDS that can be accessed just like any other dataset: -tutrvds-snglimg.png - -The mapping between a VDS and the HDF5 source datasets is persistent and transparent to an application. If a source -file is missing the fill value will be displayed. - -See the Virtual (VDS) Documentation for complete details regarding the VDS feature. - -The VDS feature was implemented using hyperslab selection (#H5Sselect_hyperslab). See the tutorial on -Reading From or Writing to a Subset of a Dataset for more information on selecting hyperslabs. - -\subsection subsec_vds_intro_model Programming Model -To create a Virtual Dataset you simply follow the HDF5 programming model and add a few additional API calls -to map the source code datasets to the VDS. - -Following are the steps for creating a Virtual Dataset: -\li Create the source datasets that will comprise the VDS -\li Create the VDS: ‐ Define a datatype and dataspace (can be unlimited) -\li Define the dataset creation property list (including fill value) -\li (Repeat for each source dataset) Map elements from the source dataset to elements of the VDS -\li Select elements in the source dataset (source selection) -\li Select elements in the virtual dataset (destination selection) -\li Map destination selections to source selections (see Functions for Working with a VDS) -\li Call H5Dcreate using the properties defined above -\li Access the VDS as a regular HDF5 dataset -\li Close the VDS when finished - -

Functions for Working with a VDS

-The #H5Pset_virtual API sets the mapping between virtual and source datasets. This is a dataset creation property list. -Using this API will change the layout of the dataset to #H5D_VIRTUAL. As with specifying any dataset creation property -list, an instance of the property list is created, modified, passed into the dataset creation call and then closed: -\code - dcpl = H5Pcreate (H5P_DATASET_CREATE); - src_space = H5screate_simple ... - status = H5Sselect_hyperslab (space, ... - status = H5Pset_virtual (dcpl, space, SRC_FILE[i], SRC_DATASET[i], src_space); - dset = H5Dcreate2 (file, DATASET, H5T_NATIVE_INT, space, H5P_DEFAULT, dcpl, H5P_DEFAULT); - status = H5Pclose (dcpl); -\endcode - -There are several other APIs introduced with Virtual Datasets, including query functions. For details -see the complete list of HDF5 library APIs that support Virtual Datasets. - -

Limitations

-This feature was introduced in HDF5-1.10. - -The number of source datasets is unlimited. However, there is a limit on the size of each source dataset. - -\subsection subsec_vds_intro_examples Programming Examples -Example 1 -This example creates three HDF5 files, each with a one-dimensional dataset of 6 elements. The datasets in these files -are the source datasets that are then used to create a 4 x 6 Virtual Dataset with a fill value of -1. The first three -rows of the VDS are mapped to the data from the three source datasets as shown below: -tutrvds-ex.png - -In this example the three source datasets are mapped to the VDS with this code: -\code> -src_space = H5Screate_simple (RANK1, dims, NULL); -for (i = 0; i < 3; i++) { - start[0] = (hsize_t)i; - // Select i-th row in the virtual dataset; selection in the source datasets is the same. - status = H5Sselect_hyperslab (space, H5S_SELECT_SET, start, NULL, count, block); - status = H5Pset_virtual (dcpl, space, SRC_FILE[i], SRC_DATASET[i], src_space); -} -endcode> - -After the VDS is created and closed, it is reopened. The property list is then queried to determine the -layout of the dataset and its mappings, and the data in the VDS is read and printed. - -This example is in the HDF5 source code and can be obtained from here: -

C Example

-For details on compiling an HDF5 application: [ Compiling HDF5 Applications ] - -

Example 2

-This example shows how to use a C-style printf statement for specifying multiple source datasets as one virtual -dataset. Only one mapping is required. In other words only one #H5Pset_virtual call is needed to map multiple datasets. -It creates a 2-dimensional unlimited VDS. Then it re-opens the file, makes queries, and reads the virtual dataset. - -The source datasets are specified as A-0, A-1, A-2, and A-3. These are mapped to the virtual dataset with one call: -\code -status = H5Pset_virtual (dcpl, vspace, SRCFILE, "A-%b", src_space); -\endcode - -The %b indicates that the block count of the selection in the dimension should be used. - -

C Example

-For details on compiling an HDF5 application: [ Compiling HDF5 Applications ] - -Using h5dump with a VDS -The h5dump utility can be used to view a VDS. The h5dump output for a VDS looks exactly like that for any other dataset. -If h5dump cannot find a source dataset then the fill value will be displayed. - -You can determine that a dataset is a VDS by looking at its properties with -\code - h5dump -p -\endcode - It will display each source dataset mapping, beginning with Mapping 0. Below is an excerpt of the output of -\code - h5dump -p -\endcode -on the vds.h5 file created in Example 1.You can see that the entire source file a.h5 is mapped to the first row of the VDS dataset. - -tutrvds-map.png - -*/ - diff --git a/doxygen/dox/Unicode.dox b/doxygen/dox/Unicode.dox new file mode 100644 index 00000000000..8195ccf4caa --- /dev/null +++ b/doxygen/dox/Unicode.dox @@ -0,0 +1,145 @@ + +/** \page UNICODE Using UTF-8 Encoding in HDF5 Applications + +\section sec_unicode_intro Introduction +Text and character data are often discussed as though text means ASCII text. We even go so far as +to call a file containing only ASCII text a plain text file. This works reasonably well for English +(though better for American English than British English), but what if that plain text file is in +French, German, Chinese, or any of several hundred other languages? This document introduces the +use of UTF-8 encoding (see note 1), enabling the use of a much more extensive and flexible character +set that can faithfully represent any of those languages. + +This document assumes a working familiarity with UTF-8 and Unicode. Any reader who is unfamiliar +with UTF-8 encoding should read the [Wikipedia UTF-8 article](https://en.wikipedia.org/wiki/UTF-8) +before proceeding; it provides an excellent primer. + +For our context, the most important UTF-8 concepts are: +\li Multi-byte and variable-size character encodings +\li Limitations of the ASCII character set +\li Risks associated with the use of the term plain text +\li Representation of multiple language alphabets or characters in a single document + +More specific technical details will only become important if they affect the specifics of +your application design or implementation. + +\section sec_unicode_support How and Where Is UTF-8 Supported in HDF5? +HDF5 uses characters in object names (which are actually link names, but that's a story for a +different article), dataset raw data, attribute names, and attribute raw data. Though the +mechanisms differ, you can use either ASCII or UTF-8 character sets in all of these situations. + +\subsection subsec_unicode_support_names Object and Attribute Names +By default, HDF5 creates object and attribute names with ASCII character encoding. An object or +attribute creation property list setting is required to create object names with UTF-8 characters. +This uses the function #H5Pset_char_encoding, which sets the character encoding used for object and attribute names. + +For example, the following call sequence could be used to create a dataset with its name encoded with the UTF-8 character set: + +\code + lcpl_id = H5Pcreate(H5P_LINK_CREATE) ; + error = H5Pset_char_encoding(lcpl_id, H5T_CSET_UTF8) ; + dataset_id = H5Dcreate2(group_id, "datos_ñ", datatype_id, dataspace_id, + lcpl_id, H5P_DEFAULT, H5P_DEFAULT) ; +\endcode + +If the character encoding of an attribute name is unknown, the combination of an +#H5Aget_create_plist call and an #H5Pget_char_encoding call will reveal that information. +If the character encoding of an object name is unknown, the information can be accessed +through the object's H5L_info_t structure which can be obtained using #H5Lvisit or #H5Lget_info_by_idx calls. + +\subsection subsec_unicode_support_char Character Datatypes in Datasets and Attributes +Like object names, HDF5 character data in datasets and attributes is encoded as ASCII by +default. Setting up attribute or dataset character data to be UTF-8-encoded is accomplished +while defining the attribute or dataset datatype. This makes use of the function #H5Tset_cset, +which sets the character encoding to be used in building a character datatype. + +For example, the following commands could be used to create an 8-character, UTF-8 encoded, +string datatype for use in either an attribute or dataset: + +\code + datatype_id = H5Tcopy(H5T_C_S1); + error = H5Tset_cset(datatype_id, H5T_CSET_UTF8); + error = H5Tset_size(datatype_id, "8"); +\endcode + +If a character or string datatype's character encoding is unknown, the combination of an +#H5Aget_type or #H5Dget_type call and an #H5Tget_cset call can be used to determine that. + +\section sec_unicode_warn Caveats, Pitfalls, and Things to Watch For +Programmers who are accustomed to using ASCII text without accommodating other text +encodings will have to be aware of certain common issues as they begin using UTF-8 encodings. + +\subsection subsec_unicode_warn_port Cross-platform Portability +Since the HDF5 Library handles datatypes directly, UTF-8 encoded text in dataset and +attribute datatypes in a well-designed HDF5 application and file should work transparently +across platforms. The same should be true of handling names of groups, datasets, committed +datatypes, and attributes within a file. + +Be aware, however, of system or application limitations once data or other information +has been extracted from an HDF5 file. The application or system must be designed to +accommodate UTF-8 encodings if the information is then used elsewhere in the application or system environment. + +Data from a UTF-8 encoded HDF5 datatype, in either a dataset or an attribute, +that has been established within an HDF5 application should "just work" within the HDF5 portions of the application. + +\subsection subsec_unicode_warn_names Filenames +Since file access is a system issue, filenames do not fall within the scope +of HDF5's UTF-8 capabilities; filenames are encoded at the system level. + +Linux and Mac OS systems normally handle UTF-8 encoded filenames correctly +while Windows systems generally do not. + +\section sec_unicode_text The *Plain Text* Illusion +Beware the use of the term *plain text*. *Plain text* is at best ambiguous, but often +misleading. Many will assume that *plain text* means ASCII, but plain text German or +French, for example, cannot be represented in ASCII. Plain text is only unambiguous +in the context of English (and even then can be problematic!). + +\subsection subsec_unicode_warn_store Storage Size +Programmers and data users accustomed to working strictly with ASCII data generally make +the reasonable assumption that 1 character, be it in an object name or in data, requires +1 byte of storage. This equation does not work when using UTF-8 or any other Unicode encoding. +With Unicode encoding, number of characters is not synonymous with number of bytes. One must +get used to thinking in terms of number of characters when talking about content, reserving +number of bytes for discussions of storage size. + +When working with Unicode text, one can no longer assume a 1:1 correspondence between the +number of characters and the data storage requirement. + +\subsection subsec_unicode_warn_sys System Dependencies +Linux, Unix, and similar systems generally handle UTF-8 encodings in correct and +predictable ways. There is an apparent consensus in the Linux community that "UTF-8 is just the right way to go." + +Mac OS systems generally handle UTF-8 encodings correctly. + +Windows systems use a different Unicode encoding, UCS-2 +(discussed in this UTF-16 article) at +the system level. Within an HDF5 file and application on a Windows system, UTF-8 encoding should +work correctly and as expected. Problems may arise, however, when that UTF-8 encoding is exposed +directly to the Windows system. For example: +\li File open and close calls on files with UTF-8 encoded names are likely to fail as the HDF5 +open and close operations interact directly with the Windows file system interface. +\li Anytime an HDF5 command-line utility (\ref H5TOOL_LS_UG or \ref H5TOOL_DP_UG, for example) emits text output, the +Windows system must interpret the character encodings. If that output is UTF-8 encoded, Windows +will correctly interpret only those characters in the ASCII subset of UTF-8. + +\section sec_unicode_common Common Characters in UTF-8 and ASCII +One interesting feature of UTF-8 and ASCII is that the ASCII character set is a discrete subset of +the UTF-8 character set. And where they overlap, the encodings are identical. This means that a +character string consisting entirely of members of the ASCII character set can be encoded in either +ASCII or UTF-8, the two encodings will be indistinguishable, and the encodings will require exactly the same storage space. + + +\section sec_unicode_also See Also + +- For object and attribute names: + * #H5Pset_char_encoding + * #H5Pget_char_encoding +- For dataset and attribute datatypes: + * #H5Tset_cset + * #H5Tget_cset +- [UTF-8 article on Wikipedia](https://en.wikipedia.org/wiki/UTF-8) + +

NOTES

+1. UTF-8 is the only Unicode standard encoding supported in HDF5. + +*/ diff --git a/doxygen/dox/UsersGuide.dox b/doxygen/dox/UsersGuide.dox index 03151473d2a..14481a10f69 100644 --- a/doxygen/dox/UsersGuide.dox +++ b/doxygen/dox/UsersGuide.dox @@ -335,6 +335,23 @@ \ref sec_map +\section sec_hl HDF5 High Level APIs +\li \ref H5DO_UG +
    +
  • \ref sec_hldo_direct_chunk +
+\li \ref H5DS_UG +
    +
  • \ref sec_dim_scales_stand +
  • \ref sec_dim_scales_concept +
  • \ref sec_dim_scales_spec +
  • \ref sec_dim_scales_api +
+\li \ref H5IM_UG +\li \ref H5LT_UG +\li \ref H5TB_UG +\li \ref H5PT_UG + \ref sec_cltools \li \ref H5TOOL_CP_UG \li \ref H5TOOL_DF_UG diff --git a/doxygen/dox/VDSTechNote.dox b/doxygen/dox/VDSTechNote.dox new file mode 100644 index 00000000000..9bb6786a132 --- /dev/null +++ b/doxygen/dox/VDSTechNote.dox @@ -0,0 +1,115 @@ + +/** \page VDSTN Introduction to the Virtual Dataset - VDS + +\section sec_vds_intro Introduction to VDS +The HDF5 Virtual Dataset (VDS) feature enables users to access data in a collection of HDF5 files as a +single HDF5 dataset and to use the HDF5 APIs to work with that dataset. + +For example, your data may be collected into four files: +tutrvds-multimgs.png + +You can map the datasets in the four files into a single VDS that can be accessed just like any other dataset: +tutrvds-snglimg.png + +The mapping between a VDS and the HDF5 source datasets is persistent and transparent to an application. If a source +file is missing the fill value will be displayed. + +See the Virtual (VDS) Documentation for complete details regarding the VDS feature. + +The VDS feature was implemented using hyperslab selection (#H5Sselect_hyperslab). See the tutorial on +Reading From or Writing to a Subset of a Dataset for more information on selecting hyperslabs. + +\subsection subsec_vds_intro_model Programming Model +To create a Virtual Dataset you simply follow the HDF5 programming model and add a few additional API calls +to map the source code datasets to the VDS. + +Following are the steps for creating a Virtual Dataset: +\li Create the source datasets that will comprise the VDS +\li Create the VDS: ‐ Define a datatype and dataspace (can be unlimited) +\li Define the dataset creation property list (including fill value) +\li (Repeat for each source dataset) Map elements from the source dataset to elements of the VDS +\li Select elements in the source dataset (source selection) +\li Select elements in the virtual dataset (destination selection) +\li Map destination selections to source selections (see Functions for Working with a VDS) +\li Call H5Dcreate using the properties defined above +\li Access the VDS as a regular HDF5 dataset +\li Close the VDS when finished + +

Functions for Working with a VDS

+The #H5Pset_virtual API sets the mapping between virtual and source datasets. This is a dataset creation property list. +Using this API will change the layout of the dataset to #H5D_VIRTUAL. As with specifying any dataset creation property +list, an instance of the property list is created, modified, passed into the dataset creation call and then closed: +\code + dcpl = H5Pcreate (H5P_DATASET_CREATE); + src_space = H5screate_simple ... + status = H5Sselect_hyperslab (space, ... + status = H5Pset_virtual (dcpl, space, SRC_FILE[i], SRC_DATASET[i], src_space); + dset = H5Dcreate2 (file, DATASET, H5T_NATIVE_INT, space, H5P_DEFAULT, dcpl, H5P_DEFAULT); + status = H5Pclose (dcpl); +\endcode + +There are several other APIs introduced with Virtual Datasets, including query functions. For details +see the complete list of HDF5 library APIs that support Virtual Datasets. + +

Limitations

+This feature was introduced in HDF5-1.10. + +The number of source datasets is unlimited. However, there is a limit on the size of each source dataset. + +\subsection subsec_vds_intro_examples Programming Examples +Example 1 +This example creates three HDF5 files, each with a one-dimensional dataset of 6 elements. The datasets in these files +are the source datasets that are then used to create a 4 x 6 Virtual Dataset with a fill value of -1. The first three +rows of the VDS are mapped to the data from the three source datasets as shown below: +tutrvds-ex.png + +In this example the three source datasets are mapped to the VDS with this code: +\code> +src_space = H5Screate_simple (RANK1, dims, NULL); +for (i = 0; i < 3; i++) { + start[0] = (hsize_t)i; + // Select i-th row in the virtual dataset; selection in the source datasets is the same. + status = H5Sselect_hyperslab (space, H5S_SELECT_SET, start, NULL, count, block); + status = H5Pset_virtual (dcpl, space, SRC_FILE[i], SRC_DATASET[i], src_space); +} +endcode> + +After the VDS is created and closed, it is reopened. The property list is then queried to determine the +layout of the dataset and its mappings, and the data in the VDS is read and printed. + +This example is in the HDF5 source code and can be obtained from here: +

C Example

+For details on compiling an HDF5 application: [ Compiling HDF5 Applications ] + +

Example 2

+This example shows how to use a C-style printf statement for specifying multiple source datasets as one virtual +dataset. Only one mapping is required. In other words only one #H5Pset_virtual call is needed to map multiple datasets. +It creates a 2-dimensional unlimited VDS. Then it re-opens the file, makes queries, and reads the virtual dataset. + +The source datasets are specified as A-0, A-1, A-2, and A-3. These are mapped to the virtual dataset with one call: +\code +status = H5Pset_virtual (dcpl, vspace, SRCFILE, "A-%b", src_space); +\endcode + +The %b indicates that the block count of the selection in the dimension should be used. + +

C Example

+For details on compiling an HDF5 application: [ Compiling HDF5 Applications ] + +Using h5dump with a VDS +The h5dump utility can be used to view a VDS. The h5dump output for a VDS looks exactly like that for any other dataset. +If h5dump cannot find a source dataset then the fill value will be displayed. + +You can determine that a dataset is a VDS by looking at its properties with +\code + h5dump -p +\endcode + It will display each source dataset mapping, beginning with Mapping 0. Below is an excerpt of the output of +\code + h5dump -p +\endcode +on the vds.h5 file created in Example 1.You can see that the entire source file a.h5 is mapped to the first row of the VDS dataset. + +tutrvds-map.png + +*/ diff --git a/doxygen/dox/VFLTechNote.dox b/doxygen/dox/VFLTechNote.dox new file mode 100644 index 00000000000..535001a2eb5 --- /dev/null +++ b/doxygen/dox/VFLTechNote.dox @@ -0,0 +1,1025 @@ + +/** \page VFLTN HDF5 Virtual File Layer + +\section sec_vfl_intro Introduction +The HDF5 file format describes how HDF5 data structures and dataset raw data are mapped +to a linear format address space and the HDF5 library implements that bidirectional mapping +in terms of an API. However, the HDF5 format specifications do not indicate how the format +address space is mapped onto storage and HDF (version 5 and earlier) simply mapped the format +address space directly onto a single file by convention. + +Since early versions of HDF5 it became apparent that users want the ability to map the +format address space onto different types of storage (a single file, multiple files, local +memory, global memory, network distributed global memory, a network protocol, etc.) with +various types of maps. For instance, some users want to be able to handle very large format +address spaces on operating systems that support only 2GB files by partitioning the format +address space into equal-sized parts each served by a separate file. Other users want the +same multi-file storage capability but want to partition the address space according to +purpose (raw data in one file, object headers in another, global heap in a third, etc.) +in order to improve I/O speeds. + +In fact, the number of storage variations is probably larger than the number of methods +that the HDF5 team is capable of implementing and supporting. Therefore, a Virtual File +Layer API is being implemented which will allow application teams or departments to design +and implement their own mapping between the HDF5 format address space and storage, with each +mapping being a separate file driver (possibly written in terms of other file drivers). The +HDF5 team will provide a small set of useful file drivers which will also serve as examples +for those who which to write their own: + + + + + + + + + + + + + + + + +
#H5FD_SEC2This is the default driver which uses Posix file-system functions +like read and write to perform I/O to a single file. All I/O requests are unbuffered +although the driver does optimize file seeking operations to some extent. +
#H5FD_STDIOThis driver uses functions from 'stdio.h' to perform buffered I/O to a single file. +
#H5FD_COREThis driver performs I/O directly to memory and can be +used to create small temporary files that never exist on permanent storage. This +type of storage is generally very fast since the I/O consists only of memory-to-memory copy operations. +
#H5FD_MPIOThis is the driver of choice for accessing files in parallel +using MPI and MPI-IO. It is only predefined if the library is compiled with parallel I/O support. +
#H5FD_FAMILYLarge format address spaces are partitioned into more +manageable pieces and sent to separate storage locations using an underlying driver +of the user's choice. \ref H5TOOL_RT_UG can be used to change the sizes of the family +members when stored as files or to convert a family of files to a single file or vice versa. +
+ +\section sec_vfl_use Using a File Driver +Most application writers will use a driver defined by the HDF5 library or contributed by another +programming team. This chapter describes how existing drivers are used. + +\subsection subsec_vfl_use_hdr Driver Header Files +Each file driver is defined in its own public header file which should be included by any +application which plans to use that driver. The predefined drivers are in header files whose +names begin with 'H5FD' followed by the driver name and '.h'. The 'hdf5.h' header file includes +all the predefined driver header files. + +Once the appropriate header file is included a symbol of the form 'H5FD_' followed by the +upper-case driver name will be the driver identification number.(The driver name is by convention +and might not apply to drivers which are not distributed with HDF5.) However, the value may +change if the library is closed (e.g., by calling #H5close) and the symbol is referenced again. + +\subsection subsec_vfl_use_create Creating and Opening Files +In order to create or open a file one must define the method by which the storage is +accessed(The access method also indicates how to translate the storage name to a storage server +such as a file, network protocol, or memory.) and does so by creating a file access property +list(The term "file access property list" is a misnomer since storage isn't required to be a file.) +which is passed to the #H5Fcreate or #H5Fopen function. A default file access property list is created +by calling #H5Pcreate and then the file driver information is inserted by calling a driver initialization +function such as #H5Pset_fapl_family: +\code +hid_t fapl = H5Pcreate(H5P_FILE_ACCESS); +size_t member_size = 100*1024*1024; /*100MB*/ +H5Pset_fapl_family(fapl, member_size, H5P_DEFAULT); +hid_t file = H5Fcreate("foo%05d.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl); +H5Pclose(fapl); +\endcode + +Each file driver will have its own initialization function whose name is H5Pset_fapl_ followed by +the driver name and which takes a file access property list as the first argument followed by additional +driver-dependent arguments. + +An alternative to using the driver initialization function is to set the driver directly using the +#H5Pset_driver function.(This function is overloaded to operate on data transfer property lists also, as described below.) +Its second argument is the file driver identifier, which may have a different numeric value from run to run +depending on the order in which the file drivers are registered with the library. The third argument encapsulates +the additional arguments of the driver initialization function. This method only works if the file driver +writer has made the driver-specific property list structure a public datatype, which is often not the case. +\code +hid_t fapl = H5Pcreate(H5P_FILE_ACCESS); +static H5FD_family_fapl_t fa = {100*1024*1024, H5P_DEFAULT}; +H5Pset_driver(fapl, H5FD_FAMILY, &fa); +hid_t file = H5Fcreate("foo.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl); +H5Pclose(fapl); +\endcode + +It is also possible to query the file driver information from a file access property list by +calling #H5Pget_driver to determine the driver and then calling a driver-defined query function +to obtain the driver information: +\code +hid_t driver = H5Pget_driver(fapl); +if (H5FD_SEC2==driver) { + /*nothing further to get*/ +} else if (H5FD_FAMILY==driver) { + hid_t member_fapl; + haddr_t member_size; + H5Pget_fapl_family(fapl, &member_size, &member_fapl); +} else if (....) { + .... +} +\endcode + +\subsection subsec_vfl_use_per Performing I/O +The #H5Dread and #H5Dwrite functions transfer data between application memory and the file. They both take +an optional data transfer property list which has some general driver-independent properties and optional +driver-defined properties. An application will typically perform I/O in one of three styles via the +#H5Dread or #H5Dwrite function: + +Like file access properties in the previous section, data transfer properties can be set using a driver +initialization function or a general purpose function. For example, to set the MPI-IO driver to use +independent access for I/O operations one would say: +\code +hid_t dxpl = H5Pcreate(H5P_DATA_XFER); +H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); +H5Dread(dataset, type, mspace, fspace, buffer, dxpl); +H5Pclose(dxpl); +\endcode + +The alternative is to initialize a driver defined C struct and pass it to the #H5Pset_driver function: +\code +hid_t dxpl = H5Pcreate(H5P_DATA_XFER); +static H5FD_mpio_dxpl_t dx = {H5FD_MPIO_INDEPENDENT}; +H5Pset_driver(dxpl, H5FD_MPIO, &dx); +H5Dread(dataset, type, mspace, fspace, buffer, dxpl); +\endcode + +The transfer property list can be queried in a manner similar to the file access property list: the driver +provides a function (or functions) to return various information about the transfer property list: +\code +hid_t driver = H5Pget_driver(dxpl); +if (H5FD_MPIO==driver) { + H5FD_mpio_xfer_t xfer_mode; + H5Pget_dxpl_mpio(dxpl, &xfer_mode); +} else { + .... +} +\endcode + +\subsection subsec_vfl_use_inter File Driver Interchangeability +The HDF5 specifications describe two things: the mapping of data onto a linear format address +space and the C API which performs the mapping. However, the mapping of the format address space +onto storage intentionally falls outside the scope of the HDF5 specs. This is a direct result of the +fact that it is not generally possible to store information about how to access storage inside the +storage itself. For instance, given only the file name '/arborea/1225/work/f%03d' the HDF5 library +is unable to tell whether the name refers to a file on the local file system, a family of files on +the local file system, a file on host 'arborea' port 1225, a family of files on a remote system, etc. + +Two ways which library could figure out where the storage is located are: storage access information +can be provided by the user, or the library can try all known file access methods. This implementation +uses the former method. + +In general, if a file was created with one driver then it isn't possible to open it with another driver. +There are of course exceptions: a file created with MPIO could probably be opened with the sec2 driver, +any file created by the sec2 driver could be opened as a family of files with one member, etc. In fact, +sometimes a file must not only be opened with the same driver but also with the same driver properties. +The predefined drivers are written in such a way that specifying the correct driver is sufficient for +opening a file. + +\section sec_vfl_imp Implementation of a Driver +A driver is simply a collection of functions and data structures which are registered with the HDF5 +library at runtime. The functions fall into these categories: +\li Functions which operate on modes +\li Functions which operate on files +\li Functions which operate on the address space +\li Functions which operate on data +\li Functions for driver initialization +\li Optimization functions + +\subsection subsec_vfl_imp_mode Mode Functions +Some drivers need information about file access and data transfers which are very specific to the driver. +The information is usually implemented as a pair of pointers to C structs which are allocated and +initialized as part of an HDF5 property list and passed down to various driver functions. There are two +classes of settings: file access modes that describe how to access the file through the driver, and +data transfer modes which are settings that control I/O operations. Each file opened by a particular +driver may have a different access mode; each dataset I/O request for a particular file may have a +different data transfer mode. + +Since each driver has its own particular requirements for various settings, each driver is responsible +for defining the mode structures that it needs. Higher layers of the library treat the structures as +opaque but must be able to copy and free them. Thus, the driver provides either the size of the +structure or a pair of function pointers for each of the mode types. + +Example: The family driver needs to know how the format address space is partitioned and the file +access property list to use for the family members. +\code +// Driver-specific file access properties +typedef struct H5FD_family_fapl_t { + hsize_t memb_size; // size of each family member + hid_t memb_fapl; // file access property list for each family member +} H5FD_family_fapl_t; + +// Driver specific data transfer properties +typedef struct H5FD_family_dxpl_t { + hid_t memb_dxpl_id; //data xfer property list of each member +} H5FD_family_dxpl_t; +\endcode +n order to copy or free one of these structures the member file access or data transfer properties must +also be copied or freed. This is done by providing a copy and close function for each structure: + +Example: The file access property list copy and close functions for the family driver: +\code +static void * +H5FD_family_fapl_copy(const void *_old_fa) +{ + const H5FD_family_fapl_t *old_fa = (const H5FD_family_fapl_t*)_old_fa; + H5FD_family_fapl_t *new_fa = malloc(sizeof(H5FD_family_fapl_t)); + assert(new_fa); + + memcpy(new_fa, old_fa, sizeof(H5FD_family_fapl_t)); + new_fa->memb_fapl_id = H5Pcopy(old_fa->memb_fapl_id); + return new_fa; +} + +static herr_t +H5FD_family_fapl_free(void *_fa) +{ + H5FD_family_fapl_t *fa = (H5FD_family_fapl_t*)_fa; + H5Pclose(fa->memb_fapl_id); + free(fa); + return 0; +} +\endcode + +Generally when a file is created or opened the file access properties for the driver are copied into the +file pointer which is returned and they may be modified from their original value (for instance, the file +family driver modifies the member size property when opening an existing family). In order to support the +#H5Fget_access_plist function the driver must provide a fapl_get callback which creates a copy of the +driver-specific properties based on a particular file. + +Example: The file family driver copies the member size file access property list into the return value: +\code +static void * +H5FD_family_fapl_get(H5FD_t *_file) +{ + H5FD_family_t *file = (H5FD_family_t*)_file; + H5FD_family_fapl_t *fa = calloc(1, sizeof(H5FD_family_fapl_t*)); + + fa->memb_size = file->memb_size; + fa->memb_fapl_id = H5Pcopy(file->memb_fapl_id); + return fa; +} +\endcode + +\subsection subsec_vfl_imp_file File Functions +The higher layers of the library expect files to have a name and allow the file to be accessed in various modes. +The driver must be able to create a new file, replace an existing file, or open an existing file. Opening or +creating a file should return a handle, a pointer to a specialization of the H5FD_t struct, which allows read-only +or read-write access and which will be passed to the other driver functions as they are called.(Read-only access is +only appropriate when opening an existing file.) +\code +typedef struct { + // Public fields + H5FD_class_t *cls; //class data defined below + + // Private fields -- driver-defined + +} H5FD_t; +\endcode + +Example: The family driver requires handles to the underlying storage, the size of the members for this +particular file (which might be different than the member size specified in the file access property list +if an existing file family is being opened), the name used to open the file in case additional members +must be created, and the flags to use for creating those additional members. The eoa member caches the +size of the format address space so the family members don't have to be queried in order to find it. +\code +// The description of a file belonging to this driver. +typedef struct H5FD_family_t { + H5FD_t pub; // public stuff, must be first + hid_t memb_fapl_id; // file access property list for members + hsize_t memb_size; // maximum size of each member file + int nmembs; // number of family members + int amembs; // number of member slots allocated + H5FD_t **memb; // dynamic array of member pointers + haddr_t eoa; // end of allocated addresses + char *name; // name generator printf format + unsigned flags; // flags for opening additional members +} H5FD_family_t; +\endcode + +Example: The sec2 driver needs to keep track of the underlying Unix file descriptor and also the +end of format address space and current Unix file size. It also keeps track of the current file +position and last operation (read, write, or unknown) in order to optimize calls to lseek. The +device and inode fields are defined on Unix in order to uniquely identify the file and will be +discussed below. +\code +typedef struct H5FD_sec2_t { + H5FD_t pub; // public stuff, must be first + int fd; // the unix file + haddr_t eoa; // end of allocated region + haddr_t eof; // end of file; current file size + haddr_t pos; // current file I/O position + int op; // last operation + dev_t device; // file device number + ino_t inode; // file i-node number +} H5FD_sec2_t; +\endcode + +\subsection subsec_vfl_imp_open Open Files +All drivers must define a function for opening/creating a file. This function should have a prototype which is: + + + + + +
static H5FD_t * open (const char *name, unsigned flags, hid_t fapl, haddr_t maxaddr)The file name name and file access property list fapl are the same as were specified in the #H5Fcreate +or #H5Fopen call. The flags are the same as in those calls also except the flag #H5F_ACC_CREAT is also +present if the call was to H5Fcreate and they are documented in the 'H5Fpublic.h' file. The maxaddr +argument is the maximum format address that the driver should be prepared to handle (the minimum address is always zero).
+ +Example: The sec2 driver opens a Unix file with the requested name and saves information which +uniquely identifies the file (the Unix device number and inode). +\code +static H5FD_t * +H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id/*unused*/, + haddr_t maxaddr) +{ + unsigned o_flags; + int fd; + struct stat sb; + H5FD_sec2_t *file=NULL; + + // Check arguments + if (!name || !*name) return NULL; + if (0==maxaddr || HADDR_UNDEF==maxaddr) return NULL; + if (ADDR_OVERFLOW(maxaddr)) return NULL; + + // Build the open flags + o_flags = (H5F_ACC_RDWR & flags) ? O_RDWR : O_RDONLY; + if (H5F_ACC_TRUNC & flags) o_flags |= O_TRUNC; + if (H5F_ACC_CREAT & flags) o_flags |= O_CREAT; + if (H5F_ACC_EXCL & flags) o_flags |= O_EXCL; + + // Open the file + if ((fd=open(name, o_flags, 0666))<0) return NULL; + if (fstat(fd, &sb)<0) { + close(fd); + return NULL; + } + + // Create the new file struct + file = calloc(1, sizeof(H5FD_sec2_t)); + file->fd = fd; + file->eof = sb.st_size; + file->pos = HADDR_UNDEF; + file->op = OP_UNKNOWN; + file->device = sb.st_dev; + file->inode = sb.st_ino; + + return (H5FD_t*)file; +} +\endcode + +\subsection subsec_vfl_imp_close Closing Files +Closing a file simply means that all cached data should be flushed to the next lower layer, the +file should be closed at the next lower layer, and all file-related data structures should be +freed. All information needed by the close function is already present in the file handle. + + + + + +
static herr_t close (H5FD_t *file)The file argument is the handle which was returned by the open function, and the close should +free only memory associated with the driver-specific part of the handle (the public parts will +have already been released by HDF5's virtual file layer).
+ +Example: The sec2 driver just closes the underlying Unix file, making sure that the actual +file size is the same as that known to the library by writing a zero to the last file position +it hasn't been written by some previous operation (which happens in the same code which flushes +the file contents and is shown below). +\code +static herr_t +H5FD_sec2_close(H5FD_t *_file) +{ + H5FD_sec2_t *file = (H5FD_sec2_t*)_file; + + if (H5FD_sec2_flush(_file)<0) return -1; + if (close(file->fd)<0) return -1; + free(file); + return 0; +} +\endcode + +\subsection subsec_vfl_imp_key File Keys +Occasionally an application will attempt to open a single file more than one time in order +to obtain multiple handles to the file. HDF5 allows the files to share information(For instance, +writing data to one handle will cause the data to be immediately visible on the other handle.) +but in order to accomplish this HDF5 must be able to tell when two names refer to the same file. +It does this by associating a driver-defined key with each file opened by a driver and comparing +the key for an open request with the keys for all other files currently open by the same driver. + + + + + +
const int cmp (const H5FD_t *f1, const H5FD_t *f2)The driver may provide a function which compares two files f1 and f2 belonging to the same +driver and returns a negative, positive, or zero value a la the strcmp function.(The ordering +is arbitrary as long as it's consistent within a particular file driver.) If this function is +not provided then HDF5 assumes that all calls to the open callback return unique files regardless +of the arguments and it is up to the application to avoid doing this if that assumption is incorrect.
+ +Each time a file is opened the library calls the cmp function to compare that file with all other files +currently open by the same driver and if one of them matches (at most one can match) then the file +which was just opened is closed and the previously opened file is used instead. + +Opening a file twice with incompatible flags will result in failure. For instance, opening a file with +the truncate flag is a two step process which first opens the file without truncation so keys can be +compared, and if no matching file is found already open then the file is closed and immediately reopened +with the truncation flag set (if a matching file is already open then the truncating open will fail). + +Example: The sec2 driver uses the Unix device and i-node as the key. They were initialized when +the file was opened. +\code +static int +H5FD_sec2_cmp(const H5FD_t *_f1, const H5FD_t *_f2) +{ + const H5FD_sec2_t *f1 = (const H5FD_sec2_t*)_f1; + const H5FD_sec2_t *f2 = (const H5FD_sec2_t*)_f2; + + if (f1->device < f2->device) return -1; + if (f1->device > f2->device) return 1; + + if (f1->inode < f2->inode) return -1; + if (f1->inode > f2->inode) return 1; + + return 0; +} +\endcode + +\subsection subsec_vfl_imp_save Saving Modes Across Opens +Some drivers may also need to store certain information in the file superblock in order +to be able to reliably open the file at a later date. This is done by three functions: +one to determine how much space will be necessary to store the information in the superblock, +one to encode the information, +and one to decode the information. These functions are optional, but if any one is defined +then the other two must also be defined. + + + + + + + + + + + + + + + + + +
FunctionDescription
static hsize_t sb_size (H5FD_t *file)The sb_size function returns the number of bytes necessary to encode +information needed later if the file is reopened.
static herr_t sb_encode (H5FD_t *file, char *name, unsigned char *buf)The sb_encode function encodes information from the file into buffer buf +allocated by the caller. It also writes an 8-character (plus null termination) into +the name argument, which should be a unique identification for the driver.
static herr_t sb_decode (H5FD_t *file, const char *name, const unsigned char *buf)The sb_decode function looks at the name decodes data from the buffer buf and +updates the file argument with the new information, advancing *p in the process.
+The part of this which is somewhat tricky is that the file must be readable before the +superblock information is decoded. File access modes fall outside the scope of the HDF5 +file format, but they are placed inside the boot block for convenience.(File access modes +do not describe data, but rather describe how the HDF5 format address space is mapped to +the underlying file(s). Thus, in general the mapping must be known before the file +superblock can be read. However, the user usually knows enough about the mapping for +the superblock to be readable and once the superblock is read the library can fill +in the missing parts of the mapping.) + +\section sec_vfl_address Address Space Functions +HDF5 does not assume that a file is a linear address space of bytes. Instead, the library +will call functions to allocate and free portions of the HDF5 format address space, which +in turn map onto functions in the file driver to allocate and free portions of file address +space. The library tells the file driver how much format address space it wants to allocate +and the driver decides what format address to use and how that format address is mapped +onto the file address space. Usually the format address is chosen so that the file address +can be calculated in constant time for data I/O operations (which are always specified by format addresses). + +\subsection subsec_vfl_address_blk Userblock and Superblock +The HDF5 format allows an optional userblock to appear before the actual HDF5 data in such +a way that if the userblock is sucked out of the file and everything remaining is +shifted downward in the file address space, then the file is still a valid HDF5 file. +The userblock size can be zero or any multiple of two greater than or equal to 512 and +the file superblock begins immediately after the userblock. + +HDF5 allocates space for the userblock and superblock by calling an allocation function +defined below, which must return a chunk of memory at format address zero on the first call. + +\subsection subsec_vfl_address_alloc Allocatiion of Format Regions +The library makes many types of allocation requests: + + + + + + + + + + + + + + + + + + + + +
#H5FD_MEM_SUPERuserblock
#H5FD_MEM_BTREEAn allocation request for a node of a B-tree. +
#H5FD_MEM_DRAWAn allocation request for the raw data of a dataset. +
#H5FD_MEM_GHEAPAn allocation request for a global heap collection. Global +heaps are used to store certain types of references such as dataset region references. +The set of all global heap collections can become quite large. +
#H5FD_MEM_LHEAPAn allocation request for a local heap. Local heaps are used +to store the names which are members of a group. The combined size of all local heaps is +a function of the number of object names in the file. +
#H5FD_MEM_OHDRAn allocation request for (part of) an object header. Object +headers are relatively small and include meta information about objects (like the data +space and type of a dataset) and attributes. +
+ +When a chunk of memory is freed the library adds it to a free list and allocation requests +are satisfied from the free list before requesting memory from the file driver. Each type of +allocation request enumerated above has its own free list, but the file driver can specify that +certain object types can share a free list. It does so by providing an array which maps a +request type to a free list. If any value of the map is H5MF_DEFAULT (zero) then the object's +own free list is used. The special value H5MF_NOLIST indicates that the library should not +attempt to maintain a free list for that particular object type, instead calling the file driver +each time an object of that type is freed. + +Mappings predefined in the 'H5FDdevelop.h' file are: + + + + + + + + + + +
#H5FD_FLMAP_SINGLEAll memory usage types are mapped to a single free list. +
#H5FD_FLMAP_DICHOTOMYMemory usage is segregated into meta data and raw data +for the purposes of memory management. +
#H5FD_FLMAP_DEFAULTEach memory usage type has its own free list. +
+ +Example: To make a map that manages object headers on one free list and everything else on +another free list one might initialize the map with the following code: (the use of #H5FD_MEM_SUPER is arbitrary) +\code +H5FD_mem_t mt, map[H5FD_MEM_NTYPES]; + +for (mt = 0; mt < H5FD_MEM_NTYPES; mt++) { + map[mt] = (H5FD_MEM_OHDR== mt) ? mt : H5FD_MEM_SUPER; +} +\endcode + +If an allocation request cannot be satisfied from the free list then one of two things happen. +If the driver defines an allocation callback then it is used to allocate space; otherwise new +memory is allocated from the end of the format address space by incrementing the end-of-address marker. + + + + + +
static haddr_t alloc (H5FD_t *file, H5MF_type_t type, hsize_t size)The file argument is the file from which space is to be allocated, type is the type of +memory being requested (from the list above) without being mapped according to the freelist +map and size is the number of bytes being requested. The library is allowed to allocate large +chunks of storage and manage them in a layer above the file driver (although the current library +doesn't do that). The allocation function should return a format address for the first byte +allocated. The allocated region extends from that address for size bytes. If the request cannot +be honored then the undefined address value is returned (#HADDR_UNDEF). The first call to this +function for a file which has never had memory allocated must return a format address of zero +or #HADDR_UNDEF since this is how the library allocates space for the userblock and/or superblock.
+ +\subsection subsec_vfl_address_free Freeing Format Regions +When the library is finished using a certain region of the format address space it will return the +space to the free list according to the type of memory being freed and the free list map described above. +If the free list has been disabled for a particular memory usage type (according to the free list map) +and the driver defines a free callback then it will be invoked. The free callback is also invoked for +all entries on the free list when the file is closed. + + + + + + +
static herr_t free (H5FD_t *file, H5MF_type_t type, haddr_t addr, hsize_t size)The file argument is the file for which space is being freed; type is the type of object being +freed (from the list above) without being mapped according to the freelist map; addr is the first +format address to free; and size is the size in bytes of the region being freed. The region being +freed may refer to just part of the region originally allocated and/or may cross allocation boundaries +provided all regions being freed have the same usage type. However, the library will never attempt +to free regions which have already been freed or which have never been allocated.
+A driver may choose to not define the free function, in which case format addresses will be leaked. +This isn't normally a huge problem since the library contains a simple free list of its own and freeing +parts of the format address space is not a common occurrence. + +\subsection subsec_vfl_address_query Querying the Address Range +Each file driver must have some mechanism for setting and querying the end of address, or +EOA, marker. The EOA marker is the first format address after the last format address ever allocated. +If the last part of the allocated address range is freed then the driver may optionally decrease the eoa marker. + + + + + +
static haddr_t get_eoa (H5FD_t *file)This function returns the current value of the EOA marker for the specified file.
+ +Example: The sec2 driver just returns the current eoa marker value which is cached in the file structure: +\code +static haddr_t +H5FD_sec2_get_eoa(H5FD_t *_file) +{ + H5FD_sec2_t *file = (H5FD_sec2_t*)_file; + return file->eoa; +} +\endcode + +The eoa marker is initially zero when a file is opened and the library may set it to some other value +shortly after the file is opened (after the superblock is read and the saved eoa marker is determined) +or when allocating additional memory in the absence of an alloc callback (described above). + +Example: The sec2 driver simply caches the eoa marker in the file structure and does not extend the +underlying Unix file. When the file is flushed or closed then the Unix file size is extended to match +the eoa marker. +\code +static herr_t +H5FD_sec2_set_eoa(H5FD_t *_file, haddr_t addr) +{ + H5FD_sec2_t *file = (H5FD_sec2_t*)_file; + file->eoa = addr; + return 0; +} +\endcode + +\section sec_vfl_data Data Functions +These functions operate on data, transferring a region of the format address space between memory and files. + +\subsection subsec_vfl_data_cont Contiguous I/O Functions +A driver must specify two functions to transfer data from the library to the file and vice versa. + + + + + + + + + +
static herr_t read (H5FD_t *file, H5FD_mem_t type, hid_t dxpl, haddr_t addr, hsize_t size, void *buf)The read function reads data from file beginning at address addr and continuing +for size bytes into the buffer buf supplied by the caller.
static herr_t write (H5FD_t *file, H5FD_mem_t type, hid_t dxpl, haddr_t addr, hsize_t size, const void *buf)The write function transfers data +in the opposite direction.
+\li Both functions take a data transfer property list dxpl which +indicates the fine points of how the data is to be transferred and which comes directly +from the #H5Dread or #H5Dwrite function. +\li Both functions receive type of data being written, +which may allow a driver to tune it's behavior for different kinds of data. +\li Both functions should return +a negative value if they fail to transfer the requested data, or non-negative if they +succeed. The library will never attempt to read from unallocated regions of the format address space. + +Example: The sec2 driver just makes system calls. It tries not to call lseek if the current operation +is the same as the previous operation and the file position is correct. It also fills the output buffer +with zeros when reading between the current EOF and EOA markers and restarts system calls which were interrupted. +\code +static herr_t +H5FD_sec2_read(H5FD_t *_file, H5FD_mem_t type/*unused*/, hid_t dxpl_id/*unused*/, + haddr_t addr, hsize_t size, void *buf/*out*/) +{ + H5FD_sec2_t *file = (H5FD_sec2_t*)_file; + ssize_t nbytes; + + assert(file && file->pub.cls); + assert(buf); + + /* Check for overflow conditions */ + if (REGION_OVERFLOW(addr, size)) return -1; + if (addr+size>file->eoa) return -1; + + /* Seek to the correct location */ + if ((addr!=file->pos || OP_READ!=file->op) && + file_seek(file->fd, (file_offset_t)addr, SEEK_SET)<0) { + file->pos = HADDR_UNDEF; + file->op = OP_UNKNOWN; + return -1; + } + + /* + * Read data, being careful of interrupted system calls, partial results, + * and the end of the file. + */ + while (size>0) { + do nbytes = read(file->fd, buf, size); + while (-1==nbytes && EINTR==errno); + if (-1==nbytes) { + /* error */ + file->pos = HADDR_UNDEF; + file->op = OP_UNKNOWN; + return -1; + } + if (0==nbytes) { + /* end of file but not end of format address space */ + memset(buf, 0, size); + size = 0; + } + assert(nbytes>=0); + assert((hsize_t)nbytes<=size); + size -= (hsize_t)nbytes; + addr += (haddr_t)nbytes; + buf = (char*)buf + nbytes; + } + + /* Update current position */ + file->pos = addr; + file->op = OP_READ; + return 0; +} +\endcode +Example: The sec2 write callback is similar except it updates the file EOF marker when extending the file. + +\subsection subsec_vfl_data_flush Flushing Cached Data +Some drivers may desire to cache data in memory in order to make larger I/O requests to the +underlying file and thus improving bandwidth. Such drivers should register a cache flushing +function so that the library can insure that data has been flushed out of the drivers in +response to the application calling #H5Fflush. + + + + + +
static herr_t flush (H5FD_t *file)Flush all data for file file to storage.
+ +Example: The sec2 driver doesn't cache any data but it also doesn't extend the Unix file as +aggressively as it should. Therefore, when finalizing a file it should write a zero to the last +byte of the allocated region so that when reopening the file later the EOF marker will be at +least as large as the EOA marker saved in the superblock (otherwise HDF5 will refuse to open +the file, claiming that the data appears to be truncated). +\code +static herr_t +H5FD_sec2_flush(H5FD_t *_file) +{ + H5FD_sec2_t *file = (H5FD_sec2_t*)_file; + + if (file->eoa>file->eof) { + if (-1==file_seek(file->fd, file->eoa-1, SEEK_SET)) return -1; + if (write(file->fd, "", 1)!=1) return -1; + file->eof = file->eoa; + file->pos = file->eoa; + file->op = OP_WRITE; + } + + return 0; +} +\endcode + +\section sec_vfl_opt Optimization Functions +The library is capable of performing several generic optimizations on I/O, but these types of +optimizations may not be appropriate for a given VFL driver. + +Each driver may provide a query function to allow the library to query whether to enable these +optimizations. If a driver lacks a query function, the library will disable all types of +optimizations which can be queried. + + + + + + +
static herr_t query (const H5FD_t *file, unsigned long *flags)This function is called by the library to query which optimizations to enable for I/O to this driver.
+ +These are the flags which are currently defined: + + + + + + + + + + + + + +
H5FD_FEAT_AGGREGATE_METADATA (0x00000001)Defining the H5FD_FEAT_AGGREGATE_METADATA for a VFL driver means that the library will attempt to allocate +a larger block for metadata and then sub-allocate each metadata request from that larger block.
H5FD_FEAT_ACCUMULATE_METADATA (0x00000002)Defining the H5FD_FEAT_ACCUMULATE_METADATA for a VFL driver means that the library will attempt to cache +metadata as it is written to the file and build up a larger block of metadata to eventually pass to the +VFL 'write' routine.
H5FD_FEAT_DATA_SIEVE (0x00000004)Defining the H5FD_FEAT_DATA_SIEVE for a VFL driver means that the library will attempt to cache raw data + as it is read from/written to a file in a "data sieve" buffer.
+ +See Rajeev Thakur's papers: +http://www.mcs.anl.gov/~thakur/papers/romio-coll.ps.gz +http://www.mcs.anl.gov/~thakur/papers/mpio-high-perf.ps.gz + +\section sec_vfl_reg Registration of a Driver +Before a driver can be used the HDF5 library needs to be told of its existence. This is done by +registering the driver, which results in a driver identification number. Instead of passing many +arguments to the registration function, the driver information is entered into a structure and the +address of the structure is passed to the registration function where it is copied. This allows +the HDF5 API to be extended while providing backward compatibility at the source level. + + + + + + +
hid_t H5FDregister (H5FD_class_t *cls)The driver described by struct cls is registered with the library and an ID number for the driver is returned.
+ +The H5FD_class_t type is a struct with the following fields: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
const char *nameA pointer to a constant, null-terminated driver name to be used for debugging purposes.
size_t fapl_sizeThe size in bytes of the file access mode structure or zero if the driver supplies a copy function +or doesn't define the structure.
void *(*fapl_copy)(const void *fapl)An optional function which copies a driver-defined file access mode structure. This field takes +precedence over fm_size when both are defined.
void (*fapl_free)(void *fapl)An optional function to free the driver-defined file access mode structure. If null, then the +library calls the C free function to free the structure.
size_t dxpl_sizeThe size in bytes of the data transfer mode structure or zero if the driver supplies a copy +function or doesn't define the structure.
void *(*dxpl_copy)(const void *dxpl)An optional function which copies a driver-defined data transfer mode structure. This field +takes precedence over xm_size when both are defined.
void (*dxpl_free)(void *dxpl)An optional function to free the driver-defined data transfer mode structure. If null, then +the library calls the C free function to free the structure.
H5FD_t *(*open)(const char *name, unsigned flags, hid_t fapl, haddr_t maxaddr)The function which opens or creates a new file.
herr_t (*close)(H5FD_t *file)The function which ends access to a file.
int (*cmp)(const H5FD_t *f1, const H5FD_t *f2)An optional function to determine whether two open files have the same key. If this function +is not present then the library assumes that two files will never be the same.
int (*query)(const H5FD_t *f, unsigned long *flags)An optional function to determine which library optimizations a driver can support.
haddr_t (*alloc)(H5FD_t *file, H5FD_mem_t type, hsize_t size)An optional function to allocate space in the file.
herr_t (*free)(H5FD_t *file, H5FD_mem_t type, haddr_t addr, hsize_t size)An optional function to free space in the file.
haddr_t (*get_eoa)(H5FD_t *file)A function to query how much of the format address space has been allocated.
herr_t (*set_eoa)(H5FD_t *file, haddr_t)A function to set the end of address space.
haddr_t (*get_eof)(H5FD_t *file)A function to return the current end-of-file marker value.
herr_t (*read)(H5FD_t *file, H5FD_mem_t type, hid_t dxpl, haddr_t addr, hsize_t size, void *buffer)A function to read data from a file.
herr_t (*write)(H5FD_t *file, H5FD_mem_t type, hid_t dxpl, haddr_t addr, hsize_t size, const void *buffer)A function to write data to a file.
herr_t (*flush)(H5FD_t *file)A function which flushes cached data to the file.
H5FD_mem_t fl_map[H5FD_MEM_NTYPES]An array which maps a file allocation request type to a free list.
+ +Example: The sec2 driver would be registered as: +\code +static const H5FD_class_t H5FD_sec2_g = { + "sec2", /*name */ + MAXADDR, /*maxaddr */ + NULL, /*sb_size */ + NULL, /*sb_encode */ + NULL, /*sb_decode */ + 0, /*fapl_size */ + NULL, /*fapl_get */ + NULL, /*fapl_copy */ + NULL, /*fapl_free */ + 0, /*dxpl_size */ + NULL, /*dxpl_copy */ + NULL, /*dxpl_free */ + H5FD_sec2_open, /*open */ + H5FD_sec2_close, /*close */ + H5FD_sec2_cmp, /*cmp */ + H5FD_sec2_query, /*query */ + NULL, /*alloc */ + NULL, /*free */ + H5FD_sec2_get_eoa, /*get_eoa */ + H5FD_sec2_set_eoa, /*set_eoa */ + H5FD_sec2_get_eof, /*get_eof */ + H5FD_sec2_read, /*read */ + H5FD_sec2_write, /*write */ + H5FD_sec2_flush, /*flush */ + H5FD_FLMAP_SINGLE, /*fl_map */ +}; + +hid_t +H5FD_sec2_init(void) +{ + if (!H5FD_SEC2_g) { + H5FD_SEC2_g = H5FDregister(&H5FD_sec2_g); + } + return H5FD_SEC2_g; +} +\endcode + +A driver can be removed from the library by unregistering it + + + + + +
herr_t H5Dunregister (hid_t driver)Where driver is the ID number returned when the driver was registered.
+Unregistering a driver makes it unusable for creating new file access or data transfer property +lists but doesn't affect any property lists or files that already use that driver. + +\subsection subsec_vfl_reg_prog Programming Note for C++ Developers Using C Functions +If a C routine that takes a function pointer as an argument is called from within C++ code, +the C routine should be returned from normally. + +Examples of this kind of routine include callbacks such as #H5Pset_elink_cb +and #H5Pset_type_conv_cb and functions such as #H5Tconvert and #H5Ewalk2. + +Exiting the routine in its normal fashion allows the HDF5 C Library to clean up +its work properly. In other words, if the C++ application jumps out of the routine +back to the C++ “catch” statement, the library is not given the opportunity to close +any temporary data structures that were set up when the routine was called. The C++ +application should save some state as the routine is started so that any problem that +occurs might be diagnosed. + +\section sec_vfl_query Querying Driver Information + + + + + +
void * H5Pget_driver_data (hid_t fapl)
void * H5Pget_driver_data (hid_t fxpl)		This function is intended to be used by driver functions, not applications. It returns a pointer +directly into the file access property list fapl which is a copy of the driver's file access mode +originally provided to the H5Pset_driver function. If its argument is a data transfer property list +fxpl then it returns a pointer to the driver-specific data transfer information instead. +
+ +\section sec_vfl_misc Miscellaneous +The various private H5F_low_* functions will be replaced by public H5FD* functions so they +can be called from drivers. + +All private functions H5F_addr_* which operate on addresses will be renamed as public functions +by removing the first underscore so they can be called by drivers. + +The haddr_t address data type will be passed by value throughout the library. The original +intent was that this type would eventually be a union of file address types for the various +drivers and may become quite large, but that was back when drivers were part of HDF5. It will +become an alias for an unsigned integer type (32 or 64 bits depending on how the library was configured). + +The various H5F*.c driver files will be renamed H5FD*.c and each will have a corresponding header +file. All driver functions except the initializer and API will be declared static. + +This documentation didn't cover optimization functions which would be useful to drivers like MPI-IO. +Some drivers may be able to perform data pipeline operations more efficiently than HDF5 and need to +be given a chance to override those parts of the pipeline. The pipeline would be designed to call +various H5FD optimization functions at various points which return one of three values: the operation +is not implemented by the driver, the operation is implemented but failed in a non-recoverable manner, +the operation is implemented and succeeded. + +Various parts of HDF5 check the only the top-level file driver and do something special if it is +the MPI-IO driver. However, we might want to be able to put the MPI-IO driver under other drivers +such as the raw part of a split driver or under a debug driver whose sole purpose is to accumulate +statistics as it passes all requests through to the MPI-IO driver. Therefore we will probably need +a function which takes a format address and or object type and returns the driver which would have +been used at the lowest level to process the request. + +*/ diff --git a/doxygen/dox/VOLConnGuide.dox b/doxygen/dox/VOLConnGuide.dox index 547d6eb76cb..00dd18e82f7 100644 --- a/doxygen/dox/VOLConnGuide.dox +++ b/doxygen/dox/VOLConnGuide.dox @@ -191,7 +191,8 @@ are for connector testing and should not be found in the wild. Values of 512 to connectors. As is the case with HDF5 filters, The HDF Group can assign you an official VOL connector value. Please -contact help@hdfgroup.org for help with this. We currently do not register connector names, though the +contact help@hdfgroup.org for help with this. We currently do not +register connector names, though the name you've chosen will appear on the registered VOL connectors page. As noted above, registered VOL connectors will be listed at: diff --git a/doxygen/examples/FileFormat.html b/doxygen/examples/FileFormat.html deleted file mode 100644 index 40d21138eb2..00000000000 --- a/doxygen/examples/FileFormat.html +++ /dev/null @@ -1,1273 +0,0 @@ - - - HDF5 File Format Discussion - - - - - - - -
-

HDF5 File Format Discussion

-

Quincey Koziol
- koziol@ncsa.uiuc.edu
- May 15, 2003 -

- -
    - -

  1. Document's Audience:

    - -
      -
    • Current H5 library designers and knowledgeable external developers.
      • -
    - -

  2. Background Reading:

    - -
    -
    HDF5 File Format Specification -
    This describes the current HDF5 file format. -
    - -

  3. Introduction:

    - -
    -
    What is this document about?
    -
    This document attempts to explain the HDF5 file format - specification with a few examples and describes some potential - improvements to the format specification. -

    -
    - -

  4. File Format Examples:

    - -

    This section has several small programs and describes the format of a file -created with each of them. -

    - -

    Example program one - Create an empty file: -

     
-#include "hdf5.h"
-#include 
-
-int main()
-{
-    hid_t fid;      /* File ID */
-    herr_t ret;     /* Generic return value */
-
-    /* Create the file */
-    fid=H5Fcreate("example1.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
-    assert(fid>=0);
-
-    /* Close the file */
-    ret=H5Fclose(fid);
-    assert(ret>=0);
-
-    return(0);
-}
-
    - -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Super Block -
    bytebytebytebyte
    \211'H''D''F'
    \r\n\032\n
    0000
    0880
    416
    0x00000003
    0

    0xffffffffffffffff

    ?

    0xffffffffffffffff

    - - - - - - - - - - - - - - - - - - - - -
    0

    928

    H5G_CACHED_STAB (1)
    0
    - - - - - - - -

    384


    96

    -
    -
    -
    -
    -
     
-%h5debug example1.h5
-
-Reading signature at address 0 (rel)
-File Super Block...
-File name:                                         example1.h5
-File access flags                                  0x00000000
-File open reference count:                         1
-Address of super block:                            0 (abs)
-Size of user block:                                0 bytes
-Super block version number:                        0
-Free list version number:                          0
-Root group symbol table entry version number:      0
-Shared header version number:                      0
-Size of file offsets (haddr_t type):               8 bytes
-Size of file lengths (hsize_t type):               8 bytes
-Symbol table leaf node 1/2 rank:                   4
-Symbol table internal node 1/2 rank:               16
-File consistency flags:                            0x00000003
-Base address:                                      0 (abs)
-Free list address:                                 UNDEF (rel)
-Address of driver information block:               UNDEF (rel)
-Root group symbol table entry:
-   Name offset into private heap:                  0
-   Object header address:                          928
-   Dirty:                                          Yes
-   Cache info type:                                Symbol Table
-   Cached information:
-      B-tree address:                              384
-      Heap address:                                96
-
    - - -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Root Group Object Header -
    bytebytebytebyte
    102
    1
    32
    0x001116
    0x010
    - - - - - - - -

    384


    96

    -
    00
    0x000
    -
    -
    -
     
-%h5debug example1.h5 928
-
-New address: 928
-Reading signature at address 928 (rel)
-Object Header...
-Dirty:                                             0
-Version:                                           1
-Header size (in bytes):                            16
-Number of links:                                   1
-Number of messages (allocated):                    2 (32)
-Number of chunks (allocated):                      1 (8)
-Chunk 0...
-   Dirty:                                          0
-   Address:                                        944
-   Size in bytes:                                  32
-Message 0...
-   Message ID (sequence number):                   0x0011 stab(0)
-   Shared message:                                 No
-   Constant:                                       Yes
-   Raw size in obj header:                         16 bytes
-   Chunk number:                                   0
-   Message Information:
-      B-tree address:                              384
-      Name heap address:                           96
-Message 1...
-   Message ID (sequence number):                   0x0000 null(0)
-   Shared message:                                 No
-   Constant:                                       No
-   Raw size in obj header:                         0 bytes
-   Chunk number:                                   0
-   Message Information:
-      
-
    - -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Root Group Local Heap -
    bytebytebytebyte
    'H''E''A''P'
    0
    256
    8
    128
    -
    -
    - -
     
-%h5debug example1.h5 96
-
-New address: 96
-Reading signature at address 96 (rel)
-Local Heap...
-Dirty:                                             0
-Header size (in bytes):                            32
-Address of heap data:                              128
-Data bytes allocated on disk:                      256
-Data bytes allocated in core:                      256
-Free Blocks (offset, size):
-   Block #0:                                        8,      248
-Percent of heap used:                              3.12%
-Data follows (`__' indicates free region)...
-     0: 00 00 00 00 00 00 00 00  __ __ __ __ __ __ __ __ ........
-    16: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-    32: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-    48: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-    64: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-    80: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-    96: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-   112: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-   128: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-   144: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-   160: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-   176: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-   192: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-   208: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-   224: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-   240: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-
-
    - -
    - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Root Group B-tree -
    bytebytebytebyte
    'T''R''E''E'
    000

    0xffffffffffffffff


    0xffffffffffffffff

    -
    -
    -
     
-%h5debug example1.h5 384 96
-
-New address: 384
-Reading signature at address 384 (rel)
-Tree type ID:                                      H5B_SNODE_ID
-Size of node:                                      544
-Size of raw (disk) key:                            8
-Dirty flag:                                        False
-Number of initial dirty children:                  0
-Level:                                             0
-Address of left sibling:                           UNDEF
-Address of right sibling:                          UNDEF
-Number of children (max):                          0 (32)
-
-
    - -

    - -

    Example program two - Create a file with a single dataset in it: -

     
-#include "hdf5.h"
-#include 
-
-int main()
-{
-    hid_t fid;      /* File ID */
-    hid_t sid;      /* Dataspace ID */
-    hid_t did;      /* Dataset ID */
-    herr_t ret;     /* Generic return value */
-
-    /* Create the file */
-    fid=H5Fcreate("example2.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
-    assert(fid>=0);
-
-    /* Create a scalar dataspace for the dataset */
-    sid=H5Screate(H5S_SCALAR);
-    assert(sid>=0);
-
-    /* Create a trivial dataset */
-    did=H5Dcreate(fid, "Dataset", H5T_NATIVE_INT, sid, H5P_DEFAULT);
-    assert(did>=0);
-
-    /* Close the dataset */
-    ret=H5Dclose(did);
-    assert(ret>=0);
-
-    /* Close the dataspace */
-    ret=H5Sclose(sid);
-    assert(ret>=0);
-
-    /* Close the file */
-    ret=H5Fclose(fid);
-    assert(ret>=0);
-
-    return(0);
-}
-
    - -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Super Block -
    bytebytebytebyte
    \211'H''D''F'
    \r\n\032\n
    0000
    0880
    416
    0x00000003
    0

    0xffffffffffffffff

    ?

    0xffffffffffffffff

    - - - - - - - - - - - - - - - - - - - - -
    0

    928

    H5G_CACHED_STAB (1)
    0
    - - - - - - - -

    384


    96

    -
    -
    -
    -
    -
     
-%h5debug example2.h5
-
-Reading signature at address 0 (rel)
-File Super Block...
-File name:                                         example2.h5
-File access flags                                  0x00000000
-File open reference count:                         1
-Address of super block:                            0 (abs)
-Size of user block:                                0 bytes
-Super block version number:                        0
-Free list version number:                          0
-Root group symbol table entry version number:      0
-Shared header version number:                      0
-Size of file offsets (haddr_t type):               8 bytes
-Size of file lengths (hsize_t type):               8 bytes
-Symbol table leaf node 1/2 rank:                   4
-Symbol table internal node 1/2 rank:               16
-File consistency flags:                            0x00000003
-Base address:                                      0 (abs)
-Free list address:                                 UNDEF (rel)
-Address of driver information block:               UNDEF (rel)
-Root group symbol table entry:
-   Name offset into private heap:                  0
-   Object header address:                          928
-   Dirty:                                          Yes
-   Cache info type:                                Symbol Table
-   Cached entry information:
-      B-tree address:                              384
-      Heap address:                                96
-
    - -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Root Group Object Header -
    bytebytebytebyte
    102
    1
    32
    0x001116
    0x010
    - - - - - - - -

    384


    96

    -
    00
    0x000
    -
    -
    -
     
-%h5debug example2.h5 928
-
-New address: 928
-Reading signature at address 928 (rel)
-Object Header...
-Dirty:                                             0
-Version:                                           1
-Header size (in bytes):                            16
-Number of links:                                   1
-Number of messages (allocated):                    2 (32)
-Number of chunks (allocated):                      1 (8)
-Chunk 0...
-   Dirty:                                          0
-   Address:                                        944
-   Size in bytes:                                  32
-Message 0...
-   Message ID:                                     0x0011 stab(0)
-   Shared message:                                 No
-   Constant:                                       Yes
-   Raw size in obj header:                         16 bytes
-   Chunk number:                                   0
-   Message Information:
-      B-tree address:                              384
-      Name heap address:                           96
-Message 1...
-   Message ID:                                     0x0000 null(0)
-   Shared message:                                 No
-   Constant:                                       No
-   Raw size in obj header:                         0 bytes
-   Chunk number:                                   0
-   Message Information:
-      
-
    - -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Root Group Local Heap -
    bytebytebytebyte
    'H''E''A''P'
    0
    256
    16
    128
    -
    -
    - -
     
-%h5debug example2.h5 96
-
-New address: 96
-Reading signature at address 96 (rel)
-Local Heap...
-Dirty:                                             0
-Header size (in bytes):                            32
-Address of heap data:                              128
-Data bytes allocated on disk:                      256
-Data bytes allocated in core:                      256
-Free Blocks (offset, size):
-   Block #0:                                       16,      240
-Percent of heap used:                              6.25%
-Data follows (`__' indicates free region)...
-      0: 00 00 00 00 00 00 00 00  44 61 74 61 73 65 74 00 ........Dataset.
-     16: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-     32: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-     48: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-     64: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-     80: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-     96: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-    112: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-    128: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-    144: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-    160: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-    176: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-    192: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-    208: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-    224: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-    240: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
-
    - -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Root Group B-tree -
    bytebytebytebyte
    'T''R''E''E'
    001

    0xffffffffffffffff


    0xffffffffffffffff


    0


    1248


    8

    -
    -
    -
     
-%h5debug example2.h5 384 96
-
-New address: 384
-Reading signature at address 384 (rel)
-Tree type ID:                                      H5B_SNODE_ID
-Size of node:                                      544
-Size of raw (disk) key:                            8
-Dirty flag:                                        False
-Number of initial dirty children:                  0
-Level:                                             0
-Address of left sibling:                           UNDEF
-Address of right sibling:                          UNDEF
-Number of children (max):                          1 (32)
-Child 0...
-   Address:                                        1248
-   Left Key:
-      Heap offset:                                 0
-      Name :
-   Right Key:
-      Heap offset:                                 8
-      Name :                                       Dataset
-
    - -
    - - - - - - - - - - - - - - - - - - - - - - - -
    - Root Group B-tree Symbol Table Node -
    bytebytebytebyte
    'S''N''O''D'
    101
    - - - - - - - - - - - - - - - - - - - - -
    8

    976

    0
    0


    0


    -
    -
    -
    -
     
-%h5debug example2.h5 1248 96
-
-New address: 1248
-Reading signature at address 1248 (rel)
-Symbol Table Node...
-Dirty:                                             No
-Size of Node (in bytes):                           328
-Number of Symbols:                                 1 of 8
-Symbol 0:
-   Name:                                           `Dataset'
-   Name offset into private heap:                  8
-   Object header address:                          976
-   Dirty:                                          No
-   Cache info type:                                Nothing Cached
-
    - -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - '/Dataset' Object Header -
    bytebytebytebyte
    Version: 1Reserved: 0Number of Header Messages: 6
    Object Reference Count: 1
    Total Object Header Size: 256
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Fill Value Header Message -
    Message Type: 0x0005Message Data Size: 8
    Flags: 0x01Reserved: 0
    Version: 1Space Allocation Time: 2 (Late)Fill Value Writing Time: 0 (At allocation)Fill Value Defined: 0 (Undefined)
    Fill Value Datatype Size: 0 (Use dataset's datatype for fill-value datatype)
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Datatype Header Message -
    Message Type: 0x0003Message Data Size: 16
    Flags: 0x01Reserved: 0
    - - - - - -
    Version: 0x1Class: 0x0 (Fixed-Point)
    -     		Fixed-Point Bit-Field: 0x08 (Little-endian, No padding, Signed)
    Size: 4
    Bit Offset: 0Bit Precision: 32
    Message Alignment Filler: -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Dataspace Header Message -
    Message Type: 0x0001Message Data Size: 8
    Flags: 0x00Reserved: 0
    Version: 1Rank: 0 (Scalar)Flags: 0x00 (No maximum dimensions, no permutation information)Reserved: 0
    Reserved: 0
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Layout Header Message -
    Message Type: 0x0008Message Data Size: 24
    Flags: 0x00Reserved: 0
    Version: 1Rank: 1 (Dataspace rank+1)Class: 1 (Contiguous)Reserved: 0
    Reserved: 0

    Address: 0xffffffffffffffff (Undefined)

    Dimension 0 Size: 4 (Datatype size)
    Message Alignment Filler: -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Modification Date & Time Header Message -
    Message Type: 0x0012Message Data Size: 8
    Flags: 0x00Reserved: 0
    Version: 1Reserved: 0
    Seconds Since Epoch: 1052401700 (2003-05-08 08:48:20 CDT)
    -
    - - - - - - - - - - - - - - - - - -
    - Null Header Message -
    Message Type: 0x0000Message Data Size: 144
    Flags: 0x00Reserved: 0
    -
    -
    -
    -
     
-%h5debug example2.h5 976
-
-New address: 976
-Reading signature at address 976 (rel)
-Object Header...
-Dirty:                                             0
-Version:                                           1
-Header size (in bytes):                            16
-Number of links:                                   1
-Number of messages (allocated):                    6 (32)
-Number of chunks (allocated):                      1 (8)
-Chunk 0...
-   Dirty:                                          0
-   Address:                                        992
-   Size in bytes:                                  256
-Message 0...
-   Message ID (sequence number):                   0x0005 `fill_new' (0)
-   Shared:                                         No
-   Constant:                                       Yes
-   Raw size in obj header:                         8 bytes
-   Chunk number:                                   0
-   Message Information:
-      Version:                                     1
-      Space Allocation Time:                       Late
-      Fill Time:                                   On Allocation
-      Fill Value Defined:                          Undefined
-      Size:                                        0
-      Data type:                                   
-Message 1...
-   Message ID (sequence number):                   0x0003 data_type(0)
-   Shared message:                                 No
-   Constant:                                       Yes
-   Raw size in obj header:                         16 bytes
-   Chunk number:                                   0
-   Message Information:
-      Type class:                                  integer
-      Size:                                        4 bytes
-      Byte order:                                  little endian
-      Precision:                                   32 bits
-      Offset:                                      0 bits
-      Low pad type:                                zero
-      High pad type:                               zero
-      Sign scheme:                                 2's comp
-Message 2...
-   Message ID (sequence number):                   0x0001 simple_dspace(0)
-   Shared message:                                 No
-   Constant:                                       No
-   Raw size in obj header:                         8 bytes
-   Chunk number:                                   0
-   Message Information:
-      Rank:                                        0
-Message 3...
-   Message ID (sequence number):                   0x0008 layout(0)
-   Shared message:                                 No
-   Constant:                                       No
-   Raw size in obj header:                         24 bytes
-   Chunk number:                                   0
-   Message Information:
-      Data address:                                UNDEF
-      Number of dimensions:                        1
-      Size:                                        {4}
-Message 4...
-   Message ID (sequence number):                   0x0012 mtime_new(0)
-   Shared message:                                 No
-   Constant:                                       No
-   Raw size in obj header:                         8 bytes
-   Chunk number:                                   0
-   Message Information:
-      Time:                                        2003-03-05 14:52:00 CST
-Message 5...
-   Message ID (sequence number):                   0x0000 null(0)
-   Shared message:                                 No
-   Constant:                                       No
-   Raw size in obj header:                         144 bytes
-   Chunk number:                                   0
-   Message Information:
-      
-
    - -

    - -
- - -
- diff --git a/doxygen/examples/H5.format.1.0.html b/doxygen/examples/H5.format.1.0.html deleted file mode 100644 index 00da963c48e..00000000000 --- a/doxygen/examples/H5.format.1.0.html +++ /dev/null @@ -1,4054 +0,0 @@ - - - - HDF5 File Format Specification - - - - -
-
- - - -
-
    -
  1. Introduction -
  2. Disk Format Level 0 - File Signature and Super Block -
  3. Disk Format Level 1 - File Infrastructure - -
      -
    1. Disk Format Level 1A - B-link Trees and B-tree Nodes -
    2. Disk Format Level 1B - Group -
    3. Disk Format Level 1C - Group Entry -
    4. Disk Format Level 1D - Local Heaps -
    5. Disk Format Level 1E - Global Heap -
    6. Disk Format Level 1F - Free-space Index -
    -     -
  4. Disk Format Level 2 - Data Objects - -
      -
    1. Disk Format Level 2a - Data Object Headers -
        -
      1. Name: NIL -
      2. Name: Simple Dataspace - -
      3. Name: Datatype -
      4. Name: Data Storage - Fill Value -
      5. Name: Reserved - not assigned yet -
      -
    -     -
- 		   -
    - -
  1. Disk Format Level 2 - Data Objects - (Continued) -
      -
    1. Disk Format Level 2a - Data Object Headers(Continued) -
        -
      1. Name: Data Storage - Compact -
      2. Name: Data Storage - External Data Files -
      3. Name: Data Storage - Layout -
      4. Name: Reserved - not assigned yet -
      5. Name: Reserved - not assigned yet -
      6. Name: Data Storage - Filter Pipeline -
      7. Name: Attribute -
      8. Name: Object Name -
      9. Name: Object Modification Date and Time -
      10. Name: Shared Object Message -
      11. Name: Object Header Continuation -
      12. Name: Group Message -
      -
    2. Disk Format: Level 2b - Shared Data Object Headers -
    3. Disk Format: Level 2c - Data Object Data Storage -
    -     -
-
-
- -

- - -

Introduction

- - - - - - - -
  -
- HDF5 Groups - 		 
  - Figure 1: Relationships among the HDF5 root group, other groups, and objects -
- 		 
  - HDF5 Objects -  
  - Figure 2: HDF5 objects -- datasets, datatypes, or dataspaces -
- 		 
- - -

The format of an HDF5 file on disk encompasses several - key ideas of the HDF4 and AIO file formats as well as - addressing some shortcomings therein. The new format is - more self-describing than the HDF4 format and is more - uniformly applied to data objects in the file. - -

An HDF5 file appears to the user as a directed graph. - The nodes of this graph are the higher-level HDF5 objects - that are exposed by the HDF5 APIs: - -

    -
  • Groups -
  • Datasets -
  • Datatypes -
  • Dataspaces -
- -

At the lowest level, as information is actually written to the disk, - an HDF5 file is made up of the following objects: -

    -
  • A super block -
  • B-tree nodes (containing either symbol nodes or raw data chunks) -
  • Object headers - -
  • Collections -
  • Local heaps -
  • Free space -
- - The HDF5 library uses these lower-level objects to represent the - higher-level objects that are then presented to the user or - to applications through the APIs. - For instance, a group is an object header that contains a message that - points to a local heap and to a B-tree which points to symbol nodes. - A dataset is an object header that contains messages that describe - datatype, space, layout, filters, external files, fill value, etc - with the layout message pointing to either a raw data chunk or to a - B-tree that points to raw data chunks. - - -

This Document

- -

This document describes the lower-level data objects; - the higher-level objects and their properties are described - in the HDF5 User Guide. - - - - - - -

Three levels of information comprise the file format. - Level 0 contains basic information for identifying and - defining information about the file. Level 1 information contains - the group information (stored as a B-tree) and is used as the - index for all the objects in the file. Level 2 is the rest - of the file and contains all of the data objects, with each object - partitioned into header information, also known as - meta information, and data. - -

The sizes of various fields in the following layout tables are - determined by looking at the number of columns the field spans - in the table. There are three exceptions: (1) The size may be - overridden by specifying a size in parentheses, (2) the size of - addresses is determined by the Size of Offsets field - in the super block, and (3) the size of size fields is determined - by the Size of Lengths field in the super block. - - - -

-

- - -

- Disk Format: Level 0 - File Signature and Super Block

- -

The super block may begin at certain predefined offsets within - the HDF5 file, allowing a block of unspecified content for - users to place additional information at the beginning (and - end) of the HDF5 file without limiting the HDF5 library's - ability to manage the objects within the file itself. This - feature was designed to accommodate wrapping an HDF5 file in - another file format or adding descriptive information to the - file without requiring the modification of the actual file's - information. The super block is located by searching for the - HDF5 file signature at byte offset 0, byte offset 512 and at - successive locations in the file, each a multiple of two of - the previous location, i.e. 0, 512, 1024, 2048, etc. - -

The super block is composed of a file signature, followed by - super block and group version numbers, information - about the sizes of offset and length values used to describe - items within the file, the size of each group page, - and a group entry for the root object in the file. - -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- HDF5 Super Block Layout -
bytebytebytebyte

HDF5 File Signature (8 bytes)

Version # of Super BlockVersion # of Global Free-space StorageVersion # of GroupReserved
Version # of Shared Header Message FormatSize of OffsetsSize of LengthsReserved (zero)
Group Leaf Node KGroup Internal Node K
File Consistency Flags
Base Address*
Address of Global Free-space Heap*
End of File Address*
Driver Information Block Address*
Root Group Address*
- - - -
-
- (Items marked with an asterisk (*) in the above table -
- are of the size specified in "Size of Offsets.") -
-
-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
File SignatureThis field contains a constant value and can be used to - quickly identify a file as being an HDF5 file. The - constant value is designed to allow easy identification of - an HDF5 file and to allow certain types of data corruption - to be detected. The file signature of an HDF5 file always - contains the following values: - -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
decimal13772687013102610
hexadecimal894844460d0a1a0a
ASCII C Notation\211HDF\r\n\032\n
-
-
- - This signature both identifies the file as an HDF5 file - and provides for immediate detection of common - file-transfer problems. The first two bytes distinguish - HDF5 files on systems that expect the first two bytes to - identify the file type uniquely. The first byte is - chosen as a non-ASCII value to reduce the probability - that a text file may be misrecognized as an HDF5 file; - also, it catches bad file transfers that clear bit - 7. Bytes two through four name the format. The CR-LF - sequence catches bad file transfers that alter newline - sequences. The control-Z character stops file display - under MS-DOS. The final line feed checks for the inverse - of the CR-LF translation problem. (This is a direct - descendent of the PNG file signature.)
Version Number of the Super BlockThis value is used to determine the format of the - information in the super block. When the format of the - information in the super block is changed, the version number - is incremented to the next integer and can be used to - determine how the information in the super block is - formatted.
Version Number of the Global Free-space HeapThis value is used to determine the format of the - information in the Global Free-space Heap.
Version Number of the GroupThis value is used to determine the format of the - information in the Group. When the format of - the information in the Group is changed, the - version number is incremented to the next integer and can be - used to determine how the information in the Group - is formatted.
Version Number of the Shared Header Message FormatThis value is used to determine the format of the - information in a shared object header message, which is - stored in the global small-data heap. Since the format - of the shared header messages differs from the private - header messages, a version number is used to identify changes - in the format.
Size of OffsetsThis value contains the number of bytes used to store - addresses in the file. The values for the addresses of - objects in the file are offsets relative to a base address, - usually the address of the super block signature. This - allows a wrapper to be added after the file is created - without invalidating the internal offset locations.
Size of LengthsThis value contains the number of bytes used to store - the size of an object.
Group Leaf Node KEach leaf node of a group B-tree will have at - least this many entries but not more than twice this - many. If a group has a single leaf node then it - may have fewer entries.
Group Internal Node KEach internal node of a group B-tree will have - at least K pointers to other nodes but not more than 2K - pointers. If the group has only one internal - node then it might have fewer than K pointers.
Bytes per B-tree PageThis value contains the number of bytes used for symbol - pairs per page of the B-trees used in the file. All - B-tree pages will have the same size per page. -
- For 32-bit file offsets, 340 objects is the maximum - per 4KB page; for 64-bit file offset, 254 objects will fit - per 4KB page. In general, the equation is: -
-    <number of objects> = -
       - FLOOR((<page size> - <offset size>) / -
          - (<Symbol size> + <offset size>)) - - 1
File Consistency FlagsThis value contains flags to indicate information - about the consistency of the information contained - within the file. Currently, the following bit flags are - defined: -
    -
  • Bit 0 set indicates that the file is opened for - write-access. -
  • Bit 1 set indicates that the file has - been verified for consistency and is guaranteed to be - consistent with the format defined in this document. -
  • Bits 2-31 are reserved for future use. -
- Bit 0 should be - set as the first action when a file is opened for write - access and should be cleared only as the final action - when closing a file. Bit 1 should be cleared during - normal access to a file and only set after the file's - consistency is guaranteed by the library or a - consistency utility.
Base AddressThis is the absolute file address of the first byte of - the HDF5 data within the file. The library currently - constrains this value to be the absolute file address - of the super block itself when creating new files; - future versions of the library may provide greater - flexibility. Unless otherwise noted, - all other file addresses are relative to this base - address.
Address of Global Free-space HeapFree-space management is not yet defined in the HDF5 - file format and is not handled by the library. - Currently this field always contains the - undefined address 0xfff...ff. - -
End of File AddressThis is the relative file address of the first byte past - the end of all HDF5 data. It is used to determine whether a - file has been accidentally truncated and as an address where - file data allocation can occur if the free list is not - used.
Driver Information Block AddressThis is the relative file address of the file driver - information block which contains driver-specific - information needed to reopen the file. If there is no - driver information block then this entry should be the - undefined address (all bits set).
Root Group AddressThis is the address of the root group (described later - in this document), which serves as the entry point into - the group graph.
-
- - -

The file driver information block is an optional region of the - file which contains information needed by the file driver in - order to reopen a file. The format of the file driver information - block is: - -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Driver Information Block -
bytebytebytebyte
VersionReserved (zero)
Driver Information Size (4 bytes)

Driver Identification (8 bytes)



Driver Information


-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
VersionThe version number of the driver information block. The - file format documented here is version zero.
Driver Information SizeThe size in bytes of the Driver Information part of this - structure.
Driver IdentificationThis is an eight-byte ASCII string without null - termination which identifies the driver and version number - of the Driver Information block. The predefined drivers - supplied with the HDF5 library are identified by the - letters NCSA followed by the first four characters of - the driver name. If the Driver Information block is not - the original version then the last letter(s) of the - identification will be replaced by a version number in - ASCII. - For example, the various versions of the family driver - will be identified by NCSAfami, NCSAfam0, - NCSAfam1, etc. - (NCSAfami is simply NCSAfamily truncated - to eight characters. Subsequent identifiers will be created by - substituting sequential numerical values for the final character, - starting with zero.) -

- Identification for user-defined drivers - is arbitrary but should be unique.

Driver InformationDriver information is stored in a format defined by the - file driver and encoded/decoded by the driver callbacks - invoked from the H5FD_sb_encode and - H5FD_sb_decode functions.
-
- - -

-

- - -

- Disk Format: Level 1 - File Infrastructure

-

Disk Format: Level 1A - B-link Trees and B-tree Nodes

- -

B-link trees allow flexible storage for objects which tend to grow - in ways that cause the object to be stored discontiguously. B-trees - are described in various algorithms books including "Introduction to - Algorithms" by Thomas H. Cormen, Charles E. Leiserson, and Ronald - L. Rivest. The B-link tree, in which the sibling nodes at a - particular level in the tree are stored in a doubly-linked list, - is described in the "Efficient Locking for Concurrent Operations - on B-trees" paper by Phillip Lehman and S. Bing Yao as published - in the ACM Transactions on Database Systems, Vol. 6, - No. 4, December 1981. - -

The B-link trees implemented by the file format contain one more - key than the number of children. In other words, each child - pointer out of a B-tree node has a left key and a right key. - The pointers out of internal nodes point to sub-trees while - the pointers out of leaf nodes point to symbol nodes and - raw data chunks. - Aside from that difference, internal nodes and leaf nodes - are identical. - -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- B-tree Nodes -
bytebytebytebyte
Node Signature
Node TypeNode LevelEntries Used
Address of Left Sibling
Address of Right Sibling
Key 0 (variable size)
Address of Child 0
Key 1 (variable size)
Address of Child 1
...
Key 2K (variable size)
Address of Child 2K
Key 2K+1 (variable size)
-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Node SignatureThe ASCII character string TREE is - used to indicate the - beginning of a B-link tree node. This gives file - consistency checking utilities a better chance of - reconstructing a damaged file.
Node TypeEach B-link tree points to a particular type of data. - This field indicates the type of data as well as - implying the maximum degree K of the tree and - the size of each Key field. -
-
-
0 -
This tree points to group nodes. -
1 -
This tree points to a new data chunk. -
-
Node LevelThe node level indicates the level at which this node - appears in the tree (leaf nodes are at level zero). Not - only does the level indicate whether child pointers - point to sub-trees or to data, but it can also be used - to help file consistency checking utilities reconstruct - damaged trees.
Entries UsedThis determines the number of children to which this - node points. All nodes of a particular type of tree - have the same maximum degree, but most nodes will point - to less than that number of children. The valid child - pointers and keys appear at the beginning of the node - and the unused pointers and keys appear at the end of - the node. The unused pointers and keys have undefined - values.
Address of Left SiblingThis is the file address of the left sibling of the - current node relative to the super block. If the current - node is the left-most node at this level then this field - is the undefined address (all bits set).
Address of Right SiblingThis is the file address of the right sibling of the - current node relative to the super block. If the current - node is the right-most node at this level then this - field is the undefined address (all bits set).
Keys and Child PointersEach tree has 2K+1 keys with 2K - child pointers interleaved between the keys. The number - of keys and child pointers actually containing valid - values is determined by the Entries Used field. If - that field is N then the B-link tree contains - N child pointers and N+1 keys.
KeyThe format and size of the key values is determined by - the type of data to which this tree points. The keys are - ordered and are boundaries for the contents of the child - pointer; that is, the key values represented by child - N fall between Key N and Key - N+1. Whether the interval is open or closed on - each end is determined by the type of data to which the - tree points. -

- The format of the key depends on the node type. - For nodes of node type 1, the key is formatted as follows: -

- - - - - - - - - - - -
Bytes 1-4Size of chunk in bytes.
Bytes 4-8Filter mask, a 32-bit bitfield indicating which - filters have been applied to that chunk.
N fields of 8 bytes eachA 64-bit index indicating the offset of the - chunk within the dataset where N is the number - of dimensions of the dataset. For example, if - a chunk in a 3-dimensional dataset begins at the - position [5,5,5], there will be three - such 8-bit indices, each with the value of - 5.
-
-

- For nodes of node type 0, the key is formatted as follows: -

- - - - - -
A single field of Size of Lengths - bytesIndicates the byte offset into the local heap - for the first object name in the subtree which - that key describes.
-
-
Child PointersThe tree node contains file addresses of subtrees or - data depending on the node level. Nodes at Level 0 point - to data addresses, either data chunk or group nodes. - Nodes at non-zero levels point to other nodes of the - same B-tree.
-
- -

- Each B-tree node looks like this: - -

- - - - - - - - - - - - - -
key[0]  child[0]  key[1]  child[1]  key[2]  ...  ...  key[N-1]  child[N-1]  key[N]
-
- - where child[i] is a pointer to a sub-tree (at a level - above Level 0) or to data (at Level 0). - Each key[i] describes an item stored by the B-tree - (a chunk or an object of a group node). The range of values - represented by child[i] are indicated by key[i] - and key[i+1]. - - -

The following question must next be answered: - "Is the value described by key[i] contained in - child[i-1] or in child[i]?" - The answer depends on the type of tree. - In trees for groups (node type 0) the object described by - key[i] is the greatest object contained in - child[i-1] while in chunk trees (node type 1) the - chunk described by key[i] is the least chunk in - child[i]. - -

That means that key[0] for group trees is sometimes unused; - it points to offset zero in the heap, which is always the - empty string and compares as "less-than" any valid object name. - -

And key[N] for chunk trees is sometimes unused; - it contains a chunk offset which compares as "greater-than" - any other chunk offset and has a chunk byte size of zero - to indicate that it is not actually allocated. - - -

Disk Format: Level 1B - Group and Symbol Nodes

- -

A group is an object internal to the file that allows - arbitrary nesting of objects (including other groups). - A group maps a set of names to a set of file - address relative to the base address. Certain meta data - for an object to which the group points can be duplicated - in the group symbol table in addition to the object header. - -

An HDF5 object name space can be stored hierarchically by - partitioning the name into components and storing each - component in a group. The group entry for a - non-ultimate component points to the group containing - the next component. The group entry for the last - component points to the object being named. - -

A group is a collection of group nodes pointed - to by a B-link tree. Each group node contains entries - for one or more symbols. If an attempt is made to add a - symbol to an already full group node containing - 2K entries, then the node is split and one node - contains K symbols and the other contains - K+1 symbols. - -

-

- - - - - - - - - - - - - - - - - - - -
- Group Node (A Leaf of a B-tree) -
bytebytebytebyte
Node Signature
Version NumberReserved for Future UseNumber of Symbols


Group Entries


-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Node SignatureThe ASCII character string SNOD is - used to indicate the - beginning of a group node. This gives file - consistency checking utilities a better chance of - reconstructing a damaged file.
Version NumberThe version number for the group node. This - document describes version 1.
Number of SymbolsAlthough all group nodes have the same length, - most contain fewer than the maximum possible number of - symbol entries. This field indicates how many entries - contain valid data. The valid entries are packed at the - beginning of the group node while the remaining - entries contain undefined values.
Group EntriesEach symbol has an entry in the group node. - The format of the entry is described below.
-
- -

- Disk Format: Level 1C - Group Entry

- -

Each group entry in a group node is designed - to allow for very fast browsing of stored objects. - Toward that design goal, the group entries - include space for caching certain constant meta data from the - object header. - -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Group Entry -
bytebytebytebyte
Name Offset (<size> bytes)
Object Header Address
Cache Type
Reserved


Scratch-pad Space (16 bytes)


-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Name OffsetThis is the byte offset into the group local - heap for the name of the object. The name is null - terminated.
Object Header AddressEvery object has an object header which serves as a - permanent location for the object's meta data. In addition - to appearing in the object header, some meta data can be - cached in the scratch-pad space.
Cache TypeThe cache type is determined from the object header. - It also determines the format for the scratch-pad space. -
-
-
0 -
No data is cached by the group entry. This - is guaranteed to be the case when an object header - has a link count greater than one. - -
1 -
Object header meta data is cached in the group - entry. This implies that the group - entry refers to another group. - -
2 -
The entry is a symbolic link. The first four bytes - of the scratch-pad space are the offset into the local - heap for the link value. The object header address - will be undefined. - -
N -
Other cache values can be defined later and - libraries that do not understand the new values will - still work properly. -
-
ReservedThese four bytes are present so that the scratch-pad - space is aligned on an eight-byte boundary. They are - always set to zero.
Scratch-pad SpaceThis space is used for different purposes, depending - on the value of the Cache Type field. Any meta-data - about a dataset object represented in the scratch-pad - space is duplicated in the object header for that - dataset. This meta data can include the datatype - and the size of the dataspace for a dataset whose datatype - is atomic and whose dataspace is fixed and less than - four dimensions. - Furthermore, no data is cached in the group - entry scratch-pad space if the object header for - the group entry has a link count greater than - one.
-
- -

Format of the Scratch-pad Space

- -

The group entry scratch-pad space is formatted - according to the value in the Cache Type field. - -

If the Cache Type field contains the value zero - (0) then no information is - stored in the scratch-pad space. - -

If the Cache Type field contains the value one - (1), then the scratch-pad space - contains cached meta data for another object header - in the following format: - -

-

- - - - - - - - - - - - - - -
- Object Header Scratch-pad Format -
bytebytebytebyte
Address of B-tree
Address of Name Heap
-
- -

-

- - - - - - - - - - - - - - - -
Field NameDescription
Address of B-treeThis is the file address for the root of the - group's B-tree.
Address of Name HeapThis is the file address for the group's local - heap, in which are stored the symbol names.
-
- - -

If the Cache Type field contains the value two - (2), then the scratch-pad space - contains cached meta data for another symbolic link - in the following format: - -

-

- - - - - - - - - - - - - -
- Symbolic Link Scratch-pad Format -
bytebytebytebyte
Offset to Link Value
-
- -

-

- - - - - - - - - - -
Field NameDescription
Offset to Link ValueThe value of a symbolic link (that is, the name of the - thing to which it points) is stored in the local heap. - This field is the 4-byte offset into the local heap for - the start of the link value, which is null terminated.
-
- -

Disk Format: Level 1D - Local Heaps

- -

A heap is a collection of small heap objects. Objects can be - inserted and removed from the heap at any time. - The address of a heap does not change once the heap is created. - References to objects are stored in the group table; - the names of those objects are stored in the local heap. - -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Local Heaps -
bytebytebytebyte
Heap Signature
Reserved (zero)
Data Segment Size
Offset to Head of Free-list (<size> bytes)
Address of Data Segment
-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Heap SignatureThe ASCII character string HEAP - is used to indicate the - beginning of a heap. This gives file consistency - checking utilities a better chance of reconstructing a - damaged file.
Data Segment SizeThe total amount of disk memory allocated for the heap - data. This may be larger than the amount of space - required by the object stored in the heap. The extra - unused space holds a linked list of free blocks.
Offset to Head of Free-listThis is the offset within the heap data segment of the - first free block (or all 0xff bytes if there is no free - block). The free block contains <size> bytes that - are the offset of the next free chunk (or all 0xff bytes - if this is the last free chunk) followed by <size> - bytes that store the size of this free chunk.
Address of Data SegmentThe data segment originally starts immediately after - the heap header, but if the data segment must grow as a - result of adding more objects, then the data segment may - be relocated, in its entirety, to another part of the - file.
-
- -

Objects within the heap should be aligned on an 8-byte boundary. - -

Disk Format: Level 1E - Global Heap

- -

Each HDF5 file has a global heap which stores various types of - information which is typically shared between datasets. The - global heap was designed to satisfy these goals: - -

    -
  1. Repeated access to a heap object must be efficient without - resulting in repeated file I/O requests. Since global heap - objects will typically be shared among several datasets, it is - probable that the object will be accessed repeatedly. - -

    -
  2. Collections of related global heap objects should result in - fewer and larger I/O requests. For instance, a dataset of - void pointers will have a global heap object for each - pointer. Reading the entire set of void pointer objects - should result in a few large I/O requests instead of one small - I/O request for each object. - -

    -
  3. It should be possible to remove objects from the global heap - and the resulting file hole should be eligible to be reclaimed - for other uses. -

    -
- -

The implementation of the heap makes use of the memory - management already available at the file level and combines that - with a new top-level object called a collection to - achieve Goal B. The global heap is the set of all collections. - Each global heap object belongs to exactly one collection and - each collection contains one or more global heap objects. For - the purposes of disk I/O and caching, a collection is treated as - an atomic object. - -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- A Global Heap Collection -
bytebytebytebyte
Magic Number
VersionReserved
Collection Size

Global Heap Object 1 - (described below)


Global Heap Object 2


...


Global Heap Object N


Global Heap Object 0 (free space)

-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Magic NumberThe magic number for global heap collections are the - four bytes G, C, O, - and L.
VersionEach collection has its own version number so that new - collections can be added to old files. This document - describes version zero of the collections. -
Collection Data SizeThis is the size in bytes of the entire collection - including this field. The default (and minimum) - collection size is 4096 bytes which is a typical file - system block size and which allows for 170 16-byte heap - objects plus their overhead.
Object 1 through NThe objects are stored in any order with no - intervening unused space.
Object 0Object 0 (zero), when present, represents the free space in - the collection. Free space always appears at the end of - the collection. If the free space is too small to store - the header for Object 0 (described below) then the - header is implied and the collection contains no free space. -
-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Global Heap Object -
bytebytebytebyte
Object IDReference Count
Reserved
Object Data Size

Object Data

-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Object IDEach object has a unique identification number within a - collection. The identification numbers are chosen so that - new objects have the smallest value possible with the - exception that the identifier 0 always refers to the - object which represents all free space within the - collection.
Reference CountAll heap objects have a reference count field. An - object which is referenced from some other part of the - file will have a positive reference count. The reference - count for Object 0 is always zero.
ReservedZero padding to align next field on an 8-byte - boundary.
Object Size This is the size of the fields - above plus the object data stored for the object. The - actual storage size is rounded up to a multiple of - eight.
Object DataThe object data is treated as a one-dimensional array - of bytes to be interpreted by the caller.
-
- -

Disk Format: Level 1F - Free-space Heap

- -

The Free-space Index is a collection of blocks of data, - dispersed throughout the file, which are currently not used by - any file objects. - -

The super block contains a pointer to root of the free-space description; - that pointer is currently (i.e., in HDF5 Release 1.2) required - to be the undefined address 0xfff...ff. - -

The free-sapce index is not otherwise publicly defined at this time. - - - - - -

-

- - -

Disk Format: Level 2 - Data Objects

- -

Data objects contain the real information in the file. These - objects compose the scientific data and other information which - are generally thought of as "data" by the end-user. All the - other information in the file is provided as a framework for - these data objects. - -

A data object is composed of header information and data - information. The header information contains the information - needed to interpret the data information for the data object as - well as additional "meta-data" or pointers to additional - "meta-data" used to describe or annotate each data object. - -

- Disk Format: Level 2a - Data Object Headers

- -

The header information of an object is designed to encompass - all the information about an object which would be desired to be - known, except for the data itself. This information includes - the dimensionality, number-type, information about how the data - is stored on disk (in external files, compressed, broken up in - blocks, etc.), as well as other information used by the library - to speed up access to the data objects or maintain a file's - integrity. The header of each object is not necessarily located - immediately prior to the object's data in the file and in fact - may be located in any position in the file. - -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Object Headers -
bytebytebytebyte
Version # of Object HeaderReservedNumber of Header Messages
Object Reference Count

Total Object Header Size

Header Message Type #1Size of Header Message Data #1
FlagsReserved

Header Message Data #1

.
.
.
Header Message Type #nSize of Header Message Data #n
FlagsReserved

Header Message Data #n

-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version number of the object headerThis value is used to determine the format of the - information in the object header. When the format of the - information in the object header is changed, the version number - is incremented and can be used to determine how the - information in the object header is formatted.
ReservedAlways set to zero.
Number of header messagesThis value determines the number of messages listed in - this object header. This provides a fast way for software - to prepare storage for the messages in the header.
Object Reference CountThis value specifies the number of references to this - object within the current file. References to the - data object from external files are not tracked.
Total Object Header SizeThis value specifies the total number of bytes of header - message data following this length field for the current - message as well as any continuation data located elsewhere - in the file.
Header Message TypeThe header message type specifies the type of - information included in the header message data following - the type along with a small amount of other information. - Bit 15 of the message type is set if the message is - constant (constant messages cannot be changed since they - may be cached in group entries throughout the - file). The header message types for the pre-defined - header messages will be included in further discussion - below.
Size of Header Message DataThis value specifies the number of bytes of header - message data following the header message type and length - information for the current message. The size includes - padding bytes to make the message a multiple of eight - bytes.
FlagsThis is a bit field with the following definition: -
-
0 -
If set, the message data is constant. This is used - for messages like the datatype message of a dataset. -
1 -
If set, the message is stored in the global heap and - the Header Message Data field contains a Shared Object - message and the Size of Header Message Data field - contains the size of that Shared Object message. -
2-7 -
Reserved -
-
Header Message DataThe format and length of this field is determined by the - header message type and size respectively. Some header - message types do not require any data and this information - can be eliminated by setting the length of the message to - zero. The data is padded with enough zeros to make the - size a multiple of eight.
-
- -

The header message types and the message data associated with - them compose the critical "meta-data" about each object. Some - header messages are required for each object while others are - optional. Some optional header messages may also be repeated - several times in the header itself, the requirements and number - of times allowed in the header will be noted in each header - message description below. - -

The following is a list of currently defined header messages: - -

-

Name: NIL

- Type: 0x0000
- Length: varies
- Status: Optional, may be repeated.
- Purpose and Description: The NIL message is used to - indicate a message - which is to be ignored when reading the header messages for a data object. - [Probably one which has been deleted for some reason.]
- Format of Data: Unspecified.
- - - - -
-

Name: Simple Dataspace

- - Type: 0x0001
- Length: Varies according to the number of dimensions, - as described in the following table
- Status: The Simple Dataspace message is required - and may not be repeated. This message is currently used with - datasets and named dataspaces.
- -

The Simple Dataspace message describes the number - of dimensions and size of each dimension that the data object - has. This message is only used for datasets which have a - simple, rectilinear grid layout; datasets requiring a more - complex layout (irregularly structured or unstructured grids, etc.) - must use the Complex Dataspace message for expressing - the space the dataset inhabits. - (Note: The Complex Dataspace functionality is - not yet implemented (as of HDF5 Release 1.2). It is not described - in this document.) - -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Simple Dataspace Message -
bytebytebytebyte
VersionDimensionalityFlagsReserved
Reserved
Dimension Size #1 (<size> bytes)
.
.
.
Dimension Size #n (<size> bytes)
Dimension Maximum #1 (<size> bytes)
.
.
.
Dimension Maximum #n (<size> bytes)
Permutation Index #1
.
.
.
Permutation Index #n
-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version This value is used to determine the format of the - Simple Dataspace Message. When the format of the - information in the message is changed, the version number - is incremented and can be used to determine how the - information in the object header is formatted.
DimensionalityThis value is the number of dimensions that the data - object has.
FlagsThis field is used to store flags to indicate the - presence of parts of this message. Bit 0 (the least - significant bit) is used to indicate that maximum - dimensions are present. Bit 1 is used to indicate that - permutation indices are present for each dimension.
Dimension Size #n (<size> bytes)This value is the current size of the dimension of the - data as stored in the file. The first dimension stored in - the list of dimensions is the slowest changing dimension - and the last dimension stored is the fastest changing - dimension.
Dimension Maximum #n (<size> bytes)This value is the maximum size of the dimension of the - data as stored in the file. This value may be the special - value <UNLIMITED> (all bits set) which indicates - that the data may expand along this dimension - indefinitely. If these values are not stored, the maximum - value of each dimension is assumed to be the same as the - current size value.
Permutation Index #n (4 bytes)This value is the index permutation used to map - each dimension from the canonical representation to an - alternate axis for each dimension. If these values are - not stored, the first dimension stored in the list of - dimensions is the slowest changing dimension and the last - dimension stored is the fastest changing dimension.
-
- - - - - - - -
-

Name: Datatype

- - Type: 0x0003
- Length: variable
- Status: One required per dataset or named datatype
- -

The datatype message defines the datatype for each data point - of a dataset. A datatype can describe an atomic type like a - fixed- or floating-point type or a compound type like a C - struct. A datatype does not, however, describe how data points - are combined to produce a dataset. Datatypes are stored on disk - as a datatype message, which is a list of datatype classes and - their associated properties. - -

-

- - - - - - - - - - - - - - - - - - - - - - -
- Datatype Message -
bytebytebytebyte
Type Class and VersionClass Bit Field
Size in Bytes (4 bytes)


Properties


-
- -

The Class Bit Field and Properties fields vary depending - on the Type Class, which is the low-order four bits of the Type - Class and Version field (the high-order four bits are the - version, which should be set to the value one). The type class - is one of 0 (fixed-point number), 1 (floating-point number), - 2 (date and time), 3 (text string), 4 (bit field), 5 (opaque), - 6 (compound), 7 (reference), 8 (enumeration), or 9 (variable-length). - The Class Bit Field is zero and the size of the - Properties field is zero except for the cases noted here. - -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Bit Field for Fixed-point Numbers (Class 0) -
BitsMeaning
0Byte Order. If zero, byte order is little-endian; - otherwise, byte order is big endian.
1, 2Padding type. Bit 1 is the lo_pad type and bit 2 - is the hi_pad type. If a datum has unused bits at either - end, then the lo_pad or hi_pad bit is copied to those - locations.
3Signed. If this bit is set then the fixed-point - number is in 2's complement form.
4-23Reserved (zero).
-
- -

-

- - - - - - - - - - - - - - -
- Properties for Fixed-point Numbers (Class 0) -
ByteByteByteByte
Bit OffsetBit Precision
-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Bit Field for Floating-point Numbers (Class 1) -
BitsMeaning
0Byte Order. If zero, byte order is little-endian; - otherwise, byte order is big endian.
1, 2, 3Padding type. Bit 1 is the low bits pad type, bit 2 - is the high bits pad type, and bit 3 is the internal bits - pad type. If a datum has unused bits at either or between - the sign bit, exponent, or mantissa, then the value of bit - 1, 2, or 3 is copied to those locations.
4-5Normalization. The value can be 0 if there is no - normalization, 1 if the most significant bit of the - mantissa is always set (except for 0.0), and 2 if the most - significant bit of the mantissa is not stored but is - implied to be set. The value 3 is reserved and will not - appear in this field.
6-7Reserved (zero).
8-15Sign. This is the bit position of the sign - bit.
16-23Reserved (zero).
-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - -
- Properties for Floating-point Numbers (Class 1) -
ByteByteByteByte
Bit OffsetBit Precision
Exponent LocationExponent Size in BitsMantissa LocationMantissa Size in Bits
Exponent Bias
-
- -

-

- - - - - - - - - - - - - - - - - - - - - -
- Bit Field for Strings (Class 3) -
BitsMeaning
0-3Padding type. This four-bit value determines the - type of padding to use for the string. The values are: - -
-
0 Null terminate. -
A zero byte marks the end of the string and is - guaranteed to be present after converting a long - string to a short string. When converting a short - string to a long string the value is padded with - additional null characters as necessary. - -

-
1 Null pad. -
Null characters are added to the end of the value - during conversions from short values to long values - but conversion in the opposite direction simply - truncates the value. - -

-
2 Space pad. -
Space characters are added to the end of the value - during conversions from short values to long values - but conversion in the opposite direction simply - truncates the value. This is the Fortran - representation of the string. - -

-
3-15 Reserved. -
These values are reserved for future use. -
-
4-7Character Set. The character set to use for - encoding the string. The only character set supported is - the 8-bit ASCII (zero) so no translations have been defined - yet.
8-23Reserved (zero).
-
- -

-

- - - - - - - - - - - - - - - - - - - - - - -
- Bit Field for Bitfield Types (Class 4) -
BitsMeaning
0Byte Order. If zero, byte order is little-endian; - otherwise, byte order is big endian.
1, 2Padding type. Bit 1 is the lo_pad type and bit 2 - is the hi_pad type. If a datum has unused bits at either - end, then the lo_pad or hi_pad bit is copied to those - locations.
3-23Reserved (zero).
-
- -

-

- - - - - - - - - - - - - - -
- Properties for Bitfield Types (Class 4) -
ByteByteByteByte
Bit OffsetBit Precision
-
- -

-

- - - - - - - - - - - - -
- Bit Field for Opaque Types (Class 5) -
BitsMeaning
0-23Reserved (zero).
-
- -

-

- - - - - - - - - - - - - -
- Properties for Opaque Types (Class 5) -
ByteByteByteByte

Null-terminated ASCII Tag
- (multiple of 8 bytes)

-
- -

-

- - - - - - - - - - - - - - - - -
- Bit Field for Compound Types (Class 6) -
BitsMeaning
0-15Number of Members. This field contains the number - of members defined for the compound datatype. The member - definitions are listed in the Properties field of the data - type message. -
15-23Reserved (zero).
-
- -

The Properties field of a compound datatype is a list of the - member definitions of the compound datatype. The member - definitions appear one after another with no intervening bytes. - The member types are described with a recursive datatype - message. - -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Properties for Compound Types (Class 6) -
ByteByteByteByte


Name (null terminated, multiple of - eight bytes)


Byte Offset of Member in Compound Instance
Dimensionalityreserved
Dimension Permutation
Reserved
Size of Dimension 0 (required)
Size of Dimension 1 (required)
Size of Dimension 2 (required)
Size of Dimension 3 (required)


Member Type Message


-
- -

-

- - - - - - - - - - - - - - - - - -
- Bit Field for Enumeration Types (Class 8) -
BitsMeaning
0-15Number of Members. The number of name/value - pairs defined for the enumeration type.
16-23Reserved (zero).
-
- -

-

- - - - - - - - - - - - - - - - - - - - - - -
- Properties for Enumeration Types (Class 8) -
ByteByteByteByte

Parent Type


Names


Values

-
- -
- - - - - - - - - - - -
Parent Type:Each enumeration type is based on some parent type, - usually an integer. The information for that parent type is - described recursively by this field.
Names:The name for each name/value pair. Each name is - stored as a null terminated ASCII string in a multiple of - eight bytes. The names are in no particular order.
Values:The list of values in the same order as the names. - The values are packed (no inter-value padding) and the - size of each value is determined by the parent type.
-
- - -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Bit Field for Variable-length Types (Class 9) -
BitsMeaning
0-3
Type
-
0 Variable-length sequence
-
This variable-length datatype can be of any sequence - of data. Variable-length sequences do not have padding - or character set information.
-
1 Variable-length string
-
This variable-length datatype is composed of a series of - characters. Variable-length strings have padding and - character set information.
-
4-7
Padding type (variable-length string only)
-
This four-bit value determines the type of padding - used for variable-length strings. The values are the same - as for the string padding type, as follows:
-
0 Null terminate
-
A zero byte marks the end of a string and is guaranteed - to be present after converting a long string to a short - string. When converting a short string to a long string, - the value is padded with additional null characters - as necessary. -
1 Null pad
-
Null characters are added to the end of the value - during conversion from a short string to a longer string. - Conversion from a long string to a shorter string - simply truncates the value.
-
2 Space pad
-
Space characters are added to the end of the value - during conversion from a short string to a longer string. - Conversion from a long string to a shorter string simply - truncates the value. - This is the Fortran representation of the string. -
-
3-15 Reserved
-
These values are reserved for future use.
-
8-11
Character set (variable-length string only)
-
This four-bit value specifies the character set - to be used for encoding the string.
-
0 8-bit ASCII
-
As of this writing (July 2002, Release 1.4.4), - 8-bit ASCII is the only character set supported. - Therefore, no translations have been defined.
-
12-23Reserved (zero).
-
- -

-

- - - - - - - - - - - - - - -
- Properties for Variable-length Types (Class 9) -
ByteByteByteByte

Parent Type

-
- -
- - - - - -
Parent Type:Each variable-length type is based on - some parent type. The information for that parent type is - described recursively by this field.
-
- - - -

- - - - -

-

Name: Data Storage - Fill Value

- Type: 0x0004
- Length: varies
- Status: Optional, may not be repeated.
- -

The fill value message stores a single data point value which - is returned to the application when an uninitialized data point - is read from the dataset. The fill value is interpreted with - the same datatype as the dataset. If no fill value message is - present then a fill value of all zero is assumed. - -

-

- - - - - - - - - - - - - - - - - -
- Fill Value Message -
bytebytebytebyte
Size (4 bytes)

Fill Value

-
- -

-

- - - - - - - - - - - - - - - -
Field NameDescription
Size (4 bytes)This is the size of the Fill Value field in bytes.
Fill ValueThe fill value. The bytes of the fill value are - interpreted using the same datatype as for the dataset.
-
- -
-

Name: Reserved - Not Assigned Yet

- Type: 0x0005
- Length: N/A
- Status: N/A
- - - -
-

Name: Data Storage - Compact

- - Type: 0x0006
- Length: varies
- Status: Optional, may not be repeated.
- -

This message indicates that the data for the data object is - stored within the current HDF file by including the actual - data as the header data for this message. The data is - stored internally in - the normal format, i.e. in one chunk, uncompressed, etc. - -

Note that one and only one of the Data Storage headers can be - stored for each data object. - -

Format of Data: The message data is actually composed - of dataset data, so the format will be determined by the dataset - format. - - - -

-

Name: Data Storage - - External Data Files

- Type: 0x0007
- Length: varies
- Status: Optional, may not be repeated.
- -

Purpose and Description: The external object message - indicates that the data for an object is stored outside the HDF5 - file. The filename of the object is stored as a Universal - Resource Location (URL) of the actual filename containing the - data. An external file list record also contains the byte offset - of the start of the data within the file and the amount of space - reserved in the file for that data. - -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- External File List Message -
bytebytebytebyte
VersionReserved
Allocated SlotsUsed Slots

Heap Address


Slot Definitions...

-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version This value is used to determine the format of the - External File List Message. When the format of the - information in the message is changed, the version number - is incremented and can be used to determine how the - information in the object header is formatted.
ReservedThis field is reserved for future use.
Allocated SlotsThe total number of slots allocated in the message. Its - value must be at least as large as the value contained in - the Used Slots field.
Used SlotsThe number of initial slots which contain valid - information. The remaining slots are zero filled.
Heap AddressThis is the address of a local name heap which contains - the names for the external files. The name at offset zero - in the heap is always the empty string.
Slot DefinitionsThe slot definitions are stored in order according to - the array addresses they represent. If more slots have - been allocated than what has been used then the defined - slots are all at the beginning of the list.
-
- -

-

- - - - - - - - - - - - - - - - - - - - - -
- External File List Slot -
bytebytebytebyte

Name Offset (<size> bytes)


File Offset (<size> bytes)


Size

-
- -

-

- - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Name Offset (<size> bytes)The byte offset within the local name heap for the name - of the file. File names are stored as a URL which has a - protocol name, a host name, a port number, and a file - name: - protocol:port//host/file. - If the protocol is omitted then "file:" is assumed. If - the port number is omitted then a default port for that - protocol is used. If both the protocol and the port - number are omitted then the colon can also be omitted. If - the double slash and host name are omitted then - "localhost" is assumed. The file name is the only - mandatory part, and if the leading slash is missing then - it is relative to the application's current working - directory (the use of relative names is not - recommended).
File Offset (<size> bytes)This is the byte offset to the start of the data in the - specified file. For files that contain data for a single - dataset this will usually be zero.
SizeThis is the total number of bytes reserved in the - specified file for raw data storage. For a file that - contains exactly one complete dataset which is not - extendable, the size will usually be the exact size of the - dataset. However, by making the size larger one allows - HDF5 to extend the dataset. The size can be set to a value - larger than the entire file since HDF5 will read zeros - past the end of the file without failing.
-
- - -
-

Name: Data Storage - Layout

- - Type: 0x0008
- Length: varies
- Status: Required for datasets, may not be repeated. - -

Purpose and Description: Data layout describes how the - elements of a multi-dimensional array are arranged in the linear - address space of the file. Two types of data layout are - supported: - -

    -
  1. The array can be stored in one contiguous area of the file. - The layout requires that the size of the array be constant and - does not permit chunking, compression, checksums, encryption, - etc. The message stores the total size of the array and the - offset of an element from the beginning of the storage area is - computed as in C. - -
  2. The array domain can be regularly decomposed into chunks and - each chunk is allocated separately. This layout supports - arbitrary element traversals, compression, encryption, and - checksums, and the chunks can be distributed across external - raw data files (these features are described in other - messages). The message stores the size of a chunk instead of - the size of the entire array; the size of the entire array can - be calculated by traversing the B-tree that stores the chunk - addresses. -
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Data Layout Message -
bytebytebytebyte
VersionDimensionalityLayout ClassReserved
Reserved

Address

Dimension 0 (4-bytes)
Dimension 1 (4-bytes)
...
-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
VersionA version number for the layout message. This - documentation describes version one.
DimensionalityAn array has a fixed dimensionality. This field - specifies the number of dimension size fields later in the - message.
Layout ClassThe layout class specifies how the other fields of the - layout message are to be interpreted. A value of one - indicates contiguous storage while a value of two - indicates chunked storage. Other values will be defined - in the future.
AddressFor contiguous storage, this is the address of the first - byte of storage. For chunked storage this is the address - of the B-tree that is used to look up the addresses of the - chunks.
DimensionsFor contiguous storage the dimensions define the entire - size of the array while for chunked storage they define - the size of a single chunk.
-
- - -
-

Name: Reserved - Not Assigned Yet

- Type: 0x0009
- Length: N/A
- Status: N/A
- Purpose and Description: N/A
- Format of Data: N/A - -
-

Name: Reserved - Not Assigned Yet

- Type: 0x000A
- Length: N/A
- Status: N/A
- Purpose and Description: N/A
- Format of Data: N/A - -
-

Name: Data Storage - Filter Pipeline

- Type: 0x000B
- Length: varies
- Status: Optional, may not be repeated. - -

Purpose and Description: This message describes the - filter pipeline which should be applied to the data stream by - providing filter identification numbers, flags, a name, an - client data. - -

-

- - - - - - - - - - - - - - - - - - - - - - - -
- Filter Pipeline Message -
bytebytebytebyte
VersionNumber of FiltersReserved
Reserved

Filter List

-
- -

-

- - - - - - - - - - - - - - - - - - - - -
Field NameDescription
VersionThe version number for this message. This document - describes version one.
Number of FiltersThe total number of filters described by this - message. The maximum possible number of filters in a - message is 32.
Filter ListA description of each filter. A filter description - appears in the next table.
-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Filter Pipeline Message -
bytebytebytebyte
Filter IdentificationName Length
FlagsClient Data Number of Values

Name


Client Data

Padding
-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Filter IdentificationThis is a unique (except in the case of testing) - identifier for the filter. Values from zero through 255 - are reserved for filters defined by the NCSA HDF5 - library. Values 256 through 511 have been set aside for - use when developing/testing new filters. The remaining - values are allocated to specific filters by contacting the - HDF5 development team. -
Name LengthEach filter has an optional null-terminated ASCII name - and this field holds the length of the name including the - null termination padded with nulls to be a multiple of - eight. If the filter has no name then a value of zero is - stored in this field.
FlagsThe flags indicate certain properties for a filter. The - bit values defined so far are: - -
-
bit 1 -
If set then the filter is an optional filter. - During output, if an optional filter fails it will be - silently removed from the pipeline. -
-
Client Data Number of ValuesEach filter can store a few integer values to control - how the filter operates. The number of entries in the - Client Data array is stored in this field.
NameIf the Name Length field is non-zero then it will - contain the size of this field, a multiple of eight. This - field contains a null-terminated, ASCII character - string to serve as a comment/name for the filter.
Client DataThis is an array of four-byte integers which will be - passed to the filter function. The Client Data Number of - Values determines the number of elements in the - array.
PaddingFour bytes of zeros are added to the message at this - point if the Client Data Number of Values field contains - an odd number.
-
- -
-

Name: Attribute

- Type: 0x000C
- Length: varies
- Status: Optional, may be repeated.
- -

Purpose and Description: The Attribute - message is used to list objects in the HDF file which are used - as attributes, or "meta-data" about the current object. An - attribute is a small dataset; it has a name, a datatype, a data - space, and raw data. Since attributes are stored in the object - header they must be relatively small (<64kb) and can be - associated with any type of object which has an object header - (groups, datasets, named types and spaces, etc.). - -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Attribute Message -
bytebytebytebyte
VersionReservedName Size
Type SizeSpace Size

Name


Type


Space


Data

-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
VersionVersion number for the message. This document describes - version 1 of attribute messages.
ReservedThis field is reserved for later use and is set to - zero.
Name SizeThe length of the attribute name in bytes including the - null terminator. Note that the Name field below may - contain additional padding not represented by this - field.
Type SizeThe length of the datatype description in the Type - field below. Note that the Type field may contain - additional padding not represented by this field.
Space SizeThe length of the dataspace description in the Space - field below. Note that the Space field may contain - additional padding not represented by this field.
NameThe null-terminated attribute name. This field is - padded with additional null characters to make it a - multiple of eight bytes.
TypeThe datatype description follows the same format as - described for the datatype object header message. This - field is padded with additional zero bytes to make it a - multiple of eight bytes.
SpaceThe dataspace description follows the same format as - described for the dataspace object header message. This - field is padded with additional zero bytes to make it a - multiple of eight bytes.
DataThe raw data for the attribute. The size is determined - from the datatype and dataspace descriptions. This - field is not padded with additional zero - bytes.
-
- -
-

Name: Object Name

- -

Type: 0x000D
- Length: varies
- Status: Optional, may not be repeated. - -

Purpose and Description: The object name or comment is - designed to be a short description of an object. An object name - is a sequence of non-zero (\0) ASCII characters with no other - formatting included by the library. - -

-

- - - - - - - - - - - - - -
- Name Message -
bytebytebytebyte

Name

-
- -

-

- - - - - - - - - - -
Field NameDescription
NameA null terminated ASCII character string.
-
- -
-

Name: Object Modification Date & Time

- -

Type: 0x000E
- Length: fixed
- Status: Optional, may not be repeated. - -

Purpose and Description: The object modification date - and time is a timestamp which indicates (using ISO-8601 date and - time format) the last modification of an object. The time is - updated when any object header message changes according to the - system clock where the change was posted. - -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Modification Time Message -
bytebytebytebyte
Year
MonthDay of Month
HourMinute
SecondReserved
-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
YearThe four-digit year as an ASCII string. For example, - 1998. All fields of this message should be interpreted - as coordinated universal time (UTC)
MonthThe month number as a two digit ASCII string where - January is 01 and December is 12.
Day of MonthThe day number within the month as a two digit ASCII - string. The first day of the month is 01.
HourThe hour of the day as a two digit ASCII string where - midnight is 00 and 11:00pm is 23.
MinuteThe minute of the hour as a two digit ASCII string where - the first minute of the hour is 00 and - the last is 59.
SecondThe second of the minute as a two digit ASCII string - where the first second of the minute is 00 - and the last is 59.
ReservedThis field is reserved and should always be zero.
-
- -
-

Name: Shared Object Message

- Type: 0x000F
- Length: 4 Bytes
- Status: Optional, may be repeated. - -

A constant message can be shared among several object headers - by writing that message in the global heap and having the object - headers all point to it. The pointing is accomplished with a - Shared Object message which is understood directly by the object - header layer of the library. It is also possible to have a - message of one object header point to a message in some other - object header, but care must be exercised to prevent cycles. - -

If a message is shared, then the message appears in the global - heap and its message ID appears in the Header Message Type - field of the object header. Also, the Flags field in the object - header for that message will have bit two set (the - H5O_FLAG_SHARED bit). The message body in the - object header will be that of a Shared Object message defined - here and not that of the pointed-to message. - -

-

- - - - - - - - - - - - - - - - - - - -
- Shared Message Message -
byte - byte - byte - byte -
VersionFlagsReserved
Reserved

Pointer

-
- -

-

- - - - - - - - - - - - - - - - - - - -
Field NameDescription
VersionThe version number for the message. This document - describes version one of shared messages.
FlagsThe Shared Message message points to a message which is - shared among multiple object headers. The Flags field - describes the type of sharing: - -
-
Bit 0 -
If this bit is clear then the actual message is the - first message in some other object header; otherwise - the actual message is stored in the global heap. - -
Bits 2-7 -
Reserved (always zero) -
-
PointerThis field points to the actual message. The format of - the pointer depends on the value of the Flags field. If - the actual message is in the global heap then the pointer - is the file address of the global heap collection that - holds the message, and a four-byte index into that - collection. Otherwise the pointer is a group entry - that points to some other object header.
-
- - -
-

Name: Object Header Continuation

-Type: 0x0010
-Length: fixed
-Status: Optional, may be repeated.
-Purpose and Description: The object header continuation is the location -in the file of more header messages for the current data object. This can be -used when header blocks are large, or likely to change over time.
-Format of Data:

- The object header continuation is formatted as follows (assuming a 4-byte -length & offset are being used in the current file): - -

-

- - - - - - - - - - - - - -
-HDF5 Object Header Continuation Message Layout -
bytebytebytebyte
Header Continuation Offset
Header Continuation Length
-
- -

-

-
The elements of the Header Continuation Message are described below: -
-
-
Header Continuation Offset: (<offset> bytes) -
This value is the offset in bytes from the beginning of the file where the -header continuation information is located. -
Header Continuation Length: (<length> bytes) -
This value is the length in bytes of the header continuation information in -the file. -
-
- - - -
-

Name: Group Message

-Type: 0x0011
-Length: fixed
-Status: Required for groups, may not be repeated.
-Purpose and Description: Each group has a B-tree and a -name heap which are pointed to by this message.
-Format of data: -

The group message is formatted as follows: - -

-

- - - - - - - - - - - - - - -
-HDF5 Object Header Group Message Layout -
bytebytebytebyte
B-tree Address
Heap Address
-
- -

-

-
The elements of the Group Message are described below: -
-
-
B-tree Address (<offset> bytes) -
This value is the offset in bytes from the beginning of the file -where the B-tree is located. -
Heap Address (<offset> bytes) -
This value is the offset in bytes from the beginning of the file -where the group name heap is located. -
-
- -

Disk Format: Level 2b - Shared Data Object Headers

-

In order to share header messages between several dataset objects, object -header messages may be placed into the global heap. Since these -messages require additional information beyond the basic object header message -information, the format of the shared message is detailed below. - -

-

- - - - - - - - - - - - - -
-HDF5 Shared Object Header Message -
bytebytebytebyte
Reference Count of Shared Header Message

Shared Object Header Message

-
- -

-

-
The elements of the shared object header message are described below: -
-
-
Reference Count of Shared Header Message: (32-bit unsigned integer) -
This value is used to keep a count of the number of dataset objects which -refer to this message from their dataset headers. When this count reaches zero, -the shared message header may be removed from the global heap. -
Shared Object Header Message: (various lengths) -
The data stored for the shared object header message is formatted in the -same way as the private object header messages described in the object header -description earlier in this document and begins with the header message Type. -
-
- - -

Disk Format: Level 2c - Data Object Data Storage

-

The data for an object is stored separately from the header -information in the file and may not actually be located in the HDF5 file -itself if the header indicates that the data is stored externally. The -information for each record in the object is stored according to the -dimensionality of the object (indicated in the dimensionality header message). -Multi-dimensional data is stored in C order [same as current scheme], i.e. the -"last" dimension changes fastest. -

Data whose elements are composed of simple number-types are stored in -native-endian IEEE format, unless they are specifically defined as being stored -in a different machine format with the architecture-type information from the -number-type header message. This means that each architecture will need to -[potentially] byte-swap data values into the internal representation for that -particular machine. -

Data with a "variable" sized number-type is stored in a data heap -internal to the HDF5 file. Global heap identifiers are stored in the -data object storage. -

Data whose elements are composed of pointer number-types are stored in several -different ways depending on the particular pointer type involved. Simple -pointers are just stored as the dataset offset of the object being pointed to with the -size of the pointer being the same number of bytes as offsets in the file. -Partial-object pointers are stored as a heap-ID which points to the following -information within the file-heap: an offset of the object pointed to, number-type -information (same format as header message), dimensionality information (same -format as header message), sub-set start and end information (i.e. a coordinate -location for each), and field start and end names (i.e. a [pointer to the] -string indicating the first field included and a [pointer to the] string name -for the last field). - -

Data of a compound datatype is stored as a contiguous stream of the items -in the structure, with each item formatted according to its datatype. - -

- - diff --git a/doxygen/examples/H5.format.1.1.html b/doxygen/examples/H5.format.1.1.html deleted file mode 100644 index 418afd5ab88..00000000000 --- a/doxygen/examples/H5.format.1.1.html +++ /dev/null @@ -1,6440 +0,0 @@ - - - - HDF5 File Format Specification Version 1.1 - - - - - -
-
- - - -
-
    -
  1. Introduction -
  2. Disk Format Level 0 - File Metadata - -
      -
    1. Disk Format Level 0A - File Signature and Super Block -
    2. Disk Format Level 0B - File Driver Info -
    -     -
  3. Disk Format Level 1 - File Infrastructure - -
      -
    1. Disk Format Level 1A - B-link Trees and B-tree Nodes -
    2. Disk Format Level 1B - Group -
    3. Disk Format Level 1C - Group Entry -
    4. Disk Format Level 1D - Local Heaps -
    5. Disk Format Level 1E - Global Heap -
    6. Disk Format Level 1F - Free-space Index -
    -     -
  4. Disk Format Level 2 - Data Objects - -
      -
    1. Disk Format Level 2a - Data Object Headers -
        -
      1. Name: NIL -
      2. Name: Simple Dataspace - -
      3. Name: Reserved - not assigned yet -
      4. Name: Datatype -
      5. Name: Data Storage - Fill Value (Old) -
      6. Name: Data Storage - Fill Value -
      -
    -     -
- 		   -
    - -
  1. Disk Format Level 2 - Data Objects - (Continued) -
      -
    1. Disk Format Level 2a - Data Object Headers(Continued) -
        - -
      1. Name: Reserved - not assigned yet -
      2. Name: Data Storage - External Data Files -
      3. Name: Data Storage - Layout -
      4. Name: Reserved - not assigned yet -
      5. Name: Reserved - not assigned yet -
      6. Name: Data Storage - Filter Pipeline -
      7. Name: Attribute -
      8. Name: Object Comment -
      9. Name: Object Modification Date and Time (Old) -
      10. Name: Shared Object Message -
      11. Name: Object Header Continuation -
      12. Name: Group Message -
      13. Name: Object Modification Date and Time -
      -
    2. Disk Format: Level 2b - Data Object Data Storage -
    -     -
  2. Appendix -
-
-
- -
-
- - -

Introduction

- - - - - - - -
  -
- HDF5 Groups - 		 
  - Figure 1: Relationships among the HDF5 root group, other groups, and objects -
- 		 
  - HDF5 Objects -  
  - Figure 2: HDF5 objects -- datasets, datatypes, or dataspaces -
- 		 
- - -

The format of an HDF5 file on disk encompasses several - key ideas of the HDF4 and AIO file formats as well as - addressing some shortcomings therein. The new format is - more self-describing than the HDF4 format and is more - uniformly applied to data objects in the file. - -

An HDF5 file appears to the user as a directed graph. - The nodes of this graph are the higher-level HDF5 objects - that are exposed by the HDF5 APIs: - -

    -
  • Groups -
  • Datasets -
  • Named datatypes -
- -

At the lowest level, as information is actually written to the disk, - an HDF5 file is made up of the following objects: -

    -
  • A super block -
  • B-tree nodes (containing either symbol nodes or raw data chunks) -
  • Object headers -
  • A global heap -
  • Local heaps -
  • Free space -
- -

The HDF5 library uses these low-level objects to represent the - higher-level objects that are then presented to the user or - to applications through the APIs. - For instance, a group is an object header that contains a message that - points to a local heap and to a B-tree which points to symbol nodes. - A dataset is an object header that contains messages that describe - datatype, space, layout, filters, external files, fill value, etc - with the layout message pointing to either a raw data chunk or to a - B-tree that points to raw data chunks. - - -

This Document

- -

This document describes the lower-level data objects; - the higher-level objects and their properties are described - in the HDF5 User Guide. - -

Three levels of information comprise the file format. - Level 0 contains basic information for identifying and - defining information about the file. Level 1 information contains - the information about the pieces of a file shared by many objects - in the file (such as a B-trees and heaps). Level 2 is the rest - of the file and contains all of the data objects, with each object - partitioned into header information, also known as - metadata, and data. - -

The sizes of various fields in the following layout tables are - determined by looking at the number of columns the field spans - in the table. There are three exceptions: (1) The size may be - overridden by specifying a size in parentheses, (2) the size of - addresses is determined by the Size of Offsets field - in the super block and is indicated in this document with a - superscripted 'O', and (3) the size of length fields is determined - by the Size of Lengths field in the super block and is - indicated in this document with a superscripted 'L'. - -

Values for all fields in this document should be treated as unsigned - integers, unless otherwise noted in the description of a field. - Additionally, all metadata fields are stored in little-endian byte - order. -

- -
-
- -

- Disk Format: Level 0 - File Metadata

- -

- Disk Format: Level 0A - File Signature and Super Block

- -

The super block may begin at certain predefined offsets within - the HDF5 file, allowing a block of unspecified content for - users to place additional information at the beginning (and - end) of the HDF5 file without limiting the HDF5 library's - ability to manage the objects within the file itself. This - feature was designed to accommodate wrapping an HDF5 file in - another file format or adding descriptive information to the - file without requiring the modification of the actual file's - information. The super block is located by searching for the - HDF5 file signature at byte offset 0, byte offset 512 and at - successive locations in the file, each a multiple of two of - the previous location, i.e. 0, 512, 1024, 2048, etc. - -

The super block is composed of a file signature, followed by - super block and group version numbers, information - about the sizes of offset and length values used to describe - items within the file, the size of each group page, - and a group entry for the root object in the file. - -
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- HDF5 Super Block Layout -
bytebytebytebyte

HDF5 File Signature (8 bytes)

Version # of Super BlockVersion # of Global Free-space StorageVersion # of Root Group Symbol Table EntryReserved (zero)
Version # of Shared Header Message FormatSize of OffsetsSize of LengthsReserved (zero)
Group Leaf Node KGroup Internal Node K
File Consistency Flags
Indexed Storage Internal Node K1Reserved (zero)1
Base AddressO
Address of Global Free-space HeapO
End of File AddressO
Driver Information Block AddressO
Root Group Symbol Table Entry
- - - - -
- (Items marked with an 'O' the above table are -
- of the size specified in "Size of Offsets.") -
- (Items marked with an '1' the above table are -
- new in version 1 of the superblock) -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
HDF5 File Signature -

This field contains a constant value and can be used to - quickly identify a file as being an HDF5 file. The - constant value is designed to allow easy identification of - an HDF5 file and to allow certain types of data corruption - to be detected. The file signature of an HDF5 file always - contains the following values: -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Decimal:13772687013102610
Hexadecimal:894844460d0a1a0a
ASCII C Notation:\211HDF\r\n\032\n
-
-
- -

This signature both identifies the file as an HDF5 file - and provides for immediate detection of common - file-transfer problems. The first two bytes distinguish - HDF5 files on systems that expect the first two bytes to - identify the file type uniquely. The first byte is - chosen as a non-ASCII value to reduce the probability - that a text file may be misrecognized as an HDF5 file; - also, it catches bad file transfers that clear bit - 7. Bytes two through four name the format. The CR-LF - sequence catches bad file transfers that alter newline - sequences. The control-Z character stops file display - under MS-DOS. The final line feed checks for the inverse - of the CR-LF translation problem. (This is a direct - descendent of the PNG file - signature.) -

- -

This field is present in version 0+ of the superblock. -

-
Version Number of the Super Block -

This value is used to determine the format of the - information in the super block. When the format of the - information in the super block is changed, the version number - is incremented to the next integer and can be used to - determine how the information in the super block is - formatted. -

- -

Values of 0 and 1 are defined for this field. -

- -

This field is present in version 0+ of the superblock. -

-
Version Number of the File Free-space Information -

This value is used to determine the format of the - information in the File Free-space Information. -

-

The only value currently valid in this field is '0', which - indicates that the free space index is formatted as described - below. -

- -

This field is present in version 0+ of the superblock. -

-
Version Number of the Root Group Symbol Table Entry -

This value is used to determine the format of the - information in the Root Group Symbol Table Entry. When the - format of the information in that field is changed, the - version number is incremented to the next integer and can be - used to determine how the information in the field - is formatted. -

-

The only value currently valid in this field is '0', which - indicates that the root group symbol table entry is formatted as - described below. -

- -

This field is present in version 0+ of the superblock. -

-
Version Number of the Shared Header Message Format -

This value is used to determine the format of the - information in a shared object header message. Since the format - of the shared header messages differs from the other private - header messages, a version number is used to identify changes - in the format. -

-

The only value currently valid in this field is '0', which - indicates that shared header messages are formatted as - described below. -

- -

This field is present in version 0+ of the superblock. -

-
Size of Offsets -

This value contains the number of bytes used to store - addresses in the file. The values for the addresses of - objects in the file are offsets relative to a base address, - usually the address of the super block signature. This - allows a wrapper to be added after the file is created - without invalidating the internal offset locations. -

- -

This field is present in version 0+ of the superblock. -

-
Size of Lengths -

This value contains the number of bytes used to store - the size of an object. -

- -

This field is present in version 0+ of the superblock. -

-
Group Leaf Node K -

Each leaf node of a group B-tree will have at - least this many entries but not more than twice this - many. If a group has a single leaf node then it - may have fewer entries. -

-

This value must be greater than zero. -

-

See the description of B-trees below. -

- -

This field is present in version 0+ of the superblock. -

-
Group Internal Node K -

Each internal node of a group B-tree will have at - least this many entries but not more than twice this - many. If the group has only one internal - node then it might have fewer entries. -

-

This value must be greater than zero. -

-

See the description of B-trees below. -

- -

This field is present in version 0+ of the superblock. -

-
File Consistency Flags -

This value contains flags to indicate information - about the consistency of the information contained - within the file. Currently, the following bit flags are - defined: -

    -
  • Bit 0 set indicates that the file is opened for - write-access. -
  • Bit 1 set indicates that the file has - been verified for consistency and is guaranteed to be - consistent with the format defined in this document. -
  • Bits 2-31 are reserved for future use. -
- Bit 0 should be - set as the first action when a file is opened for write - access and should be cleared only as the final action - when closing a file. Bit 1 should be cleared during - normal access to a file and only set after the file's - consistency is guaranteed by the library or a - consistency utility. -

- -

This field is present in version 0+ of the superblock. -

-
Indexed Storage Internal Node K -

Each internal node of a indexed storage B-tree will have at - least this many entries but not more than twice this - many. If the group has only one internal - node then it might have fewer entries. -

-

This value must be greater than zero. -

-

See the description of B-trees below. -

- -

This field is present in version 1+ of the superblock. -

-
Base Address -

This is the absolute file address of the first byte of - the HDF5 data within the file. The library currently - constrains this value to be the absolute file address - of the super block itself when creating new files; - future versions of the library may provide greater - flexibility. When opening an existing file and this address does - not match the offset of the superblock, the library assumes - that the entire contents of the HDF5 file have been adjusted in - the file and adjusts the base address and end of file address to - reflect their new positions in the file. Unless otherwise noted, - all other file addresses are relative to this base - address. -

- -

This field is present in version 0+ of the superblock. -

-
Address of Global Free-space Index -

Free-space management is not yet defined in the HDF5 - file format and is not handled by the library. - Currently this field always contains the - undefined address. -

- -

This field is present in version 0+ of the superblock. -

-
End of File Address -

This is the absolute file address of the first byte past - the end of all HDF5 data. It is used to determine whether a - file has been accidentally truncated and as an address where - file data allocation can occur if space from the free list is - not used. -

- -

This field is present in version 0+ of the superblock. -

-
Driver Information Block Address -

This is the relative file address of the file driver - information block which contains driver-specific - information needed to reopen the file. If there is no - driver information block then this entry should be the - undefined address. -

- -

This field is present in version 0+ of the superblock. -

-
Root Group Symbol Table Entry -

This is the symbol table entry - of the root group, which serves as the entry point into - the group graph for the file. -

- -

This field is present in version 0+ of the superblock. -

-
-
- -

- Disk Format: Level 0B - File Driver Info

- -

The file driver information block is an optional region of the - file which contains information needed by the file driver in - order to reopen a file. The format of the file driver information - block is: - -
-

- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Driver Information Block -
bytebytebytebyte
VersionReserved (zero)
Driver Information Size (4 bytes)

Driver Identification (8 bytes)



Driver Information (n bytes)


-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version -

The version number of the driver information block. The - file format documented here is version zero. -

-
Driver Information Size -

The size in bytes of the Driver Information part of this - structure. -

-
Driver Identification -

This is an eight-byte ASCII string without null - termination which identifies the driver and version number - of the Driver Information block. The predefined drivers - supplied with the HDF5 library are identified by the - letters NCSA followed by the first four characters of - the driver name. If the Driver Information block is not - the original version then the last letter(s) of the - identification will be replaced by a version number in - ASCII. -

-

- For example, the various versions of the multi driver - will be identified by NCSAmult. - (NCSAmult is simply NCSAmulti truncated - to eight characters. Subsequent identifiers will be created by - substituting sequential numerical values for the final character, - starting with zero.) multi driver is the only default driver that - is encoded in this field. -

-

- Identification for user-defined drivers - is eight-byte long and arbitrary but should be unique and avoid - the four character prefix "NCSA". -

-
Driver InformationDriver information is encoded/decoded in a format defined by the - file driver. multi driver is the only default driver that has driver - information stored in this field. Its format is explained in the - following block.
-
- -
-

Multi driver has the following format:

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Multi Driver Message -
bytebytebytebyte
Member MappingMember MappingMember MappingMember Mapping
Member MappingMember MappingReservedReserved

Address of Member File 1


End of Address for Member File 1


Address of Member File 2


End of Address for Member File 2


... ...


Name of Member File 1


Name of Member File 2


... ...

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Member Mapping

Multi driver enables different types of HDF5 data and - metadata to be written to separate files. These files are viewed by the - library as a single virtual HDF5 file with a single file address. - It allows maximal 6 files to be created. - In sequence, these Member Mapping fields are for super block, - B-tree, raw data, global heap, local heap, - and object header. More than one type of data can be written to the - same file.

-

These Member Mapping fields are integer values from 1 to 6 - indicating how the data can be mapped to or merged with another type of - data. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Member MappingDescription
1The super block data.
2The B-tree data.
3The raw data.
4The global heap data.
5The local heap data.
6The object header data.

- For example, if the third field has the value 3 and all the rest have the - value 1, it means there are two files, one for raw data, one for super block, - B-tree, global heap, local heap, and object header. -
Reserved

These fields are reserved and should always be zero.

Address of Member File

Specifies the virtual address. A normally eight-byte integer with - the value from 0 (zero) to maximal value, - at which the member file starts.

End of Address for Member File

The end of allocated address for the member file. A normally eight-byte - integer value.

Name of Member File

The null-terminated name of member file. Its length should be multiples of - 8 bytes. Additional bytes will be padded with NULLs. The default naming - convention is %%s-X.h5, where X is one of the letters - s (for super block), b (for B-tree), r (for raw data), - g (for global heap), l (for local heap), and o (for - object header). The name for the whole HDF5 file will substitute the %s - in the string. -

-
-
- -
-
- -

- Disk Format: Level 1 - File Infrastructure

-

Disk Format: Level 1A - B-link Trees and B-tree Nodes

- -

B-link trees allow flexible storage for objects which tend to grow - in ways that cause the object to be stored discontiguously. B-trees - are described in various algorithms books including "Introduction to - Algorithms" by Thomas H. Cormen, Charles E. Leiserson, and Ronald - L. Rivest. The B-link tree, in which the sibling nodes at a - particular level in the tree are stored in a doubly-linked list, - is described in the "Efficient Locking for Concurrent Operations - on B-trees" paper by Phillip Lehman and S. Bing Yao as published - in the ACM Transactions on Database Systems, Vol. 6, - No. 4, December 1981. - -

The B-link trees implemented by the file format contain one more - key than the number of children. In other words, each child - pointer out of a B-tree node has a left key and a right key. - The pointers out of internal nodes point to sub-trees while - the pointers out of leaf nodes point to symbol nodes and - raw data chunks. - Aside from that difference, internal nodes and leaf nodes - are identical. - -
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- B-tree Nodes -
bytebytebytebyte
Signature
Node TypeNode LevelEntries Used
Address of Left SiblingO
Address of Right SiblingO
Key 0 (variable size)
Address of Child 0O
Key 1 (variable size)
Address of Child 1O
...
Key 2K (variable size)
Address of Child 2KO
Key 2K+1 (variable size)
- - - -
- (Items marked with an 'O' the above table are -
- of the size specified in "Size of Offsets.") -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Signature -

The ASCII character string "TREE" is - used to indicate the - beginning of a B-link tree node. This gives file - consistency checking utilities a better chance of - reconstructing a damaged file. -

-
Node Type -

Each B-link tree points to a particular type of data. - This field indicates the type of data as well as - implying the maximum degree K of the tree and - the size of each Key field. -

- - - - - - - - - - - - - - -
Node TypeDescription
0This tree points to group nodes.
1This tree points to raw data chunk nodes.
-
Node Level -

The node level indicates the level at which this node - appears in the tree (leaf nodes are at level zero). Not - only does the level indicate whether child pointers - point to sub-trees or to data, but it can also be used - to help file consistency checking utilities reconstruct - damaged trees. -

-
Entries Used -

This determines the number of children to which this - node points. All nodes of a particular type of tree - have the same maximum degree, but most nodes will point - to less than that number of children. The valid child - pointers and keys appear at the beginning of the node - and the unused pointers and keys appear at the end of - the node. The unused pointers and keys have undefined - values. -

-
Address of Left Sibling -

This is the relative file address of the left sibling of - the current node. If the current - node is the left-most node at this level then this field - is the undefined address. -

-
Address of Right Sibling -

This is the relative file address of the right sibling of - the current node. If the current - node is the right-most node at this level then this - field is the undefined address. -

-
Keys and Child Pointers -

Each tree has 2K+1 keys with 2K - child pointers interleaved between the keys. The number - of keys and child pointers actually containing valid - values is determined by the node's Entries Used field. - If that field is N then the B-link tree contains - N child pointers and N+1 keys. -

-
Key -

The format and size of the key values is determined by - the type of data to which this tree points. The keys are - ordered and are boundaries for the contents of the child - pointer; that is, the key values represented by child - N fall between Key N and Key - N+1. Whether the interval is open or closed on - each end is determined by the type of data to which the - tree points. -

- -

- The format of the key depends on the node type. - For nodes of node type 0 (group nodes), the key is formatted as - follows: -

- - - - - -
A single field of Size of Lengths - bytes:Indicates the byte offset into the local heap - for the first object name in the subtree which - that key describes. -
-
-

- -

- For nodes of node type 1 (chunked raw data nodes), the key is - formatted as follows: -

- - - - - - - - - - - - - -
Bytes 1-4:Size of chunk in bytes.
Bytes 4-8:Filter mask, a 32-bit bitfield indicating which - filters have been skipped for this chunk. Each filter - has an index number in the pipeline (starting at 0, with - the first filter to apply) and if that filter is skipped, - the bit corresponding to it's index is set.
N 64-bit fields:A 64-bit index indicating the offset of the - chunk within the dataset where N is the number - of dimensions of the dataset. For example, if - a chunk in a 3-dimensional dataset begins at the - position [5,5,5], there will be three - such 64-bit indices, each with the value of - 5.
-
-

-
Child Pointer -

The tree node contains file addresses of subtrees or - data depending on the node level. Nodes at Level 0 point - to data addresses, either raw data chunk or group nodes. - Nodes at non-zero levels point to other nodes of the - same B-tree. -

-

For raw data chunk nodes, the child pointer is the address - of a single raw data chunk. For group nodes, the child pointer - points to a symbol table, which contains - information for multiple symbol table entries. -

-
-
- -

- Conceptually, each B-tree node looks like this: -

- - - - - - - - - - - - - -
key[0] child[0] key[1] child[1] key[2] ... ... key[N-1] child[N-1] key[N]
-
-
- - where child[i] is a pointer to a sub-tree (at a level - above Level 0) or to data (at Level 0). - Each key[i] describes an item stored by the B-tree - (a chunk or an object of a group node). The range of values - represented by child[i] is indicated by key[i] - and key[i+1]. - - -

The following question must next be answered: - "Is the value described by key[i] contained in - child[i-1] or in child[i]?" - The answer depends on the type of tree. - In trees for groups (node type 0) the object described by - key[i] is the greatest object contained in - child[i-1] while in chunk trees (node type 1) the - chunk described by key[i] is the least chunk in - child[i]. - -

That means that key[0] for group trees is sometimes unused; - it points to offset zero in the heap, which is always the - empty string and compares as "less-than" any valid object name. - -

And key[N] for chunk trees is sometimes unused; - it contains a chunk offset which compares as "greater-than" - any other chunk offset and has a chunk byte size of zero - to indicate that it is not actually allocated. - - -

Disk Format: Level 1B - Group and Symbol Nodes

- -

A group is an object internal to the file that allows - arbitrary nesting of objects within the file (including other groups). - A group maps a set of names in the group to a set of relative - file addresses where objects with those names are located in - the file. Certain metadata for an object to which the group points - can be cached in the group's symbol table in addition to the - object's header. - -

An HDF5 object name space can be stored hierarchically by - partitioning the name into components and storing each - component in a group. The group entry for a - non-ultimate component points to the group containing - the next component. The group entry for the last - component points to the object being named. - -

A group is a collection of group nodes pointed - to by a B-link tree. Each group node contains entries - for one or more symbols. If an attempt is made to add a - symbol to an already full group node containing - 2K entries, then the node is split and one node - contains K symbols and the other contains - K+1 symbols. - -
-

- - - - - - - - - - - - - - - - - - - -
- Group Node (A Leaf of a B-tree) -
bytebytebytebyte
Signature
Version NumberReserved (0)Number of Symbols


Group Entries


-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Signature -

The ASCII character string "SNOD" is - used to indicate the - beginning of a group node. This gives file - consistency checking utilities a better chance of - reconstructing a damaged file. -

-
Version Number -

The version number for the group node. This - document describes version 1. (There is no version '0' - of the group node) -

-
Number of Symbols -

Although all group nodes have the same length, - most contain fewer than the maximum possible number of - symbol entries. This field indicates how many entries - contain valid data. The valid entries are packed at the - beginning of the group node while the remaining - entries contain undefined values. -

-
Group Entries -

Each symbol has an entry in the group node. - The format of the entry is described below. - There are 2K entries in each group node, where - K is the "Group Leaf Node K" value from the - super block. -

-
-
- -

- Disk Format: Level 1C - Group Entry

- -

Each group entry in a group node is designed - to allow for very fast browsing of stored objects. - Toward that design goal, the group entries - include space for caching certain constant metadata from the - object header. - -
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Group Entry -
bytebytebytebyte
Name OffsetO
Object Header AddressO
Cache Type
Reserved


Scratch-pad Space (16 bytes)


- - - -
- (Items marked with an 'O' the above table are -
- of the size specified in "Size of Offsets.") -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Name Offset -

This is the byte offset into the group local - heap for the name of the object. The name is null - terminated. -

-
Object Header Address -

Every object has an object header which serves as a - permanent location for the object's metadata. In addition - to appearing in the object header, some metadata can be - cached in the scratch-pad space. -

-
Cache Type -

The cache type is determined from the object header. - It also determines the format for the scratch-pad space: -
- - - - - - - - - - - - - - - - - - - - - -
Type:Description:
0No data is cached by the group entry. This - is guaranteed to be the case when an object header - has a link count greater than one. -
1Object header metadata is cached in the group - entry. This implies that the group - entry refers to another group. -
2The entry is a symbolic link. The first four bytes - of the scratch-pad space are the offset into the local - heap for the link value. The object header address - will be undefined. -
NOther cache values can be defined later and - libraries that do not understand the new values will - still work properly. -
-

-
Reserved -

These four bytes are present so that the scratch-pad - space is aligned on an eight-byte boundary. They are - always set to zero. -

-
Scratch-pad Space -

This space is used for different purposes, depending - on the value of the Cache Type field. Any metadata - about a dataset object represented in the scratch-pad - space is duplicated in the object header for that - dataset. This metadata can include the datatype - and the size of the dataspace for a dataset whose datatype - is atomic and whose dataspace is fixed and less than - four dimensions. -

-

- Furthermore, no data is cached in the group - entry scratch-pad space if the object header for - the group entry has a link count greater than - one. -

-
-
- -

Format of the Scratch-pad Space

- -

The group entry scratch-pad space is formatted - according to the value in the Cache Type field. - -

If the Cache Type field contains the value zero - (0) then no information is - stored in the scratch-pad space. - -

If the Cache Type field contains the value one - (1), then the scratch-pad space - contains cached metadata for another object header - in the following format: - -
-

- - - - - - - - - - - - - - -
- Object Header Scratch-pad Format -
bytebytebytebyte
Address of B-treeO
Address of Name HeapO
- - - -
- (Items marked with an 'O' the above table are -
- of the size specified in "Size of Offsets.") -
-
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription
Address of B-tree -

This is the file address for the root of the - group's B-tree. -

-
Address of Name Heap -

This is the file address for the group's local - heap, in which are stored the group's symbol names. -

-
-
- - -

If the Cache Type field contains the value two - (2), then the scratch-pad space - contains cached metadata for another symbolic link - in the following format: - -
-

- - - - - - - - - - - - - -
- Symbolic Link Scratch-pad Format -
bytebytebytebyte
Offset to Link Value
-
- -
-
- - - - - - - - - - -
Field NameDescription
Offset to Link Value -

The value of a symbolic link (that is, the name of the - thing to which it points) is stored in the local heap. - This field is the 4-byte offset into the local heap for - the start of the link value, which is null terminated. -

-
-
- -

Disk Format: Level 1D - Local Heaps

- -

A heap is a collection of small heap objects. Objects can be - inserted and removed from the heap at any time. - The address of a heap does not change once the heap is created. - References to objects are stored in the group table; - the names of those objects are stored in the local heap. -

- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Local Heap -
bytebytebytebyte
Signature
VersionReserved (zero)
Data Segment SizeL
Offset to Head of Free-listL
Address of Data SegmentO
- - - - -
- (Items marked with an 'L' the above table are -
- of the size specified in "Size of Lengths.") -
- (Items marked with an 'O' the above table are -
- of the size specified in "Size of Offsets.") -
-
- -

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Signature -

The ASCII character string "HEAP" - is used to indicate the - beginning of a heap. This gives file consistency - checking utilities a better chance of reconstructing a - damaged file. -

-
Version -

Each local heap has its own version number so that new - heaps can be added to old files. This document - describes version zero (0) of the local heap. -

-
Data Segment Size -

The total amount of disk memory allocated for the heap - data. This may be larger than the amount of space - required by the objects stored in the heap. The extra - unused space in the heap holds a linked list of free blocks. -

-
Offset to Head of Free-list -

This is the offset within the heap data segment of the - first free block (or the - undefined address if there is no - free block). The free block contains "Size of Lengths" bytes that - are the offset of the next free block (or the - value '1' if this is the - last free block) followed by "Size of Lengths" bytes that store - the size of this free block. The size of the free block includes - the space used to store the offset of the next free block and - the of the current block, making the minimum size of a free block - 2 * "Size of Lengths". -

-
Address of Data Segment -

The data segment originally starts immediately after - the heap header, but if the data segment must grow as a - result of adding more objects, then the data segment may - be relocated, in its entirety, to another part of the - file. -

-
-
- -

Objects within the heap should be aligned on an 8-byte boundary. - -

Disk Format: Level 1E - Global Heap

- -

Each HDF5 file has a global heap which stores various types of - information which is typically shared between datasets. The - global heap was designed to satisfy these goals: - -

    -
  1. Repeated access to a heap object must be efficient without - resulting in repeated file I/O requests. Since global heap - objects will typically be shared among several datasets, it is - probable that the object will be accessed repeatedly. -
  2. Collections of related global heap objects should result in - fewer and larger I/O requests. For instance, a dataset of - object references will have a global heap object for each - reference. Reading the entire set of object references - should result in a few large I/O requests instead of one small - I/O request for each reference. -
  3. It should be possible to remove objects from the global heap - and the resulting file hole should be eligible to be reclaimed - for other uses. -
-

- -

The implementation of the heap makes use of the memory - management already available at the file level and combines that - with a new top-level object called a collection to - achieve Goal B. The global heap is the set of all collections. - Each global heap object belongs to exactly one collection and - each collection contains one or more global heap objects. For - the purposes of disk I/O and caching, a collection is treated as - an atomic object. -

- -

The HDF5 library creates global heap collections as needed, so there may - be multiple collections throughout the file. The set of all of them is - abstractly called the "global heap", although they don't actually link - to each other, and there is no global place in the file where you can - discover all of the collections. The collections are found simply by - finding a reference to one through another object in the file (eg. - variable-length datatype elements, etc). -

- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- A Global Heap Collection -
bytebytebytebyte
Signature
VersionReserved (zero)
Collection SizeL

Global Heap Object 1


Global Heap Object 2


...


Global Heap Object N


Global Heap Object 0 (free space)

- - - -
- (Items marked with an 'L' the above table are -
- of the size specified in "Size of Lengths.") -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Signature -

The ASCII character string "GCOL" - is used to indicate the - beginning of a collection. This gives file consistency - checking utilities a better chance of reconstructing a - damaged file. -

-
Version -

Each collection has its own version number so that new - collections can be added to old files. This document - describes version one (1) of the collections (there is no - version zero (0)). -

-
Collection Size -

This is the size in bytes of the entire collection - including this field. The default (and minimum) - collection size is 4096 bytes which is a typical file - system block size. This allows for 127 16-byte heap - objects plus their overhead (the collection header of 16 bytes - and the 16 bytes of information about each heap object). -

-
Global Heap Object 1 through N -

The objects are stored in any order with no - intervening unused space. -

-
Global Heap Object 0 -

Global Heap Object 0 (zero), when present, represents the free - space in the collection. Free space always appears at the end of - the collection. If the free space is too small to store the header - for Object 0 (described below) then the header is implied and the - collection contains no free space. -

-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Global Heap Object -
bytebytebytebyte
Heap Object IDReference Count
Reserved
Object SizeL

Object Data

- - - -
- (Items marked with an 'L' the above table are -
- of the size specified in "Size of Lengths.") -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Heap Object ID -

Each object has a unique identification number within a - collection. The identification numbers are chosen so that - new objects have the smallest value possible with the - exception that the identifier 0 always refers to the - object which represents all free space within the - collection. -

-
Reference Count -

All heap objects have a reference count field. An - object which is referenced from some other part of the - file will have a positive reference count. The reference - count for Object 0 is always zero. -

-
Reserved -

Zero padding to align next field on an 8-byte boundary. -

-
Object Size -

This is the size of the object data stored for the object. - The actual storage space allocated for the object data is rounded - up to a multiple of eight. -

-
Object Data -

The object data is treated as a one-dimensional array - of bytes to be interpreted by the caller. -

-
-
- -

Disk Format: Level 1F - Free-space Index

- -

The free-space index is a collection of blocks of data, - dispersed throughout the file, which are currently not used by - any file objects. - -

The super block contains a pointer to root of the free-space description; - that pointer is currently required to be the - undefined address. - -

The format of the free-space index is not defined at this time. - - - -
-

- -

Disk Format: Level 2 - Data Objects

- -

Data objects contain the real information in the file. These - objects compose the scientific data and other information which - are generally thought of as "data" by the end-user. All the - other information in the file is provided as a framework for - these data objects. -

- -

A data object is composed of header information and data - information. The header information contains the information - needed to interpret the data information for the data object as - well as additional "metadata" or pointers to additional - "metadata" used to describe or annotate each data object. -

- -

- Disk Format: Level 2A - Data Object Headers

- -

The header information of an object is designed to encompass - all the information about an object, except for the data itself. - This information includes - the dataspace, datatype, information about how the data - is stored on disk (in external files, compressed, broken up in - blocks, etc.), as well as other information used by the library - to speed up access to the data objects or maintain a file's - integrity. Information stored by user applications as attributes - is also stored in the object's header. The header of each object is - not necessarily located immediately prior to the object's data in the - file and in fact may be located in any position in the file. The order - of the messages in an object header is not significant. -

- -

Header messages are aligned on 8-byte boundaries. -

- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Object Headers -
bytebytebytebyte
VersionReserved (zero)Number of Header Messages
Object Reference Count
Object Header Size
Header Message Type #1Size of Header Message Data #1
Header Message #1 FlagsReserved (zero)

Header Message Data #1

.
.
.
Header Message Type #nSize of Header Message Data #n
Header Message #n FlagsReserved (zero)

Header Message Data #n

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version -

This value is used to determine the format of the - information in the object header. When the format of the - information in the object header is changed, the version number - is incremented and can be used to determine how the - information in the object header is formatted. This - document describes version one (1) (there was no version - zero (0)). -

-
Number of Header Messages -

This value determines the number of messages listed in - object headers for this object. This value includes the messages - in continuation messages for this object. -

-
Object Reference Count -

This value specifies the number of "hard links" to this object - within the current file. References to the object from external - files, "soft links" in this file and object references in this - file are not tracked. -

-
Object Header Size -

This value specifies the number of bytes of header message data - following this length field that contain object header messages - for this object header. This value does not include the size of - object header continuation blocks for this object elsewhere in the - file. -

-
Header Message Type -

This value specifies the type of information included in the - following header message data. The header message types for the - pre-defined header messages are included in sections below. -

-
Size of Header Message Data -

This value specifies the number of bytes of header - message data following the header message type and length - information for the current message. The size includes - padding bytes to make the message a multiple of eight - bytes. -

-
Header Message Flags -

This is a bit field with the following definition: - - - - - - - - - - - - - - - - - - -
BitDescription
0If set, the message data is constant. This is used - for messages like the datatype message of a dataset. -
1If set, the message is stored in the global heap. - The Header Message Data field contains a Shared Object - message and the Size of Header Message Data field - contains the size of that Shared Object message. -
2-7Reserved
-

-
Header Message Data -

The format and length of this field is determined by the - header message type and size respectively. Some header - message types do not require any data and this information - can be eliminated by setting the length of the message to - zero. The data is padded with enough zeros to make the - size a multiple of eight. -

-
-
- -

The header message types and the message data associated with - them compose the critical "metadata" about each object. Some - header messages are required for each object while others are - optional. Some optional header messages may also be repeated - several times in the header itself, the requirements and number - of times allowed in the header will be noted in each header - message description below. -

- -

The following is a list of currently defined header messages: -

- -
-

Name: NIL

- -

Header Message Type: 0x0000 -

-

Length: varies -

-

Status: Optional, may be repeated. -

-

Purpose and Description: The NIL message is used to indicate a - message which is to be ignored when reading the header messages for a - data object. [Possibly one which has been deleted for some reason.] -

-

Format of Data: Unspecified. -

- -
-

Name: Simple Dataspace

- -

Header Message Type: 0x0001 -

-

Length: Varies according to the number of dimensions, - as described in the following table. -

-

Status: Required for dataset objects, may not be - repeated. -

-

Description: The simple dataspace message describes the - number of dimensions (i.e. "rank") and size of each dimension that the - data object has. This message is only used for datasets which have a - simple, rectilinear grid layout; datasets requiring a more complex - layout (irregularly structured or unstructured grids, etc.) must use - the Complex Dataspace message for expressing the space the - dataset inhabits. (Note: The Complex Dataspace - functionality is not yet implemented and it is not described in this - document.) -

- -

Format of Data: -
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Simple Dataspace Message -
bytebytebytebyte
VersionDimensionalityFlagsReserved
Reserved
Dimension #1 SizeL
.
.
.
Dimension #n SizeL
Dimension #1 Maximum SizeL
.
.
.
Dimension #n Maximum SizeL
Permutation Index #1L
.
.
.
Permutation Index #nL
- - - -
- (Items marked with an 'L' the above table are -
- of the size specified in "Size of Lengths.") -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version -

This value is used to determine the format of the - Simple Dataspace Message. When the format of the - information in the message is changed, the version number - is incremented and can be used to determine how the - information in the object header is formatted. This - document describes version one (1) (there was no version - zero (0)). -

-
Dimensionality -

This value is the number of dimensions that the data - object has. -

-
Flags -

This field is used to store flags to indicate the - presence of parts of this message. Bit 0 (the least - significant bit) is used to indicate that maximum - dimensions are present. Bit 1 is used to indicate that - permutation indices are present. -

-
Dimension #n Size -

This value is the current size of the dimension of the - data as stored in the file. The first dimension stored in - the list of dimensions is the slowest changing dimension - and the last dimension stored is the fastest changing - dimension. -

-
Dimension #n Maximum Size -

This value is the maximum size of the dimension of the - data as stored in the file. This value may be the special - "unlimited" size which indicates - that the data may expand along this dimension indefinitely. - If these values are not stored, the maximum size of each - dimension is assumed to be the dimension's current size. -

-
Permutation Index #n -

This value is the index permutation used to map - each dimension from the canonical representation to an - alternate axis for each dimension. If these values are - not stored, the first dimension stored in the list of - dimensions is the slowest changing dimension and the last - dimension stored is the fastest changing dimension. -

-
-
- -

- - - -
-

Name: Reserved - Not Assigned Yet

- Header Message Type: 0x0002
- Length: N/A
- Status: N/A
- Format of Data: N/A
- -

Purpose and Description: This message type was skipped during - the initial specification of the file format and may be used in a - future expansion to the format. - - -

-

Name: Datatype

- -

Header Message Type: 0x0003 -

-

Length: variable -

-

Status: Required for dataset or named datatype objects, - may not be repeated. -

- -

Description: The datatype message defines the datatype - for each element of a dataset. A datatype can describe an atomic type - like a fixed- or floating-point type or a compound type like a C - struct. - Datatypes messages are stored - as a list of datatype classes and - their associated properties. -

- -

Datatype messages that are part of a dataset object, - do not describe how elements are related to one another, the dataspace - message is used for that purpose. Datatype messages that are part of - a named datatype message describe an "abstract" datatype that can be - used by other objects in the file. -

- -

Format of Data: -
-

- - - - - - - - - - - - - - - - - - - - - - - - -
- Datatype Message -
bytebytebytebyte
Class and VersionClass Bit Field, Bits 0-7Class Bit Field, Bits 8-15Class Bit Field, Bits 16-23
Size


Properties


-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Class and Version -

The version of the datatype message and the datatype's class - information are packed together in this field. The version - number is packed in the top 4 bits of the field and the class - is contained in the bottom 4 bits. -

-

The version number information is used for changes in the - format of the datatype message and is described here: - - - - - - - - - - - - - - - - - - -
VersionDescription
0Never used -
1Used by early versions of the library to encode - compound datatypes with explicit array fields. - See the compound datatype description below for - further details. -
2The current version used by the library. -
-

-

The class of the datatype determines the format for the class - bit field and properties portion of the datatype message, which - are described below. The - following classes are currently defined: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Fixed-Point
1Floating-Point
2Time
3String
4Bitfield
5Opaque
6Compound
7Reference
8Enumerated
9Variable-Length
10Array
-

-
Class Bit Fields -

The information in these bit fields is specific to each datatype - class and is described below. All bits not defined for a - datatype class are set to zero. -

-
Size -

The size of the datatype in bytes. -

-
Properties -

This variable-sized field encodes information specific to each - datatype class and is described below. If there is no - property information specified for a datatype class, the size - of this field is zero. -

-
-
-

- -

Class specific information for Fixed-Point Numbers (Class 0): - -
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Bit Field Description -
BitsMeaning
0Byte Order. If zero, byte order is little-endian; - otherwise, byte order is big endian.
1, 2Padding type. Bit 1 is the lo_pad type and bit 2 - is the hi_pad type. If a datum has unused bits at either - end, then the lo_pad or hi_pad bit is copied to those - locations.
3Signed. If this bit is set then the fixed-point - number is in 2's complement form.
4-23Reserved (zero).
-
- -
-
- - - - - - - - - - - - - - -
- Property Descriptions -
ByteByteByteByte
Bit OffsetBit Precision
-
- -
-
- - - - - - - - - - - - - - - - -
Field NameDescription
Bit Offset -

The bit offset of the first significant bit of the fixed-point - value within the datatype. The bit offset specifies the number - of bits "to the right of" the value. -

-
Bit Precision -

The number of bits of precision of the fixed-point value - within the datatype. -

-
-
-

- -

Class specific information for Floating-Point Numbers (Class 1): - -
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Bit Field Description -
BitsMeaning
0Byte Order. If zero, byte order is little-endian; - otherwise, byte order is big endian.
1, 2, 3Padding type. Bit 1 is the low bits pad type, bit 2 - is the high bits pad type, and bit 3 is the internal bits - pad type. If a datum has unused bits at either end or between - the sign bit, exponent, or mantissa, then the value of bit - 1, 2, or 3 is copied to those locations.
4-5Normalization. The value can be 0 if there is no - normalization, 1 if the most significant bit of the - mantissa is always set (except for 0.0), and 2 if the most - significant bit of the mantissa is not stored but is - implied to be set. The value 3 is reserved and will not - appear in this field.
6-7Reserved (zero).
8-15Sign Location. This is the bit position of the sign - bit. Bits are numbered with the least significant bit zero.
16-23Reserved (zero).
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
- Property Descriptions -
ByteByteByteByte
Bit OffsetBit Precision
Exponent LocationExponent SizeMantissa LocationMantissa Size
Exponent Bias
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Bit Offset -

The bit offset of the first significant bit of the floating-point - value within the datatype. The bit offset specifies the number - of bits "to the right of" the value. -

-
Bit Precision -

The number of bits of precision of the floating-point value - within the datatype. -

-
Exponent Location -

The bit position of the exponent field. Bits are numbered with - the least significant bit number zero. -

-
Exponent Size -

The size of the exponent field in bits. -

-
Mantissa Location -

The bit position of the mantissa field. Bits are numbered with - the least significant bit number zero. -

-
Mantissa Size -

The size of the mantissa field in bits. -

-
Exponent Bias -

The bias of the exponent field. -

-
-
-

- -

Class specific information for Time (Class 2): - -
-

- - - - - - - - - - - - - - - - - -
- Bit Field Description -
BitsMeaning
0Byte Order. If zero, byte order is little-endian; - otherwise, byte order is big endian.
1-23Reserved (zero).
-
- -
-
- - - - - - - - - - - -
- Property Descriptions -
ByteByte
Bit Precision
-
- -
-
- - - - - - - - - - - -
Field NameDescription
Bit Precision -

The number of bits of precision of the time value. -

-
-
-

- -

Class specific information for Strings (Class 3): - -
-

- - - - - - - - - - - - - - - - - - - - - -
- Bit Field Description -
BitsMeaning
0-3Padding type. This four-bit value determines the - type of padding to use for the string. The values are: - - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Null Terminate: A zero byte marks the end of the - string and is guaranteed to be present after - converting a long string to a short string. When - converting a short string to a long string the value is - padded with additional null characters as necessary. -
1Null Pad: Null characters are added to the end of - the value during conversions from short values to long - values but conversion in the opposite direction simply - truncates the value. -
2Space Pad: Space characters are added to the end of - the value during conversions from short values to long - values but conversion in the opposite direction simply - truncates the value. This is the Fortran - representation of the string. -
3-15Reserved -
-
4-7Character Set. The character set to use for - encoding the string. The only character set supported is - the 8-bit ASCII (zero) so no translations have been defined - yet.
8-23Reserved (zero).
-
- -

There are no properties defined for the string class. -

-

- -

Class specific information for Bitfields (Class 4): - -
-

- - - - - - - - - - - - - - - - - - - - - - -
- Bit Field Description -
BitsMeaning
0Byte Order. If zero, byte order is little-endian; - otherwise, byte order is big endian.
1, 2Padding type. Bit 1 is the lo_pad type and bit 2 - is the hi_pad type. If a datum has unused bits at either - end, then the lo_pad or hi_pad bit is copied to those - locations.
3-23Reserved (zero).
-
- -
-
- - - - - - - - - - - - - - -
- Property Description -
ByteByteByteByte
Bit OffsetBit Precision
-
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription
Bit Offset -

The bit offset of the first significant bit of the bitfield - within the datatype. The bit offset specifies the number - of bits "to the right of" the value. -

-
Bit Precision -

The number of bits of precision of the bitfield - within the datatype. -

-
-
-

- -

Class specific information for Opaque (Class 5): - -
-

- - - - - - - - - - - - - - - - - -
- Bit Field Description -
BitsMeaning
0-7Length of ASCII tag in bytes.
8-23Reserved (zero).
-
- -
-
- - - - - - - - - - - - - -
- Property Description -
ByteByteByteByte

ASCII Tag
-
-
- -
-
- - - - - - - - - - -
Field NameDescription
ASCII Tag -

This NUL-terminated string provides a description for the - opaque type. It is NUL-padded to a multiple of 8 bytes. -

-
-
-

- -

Class specific information for Compound (Class 6): - -
-

- - - - - - - - - - - - - - - - -
- Bit Field Description -
BitsMeaning
0-15Number of Members. This field contains the number - of members defined for the compound datatype. The member - definitions are listed in the Properties field of the data - type message. -
15-23Reserved (zero).
-
-

- -

The Properties field of a compound datatype is a list of the - member definitions of the compound datatype. The member - definitions appear one after another with no intervening bytes. - The member types are described with a recursive datatype - message. - -

Note that the property descriptions are different for different - versions of the datatype version. Additionally note that the version - 0 properties are deprecated and have been replaced with the version - 1 properties in versions of the HDF5 library from the 1.4 release - onward. - -
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Properties Description for Datatype Version 1 -
ByteByteByteByte

Name

Byte Offset of Member
DimensionalityReserved (zero)
Dimension Permutation
Reserved (zero)
Dimension #1 Size (required)
Dimension #2 Size (required)
Dimension #3 Size (required)
Dimension #4 Size (required)

Member Type Message

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Name -

This NUL-terminated string provides a description for the - opaque type. It is NUL-padded to a multiple of 8 bytes. -

-
Byte Offset of Member -

This is the byte offset of the member within the datatype. -

-
Dimensionality -

If set to zero, this field indicates a scalar member. If set - to a value greater than zero, this field indicates that the - member is an array of values. For array members, the size of - the array is indicated by the 'Size of Dimension n' field in - this message. -

-
Dimension Permutation -

This field was intended to allow an array field to have - it's dimensions permuted, but this was never implemented. - This field should always be set to zero. -

-
Dimension #n Size -

This field is the size of a dimension of the array field as - stored in the file. The first dimension stored in the list of - dimensions is the slowest changing dimension and the last - dimension stored is the fastest changing dimension. -

-
Member Type Message -

This field is a datatype message describing the datatype of - the member. -

-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Properties Description for Datatype Version 2 -
ByteByteByteByte

Name

Byte Offset of Member

Member Type Message

-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Name -

This NUL-terminated string provides a description for the - opaque type. It is NUL-padded to a multiple of 8 bytes. -

-
Byte Offset of Member -

This is the byte offset of the member within the datatype. -

-
Member Type Message -

This field is a datatype message describing the datatype of - the member. -

-
-
-

- -

Class specific information for Reference (Class 7): - -
-

- - - - - - - - - - - - - - - - - -
- Bit Field Description -
BitsMeaning
0-3Type. This four-bit value contains the type of reference - described. The values defined are: - - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Object Reference: A reference to another object in this - HDF5 file. -
1Dataset Region Reference: A reference to a region within - a dataset in this HDF5 file. -
2Internal Reference: A reference to a region within the - current dataset. (Not currently implemented) -
3-15Reserved -
- -
15-23Reserved (zero).
-
- -

There are no properties defined for the reference class. -

-

- -

Class specific information for Enumeration (Class 8): - -
-

- - - - - - - - - - - - - - - - - -
- Bit Field Description -
BitsMeaning
0-15Number of Members. The number of name/value - pairs defined for the enumeration type.
16-23Reserved (zero).
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Property Description -
ByteByteByteByte

Base Type


Names


Values

-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Base Type -

Each enumeration type is based on some parent type, usually an - integer. The information for that parent type is described - recursively by this field. -

-
Names -

The name for each name/value pair. Each name is stored as a null - terminated ASCII string in a multiple of eight bytes. The names - are in no particular order. -

-
Values -

The list of values in the same order as the names. The values - are packed (no inter-value padding) and the size of each value - is determined by the parent type. -

-
-
-

- - -

Class specific information for Variable-Length (Class 9): - -
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Bit Field Description -
BitsMeaning
0-3Type. This four-bit value contains the type of - variable-length datatype described. The values defined are: - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Sequence: A variable-length sequence of any sequence of - data. Variable-length sequences do not have padding or - character set information. -
1String: A variable-length sequence of characters. - Variable-length strings have padding and character set - information. -
2-15Reserved -
- -
4-7Padding type. (variable-length string only) - This four-bit value determines the type of padding - used for variable-length strings. The values are the same - as for the string padding type, as follows: - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Null terminate: A zero byte marks the end of a string - and is guaranteed to be present after converting a long - string to a short string. When converting a short string - to a long string, the value is padded with additional null - characters as necessary. -
1Null pad: Null characters are added to the end of the - value during conversion from a short string to a longer - string. Conversion from a long string to a shorter string - simply truncates the value. -
2Space pad: Space characters are added to the end of the - value during conversion from a short string to a longer - string. Conversion from a long string to a shorter string - simply truncates the value. This is the Fortran - representation of the string. -
3-15Reserved -
- - This value is set to zero for variable-length sequences. - -
8-11Character Set. (variable-length string only) - This four-bit value specifies the character set - to be used for encoding the string: - - - - - - - - - - - - - - - -
ValueDescription
0ASCII: As of this writing (July 2003, Release 1.6.0), - 8-bit ASCII is the only character set supported. Therefore, - no translations have been defined. -
1-15Reserved -
- - This value is set to zero for variable-length sequences. - -
12-23Reserved (zero).
-
- -
-
- - - - - - - - - - - - - - -
- Property Description -
ByteByteByteByte

Base Type

-
- -
-
- - - - - - - - - - - -
Field NameDescription
Base Type -

Each variable-length type is based on some parent type. The - information for that parent type is described recursively by - this field. -

-
-
-

- -

Class specific information for Array (Class 10): - -

There are no bit fields defined for the array class. -

- -

Note that the dimension information defined in the property for this - datatype class is independent of dataspace information for a dataset. - The dimension information here describes the dimensionality of the - information within a data element (or a component of an element, if the - array datatype is nested within another datatype) and the dataspace for a - dataset describes the location of the elements in a dataset. -

- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Property Description -
ByteByteByteByte
DimensionalityReserved (zero)
Dimension #1 Size
.
.
.
Dimension #n Size
Permutation Index #1
.
.
.
Permutation Index #n

Base Type

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Dimensionality -

This value is the number of dimensions that the array has. -

-
Dimension #n Size -

This value is the size of the dimension of the array - as stored in the file. The first dimension stored in - the list of dimensions is the slowest changing dimension - and the last dimension stored is the fastest changing - dimension. -

-
Permutation Index #n -

This value is the index permutation used to map - each dimension from the canonical representation to an - alternate axis for each dimension. Currently, dimension - permutations are not supported and these indices should be set - to the index position minus one (i.e. the first dimension should - be set to 0, the second dimension should be set to 1, etc.) -

-
Base Type -

Each array type is based on some parent type. The - information for that parent type is described recursively by - this field. -

-
-
- -

- -
-

Name: Data Storage - Fill Value (Old)

- -

Header Message Type: 0x0004 -

-

Length: varies -

-

Status: Optional, may not be repeated. -

- -

Description: The fill value message stores a single - data value which is returned to the application when an uninitialized - data element is read from a dataset. The fill value is interpreted - with the same datatype as the dataset. If no fill value message is - present then a fill value of all zero bytes is assumed. -

- -

This fill value message is deprecated in favor of the "new" - fill value message (Message Type 0x0005) and is only written to the - file for forward compatibility with versions of the HDF5 library before - the 1.6.0 version. Additionally, it only appears for datasets with a - user defined fill value (as opposed to the library default fill value - or an explicitly set "undefined" fill value). -

- -

Format of Data: -
-

- - - - - - - - - - - - - - - - - -
- Fill Value Message (Old) -
bytebytebytebyte
Size

Fill Value

-
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription
Size -

This is the size of the Fill Value field in bytes. -

-
Fill Value -

The fill value. The bytes of the fill value are interpreted - using the same datatype as for the dataset. -

-
-
-

- -
-

Name: Data Storage - Fill Value

- -

Header Message Type: 0x0005 -

-

Length: varies -

-

Status: Required for dataset objects, may not be repeated. -

- -

Description: The fill value message stores a single - data value which is returned to the application when an uninitialized - data element is read from a dataset. The fill value is interpreted - with the same datatype as the dataset. -

- -

Format of Data: -
-

- - - - - - - - - - - - - - - - - - - - - - - - -
- Fill Value Message -
bytebytebytebyte
VersionSpace Allocation TimeFill Value Write TimeFill Value Defined
Size

Fill Value

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version -

The version number information is used for changes in the - format of the fill value message and is described here: - - - - - - - - - - - - - - - - - - -
VersionDescription
0Never used -
1Used by version 1.6.x of the library to encode - fill values. In this version, the Size field is - always present. -
2The current version used by the library (version - 1.7.3 or later). In this version, the Size and - Fill Value fields are - only present if the Fill Value Defined field is set - to 1. -
-

-
Space Allocation Time -

When the storage space for the dataset's raw data will be - allocated. The allowed values are: - - - - - - - - - - - - - - - - - - -
ValueDescription
1Early allocation. Storage space for the entire dataset - should be allocated in the file when the dataset is - created. -
2Late allocation. Storage space for the entire dataset - should not be allocated until the dataset is written - to. -
3Incremental allocation. Storage space for the - dataset should not be allocated until the portion - of the dataset is written to. This is currently - used in conjunction with chunked data storage for - datasets. -
-

-
Fill Value Write Time -

At the time that storage space for the dataset's raw data is - allocated, this value indicates whether the fill value should - be written to the raw data storage elements. The allowed values - are: - - - - - - - - - - - - - - - - - - -
ValueDescription
0On allocation. The fill value is always written to - the raw data storage when the storage space is allocated. -
1Never. The fill value should never be written to - the raw data storage. -
2Fill value written if set by user. The fill value - will be written to the raw data storage when the storage - space is allocated only if the user explicitly set - the fill value. If the fill value is the library - default or is undefined, it will not be written to - the raw data storage. -
-

-
Fill Value Defined -

This value indicates if a fill value is defined for this - dataset. If this value is 0, the fill value is undefined. - If this value is 1, a fill value is defined for this dataset. - For version 2 or later of the fill value message, this value - controls the presence of the Size field. -

-
Size -

This is the size of the Fill Value field in bytes. This field - is not present if the Version field is >1 and the Fill Value - Defined field is set to 0. -

-
Fill Value -

The fill value. The bytes of the fill value are interpreted - using the same datatype as for the dataset. This field is - not present if the Version field is >1 and the Fill Value - Defined field is set to 0. -

-
-
-

- - - -
-

Name: Reserved - Not Assigned Yet

-

Header Message Type: 0x0006

-

Length: N/A

-

Status: N/A

-

Format of Data: N/A

- -

Purpose and Description: This message type was skipped during - the initial specification of the file format and may be used in a - future expansion to the format.

- -
-

Name: Data Storage - - External Data Files

-

Header Message Type: 0x0007

-

Length: varies

-

Status: Optional, may not be repeated.

- -

Purpose and Description: The external object message - indicates that the data for an object is stored outside the HDF5 - file. The filename of the object is stored as a Universal - Resource Location (URL) of the actual filename containing the - data. An external file list record also contains the byte offset - of the start of the data within the file and the amount of space - reserved in the file for that data.

- -

Format of Data: -
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- External File List Message -
bytebytebytebyte
VersionReserved
Allocated SlotsUsed Slots

Heap Address


Slot Definitions...

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version -

The version number information is used for changes in the format of External File - List Message and is described here: - - - - - - - - - - - -
VersionDescription
0Never used. -
1The current version used by the library. -
-

-
Reserved -

This field is reserved for future use.

-
Allocated Slots -

The total number of slots allocated in the message. Its value must be at least as - large as the value contained in the Used Slots field. (The current library simply - uses the number of Used Slots for this message)

-
Used Slots -

The number of initial slots which contains valid information.

-
Heap Address -

This is the address of a local heap which contains the names for the external - files (The local heap information can be found in Disk Format Level 1D in this - document). The name at offset zero in the heap is always the empty string.

-
Slot Definitions -

The slot definitions are stored in order according to the array addresses they - represent.

-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
- External File List Slot -
bytebytebytebyte

Name Offset(<size> bytes)


File Offset(<size> bytes)


Size

-
- -
-
- - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Name Offset(<size> bytes) -

The byte offset within the local name heap for the name - of the file. File names are stored as a URL which has a - protocol name, a host name, a port number, and a file - name: - protocol:port//host/file. - If the protocol is omitted then "file:" is assumed. If - the port number is omitted then a default port for that - protocol is used. If both the protocol and the port - number are omitted then the colon can also be omitted. If - the double slash and host name are omitted then - "localhost" is assumed. The file name is the only - mandatory part, and if the leading slash is missing then - it is relative to the application's current working - directory (the use of relative names is not - recommended).

-
File Offset(<size> bytes) -

This is the byte offset to the start of the data in the - specified file. For files that contain data for a single - dataset this will usually be zero.

-
Size -

This is the total number of bytes reserved in the - specified file for raw data storage. For a file that - contains exactly one complete dataset which is not - extendable, the size will usually be the exact size of the - dataset. However, by making the size larger one allows - HDF5 to extend the dataset. The size can be set to a value - larger than the entire file since HDF5 will read zeros - past the end of the file without failing.

-
-
- - -
-

Name: Data Storage - Layout

- -

Header Message Type: 0x0008

-

Length: varies

-

Status: Required for datasets, may not be repeated.

- -

Purpose and Description: Data layout describes how the - elements of a multi-dimensional array are arranged in the linear - address space of the file. Three types of data layout are - supported: - -

    -
  1. Contiguous: The array can be stored in one contiguous area of the file. - The layout requires that the size of the array be constant and - does not permit chunking, compression, checksums, encryption, - etc. The message stores the total size of the array and the - offset of an element from the beginning of the storage area is - computed as in C. - -
  2. Chunked: The array domain can be regularly decomposed into chunks and - each chunk is allocated separately. This layout supports - arbitrary element traversals, compression, encryption, and - checksums, and the chunks can be distributed across external - raw data files (these features are described in other - messages). The message stores the size of a chunk instead of - the size of the entire array; the size of the entire array can - be calculated by traversing the B-tree that stores the chunk - addresses. - -
  3. Compact: The array can be stored in one contiguous block, as part of - this object header message (this is called "compact" storage below). -
- -

Format of Data: -
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Data Layout Message (Versions 1 and 2) -
bytebytebytebyte
VersionDimensionalityLayout ClassReserved
Reserved

Address

Dimension 0 (4-bytes)
Dimension 1 (4-bytes)
...
Dataset Element Size (optional)
Compact Data Size (4-bytes)

Compact Data...

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version -

The version number information is used for changes in the format of the data - layout message and is described here:

- - - - - - - - - - - - - - - - - - - - -
VersionDescription
0Never used.
1Used by version 1.4 and before of the library to encode layout information. - Data space is always allocated when the data set is created.
2Used by version 1.6.x of the library to encode layout information. - Data space is allocated only when it is necessary.
-
Dimensionality

An array has a fixed dimensionality. This field - specifies the number of dimension size fields later in the - message.

Layout Class

The layout class specifies how the other fields of the - layout message are to be interpreted. A value of one - indicates contiguous storage, a value of two indicates chunked storage, - while a value of zero indicates compact storage. Other values will be defined - in the future.

Address

For contiguous storage, this is the address of the first - byte of storage. For chunked storage this is the address - of the B-tree that is used to look up the addresses of the - chunks. This field is not present for compact storage. - If the version for this message is set to 2, the address - may have the "undefined address" value, to indicate that - storage has not yet been allocated for this array.

Dimensions

For contiguous and compact storage the dimensions define - the entire size of the array while for chunked storage they define - the size of a single chunk. In all cases, they are in units of - array elements (not bytes). The first dimension stored in the list - of dimensions is the slowest changing dimension and the last - dimension stored is the fastest changing dimension. -

-
Dataset Element Size

The size of a dataset element, in bytes. This field is only - present for chunked storage. -

-
Compact Data Size

This field is only present for compact data storage. - It contains the size of the raw data for the dataset array.

Compact Data

This field is only present for compact data storage. - It contains the raw data for the dataset array.

-
- -
-

Version 3 of this message re-structured the format into specific - properties that are required for each layout class. - -
-

- - - - - - - - - - - - - - - - - - - -
- Data Layout Message (Version 3) -
bytebytebytebyte
VersionLayout Class 

Properties

-
- -
-
- - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version -

The version number information is used for changes in the format of layout message - and is described here:

- - - - - - - - - - -
VersionDescription
3Used by the version 1.6.3 and later of the library to store properties - for each layout class.
-
Layout Class

The layout class specifies how the other fields of the layout message are to be - interpreted. A value of one indicates contiguous storage, a value of two - indicates chunked storage, while a value of zero indicates compact storage.

Properties

This variable-sized field encodes information specific to each - layout class and is described below. If there is no property - information specified for a layout class, the size of this field - is zero bytes.

-
- -
-

Class-specific information for compact layout (Class 0): (Note: The dimensionality information - is in the Dataspace message) - -
-

- - - - - - - - - - - - - - - - - - -
- Property Descriptions -
bytebytebytebyte
Size 

Raw Data...

-
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription
Size

This field contains the size of the raw data for the dataset array.

Raw Data

This field contains the raw data for the dataset array.

-
- -
-

Class-specific information for contiguous layout (Class 1): (Note: The dimensionality information - is in the Dataspace message) - -
-

- - - - - - - - - - - - - - - - - -
- Property Descriptions -
bytebytebytebyte

Address


Size

-
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription
Address

This is the address of the first byte of raw data storage. - The address may have the "undefined address" value, to indicate - that storage has not yet been allocated for this array.

Size

This field contains the size allocated to store the raw data.

-
- -
-

Class-specific information for chunked layout (Class 2): - -
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Property Descriptions -
bytebytebytebyte
Dimensionality 

Address

Dimension 0 (4-bytes)
Dimension 1 (4-bytes)
...
Dataset Element Size
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Dimensionality

A chunk has a fixed dimensionality. This field specifies - the number of dimension size fields later in the message.

Address

This is the address of the B-tree that is used to look up the addresses of the - chunks. The address may have the "undefined address" value, to indicate that - storage has not yet been allocated for this array.

Dimensions

These values define the dimension size of a single chunk, in - units of array elements (not bytes). The first dimension stored in - the list of dimensions is the slowest changing dimension and the - last dimension stored is the fastest changing dimension. -

-
Dataset Element Size

The size of a dataset element, in bytes. -

-
-
- -
-

Name: Reserved - Not Assigned Yet

-

Header Message Type: 0x0009

-

Length: N/A

-

Status: N/A

-

Format of Data: N/A

- -

Purpose and Description: This message type was skipped during the initial - specification of the file format and may be used in a future expansion to the format. - -

-

Name: Reserved - Not Assigned Yet

-

Header Message Type: 0x0009

-

Length: N/A

-

Status: N/A

-

Format of Data: N/A

- -

Purpose and Description: This message type was skipped during the initial - specification of the file format and may be used in a future expansion to the format. - -

-

Name: Data Storage - Filter Pipeline

-

Header Message Type: 0x000B

-

Length: varies

-

Status: Optional, may not be repeated.

- -

Description: This message describes the - filter pipeline which should be applied to the data stream by - providing filter identification numbers, flags, a name, and - client data.

- -

Format of Data: -
-

- - - - - - - - - - - - - - - - - - - - - - - -
- Filter Pipeline Message -
bytebytebytebyte
VersionNumber of FiltersReserved
Reserved

Filter List

-
- -
-
- - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version

The version number for this message. This document - describes version 1.

Number of Filters

The total number of filters described by this - message. The maximum possible number of filters in a - message is 32.

Filter List

A description of each filter. A filter description - appears in the next table.

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Filter Description -
bytebytebytebyte
Filter IdentificationName Length
FlagsNumber of Values for Client Data

Name


Client Data

Padding
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Filter Identification -

- This value, often referred to as a filter identifier, - is designed to be a unique identifier for the filter. - Values from zero through 32,767 are reserved for filters - supported by The HDF Group in the HDF5 library and for - filters requested and supported by third parties. - Filters supported by The HDF Group are documented immediately - below. Information on 3rd-party filters can be found at - - https://github.com/HDFGroup/hdf5_plugins/blob/master/docs/RegisteredFilterPlugins.md. - 1 -

- To request a filter identifier, please contact - The HDF Group’s Help Desk at - . - You will be asked to provide the following information: -

    -
  1. Contact information for the developer requesting the - new identifier -
  2. A short description of the new filter -
  3. Links to any relevant information, including licensing - information -
-

- Values from 32768 to 65535 are reserved for non-distributed uses - (for example, internal company usage) or for application usage - when testing a feature. The HDF Group does not track or document - the use of the filters with identifiers from this range. - -

- The filters currently in library version 1.6.5 are - listed below: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
IdentificationNameDescription
1deflateGZIP deflate compression
2shuffleData element shuffling
3fletcher32Fletcher32 checksum
4szipSZIP compression
-

Name Length

Each filter has an optional null-terminated ASCII name - and this field holds the length of the name including the - null termination padded with nulls to be a multiple of - eight. If the filter has no name then a value of zero is - stored in this field.

Flags

The flags indicate certain properties for a filter. The - bit values defined so far are:

- - - - - - - - - - -
ValueDescription
bit 1If set then the filter is an optional filter. - During output, if an optional filter fails it will be - silently removed from the pipeline.
-
Client Data Number of Values

Each filter can store a few integer values to control - how the filter operates. The number of entries in the - Client Data array is stored in this field.

Name

If the Name Length field is non-zero then it will - contain the size of this field, a multiple of eight. This - field contains a null-terminated, ASCII character - string to serve as a comment/name for the filter.

Client Data

This is an array of four-byte integers which will be - passed to the filter function. The Client Data Number of - Values determines the number of elements in the array.

Padding

Four bytes of zeros are added to the message at this - point if the Client Data Number of Values field contains - an odd number.

-
-

-

- 1If you are reading - an earlier version of this document, this link may have changed. - If the link does not work, use the latest version of this document - on The HDF Group’s website, - - H5.format.html; - the link there will always be correct. - (Return) -

- -
-

Name: Attribute

-

Header Message Type: 0x000C -

Length: varies -

Status: Optional, may be repeated. - -

Description: The Attribute - message is used to list objects in the HDF file which are used - as attributes, or "metadata" about the current object. An - attribute is a small dataset; it has a name, a datatype, a data - space, and raw data. Since attributes are stored in the object - header they must be relatively small (<64KB) and can be - associated with any type of object which has an object header - (groups, datasets, named types and spaces, etc.). - -

Note: Attributes on an object must have unique names. (The HDF5 library - currently enforces this by causing the creation of an attribute with - a duplicate name to fail). Attributes on different objects may have the - same name, however. - -

Format of Data: -
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Attribute Message (Version 1) -
bytebytebytebyte
VersionReservedName Size
Datatype SizeDataspace Size

Name


Datatype


Dataspace


Data

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version

The version number information is used for changes in the format of the - attribute message and is described here:

- - - - - - - - - - - - - - - -
VersionDescription
0Never used.
1Used by the library before version 1.6 to encode attribute message. - This version does not support shared data type.
-
Reserved

This field is reserved for later use and is set to - zero.

Name Size

The length of the attribute name in bytes including the - null terminator. Note that the Name field below may - contain additional padding not represented by this - field.

Datatype Size

The length of the datatype description in the Datatype - field below. Note that the Datatype field may contain - additional padding not represented by this field.

Dataspace Size

The length of the dataspace description in the Dataspace - field below. Note that the Dataspace field may contain - additional padding not represented by this field.

Name

The null-terminated attribute name. This field is - padded with additional null characters to make it a - multiple of eight bytes.

Datatype

The datatype description follows the same format as - described for the datatype object header message. This - field is padded with additional zero bytes to make it a - multiple of eight bytes.

Dataspace

The dataspace description follows the same format as - described for the dataspace object header message. This - field is padded with additional zero bytes to make it a - multiple of eight bytes.

Data

The raw data for the attribute. The size is determined - from the datatype and dataspace descriptions. This - field is not padded with additional bytes.

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Attribute Message (Version 2) -
bytebytebytebyte
VersionFlagName Size
Type SizeSpace Size

Name


Type


Space


Data

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version

The version number information is used for changes in the format of the - attribute message and is described here:

- - - - - - - - - - -
VersionDescription
2Used by the library of version 1.6.x and after to encode attribute message. - This version supports shared data type. The fields of name, type, and space - are not padded with additional bytes of zero.
-
Flag

This field indicates whether the data type of this attribute is shared:

- - - - - - - - - - - - - - - -
ValueDescription
0Datatype is not shared.
1Datatype is shared.
-
Name Size

The length of the attribute name in bytes including the - null terminator.

Datatype Size

The length of the datatype description in the Datatype - field below.

Dataspace Size

The length of the dataspace description in the Dataspace - field below.

Name

The null-terminated attribute name. This field is not - padded with additional bytes.

Datatype

The datatype description follows the same format as - described for the datatype object header message. This - field is not padded with additional bytes.

Dataspace

The dataspace description follows the same format as - described for the dataspace object header message. This - field is not padded with additional bytes.

Data

The raw data for the attribute. The size is determined - from the datatype and dataspace descriptions. This - field is not padded with additional zero - bytes.

-
- -
-

Name: Object Comment

- -

Header Message Type: 0x000D

-

Length: varies

-

Status: Optional, may not be repeated.

- -

Description: The object comment is - designed to be a short description of an object. An object comment - is a sequence of non-zero (\0) ASCII characters with no other - formatting included by the library.

- -

Format of Data: -
-

- - - - - - - - - - - - - -
- Name Message -
bytebytebytebyte

Comment

-
- -
-
- - - - - - - - - - -
Field NameDescription
NameA null terminated ASCII character string.
-
- -
-

Name: Object Modification Date & Time (Old)

- -

Header Message Type: 0x000E

-

Length: fixed

-

Status: Optional, may not be repeated.

- -

Description: The object modification date - and time is a timestamp which indicates (using ISO-8601 date and - time format) the last modification of an object. The time is - updated when any object header message changes according to the - system clock where the change was posted. - -

This modification time message is deprecated in favor of the "new" - modification time message (Message Type 0x0012) and is no longer written - to the file in versions of the HDF5 library after the 1.6.0 version. -

- -

Format of Data: -
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Modification Time Message -
bytebytebytebyte
Year
MonthDay of Month
HourMinute
SecondReserved
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Year

The four-digit year as an ASCII string. For example, - 1998. All fields of this message should be interpreted - as coordinated universal time (UTC)

Month

The month number as a two digit ASCII string where - January is 01 and December is 12.

Day of Month

The day number within the month as a two digit ASCII - string. The first day of the month is 01.

Hour

The hour of the day as a two digit ASCII string where - midnight is 00 and 11:00pm is 23.

Minute

The minute of the hour as a two digit ASCII string where - the first minute of the hour is 00 and - the last is 59.

Second

The second of the minute as a two digit ASCII string - where the first second of the minute is 00 - and the last is 59.

Reserved

This field is reserved and should always be zero.

-
- -
-

Name: Shared Object Message

-

Header Message Type: 0x000F

-

Length: Fixed

-

Status: Optional, may be repeated.

- -

Description: A constant message can be shared among - several object headers. A Shared Object Message contains the address of - the object message to be shared. Care must be exercised to prevent cycles when a - message of one object header points to a message in some other object header. - Starting from Version 2 of the Shared Object Message, the Flags - field becomes unused. -

- -

Format of Data: -
-

- - - - - - - - - - - - - - - - - - - -
- Shared Object Message (Version 1) -
byte - byte - byte - byte -
VersionFlagsReserved
Reserved

Pointer

-
- -
-
- - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version

The version number is used when there are changes in the format - of a shared object message and is described here:

- - - - - - - - - - - - - - - -
VersionDescription
0Never used.
1Used by the library before version 1.6.1. In this version, - the Flags field is used to indicate whether the actual message is - stored in the global heap (never implemented). The Pointer field - either contains the header message address in the global heap - (never implemented) or the address of the shared object header.
-
Flags

The Shared Message message points to a message which is - shared among multiple object headers. The Flags field - describes the type of sharing:

- - - - - - - - - - - - - - - -
BitDescription
0If this bit is clear then the actual message is the - first message in some other object header; otherwise - the actual message is stored in the global heap (never - implemented).
2-7Reserved (always zero)
-
Pointer

The address of the object header - containing the message to be shared.

-
- -
-
- - - - - - - - - - - - - - - -
- Shared Object Message (Version 2) -
byte - byte - byte - byte -
VersionFlags 

Pointer

-
- -
-
- - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version

The version number is used when there are changes in the format - of a shared object message and is described here:

- - - - - - - - - - -
VersionDescription
2Used by the library of version 1.6.1 and after. In this version, - The Flags field is not used and the Pointer field contains the address - of the object header containing the message to be shared.
-
Flags

Unused.

Pointer

The address of the object header - containing the message to be shared.

-
- - -
-

Name: Object Header Continuation

-

Header Message Type: 0x0010

-

Length: fixed

-

Status: Optional, may be repeated.

-

Description: The object header continuation is the location - in the file of more header messages for the current data object. This can be - used when header blocks become too large or are likely to change over time.

- -

Format of Data: -
-

- - - - - - - - - - - - - - - - - -
- Object Header Continuation Message -
bytebytebytebyte

Offset


Length

-
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription
Offset

This value is the offset in bytes from the beginning of the file where the - header continuation information is located.

Length

This value is the length in bytes of the header continuation information in - the file.

-
- -
-

Name: Group Message

-

Header Message Type: 0x0011

-

Length: fixed

-

Status: Required for groups, may not be repeated.

-

Description: Each group has a B-tree and a - name heap which are pointed to by this message.

-

Format of data: - -
-

- - - - - - - - - - - - - - - - - -
- Group Message -
bytebytebytebyte

B-tree Address


Heap Address

-
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription
B-tree Address

This value is the offset in bytes from the beginning of the file - where the B-tree is located.

Heap Address

This value is the offset in bytes from the beginning of the file - where the group name heap is located.

-
- -
-

Name: Object Modification Date & Time

- -

Header Message Type: 0x0012

-

Length: Fixed

-

Status: Optional, may not be repeated.

- -

Description: The object modification date - and time is a timestamp which indicates the last modification of an object. - The time is updated when any object header message changes according to the - system clock where the change was posted. -

- -

Format of Data: -

- - - - - - - - - - - - - - - - - - -
- Modification Time Message -
bytebytebytebyte
VersionReserved
Seconds After Epoch
-
- -
-
- - - - - - - - - - - - - - - - - - - - -
Field NameDescription
Version

The version number is used for changes in the format of Object Modification Time - and is described here:

- - - - - - - - - - - - - - - -
VersionDescription
0Never used.
1Used by Version 1.6.1 and after of the library to encode time. In - this version, the time is the seconds after Epoch.
-
Reserved

This field is reserved and should always be zero.

Seconds After Epoch

The number of seconds since 0 hours, 0 minutes, 0 seconds, - January 1, 1970, Coordinated Universal Time.

-
- -
-

Disk Format: Level 2b - Data Object Data Storage

-

The data for an object is stored separately from the header -information in the file and may not actually be located in the HDF5 file -itself if the header indicates that the data is stored externally. The -information for each record in the object is stored according to the -dimensionality of the object (indicated in the dimensionality header message). -Multi-dimensional data is stored in C order [same as current scheme], i.e. the -"last" dimension changes fastest. -

Data whose elements are composed of simple number-types are stored in -native-endian IEEE format, unless they are specifically defined as being stored -in a different machine format with the architecture-type information from the -number-type header message. This means that each architecture will need to -[potentially] byte-swap data values into the internal representation for that -particular machine. -

Data with a variable-length datatype is stored in the global heap -of the HDF5 file. Global heap identifiers are stored in the -data object storage. -

Data whose elements are composed of pointer number-types are stored in several -different ways depending on the particular pointer type involved. Simple -pointers are just stored as the dataset offset of the object being pointed to with the -size of the pointer being the same number of bytes as offsets in the file. -Dataset region references are stored as a heap-ID which points to the following -information within the file-heap: an offset of the object pointed to, number-type -information (same format as header message), dimensionality information (same -format as header message), sub-set start and end information (i.e. a coordinate -location for each), and field start and end names (i.e. a [pointer to the] -string indicating the first field included and a [pointer to the] string name -for the last field). - -

Data of a compound datatype is stored as a contiguous stream of the items -in the structure, with each item formatted according to its datatype.

- -
-

Appendix

-

Definitions of various terms used in this document. -

-

The "undefined address" for a file is a -file address with all bits set, i.e. 0xffff...ff. -

The "unlimited size" for a size is a -value with all bits set, i.e. 0xffff...ff. - -

- - diff --git a/doxygen/examples/H5.format.2.0.html b/doxygen/examples/H5.format.2.0.html deleted file mode 100644 index 789c6e501a9..00000000000 --- a/doxygen/examples/H5.format.2.0.html +++ /dev/null @@ -1,15255 +0,0 @@ - - - -HDF5 File Format Specification Version 2.0 - - - - -
-
- - - - - - - - -
-
    -
  1. Introduction
    2. - -
      -
    1. This Document
      2. -
    2. Changes for HDF5 1.10
      3. -
    -     - -
  2. Disk Format: Level 0 - File - Metadata
    3. - -
      -
    1. Disk Format: Level 0A - Format - Signature and Superblock
      2. -
    2. Disk Format: Level 0B - File - Driver Info
      3. -
    3. Disk Format: Level 0C - - Superblock Extension
      4. -
    -     -
  3. Disk Format: Level 1 - File - Infrastructure
    4. - -
      -
    1. Disk Format: Level 1A - B-trees - and B-tree Nodes
      2. -
        -
      1. Disk Format: Level 1A1 - - Version 1 B-trees (B-link Trees)
        2. -
      2. Disk Format: Level 1A2 - - Version 2 B-trees
        3. -
      -
    2. Disk Format: Level 1B - Group - Symbol Table Nodes
      3. -
    3. Disk Format: Level 1C - - Symbol Table Entry
      4. -
    4. Disk Format: Level 1D - Local - Heaps
      5. -
    5. Disk Format: Level 1E - Global - Heap
      6. -
    6. Disk Format: Level 1F - - Fractal Heap
      7. -
    7. Disk Format: Level 1G - - Free-space Manager
      8. -
    8. Disk Format: Level 1H - Shared - Object Header Message Table
      9. -
    -     -
  4. Disk Format: Level 2 - Data - Objects
    5. - -
      -
    1. Disk Format: Level 2A - Data - Object Headers
      2. -
        -
      1. Disk Format: Level - 2A1 - Data Object Header Prefix
        2. -
          -
        1. Version 1 Data - Object Header Prefix
          2. -
        2. Version 2 Data - Object Header Prefix
          3. -
        -
      2. Disk Format: Level - 2A2 - Data Object Header Messages
        3. -
          -
        1. The NIL Message
          2. - -
        2. The Dataspace Message
          3. - -
        3. The Link Info Message
          4. - -
        -
      -
    -     -
- 		  -
    -
  1. Disk Format: Level 2 - Data - Objects (Continued)
    2. -
      -
    1. Disk Format: Level 2A - Data - Object Headers (Continued)
      2. -
        -
      1. Disk Format: Level - 2A2 - Data Object Header Messages (Continued)
        2. -
          -
        1. The Datatype Message
          2. - -
        2. The Data Storage - - Fill Value (Old) Message
          3. - -
        3. The Data Storage - Fill - Value Message
          4. - -
        4. The Link Message
          5. - -
        5. The Data Storage - - External Data Files Message
          6. - -
        6. The Data Storage - Layout - Message
          7. - -
        7. The Bogus Message
          8. - -
        8. The Group Info Message
          9. - -
        9. The Data Storage - Filter - Pipeline Message
          10. - -
        10. The Attribute Message
          11. - -
        11. The Object Comment - Message
          12. - -
        12. The Object - Modification Time (Old) Message
          13. - -
        13. The Shared Message - Table Message
          14. - -
        14. The Object Header - Continuation Message
          15. - -
        15. The Symbol Table - Message
          16. - -
        16. The Object - Modification Time Message
          17. - -
        17. The B-tree - ‘K’ Values Message
          18. - -
        18. The Driver Info Message
          19. - -
        19. The Attribute Info Message
          20. - -
        20. The Object Reference - Count Message
          21. - -
        21. The File Space Info - Message
          22. - -
        -
      -
    2. Disk Format: Level 2B - Data - Object Data Storage
      3. -
    - -
  2. Appendix A: Definitions
    3. -
  3. Appendix B: File Memory - Allocation Types
    4. -
-
-
- - - -
-
-
-

I. Introduction

 - - - - - - - - - - - - - - - - - - - - - - - -
  -
HDF5 Groups - 		 
 Figure 1: Relationships among - the HDF5 root group, other groups, and objects -
 
 HDF5 Objects 
 Figure 2: HDF5 objects -- - datasets, datatypes, or dataspaces -
 
- - -

The format of an HDF5 file on disk encompasses several key ideas - of the HDF4 and AIO file formats as well as addressing some - shortcomings therein. The new format is more self-describing than the - HDF4 format and is more uniformly applied to data objects in the file.

- -

An HDF5 file appears to the user as a directed graph. The nodes - of this graph are the higher-level HDF5 objects that are exposed by the - HDF5 APIs:

- -
    -
  • Groups
    • -
  • Datasets
    • -
  • Committed (formerly Named) datatypes
    • -
- -

At the lowest level, as information is actually written to the - disk, an HDF5 file is made up of the following objects:

-
    -
  • A superblock
    • -
  • B-tree nodes
    • -
  • Heap blocks
    • -
  • Object headers
    • -
  • Object data
    • -
  • Free space
    • -
- -

The HDF5 Library uses these low-level objects to represent the - higher-level objects that are then presented to the user or to - applications through the APIs. For instance, a group is an object - header that contains a message that points to a local heap (for storing - the links to objects in the group) and to a B-tree (which indexes the - links). A dataset is an object header that contains messages that - describe datatype, dataspace, layout, filters, external files, fill - value, and other elements with the layout message pointing to either a - raw data chunk or to a B-tree that points to raw data chunks.

- - -
-

I.A. This Document

 - -

- This document describes the lower-level data objects; the higher-level - objects and their properties are described in the HDF5 - User Guide. -

- -

- Three levels of information comprise the file format. Level 0 contains - basic information for identifying and defining information about the - file. Level 1 information contains the information about the pieces of - a file shared by many objects in the file (such as a B-trees and - heaps). Level 2 is the rest of the file and contains all of the data - objects, with each object partitioned into header information, also - known as metadata, and data. -

- -

- The sizes of various fields in the following layout tables are - determined by looking at the number of columns the field spans in the - table. There are three exceptions: (1) The size may be overridden by - specifying a size in parentheses, (2) the size of addresses is - determined by the Size of Offsets field in the superblock and - is indicated in this document with a superscripted ‘O’, and - (3) the size of length fields is determined by the Size of - Lengths field in the superblock and is indicated in this document with - a superscripted ‘L’. -

- -

Values for all fields in this document should be treated as - unsigned integers, unless otherwise noted in the description of a - field. Additionally, all metadata fields are stored in little-endian - byte order.

- -

- All checksums used in the format are computed with the Jenkins’ - lookup3 algorithm. -

- -

Whenever a bit flag or field is mentioned for an entry, bits are - numbered from the lowest bit position in the entry.

- -

Various tables in this document aligned with “This space - inserted only to align table nicely”. These entries in the table - are just to make the table presentation nicer and do not represent any - values or padding in the file.

- - -
-

I.B. Changes for HDF5 1.10

 - -

As of October 2015, changes in the file format for HDF5 1.10 have - not yet been finalized.

- - - -
-
-
-

- II. Disk Format: Level 0 - File Metadata -

- -
-

- II.A. Disk Format: Level 0A - Format - Signature and Superblock -

- -

The superblock may begin at certain predefined offsets within the - HDF5 file, allowing a block of unspecified content for users to place - additional information at the beginning (and end) of the HDF5 file - without limiting the HDF5 Library’s ability to manage the objects - within the file itself. This feature was designed to accommodate - wrapping an HDF5 file in another file format or adding descriptive - information to an HDF5 file without requiring the modification of the - actual file’s information. The superblock is located by searching - for the HDF5 format signature at byte offset 0, byte offset 512, and at - successive locations in the file, each a multiple of two of the - previous location; in other words, at these byte offsets: 0, 512, 1024, - 2048, and so on.

- -

The superblock is composed of the format signature, followed by a - superblock version number and information that is specific to each - version of the superblock. Currently, there are three versions of the - superblock format. Version 0 is the default format, while version 1 is - basically the same as version 0 with additional information when a - non-default B-tree ‘K’ value is stored. Version 2 is the - latest format, with some fields eliminated or compressed and with - superblock extension and checksum support.

- -

Version 0 and 1 of the superblock are described below:

- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Superblock (Versions 0 and 1)
bytebytebytebyte

Format Signature (8 bytes)
-
Version # of SuperblockVersion # of File’s Free Space StorageVersion # of Root Group Symbol Table EntryReserved (zero)
Version # of Shared Header Message FormatSize of OffsetsSize of LengthsReserved (zero)
Group Leaf Node KGroup Internal Node K
File Consistency Flags
Indexed Storage Internal - Node K1 - Reserved (zero)1

Base AddressO
-

Address of File Free space InfoO
-

End of File AddressO
-

Driver Information Block AddressO
-
Root Group Symbol Table Entry
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets.”)
 (Items marked with a ‘1’ in the above table are - new in version 1 of the superblock)
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Format Signature

This field contains a constant value and can be used - to quickly identify a file as being an HDF5 file. The constant - value is designed to allow easy identification of an HDF5 file and - to allow certain types of data corruption to be detected. The file - signature of an HDF5 file always contains the following values:

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Decimal:13772687013102610
Hexadecimal:894844460d0a1a0a
ASCII C Notation:\211HDF\r\n\032\n
-
-

- This signature both identifies the file as an HDF5 file and - provides for immediate detection of common file-transfer problems. - The first two bytes distinguish HDF5 files on systems that expect - the first two bytes to identify the file type uniquely. The first - byte is chosen as a non-ASCII value to reduce the probability that - a text file may be misrecognized as an HDF5 file; also, it catches - bad file transfers that clear bit 7. Bytes two through four name - the format. The CR-LF sequence catches bad file transfers that - alter newline sequences. The control-Z character stops file display - under MS-DOS. The final line feed checks for the inverse of the - CR-LF translation problem. (This is a direct descendent of the PNG - file signature.) -

-

- This field is present in version 0+ of the superblock. -

Version Number of the Superblock

This value is used to determine the format of the - information in the superblock. When the format of the information - in the superblock is changed, the version number is incremented to - the next integer and can be used to determine how the information - in the superblock is formatted.

- -

Values of 0, 1 and 2 are defined for this field. (The format - of version 2 is described below, not here)

- -

- This field is present in version 0+ of the superblock. -

Version Number of the File’s Free Space - Information

 -

This value is used to determine the format of the - file’s free space information.

-

- The only value currently valid in this field is ‘0’, - which indicates that the file’s free space is as described below. -

- -

- This field is present in version 0 and 1 of the - superblock. -

-

Version Number of the Root Group Symbol Table Entry

This value is used to determine the format of the - information in the Root Group Symbol Table Entry. When the format - of the information in that field is changed, the version number is - incremented to the next integer and can be used to determine how - the information in the field is formatted.

-

- The only value currently valid in this field is ‘0’, - which indicates that the root group symbol table entry is formatted - as described below. -

-

- This field is present in version 0 and 1 of the - superblock. -

Version Number of the Shared Header Message Format

This value is used to determine the format of the - information in a shared object header message. Since the format of - the shared header messages differs from the other private header - messages, a version number is used to identify changes in the - format.

-

- The only value currently valid in this field is ‘0’, - which indicates that shared header messages are formatted as - described below. -

- -

- This field is present in version 0 and 1 of the - superblock. -

Size of Offsets

This value contains the number of bytes used to store - addresses in the file. The values for the addresses of objects in - the file are offsets relative to a base address, usually the - address of the superblock signature. This allows a wrapper to be - added after the file is created without invalidating the internal - offset locations.

- -

- This field is present in version 0+ of the superblock. -

Size of Lengths

This value contains the number of bytes used to store - the size of an object.

-

- This field is present in version 0+ of the superblock. -

Group Leaf Node K

 -

Each leaf node of a group B-tree will have at least this many - entries but not more than twice this many. If a group has a single - leaf node then it may have fewer entries.

-

This value must be greater than zero.

-

- See the description of B-trees below. -

- -

- This field is present in version 0 and 1 of the - superblock. -

-

Group Internal Node K

 -

Each internal node of a group B-tree will have at least this - many entries but not more than twice this many. If the group has - only one internal node then it might have fewer entries.

-

This value must be greater than zero.

-

- See the description of B-trees below. -

- -

- This field is present in version 0 and 1 of the - superblock. -

-

File Consistency Flags

 -

This value contains flags to indicate information about the - consistency of the information contained within the file. - Currently, the following bit flags are defined:

-
    -
  • Bit 0 set indicates that the file is opened for - write-access.
    • -
  • Bit 1 set indicates that the file has been verified for - consistency and is guaranteed to be consistent with the format - defined in this document.
    • -
  • Bits 2-31 are reserved for future use.
    • -
Bit 0 should be set as the first action when a file is opened for - write access and should be cleared only as the final action when - closing a file. Bit 1 should be cleared during normal access to a - file and only set after the file’s consistency is guaranteed - by the library or a consistency utility. -

- -

- This field is present in version 0+ of the superblock. -

-

Indexed Storage Internal Node K

 -

Each internal node of an indexed storage B-tree will have at - least this many entries but not more than twice this many. If the - index storage B-tree has only one internal node then it might have - fewer entries.

-

This value must be greater than zero.

-

- See the description of B-trees below. -

- -

- This field is present in version 1 of the superblock. -

-

Base Address

 -

This is the absolute file address of the first byte of the - HDF5 data within the file. The library currently constrains this - value to be the absolute file address of the superblock itself when - creating new files; future versions of the library may provide - greater flexibility. When opening an existing file and this address - does not match the offset of the superblock, the library assumes - that the entire contents of the HDF5 file have been adjusted in the - file and adjusts the base address and end of file address to - reflect their new positions in the file. Unless otherwise noted, - all other file addresses are relative to this base address.

- -

- This field is present in version 0+ of the superblock. -

-

Address of Global Free-space Index

 -

- The file’s free space is not persistent for version 0 and 1 - of the superblock. Currently this field always contains the undefined address. -

- -

- This field is present in version 0 and 1 of the - superblock. -

-

End of File Address

 -

This is the absolute file address of the first byte past the - end of all HDF5 data. It is used to determine whether a file has - been accidentally truncated and as an address where file data - allocation can occur if space from the free list is not used.

- -

- This field is present in version 0+ of the superblock. -

-

Driver Information Block Address

 -

- This is the relative file address of the file driver information - block which contains driver-specific information needed to reopen - the file. If there is no driver information block then this entry - should be the undefined address. -

- -

- This field is present in version 0 and 1 of the - superblock. -

-

Root Group Symbol Table Entry

 -

- This is the symbol table entry of - the root group, which serves as the entry point into the group - graph for the file. -

- -

- This field is present in version 0 and 1 of the - superblock. -

-
-
- -
-

Version 2 of the superblock is described below:

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Superblock (Version 2)
bytebytebytebyte

Format Signature (8 bytes)
-
Version # of SuperblockSize of OffsetsSize of LengthsFile Consistency Flags

Base AddressO
-

Superblock Extension AddressO
-

End of File AddressO
-

Root Group Object Header AddressO
-
Superblock Checksum
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets.”)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Format Signature

 -

This field is the same as described for versions 0 and 1 of - the superblock.

-

Version Number of the Superblock

 -

This field has a value of 2 and has the same meaning as for - versions 0 and 1.

-

Size of Offsets

 -

This field is the same as described for versions 0 and 1 of - the superblock.

-

Size of Lengths

 -

This field is the same as described for versions 0 and 1 of - the superblock.

-

File Consistency Flags

 -

This field is the same as described for versions 0 and 1 - except that it is smaller (the number of reserved bits has been - reduced from 30 to 6).

-

Base Address

 -

This field is the same as described for versions 0 and 1 of - the superblock.

-

Superblock Extension Address

 -

- The field is the address of the object header for the superblock extension. If there is no - extension then this entry should be the undefined - address. -

-

End of File Address

 -

This field is the same as described for versions 0 and 1 of - the superblock.

-

Root Group Object Header Address

 -

- This is the address of the root group - object header, which serves as the entry point into the group - graph for the file. -

-

Superblock Checksum

 -

The checksum for the superblock.

-
-
- -
-

- II.B. Disk Format: Level 0B - File Driver - Info -

- -

- The driver information block is an optional region of the file - which contains information needed by the file driver to reopen a file. - The format is described below: -

- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - -
Driver Information Block
bytebytebytebyte
VersionReserved
Driver Information Size

Driver Identification (8 bytes)
-

-
Driver Information (variable size)
-
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

 -

The version number of the Driver Information Block. This - document describes version 0.

-

Driver Information Size

 -

- The size in bytes of the Driver Information field. -

-

Driver Identification

 -

- This is an eight-byte ASCII string without null termination which - identifies the driver and/or version number of the Driver - Information Block. The predefined driver encoded in this field by - the HDF5 Library is identified by the letters - NCSA - followed by the first four characters of the driver name. If the - Driver Information block is not the original version then the last - letter(s) of the identification will be replaced by a version - number in ASCII, starting with 0. -

-

Identification for user-defined drivers is also eight-byte - long. It can be arbitrary but should be unique to avoid the four - character prefix “NCSA”.

-

Driver Information

Driver information is stored in a format defined by the file - driver (see description below).
-
- -
The two drivers encoded in the -Driver Identification field are as follows: -
    -
  • Multi driver: -

    The identifier for this driver is “NCSAmulti”. This - driver provides a mechanism for segregating raw data and different - types of metadata into multiple files. These files are viewed by the - library as a single virtual HDF5 file with a single file address. A - maximum of 6 files will be created for the following data: - superblock, B-tree, raw data, global heap, local heap, and object - header. More than one type of data can be written to the same file.

    -
    • -
  • Family driver -

    The identifier for this driver is “NCSAfami” and is - encoded in this field for library version 1.8 and after. This driver - is designed for systems that do not support files larger than 2 - gigabytes by splitting the HDF5 file address space across several - smaller files. It does nothing to segregate metadata and raw data; - they are mixed in the address space just as they would be in a single - contiguous file.

    -
    • -
-

- The format of the Driver Information field for the above two - drivers are described below: -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Multi Driver Information
bytebytebytebyte
Member MappingMember MappingMember MappingMember Mapping
Member MappingMember MappingReservedReserved

Address of Member File 1
-

End of Address for Member File 1
-

Address of Member File 2
-

End of Address for Member File 2
-

... ...
-

Address of Member File N
-

End of Address for Member File N
-

Name of Member File 1 (variable - size)
-

Name of Member File 2 (variable - size)
-

... ...
-

Name of Member File N (variable - size)
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Member Mapping

These fields are integer values from 1 to 6 - indicating how the data can be mapped to or merged with another - type of data.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Member MappingDescription
1The superblock data.
2The B-tree data.
3The raw data.
4The global heap data.
5The local heap data.
6The object header data.
-

-

For example, if the third field has the value 3 and all the - rest have the value 1, it means there are two files: one for raw - data, and one for superblock, B-tree, global heap, local heap, and - object header.

Reserved

These fields are reserved and should always be zero.

Address of Member File N

This field Specifies the virtual address at which the - member file starts.

-

N is the number of member files.

End of Address for Member File N

This field is the end of the allocated address for - the member file.

Name of Member File N

- This field is the null-terminated name of the member file and its - length should be multiples of 8 bytes. Additional bytes will be - padded with NULLs. The default naming convention is %s-X.h5, - where X is one of the letters s (for superblock), - b (for B-tree), r (for raw data), g (for - global heap), l (for local heap), and o (for - object header). The name of the whole HDF5 file will substitute the - %s in the string. -

-
- -
-
- - - - - - - - - - - - - - -
Family Driver Information
bytebytebytebyte

Size of Member File
-
-
- -
-
- - - - - - - - - - -
Field NameDescription

Size of Member File

This field is the size of the member file in the - family of files.

-
- -
-

- II.C. Disk Format: Level 0C - Superblock - Extension -

- -

- The superblock extension is used to store superblock metadata - which is either optional, or added after the version of the superblock - was defined. Superblock extensions may only exist when version 2+ of - superblock is used. A superblock extension is an object header which - may hold the following messages: -

- - - - -
-
-
-

- III. Disk Format: Level 1 - File - Infrastructure -

- -
-

- III.A. Disk Format: Level 1A - B-trees and B-tree - Nodes -

- -

B-trees allow flexible storage for objects which tend to grow in - ways that cause the object to be stored discontiguously. B-trees are - described in various algorithms books including “Introduction to - Algorithms” by Thomas H. Cormen, Charles E. Leiserson, and Ronald - L. Rivest. B-trees are used in several places in the HDF5 file format, - when an index is needed for another data structure.

- -

The version 1 B-tree structure described below is the original - index structure, but are limited by some bugs in our implementation - (mainly in how they handle deleting records). The version 1 B-trees are - being phased out in favor of the version 2 B-trees described below, - although both types of structures may be found in the same file, - depending on application settings when creating the file.

- -
-

- III.A.1. Disk Format: Level 1A1 - Version 1 - B-trees (B-link Trees) -

- -

- Version 1 B-trees in HDF5 files an implementation of the B-link tree, - in which the sibling nodes at a particular level in the tree are stored - in a doubly-linked list, is described in the “Efficient Locking - for Concurrent Operations on B-trees” paper by Phillip Lehman and - S. Bing Yao as published in the ACM Transactions on - Database Systems, Vol. 6, No. 4, December 1981. -

- -

The B-link trees implemented by the file format contain one more - key than the number of children. In other words, each child pointer out - of a B-tree node has a left key and a right key. The pointers out of - internal nodes point to sub-trees while the pointers out of leaf nodes - point to symbol nodes and raw data chunks. Aside from that difference, - internal nodes and leaf nodes are identical.

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
B-link Tree Nodes
bytebytebytebyte
Signature
Node TypeNode LevelEntries Used

Address of Left SiblingO
-

Address of Right SiblingO
-
Key 0 (variable size)

Address of Child 0O
-
Key 1 (variable size)

Address of Child 1O
-
...
Key 2K (variable size) -

Address of Child 2KO
-
Key 2K+1 (variable size) -
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - TREE - ” is used to indicate the beginning of a B-link tree node. - This gives file consistency checking utilities a better chance of - reconstructing a damaged file. -

-

Node Type

 -

- Each B-link tree points to a particular type of data. This field - indicates the type of data as well as implying the maximum degree K - of the tree and the size of each Key field. - - -

- - - - - - - - - - - - - -
Node TypeDescription
0This tree points to group nodes.
1This tree points to raw data chunk nodes.
-

-

Node Level

 -

The node level indicates the level at which this node appears - in the tree (leaf nodes are at level zero). Not only does the level - indicate whether child pointers point to sub-trees or to data, but - it can also be used to help file consistency checking utilities - reconstruct damaged trees.

-

Entries Used

 -

This determines the number of children to which this node - points. All nodes of a particular type of tree have the same - maximum degree, but most nodes will point to less than that number - of children. The valid child pointers and keys appear at the - beginning of the node and the unused pointers and keys appear at - the end of the node. The unused pointers and keys have undefined - values.

-

Address of Left Sibling

 -

- This is the relative file address of the left sibling of the - current node. If the current node is the left-most node at this - level then this field is the undefined - address. -

-

Address of Right Sibling

 -

- This is the relative file address of the right sibling of the - current node. If the current node is the right-most node at this - level then this field is the undefined - address. -

-

Keys and Child Pointers

 -

- Each tree has 2K+1 keys with 2K child pointers - interleaved between the keys. The number of keys and child pointers - actually containing valid values is determined by the node’s - Entries Used field. If that field is N then the - B-link tree contains N child pointers and N+1 - keys. -

-

Key

 -

- The format and size of the key values is determined by the type of - data to which this tree points. The keys are ordered and are - boundaries for the contents of the child pointer; that is, the key - values represented by child N fall between Key N - and Key N+1. Whether the interval is open or closed on - each end is determined by the type of data to which the tree - points. -

- -

The format of the key depends on the node type. For nodes of - node type 0 (group nodes), the key is formatted as follows:

- - - - - -
A single field of Size of Lengths - bytes: - Indicates the byte offset into the local heap - for the first object name in the subtree which that key - describes.
-

- - -

For nodes of node type 1 (chunked raw data nodes), the key is - formatted as follows:

- - - - - - - - - - - - - -
Bytes 1-4:Size of chunk in bytes.
Bytes 4-8:Filter mask, a 32-bit bit field indicating which filters - have been skipped for this chunk. Each filter has an index number - in the pipeline (starting at 0, with the first filter to apply) - and if that filter is skipped, the bit corresponding to its index - is set.
(D + 1) 64-bit fields: - The offset of the chunk within the dataset where D - is the number of dimensions of the dataset, and the last value is - the offset within the dataset’s datatype and should always - be zero. For example, if a chunk in a 3-dimensional dataset - begins at the position [5,5,5], there will be three - such 64-bit values, each with the value of 5, - followed by a 0 value. -
-

- -

Child Pointer

 -

The tree node contains file addresses of subtrees or data - depending on the node level. Nodes at Level 0 point to data - addresses, either raw data chunks or group nodes. Nodes at non-zero - levels point to other nodes of the same B-tree.

-

- For raw data chunk nodes, the child pointer is the address of a - single raw data chunk. For group nodes, the child pointer points to - a symbol table, which contains - information for multiple symbol table entries. -

-
-
- -

Conceptually, each B-tree node looks like this:

-
- - - - - - - - - - - - - - - - - - - - - - -
key[0] child[0] key[1] child[1] key[2] ... ... key[N-1] -  child[N-1] -  key[N] -
-
-
where child[ -i] is a pointer to a sub-tree (at a level above Level 0) or to -data (at Level 0). Each key[ -i] describes an -item stored by the B-tree (a chunk or an object of a group node). -The range of values represented by child[ -i] is indicated by key[ -i] and key[ -i+1]. - - -

- The following question must next be answered: “Is the value - described by key[i] contained in child[i-1] or in child[i]?” - The answer depends on the type of tree. In trees for groups (node type - 0) the object described by key[i] is the greatest object - contained in child[i-1] while in chunk trees (node type 1) the - chunk described by key[i] is the least chunk in child[i]. -

- -

That means that key[0] for group trees is sometimes unused; it - points to offset zero in the heap, which is always the empty string and - compares as “less-than” any valid object name.

- -

- And key[N] for chunk trees is sometimes unused; it contains a - chunk offset which compares as “greater-than” any other - chunk offset and has a chunk byte size of zero to indicate that it is - not actually allocated. -

- -
-

- III.A.2. Disk Format: Level 1A2 - Version 2 - B-trees -

- -

- Version 2 B-trees are “traditional” B-trees, with one major - difference. Instead of just using a simple pointer (or address in the - file) to a child of an internal node, the pointer to the child node - contains two additional pieces of information: the number of records in - the child node itself, and the total number of records in the child - node and all its descendants. Storing this additional information - allows fast array-like indexing to locate the nth record in - the B-tree. -

- -

- The entry into a version 2 B-tree is a header which contains global - information about the structure of the B-tree. The root node - address field in the header points to the B-tree root node, which is - either an internal or leaf node, depending on the value in the - header’s depth field. An internal node consists of - records plus pointers to further leaf or internal nodes in the tree. A - leaf node consists of solely of records. The format of the records - depends on the B-tree type (stored in the header). -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Version 2 B-tree Header
bytebytebytebyte
Signature
VersionTypeThis space inserted - only to align table nicely
Node Size
Record SizeDepth
Split PercentMerge PercentThis space inserted - only to align table nicely

Root Node AddressO
-
Number of Records in Root NodeThis space inserted - only to align table nicely

Total Number of Records in B-treeL
-
Checksum
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
 (Items marked with an ‘L’ in the above table are - of the size specified in “Size of Lengths” field in the - superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - BTHD - ” is used to indicate the header of a version 2 B-link tree - node. -

-

Version

 -

The version number for this B-tree header. This document - describes version 0.

-

Type

 -

This field indicates the type of B-tree:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0A “testing” B-tree, this value should not - be used for storing records in actual HDF5 files. -
1This B-tree is used for indexing indirectly accessed, - non-filtered ‘huge’ fractal heap objects.
2This B-tree is used for indexing indirectly accessed, - filtered ‘huge’ fractal heap objects.
3This B-tree is used for indexing directly accessed, - non-filtered ‘huge’ fractal heap objects.
4This B-tree is used for indexing directly accessed, - filtered ‘huge’ fractal heap objects.
5This B-tree is used for indexing the ‘name’ - field for links in indexed groups.
6This B-tree is used for indexing the ‘creation - order’ field for links in indexed groups.
7This B-tree is used for indexing shared object header - messages.
8This B-tree is used for indexing the ‘name’ - field for indexed attributes.
9This B-tree is used for indexing the ‘creation - order’ field for indexed attributes.
-

-

The format of records for each type is described below.

-

Node Size

 -

This is the size in bytes of all B-tree nodes.

-

Record Size

 -

This field is the size in bytes of the B-tree record.

-

Depth

 -

This is the depth of the B-tree.

-

Split Percent

 -

The percent full that a node needs to increase above before - it is split.

-

Merge Percent

 -

The percent full that a node needs to be decrease below - before it is split.

-

Root Node Address

 -

- This is the address of the root B-tree node. A B-tree with no - records will have the undefined - address in this field. -

-

Number of Records in Root Node

 -

This is the number of records in the root node.

-

Total Number of Records in B-tree

 -

This is the total number of records in the entire B-tree.

-

Checksum

 -

This is the checksum for the B-tree header.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Version 2 B-tree Internal Node
bytebytebytebyte
Signature
VersionTypeRecords 0, 1, 2...N-1 (variable size)

Child Node Pointer 0O
-

Number of Records N0 for Child - Node 0 (variable size)

Total Number of Records for Child Node 0 - (optional, variable size)

Child Node Pointer 1O
-

Number of Records N1 for Child - Node 1 (variable size)

Total Number of Records for Child Node 1 - (optional, variable size)
...

Child Node Pointer NO
-

Number of Records Nn for Child - Node N (variable size)

Total Number of Records for Child Node N - (optional, variable size)
Checksum
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
-
- - -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - BTIN - ” is used to indicate the internal node of a B-link tree. -

-

Version

 -

The version number for this B-tree internal node. This - document describes version 0.

-

Type

 -

This field is the type of the B-tree node. It should always - be the same as the B-tree type in the header.

-

Records

 -

The size of this field is determined by the number of records - for this node and the record size (from the header). The format of - records depends on the type of B-tree.

-

Child Node Pointer

 -

This field is the address of the child node pointed to by the - internal node.

-

Number of Records in Child Node

 -

- This is the number of records in the child node pointed to by the - corresponding Node Pointer. -

-

The number of bytes used to store this field is determined by - the maximum possible number of records able to be stored in the - child node.

-

The maximum number of records in a child node is computed in - the following way:

-
    -
  • Subtract the fixed size overhead for the child node (for - example, its signature, version, checksum, and so on and one - pointer triplet of information for the child node (because there - is one more pointer triplet than records in each internal node)) - from the size of nodes for the B-tree. -
    • -
  • Divide that result by the size of a record plus the - pointer triplet of information stored to reach each child node - from this node.
    • -
- -

-

Note that leaf nodes do not encode any child pointer - triplets, so the maximum number of records in a leaf node is just - the node size minus the leaf node overhead, divided by the record - size.

-

- Also note that the first level of internal nodes above the leaf - nodes do not encode the Total Number of Records in Child - Node value in the child pointer triplets (since it is the same as - the Number of Records in Child Node), so the maximum - number of records in these nodes is computed with the equation - above, but using (Child Pointer, Number of - Records in Child Node) pairs instead of triplets. -

-

The number of bytes used to encode this field is the least - number of bytes required to encode the maximum number of records in - a child node value for the child nodes below this level in the - B-tree.

-

For example, if the maximum number of child records is 123, - one byte will be used to encode these values in this node; if the - maximum number of child records is 20000, two bytes will be used to - encode these values in this node; and so on. The maximum number of - bytes used to encode these values is 8 (in other words, an unsigned - 64-bit integer).

-

Total Number of Records in Child Node

 -

- This is the total number of records for the node pointed to by the - corresponding Node Pointer and all its children. This - field exists only in nodes whose depth in the B-tree node is - greater than 1 (in other words, the “twig” internal - nodes, just above leaf nodes, do not store this field in their - child node pointers). -

-

The number of bytes used to store this field is determined by - the maximum possible number of records able to be stored in the - child node and its descendants.

-

The maximum possible number of records able to be stored in a - child node and its descendants is computed iteratively, in the - following way: The maximum number of records in a leaf node is - computed, then that value is used to compute the maximum possible - number of records in the first level of internal nodes above the - leaf nodes. Multiplying these two values together determines the - maximum possible number of records in child node pointers for the - level of nodes two levels above leaf nodes. This process is - continued up to any level in the B-tree.

-

- The number of bytes used to encode this value is computed in the - same way as for the Number of Records in Child Node field. -

-

Checksum

 -

This is the checksum for this node.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - -
Version 2 B-tree Leaf Node
bytebytebytebyte
Signature
VersionTypeRecord 0, 1, 2...N-1 (variable size)
Checksum
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - BTLF - “ is used to indicate the leaf node of a version 2 B-link - tree. -

-

Version

 -

The version number for this B-tree leaf node. This document - describes version 0.

-

Type

 -

This field is the type of the B-tree node. It should always - be the same as the B-tree type in the header.

-

Records

 -

The size of this field is determined by the number of records - for this node and the record size (from the header). The format of - records depends on the type of B-tree.

-

Checksum

 -

This is the checksum for this node.

-
-
- -
-

The record layout for each stored (in other words, non-testing) - B-tree type is as follows:

- -
- - - - - - - - - - - - - - - - - - - -
Version 2 B-tree, Type 1 Record Layout - Indirectly - Accessed, Non-Filtered, ‘Huge’ Fractal Heap Objects
bytebytebytebyte

Huge Object AddressO
-

Huge Object LengthL
-

Huge Object IDL
-
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
 (Items marked with an ‘L’ in the above table are - of the size specified in “Size of Lengths” field in the - superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Huge Object Address

 -

The address of the huge object in the file.

-

Huge Object Length

 -

The length of the huge object in the file.

-

Huge Object ID

 -

The heap ID for the huge object.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
Version 2 B-tree, Type 2 Record Layout - Indirectly - Accessed, Filtered, ‘Huge’ Fractal Heap Objects
bytebytebytebyte

Filtered Huge Object AddressO
-

Filtered Huge Object LengthL
-
Filter Mask

Filtered Huge Object Memory SizeL
-

Huge Object IDL
-
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
 (Items marked with an ‘L’ in the above table are - of the size specified in “Size of Lengths” field in the - superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Filtered Huge Object Address

 -

The address of the filtered huge object in the file.

-

Filtered Huge Object Length

 -

The length of the filtered huge object in the file.

-

Filter Mask

 -

A 32-bit bit field indicating which filters have been skipped - for this chunk. Each filter has an index number in the pipeline - (starting at 0, with the first filter to apply) and if that filter - is skipped, the bit corresponding to its index is set.

-

Filtered Huge Object Memory Size

 -

The size of the de-filtered huge object in memory.

-

Huge Object ID

 -

The heap ID for the huge object.

-
-
- -
-
-
- - - - - - - - - - - - - - - - -
Version 2 B-tree, Type 3 Record Layout - Directly - Accessed, Non-Filtered, ‘Huge’ Fractal Heap Objects
bytebytebytebyte

Huge Object AddressO
-

Huge Object LengthL
-
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
 (Items marked with an ‘L’ in the above table are - of the size specified in “Size of Lengths” field in the - superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - -
Field NameDescription

Huge Object Address

 -

The address of the huge object in the file.

-

Huge Object Length

 -

The length of the huge object in the file.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - -
Version 2 B-tree, Type 4 Record Layout - Directly - Accessed, Filtered, ‘Huge’ Fractal Heap Objects
bytebytebytebyte

Filtered Huge Object AddressO
-

Filtered Huge Object LengthL
-
Filter Mask

Filtered Huge Object Memory SizeL
-
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
 (Items marked with an ‘L’ in the above table are - of the size specified in “Size of Lengths” field in the - superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Filtered Huge Object Address

 -

The address of the filtered huge object in the file.

-

Filtered Huge Object Length

 -

The length of the filtered huge object in the file.

-

Filter Mask

 -

A 32-bit bit field indicating which filters have been skipped - for this chunk. Each filter has an index number in the pipeline - (starting at 0, with the first filter to apply) and if that filter - is skipped, the bit corresponding to its index is set.

-

Filtered Huge Object Memory Size

 -

The size of the de-filtered huge object in memory.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - -
Version 2 B-tree, Type 5 Record Layout - Link Name - for Indexed Group
bytebytebytebyte
Hash of Name
ID (bytes 1-4)
ID (bytes 5-7)
-
- -
-
- - - - - - - - - - - - - - - - -
Field NameDescription

Hash

 -

This field is hash value of the name for the link. The hash - value is the Jenkins’ lookup3 checksum algorithm applied to - the link’s name.

-

ID

 -

This is a 7-byte sequence of bytes and is the heap ID for the - link record in the group’s fractal heap.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - -
Version 2 B-tree, Type 6 Record Layout - Creation - Order for Indexed Group
bytebytebytebyte

Creation Order (8 bytes)
-
ID (bytes 1-4)
ID (bytes 5-7)
-
- -
-
- - - - - - - - - - - - - - - - -
Field NameDescription

Creation Order

 -

This field is the creation order value for the link.

-

ID

 -

This is a 7-byte sequence of bytes and is the heap ID for the - link record in the group’s fractal heap.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - -
Version 2 B-tree, Type 7 Record Layout - Shared - Object Header Messages (Sub-Type 0 - Message in Heap)
bytebytebytebyte
Message LocationThis space inserted - only to align table nicely
Hash
Reference Count

Heap ID (8 bytes)
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Message Location

 -

This field Indicates the location where the message is - stored:

- - - - - - - - - - - - - -
ValueDescription
0Shared message is stored in shared message index heap.
1Shared message is stored in object header.
-

-

Hash

 -

This field is hash value of the shared message. The hash - value is the Jenkins’ lookup3 checksum algorithm applied to - the shared message.

-

Reference Count

 -

The number of objects which reference this message.

-

Heap ID

 -

This is an 8-byte sequence of bytes and is the heap ID for - the shared message in the shared message index’s fractal - heap.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
Version 2 B-tree, Type 7 Record Layout - Shared - Object Header Messages (Sub-Type 1 - Message in Object Header)
bytebytebytebyte
Message LocationThis space inserted - only to align table nicely
Hash
Reserved (zero)Message TypeObject Header Index

Object Header AddressO
-
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Message Location

 -

This field Indicates the location where the message is - stored:

- - - - - - - - - - - - - -
ValueDescription
0Shared message is stored in shared message index heap.
1Shared message is stored in object header.
-

-

Hash

 -

This field is hash value of the shared message. The hash - value is the Jenkins’ lookup3 checksum algorithm applied to - the shared message.

-

Message Type

 -

The object header message type of the shared message.

-

Object Header Index

 -

- This field indicates that the shared message is the nth - message of its type in the specified object header. -

-

Object Header Address

 -

The address of the object header containing the shared - message.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - -
Version 2 B-tree, Type 8 Record Layout - Attribute - Name for Indexed Attributes
bytebytebytebyte

Heap ID (8 bytes)
-
Message FlagsThis space inserted - only to align table nicely
Creation Order
Hash of Name
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Heap ID

 -

This is an 8-byte sequence of bytes and is the heap ID for - the attribute in the object’s attribute fractal heap.

-

Message Flags

The object header message flags for the attribute - message.

Creation Order

 -

This field is the creation order value for the attribute.

-

Hash

 -

This field is hash value of the name for the attribute. The - hash value is the Jenkins’ lookup3 checksum algorithm applied - to the attribute’s name.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - -
Version 2 B-tree, Type 9 Record Layout- Creation - Order for Indexed Attributes
bytebytebytebyte

Heap ID (8 bytes)
-
Message FlagsThis space inserted - only to align table nicely
Creation Order
-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Heap ID

 -

This is an 8-byte sequence of bytes and is the heap ID for - the attribute in the object’s attribute fractal heap.

-

Message Flags

 -

The object header message flags for the attribute message.

-

Creation Order

 -

This field is the creation order value for the attribute.

-
-
- - -
-

- III.B. Disk Format: Level 1B - Group Symbol - Table Nodes -

- -

A group is an object internal to the file that allows arbitrary - nesting of objects within the file (including other groups). A group - maps a set of link names in the group to a set of relative file - addresses of objects in the file. Certain metadata for an object to - which the group points can be cached in the group’s symbol table - entry in addition to being in the object’s header.

- -

An HDF5 object name space can be stored hierarchically by - partitioning the name into components and storing each component as a - link in a group. The link for a non-ultimate component points to the - group containing the next component. The link for the last component - points to the object being named.

- -

- One implementation of a group is a collection of symbol table nodes - indexed by a B-link tree. Each symbol table node contains entries for - one or more links. If an attempt is made to add a link to an already - full symbol table node containing 2K entries, then the node is - split and one node contains K symbols and the other contains K+1 - symbols. -

- -
- - - - - - - - - - - - - - - - - - - - - - - -
Symbol Table Node (A Leaf of a B-link tree)
bytebytebytebyte
Signature
Version NumberReserved (zero)Number of Symbols

-
Group Entries
-
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - SNOD - ” is used to indicate the beginning of a symbol table node. - This gives file consistency checking utilities a better chance of - reconstructing a damaged file. -

-

Version Number

 -

The version number for the symbol table node. This document - describes version 1. (There is no version ‘0’ of the - symbol table node)

-

Number of Entries

 -

Although all symbol table nodes have the same length, most - contain fewer than the maximum possible number of link entries. - This field indicates how many entries contain valid data. The valid - entries are packed at the beginning of the symbol table node while - the remaining entries contain undefined values.

-

Symbol Table Entries

 -

- Each link has an entry in the symbol table node. The format of the - entry is described below. There are 2K entries in each - group node, where K is the “Group Leaf Node K” - value from the superblock. -

-
-
- -
-

- III.C. Disk Format: Level 1C - Symbol - Table Entry -

- -

Each symbol table entry in a symbol table node is designed to - allow for very fast browsing of stored objects. Toward that design - goal, the symbol table entries include space for caching certain - constant metadata from the object header.

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Symbol Table Entry
bytebytebytebyte

Link Name OffsetO
-

Object Header AddressO
-
Cache Type
Reserved (zero)

-
Scratch-pad Space (16 bytes)
-
-
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Link Name Offset

 -

This is the byte offset into the group’s local heap for - the name of the link. The name is null terminated.

-

Object Header Address

 -

Every object has an object header which serves as a permanent - location for the object’s metadata. In addition to appearing - in the object header, some of the object’s metadata can be - cached in the scratch-pad space.

-

Cache Type

 -

The cache type is determined from the object header. It also - determines the format for the scratch-pad space:

- - - - - - - - - - - - - - - - - -
TypeDescription
0No data is cached by the group entry. This is guaranteed - to be the case when an object header has a link count greater - than one.
1Group object header metadata is cached in the scratch-pad - space. This implies that the symbol table entry refers to another - group.
2The entry is a symbolic link. The first four bytes of the - scratch-pad space are the offset into the local heap for the link - value. The object header address will be undefined.
-

- -

Reserved

 -

These four bytes are present so that the scratch-pad space is - aligned on an eight-byte boundary. They are always set to zero.

-

Scratch-pad Space

 -

This space is used for different purposes, depending on the - value of the Cache Type field. Any metadata about an object - represented in the scratch-pad space is duplicated in the object - header for that object.

-

Furthermore, no data is cached in the group entry scratch-pad - space if the object header for the object has a link count greater - than one.

-
-
- -
-

Format of the Scratch-pad Space

- -

The symbol table entry scratch-pad space is formatted according - to the value in the Cache Type field.

- -

- If the Cache Type field contains the value zero - (0) - then no information is stored in the scratch-pad space. -

- -

- If the Cache Type field contains the value one - (1) - , then the scratch-pad space contains cached metadata for another - object header in the following format: -

- -
- - - - - - - - - - - - - - - - - -
Object Header Scratch-pad Format
bytebytebytebyte

Address of B-treeO
-

Address of Name HeapO
-
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription

Address of B-tree

 -

This is the file address for the root of the group’s - B-tree.

-

Address of Name Heap

 -

This is the file address for the group’s local heap, in - which are stored the group’s symbol names.

-
-
- - -
-

- If the Cache Type field contains the value two - (2) - , then the scratch-pad space contains cached metadata for a symbolic - link in the following format: -

- -
- - - - - - - - - - - - - -
Symbolic Link Scratch-pad Format
bytebytebytebyte
Offset to Link Value
-
- -
-
- - - - - - - - - - -
Field NameDescription

Offset to Link Value

 -

The value of a symbolic link (that is, the name of the thing - to which it points) is stored in the local heap. This field is the - 4-byte offset into the local heap for the start of the link value, - which is null terminated.

-
-
- -
-

- III.D. Disk Format: Level 1D - Local Heaps -

- -

A local heap is a collection of small pieces of data that are - particular to a single object in the HDF5 file. Objects can be inserted - and removed from the heap at any time. The address of a heap does not - change once the heap is created. For example, a group stores addresses - of objects in symbol table nodes with the names of links stored in the - group’s local heap.

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Local Heap
bytebytebytebyte
Signature
VersionReserved (zero)

Data Segment SizeL
-

Offset to Head of Free-listL
-

Address of Data SegmentO
-
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
 (Items marked with an ‘L’ in the above table are - of the size specified in “Size of Lengths” field in the - superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - HEAP - ” is used to indicate the beginning of a heap. This gives - file consistency checking utilities a better chance of - reconstructing a damaged file. -

-

Version

 -

Each local heap has its own version number so that new heaps - can be added to old files. This document describes version zero (0) - of the local heap.

-

Data Segment Size

 -

The total amount of disk memory allocated for the heap data. - This may be larger than the amount of space required by the objects - stored in the heap. The extra unused space in the heap holds a - linked list of free blocks.

-

Offset to Head of Free-list

 -

- This is the offset within the heap data segment of the first free - block (or the undefined address if - there is no free block). The free block contains “Size of - Lengths” bytes that are the offset of the next free block (or - the value ‘1’ if this is the last free block) followed - by “Size of Lengths” bytes that store the size of this - free block. The size of the free block includes the space used to - store the offset of the next free block and the size of the current - block, making the minimum size of a free block 2 * “Size of - Lengths”. -

-

Address of Data Segment

 -

The data segment originally starts immediately after the heap - header, but if the data segment must grow as a result of adding - more objects, then the data segment may be relocated, in its - entirety, to another part of the file.

-
-
- -

Objects within a local heap should be aligned on an 8-byte - boundary.

- -
-

- III.E. Disk Format: Level 1E - Global Heap -

- -

Each HDF5 file has a global heap which stores various types of - information which is typically shared between datasets. The global heap - was designed to satisfy these goals:

- -
    -
  1. Repeated access to a heap object must be efficient without - resulting in repeated file I/O requests. Since global heap objects - will typically be shared among several datasets, it is probable that - the object will be accessed repeatedly.
    2. -
  2. Collections of related global heap objects should result in - fewer and larger I/O requests. For instance, a dataset of object - references will have a global heap object for each reference. Reading - the entire set of object references should result in a few large I/O - requests instead of one small I/O request for each reference.
    3. -
  3. It should be possible to remove objects from the global heap - and the resulting file hole should be eligible to be reclaimed for - other uses.
    4. -
- - -

- The implementation of the heap makes use of the memory management - already available at the file level and combines that with a new object - called a collection to achieve goal B. The global heap is the - set of all collections. Each global heap object belongs to exactly one - collection and each collection contains one or more global heap - objects. For the purposes of disk I/O and caching, a collection is - treated as an atomic object, addressing goal A. -

- -

When a global heap object is deleted from a collection (which - occurs when its reference count falls to zero), objects located after - the deleted object in the collection are packed down toward the - beginning of the collection and the collection’s global heap - object 0 is created (if possible) or its size is increased to account - for the recently freed space. There are no gaps between objects in each - collection, with the possible exception of the final space in the - collection, if it is not large enough to hold the header for the - collection’s global heap object 0. These features address goal C. -

- -

The HDF5 Library creates global heap collections as needed, so - there may be multiple collections throughout the file. The set of all - of them is abstractly called the “global heap”, although - they do not actually link to each other, and there is no global place - in the file where you can discover all of the collections. The - collections are found simply by finding a reference to one through - another object in the file. For example, data of variable-length - datatype elements is stored in the global heap and is accessed via a - global heap ID. The format for global heap IDs is described at the end - of this section.

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
A Global Heap Collection
bytebytebytebyte
Signature
VersionReserved (zero)

Collection SizeL
-

Global Heap Object 1
-

Global Heap Object 2
-

...
-

Global Heap Object N
-

Global Heap Object 0 (free space)
-
- - - - - - -
 (Items marked with an ‘L’ in the - above table are of the size specified in “Size of - Lengths” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - GCOL - ” is used to indicate the beginning of a collection. This - gives file consistency checking utilities a better chance of - reconstructing a damaged file. -

-

Version

 -

Each collection has its own version number so that new - collections can be added to old files. This document describes - version one (1) of the collections (there is no version zero (0)). -

-

Collection Size

 -

This is the size in bytes of the entire collection including - this field. The default (and minimum) collection size is 4096 bytes - which is a typical file system block size. This allows for 127 - 16-byte heap objects plus their overhead (the collection header of - 16 bytes and the 16 bytes of information about each heap object).

-

- Global Heap Object 1 through N -

-

The objects are stored in any order with no intervening - unused space.

-

Global Heap Object 0

 -

Global Heap Object 0 (zero), when present, represents the - free space in the collection. Free space always appears at the end - of the collection. If the free space is too small to store the - header for Object 0 (described below) then the header is implied - and the collection contains no free space.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
Global Heap Object
bytebytebytebyte
Heap Object IndexReference Count
Reserved (zero)

Object SizeL
-

Object Data
-
- - - - - - -
 (Items marked with an ‘L’ in the - above table are of the size specified in “Size of - Lengths” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Heap Object Index

 -

- Each object has a unique identification number within a collection. - The identification numbers are chosen so that new objects have the - smallest value possible with the exception that the identifier - 0 - always refers to the object which represents all free space within - the collection. -

-

Reference Count

 -

All heap objects have a reference count field. An object - which is referenced from some other part of the file will have a - positive reference count. The reference count for Object 0 is - always zero.

-

Reserved

 -

Zero padding to align next field on an 8-byte boundary.

-

Object Size

 -

This is the size of the object data stored for the object. - The actual storage space allocated for the object data is rounded - up to a multiple of eight.

-

Object Data

 -

The object data is treated as a one-dimensional array of - bytes to be interpreted by the caller.

-
- -
- -
-

The format for the ID used to locate an object in the global heap - is described here:

- -
- - - - - - - - - - - - - - - - - -
Global Heap ID
bytebytebytebyte

Collection AddressO
-
Object Index
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - -
Field NameDescription

Collection Address

 -

This field is the address of the global heap collection where - the data object is stored.

-

ID

 -

This field is the index of the data object within the global - heap collection.

-
-
- - -
-

- III.F. Disk Format: Level 1F - Fractal Heap -

- -

- Each fractal heap consists of a header and zero or more direct and - indirect blocks (described below). The header contains general - information as well as initialization parameters for the doubling - table. The Root Block Address in the header points to the - first direct or indirect block in the heap. -

- -

- Fractal heaps are based on a data structure called a doubling - table. A doubling table provides a mechanism for quickly extending an - array-like data structure that minimizes the number of empty blocks in - the heap, while retaining very fast lookup of any element within the - array. More information on fractal heaps and doubling tables can be - found in the RFC “Private Heaps in - HDF5.” -

- -

The fractal heap implements the doubling table structure with - indirect and direct blocks. Indirect blocks in the heap do not actually - contain data for objects in the heap, their “size” is - abstract - they represent the indexing structure for locating the - direct blocks in the doubling table. Direct blocks contain the actual - data for objects stored in the heap.

- -

- All indirect blocks have a constant number of block entries in each - row, called the width of the doubling table (stored in the - heap header). The number of rows for each indirect block in the heap is - determined by the size of the block that the indirect block represents - in the doubling table (calculation of this is shown below) and is - constant, except for the “root” indirect block, which - expands and shrinks its number of rows as needed. -

- -

- Blocks in the first two rows of an indirect block are Starting - Block Size number of bytes in size, and the blocks in each subsequent - row are twice the size of the blocks in the previous row. In other - words, blocks in the third row are twice the Starting Block - Size, blocks in the fourth row are four times the Starting - Block Size, and so on. Entries for blocks up to the Maximum - Direct Block Size point to direct blocks, and entries for blocks - greater than that size point to further indirect blocks (which have - their own entries for direct and indirect blocks). -

- -

- The number of rows of blocks, nrows, in an indirect block of - size iblock_size is given by the following expression:
-
nrows = (log2(iblock_size) - log2(<Starting - Block Size> * <Width>)) + 1 -

- -

- The maximum number of rows of direct blocks, max_dblock_rows, - in any indirect block of a fractal heap is given by the following - expression:

max_dblock_rows = (log2(<Max. - Direct Block Size>) - log2(<Starting Block - Size>)) + 2 -

- -

- Using the computed values for nrows and max_dblock_rows, - along with the Width of the doubling table, the number of - direct and indirect block entries (K and N in the - indirect block description, below) in an indirect block can be - computed:

K = MIN(nrows, max_dblock_rows) - * Width

If nrows is less than or - equal to max_dblock_rows, N is 0. Otherwise, N - is simply computed:

N = K - (max_dblock_rows - * Width) -

- -

The size indirect blocks on disk is determined by the number of - rows in the indirect block (computed above). The size of direct blocks - on disk is exactly the size of the block in the doubling table.

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Fractal Heap Header
bytebytebytebyte
Signature
VersionThis space inserted - only to align table nicely
Heap ID LengthI/O Filters’ Encoded Length
FlagsThis space inserted - only to align table nicely
Maximum Size of Managed Objects

Next Huge Object IDL
-

v2 B-tree Address of Huge ObjectsO
-

Amount of Free Space in Managed BlocksL
-

Address of Managed Block Free Space - ManagerO
-

Amount of Managed Space in HeapL
-

Amount of Allocated Managed Space in HeapL
-

Offset of Direct Block Allocation - Iterator in Managed SpaceL
-

Number of Managed Objects in HeapL
-

Size of Huge Objects in HeapL
-

Number of Huge Objects in HeapL
-

Size of Tiny Objects in HeapL
-

Number of Tiny Objects in HeapL
-
Table WidthThis space inserted - only to align table nicely

Starting Block SizeL
-

Maximum Direct Block SizeL
-
Maximum Heap SizeStarting # of Rows in Root Indirect Block

Address of Root BlockO
-
Current # of Rows in Root Indirect BlockThis space inserted - only to align table nicely

Size of Filtered Root Direct Block (optional)L
-
I/O Filter Mask (optional)
I/O Filter Information (optional, - variable size)
Checksum
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
 (Items marked with an ‘L’ in the above table are - of the size specified in “Size of Lengths” field in the - superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - FRHP - ” is used to indicate the beginning of a fractal heap header. - This gives file consistency checking utilities a better chance of - reconstructing a damaged file. -

-

Version

 -

This document describes version 0.

-

Heap ID Length

 -

This is the length in bytes of heap object IDs for this heap.

-

I/O Filters’ Encoded Length

 -

- This is the size in bytes of the encoded I/O Filter - Information. -

-

Flags

 -

This field is the heap status flag and is a bit field - indicating additional information about the fractal heap.

- - - - - - - - - - - - - - - - - - -
Bit(s)Description
0If set, the ID value to use for huge object has wrapped - around. If the value for the Next Huge Object ID has - wrapped around, each new huge object inserted into the heap will - require a search for an ID value. -
1If set, the direct blocks in the heap are checksummed.
2-7Reserved
-

- -

Maximum Size of Managed Objects

 -

This is the maximum size of managed objects allowed in the - heap. Objects greater than this this are ‘huge’ objects - and will be stored in the file directly, rather than in a direct - block for the heap.

-

Next Huge Object ID

 -

This is the next ID value to use for a huge object in the - heap.

-

v2 B-tree Address of Huge Objects

 -

- This is the address of the v2 B-tree used - to track huge objects in the heap. The type of records stored in - the v2 B-tree will be determined by whether the address & - length of a huge object can fit into a heap ID (if yes, it is a - “directly” accessed huge object) and whether there is a - filter used on objects in the heap. -

-

Amount of Free Space in Managed Blocks

 -

This is the total amount of free space in managed direct - blocks (in bytes).

-

Address of Managed Block Free Space Manager

 -

- This is the address of the Free-space - Manager for managed blocks. -

-

Amount of Managed Space in Heap

 -

This is the total amount of managed space in the heap (in - bytes), essentially the upper bound of the heap’s linear - address space.

-

Amount of Allocated Managed Space in Heap

 -

- This is the total amount of managed space (in bytes) actually - allocated in the heap. This can be less than the Amount of - Managed Space in Heap field, if some direct blocks in the - heap’s linear address space are not allocated. -

-

Offset of Direct Block Allocation Iterator in Managed - Space

 -

- This is the linear heap offset where the next direct block should - be allocated at (in bytes). This may be less than the Amount - of Managed Space in Heap value because the heap’s address - space is increased by a “row” of direct blocks at a - time, rather than by single direct block increments. -

-

Number of Managed Objects in Heap

 -

This is the number of managed objects in the heap.

-

Size of Huge Objects in Heap

 -

This is the total size of huge objects in the heap (in - bytes).

-

Number of Huge Objects in Heap

 -

This is the number of huge objects in the heap.

-

Size of Tiny Objects in Heap

 -

This is the total size of tiny objects that are packed in - heap IDs (in bytes).

-

Number of Tiny Objects in Heap

 -

This is the number of tiny objects that are packed in heap - IDs.

-

Table Width

 -

This is the number of columns in the doubling table for - managed blocks. This value must be a power of two.

-

Starting Block Size

 -

This is the starting block size to use in the doubling table - for managed blocks (in bytes). This value must be a power of two.

-

Maximum Direct Block Size

 -

This is the maximum size allowed for a managed direct block. - Objects inserted into the heap that are larger than this value - (less the # of bytes of direct block prefix/suffix) are stored as - ‘huge’ objects. This value must be a power of two.

-

Maximum Heap Size

 -

This is the maximum size of the heap’s linear address - space for managed objects (in bytes). The value stored is the log2 - of the actual value, that is: the # of bits of the address space. - ‘Huge’ and ‘tiny’ objects are not counted - in this value, since they do not store objects in the linear - address space of the heap.

-

Starting # of Rows in Root Indirect Block

 -

- This is the starting number of rows for the root indirect block. A - value of 0 indicates that the root indirect block will have the - maximum number of rows needed to address the heap’s Maximum - Heap Size. -

-

Address of Root Block

 -

- This is the address of the root block for the heap. It can be the undefined address if there is no data - in the heap. It either points to a direct block (if the Current - # of Rows in the Root Indirect Block value is 0), or an indirect - block. -

-

Current # of Rows in Root Indirect Block

 -

- This is the current number of rows in the root indirect block. A - value of 0 indicates that Address of Root Block points to - direct block instead of indirect block. -

-

Size of Filtered Root Direct Block

 -

- This is the size of the root direct block, if filters are applied - to heap objects (in bytes). This field is only stored in the header - if the I/O Filters’ Encoded Length is greater than - 0. -

-

I/O Filter Mask

 -

- This is the filter mask for the root direct block, if filters are - applied to heap objects. This mask has the same format as that used - for the filter mask in chunked raw data records in a v1 B-tree. This field is only stored in the - header if the I/O Filters’ Encoded Length is greater - than 0. -

-

I/O Filter Information

 -

- This is the I/O filter information encoding direct blocks and huge - objects, if filters are applied to heap objects. This field is - encoded as a Filter Pipeline message. - The size of this field is determined by I/O Filters’ - Encoded Length. -

-

Checksum

 -

This is the checksum for the header.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Fractal Heap Direct Block
bytebytebytebyte
Signature
VersionThis space inserted - only to align table nicely

Heap Header AddressO
-
Block Offset (variable size)
Checksum (optional)

Object Data (variable size)
-
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - FHDB - ” is used to indicate the beginning of a fractal heap direct - block. This gives file consistency checking utilities a better - chance of reconstructing a damaged file. -

-

Version

 -

This document describes version 0.

-

Heap Header Address

 -

This is the address for the fractal heap header that this - block belongs to. This field is principally used for file integrity - checking.

-

Block Offset

 -

- This is the offset of the block within the fractal heap’s - address space (in bytes). The number of bytes used to encode this - field is the Maximum Heap Size (in the heap’s - header) divided by 8 and rounded up to the next highest integer, - for values that are not a multiple of 8. This value is principally - used for file integrity checking. -

-

Checksum

 -

This is the checksum for the direct block.

-

- This field is only present if bit 1 of Flags in the - heap’s header is set. -

-

Object Data

 -

- This section of the direct block stores the actual data for objects - in the heap. The size of this section is determined by the direct - block’s size minus the size of the other fields stored in the - direct block (for example, the Signature, Version, - and others including the Checksum if it is present). -

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Fractal Heap Indirect Block
bytebytebytebyte
Signature
VersionThis space inserted - only to align table nicely

Heap Header AddressO
-
Block Offset (variable size)

Child Direct Block #0 AddressO
-

Size of Filtered Direct Block #0 (optional) - L
-
Filter Mask for Direct Block #0 (optional)

Child Direct Block #1 AddressO
-

Size of Filtered Direct Block #1 (optional)L
-
Filter Mask for Direct Block #1 (optional)
...

Child Direct Block #K-1 AddressO
-

Size of Filtered Direct Block #K-1 (optional)L
-
Filter Mask for Direct Block #K-1 (optional)

Child Indirect Block #0 AddressO
-

Child Indirect Block #1 AddressO
-
...

Child Indirect Block #N-1 AddressO
-
Checksum
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
 (Items marked with an ‘L’ in the above table are - of the size specified in “Size of Lengths” field in the - superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - FHIB - ” is used to indicate the beginning of a fractal heap - indirect block. This gives file consistency checking utilities a - better chance of reconstructing a damaged file. -

-

Version

 -

This document describes version 0.

-

Heap Header Address

 -

This is the address for the fractal heap header that this - block belongs to. This field is principally used for file integrity - checking.

-

Block Offset

 -

- This is the offset of the block within the fractal heap’s - address space (in bytes). The number of bytes used to encode this - field is the Maximum Heap Size (in the heap’s - header) divided by 8 and rounded up to the next highest integer, - for values that are not a multiple of 8. This value is principally - used for file integrity checking. -

-

Child Direct Block #K Address

 -

This field is the address of the child direct block. The size - of the [uncompressed] direct block can be computed by its offset in - the heap’s linear address space.

-

Size of Filtered Direct Block #K

 -

This is the size of the child direct block after passing - through the I/O filters defined for this heap (in bytes). If no I/O - filters are present for this heap, this field is not present.

-

Filter Mask for Direct Block #K

 -

- This is the I/O filter mask for the filtered direct block. This - mask has the same format as that used for the filter mask in - chunked raw data records in a v1 B-tree. If - no I/O filters are present for this heap, this field is not - present. -

-

Child Indirect Block #N Address

 -

This field is the address of the child indirect block. The - size of the indirect block can be computed by its offset in the - heap’s linear address space.

-

Checksum

 -

This is the checksum for the indirect block.

-
- -
- -
-

An object in the fractal heap is identified by means of a fractal - heap ID, which encodes information to locate the object in the heap. - Currently, the fractal heap stores an object in one of three ways, - depending on the object’s size:

- -
- - - - - - - - - - - - - - - - - - - - -
TypeDescription
Tiny -

When an object is small enough to be encoded in the heap ID, - the object’s data is embedded in the fractal heap ID itself. - There are 2 sub-types for this type of object: normal and extended. - The sub-type for tiny heap IDs depends on whether the heap ID is - large enough to store objects greater than 16 bytes or not. If the - heap ID length is 18 bytes or smaller, the ‘normal’ - tiny heap ID form is used. If the heap ID length is greater than 18 - bytes in length, the “extended” form is used. See - format description below for both sub-types.

-
Huge -

- When the size of an object is larger than Maximum Size of - Managed Objects in the Fractal Heap Header, the - object’s data is stored on its own in the file and the object - is tracked/indexed via a version 2 B-tree. All huge objects for a - particular fractal heap use the same v2 B-tree. All huge objects - for a particular fractal heap use the same format for their huge - object IDs. -

- -

Depending on whether the IDs for a heap are large enough to - hold the object’s retrieval information and whether I/O - pipeline filters are applied to the heap’s objects, 4 - sub-types are derived for huge object IDs for this heap:

- -
- - - - - - - - - - - - - - - - - - - - - - - - - -
Sub-typeDescription
Directly accessed, non-filtered -

The object’s address and length are embedded in the - fractal heap ID itself and the object is directly accessed from - them. This allows the object to be accessed without resorting - to the B-tree.

-
Directly accessed, filtered -

The filtered object’s address, length, filter mask - and de-filtered size are embedded in the fractal heap ID itself - and the object is accessed directly with them. This allows the - object to be accessed without resorting to the B-tree.

-
Indirectly accessed, non-filtered -

The object is located by using a B-tree key embedded in - the fractal heap ID to retrieve the address and length from the - version 2 B-tree for huge objects. Then, the address and length - are used to access the object.

-
Indirectly accessed, filtered -

The object is located by using a B-tree key embedded in - the fractal heap ID to retrieve the filtered object’s - address, length, filter mask and de-filtered size from the - version 2 B-tree for huge objects. Then, this information is - used to access the object.

-
-
- -
Managed -

When the size of an object does not meet the above two - conditions, the object is stored and managed via the direct and - indirect blocks based on the doubling table.

-
-
- - -

The specific format for each type of heap ID is described below: -

- -
- - - - - - - - - - - - - - - - - - - -
Fractal Heap ID for Tiny Objects (sub-type 1 - - ‘Normal’)
bytebytebytebyte
Version, Type & LengthThis space inserted - only to align table nicely

Data (variable size)
-
- -
-
- - - - - - - - - - - - - - - - -
Field NameDescription

Version, Type & Length

 -

This is a bit field with the following definition:

- - - - - - - - - - - - - - - - - - -
BitDescription
6-7The current version of ID format. This document describes - version 0.
4-5The ID type. Tiny objects have a value of 2. -
0-3The length of the tiny object. The value stored is one - less than the actual length (since zero-length objects are not - allowed to be stored in the heap). For example, an object of - actual length 1 has an encoded length of 0, an object of actual - length 2 has an encoded length of 1, and so on.
-

- -

Data

 -

This is the data for the object.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - -
Fractal Heap ID for Tiny Objects (sub-type 2 - - ‘Extended’)
bytebytebytebyte
Version, Type & LengthExtended LengthThis space inserted - only to align table nicely
Data (variable size)
-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version, Type & Length

 -

This is a bit field with the following definition:

- - - - - - - - - - - - - - - - - - -
BitDescription
6-7The current version of ID format. This document describes - version 0.
4-5The ID type. Tiny objects have a value of 2. -
0-3These 4 bits, together with the next byte, form an - unsigned 12-bit integer for holding the length of the object. - These 4-bits are bits 8-11 of the 12-bit integer. See description - for the Extended Length field below. -
-

- -

Extended Length

 -

This byte, together with the 4 bits in the previous byte, - forms an unsigned 12-bit integer for holding the length of the tiny - object. These 8 bits are bits 0-7 of the 12-bit integer formed. The - value stored is one less than the actual length (since zero-length - objects are not allowed to be stored in the heap). For example, an - object of actual length 1 has an encoded length of 0, an object of - actual length 2 has an encoded length of 1, and so on.

-

Data

 -

This is the data for the object.

-
-
- - -
-
-
- - - - - - - - - - - - - - - - - - - -
Fractal Heap ID for Huge Objects (sub-type 1 & 2): - indirectly accessed, non-filtered/filtered
bytebytebytebyte
Version & TypeThis space inserted - only to align table nicely

v2 B-tree KeyL - (variable size)
-
- - - - - - -
 (Items marked with an ‘L’ in the - above table are of the size specified in “Size of - Lengths” field in the superblock.)
-
- -
-
- - - - - - - - - - - - - - - - -
Field NameDescription

Version & Type

 -

This is a bit field with the following definition:

- - - - - - - - - - - - - - - - - - -
BitDescription
6-7The current version of ID format. This document describes - version 0.
4-5The ID type. Huge objects have a value of 1. -
0-3Reserved.
-

- -

v2 B-tree Key

- This field is the B-tree key for retrieving the information from - the version 2 B-tree for huge objects needed to access the object. - See the description of v2 B-tree records - sub-type 1 & 2 for a description of the fields. New key values are - derived from Next Huge Object ID in the Fractal - Heap Header. -

-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - -
Fractal Heap ID for Huge Objects (sub-type 3): - directly accessed, non-filtered
bytebytebytebyte
Version & TypeThis space inserted - only to align table nicely

Address O
-

Length L
-
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
 (Items marked with an ‘L’ in the above table are - of the size specified in “Size of Lengths” field in the - superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version & Type

 -

This is a bit field with the following definition:

- - - - - - - - - - - - - - - - - - -
BitDescription
6-7The current version of ID format. This document describes - version 0.
4-5The ID type. Huge objects have a value of 1. -
0-3Reserved.
-

- -

Address

This field is the address of the object in the file.

-

Length

This field is the length of the object in the file.

-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Fractal Heap ID for Huge Objects (sub-type 4): - directly accessed, filtered
bytebytebytebyte
Version & TypeThis space inserted - only to align table nicely

Address O
-

Length L
-
Filter Mask

De-filtered Size L
-
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
 (Items marked with an ‘L’ in the above table are - of the size specified in “Size of Lengths” field in the - superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version & Type

 -

This is a bit field with the following definition:

- - - - - - - - - - - - - - - - - - -
BitDescription
6-7The current version of ID format. This document describes - version 0.
4-5The ID type. Huge objects have a value of 1. -
0-3Reserved.
-

- -

Address

This field is the address of the filtered object in - the file.

Length

This field is the length of the filtered object in - the file.

Filter Mask

This field is the I/O pipeline filter mask for the - filtered object in the file.

Filtered Size

This field is the size of the de-filtered object in - the file.

-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - -
Fractal Heap ID for Managed Objects
bytebytebytebyte
Version & TypeThis space inserted - only to align table nicely
Offset (variable size)
Length (variable size)
-
- -
-
- - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version & Type

This is a bit field with the following definition:

- - - - - - - - - - - - - - - - - - -
BitDescription
6-7The current version of ID format. This document describes - version 0.
4-5The ID type. Managed objects have a value of 0. -
0-3Reserved.
-

Offset

- This field is the offset of the object in the heap. This - field’s size is the minimum number of bytes necessary to - encode the Maximum Heap Size value (from the Fractal - Heap Header). For example, if the value of the Maximum - Heap Size is less than 256 bytes, this field is 1 byte in length, - a Maximum Heap Size of 256-65535 bytes uses a 2 byte - length, and so on. -

Length

- This field is the length of the object in the heap. It is - determined by taking the minimum value of Maximum Direct - Block Size and Maximum Size of Managed Objects in the Fractal - Heap Header. Again, the minimum number of bytes needed to encode - that value is used for the size of this field. -

-
- -
-

- III.G. Disk Format: Level 1G - - Free-space Manager -

- -

Free-space managers are used to describe space within a heap or - the entire HDF5 file that is not currently used for that heap or file. -

- -

- The free-space manager header contains metadata information - about the space being tracked, along with the address of the list of free - space sections which actually describes the free space. The header - records information about free-space sections being tracked, creation - parameters for handling free-space sections of a client, and section - information used to locate the collection of free-space sections. -

- -

- The free-space section list stores a collection of free-space - sections that is specific to each client of the free-space - manager. For example, the fractal heap is a client of the free space - manager and uses it to track unused space within the heap. There are 4 - types of section records for the fractal heap, each of which has its - own format, listed below. -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Free-space Manager Header
bytebytebytebyte
Signature
VersionClient IDThis space inserted - only to align table nicely

Total Space TrackedL
-

Total Number of SectionsL
-

Number of Serialized SectionsL
-

Number of Un-Serialized SectionsL
-
Number of Section ClassesThis space inserted - only to align table nicely
Shrink PercentExpand Percent
Size of Address SpaceThis space inserted - only to align table nicely

Maximum Section Size L
-

Address of Serialized Section ListO
-

Size of Serialized Section List UsedL
-

Allocated Size of Serialized Section ListL
-
Checksum
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
 (Items marked with an ‘L’ in the above table are - of the size specified in “Size of Lengths” field in the - superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - FSHD - ” is used to indicate the beginning of the Free-space Manager - Header. This gives file consistency checking utilities a better - chance of reconstructing a damaged file. -

-

Version

 -

This is the version number for the Free-space Manager Header - and this document describes version 0.

-

Client ID

 -

This is the client ID for identifying the user of this - free-space manager:

- - - - - - - - - - - - - - - - - - -
IDDescription
0Fractal heap
1File
2+Reserved.
-

- -

Total Space Tracked

 -

This is the total amount of free space being tracked, in - bytes.

-

Total Number of Sections

 -

This is the total number of free-space sections being - tracked.

-

Number of Serialized Sections

 -

This is the number of serialized free-space sections being - tracked.

-

Number of Un-Serialized Sections

 -

This is the number of un-serialized free-space sections being - managed. Un-serialized sections are created by the free-space - client when the list of sections is read in.

-

Number of Section Classes

 -

This is the number of section classes handled by this free - space manager for the free-space client.

-

Shrink Percent

 -

This is the percent of current size to shrink the allocated - serialized free-space section list.

-

Expand Percent

 -

This is the percent of current size to expand the allocated - serialized free-space section list.

-

Size of Address Space

 -

- This is the size of the address space that free-space sections are - within. This is stored as the log2 of the actual value - (in other words, the number of bits required to store values within - that address space). -

-

Maximum Section Size

 -

This is the maximum size of a section to be tracked.

-

Address of Serialized Section List

 -

This is the address where the serialized free-space section - list is stored.

-

Size of Serialized Section List Used

 -

- This is the size of the serialized free-space section list used (in - bytes). This value must be less than or equal to the allocated - size of serialized section list, below. -

-

Allocated Size of Serialized Section List

 -

This is the size of serialized free-space section list - actually allocated (in bytes).

-

Checksum

 -

This is the checksum for the free-space manager header.

-
-
- -
-

- The free-space sections being managed are stored in a free-space - section list, described below. The sections in the free-space section - list are stored in the following way: a count of the number of sections - describing a particular size of free space and the size of the - free-space described (in bytes), followed by a list of section - description records; then another section count and size, followed by - the list of section descriptions for that size; and so on. -

- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Free-space Section List
bytebytebytebyte
Signature
VersionThis space inserted - only to align table nicely

Free-space Manager Header AddressO
-
Number of Section Records in Set #0 (variable - size)
Size of Free-space Section Described in Record - Set #0 (variable size) -
Record Set #0 Section Record #0 Offset(variable - size)
Record Set #0 Section Record #0 TypeThis space inserted - only to align table nicely
Record Set #0 Section Record #0 Data (variable - size)
...
Record Set #0 Section Record #K-1 Offset(variable - size)
Record Set #0 Section Record #K-1 TypeThis space inserted - only to align table nicely
Record Set #0 Section Record #K-1 Data (variable - size)
Number of Section Records in Set #1 (variable - size)
Size of Free-space Section Described in Record - Set #1 (variable size) -
Record Set #1 Section Record #0 Offset(variable - size)
Record Set #1 Section Record #0 TypeThis space inserted - only to align table nicely
Record Set #1 Section Record #0 Data (variable - size)
...
Record Set #1 Section Record #K-1 Offset(variable - size)
Record Set #1 Section Record #K-1 TypeThis space inserted - only to align table nicely
Record Set #1 Section Record #K-1 Data (variable - size)
...
...
Number of Section Records in Set #N-1 (variable - size)
Size of Free-space Section Described in Record - Set #N-1 (variable size) -
Record Set #N-1 Section Record #0 Offset(variable - size)
Record Set #N-1 Section Record #0 TypeThis space inserted - only to align table nicely
Record Set #N-1 Section Record #0 Data (variable - size)
...
Record Set #N-1 Section Record #K-1 Offset(variable - size)
Record Set #N-1 Section Record #K-1 TypeThis space inserted - only to align table nicely
Record Set #N-1 Section Record #K-1 Data (variable - size)
Checksum
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - FSSE - ” is used to indicate the beginning of the Free-space Section - Information. This gives file consistency checking utilities a - better chance of reconstructing a damaged file. -

-

Version

 -

This is the version number for the Free-space Section List - and this document describes version 0.

-

Free-space Manager Header Address

 -

- This is the address of the Free-space Manager Header. This - field is principally used for file integrity checking. -

-

Number of Section Records for Set #N

 -

- This is the number of free-space section records for set #N. The - length of this field is the minimum number of bytes needed to store - the number of serialized sections (from the free-space - manager header). -

- -

- The number of sets of free-space section records is determined by - the size of serialized section list in the free-space - manager header. -

-

Section Size for Record Set #N

 -

- This is the size (in bytes) of the free-space section described for - all the section records in set #N. -

- -

- The length of this field is the minimum number of bytes needed to - store the maximum section size (from the free-space - manager header). -

-

Record Set #N Section #K Offset

 -

This is the offset (in bytes) of the free-space section - within the client for the free-space manager.

- -

- The length of this field is the minimum number of bytes needed to - store the size of address space (from the free-space - manager header). -

-

Record Set #N Section #K Type

 -

- This is the type of the section record, used to decode the record - set #N section #K data information. The defined record type for file - client is: - -

- - - - - - - - - - - - - - -
TypeDescription
0File’s section (a range of actual bytes in file)
1+Reserved.
-

- -

- The defined record types for a fractal heap client are: - -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
TypeDescription
0Fractal heap “single” section
1Fractal heap “first row” section
2Fractal heap “normal row” section
3Fractal heap “indirect” section
4+Reserved.
-

- -

Record Set #N Section #K Data

 -

This is the section-type specific information for each record - in the record set, described below.

-

Checksum

 -

- This is the checksum for the Free-space Section List. -

-
-
- -
-

The section-type specific data for each free-space section record - is described below:

- -
- - - - - - -
File’s Section Data Record
No additional record data stored
-
- -
-
-
- - - - - - -
Fractal Heap “Single” Section Data - Record
No additional record data stored
-
- -
-
-
- - - - - - -
Fractal Heap “First Row” Section Data - Record
Same format as “indirect” - section data
-
- -
-
-
- - - - - - -
Fractal Heap “Normal Row” Section Data - Record
No additional record data stored
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - -
Fractal Heap “Indirect” Section Data - Record
bytebytebytebyte
Fractal Heap Indirect Block Offset (variable - size)
Block Start RowBlock Start Column
Number of BlocksThis space inserted - only to align table nicely
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Fractal Heap Block Offset

 -

The offset of the indirect block in the fractal heap’s - address space containing the empty blocks.

-

- The number of bytes used to encode this field is the minimum number - of bytes needed to encode values for the Maximum Heap Size - (in the fractal heap’s header). -

-

Block Start Row

 -

This is the row that the empty blocks start in.

-

Block Start Column

 -

This is the column that the empty blocks start in.

-

Number of Blocks

 -

This is the number of empty blocks covered by the section.

-
-
- -
-

- III.H. Disk Format: Level 1H - Shared Object - Header Message Table -

- -

- The shared object header message table is used to locate - object header messages that are shared between two or more object - headers in the file. Shared object header messages are stored and - indexed in the file in one of two ways: indexed sequentially in a shared - header message list or indexed with a v2 B-tree. The shared messages - themselves are either stored in a fractal heap (when two or more - objects share the message), or remain in an object’s header (when - only one object uses the message currently, but the message can be - shared in the future). -

- -

- The shared object header message table contains a list of - shared message index headers. Each index header records information - about the version of the index format, the index storage type, flags - for the message types indexed, the number of messages in the index, the - address where the index resides, and the fractal heap address if shared - messages are stored there. -

- -

- Each index can be either a list or a v2 B-tree and may transition - between those two forms as the number of messages in the index varies. - Each shared message record contains information used to locate the - shared message from either a fractal heap or an object header. The - types of messages that can be shared are: Dataspace, Datatype, - Fill Value, Filter Pipeline and Attribute. -

- -

- The shared object header message table is pointed to from a shared message table message in the - superblock extension for a file. This message stores the version of the - table format, along with the number of index headers in the table. -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Shared Object Header Message Table
bytebytebytebyte
Signature
Version for index #0Index Type for index #0Message Type Flags for index #0
Minimum Message Size for index #0
List Cutoff for index #0v2 B-tree Cutoff for index #0
Number of Messages for index #0This space inserted - only to align table nicely

Index AddressO for index #0
-

Fractal Heap AddressO for - index #0
-
...
...
Version for index #N-1Index Type for index #N-1Message Type Flags for index #N-1
Minimum Message Size for index #N-1
List Cutoff for index #N-1v2 B-tree Cutoff for index #N-1
Number of Messages for index #N-1This space inserted - only to align table nicely

Index AddressO for index #N-1
-

Fractal Heap AddressO for - index #N-1
-
Checksum
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - SMTB - ” is used to indicate the beginning of the Shared Object - Header Message table. This gives file consistency checking - utilities a better chance of reconstructing a damaged file. -

-

Version for index #N

 -

This is the version number for the list of shared object - header message indexes and this document describes version 0.

-

Index Type for index #N

 -

The type of index can be an unsorted list or a v2 B-tree.

-

Message Type Flags for index #N

 -

This field indicates the type of messages tracked in the - index, as follows:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
BitsDescription
0If set, the index tracks Dataspace Messages. -
1If set, the message tracks Datatype Messages. -
2If set, the message tracks Fill Value Messages. -
3If set, the message tracks Filter Pipeline - Messages. -
4If set, the message tracks Attribute Messages. -
5-15Reserved (zero).
-

- - -

An index can track more than one type of message, but each - type of message can only by in one index.

-

Minimum Message Size for index #N

 -

This is the message size sharing threshold for the index. If - the encoded size of the message is less than this value, the - message is not shared.

-

List Cutoff for index #N

 -

This is the cutoff value for the indexing of messages to - switch from a list to a v2 B-tree. If the number of messages is - greater than this value, the index should be a v2 B-tree.

-

v2 B-tree Cutoff for index #N

 -

This is the cutoff value for the indexing of messages to - switch from a v2 B-tree back to a list. If the number of messages - is less than this value, the index should be a list.

-

Number of Messages for index #N

 -

The number of shared messages being tracked for the index.

-

Index Address for index #N

 -

This field is the address of the list or v2 B-tree where the - index nodes reside.

-

Fractal Heap Address for index #N

 -

This field is the address of the fractal heap if shared - messages are stored there.

-

Checksum

 -

This is the checksum for the table.

-
-
- -
-

- Shared messages are indexed either with a shared message - record list, described below, or using a v2 B-tree (using record type - 7). The number of records in the shared message record list is - determined in the index’s entry in the shared object - header message table. -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Shared Message Record List
bytebytebytebyte
Signature
Shared Message Record #0
Shared Message Record #1
...
Shared Message Record #N-1
Checksum
-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - SMLI - ” is used to indicate the beginning of a list of index nodes. - This gives file consistency checking utilities a better chance of - reconstructing a damaged file. -

-

Shared Message Record #N

 -

- The record for locating the shared message, either in the fractal - heap for the index, or an object header (see format for index - nodes below). -

-

Checksum

 -

This is the checksum for the list.

-
-
- -
-

The record for each shared message in an index is stored in one - of the following forms:

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - -
Shared Message Record, for messages stored in a - fractal heap
bytebytebytebyte
Message LocationThis space inserted - only to align table nicely
Hash Value
Reference Count

Fractal Heap ID
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Message Location

 -

This has a value of 0 indicating that the message is stored - in the heap.

-

Hash Value

 -

This is the hash value for the message.

-

Reference Count

 -

This is the number of times the message is used in the file. -

-

Fractal Heap ID

 -

This is an 8-byte fractal heap ID for the message as stored - in the fractal heap for the index.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Shared Message Record, for messages stored in an - object header
bytebytebytebyte
Message LocationThis space inserted - only to align table nicely
Hash Value
ReservedMessage TypeCreation Index

Object Header AddressO
-
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Message Location

 -

This has a value of 1 indicating that the message is stored - in an object header.

-

Hash Value

 -

This is the hash value for the message.

-

Message Type

 -

This is the message type in the object header.

-

Creation Index

 -

This is the creation index of the message within the object - header.

-

Object Header Address

 -

This is the address of the object header where the message is - located.

-
-
- - - -
-
-
-

- IV. Disk Format: Level 2 - Data Objects -

- -

Data objects contain the “real” user-visible - information in the file. These objects compose the scientific data and - other information which are generally thought of as “data” - by the end-user. All the other information in the file is provided as a - framework for storing and accessing these data objects.

- -

A data object is composed of header and data information. The - header information contains the information needed to interpret the - data information for the object as well as additional - “metadata” or pointers to additional “metadata” - used to describe or annotate each object.

- -
-

- IV.A. Disk Format: Level 2A - Data Object - Headers -

- -

The header information of an object is designed to encompass all - of the information about an object, except for the data itself. This - information includes the dataspace, the datatype, information about how - the data is stored on disk (in external files, compressed, broken up in - blocks, and so on), as well as other information used by the library to - speed up access to the data objects or maintain a file’s - integrity. Information stored by user applications as attributes is - also stored in the object’s header. The header of each object is - not necessarily located immediately prior to the object’s data in - the file and in fact may be located in any position in the file. The - order of the messages in an object header is not significant.

- -

Object headers are composed of a prefix and a set of messages. - The prefix contains the information needed to interpret the messages - and a small amount of metadata about the object, and the messages - contain the majority of the metadata about the object.

- -
-

- IV.A.1. Disk Format: Level 2A1 - Data - Object Header Prefix -

- -
-

- IV.A.1.a. Version 1 Data Object - Header Prefix -

- -

Header messages are aligned on 8-byte boundaries for version 1 - object headers.

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Version 1 Object Header
bytebytebytebyte
VersionReserved (zero)Total Number of Header Messages
Object Reference Count
Object Header Size
Header Message Type #1Size of Header Message Data #1
Header Message #1 FlagsReserved (zero)

Header Message Data #1
-
.
.
.
Header Message Type #nSize of Header Message Data #n
Header Message #n FlagsReserved (zero)

Header Message Data #n
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

 -

This value is used to determine the format of the information - in the object header. When the format of the object header is - changed, the version number is incremented and can be used to - determine how the information in the object header is formatted. - This is version one (1) (there was no version zero (0)) of the - object header.

-

Total Number of Header Messages

 -

This value determines the total number of messages listed in - object headers for this object. This value includes the messages in - continuation messages for this object.

-

Object Reference Count

 -

This value specifies the number of “hard links” - to this object within the current file. References to the object - from external files, “soft links” in this file and - object references in this file are not tracked.

-

Object Header Size

 -

This value specifies the number of bytes of header message - data following this length field that contain object header - messages for this object header. This value does not include the - size of object header continuation blocks for this object elsewhere - in the file.

-

Header Message #n Type

 -

This value specifies the type of information included in the - following header message data. The message types for header - messages are defined in sections below.

-

Size of Header Message #n Data

 -

This value specifies the number of bytes of header message - data following the header message type and length information for - the current message. The size includes padding bytes to make the - message a multiple of eight bytes.

-

Header Message #n Flags

 -

This is a bit field with the following definition:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
BitDescription
0If set, the message data is constant. This is used for - messages like the datatype message of a dataset.
1If set, the message is shared and stored in - another location than the object header. The Header Message Data - field contains a Shared Message (described in the Data Object Header Messages - section below) and the Size of Header Message Data field contains - the size of that Shared Message. -
2If set, the message should not be shared.
3If set, the HDF5 decoder should fail to open this object - if it does not understand the message’s type and the file - is open with permissions allowing write access to the file. - (Normally, unknown messages can just be ignored by HDF5 decoders) -
4If set, the HDF5 decoder should set bit 5 of this - message’s flags (in other words, this bit field) if it does - not understand the message’s type and the object is - modified in any way. (Normally, unknown messages can just be - ignored by HDF5 decoders)
5If set, this object was modified by software that did not - understand this message. (Normally, unknown messages should just - be ignored by HDF5 decoders) (Can be used to invalidate an index - or a similar feature)
6If set, this message is shareable.
7If set, the HDF5 decoder should always fail to open this - object if it does not understand the message’s type - (whether it is open for read-only or read-write access). - (Normally, unknown messages can just be ignored by HDF5 decoders) -
-

- -

Header Message #n Data

 -

The format and length of this field is determined by the - header message type and size respectively. Some header message - types do not require any data and this information can be - eliminated by setting the length of the message to zero. The data - is padded with enough zeroes to make the size a multiple of eight. -

-
-
- -
-

- IV.A.1.b. Version 2 Data Object - Header Prefix -

- -

Note that the “total number of messages” field has - been dropped from the data object header prefix in this version. The - number of messages in the data object header is just determined by the - messages encountered in all the object header blocks.

- -

- Note also that the fields and messages in this version of data object - headers have no alignment or padding bytes inserted - they are - stored packed together. -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Version 2 Object Header
bytebytebytebyte
Signature
VersionFlagsThis space inserted - only to align table nicely
Access time (optional)
Modification Time (optional)
Change Time (optional)
Birth Time (optional)
Maximum # of compact attributes (optional)Minimum # of dense attributes (optional)
Size of Chunk #0 (variable size)This space inserted - only to align table nicely
Header Message Type #1Size of Header Message Data #1Header Message #1 Flags
Header Message #1 Creation Order (optional)This space inserted - only to align table nicely

Header Message Data #1
-
.
.
.
Header Message Type #nSize of Header Message Data #nHeader Message #n Flags
Header Message #n Creation Order (optional)This space inserted - only to align table nicely

Header Message Data #n
-
Gap (optional, variable size)
Checksum
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - OHDR - ” is used to indicate the beginning of an object header. This - gives file consistency checking utilities a better chance of - reconstructing a damaged file. -

-

Version

 -

This field has a value of 2 indicating version 2 of the - object header.

-

Flags

 -

This field is a bit field indicating additional information - about the object header.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Bit(s)Description
0-1This two bit field determines the size of the Size - of Chunk #0 field. The values are: - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0The Size of Chunk #0 field is 1 byte. -
1The Size of Chunk #0 field is 2 bytes. -
2The Size of Chunk #0 field is 4 bytes. -
3The Size of Chunk #0 field is 8 bytes. -
-

-
2If set, attribute creation order is tracked.
3If set, attribute creation order is indexed.
4If set, non-default attribute storage phase change values - are stored.
5If set, access, modification, change and birth times are - stored.
6-7Reserved
-

- -

Access Time

 -

This 32-bit value represents the number of seconds after the - UNIX epoch when the object’s raw data was last accessed (in - other words, read or written).

-

- This field is present if bit 5 of flags is set. -

-

Modification Time

 -

This 32-bit value represents the number of seconds after the - UNIX epoch when the object’s raw data was last modified (in - other words, written).

-

- This field is present if bit 5 of flags is set. -

-

Change Time

 -

This 32-bit value represents the number of seconds after the - UNIX epoch when the object’s metadata was last changed.

-

- This field is present if bit 5 of flags is set. -

-

Birth Time

 -

This 32-bit value represents the number of seconds after the - UNIX epoch when the object was created.

-

- This field is present if bit 5 of flags is set. -

-

Maximum # of compact attributes

 -

This is the maximum number of attributes to store in the - compact format before switching to the indexed format.

-

- This field is present if bit 4 of flags is set. -

-

Minimum # of dense attributes

 -

This is the minimum number of attributes to store in the - indexed format before switching to the compact format.

-

- This field is present if bit 4 of flags is set. -

-

Size of Chunk #0

 -

This unsigned value specifies the number of bytes of header - message data following this field that contain object header - information.

-

This value does not include the size of object header - continuation blocks for this object elsewhere in the file.

-

- The length of this field varies depending on bits 0 and 1 of the flags - field. -

-

Header Message #n Type

 -

Same format as version 1 of the object header, described - above.

-

Size of Header Message #n Data

 -

- This value specifies the number of bytes of header message data - following the header message type and length information for the - current message. The size of messages in this version does not - include any padding bytes. -

-

Header Message #n Flags

 -

Same format as version 1 of the object header, described - above.

-

Header Message #n Creation Order

 -

This field stores the order that a message of a given type - was created in.

-

- This field is present if bit 2 of flags is set. -

-

Header Message #n Data

 -

Same format as version 1 of the object header, described - above.

-

Gap

 -

A gap in an object header chunk is inferred by the end of the - messages for the chunk before the beginning of the chunk’s - checksum. Gaps are always smaller than the size of an object header - message prefix (message type + message size + message flags).

-

Gaps are formed when a message (typically an attribute - message) in an earlier chunk is deleted and a message from a later - chunk that does not quite fit into the free space is moved into the - earlier chunk.

-

Checksum

 -

This is the checksum for the object header chunk.

-
-
- -

The header message types and the message data associated with - them compose the critical “metadata” about each object. - Some header messages are required for each object while others are - optional. Some optional header messages may also be repeated several - times in the header itself, the requirements and number of times - allowed in the header will be noted in each header message description - below.

- - -
-

- IV.A.2. Disk Format: Level 2A2 - - Data Object Header Messages -

- -

Data object header messages are small pieces of metadata that are - stored in the data object header for each object in an HDF5 file. Data - object header messages provide the metadata required to describe an - object and its contents, as well as optional pieces of metadata that - annotate the meaning or purpose of the object.

- -

- Data object header messages are either stored directly in the data - object header for the object or are shared between multiple objects in - the file. When a message is shared, a flag in the Message - Flags indicates that the actual Message Data portion of that - message is stored in another location (such as another data object - header, or a heap in the file) and the Message Data field - contains the information needed to locate the actual information for - the message. -

- -

The format of shared message data is described here:

- -
- - - - - - - - - - - - - - - - - - - - - - - -
Shared Message (Version 1)
bytebytebytebyte
VersionTypeReserved (zero)
Reserved (zero)

AddressO
-
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number is used when there are changes in - the format of a shared object message and is described here:

- - - - - - - - - - - - - - - -
VersionDescription
0Never used.
1Used by the library before version 1.6.1.
-

Type

The type of shared message location:

- - - - - - - - - - -
ValueDescription
0Message stored in another object’s header (a committed - message). -
-

Address

The address of the object header containing the - message to be shared.

-
- -
-
-
- - - - - - - - - - - - - - - - - - - -
Shared Message (Version 2)
bytebytebytebyte
VersionTypeThis space inserted - only to align table nicely

AddressO
-
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
-
- -
-
- - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number is used when there are changes in - the format of a shared object message and is described here:

- - - - - - - - - - -
VersionDescription
2Used by the library of version 1.6.1 and after.
-

Type

The type of shared message location:

- - - - - - - - - - -
ValueDescription
0Message stored in another object’s header (a committed - message). -
-

Address

The address of the object header containing the - message to be shared.

-
- -
-
-
- - - - - - - - - - - - - - - - - - - -
Shared Message (Version 3)
bytebytebytebyte
VersionTypeThis space inserted - only to align table nicely
Location (variable size)
-
- -
-
- - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number indicates changes in the format of - shared object message and is described here:

- - - - - - - - - - -
VersionDescription
3Used by the library of version 1.8 and after. In this - version, the Type field can indicate that the message is - stored in the fractal heap. -
-

Type

The type of shared message location:

- - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Message is not shared and is not shareable.
1Message stored in file’s shared object - header message heap (a shared message). -
2Message stored in another object’s header (a committed - message). -
3Message stored is not shared, but is shareable.
-

Location

- This field contains either a Size of Offsets-bytes address - of the object header containing the message to be shared, or an - 8-byte fractal heap ID for the message in the file’s shared - object header message heap. -

-
- - -

The following is a list of currently defined header messages:

- -
-

- IV.A.2.a. The NIL Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: NIL
Header Message Type: 0x0000
Length: Varies
Status: Optional; may be repeated.
Description:The NIL message is used to indicate a message which is to be - ignored when reading the header messages for a data object. - [Possibly one which has been deleted for some reason.]
Format of Data: Unspecified
-
- - - -
-

- IV.A.2.b. The Dataspace Message -

- - -
- - - - - - - - - - - - - - - - - - - - - -
Header Message Name: Dataspace
Header Message Type: 0x0001
Length: Varies according to the number of - dimensions, as described in the following table.
Status: Required for dataset objects; may - not be repeated.
Description:The dataspace message describes the number of dimensions (in - other words, “rank”) and size of each dimension that the - data object has. This message is only used for datasets which have a - simple, rectilinear, array-like layout; datasets requiring a more - complex layout are not yet supported.
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Dataspace Message - Version 1
bytebytebytebyte
VersionDimensionalityFlagsReserved
Reserved

Dimension #1 SizeL
-
.
.
.

Dimension #n SizeL
-

Dimension #1 Maximum SizeL (optional)
-
.
.
.

Dimension #n Maximum SizeL (optional)
-

Permutation Index #1L (optional)
-
.
.
.

Permutation Index #nL (optional)
-
- - - - - - -
 (Items marked with an ‘L’ in the - above table are of the size specified in “Size of - Lengths” field in the superblock.)
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

 -

This value is used to determine the format of the Dataspace - Message. When the format of the information in the message is - changed, the version number is incremented and can be used to - determine how the information in the object header is formatted. - This document describes version one (1) (there was no version zero - (0)).

-

Dimensionality

 -

This value is the number of dimensions that the data object - has.

-

Flags

 -

This field is used to store flags to indicate the presence of - parts of this message. Bit 0 (the least significant bit) is used to - indicate that maximum dimensions are present. Bit 1 is used to - indicate that permutation indices are present.

-

Dimension #n Size

 -

This value is the current size of the dimension of the data - as stored in the file. The first dimension stored in the list of - dimensions is the slowest changing dimension and the last dimension - stored is the fastest changing dimension.

-

Dimension #n Maximum Size

 -

- This value is the maximum size of the dimension of the data as - stored in the file. This value may be the special “unlimited” size which indicates - that the data may expand along this dimension indefinitely. If - these values are not stored, the maximum size of each dimension is - assumed to be the dimension’s current size. -

-

Permutation Index #n

 -

This value is the index permutation used to map each - dimension from the canonical representation to an alternate axis - for each dimension. If these values are not stored, the first - dimension stored in the list of dimensions is the slowest changing - dimension and the last dimension stored is the fastest changing - dimension.

-
-
- - - -
-

Version 2 of the dataspace message dropped the optional - permutation index value support, as it was never implemented in the - HDF5 Library:

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Dataspace Message - Version 2
bytebytebytebyte
VersionDimensionalityFlagsType

Dimension #1 SizeL
-
.
.
.

Dimension #n SizeL
-

Dimension #1 Maximum SizeL (optional)
-
.
.
.

Dimension #n Maximum SizeL (optional)
-
- - - - - - -
 (Items marked with an ‘L’ in the - above table are of the size specified in “Size of - Lengths” field in the superblock.)
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

 -

This value is used to determine the format of the Dataspace - Message. This field should be ‘2’ for version 2 format - messages.

-

Dimensionality

 -

This value is the number of dimensions that the data object - has.

-

Flags

 -

This field is used to store flags to indicate the presence of - parts of this message. Bit 0 (the least significant bit) is used to - indicate that maximum dimensions are present.

-

Type

 -

This field indicates the type of the dataspace:

- - - - - - - - - - - - - - - - - - -
ValueDescription
0A scalar dataspace; in other words, a dataspace - with a single, dimensionless element. -
1A simple dataspace; in other words, a dataspace - with a rank > 0 and an appropriate # of dimensions. -
2A null dataspace; in other words, a dataspace - with no elements. -
-

-

Dimension #n Size

 -

This value is the current size of the dimension of the data - as stored in the file. The first dimension stored in the list of - dimensions is the slowest changing dimension and the last dimension - stored is the fastest changing dimension.

-

Dimension #n Maximum Size

 -

- This value is the maximum size of the dimension of the data as - stored in the file. This value may be the special “unlimited” size which indicates - that the data may expand along this dimension indefinitely. If - these values are not stored, the maximum size of each dimension is - assumed to be the dimension’s current size. -

-
-
- - - - - -
-

- IV.A.2.c. The Link Info Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Link Info
Header Message Type: 0x002
Length: Varies
Status: Optional; may not be repeated.
Description:The link info message tracks variable information about the - current state of the links for a “new style” - group’s behavior. Variable information will be stored in this - message and constant information will be stored in the Group Info message. -
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Link Info
bytebytebytebyte
VersionFlagsThis space inserted - only to align table nicely

Maximum Creation Index (8 bytes, - optional)
-

Fractal Heap AddressO
-

Address of v2 B-tree for Name IndexO
-

Address of v2 B-tree for Creation Order - IndexO (optional)
-
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

 -

The version number for this message. This document describes - version 0.

-

Flags

This field determines various optional aspects of the - link info message:

- - - - - - - - - - - - - - - - - - -
BitDescription
0If set, creation order for the links is tracked.
1If set, creation order for the links is indexed.
2-7Reserved
-

Maximum Creation Index

This 64-bit value is the maximum creation order index - value stored for a link in this group.

-

- This field is present if bit 0 of flags is set. -

Fractal Heap Address

 -

- This is the address of the fractal heap to store dense links. Each - link stored in the fractal heap is stored as a Link Message. -

-

- If there are no links in the group, or the group’s links are - stored “compactly” (as object header messages), this - value will be the undefined address. -

-

Address of v2 B-tree for Name Index

This is the address of the version 2 B-tree to index - names of links.

-

- If there are no links in the group, or the group’s links are - stored “compactly” (as object header messages), this - value will be the undefined address. -

Address of v2 B-tree for Creation Order Index

This is the address of the version 2 B-tree to index - creation order of links.

-

- If there are no links in the group, or the group’s links are - stored “compactly” (as object header messages), this - value will be the undefined address. -

-

- This field exists if bit 1 of flags is set. -

-
- - -
-

- IV.A.2.d. The Datatype Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Datatype
Header Message Type: 0x0003
Length: Variable
Status: Required for dataset or committed - datatype (formerly named datatype) objects; may not be repeated.
Description:

The datatype message defines the datatype for each - element of a dataset or a common datatype for sharing between - multiple datasets. A datatype can describe an atomic type like a - fixed- or floating-point type or more complex types like a C struct - (compound datatype), array (array datatype) or C++ vector - (variable-length datatype).

-

Datatype messages that are part of a dataset object do not - describe how elements are related to one another; the dataspace - message is used for that purpose. Datatype messages that are part - of a committed datatype (formerly named datatype) message describe - a common datatype that can be shared by multiple datasets in the - file.

Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - -
Datatype Message
bytebytebytebyte
Class and VersionClass Bit Field, Bits 0-7Class Bit Field, Bits 8-15Class Bit Field, Bits 16-23
Size

-
Properties
-
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Class and Version

 -

The version of the datatype message and the datatype’s - class information are packed together in this field. The version - number is packed in the top 4 bits of the field and the class is - contained in the bottom 4 bits.

-

The version number information is used for changes in the - format of the datatype message and is described here:

- - - - - - - - - - - - - - - - - - - - - - -
VersionDescription
0Never used
1Used by early versions of the library to encode compound - datatypes with explicit array fields. See the compound datatype - description below for further details.
2Used when an array datatype needs to be encoded.
3Used when a VAX byte-ordered type needs to be encoded. - Packs various other datatype classes more efficiently also.
-

- -

The class of the datatype determines the format for the class - bit field and properties portion of the datatype message, which are - described below. The following classes are currently defined:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Fixed-Point
1Floating-Point
2Time
3String
4Bit field
5Opaque
6Compound
7Reference
8Enumerated
9Variable-Length
10Array
-

- -

Class Bit Fields

 -

The information in these bit fields is specific to each - datatype class and is described below. All bits not defined for a - datatype class are set to zero.

-

Size

 -

The size of a datatype element in bytes.

-

Properties

 -

This variable-sized sequence of bytes encodes information - specific to each datatype class and is described for each class - below. If there is no property information specified for a datatype - class, the size of this field is zero bytes.

-
-
- - -
-

Class specific information for Fixed-Point Numbers (Class 0):

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
Fixed-point Bit Field Description
BitsMeaning

0

- Byte Order. If zero, byte order is little-endian; otherwise, - byte order is big endian. -

1, 2

- Padding type. Bit 1 is the lo_pad bit and bit 2 is the - hi_pad bit. If a datum has unused bits at either end, then the - lo_pad or hi_pad bit is copied to those locations. -

3

- Signed. If this bit is set then the fixed-point number is in - 2’s complement form. -

4-23

Reserved (zero).

-
- -
-
- - - - - - - - - - - - - - -
Fixed-Point Property Description
ByteByteByteByte
Bit OffsetBit Precision
-
- -
-
- - - - - - - - - - - - - - - - -
Field NameDescription

Bit Offset

 -

The bit offset of the first significant bit of the - fixed-point value within the datatype. The bit offset specifies the - number of bits “to the right of” the value (which are - set to the lo_pad bit value).

-

Bit Precision

 -

The number of bits of precision of the fixed-point value - within the datatype. This value, combined with the datatype - element’s size and the Bit Offset field specifies the number - of bits “to the left of” the value (which are set to - the hi_pad bit value).

-
-
- - -
-

Class specific information for Floating-Point Numbers (Class 1):

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Floating-Point Bit Field Description
BitsMeaning

0, 6

- Byte Order. These two non-contiguous bits specify the - “endianness” of the bytes in the datatype element. -

- - - - - - - - - - - - - - - - - - - - - - - - - - - -
Bit 6Bit 0Description
00Byte order is little-endian
01Byte order is big-endian
10Reserved
11Byte order is VAX-endian
-

1, 2, 3

- Padding type. Bit 1 is the low bits pad type, bit 2 is the - high bits pad type, and bit 3 is the internal bits pad type. If a - datum has unused bits at either end or between the sign bit, - exponent, or mantissa, then the value of bit 1, 2, or 3 is copied - to those locations. -

4-5

- Mantissa Normalization. This 2-bit bit field specifies how - the most significant bit of the mantissa is managed. -

- - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0No normalization
1The most significant bit of the mantissa is always set - (except for 0.0).
2The most significant bit of the mantissa is not stored, - but is implied to be set.
3Reserved.
-

7

Reserved (zero).

8-15

- Sign Location. This is the bit position of the sign bit. - Bits are numbered with the least significant bit zero. -

16-23

Reserved (zero).

-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
Floating-Point Property Description
ByteByteByteByte
Bit OffsetBit Precision
Exponent LocationExponent SizeMantissa LocationMantissa Size
Exponent Bias
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Bit Offset

 -

The bit offset of the first significant bit of the - floating-point value within the datatype. The bit offset specifies - the number of bits “to the right of” the value.

-

Bit Precision

 -

The number of bits of precision of the floating-point value - within the datatype.

-

Exponent Location

 -

The bit position of the exponent field. Bits are numbered - with the least significant bit number zero.

-

Exponent Size

 -

The size of the exponent field in bits.

-

Mantissa Location

 -

The bit position of the mantissa field. Bits are numbered - with the least significant bit number zero.

-

Mantissa Size

 -

The size of the mantissa field in bits.

-

Exponent Bias

 -

The bias of the exponent field.

-
-
- - -
-

Class specific information for Time (Class 2):

- - -
- - - - - - - - - - - - - - - - - -
Time Bit Field Description
BitsMeaning

0

- Byte Order. If zero, byte order is little-endian; otherwise, - byte order is big endian. -

1-23

Reserved (zero).

-
- -
-
- - - - - - - - - - - -
Time Property Description
ByteByte
Bit Precision
-
- -
-
- - - - - - - - - - - -
Field NameDescription

Bit Precision

 -

The number of bits of precision of the time value.

-
-
- - -
-

Class specific information for Strings (Class 3):

- - -
- - - - - - - - - - - - - - - - - - - - - - -
String Bit Field Description
BitsMeaning

0-3

- Padding type. This four-bit value determines the type of - padding to use for the string. The values are: - -

- - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Null Terminate: A zero byte marks the end of the string - and is guaranteed to be present after converting a long string to - a short string. When converting a short string to a long string - the value is padded with additional null characters as necessary. -
1Null Pad: Null characters are added to the end of the - value during conversions from short values to long values but - conversion in the opposite direction simply truncates the value. -
2Space Pad: Space characters are added to the end of the - value during conversions from short values to long values but - conversion in the opposite direction simply truncates the value. - This is the Fortran representation of the string.
3-15Reserved
-

4-7

- Character Set. The character set used to encode the string. -

- - - - - - - - - - - - - - - - - - - - -
ValueDescription
0ASCII character set encoding
1UTF-8 character set encoding
2-15Reserved
-

8-23

Reserved (zero).

-
- -

There are no properties defined for the string class.

- - -

Class specific information for bit fields (Class 4):

- -
- - - - - - - - - - - - - - - - - - - - - - -
Bitfield Bit Field Description
BitsMeaning

0

- Byte Order. If zero, byte order is little-endian; otherwise, - byte order is big endian. -

1, 2

- Padding type. Bit 1 is the lo_pad type and bit 2 is the - hi_pad type. If a datum has unused bits at either end, then the - lo_pad or hi_pad bit is copied to those locations. -

3-23

Reserved (zero).

-
- -
-
- - - - - - - - - - - - - - -
Bit Field Property Description
ByteByteByteByte
Bit OffsetBit Precision
-
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription

Bit Offset

 -

The bit offset of the first significant bit of the bit field - within the datatype. The bit offset specifies the number of bits - “to the right of” the value.

-

Bit Precision

 -

The number of bits of precision of the bit field within the - datatype.

-
-
- - -
-

Class specific information for Opaque (Class 5):

- -
- - - - - - - - - - - - - - - - - -
Opaque Bit Field Description
BitsMeaning

0-7

Length of ASCII tag in bytes.

8-23

Reserved (zero).

-
- -
-
- - - - - - - - - - - - - -
Opaque Property Description
ByteByteByteByte

ASCII Tag

-
- -
-
- - - - - - - - - - -
Field NameDescription

ASCII Tag

 -

This NUL-terminated string provides a description for the - opaque type. It is NUL-padded to a multiple of 8 bytes.

-
-
- - -
-

Class specific information for Compound (Class 6):

- -
- - - - - - - - - - - - - - - - - -
Compound Bit Field Description
BitsMeaning

0-15

- Number of Members. This field contains the number of members - defined for the compound datatype. The member definitions are - listed in the Properties field of the data type message. -

16-23

Reserved (zero).

-
- - -

The Properties field of a compound datatype is a list of the - member definitions of the compound datatype. The member definitions - appear one after another with no intervening bytes. The member types - are described with a (recursively) encoded datatype message.

- -

Note that the property descriptions are different for different - versions of the datatype version. Additionally note that the version 0 - datatype encoding is deprecated and has been replaced with later - encodings in versions of the HDF5 Library from the 1.4 release onward.

- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Compound Properties Description for Datatype - Version 1
ByteByteByteByte

Name
-
Byte Offset of Member
DimensionalityReserved (zero)
Dimension Permutation
Reserved (zero)
Dimension #1 Size (required)
Dimension #2 Size (required)
Dimension #3 Size (required)
Dimension #4 Size (required)

Member Type Message
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Name

 -

This NUL-terminated string provides a description for the - opaque type. It is NUL-padded to a multiple of 8 bytes.

-

Byte Offset of Member

 -

This is the byte offset of the member within the datatype.

-

Dimensionality

 -

If set to zero, this field indicates a scalar member. If set - to a value greater than zero, this field indicates that the member - is an array of values. For array members, the size of the array is - indicated by the ‘Size of Dimension n’ field in this - message.

-

Dimension Permutation

 -

This field was intended to allow an array field to have its - dimensions permuted, but this was never implemented. This field - should always be set to zero.

-

Dimension #n Size

 -

This field is the size of a dimension of the array field as - stored in the file. The first dimension stored in the list of - dimensions is the slowest changing dimension and the last dimension - stored is the fastest changing dimension.

-

Member Type Message

 -

This field is a datatype message describing the datatype of - the member.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - -
Compound Properties Description for Datatype - Version 2
ByteByteByteByte

Name
-
Byte Offset of Member

Member Type Message
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Name

 -

This NUL-terminated string provides a description for the - opaque type. It is NUL-padded to a multiple of 8 bytes.

-

Byte Offset of Member

 -

This is the byte offset of the member within the datatype.

-

Member Type Message

 -

This field is a datatype message describing the datatype of - the member.

-
-
- - -
-
-
- - - - - - - - - - - - - - - - - - - - - - -
Compound Properties Description for Datatype - Version 3
ByteByteByteByte

Name
-
Byte Offset of Member (variable size)

Member Type Message
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Name

- This NUL-terminated string provides a description for the opaque - type. It is not NUL-padded to a multiple of 8 bytes. -

Byte Offset of Member

This is the byte offset of the member within the - datatype. The field size is the minimum number of bytes necessary, - based on the size of the datatype element. For example, a datatype - element size of less than 256 bytes uses a 1 byte length, a - datatype element size of 256-65535 bytes uses a 2 byte length, and - so on.

Member Type Message

This field is a datatype message describing the - datatype of the member.

-
- - -
-

Class specific information for Reference (Class 7):

- -
- - - - - - - - - - - - - - - - - -
Reference Bit Field Description
BitsMeaning

0-3

- Type. This four-bit value contains the type of reference - described. The values defined are: - -

- - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Object Reference: A reference to another object in this - HDF5 file.
1Dataset Region Reference: A reference to a region within - a dataset in this HDF5 file.
2-15Reserved
-

4-23

Reserved (zero).

-
- -

There are no properties defined for the reference class.

- - -
-

Class specific information for Enumeration (Class 8):

- -
- - - - - - - - - - - - - - - - - -
Enumeration Bit Field Description
BitsMeaning

0-15

- Number of Members. The number of name/value pairs defined - for the enumeration type. -

16-23

Reserved (zero).

-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - -
Enumeration Property Description for Datatype - Versions 1 & 2
ByteByteByteByte

Base Type
-

Names
-

Values
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Base Type

 -

Each enumeration type is based on some parent type, usually - an integer. The information for that parent type is described - recursively by this field.

-

Names

 -

The name for each name/value pair. Each name is stored as a - null terminated ASCII string in a multiple of eight bytes. The - names are in no particular order.

-

Values

 -

The list of values in the same order as the names. The values - are packed (no inter-value padding) and the size of each value is - determined by the parent type.

-
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - -
Enumeration Property Description for Datatype - Version 3
ByteByteByteByte

Base Type
-

Names
-

Values
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Base Type

 -

Each enumeration type is based on some parent type, usually - an integer. The information for that parent type is described - recursively by this field.

-

Names

 -

- The name for each name/value pair. Each name is stored as a null - terminated ASCII string, not padded to a multiple of eight - bytes. The names are in no particular order. -

-

Values

 -

The list of values in the same order as the names. The values - are packed (no inter-value padding) and the size of each value is - determined by the parent type.

-
-
- - - -
-

Class specific information for Variable-Length (Class 9):

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
Variable-Length Bit Field Description
BitsMeaning

0-3

- Type. This four-bit value contains the type of - variable-length datatype described. The values defined are: - -

- - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Sequence: A variable-length sequence of any datatype. - Variable-length sequences do not have padding or character set - information.
1String: A variable-length sequence of characters. - Variable-length strings have padding and character set - information.
2-15Reserved
-

4-7

- Padding type. (variable-length string only) This four-bit - value determines the type of padding used for variable-length - strings. The values are the same as for the string padding type, as - follows: -

- - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Null terminate: A zero byte marks the end of a string and - is guaranteed to be present after converting a long string to a - short string. When converting a short string to a long string, - the value is padded with additional null characters as necessary. -
1Null pad: Null characters are added to the end of the - value during conversion from a short string to a longer string. - Conversion from a long string to a shorter string simply - truncates the value.
2Space pad: Space characters are added to the end of the - value during conversion from a short string to a longer string. - Conversion from a long string to a shorter string simply - truncates the value. This is the Fortran representation of the - string.
3-15Reserved
-

- -

This value is set to zero for variable-length sequences.

8-11

- Character Set. (variable-length string only) This four-bit - value specifies the character set to be used for encoding the - string: -

- - - - - - - - - - - - - - - - - - - - -
ValueDescription
0ASCII character set encoding
1UTF-8 character set encoding
2-15Reserved
-

- -

This value is set to zero for variable-length sequences.

12-23

Reserved (zero).

-
- -
-
-
- - - - - - - - - - - - - - -
Variable-Length Property Description
ByteByteByteByte

Base Type
-
-
- -
-
- - - - - - - - - - - -
Field NameDescription

Base Type

 -

Each variable-length type is based on some parent type. The - information for that parent type is described recursively by this - field.

-
-
- - -
-

Class specific information for Array (Class 10):

- -

There are no bit fields defined for the array class.

- -

Note that the dimension information defined in the property for - this datatype class is independent of dataspace information for a - dataset. The dimension information here describes the dimensionality of - the information within a data element (or a component of an element, if - the array datatype is nested within another datatype) and the dataspace - for a dataset describes the size and locations of the elements in a - dataset.

- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Array Property Description for Datatype Version 2
ByteByteByteByte
DimensionalityReserved (zero)
Dimension #1 Size
.
.
.
Dimension #n Size
Permutation Index #1
.
.
.
Permutation Index #n

Base Type
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Dimensionality

 -

This value is the number of dimensions that the array has.

-

Dimension #n Size

 -

This value is the size of the dimension of the array as - stored in the file. The first dimension stored in the list of - dimensions is the slowest changing dimension and the last dimension - stored is the fastest changing dimension.

-

Permutation Index #n

 -

This value is the index permutation used to map each - dimension from the canonical representation to an alternate axis - for each dimension. Currently, dimension permutations are not - supported, and these indices should be set to the index position - minus one. In other words, the first dimension should be set to 0, - the second dimension should be set to 1, and so on.

-

Base Type

 -

Each array type is based on some parent type. The information - for that parent type is described recursively by this field.

-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Array Property Description for Datatype Version 3
ByteByteByteByte
DimensionalityThis space inserted - only to align table nicely
Dimension #1 Size
.
.
.
Dimension #n Size

Base Type
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Dimensionality

 -

This value is the number of dimensions that the array has.

-

Dimension #n Size

 -

This value is the size of the dimension of the array as - stored in the file. The first dimension stored in the list of - dimensions is the slowest changing dimension and the last dimension - stored is the fastest changing dimension.

-

Base Type

 -

Each array type is based on some parent type. The information - for that parent type is described recursively by this field.

-
-
- - - -
-

- IV.A.2.e. The Data Storage - Fill - Value (Old) Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Fill Value (old)
Header Message Type: 0x0004
Length: Varies
Status: Optional; may not be repeated.
Description:

The fill value message stores a single data value - which is returned to the application when an uninitialized data - element is read from a dataset. The fill value is interpreted with - the same datatype as the dataset. If no fill value message is - present then a fill value of all zero bytes is assumed.

-

This fill value message is deprecated in favor of the - “new” fill value message (Message Type 0x0005) and is - only written to the file for forward compatibility with versions of - the HDF5 Library before the 1.6.0 version. Additionally, it only - appears for datasets with a user-defined fill value (as opposed to - the library default fill value or an explicitly set - “undefined” fill value).

Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - -
Fill Value Message (Old)
bytebytebytebyte
Size

Fill Value (optional, variable - size)
-
-
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription

Size

 -

This is the size of the Fill Value field in bytes.

-

Fill Value

 -

The fill value. The bytes of the fill value are interpreted - using the same datatype as for the dataset.

-
-
- - -
-

- IV.A.2.f. The Data Storage - Fill Value - Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Fill Value
Header Message Type: 0x0005
Length: Varies
Status: Required for dataset objects; may - not be repeated.
Description:The fill value message stores a single data value which is - returned to the application when an uninitialized data element is - read from a dataset. The fill value is interpreted with the same - datatype as the dataset.
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - -
Fill Value Message - Versions 1 & 2
bytebytebytebyte
VersionSpace Allocation TimeFill Value Write TimeFill Value Defined
Size (optional)

Fill Value (optional, variable - size)
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

 -

The version number information is used for changes in the - format of the fill value message and is described here:

- - - - - - - - - - - - - - - - - - - - - - -
VersionDescription
0Never used
1Initial version of this message.
2In this version, the Size and Fill Value fields are only - present if the Fill Value Defined field is set to 1.
3This version packs the other fields in the message more - efficiently than version 2.
-

-

-

Space Allocation Time

 -

When the storage space for the dataset’s raw data will - be allocated. The allowed values are:

- - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Not used.
1Early allocation. Storage space for the entire dataset - should be allocated in the file when the dataset is created.
2Late allocation. Storage space for the entire dataset - should not be allocated until the dataset is written to.
3Incremental allocation. Storage space for the dataset - should not be allocated until the portion of the dataset is - written to. This is currently used in conjunction with chunked - data storage for datasets.
-

- -

Fill Value Write Time

 -

At the time that storage space for the dataset’s raw - data is allocated, this value indicates whether the fill value - should be written to the raw data storage elements. The allowed - values are:

- - - - - - - - - - - - - - - - - - -
ValueDescription
0On allocation. The fill value is always written to the - raw data storage when the storage space is allocated.
1Never. The fill value should never be written to the raw - data storage.
2Fill value written if set by user. The fill value will be - written to the raw data storage when the storage space is - allocated only if the user explicitly set the fill value. If the - fill value is the library default or is undefined, it will not be - written to the raw data storage.
-

- -

Fill Value Defined

 -

This value indicates if a fill value is defined for this - dataset. If this value is 0, the fill value is undefined. If this - value is 1, a fill value is defined for this dataset. For version 2 - or later of the fill value message, this value controls the - presence of the Size and Fill Value fields.

-

Size

 -

This is the size of the Fill Value field in bytes. This field - is not present if the Version field is greater than 1, and the Fill - Value Defined field is set to 0.

-

Fill Value

 -

The fill value. The bytes of the fill value are interpreted - using the same datatype as for the dataset. This field is not - present if the Version field is greater than 1, and the Fill Value - Defined field is set to 0.

-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - -
Fill Value Message - Version 3
bytebytebytebyte
VersionFlagsThis space inserted - only to align table nicely
Size (optional)

Fill Value (optional, variable - size)
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

 -

The version number information is used for changes in the - format of the fill value message and is described here:

- - - - - - - - - - - - - - - - - - - - - - -
VersionDescription
0Never used
1Initial version of this message.
2In this version, the Size and Fill Value fields are only - present if the Fill Value Defined field is set to 1.
3This version packs the other fields in the message more - efficiently than version 2.
-

- -

Flags

 -

When the storage space for the dataset’s raw data will - be allocated. The allowed values are:

- - - - - - - - - - - - - - - - - - - - - - - - - - -
BitsDescription
0-1Space Allocation Time, with the same values as versions 1 - and 2 of the message.
2-3Fill Value Write Time, with the same values as versions 1 - and 2 of the message.
4Fill Value Undefined, indicating that the fill value has - been marked as “undefined” for this dataset. Bits 4 - and 5 cannot both be set.
5Fill Value Defined, with the same values as versions 1 - and 2 of the message. Bits 4 and 5 cannot both be set.
6-7Reserved (zero).
-

- -

Size

 -

This is the size of the Fill Value field in bytes. This field - is not present if the Version field is greater than 1, and the Fill - Value Defined flag is set to 0.

-

Fill Value

 -

The fill value. The bytes of the fill value are interpreted - using the same datatype as for the dataset. This field is not - present if the Version field is greater than 1, and the Fill Value - Defined flag is set to 0.

-
-
- - -
-

- IV.A.2.g. The Link Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Link
Header Message Type: 0x0006
Length: Varies
Status: Optional; may be repeated.
Description:

This message encodes the information for a link in a - group’s object header, when the group is storing its links - “compactly”, or in the group’s fractal heap, when - the group is storing its links “densely”.

-

- A group is storing its links compactly when the fractal heap - address in the Link Info - Message is set to the “undefined address” value. -

Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Link Message
bytebytebytebyte
VersionFlagsLink type (optional)This space inserted only to align - table nicely

Creation Order (8 bytes, - optional)
-
Link Name Character Set (optional)Length of Link Name (variable size)This space inserted - only to align table nicely
Link Name (variable size)

Link Information (variable size)
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number for this message. This document - describes version 1.

Flags

This field contains information about the link and - controls the presence of other fields below.

- - - - - - - - - - - - - - - - - - - - - - - - - - -
BitsDescription
0-1Determines the size of the Length of Link Name - field. - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0The size of the Length of Link Name field is - 1 byte. -
1The size of the Length of Link Name field is - 2 bytes. -
2The size of the Length of Link Name field is - 4 bytes. -
3The size of the Length of Link Name field is - 8 bytes. -
-
2Creation Order Field Present: if set, the Creation - Order field is present. If not set, creation order information - is not stored for links in this group. -
3Link Type Field Present: if set, the link is not a hard - link and the Link Type field is present. If not set, the - link is a hard link. -
4Link Name Character Set Field Present: if set, the link - name is not represented with the ASCII character set and the Link - Name Character Set field is present. If not set, the link name - is represented with the ASCII character set. -
5-7Reserved (zero).
-

Link type

This is the link class type and can be one of the - following values:

- - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0A hard link (should never be stored in the file)
1A soft link.
2-63Reserved for future HDF5 internal use.
64An external link.
65-255Reserved, but available for user-defined link types.
-

- -

- This field is present if bit 3 of Flags is set. -

Creation Order

This 64-bit value is an index of the link’s - creation time within the group. Values start at 0 when the group is - created an increment by one for each link added to the group. - Removing a link from a group does not change existing links’ - creation order field.

-

- This field is present if bit 2 of Flags is set. -

Link Name Character Set

This is the character set for encoding the - link’s name:

- - - - - - - - - - - - - - - -
ValueDescription
0ASCII character set encoding (this should never be stored - in the file)
1UTF-8 character set encoding
-

- -

- This field is present if bit 4 of Flags is set. -

Length of link name

- This is the length of the link’s name. The size of this field - depends on bits 0 and 1 of Flags. -

Link name

This is the name of the link, non-NULL terminated.

Link information

- The format of this field depends on the link type. -

-

- For hard links, the field is formatted as follows: - -

- - - - - -
Size of Offsets bytes:The address of the object header for the - object that the link points to.
-

- -

- For soft links, the field is formatted as follows: - -

- - - - - - - - - -
Bytes 1-2:Length of soft link value.
Length of soft link value bytes:A non-NULL-terminated string storing the value of the - soft link.
-

- -

- For external links, the field is formatted as follows: - -

- - - - - - - - - -
Bytes 1-2:Length of external link value.
Length of external link value bytes:The first byte contains the version number in the upper 4 - bits and flags in the lower 4 bits for the external link. Both - version and flags are defined to be zero in this document. The - remaining bytes consist of two NULL-terminated strings, with no - padding between them. The first string is the name of the HDF5 - file containing the object linked to and the second string is the - full path to the object linked to, within the HDF5 file’s - group hierarchy.
-

- -

- For user-defined links, the field is formatted as follows: - -

- - - - - - - - - -
Bytes 1-2:Length of user-defined data.
Length of user-defined link value bytes:The data supplied for the user-defined link type.
-

-
- -
-

- IV.A.2.h. The Data Storage - - External Data Files Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: External Data Files
Header Message Type: 0x0007
Length: Varies
Status: Optional; may not be repeated.
Description:The external data storage message indicates that the data - for an object is stored outside the HDF5 file. The filename of the - object is stored as a Universal Resource Location (URL) of the - actual filename containing the data. An external file list record - also contains the byte offset of the start of the data within the - file and the amount of space reserved in the file for that data.
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
External File List Message
bytebytebytebyte
VersionReserved (zero)
Allocated SlotsUsed Slots

Heap AddressO
-

Slot Definitions...
-
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

 -

The version number information is used for changes in the - format of External Data Storage Message and is described here:

- - - - - - - - - - - - - -
VersionDescription
0Never used.
1The current version used by the library.
-

- -

Allocated Slots

 -

The total number of slots allocated in the message. Its value - must be at least as large as the value contained in the Used Slots - field. (The current library simply uses the number of Used Slots - for this message)

-

Used Slots

 -

The number of initial slots which contains valid information.

-

Heap Address

 -

This is the address of a local heap which contains the names - for the external files (The local heap information can be found in - Disk Format Level 1D in this document). The name at offset zero in - the heap is always the empty string.

-

Slot Definitions

 -

The slot definitions are stored in order according to the - array addresses they represent.

-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
External File List Slot
bytebytebytebyte

Name Offset in Local HeapL
-

Offset in External Data FileL
-

Data Size in External FileL
-
- - - - - - -
 (Items marked with an ‘L’ in the - above table are of the size specified in “Size of - Lengths” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Name Offset in Local Heap

 -

- The byte offset within the local name heap for the name of the - file. File names are stored as a URL which has a protocol name, a - host name, a port number, and a file name: - - protocol:port//host/file - - . If the protocol is omitted then “file:” is assumed. - If the port number is omitted then a default port for that protocol - is used. If both the protocol and the port number are omitted then - the colon can also be omitted. If the double slash and host name - are omitted then “localhost” is assumed. The file name - is the only mandatory part, and if the leading slash is missing - then it is relative to the application’s current working - directory (the use of relative names is not recommended). -

-

Offset in External Data File

 -

This is the byte offset to the start of the data in the - specified file. For files that contain data for a single dataset - this will usually be zero.

-

Data Size in External File

 -

This is the total number of bytes reserved in the specified - file for raw data storage. For a file that contains exactly one - complete dataset which is not extendable, the size will usually be - the exact size of the dataset. However, by making the size larger - one allows HDF5 to extend the dataset. The size can be set to a - value larger than the entire file since HDF5 will read zeroes past - the end of the file without failing.

-
-
- - -
-

- IV.A.2.i. The Data Storage - Layout Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Data Storage - - Layout
Header Message Type: 0x0008
Length: Varies
Status: Required for datasets; may not be - repeated.
Description:Data layout describes how the elements of a - multi-dimensional array are stored in the HDF5 file. Three types of - data layout are supported: -
    -
  1. Contiguous: The array is stored in one contiguous area of - the file. This layout requires that the size of the array be - constant: data manipulations such as chunking, compression, - checksums, or encryption are not permitted. The message stores the - total storage size of the array. The offset of an element from the - beginning of the storage area is computed as in a C array.
    2. -
  2. Chunked: The array domain is regularly decomposed into - chunks, and each chunk is allocated and stored separately. This - layout supports arbitrary element traversals, compression, - encryption, and checksums. (these features are described in other - messages). The message stores the size of a chunk instead of the - size of the entire array; the storage size of the entire array can - be calculated by traversing the B-tree that stores the chunk - addresses.
    3. -
  3. Compact: The array is stored in one contiguous block, as - part of this object header message.
    4. -
-
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Data Layout Message (Versions 1 and 2)
bytebytebytebyte
VersionDimensionalityLayout ClassReserved (zero)
Reserved (zero)

Data AddressO (optional)
-
Dimension 0 Size
Dimension 1 Size
...
Dimension #n Size
Dataset Element Size (optional)
Compact Data Size (optional)

Compact Data... (variable size, - optional)
-
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

 -

The version number information is used for changes in the - format of the data layout message and is described here:

- - - - - - - - - - - - - - - - - - - - -
VersionDescription
0Never used.
1Used by version 1.4 and before of the library to encode - layout information. Data space is always allocated when the data - set is created.
2Used by version 1.6.x of the library to encode layout - information. Data space is allocated only when it is necessary.
-

-

Dimensionality

An array has a fixed dimensionality. This field - specifies the number of dimension size fields later in the message. - The value stored for chunked storage is 1 greater than the number - of dimensions in the dataset’s dataspace. For example, 2 is - stored for a 1 dimensional dataset.

Layout Class

The layout class specifies the type of storage for - the data and how the other fields of the layout message are to be - interpreted.

- - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Compact Storage
1Contiguous Storage
2Chunked Storage
-

Data Address

For contiguous storage, this is the address of the - raw data in the file. For chunked storage this is the address of - the v1 B-tree that is used to look up the addresses of the chunks. - This field is not present for compact storage. If the version for - this message is greater than 1, the address may have the - “undefined address” value, to indicate that storage has - not yet been allocated for this array.

Dimension #n Size

For contiguous and compact storage the dimensions - define the entire size of the array while for chunked storage they - define the size of a single chunk. In all cases, they are in units - of array elements (not bytes). The first dimension stored in the - list of dimensions is the slowest changing dimension and the last - dimension stored is the fastest changing dimension.

Dataset Element Size

The size of a dataset element, in bytes. This field - is only present for chunked storage.

Compact Data Size

This field is only present for compact data storage. - It contains the size of the raw data for the dataset array, in - bytes.

Compact Data

This field is only present for compact data storage. - It contains the raw data for the dataset array.

-
- -
-

Version 3 of this message re-structured the format into specific - properties that are required for each layout class.

- - -
- - - - - - - - - - - - - - - - - - - -
- Data Layout Message (Version 3) -
bytebytebytebyte
VersionLayout ClassThis space inserted - only to align table nicely

Properties (variable size)
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

 -

The version number information is used for changes in the - format of layout message and is described here:

- - - - - - - - - - -
VersionDescription
3Used by the version 1.6.3 and later of the library to - store properties for each layout class.
-

-

Layout Class

The layout class specifies the type of storage for - the data and how the other fields of the layout message are to be - interpreted.

- - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Compact Storage
1Contiguous Storage
2Chunked Storage
-

Properties

This variable-sized field encodes information - specific to each layout class and is described below. If there is - no property information specified for a layout class, the size of - this field is zero bytes.

-
- -
-

Class-specific information for compact layout (Class 0): (Note: - The dimensionality information is in the Dataspace message)

- - -
- - - - - - - - - - - - - - - - - - -
Compact Storage Property Description
bytebytebytebyte
SizeThis space inserted - only to align table nicely

Raw Data... (variable size)
-
-
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription

Size

This field contains the size of the raw data for the - dataset array, in bytes.

Raw Data

This field contains the raw data for the dataset - array.

-
- - -
-

Class-specific information for contiguous layout (Class 1): - (Note: The dimensionality information is in the Dataspace message)

- - -
- - - - - - - - - - - - - - - - - -
Contiguous Storage Property Description
bytebytebytebyte

AddressO
-

SizeL
-
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
 (Items marked with an ‘L’ in the above table are - of the size specified in “Size of Lengths” field in the - superblock.)
- -
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription

Address

This is the address of the raw data in the file. The - address may have the “undefined address” value, to - indicate that storage has not yet been allocated for this array.

Size

This field contains the size allocated to store the - raw data, in bytes.

-
- - -
-

Class-specific information for chunked layout (Class 2):

- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Chunked Storage Property Description
bytebytebytebyte
DimensionalityThis space inserted - only to align table nicely

AddressO
-
Dimension 0 Size
Dimension 1 Size
...
Dimension #n Size
Dataset Element Size
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Dimensionality

A chunk has a fixed dimensionality. This field - specifies the number of dimension size fields later in the message.

Address

This is the address of the v1 B-tree that is used to - look up the addresses of the chunks that actually store portions of - the array data. The address may have the “undefined - address” value, to indicate that storage has not yet been - allocated for this array.

Dimension #n Size

These values define the dimension size of a single - chunk, in units of array elements (not bytes). The first dimension - stored in the list of dimensions is the slowest changing dimension - and the last dimension stored is the fastest changing dimension.

Dataset Element Size

The size of a dataset element, in bytes.

-
- -
-

- IV.A.2.j. The Bogus Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Bogus
Header Message Type: 0x0009
Length: 4 bytes
Status: For testing only; should never be - stored in a valid file.
Description:This message is used for testing the HDF5 Library’s - response to an “unknown” message type and should never - be encountered in a valid HDF5 file.
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - -
Bogus Message
bytebytebytebyte
Bogus Value
-
- -
-
- - - - - - - - - - -
Field NameDescription

Bogus Value

 -

- This value should always be: - 0xdeadbeef - . -

-
-
- -
-

- IV.A.2.k. The Group Info Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Group Info
Header Message Type: 0x000A
Length: Varies
Status: Optional; may not be repeated.
Description:

- This message stores information for the constants defining a - “new style” group’s behavior. Constant - information will be stored in this message and variable information - will be stored in the Link Info - message. -

-

Note: the “estimated entry” information below is - used when determining the size of the object header for the group - when it is created.

Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - -
Group Info Message
bytebytebytebyte
VersionFlagsLink Phase Change: Maximum Compact Value (optional)
Link Phase Change: Minimum Dense Value (optional)Estimated Number of Entries (optional)
Estimated Link Name Length of Entries (optional)This space inserted - only to align table nicely
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number for this message. This document - describes version 0.

Flags

This is the group information flag with the following - definition:

- - - - - - - - - - - - - - - - - - -
BitDescription
0If set, link phase change values are stored.
1If set, the estimated entry information is non-default - and is stored.
2-7Reserved
-

Link Phase Change: Maximum Compact Value

The is the maximum number of links to store - “compactly” (in the group’s object header).

-

- This field is present if bit 0 of Flags is set. -

Link Phase Change: Minimum Dense Value

- This is the minimum number of links to store “densely” - (in the group’s fractal heap). The fractal heap’s - address is located in the Link Info - message. -

-

- This field is present if bit 0 of Flags is set. -

Estimated Number of Entries

This is the estimated number of entries in groups.

-

- If this field is not present, the default value of - 4 - will be used for the estimated number of group entries. -

-

- This field is present if bit 1 of Flags is set. -

Estimated Link Name Length of Entries

This is the estimated length of entry name.

-

- If this field is not present, the default value of - 8 - will be used for the estimated link name length of group entries. -

-

- This field is present if bit 1 of Flags is set. -

-
-

- -
-

- IV.A.2.l. The Data Storage - Filter - Pipeline Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Data Storage - - Filter Pipeline
Header Message Type: 0x000B
Length: Varies
Status: Optional; may not be repeated.
Description:

This message describes the filter pipeline which - should be applied to the data stream by providing filter - identification numbers, flags, a name, and client data.

-

This message may be present in the object headers of both - dataset and group objects. For datasets, it specifies the filters - to apply to raw data. For groups, it specifies the filters to apply - to the group’s fractal heap. Currently, only datasets using - chunked data storage use the filter pipeline on their raw data.

Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - -
Filter Pipeline Message - Version 1
bytebytebytebyte
VersionNumber of FiltersReserved (zero)
Reserved (zero)

Filter Description List (variable - size)
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number for this message. This table - describes version 1.

Number of Filters

The total number of filters described in this - message. The maximum possible number of filters in a message is 32.

Filter Description List

A description of each filter. A filter description - appears in the next table.

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Filter Description
bytebytebytebyte
Filter Identification ValueName Length
FlagsNumber Client Data Values

Name (variable size, optional)
-

Client Data (variable size, - optional)
-
Padding (variable size, optional)
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Filter Identification Value

 -

- This value, often referred to as a filter identifier, is designed - to be a unique identifier for the filter. Values from zero through - 32,767 are reserved for filters supported by The HDF Group in the - HDF5 Library and for filters requested and supported by third - parties. Filters supported by The HDF Group are documented - immediately below. Information on 3rd-party filters can be found at - The HDF Group’s - Registered Filters page. -

- -

- To request a filter identifier, please contact The HDF - Group’s Help Desk at The HDF Group Help Desk. - You will be asked to provide the following information: -

-
    -
  1. Contact information for the developer requesting the new - identifier
    2. -
  2. A short description of the new filter
    3. -
  3. Links to any relevant information, including licensing - information
    4. -
-

Values from 32768 to 65535 are reserved for non-distributed - uses (for example, internal company usage) or for application usage - when testing a feature. The HDF Group does not track or document - the use of the filters with identifiers from this range.

- -

The filters currently in library version 1.8.0 are listed - below:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
IdentificationNameDescription
0N/AReserved
1deflateGZIP deflate compression
2shuffleData element shuffling
3fletcher32Fletcher32 checksum
4szipSZIP compression
5nbitN-bit packing
6scaleoffsetScale and offset encoded values
-

-

Name Length

Each filter has an optional null-terminated ASCII - name and this field holds the length of the name including the null - termination padded with nulls to be a multiple of eight. If the - filter has no name then a value of zero is stored in this field.

Flags

The flags indicate certain properties for a filter. - The bit values defined so far are:

- - - - - - - - - - - - - - - -
BitDescription
0If set then the filter is an optional filter. During - output, if an optional filter fails it will be silently skipped - in the pipeline.
1-15Reserved (zero)
-

Number of Client Data Values

- Each filter can store integer values to control how the filter - operates. The number of entries in the Client Data array - is stored in this field. -

Name

- If the Name Length field is non-zero then it will contain - the size of this field, padded to a multiple of eight. This field - contains a null-terminated, ASCII character string to serve as a - comment/name for the filter. -

Client Data

- This is an array of four-byte integers which will be passed to the - filter function. The Client Data Number of Values - determines the number of elements in the array. -

Padding

Four bytes of zeroes are added to the message at this - point if the Client Data Number of Values field contains an odd - number.

-
- -
-
- - - - - - - - - - - - - - - - - - - -
Filter Pipeline Message - Version 2
bytebytebytebyte
VersionNumber of FiltersThis space inserted - only to align table nicely

Filter Description List (variable - size)
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number for this message. This table - describes version 2.

Number of Filters

The total number of filters described in this - message. The maximum possible number of filters in a message is 32.

Filter Description List

A description of each filter. A filter description - appears in the next table.

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
Filter Description
bytebytebytebyte
Filter Identification ValueName Length (optional)
FlagsNumber Client Data Values

Name (variable size, optional)
-

Client Data (variable size, - optional)
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Filter Identification Value

 -

- This value, often referred to as a filter identifier, is designed - to be a unique identifier for the filter. Values from zero through - 32,767 are reserved for filters supported by The HDF Group in the - HDF5 Library and for filters requested and supported by third - parties. Filters supported by The HDF Group are documented - immediately below. Information on 3rd-party filters can be found at - The HDF Group’s - Registered Filters page. -

- -

- To request a filter identifier, please contact The HDF - Group’s Help Desk at The HDF Group Help Desk. - You will be asked to provide the following information: -

-
    -
  1. Contact information for the developer requesting the new - identifier
    2. -
  2. A short description of the new filter
    3. -
  3. Links to any relevant information, including licensing - information
    4. -
-

Values from 32768 to 65535 are reserved for non-distributed - uses (for example, internal company usage) or for application usage - when testing a feature. The HDF Group does not track or document - the use of the filters with identifiers from this range.

- -

The filters currently in library version 1.8.0 are listed - below:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
IdentificationNameDescription
0N/AReserved
1deflateGZIP deflate compression
2shuffleData element shuffling
3fletcher32Fletcher32 checksum
4szipSZIP compression
5nbitN-bit packing
6scaleoffsetScale and offset encoded values
-

-

Name Length

Each filter has an optional null-terminated ASCII - name and this field holds the length of the name including the null - termination padded with nulls to be a multiple of eight. If the - filter has no name then a value of zero is stored in this field.

-

- Filters with IDs less than 256 (in other words, filters that are - defined in this format documentation) do not store the Name - Length or Name fields. -

Flags

The flags indicate certain properties for a filter. - The bit values defined so far are:

- - - - - - - - - - - - - - - -
BitDescription
0If set then the filter is an optional filter. During - output, if an optional filter fails it will be silently skipped - in the pipeline.
1-15Reserved (zero)
-

Number of Client Data Values

- Each filter can store integer values to control how the filter - operates. The number of entries in the Client Data array - is stored in this field. -

Name

- If the Name Length field is non-zero then it will contain - the size of this field, not padded to a multiple of eight. - This field contains a non-null-terminated, ASCII character - string to serve as a comment/name for the filter. -

-

- Filters that are defined in this format documentation such as - deflate and shuffle do not store the Name Length or Name - fields. -

Client Data

- This is an array of four-byte integers which will be passed to the - filter function. The Client Data Number of Values - determines the number of elements in the array. -

-
- -
-

- IV.A.2.m. The Attribute Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Attribute
Header Message Type: 0x000C
Length: Varies
Status: Optional; may be repeated.
Description:

- The Attribute message is used to store objects in the HDF5 - file which are used as attributes, or “metadata” about - the current object. An attribute is a small dataset; it has a name, - a datatype, a dataspace, and raw data. Since attributes are stored - in the object header, they should be relatively small (in other - words, less than 64KB). They can be associated with any type of - object which has an object header (groups, datasets, or committed - (named) datatypes). -

-

- In 1.8.x versions of the library, attributes can be larger than - 64KB. See the - “Special Issues” section of the Attributes chapter in - the HDF5 User Guide for more information. -

-

Note: Attributes on an object must have unique names: the - HDF5 Library currently enforces this by causing the creation of an - attribute with a duplicate name to fail. Attributes on different - objects may have the same name, however.

Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Attribute Message (Version 1)
bytebytebytebyte
VersionReserved (zero)Name Size
Datatype SizeDataspace Size

Name (variable size)
-

Datatype (variable size)
-

Dataspace (variable size)
-

Data (variable size)
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number information is used for changes in - the format of the attribute message and is described here:

- - - - - - - - - - - - - - - -
VersionDescription
0Never used.
1Used by the library before version 1.6 to encode - attribute message. This version does not support shared - datatypes.
-

Name Size

- The length of the attribute name in bytes including the null - terminator. Note that the Name field below may contain - additional padding not represented by this field. -

Datatype Size

- The length of the datatype description in the Datatype - field below. Note that the Datatype field may contain - additional padding not represented by this field. -

Dataspace Size

- The length of the dataspace description in the Dataspace - field below. Note that the Dataspace field may contain - additional padding not represented by this field. -

Name

The null-terminated attribute name. This field is - padded with additional null characters to make it a multiple of - eight bytes.

Datatype

The datatype description follows the same format as - described for the datatype object header message. This field is - padded with additional zero bytes to make it a multiple of eight - bytes.

Dataspace

The dataspace description follows the same format as - described for the dataspace object header message. This field is - padded with additional zero bytes to make it a multiple of eight - bytes.

Data

- The raw data for the attribute. The size is determined from the - datatype and dataspace descriptions. This field is not - padded with additional bytes. -

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Attribute Message (Version 2)
bytebytebytebyte
VersionFlagsName Size
Datatype SizeDataspace Size

Name (variable size)
-

Datatype (variable size)
-

Dataspace (variable size)
-

Data (variable size)
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number information is used for changes in - the format of the attribute message and is described here:

- - - - - - - - - - -
VersionDescription
2Used by the library of version 1.6.x and after to encode - attribute messages. This version supports shared datatypes. The - fields of name, datatype, and dataspace are not padded with - additional bytes of zero.
-

Flags

This bit field contains extra information about - interpreting the attribute message:

- - - - - - - - - - - - - - - -
BitDescription
0If set, datatype is shared.
1If set, dataspace is shared.
-

Name Size

The length of the attribute name in bytes including - the null terminator.

Datatype Size

- The length of the datatype description in the Datatype - field below. -

Dataspace Size

- The length of the dataspace description in the Dataspace - field below. -

Name

- The null-terminated attribute name. This field is not - padded with additional bytes. -

Datatype

The datatype description follows the same format as - described for the datatype object header message.

-

- If the Flag field indicates this attribute’s - datatype is shared, this field will contain a “shared - message” encoding instead of the datatype encoding. -

-

- This field is not padded with additional bytes. -

Dataspace

The dataspace description follows the same format as - described for the dataspace object header message.

-

- If the Flag field indicates this attribute’s - dataspace is shared, this field will contain a “shared - message” encoding instead of the dataspace encoding. -

-

- This field is not padded with additional bytes. -

Data

The raw data for the attribute. The size is - determined from the datatype and dataspace descriptions.

-

- This field is not padded with additional zero bytes. -

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Attribute Message (Version 3)
bytebytebytebyte
VersionFlagsName Size
Datatype SizeDataspace Size
Name Character Set EncodingThis space inserted - only to align table nicely

Name (variable size)
-

Datatype (variable size)
-

Dataspace (variable size)
-

Data (variable size)
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number information is used for changes in - the format of the attribute message and is described here:

- - - - - - - - - - -
VersionDescription
3Used by the library of version 1.8.x and after to encode - attribute messages. This version supports attributes with - non-ASCII names.
-

Flags

This bit field contains extra information about - interpreting the attribute message:

- - - - - - - - - - - - - - - -
BitDescription
0If set, datatype is shared.
1If set, dataspace is shared.
-

Name Size

The length of the attribute name in bytes including - the null terminator.

Datatype Size

- The length of the datatype description in the Datatype - field below. -

Dataspace Size

- The length of the dataspace description in the Dataspace - field below. -

Name Character Set Encoding

The character set encoding for the attribute’s - name:

- - - - - - - - - - - - - - - -
ValueDescription
0ASCII character set encoding
1UTF-8 character set encoding
-

Name

- The null-terminated attribute name. This field is not - padded with additional bytes. -

Datatype

The datatype description follows the same format as - described for the datatype object header message.

-

- If the Flag field indicates this attribute’s - datatype is shared, this field will contain a “shared - message” encoding instead of the datatype encoding. -

-

- This field is not padded with additional bytes. -

Dataspace

The dataspace description follows the same format as - described for the dataspace object header message.

-

- If the Flag field indicates this attribute’s - dataspace is shared, this field will contain a “shared - message” encoding instead of the dataspace encoding. -

-

- This field is not padded with additional bytes. -

Data

The raw data for the attribute. The size is - determined from the datatype and dataspace descriptions.

-

- This field is not padded with additional zero bytes. -

-
- -
-

- IV.A.2.n. The Object Comment Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Object Comment
Header Message Type: 0x000D
Length: Varies
Status: Optional; may not be repeated.
Description:The object comment is designed to be a short description of - an object. An object comment is a sequence of non-zero (\0) - ASCII characters with no other formatting included by the library. -
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - -
Name Message
bytebytebytebyte

Comment (variable size)
-
-
- -
-
- - - - - - - - - - -
Field NameDescription

Name

A null terminated ASCII character string.

-
- -
-

- IV.A.2.o. The Object - Modification Time (Old) Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Object Modification - Time (Old)
Header Message Type: 0x000E
Length: Fixed
Status: Optional; may not be repeated.
Description:

The object modification date and time is a timestamp - which indicates (using ISO-8601 date and time format) the last - modification of an object. The time is updated when any object - header message changes according to the system clock where the - change was posted. All fields of this message should be interpreted - as coordinated universal time (UTC).

-

- This modification time message is deprecated in favor of the - “new” Object - Modification Time message and is no longer written to the file in - versions of the HDF5 Library after the 1.6.0 version. -

Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Modification Time Message
bytebytebytebyte
Year
MonthDay of Month
HourMinute
SecondReserved
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Year

- The four-digit year as an ASCII string. For example, - 1998 - . -

Month

- The month number as a two digit ASCII string where January is - 01 - and December is - 12 - . -

Day of Month

- The day number within the month as a two digit ASCII string. The - first day of the month is - 01 - . -

Hour

- The hour of the day as a two digit ASCII string where midnight is - 00 - and 11:00pm is - 23 - . -

Minute

- The minute of the hour as a two digit ASCII string where the first - minute of the hour is - 00 - and the last is - 59 - . -

Second

- The second of the minute as a two digit ASCII string where the - first second of the minute is - 00 - and the last is - 59 - . -

Reserved

This field is reserved and should always be zero.

-
- -
-

- IV.A.2.p. The Shared Message Table - Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Shared Message Table
Header Message Type: 0x000F
Length: Fixed
Status: Optional; may not be repeated.
Description:This message is used to locate the table of shared object - header message (SOHM) indexes. Each index consists of information to - find the shared messages from either the heap or object header. This - message is only found in the superblock extension. -
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - -
Shared Message Table Message
bytebytebytebyte
VersionThis space inserted - only to align table nicely

Shared Object Header Message Table - AddressO
-
Number of IndicesThis space inserted - only to align table nicely
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number for this message. This document - describes version 0.

Shared Object Header Message Table Address

This field is the address of the master table for - shared object header message indexes.

Number of Indices

This field is the number of indices in the master - table.

-
- -
-

- IV.A.2.q. The Object Header - Continuation Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Object Header - Continuation
Header Message Type: 0x0010
Length: Fixed
Status: Optional; may be repeated.
Description:The object header continuation is the location in the file - of a block containing more header messages for the current data - object. This can be used when header blocks become too large or are - likely to change over time.
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - -
Object Header Continuation Message
bytebytebytebyte

OffsetO
-

LengthL
-
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
 (Items marked with an ‘L’ in the above table are - of the size specified in “Size of Lengths” field in the - superblock.)
- -
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription

Offset

This value is the address in the file where the - header continuation block is located.

Length

This value is the length in bytes of the header - continuation block in the file.

-
-
- -

The format of the header continuation block that this message - points to depends on the version of the object header that the message - is contained within.

- -

- Continuation blocks for version 1 object headers have no special - formatting information; they are merely a list of object header message - info sequences (type, size, flags, reserved bytes and data for each - message sequence). See the description of Version 1 Data Object Header Prefix. -

- -

- Continuation blocks for version 2 object headers do have - special formatting information as described here (see also the - description of Version 2 Data - Object Header Prefix.): -

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Version 2 Object Header Continuation Block
bytebytebytebyte
Signature
Header Message Type #1Size of Header Message Data #1Header Message #1 Flags
Header Message #1 Creation Order (optional)This space inserted - only to align table nicely

Header Message Data #1
-
.
.
.
Header Message Type #nSize of Header Message Data #nHeader Message #n Flags
Header Message #n Creation Order (optional)This space inserted - only to align table nicely

Header Message Data #n
-
Gap (optional, variable size)
Checksum
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Signature

 -

- The ASCII character string “ - OCHK - ” is used to indicate the beginning of an object header - continuation block. This gives file consistency checking utilities - a better chance of reconstructing a damaged file. -

-

Header Message #n Type

 -

Same format as version 1 of the object header, described - above.

-

Size of Header Message #n Data

 -

Same format as version 1 of the object header, described - above.

-

Header Message #n Flags

 -

Same format as version 1 of the object header, described - above.

-

Header Message #n Creation Order

 -

This field stores the order that a message of a given type - was created in.

-

- This field is present if bit 2 of flags is set. -

-

Header Message #n Data

 -

Same format as version 1 of the object header, described - above.

-

Gap

 -

A gap in an object header chunk is inferred by the end of the - messages for the chunk before the beginning of the chunk’s - checksum. Gaps are always smaller than the size of an object header - message prefix (message type + message size + message flags).

-

Gaps are formed when a message (typically an attribute - message) in an earlier chunk is deleted and a message from a later - chunk that does not quite fit into the free space is moved into the - earlier chunk.

-

Checksum

 -

This is the checksum for the object header chunk.

-
-
- -
-

- IV.A.2.r. The Symbol Table Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Symbol Table Message
Header Message Type: 0x0011
Length: Fixed
Status: Required for “old - style” groups; may not be repeated.
Description:Each “old style” group has a v1 B-tree and a - local heap for storing symbol table entries, which are located with - this message.
Format of data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - -
- Symbol Table Message -
bytebytebytebyte

v1 B-tree AddressO
-

Local Heap AddressO
-
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription

v1 B-tree Address

This value is the address of the v1 B-tree containing - the symbol table entries for the group.

Local Heap Address

This value is the address of the local heap - containing the link names for the symbol table entries for the - group.

-
- -
-

- IV.A.2.s. The Object Modification - Time Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Object Modification - Time
Header Message Type: 0x0012
Length: Fixed
Status: Optional; may not be repeated.
Description:The object modification time is a timestamp which indicates - the time of the last modification of an object. The time is updated - when any object header message changes according to the system clock - where the change was posted.
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - -
Modification Time Message
bytebytebytebyte
VersionReserved (zero)
Seconds After UNIX Epoch
-
- -
-
- - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number is used for changes in the format - of Object Modification Time and is described here:

- - - - - - - - - - - - - - - -
VersionDescription
0Never used.
1Used by Version 1.6.1 and after of the library to encode - time. In this version, the time is the seconds after Epoch.
-

Seconds After UNIX Epoch

A 32-bit unsigned integer value that stores the - number of seconds since 0 hours, 0 minutes, 0 seconds, January 1, - 1970, Coordinated Universal Time.

-
- -
-

- IV.A.2.t. The B-tree ‘K’ - Values Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: B-tree - ‘K’ Values
Header Message Type: 0x0013
Length: Fixed
Status: Optional; may not be repeated.
Description:This message retrieves non-default ‘K’ values - for internal and leaf nodes of a group or indexed storage v1 - B-trees. This message is only found in the superblock - extension. -
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - -
B-tree ‘K’ Values Message
bytebytebytebyte
VersionIndexed Storage Internal Node KThis space inserted only to align - table nicely
Group Internal Node KGroup Leaf Node K
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number for this message. This document - describes version 0.

Indexed Storage Internal Node K

This is the node ‘K’ value for each - internal node of an indexed storage v1 B-tree. See the description - of this field in version 0 and 1 of the superblock as well the - section on v1 B-trees.

Group Internal Node K

This is the node ‘K’ value for each - internal node of a group v1 B-tree. See the description of this - field in version 0 and 1 of the superblock as well as the section - on v1 B-trees.

Group Leaf Node K

This is the node ‘K’ value for each leaf - node of a group v1 B-tree. See the description of this field in - version 0 and 1 of the superblock as well as the section on v1 - B-trees.

-
- -
-

- IV.A.2.u. The Driver Info Message -

- - -
- - - - - - - - - - - - - - - - - - - - - -
Header Message Name: Driver Info
Header Message Type: 0x0014
Length: Varies
Status: Optional; may not be repeated.
Description:This message contains information needed by the file driver - to reopen a file. This message is only found in the - superblock extension: see the - “Disk Format: Level 0C - Superblock Extension” section - for more information. For more information on the fields in the - driver info message, see the “Disk - Format : Level 0B - File Driver Info” section; those who use - the multi and family file drivers will find this section - particularly helpful. -
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
Driver Info Message
bytebytebytebyte
VersionThis space inserted - only to align table nicely

Driver Identification
Driver Information SizeThis space inserted - only to align table nicely

-
Driver Information (variable size)
-
-
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number for this message. This document - describes version 0.

Driver Identification

This is an eight-byte ASCII string without null - termination which identifies the driver.

Driver Information Size

- The size in bytes of the Driver Information field of this - message. -

Driver Information

Driver information is stored in a format defined by - the file driver.

-
- -
-

- IV.A.2.v. The Attribute Info Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Attribute Info
Header Message Type: 0x0015
Length: Varies
Status: Optional; may not be repeated.
Description:This message stores information about the attributes on an - object, such as the maximum creation index for the attributes - created and the location of the attribute storage when the - attributes are stored “densely”.
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - -
Attribute Info Message
bytebytebytebyte
VersionFlagsMaximum Creation Index (optional)

Fractal Heap AddressO
-

Attribute Name v2 B-tree AddressO
-

Attribute Creation Order v2 B-tree - AddressO (optional)
-
- - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number for this message. This document - describes version 0.

Flags

This is the attribute index information flag with the - following definition:

- - - - - - - - - - - - - - - - - - -
BitDescription
0If set, creation order for attributes is tracked.
1If set, creation order for attributes is indexed.
2-7Reserved
-

Maximum Creation Index

The is the maximum creation order index value for the - attributes on the object.

-

- This field is present if bit 0 of Flags is set. -

Fractal Heap Address

This is the address of the fractal heap to store - dense attributes.

Attribute Name v2 B-tree Address

This is the address of the version 2 B-tree to index - the names of densely stored attributes.

Attribute Creation Order v2 B-tree Address

This is the address of the version 2 B-tree to index - the creation order of densely stored attributes.

-

- This field is present if bit 1 of Flags is set. -

-
- -
-

- IV.A.2.w. The Object Reference Count - Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: Object Reference - Count
Header Message Type: 0x0016
Length: Fixed
Status: Optional; may not be repeated.
Description:This message stores the number of hard links (in groups or - objects) pointing to an object: in other words, its reference - count. -
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - -
Object Reference Count
bytebytebytebyte
VersionThis space inserted - only to align table nicely
Reference count
-
- -
-
- - - - - - - - - - - - - - - - -
Field NameDescription

Version

The version number for this message. This document - describes version 0.

Reference Count

The unsigned 32-bit integer is the reference count - for the object. This message is only present in “version - 2” (or later) object headers, and if not present those object - header versions, the reference count for the object is assumed to - be 1.

-
- -
-

- IV.A.2.x. The File Space Info Message -

- - -
- - - - - - - - - - - - - - - - - - - - -
Header Message Name: File Space Info
Header Message Type: 0x0018
Length: Fixed
Status: Optional; may not be repeated.
Description:This message stores the file space management strategy (see - description below) that the library uses in handling file space - request for the file. It also contains the free-space section - threshold used by the library’s free-space managers for the - file. If the strategy is 1, this message also contains the addresses - of the file’s free-space managers which track free space for - each type of file space allocation. There are six basic types of - file space allocation: superblock, B-tree, raw data, global heap, - local heap, and object header. See the description of Free-space Manager as well the - description of allocation types in Appendix - B. -
Format of Data: See the tables below.
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
File Space Info
bytebytebytebyte
VersionStrategyThresholdL
Super-block Free-space Manager AddressO
B-tree Free-space Manager AddressO
Raw Data Free-space Manager AddressO
Global Heap Free-space Manager AddressO
Local Heap Free-space Manager AddressO
Object Header Free-space Manager AddressO
- - - - - - - - - - -
 (Items marked with an ‘O’ in the - above table are of the size specified in “Size of - Offsets” field in the superblock.)
 (Items marked with an ‘L’ in the above table are - of the size specified in “Size of Lengths” field in the - superblock.)
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Field NameDescription

Version

This is the version number of this message. This - document describes version 0.

Strategy

This is the file space management strategy for the - file. There are four types of strategies:

- - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
1With this strategy, the HDF5 Library’s free-space - managers track the free space that results from the manipulation - of HDF5 objects in the HDF5 file. The free space information is - saved when the file is closed, and reloaded when the file is - reopened.
When space is needed for file metadata or raw - data, the HDF5 Library first requests space from the - library’s free-space managers. If the request is not - satisfied, the library requests space from the aggregators. If - the request is still not satisfied, the library requests space - from the virtual file driver. That is, the library will use all - of the mechanisms for allocating space. -
2This is the HDF5 Library’s default file space - management strategy. With this strategy, the library’s - free-space managers track the free space that results from the - manipulation of HDF5 objects in the HDF5 file. The free space - information is NOT saved when the file is closed and the free - space that exists upon file closing becomes unaccounted space in - the file.
As with strategy #1, the library will try all - of the mechanisms for allocating space. When space is needed for - file metadata or raw data, the library first requests space from - the free-space managers. If the request is not satisfied, the - library requests space from the aggregators. If the request is - still not satisfied, the library requests space from the virtual - file driver. -
3With this strategy, the HDF5 Library does not track free - space that results from the manipulation of HDF5 objects in the - HDF5 file and the free space becomes unaccounted space in the - file.
When space is needed for file metadata or raw data, - the library first requests space from the aggregators. If the - request is not satisfied, the library requests space from the - virtual file driver. -
4With this strategy, the HDF5 Library does not track free - space that results from the manipulation of HDF5 objects in the - HDF5 file and the free space becomes unaccounted space in the - file.
When space is needed for file metadata or raw data, - the library requests space from the virtual file driver. -
-

Threshold

- This is the free-space section threshold. The library’s - free-space managers will track only free-space sections with size - greater than or equal to threshold. The default is to - track free-space sections of all sizes. -

Superblock Free-space Manager Address

This is the address of the free-space manager for - H5FD_MEM_SUPER allocation type.

B-tree Free-space Manager Address

This is the address of the free-space manager for - H5FD_MEM_BTREE allocation type.

Raw Data Free-space Manager Address

This is the address of the free-space manager for - H5FD_MEM_DRAW allocation type.

Global Heap Free-space Manager Address

This is the address of the free-space manager for - H5FD_MEM_GHEAP allocation type.

Local Heap Free-space Manager Address

This is the address of the free-space manager for - H5FD_MEM_LHEAP allocation type.

Object Header Free-space Manager Address

This is the address of the free-space manager for - H5FD_MEM_OHDR allocation type.

-
-
- - -
-

- IV.B. Disk Format: Level 2B - Data Object - Data Storage -

- -

The data for an object is stored separately from its header - information in the file and may not actually be located in the HDF5 - file itself if the header indicates that the data is stored externally. - The information for each record in the object is stored according to - the dimensionality of the object (indicated in the dataspace header - message). Multi-dimensional array data is stored in C order; in other - words, the “last” dimension changes fastest.

- -

Data whose elements are composed of atomic datatypes are stored - in IEEE format, unless they are specifically defined as being stored in - a different machine format with the architecture-type information from - the datatype header message. This means that each architecture will - need to [potentially] byte-swap data values into the internal - representation for that particular machine.

- -

Data with a variable-length datatype is stored in the global heap - of the HDF5 file. Global heap identifiers are stored in the data object - storage.

- -

Data whose elements are composed of reference datatypes are - stored in several different ways depending on the particular reference - type involved. Object pointers are just stored as the offset of the - object header being pointed to with the size of the pointer being the - same number of bytes as offsets in the file.

- -

Dataset region references are stored as a heap-ID which points to - the following information within the file-heap: an offset of the object - pointed to, number-type information (same format as header message), - dimensionality information (same format as header message), sub-set - start and end information (in other words, a coordinate location for - each), and field start and end names (in other words, a [pointer to - the] string indicating the first field included and a [pointer to the] - string name for the last field).

- -

Data of a compound datatype is stored as a contiguous stream of - the items in the structure, with each item formatted according to its - datatype.

- - - -
-
-
-

- V. Appendix A: Definitions -

- -

Definitions of various terms used in this document are included - in this section.

- -
- - - - - - - - - - - - - - - - -
TermDefinition
Undefined AddressThe undefined address for a - file is a file address with all bits set: in other words, 0xffff...ff. -
Unlimited SizeThe unlimited size for a size is - a value with all bits set: in other words, 0xffff...ff. -
-
- - - -
-
-
-

- VI. Appendix B: File Memory Allocation Types -

- -

There are six basic types of file memory allocation as follows:

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Basic Allocation TypeDescription
H5FD_MEM_SUPERFile memory allocated for Superblock.
H5FD_MEM_BTREEFile memory allocated for B-tree.
H5FD_MEM_DRAWFile memory allocated for raw data.
H5FD_MEM_GHEAPFile memory allocated for Global Heap.
H5FD_MEM_LHEAPFile memory allocated for Local Heap.
H5FD_MEM_OHDRFile memory allocated for Object Header.
-
- -

There are other file memory allocation types that are mapped to - the above six basic allocation types because they are similar in - nature. The mapping is listed in the following table:

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Basic Allocation TypeMapping of Allocation Types to Basic Allocation Types
H5FD_MEM_SUPERnone
H5FD_MEM_BTREEH5FD_MEM_SOHM_INDEX
H5FD_MEM_DRAWH5FD_MEM_FHEAP_HUGE_OBJ
H5FD_MEM_GHEAPnone
H5FD_MEM_LHEAPH5FD_MEM_FHEAP_DBLOCK, H5FD_MEM_FSPACE_SINFO
H5FD_MEM_OHDRH5FD_MEM_FHEAP_HDR, H5FD_MEM_FHEAP_IBLOCK, - H5FD_MEM_FSPACE_HDR, H5FD_MEM_SOHM_TABLE
-
- -

Allocation types that are mapped to basic allocation types are - described below:

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Allocation TypeDescription
H5FD_MEM_FHEAP_HDRFile memory allocated for Fractal Heap Header.
H5FD_MEM_FHEAP_DBLOCKFile memory allocated for Fractal Heap Direct - Blocks.
H5FD_MEM_FHEAP_IBLOCKFile memory allocated for Fractal Heap Indirect - Blocks.
H5FD_MEM_FHEAP_HUGE_OBJFile memory allocated for huge objects in the fractal heap.
H5FD_MEM_FSPACE_HDRFile memory allocated for Free-space Manager - Header.
H5FD_MEM_FSPACE_SINFOFile memory allocated for Free-space Section List - of the free-space manager. -
H5FD_MEM_SOHM_TABLEFile memory allocated for Shared Object Header - Message Table.
H5FD_MEM_SOHM_INDEXFile memory allocated for Shared Message Record - List.
-
-
- - diff --git a/doxygen/examples/H5.format.html b/doxygen/examples/H5.format.html deleted file mode 100644 index 2272a2b5eff..00000000000 --- a/doxygen/examples/H5.format.html +++ /dev/null @@ -1,20546 +0,0 @@ - - - - HDF5 File Format Specification Version 3.0 - - - - - - - - - - -
-
- - - - - - - -
-
    -
  1. Introduction
    2. - -
      -
    1. This Document
      2. -
    2. Changes for HDF5 1.12
      3. -
    3. Changes for HDF5 1.10
      4. -
    -     - -
  2. Disk Format: Level 0 - File Metadata
    3. - -
      -
    1. Disk Format: Level 0A - Format Signature - and Superblock
      2. -
    2. Disk Format: Level 0B - File Driver - Info
      3. -
    3. Disk Format: Level 0C - Superblock - Extension
      4. -
    -     -
  3. Disk Format: Level 1 - File Infrastructure
    4. - -
      -
    1. Disk Format: Level 1A - B-trees and B-tree - Nodes -
        -
      1. Disk Format: Level 1A1 - Version 1 - B-trees
        2. -
      2. Disk Format: Level 1A2 - Version 2 - B-trees
        3. -
      -
      2. -
    2. Disk Format: Level 1B - Group Symbol - Table Nodes
      3. -
    3. Disk Format: Level 1C - Symbol - Table Entry
      4. -
    4. Disk Format: Level 1D - Local Heaps
      5. -
    5. Disk Format: Level 1E - Global Heap
      6. -
    6. Disk Format: Level 1F - Global Heap - Block for Virtual Datasets
      7. -
    7. Disk Format: Level 1G - Fractal Heap
      8. -
    8. Disk Format: Level 1H - Free-space - Manager
      9. -
    9. Disk Format: Level 1I - Shared Object - Header Message Table
      10. -
    -     -
  4. Disk Format: Level 2 - Data Objects
    5. - -
      -
    1. Disk Format: Level 2A - Data Object Headers
      2. -
        -
      1. Disk Format: Level 2A1 - - Data Object Header Prefix -
          -
        1. Version 1 Data - Object Header Prefix
          2. -
        2. Version 2 Data - Object Header Prefix
          3. -
        -
        2. -
      2. Disk Format: Level 2A2 - - Data Object Header Messages
        3. -
          -
        1. The NIL Message
          2. -
        2. The Dataspace Message
          3. -
        3. The Link Info Message
          4. -
        4. The Datatype Message
          5. -
        5. The Data Storage - - Fill Value (Old) Message
          6. -
        -
      -
    -     -
- 		  -
    -
  1. Disk Format: Level 2 - Data - Objects (Continued)
    2. -
      -
    1. Disk Format: Level 2A - Data Object - Headers (Continued) -
        -
      1. Disk Format: Level 2A2 - - Data Object Header Messages (Continued)
        2. -
          -
        1. The Data Storage - - Fill Value Message
          2. -
        2. The Link Message
          3. -
        3. The Data Storage - - External Data Files Message
          4. -
        4. The Data Layout Message
          5. -
        5. The Bogus Message
          6. -
        6. The Group Info - Message
          7. -
        7. The Data Storage - - Filter Pipeline Message
          8. -
        8. The Attribute - Message
          9. -
        9. The Object Comment - Message
          10. -
        10. The Object - Modification Time (Old) Message
          11. -
        11. The Shared Message - Table Message
          12. -
        12. The Object Header - Continuation Message
          13. -
        13. The Symbol - Table Message
          14. -
        14. The Object - Modification Time Message
          15. -
        15. The B-tree - ‘K’ Values Message
          16. -
        16. The Driver Info - Message
          17. -
        17. The Attribute Info - Message
          18. -
        18. The Object Reference - Count Message
          19. -
        19. The File Space Info - Message
          20. -
        -
      -
      2. -
    2. Disk Format: Level 2B - Data Object Data Storage
      3. -
    -     -
  2. Appendix A: Definitions
    3. -
  3. Appendix B: File Space Allocation - Types
    4. -
  4. - Appendix C: Types of Indexes for Dataset Chunks
    5. - -
      -
    1. The Single Chunk Index
      2. -
    2. The Implicit Index
      3. -
    3. The Fixed Array Index
      4. -
    4. The Extensible Array Index
      5. -
    5. The Version 2 B-trees Index
      6. -
    -     -
  5. - Appendix D: Encoding for Dataspace and Reference
    6. - -
      -
    1. Dataspace Encoding
      2. -
    2. Reference Encoding (Revised)
      3. -
    3. Reference Encoding (Backward Compatibility)
      4. -
    -     -
-
-
- - -

I. Introduction

 - - - - - - - -
  -
- HDF5 Groups - 		 
  - Figure 1: Relationships among the HDF5 root group, other groups, and objects -
- 		 
  - HDF5 Objects -  
  - Figure 2: HDF5 objects -- datasets, datatypes, or dataspaces -
- 		 
- - -

The format of an HDF5 file on disk encompasses several - key ideas of the HDF4 and AIO file formats as well as - addressing some shortcomings therein. The new format is - more self-describing than the HDF4 format and is more - uniformly applied to data objects in the file.

- -

An HDF5 file appears to the user as a directed graph. - The nodes of this graph are the higher-level HDF5 objects - that are exposed by the HDF5 APIs:

- -
    -
  • Groups
    • -
  • Datasets
    • -
  • Committed (formerly Named) datatypes
    • -
- -

At the lowest level, as information is actually written to the disk, - an HDF5 file is made up of the following objects:

-
    -
  • A superblock
    • -
  • B-tree nodes
    • -
  • Heap blocks
    • -
  • Object headers
    • -
  • Object data
    • -
  • Free space
    • -
- -

The HDF5 Library uses these low-level objects to represent the - higher-level objects that are then presented to the user or - to applications through the APIs. For instance, a group is an - object header that contains a message that points to a local - heap (for storing the links to objects in the group) and to a - B-tree (which indexes the links). A dataset is an object header - that contains messages that describe the datatype, dataspace, - layout, filters, external files, fill value, and other elements - with the layout message pointing to either a raw data chunk or - to a B-tree that points to raw data chunks.

- - -

I.A. This Document

 - -

This document describes the lower-level data objects; - the higher-level objects and their properties are described - in the HDF5 User Guide.

- -

Three levels of information comprise the file format. - Level 0 contains basic information for identifying and - defining information about the file. Level 1 information contains - the information about the pieces of a file shared by many objects - in the file (such as B-trees and heaps). Level 2 is the rest - of the file and contains all of the data objects with each object - partitioned into header information, also known as - metadata, and data.

- -

The various components of the lower-level data objects are - described in pairs of tables. The first table shows the format - layout, and the second table describes the fields. The titles - of format layout tables begin with “Layout”. The - titles of the tables where the fields are described begin with - “Fields”. For example, the table that describes the - format of the version 2 B-tree header has - a title of “Layout: Version 2 B-tree Header”, and the - fields in the version 2 B-tree header are described in the table - titled “Fields: Version 2 B-tree Header”. - -

The sizes of various fields in the following layout tables are - determined by looking at the number of columns the field spans - in the table. There are exceptions:

-
    -
  • The size may be overridden by specifying a size in - parentheses
    • -
  • The size of addresses is determined by the - Size of Offsets field - in the superblock and is indicated in this document with a - superscripted ‘O’
    • -
  • The size of length fields is determined by the - Size of Lengths field in - the superblock and is indicated in this document with a - superscripted ‘L’
    • -
- -

Values for all fields in this document should be treated as unsigned - integers, unless otherwise noted in the description of a field. - Additionally, all metadata fields are stored in little-endian byte - order. -

- -

All checksums used in the format are computed with the - Jenkins’ - lookup3 algorithm. -

- -

Whenever a bit flag or field is mentioned for an entry, bits are - numbered from the lowest bit position in the entry. -

- -

Various format tables in this document have cells with - “This space inserted only to align table nicely”. These - entries in the table are just to make the table presentation nicer - and do not represent any values or padding in the file. -

- - -

I.B. Changes for HDF5 2.0

 -

The following sections have been - changed or added for the 2.0 release:

-
    -
  • Under “The Datatype Message”, - in the Description for “Fields:Datatype Message”, - version 5 was added, as well as the new Complex class (11).
    • -
- - -

I.B. Changes for HDF5 1.12

 -

The following sections have been - changed or added for the 1.12 release:

- - - - -

I.C. Changes for HDF5 1.10

 - -

The following sections have been - changed or added for the 1.10 release:

- - - - -

- II. Disk Format: Level 0 - File Metadata

- - - -

- II.A. Disk Format: Level 0A - Format Signature and Superblock

- -

The superblock may begin at certain predefined offsets within - the HDF5 file, allowing a block of unspecified content for - users to place additional information at the beginning (and - end) of the HDF5 file without limiting the HDF5 Library’s - ability to manage the objects within the file itself. This - feature was designed to accommodate wrapping an HDF5 file in - another file format or adding descriptive information to an HDF5 - file without requiring the modification of the actual file’s - information. The superblock is located by searching for the - HDF5 format signature at byte offset 0, byte offset 512, and at - successive locations in the file, each a multiple of two of - the previous location; in other words, at these byte offsets: - 0, 512, 1024, 2048, and so on.

- -

The superblock is composed of the format signature, followed by a - superblock version number and information that is specific to each - version of the superblock. - -

Currently, there are four versions of the superblock format: -

    -
  • Version 0 is the default format.
    • -
  • Version 1 is the same as version 0 but with the - “Indexed Storage Internal Node K” field - for storing non-default B-tree ‘K’ value.
    • -
  • Version 2 has some fields eliminated and compressed from - superblock format versions 0 and 1. It has added checksum support - and superblock extension to store additional superblock - metadata.
    • -
  • Version 3 is the same as version 2 except that the field - “File Consistency Flags” is used for file - locking. This format version will enable support for the latest - version.
    • -
- -

Versions 0 and 1 of the superblock are described below:

- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Superblock (Versions 0 and 1) -
bytebytebytebyte

Format Signature - (8 bytes)

Version # of SuperblockVersion # of File’s Free Space StorageVersion # of Root Group Symbol Table EntryReserved (zero)
Version Number of Shared Header Message FormatSize of OffsetsSize of LengthsReserved (zero)
Group Leaf Node KGroup Internal Node K
File Consistency Flags
Indexed Storage Internal Node K1Reserved - (zero)1

Base AddressO


Address of File Free space InfoO


End of File AddressO


Driver Information Block AddressO

Root Group Symbol Table Entry
- - - - - - - - -
  - (Items marked with a ‘1’ in the above table are - new in version 1 of the superblock.) -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Superblock (Versions 0 and 1) -
Field NameDescription

Format Signature

This field contains a constant value and can be used to - quickly identify a file as being an HDF5 file. The - constant value is designed to allow easy identification of - an HDF5 file and to allow certain types of data corruption - to be detected. The file signature of an HDF5 file always - contains the following values:

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Decimal:13772687013102610
Hexadecimal:894844460d0a1a0a
ASCII C Notation:\211HDF\r\n\032\n
-
-

This signature both identifies the file as an HDF5 file - and provides for immediate detection of common - file-transfer problems. The first two bytes distinguish - HDF5 files on systems that expect the first two bytes to - identify the file type uniquely. The first byte is - chosen as a non-ASCII value to reduce the probability - that a text file may be misrecognized as an HDF5 file; - also, it catches bad file transfers that clear bit - 7. Bytes two through four name the format. The CR-LF - sequence catches bad file transfers that alter newline - sequences. The control-Z character stops file display - under MS-DOS. The final line feed checks for the inverse - of the CR-LF translation problem. (This is a direct - descendent of the - PNG file - signature.)

-

This field is present in version 0+ of the superblock. -

Version Number of the Superblock

This value is used to determine the format of the - information in the superblock. When the format of the - information in the superblock is changed, the version number - is incremented to the next integer and can be used to - determine how the information in the superblock is - formatted.

- -

Values of 0, 1 and 2 are defined for this field (the - format of version 2 is described below, not here). -

- -

This field is present in version 0+ of the superblock. -

-

Version Number of the File’s Free Space - Information

 -

This value is used to determine the format of the - file’s free space information. -

-

The only value currently valid in this field is ‘0’, which - indicates that the file’s free space is as described - below. -

- -

This field is present in versions 0 and 1 of the - superblock. -

-

Version Number of the Root Group Symbol Table - Entry

This value is used to determine the format of the - information in the Root Group Symbol Table Entry. When the - format of the information in that field is changed, the - version number is incremented to the next integer and can be - used to determine how the information in the field - is formatted.

-

The only value currently valid in this field is ‘0’, - which indicates that the root group symbol table entry is - formatted as described below.

-

This field is present in version 0 and 1 of the - superblock.

-

Version Number of the Shared Header Message Format

This value is used to determine the format of the - information in a shared object header message. Since the format - of the shared header messages differs from the other private - header messages, a version number is used to identify changes - in the format. -

-

The only value currently valid in this field is ‘0’, which - indicates that shared header messages are formatted as - described below. -

- -

This field is present in version 0 and 1 of the superblock. -

-

Size of Offsets

This value contains the number of bytes used to store - addresses in the file. The values for the addresses of - objects in the file are offsets relative to a base address, - usually the address of the superblock signature. This - allows a wrapper to be added after the file is created - without invalidating the internal offset locations. -

- -

This field is present in version 0+ of the superblock. -

-

Size of Lengths

This value contains the number of bytes used to store - the size of an object. -

-

This field is present in version 0+ of the superblock. -

-

Group Leaf Node K

 -

Each leaf node of a group B-tree will have at - least this many entries but not more than twice this - many. If a group has a single leaf node then it - may have fewer entries. -

-

This value must be greater than zero. -

-

See the description of B-trees below. -

- -

This field is present in version 0 and 1 of the superblock. -

-

Group Internal Node K

 -

Each internal node of a group B-tree will have at - least this many entries but not more than twice this - many. If the group has only one internal - node then it might have fewer entries. -

-

This value must be greater than zero. -

-

See the description of B-trees below. -

- -

This field is present in version 0 and 1 of the superblock. -

-

File Consistency Flags

 -

This field is unused and should be ignored. -

-

This field is present in version 0+ of the superblock. -

-

Indexed Storage Internal Node K

 -

Each internal node of an indexed storage B-tree will have at - least this many entries but not more than twice this - many. If the index storage B-tree has only one internal - node then it might have fewer entries. -

-

This value must be greater than zero. -

-

See the description of B-trees below. -

- -

This field is present in version 1 of the superblock. -

-

Base Address

 -

This is the absolute file address of the first byte of - the HDF5 data within the file. The library currently - constrains this value to be the absolute file address - of the superblock itself when creating new files; - future versions of the library may provide greater - flexibility. When opening an existing file and this address does - not match the offset of the superblock, the library assumes - that the entire contents of the HDF5 file have been adjusted in - the file and adjusts the base address and end of file address to - reflect their new positions in the file. Unless otherwise noted, - all other file addresses are relative to this base - address. -

- -

This field is present in version 0+ of the superblock. -

-

Address of Global Free-space Index

 -

The file’s free space is not persistent for version 0 and 1 of - the superblock. - Currently this field always contains the - undefined address. -

- -

This field is present in version 0 and 1 of the superblock. -

-

End of File Address

 -

This is the absolute file address of the first byte past - the end of all HDF5 data. It is used to determine whether a - file has been accidentally truncated and as an address where - file data allocation can occur if space from the free list is - not used. -

- -

This field is present in version 0+ of the superblock. -

-

Driver Information Block Address

 -

This is the relative file address of the file driver - information block which contains driver-specific - information needed to reopen the file. If there is no - driver information block then this entry should be the - undefined address. -

- -

This field is present in version 0 and 1 of the superblock. -

-

Root Group Symbol Table Entry

 -

This is the symbol table entry - of the root group, which serves as the entry point into - the group graph for the file. -

- -

This field is present in version 0 and 1 of the superblock. -

-
-
- -
-
-
-

Versions 2 and 3 of the superblock are described below:

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Superblock (Versions 2 and 3) -
bytebytebytebyte

Format Signature - (8 bytes)

Version # of SuperblockSize of OffsetsSize of LengthsFile Consistency Flags

Base AddressO


Superblock Extension AddressO


End of File AddressO


Root Group Object Header AddressO

Superblock Checksum
- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Superblock (Versions 2 and 3) -
Field NameDescription

Format Signature

 -

This field is the same as described for versions 0 and 1 of the - superblock. -

Version Number of the Superblock

 -

This field has a value of 2 and has the same meaning as for - versions 0 and 1. -

-

Size of Offsets

 -

This field is the same as described for - versions 0 and 1 of the - superblock. -

-

Size of Lengths

 -

This field is the same as described for - versions 0 and 1 of the - superblock. -

-

File Consistency Flags

 -

For superblock version - 2: This field is unused and should be ignored.

-

For superblock version - 3: This value contains flags to ensure file consistency for - file locking. Currently, the following bit flags are defined: -

    -
  • Bit 0 if set indicates that the file has been opened for - write access.
    • -
  • Bit 1 is reserved for future use.
    • -
  • Bit 2 if set indicates that the file has been opened for - single-writer/multiple-reader (SWMR) write access.
    • -
  • Bits 3-7 are reserved for future use.
    • -
-

- Bit 0 should be set as the first action when a file has been - opened for write access. Bit 2 should be set when a file - has been opened for SWMR write access. These two bits should - be cleared only as the final action when closing a file. -

-

This field is present in version 0+ of the superblock. -

-

The size of this - field has been reduced from 4 bytes in superblock format - versions 0 and 1 to 1 byte. -

-

Base Address

 -

This field is the same as described for versions 0 and - 1 of the superblock. -

-

Superblock Extension Address

 -

The field is the address of the object header for the - superblock extension. - If there is no extension then this entry should be the - undefined address. -

-

End of File Address

 -

This field is the same as described for versions 0 and 1 of the - superblock. -

-

Root Group Object Header Address

 -

This is the address of - the root group object header, - which serves as the entry point into the group graph for the file. -

-

Superblock Checksum

 -

The checksum for the superblock. -

-
-
- -
- -

- II.B. Disk Format: Level 0B - File Driver Info

- -

The driver information block is an optional region of the - file which contains information needed by the file driver - to reopen a file. The format is described below:

- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Driver Information Block -
bytebytebytebyte
VersionReserved
Driver Information Size

Driver Identification - (8 bytes)



Driver Information - (variable size)


-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Driver Information Block -
Field NameDescription

Version

 -

The version number of the Driver Information Block. - This document describes version 0. -

-

Driver Information Size

 -

The size in bytes of the Driver Information field. -

-

Driver Identification

 -

This is an eight-byte ASCII string without null - termination which identifies the driver and/or version number - of the Driver Information Block. The predefined driver encoded - in this field by the HDF5 Library is identified by the - letters NCSA followed by the first four characters of - the driver name. If the Driver Information block is not - the original version then the last letter(s) of the - identification will be replaced by a version number in - ASCII, starting with 0. -

-

- Identification for user-defined drivers is also eight-byte long. - It can be arbitrary but should be unique to avoid - the four character prefix “NCSA”. -

-

Driver Information

Driver information is stored in a format defined by the - file driver (see description below).
-
- -
-

The two drivers encoded in the Driver Identification - field are as follows:

-
    -
  • - Multi driver: -

    - The identifier for this driver is “NCSAmulti”. - This driver provides a mechanism for segregating raw data and different types of metadata - into multiple files. - These files are viewed by the library as a single virtual HDF5 file with a single file address. - A maximum of 6 files will be created for the following data: - superblock, B-tree, raw data, global heap, local heap, and object header. - More than one type of data can be written to the same file. -

    • -
  • - Family driver -

    - The identifier for this driver is “NCSAfami” and is encoded in this field for library version 1.8 and after. - This driver is designed for systems that do not support files larger than 2 gigabytes - by splitting the HDF5 file address space across several smaller files. - It does nothing to segregate metadata and raw data; - they are mixed in the address space just as they would be in a single contiguous file. -

    • -
-

The format of the Driver Information field for the - above two drivers are described below:

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Multi Driver Information -
bytebytebytebyte
Member MappingMember MappingMember MappingMember Mapping
Member MappingMember MappingReservedReserved

Address of Member File 1


End of Address for Member File 1


Address of Member File 2


End of Address for Member File 2


... ...


Address of Member File N


End of Address for Member File N


Name of Member File 1 - (variable size)


Name of Member File 2 - (variable size)


... ...


Name of Member File N - (variable size)

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Multi Driver Information -
Field NameDescription

Member Mapping

These fields are integer values from 1 to 6 - indicating how the data can be mapped to or merged with another type of - data. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Member MappingDescription
1The superblock data.
2The B-tree data.
3The raw data.
4The global heap data.
5The local heap data.
6The object header data.

-

For example, if the third field has the value 3 and all the rest have the - value 1, it means there are two files: one for raw data, and one for superblock, - B-tree, global heap, local heap, and object header.

-

Reserved

These fields are reserved and should always be zero.

Address of Member File N

This field specifies the virtual address at which the member file starts.

-

N is the number of member files.

-

End of Address for Member File N

This field is the end of the allocated address for the member file. -

Name of Member File N

This field is the null-terminated name of the member file and - its length should be multiples of 8 bytes. - Additional bytes will be padded with NULLs. The default naming - convention is %s-X.h5, where X is one of the letters - s (for superblock), b (for B-tree), r (for raw data), - g (for global heap), l (for local heap), and o (for - object header). The name of the whole HDF5 file will substitute the %s - in the string. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - -
- Layout: Family Driver Information -
bytebytebytebyte

Size of Member File

-
- -
-
- - - - - - - - - - - -
- Fields: Family Driver Information -
Field NameDescription

Size of Member File

This field is the size of the member file in the family of files.

-
- -

- II.C. Disk Format: Level 0C - Superblock Extension

- -

The superblock extension is used to store superblock metadata - which is either optional, or added after the version of the superblock - was defined. Superblock extensions may only exist when version 2 - or later of the superblock is used. A superblock extension is an object - header which may hold the following messages:

- - - - -

- III. Disk Format: Level 1 - File Infrastructure

- -

- III.A. Disk Format: Level 1A - B-trees and B-tree Nodes

- -

B-trees allow flexible storage for objects which tend to grow - in ways that cause the object to be stored discontiguously. B-trees - are described in various algorithms books including “Introduction to - Algorithms” by Thomas H. Cormen, Charles E. Leiserson, and Ronald - L. Rivest. B-trees are used in several places in the HDF5 file format, - when an index is needed for another data structure.

- -

The version 1 B-tree structure described below is the original - index structure. The version 1 B-trees are being phased out in - favor of the version 2 B-trees described below. Note that both - types of structures may be found in the same file depending on - the application settings when creating the file.

- -

- III.A.1. Disk Format: Level 1A1 - Version 1 B-trees

- -

Version 1 B-trees in HDF5 files are an implementation of the - B-link tree. The sibling nodes at a particular level in - the tree are stored in a doubly-linked list. See the - “Efficient Locking for Concurrent Operations on B-trees” - paper by Phillip Lehman and S. Bing Yao as published in the - ACM Transactions on Database Systems, Vol. 6, No. 4, - December 1981.

- -

The B-trees implemented by the file format contain one more - key than the number of children. In other words, each child - pointer out of a B-tree node has a left key and a right key. - The pointers out of internal nodes point to sub-trees while - the pointers out of leaf nodes point to symbol nodes and - raw data chunks. - Aside from that difference, internal nodes and leaf nodes - are identical.

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: B-tree Nodes -
bytebytebytebyte
Signature
Node TypeNode LevelEntries Used

Address of Left SiblingO


Address of Right SiblingO

Key 1 (variable size)

Address of Child 1O

Key 2 (variable size)

Address of Child 2O

...
Key 2K (variable size)

Address of Child 2KO

Key 2K+1 - (variable size)
- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: B-tree Nodes -
Field NameDescription

Signature

 -

The ASCII character string “TREE” - is used to indicate the beginning of a B-tree node. This - gives file consistency checking utilities a better chance - of reconstructing a damaged file. -

-

Node Type

 -

Each B-tree points to a particular type of data. - This field indicates the type of data as well as - implying the maximum degree K of the tree and - the size of each Key field. - - - - - - - - - - - - - - - -
Node TypeDescription
0This tree points to group nodes.
1This tree points to raw data chunk nodes.

-

Node Level

 -

The node level indicates the level at which this node - appears in the tree (leaf nodes are at level zero). Not - only does the level indicate whether child pointers - point to sub-trees or to data, but it can also be used - to help file consistency checking utilities reconstruct - damaged trees. -

-

Entries Used

 -

This determines the number of children to which this - node points. All nodes of a particular type of tree - have the same maximum degree, but most nodes will point - to less than that number of children. The valid child - pointers and keys appear at the beginning of the node - and the unused pointers and keys appear at the end of - the node. The unused pointers and keys have undefined - values. -

-

Address of Left Sibling

 -

This is the relative file address of the left sibling of - the current node. If the current - node is the left-most node at this level then this field - is the undefined address. -

-

Address of Right Sibling

 -

This is the relative file address of the right sibling of - the current node. If the current - node is the right-most node at this level then this - field is the undefined address. -

-

Keys and Child Pointers

 -

Each tree has 2K+1 keys with 2K - child pointers interleaved between the keys. The number - of keys and child pointers actually containing valid - values is determined by the node’s Entries - Used field. If that field is N, then the - B-tree contains N child pointers and - N+1 keys. -

-

Key

 -

The format and size of the key values is determined by - the type of data to which this tree points. The keys are - ordered and are boundaries for the contents of the child - pointer; that is, the key values represented by child - N fall between Key N and Key - N+1. Whether the interval is open or closed on - each end is determined by the type of data to which the - tree points. -

- -

- The format of the key depends on the node type. - For nodes of node type 0 (group nodes), the key is formatted as - follows: - - - - - - -
A single field of - Size of Lengths - bytes:Indicates the byte offset into the local heap - for the first object name in the subtree which - that key describes. -
-

- - -

- For nodes of node type 1 (chunked raw data nodes), the key is - formatted as follows: - - - - - - - - - - - - - - -
Bytes 1-4:Size of chunk in bytes.
Bytes 4-8:Filter mask, a 32-bit bit field indicating which - filters have been skipped for this chunk. Each filter - has an index number in the pipeline (starting at 0, with - the first filter to apply) and if that filter is skipped, - the bit corresponding to its index is set.
(D + 1) 64-bit fields:The offset of the - chunk within the dataset where D is the number - of dimensions of the dataset, and the last value is the - offset within the dataset’s datatype and should - always be zero. For example, if - a chunk in a 3-dimensional dataset begins at the - position [5,5,5], there will be three - such 64-bit values, each with the value of - 5, followed by a 0 value.
-

- -

Child Pointer

 -

The tree node contains file addresses of subtrees or - data depending on the node level. Nodes at Level 0 point - to data addresses, either raw data chunks or group nodes. - Nodes at non-zero levels point to other nodes of the - same B-tree. -

-

For raw data chunk nodes, the child pointer is the address - of a single raw data chunk. For group nodes, the child pointer - points to a symbol table, which contains - information for multiple symbol table entries. -

-
-
- -

- Conceptually, each B-tree node looks like this:

-
- - - - - - - - - - - - - -
key[0] child[0] key[1] child[1] key[2] ... ... key[N-1] child[N-1] key[N]
-
-
- - where child[i] is a pointer to a sub-tree (at a level - above Level 0) or to data (at Level 0). - Each key[i] describes an item stored by the B-tree - (a chunk or an object of a group node). The range of values - represented by child[i] is indicated by key[i] - and key[i+1]. - - -

The following question must next be answered: - “Is the value described by key[i] contained in - child[i-1] or in child[i]?” - The answer depends on the type of tree. - In trees for groups (node type 0), the object described by - key[i] is the greatest object contained in - child[i-1] while in chunk trees (node type 1) the - chunk described by key[i] is the least chunk in - child[i].

- -

That means that key[0] for group trees is sometimes unused; - it points to offset zero in the heap, which is always the - empty string and compares as “less-than” any valid - object name.

- -

And key[N] for chunk trees is sometimes unused; - it contains a chunk offset which compares as “greater-than” - any other chunk offset and has a chunk byte size of zero - to indicate that it is not actually allocated.

- -

- III.A.2. Disk Format: Level 1A2 - Version 2 B-trees

- -

Version 2 (v2) B-trees are “traditional” B-trees - with one major difference. Instead of just using a simple pointer - (or address in the file) to a child of an internal node, the pointer - to the child node contains two additional pieces of information: - the number of records in the child node itself, and the total number - of records in the child node and all its descendants. Storing this - additional information allows fast array-like indexing to locate - the nth record in the B-tree.

- -

The entry into a version 2 B-tree is a header which contains global - information about the structure of the B-tree. The root node - address - field in the header points to the B-tree root node, which is either an - internal or leaf node, depending on the value in the header’s - depth field. An internal node consists of records plus - pointers to further leaf or internal nodes in the tree. A leaf node - consists of solely of records. The format of the records depends on - the B-tree type (stored in the header).

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 B-tree Header -
bytebytebytebyte
Signature
VersionTypeThis space inserted only to align table nicely
Node Size
Record SizeDepth
Split PercentMerge PercentThis space inserted only to align table nicely

Root Node AddressO

Number of Records in Root NodeThis space inserted only to align table nicely

Total Number of Records in B-treeL

Checksum
- - - - - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree Header -
Field NameDescription

Signature

 -

The ASCII character string “BTHD” - is used to indicate the header of a version 2 (v2) B-tree - node. -

-

Version

 -

The version number for this B-tree header. This document - describes version 0. -

-

Type

 -

This field indicates the type of B-tree: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0This B-tree is used for testing only. This - value should not be used for storing - records in actual HDF5 files. -
1This B-tree is used for indexing indirectly accessed, - non-filtered ‘huge’ fractal heap objects. -
2This B-tree is used for indexing indirectly accessed, - filtered ‘huge’ fractal heap objects. -
3This B-tree is used for indexing directly accessed, - non-filtered ‘huge’ fractal heap objects. -
4This B-tree is used for indexing directly accessed, - filtered ‘huge’ fractal heap objects. -
5This B-tree is used for indexing the ‘name’ field for - links in indexed groups. -
6This B-tree is used for indexing the ‘creation order’ - field for links in indexed groups. -
7This B-tree is used for indexing shared object header - messages. -
8This B-tree is used for indexing the ‘name’ field for - indexed attributes. -
9This B-tree is used for indexing the ‘creation order’ - field for indexed attributes. -
10This B-tree is used for indexing chunks of - datasets with no filters and with more than one - dimension of unlimited extent. -
11This B-tree is used for indexing chunks of - datasets with filters and more than one dimension - of unlimited extent. -

-

The format of records for each type is described below.

-

Node Size

 -

This is the size in bytes of all B-tree nodes. -

-

Record Size

 -

This field is the size in bytes of the B-tree record. -

-

Depth

 -

This is the depth of the B-tree. -

-

Split Percent

 -

The percent full that a node needs to increase above before it - is split. -

-

Merge Percent

 -

The percent full that a node needs to be decrease below before it - is split. -

-

Root Node Address

 -

This is the address of the root B-tree node. A B-tree with - no records will have the undefined - address in this field. -

-

Number of Records in Root Node

 -

This is the number of records in the root node. -

-

Total Number of Records in B-tree

 -

This is the total number of records in the entire B-tree. -

-

Checksum

 -

This is the checksum for the B-tree header. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 B-tree Internal Node -
bytebytebytebyte
Signature
VersionTypeRecords 0, 1, 2...N-1 (variable size)

Child Node Pointer 0O


Number of Records N0 for Child - Node 0 (variable size)

Total Number of Records for Child Node 0 - (optional, variable size)

Child Node Pointer 1O


Number of Records N1 for - Child Node 1 (variable size)

Total Number of Records for Child Node 1 - (optional, variable size)
...

Child Node Pointer NO


Number of Records Nn for - Child Node N (variable size)

Total Number of Records for Child Node N - (optional, variable size)
Checksum
- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
-
- - -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree Internal Node -
Field NameDescription

Signature

 -

The ASCII character string “BTIN” is - used to indicate the internal node of a B-tree. -

-

Version

 -

The version number for this B-tree internal node. - This document describes version 0. -

-

Type

 -

This field is the type of the B-tree node. It should always - be the same as the B-tree type in the header. -

-

Records

 -

The size of this field is determined by the number of records - for this node and the record size (from the header). The format - of records depends on the type of B-tree. -

-

Child Node Pointer

 -

This field is the address of the child node pointed to by the - internal node. -

-

Number of Records in Child Node

 -

This is the number of records in the child node pointed to by - the corresponding Node Pointer. -

-

The number of bytes used to store this field is determined by - the maximum possible number of records able to be stored in the - child node. -

-

- The maximum number of records in a child node is computed - in the following way: - -

    -
  • Subtract the fixed size overhead for - the child node (for example, its signature, version, - checksum, and so on and one pointer triplet - of information for the child node (because there is one - more pointer triplet than records in each internal node)) - from the size of nodes for the B-tree.
    • -
  • Divide that result by the size of a record plus the - pointer triplet of information stored to reach each - child node from this node.
    • -
- -

-

- Note that leaf nodes do not encode any - child pointer triplets, so the maximum number of records in a - leaf node is just the node size minus the leaf node overhead, - divided by the record size. -

-

- Also note that the first level of internal nodes above the - leaf nodes do not encode the Total Number of Records in Child - Node value in the child pointer triplets (since it is the - same as the Number of Records in Child Node), so the - maximum number of records in these nodes is computed with the - equation above, but using (Child Pointer, Number of - Records in Child Node) pairs instead of triplets. -

-

- The number of - bytes used to encode this field is the least number of bytes - required to encode the maximum number of records in a child - node value for the child nodes below this level - in the B-tree. -

-

- For example, if the maximum number of child records is - 123, one byte will be used to encode these values in this - node; if the maximum number of child records is - 20000, two bytes will be used to encode these values in this - node; and so on. The maximum number of bytes used to - encode these values is 8 (in other words, an unsigned - 64-bit integer). -

-

Total Number of Records in Child Node

 -

This is the total number of records for the node pointed to by - the corresponding Node Pointer and all its children. - This field exists only in nodes whose depth in the B-tree node - is greater than 1 (in other words, the “twig” - internal nodes, just above leaf nodes, do not store this - field in their child node pointers). -

-

The number of bytes used to store this field is determined by - the maximum possible number of records able to be stored in the - child node and its descendants. -

-

- The maximum possible number of records able to be stored in a - child node and its descendants is computed iteratively, in the - following way: The maximum number of records in a leaf node - is computed, then that value is used to compute the maximum - possible number of records in the first level of internal nodes - above the leaf nodes. Multiplying these two values together - determines the maximum possible number of records in child node - pointers for the level of nodes two levels above leaf nodes. - This process is continued up to any level in the B-tree. -

-

- The number of bytes used to encode this value is computed in - the same way as for the Number of Records in Child Node - field. -

-

Checksum

 -

This is the checksum for this node. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 B-tree Leaf Node -
bytebytebytebyte
Signature
VersionTypeRecord 0, 1, 2...N-1 (variable size)
Checksum
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree Leaf Node -
Field NameDescription

Signature

 -

The ASCII character string “BTLF“ - is used to indicate the leaf node of a version 2 (v2) B-tree. -

-

Version

 -

The version number for this B-tree leaf node. - This document describes version 0. -

-

Type

 -

This field is the type of the B-tree node. It should always - be the same as the B-tree type in the header. -

-

Records

 -

The size of this field is determined by the number of records - for this node and the record size (from the header). The format - of records depends on the type of B-tree. -

-

Checksum

 -

This is the checksum for this node. -

-
-
- -
-
-
-

The record layout for each stored (in other words, non-testing) - B-tree type is as follows:

- -
- - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 B-tree, Type 1 Record Layout - Indirectly - Accessed, Non-filtered, ‘Huge’ Fractal Heap Objects -
bytebytebytebyte

Huge Object AddressO


Huge Object LengthL


Huge Object IDL

- - - - - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree, Type 1 Record Layout - Indirectly - Accessed, Non-filtered, ‘Huge’ Fractal Heap Objects -
Field NameDescription

Huge Object Address

 -

The address of the huge object in the file. -

-

Huge Object Length

 -

The length of the huge object in the file. -

-

Huge Object ID

 -

The heap ID for the huge object. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 B-tree, Type 2 Record Layout - Indirectly - Accessed, Filtered, ‘Huge’ Fractal Heap Objects -
bytebytebytebyte

Filtered Huge Object AddressO


Filtered Huge Object LengthL

Filter Mask

Filtered Huge Object Memory SizeL


Huge Object IDL

- - - - - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree, Type 2 Record Layout - Indirectly - Accessed, Filtered, ‘Huge’ Fractal Heap Objects -
Field NameDescription

Filtered Huge Object Address

 -

The address of the filtered huge object in the file. -

-

Filtered Huge Object Length

 -

The length of the filtered huge object in the file. -

-

Filter Mask

 -

A 32-bit bit field indicating which filters have been skipped for - this chunk. Each filter has an index number in the pipeline - (starting at 0, with the first filter to apply) and if that - filter is skipped, the bit corresponding to its index is set. -

-

Filtered Huge Object Memory Size

 -

The size of the de-filtered huge object in memory. -

-

Huge Object ID

 -

The heap ID for the huge object. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - -
- Layout: Version 2 B-tree, Type 3 Record Layout - Directly - Accessed, Non-filtered, ‘Huge’ Fractal Heap Objects -
bytebytebytebyte

Huge Object AddressO


Huge Object LengthL

- - - - - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree, Type 3 Record Layout - Directly - Accessed, Non-filtered, ‘Huge’ Fractal Heap Objects -
Field NameDescription

Huge Object Address

 -

The address of the huge object in the file. -

-

Huge Object Length

 -

The length of the huge object in the file. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 B-tree, Type 4 Record Layout - Directly - Accessed, Filtered, ‘Huge’ Fractal Heap Objects -
bytebytebytebyte

Filtered Huge Object AddressO


Filtered Huge Object LengthL

Filter Mask

Filtered Huge Object Memory SizeL

- - - - - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree, Type 4 Record Layout - Directly - Accessed, Filtered, ‘Huge’ Fractal Heap Objects -
Field NameDescription

Filtered Huge Object Address

 -

The address of the filtered huge object in the file. -

-

Filtered Huge Object Length

 -

The length of the filtered huge object in the file. -

-

Filter Mask

 -

A 32-bit bit field indicating which filters have been skipped for - this chunk. Each filter has an index number in the pipeline - (starting at 0, with the first filter to apply) and if that - filter is skipped, the bit corresponding to its index is set. -

-

Filtered Huge Object Memory Size

 -

The size of the de-filtered huge object in memory. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 B-tree, Type 5 Record Layout - Link Name - for Indexed Group -
bytebytebytebyte
Hash of Name
ID (bytes 1-4)
ID (bytes 5-7)
-
- -
-
- - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree, Type 5 Record Layout - Link Name - for Indexed Group -
Field NameDescription

Hash

 -

This field is hash value of the name for the link. The hash - value is the Jenkins’ lookup3 checksum algorithm applied to - the link’s name. -

-

ID

 -

This is a 7-byte sequence of bytes and is the heap ID for the - link record in the group’s fractal heap.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 B-tree, Type 6 Record Layout - Creation - Order for Indexed Group -
bytebytebytebyte

Creation Order - (8 bytes)

ID (bytes 1-4)
ID (bytes 5-7)
-
- -
-
- - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree, Type 6 Record Layout - Creation - Order for Indexed Group -
Field NameDescription

Creation Order

 -

This field is the creation order value for the link. -

-

ID

 -

This is a 7-byte sequence of bytes and is the heap ID for the - link record in the group’s fractal heap.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 B-tree, Type 7 Record Layout - Shared - Object Header Messages (Sub-type 0 - Message in Heap) -
bytebytebytebyte
Message LocationThis space inserted only to align table nicely
Hash
Reference Count

Heap ID (8 bytes)

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree, Type 7 Record Layout - Shared - Object Header Messages (Sub-type 0 - Message in Heap) -
Field NameDescription

Message Location

 -

This field Indicates the location where the message is stored: - - - - - - - - - - - - - -
ValueDescription
0Shared message is stored in shared message index heap. -
1Shared message is stored in object header. -

-

Hash

 -

This field is hash value of the shared message. The hash - value is the Jenkins’ lookup3 checksum algorithm applied to - the shared message.

-

Reference Count

 -

The number of objects which reference this message.

-

Heap ID

 -

This is an 8-byte sequence of bytes and is the heap ID for the - shared message in the shared message index’s fractal heap.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 B-tree, Type 7 Record Layout - Shared - Object Header Messages (Sub-type 1 - Message in Object Header) -
bytebytebytebyte
Message LocationThis space inserted only to align table nicely
Hash
Reserved (zero)Message TypeObject Header Index

Object Header AddressO

- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree, Type 7 Record Layout - Shared - Object Header Messages (Sub-type 1 - Message in Object Header) -
Field NameDescription

Message Location

 -

This field Indicates the location where the message is stored: - - - - - - - - - - - - - -
ValueDescription
0Shared message is stored in shared message index heap. -
1Shared message is stored in object header. -

-

Hash

 -

This field is hash value of the shared message. The hash - value is the Jenkins’ lookup3 checksum algorithm applied to - the shared message.

-

Message Type

 -

The object header message type of the shared message.

-

Object Header Index

 -

This field indicates that the shared message is the nth message - of its type in the specified object header.

-

Object Header Address

 -

The address of the object header containing the shared message.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 B-tree, Type 8 Record Layout - Attribute - Name for Indexed Attributes -
bytebytebytebyte

Heap ID (8 bytes)

Message FlagsThis space inserted only to align table nicely
Creation Order
Hash of Name
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree, Type 8 Record Layout - Attribute - Name for Indexed Attributes -
Field NameDescription

Heap ID

 -

This is an 8-byte sequence of bytes and is the heap ID for the - attribute in the object’s attribute fractal heap.

-

Message Flags

The object header message flags for the attribute message.

-

Creation Order

 -

This field is the creation order value for the attribute. -

-

Hash

 -

This field is hash value of the name for the attribute. The hash - value is the Jenkins’ lookup3 checksum algorithm applied to - the attribute’s name. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 B-tree, Type 9 Record Layout - Creation - Order for Indexed Attributes -
bytebytebytebyte

Heap ID (8 bytes)

Message Flags - This space inserted only to align table nicely
Creation Order
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree, Type 9 Record Layout - Creation - Order for Indexed Attributes -
Field NameDescription

Heap ID

 -

This is an 8-byte sequence of bytes and is the heap ID for the - attribute in the object’s attribute fractal heap.

-

Message Flags

 -

The object header message flags for the attribute message.

-

Creation Order

 -

This field is the creation order value for the attribute. -

-
-
- -
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - Layout: Version 2 B-tree, Type 10 Record Layout - - Non-filtered Dataset Chunks -
bytebytebytebyte

AddressO


Dimension 0 Scaled Offset - (8 bytes)


Dimension 1 Scaled Offset - (8 bytes)


...


Dimension #n Scaled Offset - (8 bytes)

- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree, Type 10 Record Layout - - Non-filtered Dataset Chunks -
Field NameDescription

Address

 -

This field is the address of the dataset chunk in the file.

-

Dimension #n Scaled Offset

 -

This field is the scaled offset of the chunk within the - dataset. n is the number of dimensions for the - dataset. The first scaled offset stored in the list is for - the slowest changing dimension, and the last scaled offset - stored is for the fastest changing dimension. Scaled offset - is calculated by dividing the chunk dimension sizes into - the chunk offsets.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - Layout: Version 2 B-tree, Type 11 Record Layout - Filtered - Dataset Chunks -
bytebytebytebyte

AddressO


Chunk Size - (variable size; at most 8 bytes)

Filter Mask

Dimension 0 Scaled Offset - (8 bytes)


Dimension 1 Scaled Offset - (8 bytes)


...


Dimension #n Scaled Offset - (8 bytes)

- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 B-tree, Type 11 Record Layout - Filtered - Dataset Chunks -
Field NameDescription

Address

 -

This field is the address of the dataset chunk in the file.

-

Chunk Size

 -

This field is the size of the dataset chunk in bytes.

-

Filter Mask

 -

This field is the filter mask which indicates the filter - to skip for the dataset chunk. Each filter has an index - number in the pipeline and if that filter is skipped, - the bit corresponding to its index is set.

-

Dimension #n Scaled Offset

 -

This field is the scaled offset of the chunk within - the dataset. n is the number of dimensions for - the dataset. The first scaled offset stored in the list - is for the slowest changing dimension, and the last scaled - offset stored is for the fastest changing dimension.

-
-
- -

- III.B. Disk Format: Level 1B - Group Symbol Table Nodes

- -

A group is an object internal to the file that allows - arbitrary nesting of objects within the file (including other - groups). A group maps a set of link names in the group to a set - of relative file addresses of objects in the file. Certain metadata - for an object to which the group points can be cached in the - group’s symbol table entry in addition to being in the - object’s header.

- -

An HDF5 object name space can be stored hierarchically by - partitioning the name into components and storing each - component as a link in a group. The link for a - non-ultimate component points to the group containing - the next component. The link for the last - component points to the object being named.

- -

One implementation of a group is a collection of symbol table - nodes indexed by a B-tree. Each symbol table node contains entries - for one or more links. If an attempt is made to add a link to an - already full symbol table node containing 2K entries, then - the node is split and one node contains K symbols and the - other contains K+1 symbols.

- -
- - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Symbol Table Node (A Leaf of a B-tree) -
bytebytebytebyte
Signature
Version NumberReserved (zero)Number of Symbols


Group Entries


-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Symbol Table Node (A Leaf of a B-tree) -
Field NameDescription

Signature

 -

The ASCII character string “SNOD” is - used to indicate the - beginning of a symbol table node. This gives file - consistency checking utilities a better chance of - reconstructing a damaged file. -

-

Version Number

 -

The version number for the symbol table node. This - document describes version 1. (There is no version ‘0’ - of the symbol table node) -

-

Number of Entries

 -

Although all symbol table nodes have the same length, - most contain fewer than the maximum possible number of - link entries. This field indicates how many entries - contain valid data. The valid entries are packed at the - beginning of the symbol table node while the remaining - entries contain undefined values. -

-

Symbol Table Entries

 -

Each link has an entry in the symbol table node. - The format of the entry is described below. - There are 2K entries in each group node, where - K is the “Group Leaf Node K” value from the - superblock. -

-
-
- -

- III.C. Disk Format: Level 1C - Symbol Table Entry

- -

Each symbol table entry in a symbol table node is designed - to allow for very fast browsing of stored objects. - Toward that design goal, the symbol table entries - include space for caching certain constant metadata from the - object header.

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Symbol Table Entry -
bytebytebytebyte

Link Name OffsetO


Object Header AddressO

Cache Type
Reserved (zero)


Scratch-pad Space - (16 bytes)


- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Symbol Table Entry -
Field NameDescription

Link Name Offset

 -

This is the byte offset into the group’s local - heap for the name of the link. The name is null - terminated. -

-

Object Header Address

 -

Every object has an object header which serves as a - permanent location for the object’s metadata. In addition - to appearing in the object header, some of the object’s metadata - can be cached in the scratch-pad space. -

-

Cache Type

 -

The cache type is determined from the object header. - It also determines the format for the scratch-pad space: - - - - - - - - - - - - - - - - - - -
TypeDescription
0No data is cached by the group entry. This - is guaranteed to be the case when an object header - has a link count greater than one. -
1Group object header metadata is cached in the - scratch-pad space. This implies that the symbol table - entry refers to another group. -
2The entry is a symbolic link. The first four bytes - of the scratch-pad space are the offset into the local - heap for the link value. The object header address - will be undefined. -

- -

Reserved

 -

These four bytes are present so that the scratch-pad - space is aligned on an eight-byte boundary. They are - always set to zero. -

-

Scratch-pad Space

 -

This space is used for different purposes, depending - on the value of the Cache Type field. Any metadata - about an object represented in the scratch-pad - space is duplicated in the object header for that - object. -

-

- Furthermore, no data is cached in the group - entry scratch-pad space if the object header for - the object has a link count greater than one. -

-
-
- -

Format of the Scratch-pad Space

- -

The symbol table entry scratch-pad space is formatted - according to the value in the Cache Type field.

- -

If the Cache Type field contains the value zero - (0) then no information is - stored in the scratch-pad space.

- -

If the Cache Type field contains the value one - (1), then the scratch-pad space - contains cached metadata for another object header - in the following format:

- -
- - - - - - - - - - - - - - - - - -
- Layout: Object Header Scratch-pad Format -
bytebytebytebyte

Address of B-treeO


Address of Name HeapO

- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - -
- Fields: Object Header Scratch-pad Format -
Field NameDescription

Address of B-tree

 -

This is the file address for the root of the - group’s B-tree. -

-

Address of Name Heap

 -

This is the file address for the group’s local - heap, in which are stored the group’s symbol names. -

-
-
- - -
-
-
-

If the Cache Type field contains the value two - (2), then the scratch-pad space - contains cached metadata for a symbolic link - in the following format:

- -
- - - - - - - - - - - - - -
- Layout: Symbolic Link Scratch-pad Format -
bytebytebytebyte
Offset to Link Value
-
- -
-
- - - - - - - - - - - -
- Fields: Symbolic Link Scratch-pad Format -
Field NameDescription

Offset to Link Value

 -

The value of a symbolic link (that is, the name of the - thing to which it points) is stored in the local heap. - This field is the 4-byte offset into the local heap for - the start of the link value, which is null terminated. -

-
-
- -

- III.D. Disk Format: Level 1D - Local Heaps

- -

A local heap is a collection of small pieces of data that are particular - to a single object in the HDF5 file. Objects can be - inserted and removed from the heap at any time. - The address of a heap does not change once the heap is created. - For example, a group stores addresses of objects in symbol table nodes - with the names of links stored in the group’s local heap. -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Local Heap -
bytebytebytebyte
Signature
VersionReserved (zero)

Data Segment SizeL


Offset to Head of Free-listL


Address of Data SegmentO

- - - - - - - - -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Local Heap -
Field NameDescription

Signature

 -

The ASCII character string “HEAP” - is used to indicate the - beginning of a heap. This gives file consistency - checking utilities a better chance of reconstructing a - damaged file. -

-

Version

 -

Each local heap has its own version number so that new - heaps can be added to old files. This document - describes version zero (0) of the local heap. -

-

Data Segment Size

 -

The total amount of disk memory allocated for the heap - data. This may be larger than the amount of space - required by the objects stored in the heap. The extra - unused space in the heap holds a linked list of free blocks. -

-

Offset to Head of Free-list

 -

This is the offset within the heap data segment of the - first free block (or the - undefined address if there is no - free block). The free block contains - Size of Lengths bytes that - are the offset of the next free block (or the - value ‘1’ if this is the - last free block) followed by Size of Lengths bytes that store - the size of this free block. The size of the free block includes - the space used to store the offset of the next free block and - the size of the current block, making the minimum size of a free - block 2 * Size of Lengths. -

-

Address of Data Segment

 -

The data segment originally starts immediately after - the heap header, but if the data segment must grow as a - result of adding more objects, then the data segment may - be relocated, in its entirety, to another part of the - file. -

-
-
- -

Objects within a local heap should be aligned on an 8-byte boundary.

- -

- III.E. Disk Format: Level 1E - Global Heap

- -

Each HDF5 file has a global heap which stores various types of - information which is typically shared between datasets. The - global heap was designed to satisfy these goals:

- -
    -
  1. Repeated access to a heap object must be efficient without - resulting in repeated file I/O requests. Since global heap - objects will typically be shared among several datasets, it is - probable that the object will be accessed repeatedly.
    2. -
  2. Collections of related global heap objects should result in - fewer and larger I/O requests. For instance, a dataset of - object references will have a global heap object for each - reference. Reading the entire set of object references - should result in a few large I/O requests instead of one small - I/O request for each reference.
    3. -
  3. It should be possible to remove objects from the global heap - and the resulting file hole should be eligible to be reclaimed - for other uses.
    4. -
- - -

The implementation of the heap makes use of the memory management - already available at the file level and combines that with a new - object called a collection to achieve goal B. The global heap - is the set of all collections. Each global heap object belongs to - exactly one collection, and each collection contains one or more global - heap objects. For the purposes of disk I/O and caching, a collection is - treated as an atomic object, addressing goal A. -

- -

When a global heap object is deleted from a collection (which - occurs when its reference count falls to zero), objects located - after the deleted object in the collection are packed down toward - the beginning of the collection, and the collection’s - global heap object 0 is created (if possible), or its size is - increased to account for the recently freed space. There are - no gaps between objects in each collection, with the possible - exception of the final space in the collection, if it is not - large enough to hold the header for the collection’s - global heap object 0. These features address goal C. -

- -

The HDF5 Library creates global heap collections as needed, so there may - be multiple collections throughout the file. The set of all of them is - abstractly called the “global heap”, although they do not actually link - to each other, and there is no global place in the file where you can - discover all of the collections. The collections are found simply by - finding a reference to one through another object in the file. For - example, data of variable-length datatype elements is stored in the - global heap and is accessed via a global heap ID. The format for - global heap IDs is described at the end of this section. -

- -

For more information on global heaps for virtual datasets, see - “Disk Format: Level 1F - Global Heap - Block for Virtual Datasets.”

-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: A Global Heap Collection -
bytebytebytebyte
Signature
VersionReserved (zero)

Collection SizeL


Global Heap Object 1


Global Heap Object 2


...


Global Heap Object N


Global Heap Object 0 (free space)

- - - - - -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: A Global Heap Collection -
Field NameDescription

Signature

 -

The ASCII character string “GCOL” - is used to indicate the - beginning of a collection. This gives file consistency - checking utilities a better chance of reconstructing a - damaged file. -

-

Version

 -

Each collection has its own version number so that new - collections can be added to old files. This document - describes version one (1) of the collections (there is no - version zero (0)). -

-

Collection Size

 -

This is the size in bytes of the entire collection - including this field. The default (and minimum) - collection size is 4096 bytes which is a typical file - system block size. This allows for 127 16-byte heap - objects plus their overhead (the collection header of 16 bytes - and the 16 bytes of information about each heap object). -

-

Global Heap Object 1 through N

 -

The objects are stored in any order with no - intervening unused space. -

-

Global Heap Object 0

 -

Global Heap Object 0 (zero), when present, represents the free - space in the collection. Free space always appears at the end of - the collection. If the free space is too small to store the header - for Object 0 (described below) then the header is implied and is not - written. -

- The field Object Size for Object 0 indicates the - amount of possible free space in the collection including the 16-byte - header size of Object 0. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Global Heap Object -
bytebytebytebyte
Heap Object IndexReference Count
Reserved (zero)

Object SizeL


Object Data

- - - - - -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Global Heap Object -
Field NameDescription

Heap Object Index

 -

Each object has a unique identification number within a - collection. The identification numbers are chosen so that - new objects have the smallest value possible with the - exception that the identifier 0 always refers to the - object which represents all free space within the - collection. -

-

Reference Count

 -

All heap objects have a reference count field. An - object which is referenced from some other part of the - file will have a positive reference count. The reference - count for Object 0 is always zero. -

-

Reserved

 -

Zero padding to align next field on an 8-byte boundary. -

-

Object Size

 -

This is the size of the object data stored for the object. - The actual storage space allocated for the object data is rounded - up to a multiple of eight. -

-

Object Data

 -

The object data is treated as a one-dimensional array - of bytes to be interpreted by the caller. -

-
- -
- -
-
-
-

- - The format for the ID used to locate an object in the global heap is - described here:

- -
- - - - - - - - - - - - - - - - - -
- Layout: Global Heap ID -
bytebytebytebyte

Collection AddressO

Object Index
- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - -
- Fields: Global Heap ID -
Field NameDescription

Collection Address

 -

This field is the address of the global heap collection - where the data object is stored. -

-

ID

 -

This field is the index of the data object within the - global heap collection. -

-
-
- - - -

III.F. Disk Format: Level 1F - Global - Heap Block for Virtual Datasets

- -

The layout for the global heap block used with virtual datasets is - described below. For more information on global heaps, see - “Disk Format: Level 1E - Global Heap.”

- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Global Heap Block for Virtual Dataset -
bytebytebytebyte
VersionThis space inserted - only to align table nicely

Num EntriesL


Source Filename #1 (variable size)


Source Dataset #1 (variable - size)


Source Selection #1 (variable - size)


Virtual Selection #1 (variable - size)

.
.
.

Source Filename #n (variable - size)


Source Dataset #n (variable - size)


Source Selection #n (variable - size)


Virtual Selection #n (variable - size)

Checksum
- - - - - -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Global Heap Block for Virtual Dataset -
Field NameDescription

Version

 -

The version number for the block; the value is 0.

-

Num Entries

The number of entries in the block.

-

Source Filename #n

 -

The source file name where the source dataset is located. -

-

Source Dataset #n

The source dataset name that is mapped to the - virtual dataset.

Source Selection #n

 -

The dataspace selection in the - source dataset that is mapped to the virtual selection. -

-

Virtual Selection #n

 -

This is the dataspace selection in the virtual dataset that is - mapped to the source selection. -

-

Checksum

 -

This is the checksum for the block.

-
-
-
- -

- III.G. Disk Format: Level 1G - Fractal Heap

- -

- Each fractal heap consists of a header and zero or more direct and - indirect blocks (described below). The header contains general - information as well as - initialization parameters for the doubling table. The Address - of Root Block field in the header points to the first direct or - indirect block in the heap. -

- -

- Fractal heaps are based on a data structure called a doubling - table. A doubling table provides a mechanism for quickly - extending an array-like data structure that minimizes the number of - empty blocks in the heap, while retaining very fast lookup of any - element within the array. More information on fractal heaps and - doubling tables can be found in the RFC - “Private - Heaps in HDF5.” -

- -

- The fractal heap implements the doubling table structure with - indirect and direct blocks. - Indirect blocks in the heap do not actually contain data for - objects in the heap, their “size” is abstract - - they represent the indexing structure for locating the - direct blocks in the doubling table. - Direct blocks - contain the actual data for objects stored in the heap. -

- -

- All indirect blocks have a constant number of block entries in each - row, called the width of the doubling table - (see Table Width field in the header). - - The number - of rows for each indirect block in the heap is determined by the - size of the block that the indirect block represents in the - doubling table (calculation of this is shown below) and is - constant, except for the “root” - indirect block, which expands and shrinks its number of rows as - needed. -

- -

- Blocks in the first two rows of an indirect block - are Starting Block Size number of bytes in size. - For example, if the row width of the doubling table is 4, - then the first eight block entries in the - indirect block are Starting Block Size number of bytes in size. - The blocks in each subsequent row are twice the size of - the blocks in the previous row. In other words, blocks in - the third row are twice the Starting Block Size, - blocks in the fourth row are four times the - Starting Block Size, and so on. Entries for - blocks up to the Maximum Direct Block Size point to - direct blocks, and entries for blocks greater than that size - point to further indirect blocks (which have their own - entries for direct and indirect blocks). - Starting Block Size and - Maximum Direct Block Size are fields - stored in the header. -

- -

- The number of rows of blocks, nrows, in an - indirect block is calculated by the following expression: -

- nrows = (log2(block_size) - - log2(<Starting Block Size>)) + 1 -

-where block_size is the size of the block that the indirect block -represents in the doubling table. -For example, to represent a block with block_size equals to 1024, -and Starting Block Size equals to 256, -three rows are needed. -

- The maximum number of rows of direct blocks, max_dblock_rows, - in any indirect block of a fractal heap is given by the - following expression: -

- max_dblock_rows = - (log2(<Maximum Direct Block Size>) - - log2(<Starting Block Size>)) + 2 -

-

- Using the computed values for nrows and - max_dblock_rows, along with the width of the - doubling table, the number of direct and indirect block entries - (K and N in the indirect block description, below) - in an indirect block can be computed: -

- K = MIN(nrows, max_dblock_rows) * - <Table Width> - -

- If nrows is less than or equal to max_dblock_rows, - N is 0. Otherwise, N is simply computed: -

- N = K - (max_dblock_rows * - <Table Width>) -

- -

- The size of indirect blocks on disk is determined by the number - of rows in the indirect block (computed above). The size of direct - blocks on disk is exactly the size of the block in the doubling - table. -

-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Fractal Heap Header -
bytebytebytebyte
Signature
VersionThis space inserted only to align table nicely
Heap ID LengthI/O Filters’ Encoded Length
FlagsThis space inserted only to align table nicely
Maximum Size of Managed Objects

Next Huge Object IDL


v2 B-tree Address of Huge ObjectsO


Amount of Free Space in Managed BlocksL


Address of Managed Block Free Space ManagerO


Amount of Managed Space in HeapL


Amount of Allocated Managed Space in HeapL


Offset of Direct Block Allocation Iterator in Managed SpaceL


Number of Managed Objects in HeapL


Size of Huge Objects in HeapL


Number of Huge Objects in HeapL


Size of Tiny Objects in HeapL


Number of Tiny Objects in HeapL

Table WidthThis space inserted only to align table nicely

Starting Block SizeL


Maximum Direct Block SizeL

Maximum Heap SizeStarting # of Rows in Root Indirect Block

Address of Root BlockO

Current # of Rows in Root Indirect BlockThis space inserted only to align table nicely

Size of Filtered Root Direct Block (optional)L

I/O Filter Mask (optional)
I/O Filter Information (optional, variable size)
Checksum
- - - - - - - - -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Fractal Heap Header -
Field NameDescription

Signature

 -

The ASCII character string “FRHP” - is used to indicate the - beginning of a fractal heap header. This gives file consistency - checking utilities a better chance of reconstructing a - damaged file. -

-

Version

 -

This document describes version 0.

-

Heap ID Length

 -

This is the length in bytes of heap object IDs for this heap.

-

I/O Filters’ Encoded Length

 -

This is the size in bytes of the encoded I/O Filter Information. -

-

Flags

 -

This field is the heap status flag and is a bit field - indicating additional information about the fractal heap. - - - - - - - - - - - - - - - - - - -
Bit(s)Description
0If set, the ID value to use for huge object has wrapped - around. If the value for the Next Huge Object ID - has wrapped around, each new huge object inserted into the - heap will require a search for an ID value. -
1If set, the direct blocks in the heap are checksummed. -
2-7Reserved

- -

Maximum Size of Managed Objects

 -

This is the maximum size of managed objects allowed in the heap. - Objects greater than this this are ‘huge’ objects and will be - stored in the file directly, rather than in a direct block for - the heap. -

-

Next Huge Object ID

 -

This is the next ID value to use for a huge object in the heap. -

-

v2 B-tree Address of Huge Objects

 -

This is the address of the v2 B-tree - used to track huge objects in the heap. The type of records - stored in the v2 B-tree will - be determined by whether the address and length of a huge object - can fit into a heap ID (if yes, it is a “directly” accessed - huge object) and whether there is a filter used on objects - in the heap. -

-

Amount of Free Space in Managed Blocks

 -

This is the total amount of free space in managed direct blocks - (in bytes). -

-

Address of Managed Block Free Space Manager

 -

This is the address of the - Free-space Manager for - managed blocks. -

-

Amount of Managed Space in Heap

 -

This is the total amount of managed space in the heap (in bytes), - essentially the upper bound of the heap’s linear address space. -

-

Amount of Allocated Managed Space in Heap

 -

This is the total amount of managed space (in bytes) actually - allocated in - the heap. This can be less than the Amount of Managed Space - in Heap field, if some direct blocks in the heap’s linear - address space are not allocated. -

-

Offset of Direct Block Allocation Iterator in Managed Space

 -

This is the linear heap offset where the next direct - block should be allocated at (in bytes). This may be less than - the Amount of Managed Space in Heap value because the - heap’s address space is increased by a “row” of direct blocks - at a time, rather than by single direct block increments. -

-

Number of Managed Objects in Heap

 -

This is the number of managed objects in the heap. -

-

Size of Huge Objects in Heap

 -

This is the total size of huge objects in the heap (in bytes). -

-

Number of Huge Objects in Heap

 -

This is the number of huge objects in the heap. -

-

Size of Tiny Objects in Heap

 -

This is the total size of tiny objects that are packed in heap - IDs (in bytes). -

-

Number of Tiny Objects in Heap

 -

This is the number of tiny objects that are packed in heap IDs. -

-

Table Width

 -

This is the number of columns in the doubling table for managed - blocks. This value must be a power of two. -

-

Starting Block Size

 -

This is the starting block size to use in the doubling table for - managed blocks (in bytes). This value must be a power of two. -

-

Maximum Direct Block Size

 -

This is the maximum size allowed for a managed direct block. - Objects inserted into the heap that are larger than this value - (less the number of bytes of direct block prefix/suffix) - are stored as ‘huge’ objects. This value must be a power of - two. -

-

Maximum Heap Size

 -

This is the maximum size of the heap’s linear address space for - managed objects (in bytes). The value stored is the log2 of - the actual value, that is: the number of bits of the address space. - ‘Huge’ and ‘tiny’ objects are not counted in this value, since - they do not store objects in the linear address space of the - heap. -

-

Starting # of Rows in Root Indirect Block

 -

This is the starting number of rows for the root indirect block. - A value of 0 indicates that the root indirect block will have - the maximum number of rows needed to address the heap’s Maximum - Heap Size. -

-

Address of Root Block

 -

This is the address of the root block for the heap. It can - be the undefined address if - there is no data in the heap. It either points to a direct - block (if the Current # of Rows in the Root Indirect - Block value is 0), or an indirect block. -

-

Current # of Rows in Root Indirect Block

 -

This is the current number of rows in the root indirect block. - A value of 0 indicates that Address of Root Block - points to direct block instead of indirect block. -

-

Size of Filtered Root Direct Block

 -

This is the size of the root direct block, if filters are - applied to heap objects (in bytes). This field is only - stored in the header if the I/O Filters’ Encoded Length - is greater than 0. -

-

I/O Filter Mask

 -

This is the filter mask for the root direct block, if filters - are applied to heap objects. This mask has the same format as - that used for the filter mask in chunked raw data records in a - v1 B-tree. - This field is only - stored in the header if the I/O Filters’ Encoded Length - is greater than 0. -

-

I/O Filter Information

 -

This is the I/O filter information encoding direct blocks and - huge objects, if filters are applied to heap objects. This - field is encoded as a Filter Pipeline - message. - The size of this field is determined by I/O Filters’ - Encoded Length. -

-

Checksum

 -

This is the checksum for the header.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Fractal Heap Direct Block -
bytebytebytebyte
Signature
VersionThis space inserted only to align table nicely

Heap Header AddressO

Block Offset (variable size)
Checksum (optional)

Object Data (variable size)

- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Fractal Heap Direct Block -
Field NameDescription

Signature

 -

The ASCII character string “FHDB” - is used to indicate the - beginning of a fractal heap direct block. This gives file consistency - checking utilities a better chance of reconstructing a - damaged file. -

-

Version

 -

This document describes version 0.

-

Heap Header Address

 -

This is the address for the fractal heap header that this - block belongs to. This field is principally used for file - integrity checking. -

-

Block Offset

 -

This is the offset of the block within the fractal heap’s - address space (in bytes). The number of bytes used to encode - this field is the Maximum Heap Size (in the heap’s - header) divided by 8 and rounded up to the next highest integer, - for values that are not a multiple of 8. This value is - principally used for file integrity checking. -

-

Checksum

 -

This is the checksum for the direct block.

-

This field is only present if bit 1 of Flags in the - heap’s header is set.

-

Object Data

 -

This section of the direct block stores the actual data for - objects in the heap. The size of this section is determined by - the direct block’s size minus the size of the other fields - stored in the direct block (for example, the Signature, - Version, and others including the Checksum if it is - present). -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Fractal Heap Indirect Block -
bytebytebytebyte
Signature
VersionThis space inserted only to align table nicely

Heap Header AddressO

Block Offset (variable size)

Child Direct Block #0 AddressO


Size of Filtered Direct Block #0 (optional) L

Filter Mask for Direct Block #0 (optional)

Child Direct Block #1 AddressO


Size of Filtered Direct Block #1 (optional)L

Filter Mask for Direct Block #1 (optional)
...

Child Direct Block #K-1 AddressO


Size of Filtered Direct Block #K-1 (optional)L

Filter Mask for Direct Block #K-1 (optional)

Child Indirect Block #0 AddressO


Child Indirect Block #1 AddressO

...

Child Indirect Block #N-1 AddressO

Checksum
- - - - - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Fractal Heap Indirect Block -
Field NameDescription

Signature

 -

The ASCII character string “FHIB” is used to - indicate the beginning of a fractal heap indirect block. This - gives file consistency checking utilities a better chance of - reconstructing a damaged file. -

-

Version

 -

This document describes version 0.

-

Heap Header Address

 -

This is the address for the fractal heap header that this - block belongs to. This field is principally used for file - integrity checking. -

-

Block Offset

 -

This is the offset of the block within the fractal heap’s - address space (in bytes). The number of bytes used to encode - this field is the Maximum Heap Size (in the heap’s - header) divided by 8 and rounded up to the next highest integer, - for values that are not a multiple of 8. This value is - principally used for file integrity checking. -

-

Child Direct Block #K Address

 -

This field is the address of the child direct block. - The size of the [uncompressed] direct block can be computed by - its offset in the heap’s linear address space. -

-

Size of Filtered Direct Block #K

 -

This is the size of the child direct block after passing through - the I/O filters defined for this heap (in bytes). If no I/O - filters are present for this heap, this field is not present. -

-

Filter Mask for Direct Block #K

 -

This is the I/O filter mask for the filtered direct block. - This mask has the same format as that used for the filter mask - in chunked raw data records in a v1 B-tree. - If no I/O filters are present for this heap, this field is not - present. -

-

Child Indirect Block #N Address

 -

This field is the address of the child indirect block. - The size of the indirect block can be computed by - its offset in the heap’s linear address space. -

-

Checksum

 -

This is the checksum for the indirect block.

-
- -
- -
-

An object in the fractal heap is identified by means of a fractal heap ID, - which encodes information to locate the object in the heap. - Currently, the fractal heap stores an object in one of three ways, - depending on the object’s size:

- -
- - - - - - - - - - - - - - - - - - - - -
TypeDescription
Tiny -

When an object is small enough to be encoded in the - heap ID, the object’s data is embedded in the fractal - heap ID itself. There are two sub-types for this type of - object: normal and extended. The sub-type for tiny heap - IDs depends on whether the heap ID is large enough to - store objects greater than 16 bytes or not. If the - heap ID length is 18 bytes or smaller, the - ‘normal’ tiny heap ID form is used. If the - heap ID length is greater than 18 bytes in length, the - “extended” form is used. See the format - description below for both sub-types. -

-
Huge -

When the size of an object is larger than Maximum Size of - Managed Objects in the Fractal Heap Header, the - object’s data is stored on its own in the file and the object - is tracked/indexed via a version 2 B-tree. All huge objects - for a particular fractal heap use the same v2 B-tree. All huge - objects for a particular fractal heap use the same format for - their huge object IDs. -

- -

Depending on whether the IDs for a heap are large enough to hold - the object’s retrieval information and whether I/O pipeline filters - are applied to the heap’s objects, 4 sub-types are derived for - huge object IDs for this heap:

- -
- - - - - - - - - - - - - - - - - - - - - - - - - -
Sub-typeDescription
Directly accessed, non-filtered -

The object’s address and length are embedded in the - fractal heap ID itself and the object is directly accessed - from them. This allows the object to be accessed without - resorting to the B-tree. -

-
Directly accessed, filtered -

The filtered object’s address, length, filter mask and - de-filtered size are embedded in the fractal heap ID itself - and the object is accessed directly with them. This allows - the object to be accessed without resorting to the B-tree. -

-
Indirectly accessed, non-filtered -

The object is located by using a B-tree key embedded in - the fractal heap ID to retrieve the address and length from - the version 2 B-tree for huge objects. Then, the address - and length are used to access the object. -

-
Indirectly accessed, filtered -

The object is located by using a B-tree key embedded in - the fractal heap ID to retrieve the filtered object’s - address, length, filter mask and de-filtered size from the - version 2 B-tree for huge objects. Then, this information - is used to access the object. -

-
-
- -
Managed -

When the size of an object does not meet the above two - conditions, the object is stored and managed via the direct and - indirect blocks based on the doubling table. -

-
-
- - -
-

The specific format for each type of heap ID is described below: -

- -
- - - - - - - - - - - - - - - - - - - -
- Layout: Fractal Heap ID for Tiny Objects (Sub-type 1 - - ‘Normal’) -
bytebytebytebyte
Version, Type, and LengthThis space inserted only to align table nicely

Data (variable size)
-
- -
-
- - - - - - - - - - - - - - - - - -
- Fields: Fractal Heap ID for Tiny Objects (Sub-type 1 - - ‘Normal’) -
Field NameDescription

Version, Type, and Length

 -

This is a bit field with the following definition: - - - - - - - - - - - - - - - - - - -
BitDescription
6-7The current version of ID format. This document - describes version 0. -
4-5The ID type. Tiny objects have a value of 2. -
0-3The length of the tiny object. The value stored - is one less than the actual length (since zero-length - objects are not allowed to be stored in the heap). - For example, an object of actual length 1 has an - encoded length of 0, an object of actual length 2 - has an encoded length of 1, and so on. -

- -

Data

 -

This is the data for the object. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - -
- Layout: Fractal Heap ID for Tiny Objects (Sub-type 2 - - ‘Extended’) -
bytebytebytebyte
Version, Type, and LengthExtended LengthThis space inserted only to align table nicely
Data (variable size)
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Fields: Fractal Heap ID for Tiny Objects (Sub-type 2 - - ‘Extended’) -
Field NameDescription

Version, Type, and Length

 -

This is a bit field with the following definition: - - - - - - - - - - - - - - - - - - -
BitDescription
6-7The current version of ID format. This document - describes version 0. -
4-5The ID type. Tiny objects have a value of 2. -
0-3These 4 bits, together with the next byte, form an - unsigned 12-bit integer for holding the length of the - object. These 4-bits are bits 8-11 of the 12-bit integer. - See description for the Extended Length field below. -

- -

Extended Length

 -

This byte, together with the 4 bits in the previous byte, - forms an unsigned 12-bit integer for holding the length of - the tiny object. These 8 bits are bits 0-7 of the 12-bit - integer formed. The value stored is one less than the actual - length (since zero-length objects are not allowed to be - stored in the heap). For example, an object of actual length - 1 has an encoded length of 0, an object of actual length - 2 has an encoded length of 1, and so on. -

-

Data

 -

This is the data for the object. -

-
-
- - -
-
-
-
- - - - - - - - - - - - - - - - - - - -
- Layout: Fractal Heap ID for Huge Objects (Sub-types 1 and 2): - Indirectly Accessed, Non-filtered/Filtered -
bytebytebytebyte
Version and TypeThis space inserted - only to align table nicely

v2 B-tree KeyL (variable size)

- - - - - -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
-
- -
-
- - - - - - - - - - - - - - - - - -
- Fields: Fractal Heap ID for Huge Objects (Sub-types 1 and 2): - Indirectly Accessed, Non-filtered/Filtered -
Field NameDescription

Version and Type

 -

This is a bit field with the following definition: - - - - - - - - - - - - - - - - - - -
BitDescription
6-7The current version of ID format. This document - describes version 0. -
4-5The ID type. Huge objects have a value of 1. -
0-3Reserved. -

- -

v2 B-tree Key

This field is the B-tree key for retrieving the information - from the version 2 B-tree for huge objects needed to access the - object. See the description of v2 B-tree - records sub-types 1 and 2 for a description of the fields. New key - values are derived from Next Huge Object ID in the - Fractal Heap Header.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Fractal Heap ID for Huge Objects (Sub-type 3): - Directly Accessed, Non-filtered -
bytebytebytebyte
Version and TypeThis space inserted only to align table nicely

Address O


Length L

- - - - - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - -
- Fields: Fractal Heap ID for Huge Objects (Sub-type 3): - Directly Accessed, Non-filtered -
Field NameDescription

Version and Type

 -

This is a bit field with the following definition: - - - - - - - - - - - - - - - - - - -
BitDescription
6-7The current version of ID format. This document - describes version 0. -
4-5The ID type. Huge objects have a value of 1. -
0-3Reserved. -

- -

Address

This field is the address of the object in the file.

-

Length

This field is the length of the object in the file.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Fractal Heap ID for Huge Objects (Sub-type 4): - Directly Accessed, Filtered -
bytebytebytebyte
Version and TypeThis space inserted only to align table nicely

Address O


Length L

Filter Mask

De-filtered Size L

- - - - - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Fractal Heap ID for Huge Objects (Sub-type 4): - Directly Accessed, Filtered -
Field NameDescription

Version and Type

 -

This is a bit field with the following definition: - - - - - - - - - - - - - - - - - - -
BitDescription
6-7The current version of ID format. This document - describes version 0. -
4-5The ID type. Huge objects have a value of 1. -
0-3Reserved. -

- -

Address

This field is the address of the filtered object in the file.

-

Length

This field is the length of the filtered object in the file.

-

Filter Mask

This field is the I/O pipeline filter mask for the - filtered object in the file.

-

Filtered Size

This field is the size of the de-filtered object in the file.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - -
- Layout: Fractal Heap ID for Managed Objects -
bytebytebytebyte
Version and TypeThis space inserted only to align table nicely
Offset (variable size)
Length (variable size)
-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
- Fields: Fractal Heap ID for Managed Objects -
Field NameDescription

Version and Type

This is a bit field with the following definition: - - - - - - - - - - - - - - - - - - -
BitDescription
6-7The current version of ID format. This document - describes version 0. -
4-5The ID type. Managed objects have a value of 0. -
0-3Reserved. -

-

Offset

This field is the offset of the object in the heap. - This field’s size is the minimum number of bytes - necessary to encode the Maximum Heap Size value - (from the Fractal Heap Header). For example, if the - value of the Maximum Heap Size is less than 256 bytes, - this field is 1 byte in length, a Maximum Heap Size - of 256-65535 bytes uses a 2 byte length, and so on.

Length

This field is the length of the object in the heap. It - is determined by taking the minimum value of Maximum - Direct Block Size and Maximum Size of Managed - Objects in the Fractal Heap Header. Again, - the minimum number of bytes needed to encode that value is - used for the size of this field.

-
- -

- III.H. Disk Format: Level 1H - Free-space Manager

- -

- Free-space managers are used to describe space within a heap or - the entire HDF5 file that is not currently used for that heap or - file. -

- -

- The free-space manager header contains metadata information - about the space being tracked, along with the address of the list - of free space sections which actually describes the free - space. The header records information about free-space sections being - tracked, creation parameters for handling free-space sections of a - client, and section information used to locate the collection of - free-space sections. -

- -

- The free-space section list stores a collection of - free-space sections that is specific to each client of the - free-space manager. - - For example, the fractal heap is a client of the free space manager - and uses it to track unused space within the heap. There are 4 - types of section records for the fractal heap, each of which has - its own format, listed below. -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Free-space Manager Header -
bytebytebytebyte
Signature
VersionClient IDThis space inserted only to align table nicely

Total Space TrackedL


Total Number of SectionsL


Number of Serialized SectionsL


Number of Un-Serialized SectionsL

Number of Section ClassesThis space inserted only to align table nicely
Shrink PercentExpand Percent
Size of Address SpaceThis space inserted only to align table nicely

Maximum Section Size L


Address of Serialized Section ListO


Size of Serialized Section List UsedL


Allocated Size of Serialized Section ListL

Checksum
- - - - - - - - -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Free-space Manager Header -
Field NameDescription

Signature

 -

The ASCII character string “FSHD” - is used to indicate the beginning of the Free-space Manager - Header. This gives file consistency checking utilities a - better chance of reconstructing a damaged file. -

-

Version

 -

This is the version number for the Free-space Manager Header - and this document describes version 0.

-

Client ID

 -

This is the client ID for identifying the user of this - free-space manager: - - - - - - - - - - - - - - - - - - - -
IDDescription
0Fractal heap -
1File -
2+Reserved. -

- -

Total Space Tracked

 -

This is the total amount of free space being tracked, in bytes. -

-

Total Number of Sections

 -

This is the total number of free-space sections being tracked. -

-

Number of Serialized Sections

 -

This is the number of serialized free-space sections being - tracked. -

-

Number of Un-Serialized Sections

 -

This is the number of un-serialized free-space sections being - managed. Un-serialized sections are created by the free-space - client when the list of sections is read in. -

-

Number of Section Classes

 -

This is the number of section classes handled by this free space - manager for the free-space client. -

-

Shrink Percent

 -

This is the percent of current size to shrink the allocated - serialized free-space section list. -

-

Expand Percent

 -

This is the percent of current size to expand the allocated - serialized free-space section list. -

-

Size of Address Space

 -

This is the size of the address space that free-space sections - are within. This is stored as the log2 of the - actual value (in other words, the number of bits required - to store values within that address space). -

-

Maximum Section Size

 -

This is the maximum size of a section to be tracked. -

-

Address of Serialized Section List

 -

This is the address where the serialized free-space section - list is stored. -

-

Size of Serialized Section List Used

 -

This is the size of the serialized free-space section - list used (in bytes). This value must be less than - or equal to the allocated size of serialized section - list, below. -

-

Allocated Size of Serialized Section List

 -

This is the size of serialized free-space section list - actually allocated (in bytes). -

-

Checksum

 -

This is the checksum for the free-space manager header.

-
-
- -
-

The free-space sections being managed are stored in a - free-space section list, described below. The sections - in the free-space section list are stored in the following way: - a count of the number of sections describing a particular size of - free space and the size of the free-space described (in bytes), - followed by a list of section description records; then another - section count and size, followed by the list of section - descriptions for that size; and so on.

- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Free-space Section List -
bytebytebytebyte
Signature
VersionThis space inserted only to align table nicely

Free-space Manager Header AddressO

Number of Section Records in Set #0 (variable size)
Size of Free-space Section Described in Record Set #0 (variable size)
Record Set #0 Section Record #0 Offset(variable size)
Record Set #0 Section Record #0 TypeThis space inserted only to align table nicely
Record Set #0 Section Record #0 Data (variable size)
...
Record Set #0 Section Record #K-1 Offset(variable size)
Record Set #0 Section Record #K-1 TypeThis space inserted only to align table nicely
Record Set #0 Section Record #K-1 Data (variable size)
Number of Section Records in Set #1 (variable size)
Size of Free-space Section Described in Record Set #1 (variable size)
Record Set #1 Section Record #0 Offset(variable size)
Record Set #1 Section Record #0 TypeThis space inserted only to align table nicely
Record Set #1 Section Record #0 Data (variable size)
...
Record Set #1 Section Record #K-1 Offset(variable size)
Record Set #1 Section Record #K-1 TypeThis space inserted only to align table nicely
Record Set #1 Section Record #K-1 Data (variable size)
...
...
Number of Section Records in Set #N-1 (variable size)
Size of Free-space Section Described in Record Set #N-1 (variable size)
Record Set #N-1 Section Record #0 Offset(variable size)
Record Set #N-1 Section Record #0 TypeThis space inserted only to align table nicely
Record Set #N-1 Section Record #0 Data (variable size)
...
Record Set #N-1 Section Record #K-1 Offset(variable size)
Record Set #N-1 Section Record #K-1 TypeThis space inserted only to align table nicely
Record Set #N-1 Section Record #K-1 Data (variable size)
Checksum
- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Free-space Section List -
Field NameDescription

Signature

 -

The ASCII character string “FSSE” - is used to indicate the beginning of the Free-space Section - Information. This gives file consistency checking utilities - a better chance of reconstructing a damaged file. -

-

Version

 -

This is the version number for the Free-space Section List - and this document describes version 0.

-

Free-space Manager Header Address

 -

This is the address of the Free-space Manager Header. - This field is principally used for file - integrity checking. -

-

Number of Section Records for Set #N

 -

This is the number of free-space section records for set #N. - The length of this field is the minimum number of bytes needed - to store the number of serialized sections (from the - free-space manager header). -

- -

- The number of sets of free-space section records is - determined by the size of serialized section list in - the free-space manager header. -

-

Section Size for Record Set #N

 -

This is the size (in bytes) of the free-space section described - for all the section records in set #N. -

- -

- The length of this field is the minimum number of bytes needed - to store the maximum section size (from the - free-space manager header). -

-

Record Set #N Section #K Offset

 -

This is the offset (in bytes) of the free-space section within - the client for the free-space manager. -

- -

- The length of this field is the minimum number of bytes needed - to store the size of address space (from the - free-space manager header). -

-

Record Set #N Section #K Type

 -

This is the type of the section record, used to decode the - record set #N section #K data information. The defined - record type for file client is: - - - - - - - - - - - - - - - -
TypeDescription
0File’s section (a range of actual bytes in file) -
1+Reserved. -

- -

The defined record types for a fractal heap client are: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
TypeDescription
0Fractal heap “single” section -
1Fractal heap “first row” section -
2Fractal heap “normal row” section -
3Fractal heap “indirect” section -
4+Reserved. -

- -

Record Set #N Section #K Data

 -

This is the section-type specific information for each record - in the record set, described below. -

-

Checksum

 -

This is the checksum for the Free-space Section List. -

-
-
- -
-

- The section-type specific data for each free-space section record is - described below: -

- -
- - - - - - -
- Layout: File’s Section Data Record -
No additional record data stored
-
- -
-
-
-
- - - - - - -
- Layout: Fractal Heap “Single” Section Data Record -
No additional record data stored
-
- -
-
-
-
- - - - - - -
- Layout: Fractal Heap “First Row” Section Data - Record -
Same format as “indirect” - section data
-
- -
-
-
-
- - - - - - -
- Layout: Fractal Heap “Normal Row” Section Data - Record -
No additional record data stored
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Fractal Heap “Indirect” Section - Data Record -
bytebytebytebyte
Fractal Heap Indirect Block Offset (variable size)
Block Start RowBlock Start Column
Number of BlocksThis space inserted only to align table nicely
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Fractal Heap “Indirect” Section - Data Record -
Field NameDescription

Fractal Heap Block Offset

 -

The offset of the indirect block in the fractal heap’s address - space containing the empty blocks. -

-

- The number of bytes used to encode this field is the minimum - number of bytes needed to encode values for the Maximum - Heap Size (in the fractal heap’s header). -

-

Block Start Row

 -

This is the row that the empty blocks start in. -

-

Block Start Column

 -

This is the column that the empty blocks start in. -

-

Number of Blocks

 -

This is the number of empty blocks covered by the section. -

-
-
- -

- III.I. Disk Format: Level 1I - Shared Object Header Message Table

- -

- The shared object header message table is used to locate - object - header messages that are shared between two or more object headers - in the file. Shared object header messages are stored and indexed - in the file in one of two ways: indexed sequentially in a - shared header message list or indexed with a v2 B-tree. - The shared messages themselves are either stored in a fractal - heap (when two or more objects share the message), or remain in an - object’s header (when only one object uses the message currently, - but the message can be shared in the future). -

- -

- The shared object header message table - contains a list of shared message index headers. Each index header - records information about the version of the index format, the index - storage type, flags for the message types indexed, the number of - messages in the index, the address where the index resides, - and the fractal heap address if shared messages are stored there. -

- -

- Each index can be either a list or a v2 B-tree and may transition - between those two forms as the number of messages in the index - varies. Each shared message record contains information used to - locate the shared message from either a fractal heap or an object - header. The types of messages that can be shared are: Dataspace, - Datatype, Fill Value, Filter Pipeline and Attribute. -

- -

- The shared object header message table is pointed to - from a shared message table message - in the superblock extension for a file. This message stores the - version of the table format, along with the number of index headers - in the table. -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Shared Object Header Message Table -
bytebytebytebyte
Signature
Version for index #0Index Type for index #0Message Type Flags for index #0
Minimum Message Size for index #0
List Cutoff for index #0v2 B-tree Cutoff for index #0
Number of Messages for index #0This space inserted only to align table nicely

Index AddressO for index #0


Fractal Heap AddressO for index #0

...
...
Version for index #N-1Index Type for index #N-1Message Type Flags for index #N-1
Minimum Message Size for index #N-1
List Cutoff for index #N-1v2 B-tree Cutoff for index #N-1
Number of Messages for index #N-1This space inserted only to align table nicely

Index AddressO for index #N-1


Fractal Heap AddressO for index #N-1

Checksum
- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Shared Object Header Message Table -
Field NameDescription

Signature

 -

The ASCII character string “SMTB” - is used to indicate the beginning of the Shared Object - Header Message table. This gives file consistency checking - utilities a better chance of reconstructing a damaged file. -

-

Version for index #N

 -

This is the version number for the list of shared object header message - indexes and this document describes version 0.

-

Index Type for index #N

 -

The type of index can be an unsorted list or a v2 B-tree. -

-

Message Type Flags for index #N

 -

This field indicates the type of messages tracked in the index, - as follows: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
BitsDescription
0If set, the index tracks Dataspace Messages. -
1If set, the message tracks Datatype Messages. -
2If set, the message tracks Fill Value Messages. -
3If set, the message tracks Filter Pipeline Messages. -
4If set, the message tracks Attribute Messages. -
5-15Reserved (zero). -

- - -

- An index can track more than one type of message, but each type - of message can only by in one index. -

-

Minimum Message Size for index #N

 -

This is the message size sharing threshold for the index. - If the encoded size of the message is less than this value, the - message is not shared. -

-

List Cutoff for index #N

 -

This is the cutoff value for the indexing of messages to - switch from a list to a v2 B-tree. If the number of messages - is greater than this value, the index should be a v2 B-tree. -

-

v2 B-tree Cutoff for index #N

 -

This is the cutoff value for the indexing of messages - to switch from a v2 B-tree back to a list. If the number - of messages is less than this value, the index should be - a list. -

-

Number of Messages for index #N

 -

The number of shared messages being tracked for the index. -

-

Index Address for index #N

 -

This field is the address of the list or v2 B-tree where the - index nodes reside. -

-

Fractal Heap Address for index #N

 -

This field is the address of the fractal heap if shared messages - are stored there. -

-

Checksum

 -

This is the checksum for the table.

-
-
- -
-

- Shared messages are indexed either with a shared message record - list, described below, or using a v2 B-tree (using record type 7). - The number of records in the shared message record list is - determined in the index’s entry in the shared object header message - table. -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Shared Message Record List -
bytebytebytebyte
Signature
Shared Message Record #0
Shared Message Record #1
...
Shared Message Record #N-1
Checksum
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Fields: Shared Message Record List -
Field NameDescription

Signature

 -

The ASCII character string “SMLI” - is used to indicate the beginning of a list of index nodes. - This gives file consistency checking utilities a better - chance of reconstructing a damaged file. -

-

Shared Message Record #N

 -

The record for locating the shared message, either in the - fractal heap for the index, or an object header (see format for - index nodes below). -

-

Checksum

 -

This is the checksum for the list. -

-
-
- -
-

- The record for each shared message in an index is stored in one - of the following forms: -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Shared Message Record for Messages Stored in a - Fractal Heap -
bytebytebytebyte
Message LocationThis space inserted only to align table nicely
Hash Value
Reference Count

Fractal Heap ID

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Shared Message Record for Messages Stored in a - Fractal Heap -
Field NameDescription

Message Location

 -

This has a value of 0 indicating that the message is stored in - the heap. -

-

Hash Value

 -

This is the hash value for the message. -

-

Reference Count

 -

This is the number of times the message is used in the file. -

-

Fractal Heap ID

 -

This is an 8-byte fractal heap ID for the message as stored in - the fractal heap for the index. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Shared Message Record for Messages Stored in an - Object Header -
bytebytebytebyte
Message LocationThis space inserted only to align table nicely
Hash Value
ReservedMessage TypeCreation Index

Object Header AddressO

- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Shared Message Record for Messages Stored in an - Object Header -
Field NameDescription

Message Location

 -

This has a value of 1 indicating that the message is stored in - an object header. -

-

Hash Value

 -

This is the hash value for the message. -

-

Message Type

 -

This is the message type in the object header. -

-

Creation Index

 -

This is the creation index of the message within the object - header. -

-

Object Header Address

 -

This is the address of the object header where the message is - located. -

-
-
- -

- IV. Disk Format: Level 2 - Data Objects

- -

Data objects contain the “real” user-visible information in the file. - These objects compose the scientific data and other information which - are generally thought of as “data” by the end-user. All the - other information in the file is provided as a framework for - storing and accessing these data objects. -

- -

A data object is composed of header and data - information. The header information contains the information - needed to interpret the data information for the object as - well as additional “metadata” or pointers to additional - “metadata” used to describe or annotate each object. -

- -

- IV.A. Disk Format: Level 2A - Data Object Headers

- -

The header information of an object is designed to encompass - all of the information about an object, except for the data itself. - This information includes the dataspace, the datatype, information - about how the data is stored on disk (in external files, compressed, - broken up in blocks, and so on), as well as other information used - by the library to speed up access to the data objects or maintain - a file’s integrity. Information stored by user applications - as attributes is also stored in the object’s header. The header - of each object is not necessarily located immediately prior to the - object’s data in the file and in fact may be located in any - position in the file. The order of the messages in an object header - is not significant.

- -

Object headers are composed of a prefix and a set of messages. The - prefix contains the information needed to interpret the messages and - a small amount of metadata about the object, and the messages contain - the majority of the metadata about the object. -

- -

- IV.A.1. Disk Format: Level 2A1 - Data Object Header Prefix

- - - -

- IV.A.1.a. Version 1 Data Object Header Prefix

- -

Header messages are aligned on 8-byte boundaries for version 1 - object headers. -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 1 Object Header -
bytebytebytebyte
VersionReserved (zero)Total Number of Header Messages
Object Reference Count
Object Header Size
Reserved (zero)
Header Message Type #1Size of Header Message Data #1
Header Message #1 FlagsReserved (zero)

Header Message Data #1

.
.
.
Header Message Type #nSize of Header Message Data #n
Header Message #n FlagsReserved (zero)

Header Message Data #n

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 1 Object Header -
Field NameDescription

Version

 -

This value is used to determine the format of the - information in the object header. When the format of the - object header is changed, the version number - is incremented and can be used to determine how the - information in the object header is formatted. This - is version one (1) (there was no version zero (0)) of the - object header. -

-

Total Number of Header Messages

 -

This value determines the total number of messages listed in - object headers for this object. This value includes the messages - in continuation messages for this object. -

-

Object Reference Count

 -

This value specifies the number of “hard links” to this object - within the current file. References to the object from external - files, “soft links” in this file and object references in this - file are not tracked. -

-

Object Header Size

 -

This value specifies the number of bytes of header message data - following this length field that contain object header messages - for this object header. This value does not include the size of - object header continuation blocks for this object elsewhere in the - file. -

-

Header Message #n Type

 -

This value specifies the type of information included in the - following header message data. The message types for - header messages are defined in sections below. -

-

Size of Header Message #n Data

 -

This value specifies the number of bytes of header - message data following the header message type and length - information for the current message. The size includes - padding bytes to make the message a multiple of eight - bytes. -

-

Header Message #n Flags

 -

This is a bit field with the following definition: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
BitDescription
0If set, the message data is constant. This is used - for messages like the datatype message of a dataset. -
1If set, the message is shared and stored - in another location than the object header. The Header - Message Data field contains a Shared Message - (described in the Data Object Header Messages - section below) - and the Size of Header Message Data field - contains the size of that Shared Message. -
2If set, the message should not be shared. -
3If set, the HDF5 decoder should fail to open this object - if it does not understand the message’s type and the file - is open with permissions allowing write access to the file. - (Normally, unknown messages can just be ignored by HDF5 - decoders) -
4If set, the HDF5 decoder should set bit 5 of this - message’s flags (in other words, this bit field) - if it does not understand the message’s type - and the object is modified in any way. (Normally, - unknown messages can just be ignored by HDF5 - decoders) -
5If set, this object was modified by software that did not - understand this message. - (Normally, unknown messages should just be ignored by HDF5 - decoders) (Can be used to invalidate an index or a similar - feature) -
6If set, this message is shareable. -
7If set, the HDF5 decoder should always fail to open this - object if it does not understand the message’s type (whether - it is open for read-only or read-write access). (Normally, - unknown messages can just be ignored by HDF5 decoders) -

- -

Header Message #n Data

 -

The format and length of this field is determined by the - header message type and size respectively. Some header - message types do not require any data and this information - can be eliminated by setting the length of the message to - zero. The data is padded with enough zeroes to make the - size a multiple of eight. -

-
-
- -

- IV.A.1.b. Version 2 Data Object Header Prefix

- -

Note that the “total number of messages” field has been dropped from - the data object header prefix in this version. The number of messages - in the data object header is just determined by the messages encountered - in all the object header blocks.

- -

Note also that the fields and messages in this version of data object - headers have no alignment or padding bytes inserted - they are - stored packed together.

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 Object Header -
bytebytebytebyte
Signature
VersionFlagsThis space inserted only to align table nicely
Access time (optional)
Modification Time (optional)
Change Time (optional)
Birth Time (optional)
Maximum # of compact attributes (optional)Minimum # of dense attributes (optional)
Size of Chunk #0 (variable size)This space inserted only to align table nicely
Header Message Type #1Size of Header Message Data #1Header Message #1 Flags
Header Message #1 Creation Order (optional)This space inserted only to align table nicely

Header Message Data #1

.
.
.
Header Message Type #nSize of Header Message Data #nHeader Message #n Flags
Header Message #n Creation Order (optional)This space inserted only to align table nicely

Header Message Data #n

Gap (optional, variable size)
Checksum
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 Object Header -
Field NameDescription

Signature

 -

The ASCII character string “OHDR” - is used to indicate the beginning of an object header. This - gives file consistency checking utilities a better chance - of reconstructing a damaged file. -

-

Version

 -

This field has a value of 2 indicating version 2 of the object header. -

-

Flags

 -

This field is a bit field indicating additional information - about the object header. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Bit(s)Description
0-1This two bit field determines the size of the - Size of Chunk #0 field. The values are: - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0The Size of Chunk #0 field is 1 byte. -
1The Size of Chunk #0 field is 2 bytes. -
2The Size of Chunk #0 field is 4 bytes. -
3The Size of Chunk #0 field is 8 bytes. -
-
2If set, attribute creation order is tracked.
3If set, attribute creation order is indexed.
4If set, non-default attribute storage phase change - values are stored.
5If set, access, modification, change and birth times - are stored.
6-7Reserved

- -

Access Time

 -

This 32-bit value represents the number of seconds after the - UNIX epoch when the object’s raw data was last accessed - (in other words, read or written). -

-

This field is present if bit 5 of flags is set. -

-

Modification Time

 -

This 32-bit value represents the number of seconds after - the UNIX epoch when the object’s raw data was last - modified (in other words, written). -

-

This field is present if bit 5 of flags is set. -

-

Change Time

 -

This 32-bit value represents the number of seconds after the - UNIX epoch when the object’s metadata was last changed. -

-

This field is present if bit 5 of flags is set. -

-

Birth Time

 -

This 32-bit value represents the number of seconds after the - UNIX epoch when the object was created. -

-

This field is present if bit 5 of flags is set. -

-

Maximum # of compact attributes

 -

This is the maximum number of attributes to store in the compact - format before switching to the indexed format. -

-

This field is present if bit 4 of flags is set. -

-

Minimum # of dense attributes

 -

This is the minimum number of attributes to store in the indexed - format before switching to the compact format. -

-

This field is present if bit 4 of flags is set. -

-

Size of Chunk #0

 -

- This unsigned value specifies the number of bytes of header - message data following this field that contain object header - information. -

-

- This value does not include the size of object header - continuation blocks for this object elsewhere in the file. -

-

- The length of this field varies depending on bits 0 and 1 of - the flags field. -

-

Header Message #n Type

 -

Same format as version 1 of the object header, described above. -

-

Size of Header Message #n Data

 -

This value specifies the number of bytes of header - message data following the header message type and length - information for the current message. The size of messages - in this version does not include any padding bytes. -

-

Header Message #n Flags

 -

Same format as version 1 of the object header, described above. -

-

Header Message #n Creation Order

 -

This field stores the order that a message of a given type - was created in. -

-

This field is present if bit 2 of flags is set. -

-

Header Message #n Data

 -

Same format as version 1 of the object header, described above. -

-

Gap

 -

A gap in an object header chunk is inferred by the end of the - messages for the chunk before the beginning of the chunk’s - checksum. Gaps are always smaller than the size of an - object header message prefix (message type + message size + - message flags). -

-

Gaps are formed when a message (typically an attribute message) - in an earlier chunk is deleted and a message from a later - chunk that does not quite fit into the free space is moved - into the earlier chunk. -

-

Checksum

 -

This is the checksum for the object header chunk. -

-
-
- -

The header message types and the message data associated with - them compose the critical “metadata” about each object. Some - header messages are required for each object while others are - optional. Some optional header messages may also be repeated - several times in the header itself, the requirements and number - of times allowed in the header will be noted in each header - message description below. -

- - -

- IV.A.2. Disk Format: Level 2A2 - Data Object Header Messages

- -

Data object header messages are small pieces of metadata that are - stored in the data object header for each object in an HDF5 file. - Data object header messages provide the metadata required to describe - an object and its contents, as well as optional pieces of metadata - that annotate the meaning or purpose of the object. -

- -

Data object header messages are either stored directly in the data - object header for the object or are shared between multiple objects - in the file. When a message is shared, a flag in the Message Flags - indicates that the actual Message Data - portion of that message is stored in another location (such as another - data object header, or a heap in the file) and the Message Data - field contains the information needed to locate the actual information - for the message. -

- -

- The format of shared message data is described here:

- -
- - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Shared Message (Version 1) -
bytebytebytebyte
VersionTypeReserved (zero)
Reserved (zero)

AddressO

- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - -
- Fields: Shared Message (Version 1) -
Field NameDescription

Version

The version number is used when there are changes in the format - of a shared object message and is described here: - - - - - - - - - - - - - - - -
VersionDescription
0Never used.
1Used by the library before version 1.6.1. -

-

Type

The type of shared message location: - - - - - - - - - - -
ValueDescription
0Message stored in another object’s header (a committed - message). -

-

Address

The address of the object header - containing the message to be shared.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - -
- Layout: Shared Message (Version 2) -
bytebytebytebyte
VersionTypeThis space inserted only to align table nicely

AddressO

- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
- Fields: Shared Message (Version 2) -
Field NameDescription

Version

The version number is used when there are changes in the format - of a shared object message and is described here: - - - - - - - - - - -
VersionDescription
2Used by the library of version 1.6.1 and after. -

-

Type

The type of shared message location: - - - - - - - - - - -
ValueDescription
0Message stored in another object’s header (a committed - message). -

-

Address

The address of the object header - containing the message to be shared.

-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - -
- Layout: Shared Message (Version 3) -
bytebytebytebyte
VersionTypeThis space inserted only to align table nicely
Location (variable size)
-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
- Fields: Shared Message (Version 3) -
Field NameDescription

Version

The version number indicates changes in the format of shared - object message and is described here: - - - - - - - - - - -
VersionDescription
3Used by the library of version 1.8 and after. In this - version, the Type field can indicate that - the message is stored in the fractal heap. -

-

Type

The type of shared message location: - - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Message is not shared and is not shareable. -
1Message stored in file’s shared object header message - heap (a shared message). -
2Message stored in another object’s header (a committed - message). -
3Message stored is not shared, but is shareable. -

-

Location

This field contains either a - Size of Offsets-bytes address of the object header - containing the message to be shared, or an 8-byte fractal heap - ID for the message in the file’s shared object header - message heap. -

-
-
- - -

The following is a list of currently defined header messages: -

- -

IV.A.2.a. The NIL Message

- - -
- - - - - - - - -
Header Message Name: NIL
Header Message Type: 0x0000
Length: Varies
Status: Optional; may be repeated.
Description:The NIL message is used to indicate a message which is to be - ignored when reading the header messages for a data object. - [Possibly one which has been deleted for some reason.] -
Format of Data: Unspecified
- - - -

IV.A.2.b. The Dataspace Message

- - -
- - - - - - - - - - -
Header Message Name: Dataspace
Header Message Type: 0x0001
Length: Varies according to the number of - dimensions, as described in the following table.
Status: Required for dataset objects; - may not be repeated.
Description:The dataspace message describes the number of dimensions (in - other words, “rank”) and size of each dimension that - the data object has. This message is only used for datasets which - have a simple, rectilinear, array-like layout; datasets requiring - a more complex layout are not yet supported. -
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Dataspace Message - Version 1 -
bytebytebytebyte
VersionDimensionalityFlagsReserved
Reserved

Dimension #1 SizeL

.
.
.

Dimension #n SizeL


Dimension #1 Maximum SizeL (optional)

.
.
.

Dimension #n Maximum SizeL (optional)


Permutation Index #1L (optional)

.
.
.

Permutation Index #nL (optional)

- - - - - -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Dataspace Message - Version 1 -
Field NameDescription

Version

 -

This value is used to determine the format of the - Dataspace Message. When the format of the - information in the message is changed, the version number - is incremented and can be used to determine how the - information in the object header is formatted. This - document describes version one (1) (there was no version - zero (0)). -

-

Dimensionality

 -

This value is the number of dimensions that the data - object has. -

-

Flags

 -

This field is used to store flags to indicate the - presence of parts of this message. Bit 0 (the least - significant bit) is used to indicate that maximum - dimensions are present. Bit 1 is used to indicate that - permutation indices are present. -

-

Dimension #n Size

 -

This value is the current size of the dimension of the - data as stored in the file. The first dimension stored in - the list of dimensions is the slowest changing dimension - and the last dimension stored is the fastest changing - dimension. -

-

Dimension #n Maximum Size

 -

This value is the maximum size of the dimension of the - data as stored in the file. This value may be the special - “unlimited” size which indicates - that the data may expand along this dimension indefinitely. - If these values are not stored, the maximum size of each - dimension is assumed to be the dimension’s current size. -

-

Permutation Index #n

 -

This value is the index permutation used to map - each dimension from the canonical representation to an - alternate axis for each dimension. If these values are - not stored, the first dimension stored in the list of - dimensions is the slowest changing dimension and the last - dimension stored is the fastest changing dimension. -

-
-
- - - -
-

Version 2 of the dataspace message dropped the optional - permutation index value support, as it was never implemented in the - HDF5 Library:

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Dataspace Message - Version 2 -
bytebytebytebyte
VersionDimensionalityFlagsType

Dimension #1 SizeL

.
.
.

Dimension #n SizeL


Dimension #1 Maximum SizeL (optional)

.
.
.

Dimension #n Maximum SizeL (optional)

- - - - - -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Dataspace Message - Version 2 -
Field NameDescription

Version

 -

This value is used to determine the format of the - Dataspace Message. This field should be ‘2’ for version 2 - format messages. -

-

Dimensionality

 -

This value is the number of dimensions that the data object has. -

-

Flags

 -

This field is used to store flags to indicate the - presence of parts of this message. Bit 0 (the least - significant bit) is used to indicate that maximum - dimensions are present. -

-

Type

 -

This field indicates the type of the dataspace: - - - - - - - - - - - - - - - - - - -
ValueDescription
0A scalar dataspace; in other words, - a dataspace with a single, dimensionless element. -
1A simple dataspace; in other words, - a dataspace with a rank greater than 0 and an - appropriate number of dimensions. -
2A null dataspace; in other words, - a dataspace with no elements. -

-

Dimension #n Size

 -

This value is the current size of the dimension of the - data as stored in the file. The first dimension stored in - the list of dimensions is the slowest changing dimension - and the last dimension stored is the fastest changing - dimension. -

-

Dimension #n Maximum Size

 -

This value is the maximum size of the dimension of the - data as stored in the file. This value may be the special - “unlimited” size which indicates - that the data may expand along this dimension indefinitely. - If these values are not stored, the maximum size of each - dimension is assumed to be the dimension’s current size. -

-
-
- - - - - -

IV.A.2.c. The Link Info Message

- - -
- - - - - - - - -
Header Message Name: Link Info
Header Message Type: 0x002
Length: Varies
Status: Optional; may not be - repeated.
Description:The link info message tracks variable information about the - current state of the links for a “new style” - group’s behavior. Variable information will be stored in - this message and constant information will be stored in the - Group Info message. -
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Link Info -
bytebytebytebyte
VersionFlagsThis space inserted only to align table nicely

Maximum Creation Index (8 bytes, optional)


Fractal Heap AddressO


Address of v2 B-tree for Name IndexO


Address of v2 B-tree for Creation Order IndexO (optional)

- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Link Info -
Field NameDescription

Version

 -

The version number for this message. This document describes - version 0.

-

Flags

This field determines various optional aspects of the link - info message: - - - - - - - - - - - - - - - - - - - -
BitDescription
0If set, creation order for the links is tracked. -
1If set, creation order for the links is indexed. -
2-7Reserved

- -

Maximum Creation Index

This 64-bit value is the maximum creation order index value - stored for a link in this group.

-

This field is present if bit 0 of flags is set.

-

Fractal Heap Address

 -

- This is the address of the fractal heap to store dense links. - Each link stored in the fractal heap is stored as a - Link Message. -

-

- If there are no links in the group, or the group’s links - are stored “compactly” (as object header messages), this - value will be the undefined - address. -

-

Address of v2 B-tree for Name Index

This is the address of the version 2 B-tree to index names of links.

-

If there are no links in the group, or the group’s links - are stored “compactly” (as object header messages), this - value will be the undefined - address. -

-

Address of v2 B-tree for Creation Order Index

This is the address of the version 2 B-tree to index creation order of links.

-

If there are no links in the group, or the group’s links - are stored “compactly” (as object header messages), this - value will be the undefined - address. -

-

This field exists if bit 1 of flags is set.

-
-
- - -

IV.A.2.d. The Datatype Message

- - -
- - - - - - - - -
Header Message Name: Datatype
Header Message Type: 0x0003 -
Length: Variable
Status: Required for dataset or committed - datatype (formerly named datatype) objects; may not be repeated. -
Description:

The datatype message defines the datatype for each element - of a dataset or a common datatype for sharing between multiple - datasets. A datatype can describe an atomic type like a fixed- - or floating-point type or more complex types like a C struct - (compound datatype), array (array datatype), or C++ vector - (variable-length datatype).

-

Datatype messages that are part of a dataset object do not - describe how elements are related to one another; the dataspace - message is used for that purpose. Datatype messages that are part of - a committed datatype (formerly named datatype) message describe - a common datatype that can be shared by multiple datasets in the - file.

-
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Datatype Message -
bytebytebytebyte
Class and VersionClass Bit Field, Bits 0-7Class Bit Field, Bits 8-15Class Bit Field, Bits 16-23
Size


Properties


-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Datatype Message -
Field NameDescription

Class and Version

 -

The version of the datatype message and the datatype’s class - information are packed together in this field. The version - number is packed in the top 4 bits of the field and the class - is contained in the bottom 4 bits. -

-

The version number information is used for changes in the - format of the datatype message and is described here: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
VersionDescription
0Never used -
1Used by early versions of the library to encode - compound datatypes with explicit array fields. - See the compound datatype description below for - further details. -
2Used when an array datatype needs to be encoded. -
3Used when a VAX byte-ordered type needs to be - encoded. Packs various other datatype classes more - efficiently also. -
4Used to encode the revised reference datatype. -
5Used when a complex number datatype needs to be encoded. -

- -

The class of the datatype determines the format for the class - bit field and properties portion of the datatype message, which - are described below. The - following classes are currently defined: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Fixed-Point
1Floating-Point
2 Time
3String
4Bit field
5Opaque
6Compound
7Reference
8Enumerated
9Variable-Length
10Array
11Complex

- -

Class Bit Fields

 -

The information in these bit fields is specific to each datatype - class and is described below. All bits not defined for a - datatype class are set to zero. -

-

Size

 -

The size of a datatype element in bytes. -

-

Properties

 -

This variable-sized sequence of bytes encodes information - specific to each datatype class and is described for each class - below. If there is no property information specified for a - datatype class, the size of this field is zero bytes. -

-
-
- - -
-
- -

Class specific information for the Fixed-point Numbers class - (Class 0):

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Bits: Fixed-point Bit Field Description -
BitsMeaning

0

Byte Order. If zero, byte order is little-endian; - otherwise, byte order is big endian.

1, 2

Padding type. Bit 1 is the lo_pad bit and bit 2 - is the hi_pad bit. If a datum has unused bits at either - end, then the lo_pad or hi_pad bit is copied to those - locations.

3

Signed. If this bit is set then the fixed-point - number is in 2’s complement form.

4-23

Reserved (zero).

-
- -
-
- - - - - - - - - - - - - - -
- Layout: Fixed-point Property Description -
ByteByteByteByte
Bit OffsetBit Precision
-
- -
-
- - - - - - - - - - - - - - - - - -
- Fields: Fixed-point Property Description -
Field NameDescription

Bit Offset

 -

The bit offset of the first significant bit of the fixed-point - value within the datatype. The bit offset specifies the number - of bits “to the right of” the value (which are set to the - lo_pad bit value). -

-

Bit Precision

 -

The number of bits of precision of the fixed-point value - within the datatype. This value, combined with the datatype - element’s size and the Bit Offset field specifies the number - of bits “to the left of” the value (which are set to the - hi_pad bit value). -

-
-
- - -
-
- -

Class specific information for the Floating-point Numbers class - (Class 1):

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Bits: Floating-point Bit Field Description -
BitsMeaning

0, 6

Byte Order. These two non-contiguous bits specify the - “endianness” of the bytes in the datatype element. - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Bit 6Bit 0Description
00Byte order is little-endian -
01Byte order is big-endian -
10Reserved -
11Byte order is VAX-endian -

-

1, 2, 3

Padding type. Bit 1 is the low bits pad type, bit 2 - is the high bits pad type, and bit 3 is the internal bits - pad type. If a datum has unused bits at either end or between - the sign bit, exponent, or mantissa, then the value of bit - 1, 2, or 3 is copied to those locations.

4-5

Mantissa Normalization. This 2-bit bit field specifies - how the most significant bit of the mantissa is managed. - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0No normalization -
1The most significant bit of the mantissa is always set - (except for 0.0). -
2The most significant bit of the mantissa is not stored, - but is implied to be set. -
3Reserved. -

-

7

Reserved (zero).

8-15

Sign Location. This is the bit position of the sign - bit. Bits are numbered with the least significant bit zero.

16-23

Reserved (zero).

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Floating-point Property Description -
ByteByteByteByte
Bit OffsetBit Precision
Exponent LocationExponent SizeMantissa LocationMantissa Size
Exponent Bias
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Floating-point Property Description -
Field NameDescription

Bit Offset

 -

The bit offset of the first significant bit of the floating-point - value within the datatype. The bit offset specifies the number - of bits “to the right of” the value. -

-

Bit Precision

 -

The number of bits of precision of the floating-point value - within the datatype. -

-

Exponent Location

 -

The bit position of the exponent field. Bits are numbered with - the least significant bit number zero. -

-

Exponent Size

 -

The size of the exponent field in bits. -

-

Mantissa Location

 -

The bit position of the mantissa field. Bits are numbered with - the least significant bit number zero. -

-

Mantissa Size

 -

The size of the mantissa field in bits. -

-

Exponent Bias

 -

The bias of the exponent field. -

-
-
- - -
-
- -

Class specific information for the Time class (Class 2):

- - -
- - - - - - - - - - - - - - - - - -
- Bits: Time Bit Field Description -
BitsMeaning

0

Byte Order. If zero, byte order is little-endian; - otherwise, byte order is big endian.

1-23

Reserved (zero).

-
- -
-
- - - - - - - - - - - -
- Layout: Time Property Description -
ByteByte
Bit Precision
-
- -
-
- - - - - - - - - - - - -
- Fields: Time Property Description -
Field NameDescription

Bit Precision

 -

The number of bits of precision of the time value. -

-
-
- - -
- -

Class specific information for the Strings class (Class 3):

- - -
- - - - - - - - - - - - - - - - - - - - - - -
- Bits: String Bit Field Description -
BitsMeaning

0-3

Padding type. This four-bit value determines the - type of padding to use for the string. The values are: - - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Null Terminate: A zero byte marks the end of the - string and is guaranteed to be present after - converting a long string to a short string. When - converting a short string to a long string the value is - padded with additional null characters as necessary. -
1Null Pad: Null characters are added to the end of - the value during conversions from short values to long - values but conversion in the opposite direction simply - truncates the value. -
2Space Pad: Space characters are added to the end of - the value during conversions from short values to long - values but conversion in the opposite direction simply - truncates the value. This is the Fortran - representation of the string. -
3-15Reserved -

-

4-7

Character Set. The character set used to - encode the string. - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0ASCII character set encoding -
1UTF-8 character set encoding -
2-15Reserved -

-

8-23

Reserved (zero).

-
- -

There are no properties defined for the string class. -

- -
-
- -

Class specific information for the Bit Fields class (Class 4):

- -
- - - - - - - - - - - - - - - - - - - - - - -
- Bits: Bitfield Bit Field Description -
BitsMeaning

0

Byte Order. If zero, byte order is little-endian; - otherwise, byte order is big endian.

1, 2

Padding type. Bit 1 is the lo_pad type and bit 2 - is the hi_pad type. If a datum has unused bits at either - end, then the lo_pad or hi_pad bit is copied to those - locations.

3-23

Reserved (zero).

-
- -
-
- - - - - - - - - - - - - - -
- Layout: Bit Field Property Description -
ByteByteByteByte
Bit OffsetBit Precision
-
- -
-
- - - - - - - - - - - - - - - - -
- Fields: Bit Field Property Description -
Field NameDescription

Bit Offset

 -

The bit offset of the first significant bit of the bit field - within the datatype. The bit offset specifies the number - of bits “to the right of” the value. -

-

Bit Precision

 -

The number of bits of precision of the bit field - within the datatype. -

-
-
- - -
-
- -

Class specific information for the Opaque class (Class 5):

- -
- - - - - - - - - - - - - - - - - -
- Bits: Opaque Bit Field Description -
BitsMeaning

0-7

Length of ASCII tag in bytes.

8-23

Reserved (zero).

-
- -
-
- - - - - - - - - - - - - -
- Layout: Opaque Property Description -
ByteByteByteByte

ASCII Tag
-
-
- -
-
- - - - - - - - - - - -
- Fields: Opaque Property Description -
Field NameDescription

ASCII Tag

 -

This NUL-terminated string provides a description for the - opaque type. It is NUL-padded to a multiple of 8 bytes. -

-
-
- - -
-
- -

Class specific information for the Compound class (Class 6):

- -
- - - - - - - - - - - - - - - - - -
- Bits: Compound Bit Field Description -
BitsMeaning

0-15

Number of Members. This field contains the number - of members defined for the compound datatype. The member - definitions are listed in the Properties field of the data - type message.

16-23

Reserved (zero).

-
- - -

The Properties field of a compound datatype is a list of the - member definitions of the compound datatype. The member - definitions appear one after another with no intervening bytes. - The member types are described with a (recursively) encoded datatype - message.

- -

Note that the property descriptions are different for different - versions of the datatype version. Additionally note that the version - 0 datatype encoding is deprecated and has been replaced with later - encodings in versions of the HDF5 Library from the 1.4 release - onward.

- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Compound Properties Description for Datatype Version 1 -
ByteByteByteByte

Name

Byte Offset of Member
DimensionalityReserved (zero)
Dimension Permutation
Reserved (zero)
Dimension #1 Size (required)
Dimension #2 Size (required)
Dimension #3 Size (required)
Dimension #4 Size (required)

Member Type Message

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Compound Properties Description for Datatype Version 1 -
Field NameDescription

Name

 -

This NUL-terminated string provides a description for the - opaque type. It is NUL-padded to a multiple of 8 bytes. -

-

Byte Offset of Member

 -

This is the byte offset of the member within the datatype. -

-

Dimensionality

 -

If set to zero, this field indicates a scalar member. If set - to a value greater than zero, this field indicates that the - member is an array of values. For array members, the size of - the array is indicated by the ‘Size of Dimension n’ field in - this message. -

-

Dimension Permutation

 -

This field was intended to allow an array field to have - its dimensions permuted, but this was never implemented. - This field should always be set to zero. -

-

Dimension #n Size

 -

This field is the size of a dimension of the array field as - stored in the file. The first dimension stored in the list of - dimensions is the slowest changing dimension and the last - dimension stored is the fastest changing dimension. -

-

Member Type Message

 -

This field is a datatype message describing the datatype of - the member. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - -
- Layout: Compound Properties Description for Datatype Version 2 -
ByteByteByteByte

Name

Byte Offset of Member

Member Type Message

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Fields: Compound Properties Description for Datatype Version 2 -
Field NameDescription

Name

 -

This NUL-terminated string provides a description for the - opaque type. It is NUL-padded to a multiple of 8 bytes. -

-

Byte Offset of Member

 -

This is the byte offset of the member within the datatype. -

-

Member Type Message

 -

This field is a datatype message describing the datatype of - the member. -

-
-
- - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - -
- Layout: Compound Properties Description for Datatype Version 3 -
ByteByteByteByte

Name

Byte Offset of Member (variable size)

Member Type Message

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Fields: Compound Properties Description for Datatype Version 3 -
Field NameDescription

Name

This NUL-terminated string provides a description for the - opaque type. It is not NUL-padded to a multiple of 8 - bytes.

Byte Offset of Member

This is the byte offset of the member within the datatype. - The field size is the minimum number of bytes necessary, - based on the size of the datatype element. For example, a - datatype element size of less than 256 bytes uses a 1 byte - length, a datatype element size of 256-65535 bytes uses a - 2 byte length, and so on.

Member Type Message

This field is a datatype message describing the datatype of - the member.

-
- - -
-
- -

Class specific information for the Reference class (Class 7):

- -

Note that for region references, the stored data is - a Global Heap ID pointing to information - about the region stored in the global heap. -

- - -
- - - - - - - - - - - - - - - - - -
- Bits: Reference Bit Field Description for Datatype Version < 4 -
BitsMeaning

0-3

Type. This four-bit value contains the reference types which are supported for - backward compatibility. The values defined are: - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Object Reference (H5R_OBJECT1): A reference to another object in this - HDF5 file. -
1Dataset Region Reference (H5R_DATASET_REGION1): A reference to a region within - a dataset in this HDF5 file. -
2-15Reserved -

- -

4-23

Reserved (zero).

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Bits: Reference Bit Field Description for Datatype Version 4 -
BitsMeaning

0-3

Type. This four-bit value contains the revised reference types. - The values defined are: - - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
2Object Reference (H5R_OBJECT2): A reference to another object - in this file or an external file. -
3Dataset Region Reference (H5R_DATASET_REGION2): A reference to a region within - a dataset in this file or an external file. -
4Attribute Reference (H5R_ATTR): A reference to an attribute attached to an - object in this file or an external file. -
5-15Reserved -

- -

4-7

Version. This four-bit value contains the version for encoding - the revised reference types. The values defined are: - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Unused -
1The version for encoding the revised reference types: Object Reference (2), - Dataset Region Reference (3) and Attribute Reference (4). -
2-15Reserved -

- -

8-23

Reserved (zero).

-
- -

There are no properties defined for the reference class. -

- - -
-
- -

Class specific information for the Enumeration class (Class 8):

- -
- - - - - - - - - - - - - - - - - -
- Bits: Enumeration Bit Field Description -
BitsMeaning

0-15

Number of Members. The number of name/value - pairs defined for the enumeration type.

16-23

Reserved (zero).

-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - -
- Layout: Enumeration Property Description for Datatype Versions - 1 and 2 -
ByteByteByteByte

Base Type


Names


Values

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Fields: Enumeration Property Description for Datatype Versions - 1 and 2 -
Field NameDescription

Base Type

 -

Each enumeration type is based on some parent type, usually an - integer. The information for that parent type is described - recursively by this field. -

-

Names

 -

The name for each name/value pair. Each name is stored as a null - terminated ASCII string in a multiple of eight bytes. The names - are in no particular order. -

-

Values

 -

The list of values in the same order as the names. The values - are packed (no inter-value padding) and the size of each value - is determined by the parent type. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - -
- Layout: Enumeration Property Description for Datatype Version 3 -
ByteByteByteByte

Base Type


Names


Values

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Fields: Enumeration Property Description for Datatype Version 3 -
Field NameDescription

Base Type

 -

Each enumeration type is based on some parent type, usually an - integer. The information for that parent type is described - recursively by this field. -

-

Names

 -

The name for each name/value pair. Each name is stored as a null - terminated ASCII string, not padded to a multiple of - eight bytes. The names are in no particular order. -

-

Values

 -

The list of values in the same order as the names. The values - are packed (no inter-value padding) and the size of each value - is determined by the parent type. -

-
-
- - - -
- -

Class specific information for the Variable-length class (Class 9):

- -

Note that data with a variable length type is stored on the global heap. - Locations that would normally store the data directly (e.g. attribute message) - will instead contain a Global Heap ID. -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Bits: Variable-length Bit Field Description -
BitsMeaning

0-3

Type. This four-bit value contains the type of - variable-length datatype described. The values defined are: - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Sequence: A variable-length sequence of any datatype. - Variable-length sequences do not have padding or - character set information. -
1String: A variable-length sequence of characters. - Variable-length strings have padding and character set - information. -
2-15Reserved -

- -

4-7

Padding type. (variable-length string only) - This four-bit value determines the type of padding - used for variable-length strings. The values are the same - as for the string padding type, as follows: - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Null terminate: A zero byte marks the end of a string - and is guaranteed to be present after converting a long - string to a short string. When converting a short string - to a long string, the value is padded with additional null - characters as necessary. -
1Null pad: Null characters are added to the end of the - value during conversion from a short string to a longer - string. Conversion from a long string to a shorter string - simply truncates the value. -
2Space pad: Space characters are added to the end of the - value during conversion from a short string to a longer - string. Conversion from a long string to a shorter string - simply truncates the value. This is the Fortran - representation of the string. -
3-15Reserved -

- -

This value is set to zero for variable-length sequences.

- -

8-11

Character Set. (variable-length string only) - This four-bit value specifies the character set - to be used for encoding the string: - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0ASCII character set encoding -
1UTF-8 character set encoding -
2-15Reserved -

- -

This value is set to zero for variable-length sequences.

- -

12-23

Reserved (zero).

-
- -
-
- - - - - - - - - - - - - - -
- Layout: Variable-length Property Description -
ByteByteByteByte

Parent Type Message

-
- -
-
- - - - - - - - - - - - -
- Fields: Variable-length Property Description -
Field NameDescription

Parent Type

 -

Each variable-length type is based on some parent type. - This field contains the datatype message describing that parent type. - In the case of nested variable-length types, this parent datatype message will - recursively contain all parent datatype messages. - - Variable-length strings are considered to have the parent type `H5T_NATIVE_UCHAR`. -

-
-
- - -
-
- -

Class specific information for the Array class (Class 10):

- -

There are no bit fields defined for the array class. -

- -

Note that the dimension information defined in the property for this - datatype class is independent of dataspace information for a dataset. - The dimension information here describes the dimensionality of the - information within a data element (or a component of an element, if the - array datatype is nested within another datatype) and the dataspace for a - dataset describes the size and locations of the elements in a dataset. -

- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Array Property Description for Datatype Version 2 -
ByteByteByteByte
DimensionalityReserved (zero)
Dimension #1 Size
.
.
.
Dimension #n Size
Permutation Index #1
.
.
.
Permutation Index #n

Base Type

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Array Property Description for Datatype Version 2 -
Field NameDescription

Dimensionality

 -

This value is the number of dimensions that the array has. -

-

Dimension #n Size

 -

This value is the size of the dimension of the array - as stored in the file. The first dimension stored in - the list of dimensions is the slowest changing dimension - and the last dimension stored is the fastest changing - dimension. -

-

Permutation Index #n

 -

This value is the index permutation used to map - each dimension from the canonical representation to an - alternate axis for each dimension. Currently, dimension - permutations are not supported, and these indices should - be set to the index position minus one. In other words, - the first dimension should be set to 0, the second dimension - should be set to 1, and so on. -

-

Base Type

 -

Each array type is based on some parent type. The - information for that parent type is described recursively by - this field. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Array Property Description for Datatype Version 3 -
ByteByteByteByte
DimensionalityThis space inserted only to align table nicely
Dimension #1 Size
.
.
.
Dimension #n Size

Base Type

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Fields: Array Property Description for Datatype Version 3 -
Field NameDescription

Dimensionality

 -

This value is the number of dimensions that the array has. -

-

Dimension #n Size

 -

This value is the size of the dimension of the array - as stored in the file. The first dimension stored in - the list of dimensions is the slowest changing dimension - and the last dimension stored is the fastest changing - dimension. -

-

Base Type

 -

Each array type is based on some parent type. The - information for that parent type is described recursively by - this field. -

-
-
- -
-
- -

Class specific information for the Complex class (Class 11):

- -
- - - - - - - - - - - - - - - - - - - - - - -
- Bits: Complex Bit Field Description -
BitsMeaning

0

Homogeneous. If zero, each part of the complex number - datatype is a different floating point datatype (heterogeneous). - Otherwise, each part of the complex number datatype is the same - floating point datatype (homogeneous). Currently, only homogeneous - complex number datatypes are supported.

1,2

Complex number form. This two-bit value contains the type of - complex number datatype described. The values defined are: - - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Rectangular -
1Polar -
2Exponential -
3Reserved -

- -

Currently, only rectangular complex number datatypes are supported.

-

3-23

Reserved (zero).

-
- -
-
- - - - - - - - - - - - - -
- Layout: Complex Property Description -
ByteByteByteByte

Parent Type Message

-
- -
-
- - - - - - - - - - - -
- Fields: Complex Property Description -
Field NameDescription

Parent Type Message

 -

Each complex number type is based on a parent floating point type. - This field contains the datatype message describing that parent type. -

-
-
- - -

IV.A.2.e. The Data Storage - - Fill Value (Old) Message

- - -
- - - - - - - - -
Header Message Name: Fill Value - (old)
Header Message Type: 0x0004
Length: Varies
Status: Optional; may not be - repeated.
Description:

The fill value message stores a single data value which - is returned to the application when an uninitialized data element - is read from a dataset. The fill value is interpreted with the - same datatype as the dataset. If no fill value message is present - then a fill value of all zero bytes is assumed.

-

This fill value message is deprecated in favor of the - “new” fill value message (Message Type 0x0005) and - is only written to the file for forward compatibility with - versions of the HDF5 Library before the 1.6.0 version. - Additionally, it only appears for datasets with a user-defined - fill value (as opposed to the library default fill value or an - explicitly set “undefined” fill value).

-
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - -
- Layout: Fill Value Message (Old) -
bytebytebytebyte
Size

Fill Value (optional, variable size)

-
- -
-
- - - - - - - - - - - - - - - - -
- Fields: Fill Value Message (Old) -
Field NameDescription

Size

 -

This is the size of the Fill Value field in bytes. -

-

Fill Value

 -

The fill value. The bytes of the fill value are interpreted - using the same datatype as for the dataset. -

-
-
- - -

IV.A.2.f. The Data Storage - - Fill Value Message

- - -
- - - - - - - - -
Header Message Name: Fill - Value
Header Message Type: 0x0005
Length: Varies
Status: Required for dataset objects; - may not be repeated.
Description:The fill value message stores a single data value which is - returned to the application when an uninitialized data element - is read from a dataset. The fill value is interpreted with the - same datatype as the dataset.
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Fill Value Message - Versions 1 and 2 -
bytebytebytebyte
VersionSpace Allocation TimeFill Value Write TimeFill Value Defined
Size (optional)

Fill Value (optional, variable size)

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Fill Value Message - Versions 1 and 2 -
Field NameDescription

Version

 -

The version number information is used for changes in the - format of the fill value message and is described here: - - - - - - - - - - - - - - - - - - - - - - -
VersionDescription
0Never used -
1Initial version of this message. -
2In this version, the Size and Fill Value fields are - only present if the Fill Value Defined field is set - to 1. -
3This version packs the other fields in the message - more efficiently than version 2. -

- -

Space Allocation Time

 -

When the storage space for the dataset’s raw data will be - allocated. The allowed values are: - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Not used. -
1Early allocation. Storage space for the entire dataset - should be allocated in the file when the dataset is - created. -
2Late allocation. Storage space for the entire dataset - should not be allocated until the dataset is written - to. -
3Incremental allocation. Storage space for the - dataset should not be allocated until the portion - of the dataset is written to. This is currently - used in conjunction with chunked data storage for - datasets. -

- -

Fill Value Write Time

 -

At the time that storage space for the dataset’s raw data is - allocated, this value indicates whether the fill value should - be written to the raw data storage elements. The allowed values - are: - - - - - - - - - - - - - - - - - - -
ValueDescription
0On allocation. The fill value is always written to - the raw data storage when the storage space is allocated. -
1Never. The fill value should never be written to - the raw data storage. -
2Fill value written if set by user. The fill value - will be written to the raw data storage when the storage - space is allocated only if the user explicitly set - the fill value. If the fill value is the library - default or is undefined, it will not be written to - the raw data storage. -

- -

Fill Value Defined

 -

This value indicates if a fill value is defined for this - dataset. If this value is 0, the fill value is undefined. - If this value is 1, a fill value is defined for this dataset. - For version 2 or later of the fill value message, this value - controls the presence of the Size and Fill Value fields. -

-

Size

 -

This is the size of the Fill Value field in bytes. This field - is not present if the Version field is greater than 1, - and the Fill Value Defined field is set to 0. -

-

Fill Value

 -

The fill value. The bytes of the fill value are interpreted - using the same datatype as for the dataset. This field is - not present if the Version field is greater than 1, - and the Fill Value Defined field is set to 0. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Fill Value Message - Version 3 -
bytebytebytebyte
VersionFlagsThis space inserted only to align table nicely
Size (optional)

Fill Value (optional, variable size)

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Fill Value Message - Version 3 -
Field NameDescription

Version

 -

The version number information is used for changes in the - format of the fill value message and is described here: - - - - - - - - - - - - - - - - - - - - - - -
VersionDescription
0Never used -
1Initial version of this message. -
2In this version, the Size and Fill Value fields are - only present if the Fill Value Defined field is set - to 1. -
3This version packs the other fields in the message - more efficiently than version 2. -

- -

Flags

 -

When the storage space for the dataset’s raw data will be - allocated. The allowed values are: - - - - - - - - - - - - - - - - - - - - - - - - - - -
BitsDescription
0-1Space Allocation Time, with the same - values as versions 1 and 2 of the message. -
2-3Fill Value Write Time, with the same - values as versions 1 and 2 of the message. -
4Fill Value Undefined, indicating that the fill - value has been marked as “undefined” for this dataset. - Bits 4 and 5 cannot both be set. -
5Fill Value Defined, with the same values as - versions 1 and 2 of the message. - Bits 4 and 5 cannot both be set. -
6-7Reserved (zero). -

- -

Size

 -

This is the size of the Fill Value field in bytes. This field - is not present if the Version field is greater than 1, - and the Fill Value Defined flag is set to 0. -

-

Fill Value

 -

The fill value. The bytes of the fill value are interpreted - using the same datatype as for the dataset. This field is - not present if the Version field is greater than 1, - and the Fill Value Defined flag is set to 0. -

-
-
- - -

IV.A.2.g. The Link Message

- - -
- - - - - - - - -
Header Message Name: Link
Header Message Type: 0x0006
Length: Varies
Status: Optional; may be - repeated.
Description:

This message encodes the information for a link in a - group’s object header, when the group is storing its links - “compactly”, or in the group’s fractal heap, - when the group is storing its links “densely”.

-

A group is storing its links compactly when the fractal heap - address in the Link Info - Message is set to the “undefined address” - value.

Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Link Message -
bytebytebytebyte
VersionFlagsLink type (optional)This space inserted only to align table nicely

Creation Order (8 bytes, optional)

Link Name Character Set (optional)Length of Link Name (variable size)This space inserted only to align table nicely
Link Name (variable size)

Link Information (variable size)

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Link Message -
Field NameDescription

Version

The version number for this message. This document describes version 1.

-

Flags

This field contains information about the link and controls - the presence of other fields below. - - - - - - - - - - - - - - - - - - - - - - - - - - -
BitsDescription
0-1Determines the size of the Length of Link Name - field. - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0The size of the Length of Link Name - field is 1 byte. -
1The size of the Length of Link Name - field is 2 bytes. -
2The size of the Length of Link Name - field is 4 bytes. -
3The size of the Length of Link Name - field is 8 bytes. -
-
2Creation Order Field Present: if set, the Creation - Order field is present. If not set, creation order - information is not stored for links in this group. -
3Link Type Field Present: if set, the link is not - a hard link and the Link Type field is present. - If not set, the link is a hard link. -
4Link Name Character Set Field Present: if set, the - link name is not represented with the ASCII character - set and the Link Name Character Set field is - present. If not set, the link name is represented with - the ASCII character set. -
5-7Reserved (zero). -

- -

Link type

This is the link class type and can be one of the following - values: - - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0A hard link (should never be stored in the file) -
1A soft link. -
2-63Reserved for future HDF5 internal use. -
64An external link. -
65-255Reserved, but available for user-defined link types. -

- -

This field is present if bit 3 of Flags is set.

-

Creation Order

This 64-bit value is an index of the link’s creation time within - the group. Values start at 0 when the group is created an increment - by one for each link added to the group. Removing a link from a - group does not change existing links’ creation order field. -

-

This field is present if bit 2 of Flags is set.

-

Link Name Character Set

This is the character set for encoding the link’s name: - - - - - - - - - - - - - - - -
ValueDescription
0ASCII character set encoding (this should never be stored - in the file) -
1UTF-8 character set encoding -

- -

This field is present if bit 4 of Flags is set.

-

Length of link name

This is the length of the link’s name. The size of this field - depends on bits 0 and 1 of Flags.

-

Link name

This is the name of the link, non-NULL terminated.

-

Link information

The format of this field depends on the link type.

-

For hard links, the field is formatted as follows: - - - - - - -
- Size of Offsets bytes:The address of the object header for the object that the - link points to. -
-

- -

- For soft links, the field is formatted as follows: - - - - - - - - - - -
Bytes 1-2:Length of soft link value.
Length of soft link value bytes:A non-NULL-terminated string storing the value of the - soft link. -
-

- -

- For external links, the field is formatted as follows: - - - - - - - - - - -
Bytes 1-2:Length of external link value.
Length of external link value bytes:The first byte contains the version number in the - upper 4 bits and flags in the lower 4 bits for the external - link. Both version and flags are defined to be zero in - this document. The remaining bytes consist of two - NULL-terminated strings, with no padding between them. - The first string is the name of the HDF5 file containing - the object linked to and the second string is the full path - to the object linked to, within the HDF5 file’s - group hierarchy. -
-

- -

- For user-defined links, the field is formatted as follows: - - - - - - - - - - -
Bytes 1-2:Length of user-defined data.
Length of user-defined link value bytes:The data supplied for the user-defined link type.
-

- -
-
- -

IV.A.2.h. The Data Storage - - External Data Files Message

- - -
- - - - - - - - -
Header Message Name: External - Data Files
Header Message Type: 0x0007
Length: Varies
Status: Optional; may not be - repeated.
Description:The external data storage message indicates that the data - for an object is stored outside the HDF5 file. The filename of - the object is stored as a Universal Resource Location (URL) of - the actual filename containing the data. An external file list - record also contains the byte offset of the start of the data - within the file and the amount of space reserved in the file - for that data.
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: External File List Message -
bytebytebytebyte
VersionReserved (zero)
Allocated SlotsUsed Slots

Heap AddressO


Slot Definitions...

- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: External File List Message -
Field NameDescription

Version

 -

The version number information is used for changes in the format of - External Data Storage Message and is described here: - - - - - - - - - - - - - -
VersionDescription
0Never used.
1The current version used by the library.

- -

Allocated Slots

 -

The total number of slots allocated in the message. Its value must be at least as - large as the value contained in the Used Slots field. (The current library simply - uses the number of Used Slots for this message)

-

Used Slots

 -

The number of initial slots which contains valid information.

-

Heap Address

 -

This is the address of a local heap which contains the names for the external - files (The local heap information can be found in Disk Format Level 1D in this - document). The name at offset zero in the heap is always the empty string.

-

Slot Definitions

 -

The slot definitions are stored in order according to the array addresses they - represent.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - -
- Layout: External File List Slot -
bytebytebytebyte

Name Offset in Local HeapL


Offset in External Data FileL


Data Size in External FileL

- - - - - -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - -
- Fields: External File List Slot -
Field NameDescription

Name Offset in Local Heap

 -

The byte offset within the local name heap for the name - of the file. File names are stored as a URL which has a - protocol name, a host name, a port number, and a file - name: - protocol:port//host/file. - If the protocol is omitted then “file:” is assumed. If - the port number is omitted then a default port for that - protocol is used. If both the protocol and the port - number are omitted then the colon can also be omitted. If - the double slash and host name are omitted then - “localhost” is assumed. The file name is the only - mandatory part, and if the leading slash is missing then - it is relative to the application’s current working - directory (the use of relative names is not - recommended). -

-

Offset in External Data File

 -

This is the byte offset to the start of the data in the - specified file. For files that contain data for a single - dataset this will usually be zero.

-

Data Size in External File

 -

This is the total number of bytes reserved in the - specified file for raw data storage. For a file that - contains exactly one complete dataset which is not - extendable, the size will usually be the exact size of the - dataset. However, by making the size larger one allows - HDF5 to extend the dataset. The size can be set to a value - larger than the entire file since HDF5 will read zeroes - past the end of the file without failing.

-
-
- - -

IV.A.2.i. The Data Layout Message

- - -
- - - - - - - - -
Header Message Name: Data Layout
Header Message Type: 0x0008
Length: Varies
Status: Required for datasets; may not - be repeated.
Description:The Data Layout message - describes how the elements of a multi-dimensional array are stored - in the HDF5 file. Four types of data layout are supported: -
    -
  1. Contiguous: The array is stored in one contiguous area of - the file. This layout requires that the size of the array be - constant: data manipulations such as chunking, compression, - checksums, or encryption are not permitted. The message stores - the total storage size of the array. The offset of an element - from the beginning of the storage area is computed as in a C - array.
    2. -
  2. Chunked: The array domain is regularly decomposed into - chunks, and each chunk is allocated and stored separately. This - layout supports arbitrary element traversals, compression, - encryption, and checksums (these features are described - in other messages). The message stores the size of a chunk - instead of the size of the entire array; the storage size of - the entire array can be calculated by traversing the chunk index - that stores the chunk addresses.
    3. -
  3. Compact: The array is stored in one contiguous block as - part of this object header message.
    4. -
  4. Virtual: This is only supported for version 4 of the Data - Layout message. The message stores information that is used to - locate the global heap collection containing the Virtual Dataset - (VDS) mapping information. The mapping associates the VDS to - the source dataset elements that are stored across a collection - of HDF5 files.
    5. -
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Data Layout Message (Versions 1 and 2) -
bytebytebytebyte
VersionDimensionalityLayout ClassReserved (zero)
Reserved (zero)

Data AddressO (optional)

Dimension 1 Size
Dimension 2 Size
...
Dimension #n Size
Dataset Element Size (optional)
Compact Data Size (optional)

Compact Data... (variable size, optional)

- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Data Layout Message (Versions 1 and 2) -
Field NameDescription

Version

 -

The version number information is used for changes in the format of the data - layout message and is described here: - - - - - - - - - - - - - - - - - - - - -
VersionDescription
0Never used.
1Used by version 1.4 and before of the library to encode layout information. - Data space is always allocated when the data set is created.
2Used by version 1.6.[0,1,2] of the library to encode layout information. - Data space is allocated only when it is necessary.

-

Dimensionality

An array has a fixed dimensionality. This field - specifies the number of dimension size fields later in the - message. The value stored for chunked storage is 1 greater than - the number of dimensions in the dataset’s dataspace. - For example, 2 is stored for a 1 dimensional dataset. -

-

Layout Class

The layout class specifies the type of storage for the data - and how the other fields of the layout message are to be - interpreted. - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Compact Storage -
1Contiguous Storage -
2Chunked Storage -
-

-

Data Address

For contiguous storage, this is the address of the raw - data in the file. For chunked storage this is the address - of the v1 B-tree that is used to look up the addresses of the - chunks. This field is not present for compact storage. - If the version for this message is greater than 1, the address - may have the “undefined address” value, to indicate that - storage has not yet been allocated for this array.

-

Dimension #n Size

For contiguous and compact storage the dimensions define - the entire size of the array while for chunked storage they define - the size of a single chunk. In all cases, they are in units of - array elements (not bytes). The first dimension stored in the list - of dimensions is the slowest changing dimension and the last - dimension stored is the fastest changing dimension. -

-

Dataset Element Size

The size of a dataset element, in bytes. This field is only - present for chunked storage. -

-

Compact Data Size

This field is only present for compact data storage. - It contains the size of the raw data for the dataset array, in - bytes.

-

Compact Data

This field is only present for compact data storage. - It contains the raw data for the dataset array.

-
-
- -
-

Version 3 of this message re-structured the format into specific - properties that are required for each layout class.

- - -
- - - - - - - - - - - - - - - - - - - -
- Layout: Data Layout Message (Version 3) -
bytebytebytebyte
VersionLayout ClassThis space inserted only to align table nicely

Properties (variable size)

-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
- Fields: Data Layout Message (Version 3) -
Field NameDescription

Version

 -

The version number information is used for changes in the format of layout message - and is described here: - - - - - - - - - - -
VersionDescription
3Used by the version 1.6.3 and later of the library to store properties - for each layout class.

-

Layout Class

The layout class specifies the type of storage for the data - and how the other fields of the layout message are to be - interpreted. - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Compact Storage -
1Contiguous Storage -
2Chunked Storage -
-

-

Properties

This variable-sized field encodes information specific to each - layout class and is described below. If there is no property - information specified for a layout class, the size of this field - is zero bytes.

-
- -
- -

Class-specific information for compact storage (layout class 0): (Note: The dimensionality information - is in the Dataspace message)

- - -
- - - - - - - - - - - - - - - - - - -
- Layout: Compact Storage Property Description -
bytebytebytebyte
SizeThis space inserted only to align table nicely

Raw Data... (variable size)

-
- -
-
- - - - - - - - - - - - - - - - -
- Fields: Compact Storage Property Description -
Field NameDescription

Size

This field contains the size of the raw data for the dataset - array, in bytes. -

-

Raw Data

This field contains the raw data for the dataset array.

-
- - -
- -

Class-specific information for contiguous storage (layout class 1): - (Note: The dimensionality information is in the Dataspace message)

- - -
- - - - - - - - - - - - - - - - - -
- Layout: Contiguous Storage Property Description -
bytebytebytebyte

AddressO


SizeL

- - - - - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
-
- -
-
- - - - - - - - - - - - - - - - -
- Fields: Contiguous Storage Property Description -
Field NameDescription

Address

This is the address of the raw data in the file. - The address may have the “undefined address” value, to indicate - that storage has not yet been allocated for this array.

Size

This field contains the size allocated to store the raw data, - in bytes. -

-
-
- - -
-

Class-specific information for chunked storage (layout class 2):

- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Chunked Storage Property Description -
bytebytebytebyte
DimensionalityThis space inserted only to align table nicely

AddressO

Dimension 0 Size
Dimension 1 Size
...
Dimension #n Size
Dataset Element Size
- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Chunked Storage Property Description -
Field NameDescription

Dimensionality

A chunk has a fixed dimensionality. This field specifies - the number of dimension size fields later in the message.

Address

This is the address of the v1 B-tree - that is used to look up the - addresses of the chunks that actually store portions of the array - data. The address may have the “undefined address” value, to - indicate that storage has not yet been allocated for this array.

Dimension #n Size

These values define the dimension size of a single chunk, in - units of array elements (not bytes). The first dimension stored in - the list of dimensions is the slowest changing dimension and the - last dimension stored is the fastest changing dimension. -

-

Dataset Element Size

The size of a dataset element, in bytes. -

-
-
- - -
- -

- Version 4 of this message is similar to version 3 but has - additional information for the virtual layout class as well as - indexing information for the chunked layout class.

- -
- - - - - - - - - - - - - - - - - - - -
- Layout: Data Layout Message (Version 4) -
bytebytebytebyte
VersionLayout ClassThis space inserted - only to align table nicely

Properties (variable size)

-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
- Fields: Data Layout Message (Version 4) -
Field NameDescription

Version

 -

The value for this field is 4 and is used by version 1.10.0 - and later of the library to store properties for each layout - class and indexing information for the chunked layout. -

-

Layout Class

The layout class specifies the type of storage for the data - and how the other fields of the layout message are to be - interpreted. - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Compact Storage -
1Contiguous Storage -
2Chunked Storage -
3Virtual Storage -
-

-

Properties

This variable-sized field encodes information specific to a - layout class as follows: - - - - - - - - - - - - - - - - - - - - - - - - - -
Layout ClassDescription
Compact StorageSee Compact Storage - Property Description for the version 3 -Data Layout message. -
Contiguous StorageSee Contiguous Storage - Property Description for the version 3 -Data Layout message. -
Chunked StorageSee Chunked Storage - Property Description below. -
Virtual StorageSee Virtual Storage - Property Description below. -
- -

-
- -
- -

Class-specific information for chunked storage (layout - class 2):

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Chunked Storage Property Description -
bytebytebytebyte
FlagsDimensionalityDimension Size Encoded LengthThis space inserted to align table nicely
Dimension 0 Size (variable size)
Dimension 1 Size (variable size)
...
Dimension #n Size (variable size)
Chunk Indexing TypeThis space inserted only to align table nicely
Indexing Type Information (variable size)

AddressO

- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Chunked Storage Property Description -
Field NameDescription

Flags

This is the chunked layout feature flag:

- - - - - - - - - - - - - - - - - -
ValueDescription
DONT_FILTER_PARTIAL_BOUND_CHUNKS (bit 0)Do not apply filter to a partial edge chunk. - -
SINGLE_INDEX_WITH_FILTER (bit 1)A filtered chunk for Single Chunk indexing. -
- -

Dimensionality

A chunk has fixed dimension. This field specifies - the number of Dimension Size fields later in the message.

Dimension Size Encoded Length

 -

This is the size in bytes used to encode Dimension Size. -

-

Dimension #n Size

These values define the dimension size of a single chunk, in - units of array elements (not bytes). The first dimension stored in - the list of dimensions is the slowest changing dimension and the - last dimension stored is the fastest changing dimension. -

-

Chunk Indexing Type

There are five indexing types used to look up addresses - of the chunks. For more information on each type, see - “Appendix C: Types of Indexes for - Dataset Chunks.” - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
1Single Chunk indexing type. -
2Implicit indexing type. -
3Fixed Array indexing type. -
4Extensible Array indexing type. -
5Version 2 B-tree indexing type. -
-

-

Indexing Type Information

This variable-sized field encodes information specific to - an indexing type. More information on what is encoded with - each type can be found below this table. -

-

-

Address

This is the address specific to an indexing type. - The address may be undefined if the chunk or index storage is not allocated yet. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
Single Chunk indexAddress of the single chunk.
Implicit indexAddress of the array of dataset chunks.
Fixed Array indexAddress of the index.
Extensible Array indexAddress of the index.
Version 2 B-tree indexAddress of the index.
- -

-
-
- -
- -
    -
  1. - - Index-specific information for Single Chunk: -
    2. - -

    The following information exists only when the chunk is filtered. - In other words, when DONT_FILTER_PARTIAL_BOUND_CHUNKS - (bit 0) is enabled in the field flags.

    - -
    - - - - - - - - - - - - - - - - - - -
    - Layout: Single Chunk Indexing Information -
    bytebytebytebyte

    Size of filtered chunkL

    Filters for chunk
    - - - - - -
      - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
    -
    - -
    -
    - - - - - - - - - - - - - - - - -
    - Fields: Single Chunk Indexing Information -
    Field NameDescription

    Size of filtered chunk

    This field is the size of a filtered chunk.

    Filters for chunk

    This field contains filters for the chunk.

    -
    -

    - -
    - -
  2. - - Index-specific information for Implicit: -
    3. - -
    - - - - - - - - - - - - - - -
    - Layout: Implicit Indexing Information -
    bytebytebytebyte
    - No specific indexing information
    -
    - -
    -
  3. - - Index-specific information for Fixed Array: -
    4. - -
    - - - - - - - - - - - - - - - -
    - Layout: Fixed Array Indexing Information -
    bytebytebytebyte
    Page BitsThis space inserted only to align table nicely
    -
    - -
    -
    - - - - - - - - - - - - -
    - Fields: Fixed Array Indexing Information -
    Field NameDescription

    Page Bits

    This field contains the number of bits needed to store the - maximum number of elements in a data block page.

    -
    -

    - -
    -
  4. - - Index-specific information for Extensible Array: -
    5. - -
    - - - - - - - - - - - - - - - - - - - - - -
    - Layout: Extensible Array Indexing Information -
    bytebytebytebyte
    Max BitsIndex ElementsMin PointersMin Elements
    Page BitsThis space inserted only to align table nicely
    -
    - -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - Fields: Extensible Array Indexing Information -
    Field NameDescription

    Max Bits

    This field contains the number of bits needed to store the maximum number of elements - in the array. -

    -

    Index Elements

    This field contains the number of elements to store in the - index block. -

    -

    Min Pointers

    This field contains the minimum number of data block pointers - for a superblock. -

    -

    Min Elements

    This field contains the minimum number of elements per data block. -

    -

    Page Bits

    This field contains the number of bits needed to store the - maximum number of elements in a data block page. -

    -
    -
    -

    -
    - -
  5. - - Index-specific information for Version 2 B-tree: -
    6. - -
    - - - - - - - - - - - - - - - - - - - -
    - Layout: Version 2 B-tree Indexing Information -
    bytebytebytebyte
    Node Size
    Split PercentMerge Percent - This space inserted only to align table nicely
    -
    - -
    -
    - - - - - - - - - - - - - - - - - - - - - -
    - Fields: Version 2 B-tree Indexing Information -
    Field NameDescription

    Node Size

    This field is the size in bytes of a B-tree node. -

    -

    Split Percent

    This field is the percentage full of a B-tree node at which to split the node.

    Merge Percent

    This field is the percentage full of a B-tree node at which to merge the node.

    -
    -
- - - -
- -

- Class-specific information for virtual storage (layout class 3):

- -
- - - - - - - - - - - - - - - - - - -
- Layout: Virtual Storage Property Description -
bytebytebytebyte

AddressO

Index
- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
-
- -
-
- - - - - - - - - - - - - - - - -
- Fields: Virtual Storage Property Description -
Field NameDescription

Address

This is the address of the global heap collection where - the VDS mapping entries are stored. - See “Disk Format: Level 1F - - Global Heap Block for Virtual Datasets.” -

Index

This is the index of the data object within the global heap collection. -

-
-
- -

IV.A.2.j. The Bogus Message

- - -
- - - - - - - - -
Header Message Name: Bogus
Header Message Type: 0x0009
Length: 4 bytes
Status: For testing only; should never - be stored in a valid file.
Description:This message is used for testing the HDF5 Library’s - response to an “unknown” message type and should - never be encountered in a valid HDF5 file.
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - -
- Layout: Bogus Message -
bytebytebytebyte
Bogus Value
-
- -
-
- - - - - - - - - - - -
- Fields: Bogus Message -
Field NameDescription

Bogus Value

 -

This value should always be: 0xdeadbeef.

-
-
- -

IV.A.2.k. The Group Info Message -

- - -
- - - - - - - - -
Header Message Name: Group Info
Header Message Type: 0x000A
Length: Varies
Status: Optional; may not be - repeated.
Description:

This message stores information for the constants defining - a “new style” group’s behavior. Constant - information will be stored in this message and variable - information will be stored in the - Link Info message.

-

Note: the “estimated entry” information below is - used when determining the size of the object header for the - group when it is created.

Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Group Info Message -
bytebytebytebyte
VersionFlagsLink Phase Change: Maximum Compact Value (optional)
Link Phase Change: Minimum Dense Value (optional)Estimated Number of Entries (optional)
Estimated Link Name Length of Entries (optional)This space inserted only to align table nicely
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Group Info Message -
Field NameDescription

Version

The version number for this message. This document describes version 0.

-

Flags

This is the group information flag with the following definition: - - - - - - - - - - - - - - - - - - - -
BitDescription
0If set, link phase change values are stored. -
1If set, the estimated entry information is non-default - and is stored. -
2-7Reserved

-

Link Phase Change: Maximum Compact Value

The is the maximum number of links to store “compactly” (in - the group’s object header).

-

This field is present if bit 0 of Flags is set.

-

Link Phase Change: Minimum Dense Value

This is the minimum number of links to store “densely” (in - the group’s fractal heap). The fractal heap’s address is - located in the Link Info - message.

-

This field is present if bit 0 of Flags is set.

-

Estimated Number of Entries

This is the estimated number of entries in groups.

-

If this field is not present, the default value of 4 - will be used for the estimated number of group entries.

-

This field is present if bit 1 of Flags is set.

-

Estimated Link Name Length of Entries

This is the estimated length of entry name.

-

If this field is not present, the default value of 8 - will be used for the estimated link name length of group entries.

-

This field is present if bit 1 of Flags is set.

-
-
- - -

IV.A.2.l. The Data Storage - Filter - Pipeline Message

- - -
- - - - - - - - -
Header Message Name: - Data Storage - Filter Pipeline
Header Message Type: 0x000B
Length: Varies
Status: Optional; may not be - repeated.
Description:

This message describes the filter pipeline which should - be applied to the data stream by providing filter identification - numbers, flags, a name, and client data.

-

This message may be present in the object headers of both - dataset and group objects. For datasets, it specifies the - filters to apply to raw data. For groups, it specifies the - filters to apply to the group’s fractal heap. Currently, - only datasets using chunked data storage use the filter - pipeline on their raw data.

Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Filter Pipeline Message - Version 1 -
bytebytebytebyte
VersionNumber of FiltersReserved (zero)
Reserved (zero)

Filter Description List (variable size)

-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
- Fields: Filter Pipeline Message - Version 1 -
Field NameDescription

Version

The version number for this message. This table - describes version 1.

Number of Filters

The total number of filters described in this - message. The maximum possible number of filters in a - message is 32.

Filter Description List

A description of each filter. A filter description - appears in the next table.

-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Filter Description - Version 1 -
bytebytebytebyte
Filter Identification ValueName Length
FlagsNumber Client Data Values

Name (variable size, optional)


Client Data (variable size, optional)

Padding (variable size, optional)
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Filter Description - Version 1 -
Field NameDescription

Filter Identification Value

 -

- This value, often referred to as a filter identifier, - is designed to be a unique identifier for the filter. - Values from zero through 32,767 are reserved for filters - supported by The HDF Group in the HDF5 Library and for - filters requested and supported by third parties. - Filters supported by The HDF Group are documented immediately - below. Information on 3rd-party filters can be found at - The HDF Group’s - - Registered Filters page.

- -

- To request a filter identifier, please contact - The HDF Group’s Help Desk at - The HDF Group Help Desk. - You will be asked to provide the following information:

-
    -
  1. Contact information for the developer requesting the - new identifier
    2. -
  2. A short description of the new filter
    3. -
  3. Links to any relevant information, including licensing - information
    4. -
-

- Values from 32768 to 65535 are reserved for non-distributed uses - (for example, internal company usage) or for application usage - when testing a feature. The HDF Group does not track or document - the use of the filters with identifiers from this range.

- -

- The filters currently in library version 1.8.0 are - listed below: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
IdentificationNameDescription
0N/AReserved
1deflateGZIP deflate compression
2shuffleData element shuffling
3fletcher32Fletcher32 checksum
4szipSZIP compression
5nbitN-bit packing
6scaleoffsetScale and offset encoded values
-

Name Length

Each filter has an optional null-terminated ASCII name - and this field holds the length of the name including the - null termination padded with nulls to be a multiple of - eight. If the filter has no name then a value of zero is - stored in this field.

Flags

The flags indicate certain properties for a filter. The - bit values defined so far are: - - - - - - - - - - - - - - - -
BitDescription
0If set then the filter is an optional filter. - During output, if an optional filter fails it will be - silently skipped in the pipeline.
1-15Reserved (zero)

-

Number of Client Data Values

Each filter can store integer values to control - how the filter operates. The number of entries in the - Client Data array is stored in this field.

Name

If the Name Length field is non-zero then it will - contain the size of this field, padded to a multiple of eight. This - field contains a null-terminated, ASCII character string to serve - as a comment/name for the filter.

Client Data

This is an array of four-byte integers which will be - passed to the filter function. The Client Data Number of - Values determines the number of elements in the array.

Padding

Four bytes of zeroes are added to the message at this - point if the Client Data Number of Values field contains - an odd number.

-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - -
- Layout: Filter Pipeline Message - Version 2 -
bytebytebytebyte
VersionNumber of FiltersThis space inserted only to align table nicely

Filter Description List (variable size)

-
- -
-
- - - - - - - - - - - - - - - - - - - - - -
- Fields: Filter Pipeline Message - Version 2 -
Field NameDescription

Version

The version number for this message. This table - describes version 2.

Number of Filters

The total number of filters described in this - message. The maximum possible number of filters in a - message is 32.

Filter Description List

A description of each filter. A filter description - appears in the next table.

-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Filter Description - Version 2 -
bytebytebytebyte
Filter Identification ValueName Length (optional)
FlagsNumber Client Data Values

Name (variable size, optional)


Client Data (variable size, optional)

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Filter Description - Version 2 -
Field NameDescription

Filter Identification Value

 -

- This value, often referred to as a filter identifier, - is designed to be a unique identifier for the filter. - Values from zero through 32,767 are reserved for filters - supported by The HDF Group in the HDF5 Library and for - filters requested and supported by third parties. - Filters supported by The HDF Group are documented immediately - below. Information on 3rd-party filters can be found at - The HDF Group’s - - Registered Filters page.

- -

- To request a filter identifier, please contact - The HDF Group’s Help Desk at - The HDF Group Help Desk. - You will be asked to provide the following information:

-
    -
  1. Contact information for the developer requesting the - new identifier
    2. -
  2. A short description of the new filter
    3. -
  3. Links to any relevant information, including licensing - information
    4. -
-

- Values from 32768 to 65535 are reserved for non-distributed uses - (for example, internal company usage) or for application usage - when testing a feature. The HDF Group does not track or document - the use of the filters with identifiers from this range.

- -

- The filters currently in library version 1.8.0 are - listed below: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
IdentificationNameDescription
0N/AReserved
1deflateGZIP deflate compression
2shuffleData element shuffling
3fletcher32Fletcher32 checksum
4szipSZIP compression
5nbitN-bit packing
6scaleoffsetScale and offset encoded values
-

Name Length

Each filter has an optional null-terminated ASCII name - and this field holds the length of the name including the - null termination padded with nulls to be a multiple of - eight. If the filter has no name then a value of zero is - stored in this field.

-

Filters with IDs less than 256 (in other words, filters - that are defined in this format documentation) do not store - the Name Length or Name fields. -

-

Flags

The flags indicate certain properties for a filter. The - bit values defined so far are: - - - - - - - - - - - - - - - -
BitDescription
0If set then the filter is an optional filter. - During output, if an optional filter fails it will be - silently skipped in the pipeline.
1-15Reserved (zero)

-

Number of Client Data Values

Each filter can store integer values to control - how the filter operates. The number of entries in the - Client Data array is stored in this field.

Name

If the Name Length field is non-zero, then it will - contain the size of this field, not padded to a multiple - of eight. This field contains a non-null-terminated, - ASCII character string to serve as a comment/name for the filter. -

-

Filters that are defined in this format documentation - such as deflate and shuffle do not store the Name - Length or Name fields. -

-

Client Data

This is an array of four-byte integers which will be - passed to the filter function. The Client Data Number of - Values determines the number of elements in the array.

-
-
- -

IV.A.2.m. The Attribute Message

- - -
- - - - - - - - -
Header Message Name: Attribute
Header Message Type: 0x000C
Length: Varies
Status: Optional; may be - repeated.
Description:

The Attribute message is used to store objects - in the HDF5 file which are used as attributes, or - “metadata” about the current object. An attribute - is a small dataset; it has a name, a datatype, a dataspace, and - raw data. Since attributes are stored in the object header, they - should be relatively small (in other words, less than 64KB). - They can be associated with any type of object which has an - object header (groups, datasets, or committed (named) - datatypes).

-

In 1.8.x versions of the library, attributes can be larger - than 64KB. See the - - “Special Issues” section of the Attributes chapter - in the HDF5 User’s Guide for more information.

-

Note: Attributes on an object must have unique names: - the HDF5 Library currently enforces this by causing the - creation of an attribute with a duplicate name to fail. - Attributes on different objects may have the same name, - however.

Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Attribute Message (Version 1) -
bytebytebytebyte
VersionReserved (zero)Name Size
Datatype SizeDataspace Size

Name (variable size)


Datatype (variable size)


Dataspace (variable size)


Data (variable size)

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Attribute Message (Version 1) -
Field NameDescription

Version

The version number information is used for changes in the format of the - attribute message and is described here: - - - - - - - - - - - - - - - -
VersionDescription
0Never used.
1Used by the library before version 1.6 to encode attribute message. - This version does not support shared datatypes.

-

Name Size

The length of the attribute name in bytes including the - null terminator. Note that the Name field below may - contain additional padding not represented by this - field.

Datatype Size

The length of the datatype description in the Datatype - field below. Note that the Datatype field may contain - additional padding not represented by this field.

Dataspace Size

The length of the dataspace description in the Dataspace - field below. Note that the Dataspace field may contain - additional padding not represented by this field.

Name

The null-terminated attribute name. This field is - padded with additional null characters to make it a - multiple of eight bytes.

Datatype

The datatype description follows the same format as - described for the datatype object header message. This - field is padded with additional zero bytes to make it a - multiple of eight bytes.

Dataspace

The dataspace description follows the same format as - described for the dataspace object header message. This - field is padded with additional zero bytes to make it a - multiple of eight bytes.

Data

The raw data for the attribute. The size is determined - from the datatype and dataspace descriptions. This - field is not padded with additional bytes.

-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Attribute Message (Version 2) -
bytebytebytebyte
VersionFlagsName Size
Datatype SizeDataspace Size

Name (variable size)


Datatype (variable size)


Dataspace (variable size)


Data (variable size)

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Attribute Message (Version 2) -
Field NameDescription

Version

The version number information is used for changes in the - format of the attribute message and is described here: - - - - - - - - - - -
VersionDescription
2Used by the library of version 1.6.x and after to encode - attribute messages. - This version supports shared datatypes. The fields of - name, datatype, and dataspace are not padded with - additional bytes of zero. -

-

Flags

This bit field contains extra information about - interpreting the attribute message: - - - - - - - - - - - - - - - - -
BitDescription
0If set, datatype is shared.
1If set, dataspace is shared.

-

Name Size

The length of the attribute name in bytes including the - null terminator.

Datatype Size

The length of the datatype description in the Datatype - field below.

Dataspace Size

The length of the dataspace description in the Dataspace - field below.

Name

The null-terminated attribute name. This field is not - padded with additional bytes.

Datatype

The datatype description follows the same format as - described for the datatype object header message. -

-

If the - Flag field indicates this attribute’s datatype is - shared, this field will contain a “shared message” encoding - instead of the datatype encoding. -

-

This field is not padded with additional bytes. -

-

Dataspace

The dataspace description follows the same format as - described for the dataspace object header message. -

-

If the - Flag field indicates this attribute’s dataspace is - shared, this field will contain a “shared message” encoding - instead of the dataspace encoding. -

-

This field is not padded with additional bytes.

-

Data

The raw data for the attribute. The size is determined - from the datatype and dataspace descriptions. -

-

This field is not padded with additional zero bytes. -

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Attribute Message (Version 3) -
bytebytebytebyte
VersionFlagsName Size
Datatype SizeDataspace Size
Name Character Set EncodingThis space inserted only to align table nicely

Name (variable size)


Datatype (variable size)


Dataspace (variable size)


Data (variable size)

-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Attribute Message (Version 3) -
Field NameDescription

Version

The version number information is used for changes in the - format of the attribute message and is described here: - - - - - - - - - - -
VersionDescription
3Used by the library of version 1.8.x and after to - encode attribute messages. - This version supports attributes with non-ASCII names. -

-

Flags

This bit field contains extra information about - interpreting the attribute message: - - - - - - - - - - - - - - - - -
BitDescription
0If set, datatype is shared.
1If set, dataspace is shared.

-

Name Size

The length of the attribute name in bytes including the - null terminator.

Datatype Size

The length of the datatype description in the Datatype - field below.

Dataspace Size

The length of the dataspace description in the Dataspace - field below.

Name Character Set Encoding

The character set encoding for the attribute’s name: - - - - - - - - - - - - - - - -
ValueDescription
0ASCII character set encoding -
1UTF-8 character set encoding -
-

-

Name

The null-terminated attribute name. This field is not - padded with additional bytes.

Datatype

The datatype description follows the same format as - described for the datatype object header message. -

-

If the - Flag field indicates this attribute’s datatype is - shared, this field will contain a “shared message” encoding - instead of the datatype encoding. -

-

This field is not padded with additional bytes. -

-

Dataspace

The dataspace description follows the same format as - described for the dataspace object header message. -

-

If the - Flag field indicates this attribute’s dataspace is - shared, this field will contain a “shared message” encoding - instead of the dataspace encoding. -

-

This field is not padded with additional bytes.

-

Data

The raw data for the attribute. The size is determined - from the datatype and dataspace descriptions. -

-

This field is not padded with additional zero bytes. -

-
-
- -

IV.A.2.n. The Object Comment - Message

- - -
- - - - - - - - -
Header Message Name: Object - Comment
Header Message Type: 0x000D
Length: Varies
Status: Optional; may not be - repeated.
Description:The object comment is designed to be a short description of - an object. An object comment is a sequence of non-zero - (\0) ASCII characters with no other formatting - included by the library.
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - -
- Layout: Object Comment Message -
bytebytebytebyte

Comment (variable size)

-
- -
-
- - - - - - - - - - - -
- Fields: Object Comment Message -
Field NameDescription

Name

A null terminated ASCII character string.

-
- -

IV.A.2.o. The Object - Modification Time (Old) Message

- - -
- - - - - - - - -
Header Message Name: Object - Modification Time (Old)
Header Message Type: 0x000E
Length: Fixed
Status: Optional; may not be - repeated.
Description:

The object modification date and time is a timestamp - which indicates (using ISO-8601 date and time format) the last - modification of an object. The time is updated when any object - header message changes according to the system clock where the - change was posted. All fields of this message should be - interpreted as coordinated universal time (UTC).

-

This modification time message is deprecated in favor of - the “new” Object - Modification Time message and is no longer written to the - file in versions of the HDF5 Library after the 1.6.0 - version.

Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Modification Time Message (Old) -
bytebytebytebyte
Year
MonthDay of Month
HourMinute
SecondReserved
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Modification Time Message (Old) -
Field NameDescription

Year

The four-digit year as an ASCII string. For example, - 1998. -

Month

The month number as a two digit ASCII string where - January is 01 and December is 12.

Day of Month

The day number within the month as a two digit ASCII - string. The first day of the month is 01.

Hour

The hour of the day as a two digit ASCII string where - midnight is 00 and 11:00pm is 23.

Minute

The minute of the hour as a two digit ASCII string where - the first minute of the hour is 00 and - the last is 59.

Second

The second of the minute as a two digit ASCII string - where the first second of the minute is 00 - and the last is 59.

Reserved

This field is reserved and should always be zero.

-
- -

IV.A.2.p. The Shared Message Table - Message

- - -
- - - - - - - - -
Header Message Name: Shared Message - Table
Header Message Type: 0x000F
Length: Fixed
Status: Optional; may not be - repeated.
Description:This message is used to locate the table of shared object - header message (SOHM) indexes. Each index consists of information - to find the shared messages from either the heap or object header. - This message is only found in the superblock - extension.
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Shared Message Table Message -
bytebytebytebyte
VersionThis space inserted only to align table nicely

Shared Object Header Message Table AddressO

Number of IndicesThis space inserted only to align table nicely
- - - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Fields: Shared Message Table Message -
Field NameDescription

Version

The version number for this message. This document describes version 0.

Shared Object Header Message Table Address

This field is the address of the master table for shared - object header message indexes.

-

Number of Indices

This field is the number of indices in the master table. -

-
- -

IV.A.2.q. The Object Header - Continuation Message

- - -
- - - - - - - - -
Header Message Name: Object Header - Continuation
Header Message Type: 0x0010
Length: Fixed
Status: Optional; may be - repeated.
Description:The object header continuation is the location in the file - of a block containing more header messages for the current data - object. This can be used when header blocks become too large or - are likely to change over time.
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - -
- Layout: Object Header Continuation Message -
bytebytebytebyte

OffsetO


LengthL

- - - - - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - -
- Fields: Object Header Continuation Message -
Field NameDescription

Offset

This value is the address in the file where the - header continuation block is located.

Length

This value is the length in bytes of the header continuation - block in the file.

-
-
- -

The format of the header continuation block that this message points - to depends on the version of the object header that the message is - contained within. -

- -

- Continuation blocks for version 1 object headers have no special - formatting information; they are merely a list of object header - message info sequences (type, size, flags, reserved bytes and data - for each message sequence). See the description - of Version 1 Data Object Header Prefix. -

- -

Continuation blocks for version 2 object headers do have - special formatting information as described here - (see also the description of - Version 2 Data Object Header Prefix.): -

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 Object Header Continuation Block -
bytebytebytebyte
Signature
Header Message Type #1Size of Header Message Data #1Header Message #1 Flags
Header Message #1 Creation Order (optional)This space inserted only to align table nicely

Header Message Data #1

.
.
.
Header Message Type #nSize of Header Message Data #nHeader Message #n Flags
Header Message #n Creation Order (optional)This space inserted only to align table nicely

Header Message Data #n

Gap (optional, variable size)
Checksum
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 Object Header Continuation Block -
Field NameDescription

Signature

 -

The ASCII character string “OCHK” - is used to indicate the beginning of an object header - continuation block. This gives file consistency checking - utilities a better chance of reconstructing a damaged file. -

-

Header Message #n Type

 -

Same format as version 1 of the object header, described above. -

Size of Header Message #n Data

 -

Same format as version 1 of the object header, described above. -

Header Message #n Flags

 -

Same format as version 1 of the object header, described above. -

Header Message #n Creation Order

 -

This field stores the order that a message of a given type - was created in.

-

This field is present if bit 2 of flags is set.

-

Header Message #n Data

 -

Same format as version 1 of the object header, described above. -

Gap

 -

A gap in an object header chunk is inferred by the end of the - messages for the chunk before the beginning of the chunk’s - checksum. Gaps are always smaller than the size of an - object header message prefix (message type + message size + - message flags).

-

Gaps are formed when a message (typically an attribute message) - in an earlier chunk is deleted and a message from a later - chunk that does not quite fit into the free space is moved - into the earlier chunk.

-

Checksum

 -

This is the checksum for the object header chunk. -

-
-
- -

IV.A.2.r. The Symbol Table - Message

- - -
- - - - - - - - -
Header Message Name: Symbol Table - Message
Header Message Type: 0x0011
Length: Fixed
Status: Required for - “old style” groups; may not be repeated.
Description:Each “old style” group has a v1 B-tree and a - local heap for storing symbol table entries, which are located - with this message.
Format of data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - -
- Layout: Symbol Table Message -
bytebytebytebyte

v1 B-tree AddressO


Local Heap AddressO

- - - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - -
- Fields: Symbol Table Message -
Field NameDescription

v1 B-tree Address

This value is the address of the v1 B-tree containing the - symbol table entries for the group.

Local Heap Address

This value is the address of the local heap containing - the link names for the symbol table entries for the group.

-
- -

IV.A.2.s. The Object - Modification Time Message

- - -
- - - - - - - - -
Header Message Name: Object - Modification Time
Header Message Type: 0x0012
Length: Fixed
Status: Optional; may not be - repeated.
Description:The object modification time is a timestamp which indicates - the time of the last modification of an object. The time is - updated when any object header message changes according to - the system clock where the change was posted.
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - -
- Layout: Modification Time Message -
bytebytebytebyte
VersionReserved (zero)
Seconds After UNIX Epoch
-
- -
-
- - - - - - - - - - - - - - - - -
- Fields: Modification Time Message -
Field NameDescription

Version

The version number is used for changes in the format of Object Modification Time - and is described here: - - - - - - - - - - - - - - - -
VersionDescription
0Never used.
1Used by Version 1.6.1 and after of the library to encode time. In - this version, the time is the seconds after Epoch.

-

Seconds After UNIX Epoch

A 32-bit unsigned integer value that stores the number of - seconds since 0 hours, 0 minutes, 0 seconds, January 1, 1970, - Coordinated Universal Time.

-
- -

IV.A.2.t. The B-tree - ‘K’ Values Message

- - -
- - - - - - - - -
Header Message Name: B-tree - ‘K’ Values
Header Message Type: 0x0013
Length: Fixed
Status: Optional; may not be - repeated.
Description:This message retrieves non-default ‘K’ values - for internal and leaf nodes of a group or indexed storage v1 - B-trees. This message is only found in the superblock - extension.
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - -
- Layout: B-tree ‘K’ Values Message -
bytebytebytebyte
VersionIndexed Storage Internal Node KThis space inserted only to align table nicely
Group Internal Node KGroup Leaf Node K
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: B-tree ‘K’ Values Message -
Field NameDescription

Version

The version number for this message. This document describes - version 0.

-

Indexed Storage Internal Node K

This is the node ‘K’ value for each internal node of an - indexed storage v1 B-tree. See the description of this field - in version 0 and 1 of the superblock as well the section on - v1 B-trees. -

-

Group Internal Node K

This is the node ‘K’ value for each internal node of a group - v1 B-tree. See the description of this field in version 0 and - 1 of the superblock as well as the section on v1 B-trees. -

-

Group Leaf Node K

This is the node ‘K’ value for each leaf node of a group v1 - B-tree. See the description of this field in version 0 and 1 - of the superblock as well as the section on v1 B-trees. -

-
-
- -

IV.A.2.u. The Driver Info - Message

- - -
- - - - - - - - - -
Header Message Name: Driver - Info
Header Message Type: 0x0014
Length: Varies
Status: Optional; may not be - repeated.
- Description:This message contains information needed by the file driver - to reopen a file. This message is only found in the - superblock extension: see the - “Disk Format: Level 0C - Superblock Extension” - section for more information. For more information on the fields - in the driver info message, see the - “Disk Format: Level 0B - File Driver Info” - section; those who use the multi and family file drivers will - find this section particularly helpful.
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Driver Info Message -
bytebytebytebyte
VersionThis space inserted only to align table nicely

Driver Identification
Driver Information SizeThis space inserted only to align table nicely


Driver Information (variable size)


-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Driver Info Message -
Field NameDescription

Version

The version number for this message. This document describes - version 0.

-

Driver Identification

This is an eight-byte ASCII string without null termination which - identifies the driver. -

-

Driver Information Size

The size in bytes of the Driver Information field of this - message.

-

Driver Information

Driver information is stored in a format defined by the file driver.

-
-
- -

IV.A.2.v. The Attribute Info - Message

- - -
- - - - - - - - -
Header Message Name: Attribute - Info
Header Message Type: 0x0015
Length: Varies
Status: Optional; may not be - repeated.
Description:This message stores information about the attributes on an - object, such as the maximum creation index for the attributes - created and the location of the attribute storage when the - attributes are stored “densely”.
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Attribute Info Message -
bytebytebytebyte
VersionFlagsMaximum Creation Index (optional)

Fractal Heap AddressO


Attribute Name v2 B-tree AddressO


Attribute Creation Order v2 B-tree AddressO (optional)

- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Attribute Info Message -
Field NameDescription

Version

The version number for this message. This document describes - version 0.

-

Flags

This is the attribute index information flag with the - following definition: - - - - - - - - - - - - - - - - - - - -
BitDescription
0If set, creation order for attributes is tracked. -
1If set, creation order for attributes is indexed. -
2-7Reserved

- -

Maximum Creation Index

The is the maximum creation order index value for the - attributes on the object.

-

This field is present if bit 0 of Flags is set.

-

Fractal Heap Address

This is the address of the fractal heap to store dense - attributes. - Each attribute stored in the fractal heap is described by - the Attribute Message. -

-

Attribute Name v2 B-tree Address

This is the address of the version 2 B-tree to index the - names of densely stored attributes.

-

Attribute Creation Order v2 B-tree Address

This is the address of the version 2 B-tree to index the - creation order of densely stored attributes.

-

This field is present if bit 1 of Flags is set.

-
-
- -

IV.A.2.w. The Object Reference - Count Message

- - -
- - - - - - - - -
Header Message Name: Object Reference - Count
Header Message Type: 0x0016
Length: Fixed
Status: Optional; may not be - repeated.
Description:This message stores the number of hard links (in groups or - objects) pointing to an object: in other words, its - reference count.
Format of Data: See the tables - below.
- - -
- - - - - - - - - - - - - - - - - - -
- Layout: Object Reference Count -
bytebytebytebyte
VersionThis space inserted only to align table nicely
Reference count
-
- -
-
- - - - - - - - - - - - - - - - - -
- Fields: Object Reference Count -
Field NameDescription

Version

The version number for this message. This document describes - version 0.

-

Reference Count

The unsigned 32-bit integer is the reference count for the - object. This message is only present in “version 2” - (or later) object headers, and if not present those object - header versions, the reference count for the object is assumed - to be 1.

-
-
- -
- -

IV.A.2.x. The File Space Info - Message

- -
- - - - - - - -

-

-
Header Message Name: File Space - Info
Header Message Type: 0x0017
Length: Fixed
Status: Optional; may not be - repeated.
- Description:This message stores the file space management information - that the library uses in handling file space - requests for the file. Version 0 of the message is used for release 1.10.0 only. - Version 1 of the message is used for release 1.10.1+. - There is no File Space Info message before release 1.10 as the library does - not track file space across multiple file opens. -

- Note that version 0 is deprecated starting release 1.10.1. - That means when the 1.10.1+ library opens an HDF5 file with a version 0 message, - the library will decode and map the message to version 1. - On file close, it will encode the message as a version 1 message. -

- The library uses the following three mechanisms to manage file space in an HDF5 file: -

    -
  • Free-space managers -
    They track free-space sections of various sizes in the file that are not currently - allocated. Each free-space manager corresponds to a file space type. - There are two main groups of file space types: metadata and raw data. - Metadata is further divided into five types: superblock, B-tree, global heap, - local heap, and object header. - See the description of Free-space - Manager as well the description of file space allocation types in - Appendix B -
    • -
  • Aggregators -
    The library manages two aggregators, one for metadata and one for raw data. - Aggregator is a contiguous block of free-space in the file. - The size of each aggregator is tunable via public routines - H5Pset_meta_block_size and H5Pset_small_data_block_size respectively. -
    • -
  • Virtual file drivers -
    The library's virtual file driver interface dispatches requests for additional - space to the allocation routine of the file driver associated with the file. - For example, if the sec2 file driver is being used, its allocation routine will - increase the size of the file to service the requests. -
    • -
-

- For release 1.10.0, the library derives the following four file space strategies - based on the mechanisms: -

    -
  • H5F_FILE_SPACE_ALL -
      -
    • Mechanisms used: free-space managers, aggregators, and virtual file drivers
      • -
    • Does not persist free-space across file opens
      • -
    • This strategy is the library default
      • -
    -
    • -
  • H5F_FILE_SPACE_ALL_PERSIST
    • -
      -
    • Mechanisms used: free-space managers, aggregators, and virtual file drivers
      • -
    • Persist free-space across file opens
      • -
    -
  • H5F_FILE_SPACE_AGGR_VFD
    • -
      -
    • Mechanisms used: aggregators and virtual file drivers
      • -
    • Does not persist free-space across file opens
      • -
    -
  • H5F_FILE_SPACE_VFD
    • -
      -
    • Mechanisms used: virtual file drivers
      • -
    • Does not persist free-space across file opens
      • -
    -
- For release 1.10.1+, the free-space manager mechanism is modified to handle paged aggregation - which aggregates small metadata and raw data allocations into constant-sized well-aligned pages - to allow efficient I/O accesses. - With the support of this feature, the library derives the following four file space strategies: -
    -
  • H5F_FSPACE_STRATEGY_FSM_AGGR
    • -
      -
    • Mechanisms used: free-space managers, aggregators, and virtual file drivers
      • -
    • This strategy is the library default
      • -
    -
  • H5F_FSPACE_STRATEGY_PAGE
    • -
      -
    • Mechanisms used: free-space managers with embedded paged aggregation and virtual file drivers
      • -
    -
  • H5F_FSPACE_STRATEGY_AGGR
    • -
      -
    • Mechanisms used: aggregators and virtual file drivers
      • -
    -
  • H5F_FSPACE_STRATEGY_NONE
    • -
      -
    • Mechanisms used: virtual file drivers
      • -
    -
- The default is not persisting free-space across file opens for the above four strategies. - User can use the public routine H5Pset_file_space_strategy to request - persisting free-space. -
Format of Data: See the tables - below.
-

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: File Space Info - Version 0 -
bytebytebytebyte
VersionStrategyThresholdL

Free-space manager addressO for H5FD_MEM_SUPER


Free-space manager address0 for H5FD_MEM_BTREE


Free-space manager address0 for H5FD_MEM_DRAW


Free-space manager address0 for H5FD_MEM_GHEAP


Free-space manager address0 for H5FD_MEM_LHEAP


Free-space manager address0 for H5FD_MEM_OHDR

- - - - - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: File Space Info -
Field NameDescription

Version

This is version 0 of this message.

-

Strategy

This is the file space strategy used to manage file space. - There are four types: - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
1H5F_FILE_SPACE_ALL_PERSIST
2H5F_FILE_SPACE_ALL
3H5F_FILE_SPACE_AGGR_VFD
4H5F_FILE_SPACE_VFD

-

Threshold

This is the smallest free-space section size that the - free-space manager will track. -

Free-space manager addresses

These are the six free-space manager addresses for the - six file space allocation types: -

    -
  • H5FD_MEM_SUPER
    • -
  • H5FD_MEM_BTREE
    • -
  • H5FD_MEM_DRAW
    • -
  • H5FD_MEM_GHEAP
    • -
  • H5FD_MEM_LHEAP
    • -
  • H5FD_MEM_OHDR
    • -
- Note that these six fields exist only if the value for the field - “Strategy” is H5F_FILE_SPACE_ALL_PERSIST. -

-
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: File Space Info - Version 1 -
bytebytebytebyte
VersionStrategyPersisting free-spaceThis space inserted only to align table nicely
Free-space Section ThresholdL
File Space Page Size
Page-end Metadata thresholdThis space inserted only to align table nicely

EOA0


AddressO of small-sized free-space manager for H5FD_MEM_SUPER


AddressO of small-sized free-space manager for H5FD_MEM_BTREE


AddressO of small-sized free-space manager for H5FM_MEM_DRAW


AddressO of small-sized free-space manager for H5FD_MEM_GHEAP


AddressO of small-sized free-space manager for H5FD_MEM_LHEAP


AddressO of small-sized free-space manager for H5FD_MEM_OHDR


AddressO of large-sized free-space manager for H5FD_MEM_SUPER


AddressO of large-sized free-space manager for H5FD_MEM_BTREE


AddressO of large-sized free-space manager for H5FM_MEM_DRAW


AddressO of large-sized free-space manager for H5FD_MEM_GHEAP


AddressO of large-sized free-space manager for H5FD_MEM_LHEAP


AddressO of large-sized free-space manager for H5FD_MEM_OHDR

- - - - - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: File Space Info -
Field NameDescription

Version

This is version 1 of this message.

-

Strategy

This is the file space strategy used to manage file space. - There are four types: - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0H5F_FSPACE_STRATEGY_FSM_AGGR
1H5F_FSPACE_STRATEGY_PAGE
2H5F_FSPACE_STRATEGY_AGGR
3H5F_FSPACE_STRATEGY_NONE

-

Persisting free-space

True or false in persisting free-space. -

Free-space Section Threshold

This is the smallest free-space section size that the - free-space manager will track. -

File space page size

This is the file space page size, which is used when the paged aggregation feature - is enabled. -

Page-end metadata threshold

This is the smallest free-space section size at the end of a page that - the free-space manager will track. This is used when the paged aggregation feature - is enabled. -

EOA

The EOA before the allocation of free-space manager header and section info for the - self-referential free-space managers when persisting free-space. -
- Note that self-referential free-space managers are managers that involve file space - allocation for the managers' free-space header and section info. -

Addresses of small-sized free-space managers

These are the addresses of the six small-sized free-space managers for - the six file space allocation types: -

-
    -
  • H5FD_MEM_SUPER
    • -
  • H5FD_MEM_BTREE
    • -
  • H5FD_MEM_DRAW
    • -
  • H5FD_MEM_GHEAP
    • -
  • H5FD_MEM_LHEAP
    • -
  • H5FD_MEM_OHDR
    • -
- Note that these six fields exist only if the value for the field - “Persisting free-space” is true. - -

Addresses of large-sized free-space managers

These are the addresses of the six large-sized free-space managers for - the six file space allocation types: -

-
    -
  • H5FD_MEM_SUPER
    • -
  • H5FD_MEM_BTREE
    • -
  • H5FD_MEM_DRAW
    • -
  • H5FD_MEM_GHEAP
    • -
  • H5FD_MEM_LHEAP
    • -
  • H5FD_MEM_OHDR
    • -
- Note that these six fields exist only if the value for the field - “Persisting free-space” is true. - -
-
- -

- IV.B. Disk Format: Level 2B - Data Object Data Storage

- -

The data for an object is stored separately from its header - information in the file and may not actually be located in the HDF5 file - itself if the header indicates that the data is stored externally. The - information for each record in the object is stored according to the - dimensionality of the object (indicated in the dataspace header message). - Multi-dimensional array data is stored in C order; in other words, the - “last” dimension changes fastest.

- -

Data whose elements are composed of atomic datatypes are stored in IEEE - format, unless they are specifically defined as being stored in a different - machine format with the architecture-type information from the datatype - header message. This means that each architecture will need to [potentially] - byte-swap data values into the internal representation for that particular - machine.

- -

Data with a variable-length datatype is stored in the global heap - of the HDF5 file. Global heap identifiers are stored in the - data object storage.

- -

Data whose elements are composed of reference datatypes are stored in - several different ways depending on the particular reference type involved. - Object pointers are just stored as the offset of the object header being - pointed to with the size of the pointer being the same number of bytes as - offsets in the file.

- -

Dataset region references are stored as a heap-ID which points to - the following information within the file-heap: an offset of the object - pointed to, number-type information (same format as header message), - dimensionality information (same format as header message), sub-set start - and end information (in other words, a coordinate location for each), - and field start and end names (in other words, a [pointer to the] string - indicating the first field included and a [pointer to the] string name - for the last field).

- -

Data of a compound datatype is stored as a contiguous stream of the items - in the structure, with each item formatted according to its datatype. -

- Description of datatypes for variable-length, references and compound classes can be found - in Datatype Message. -

- Information about global heap and heap ID can be found in Global Heap. -

- For reference datatype, - see also the encoding description for Reference Encoding (Revised) and - Reference Encoding (Backward Compatibility) in Appendix D. -

- -

- V. Appendix A: Definitions

- -

Definitions of various terms used in this document are included in - this section.

- -
- - - - - - - - - - - - - - - - -
TermDefinition
Undefined AddressThe undefined - address for a file is a file address with all bits - set: in other words, 0xffff...ff.
Unlimited SizeThe unlimited size - for a size is a value with all bits set: in other words, - 0xffff...ff.
-
- - -

- VI. Appendix B: File Space Allocation Types

- -

There are six basic types of file space allocation as follows: -

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Basic Allocation TypeDescription
H5FD_MEM_SUPERFile space allocated for Superblock.
H5FD_MEM_BTREEFile space allocated for B-tree.
H5FD_MEM_DRAWFile space allocated for raw data.
H5FD_MEM_GHEAPFile space allocated for Global Heap.
H5FD_MEM_LHEAPFile space allocated for Local Heap.
H5FD_MEM_OHDRFile space allocated for Object Header.
-
- -
-

There are other file space allocation types that are mapped to the - above six basic types because they are similar in nature. - The mapping and the corresponding description are listed in the following two tables: -

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Basic Allocation TypeMapping of Allocation Types to Basic Allocation Types
H5FD_MEM_SUPERnone
H5FD_MEM_BTREEH5FD_MEM_SOHM_INDEX
H5FD_MEM_DRAWH5FD_MEM_FHEAP_HUGE_OBJ
H5FD_MEM_GHEAPnone
H5FD_MEM_LHEAPH5FD_MEM_FHEAP_DBLOCK, H5FD_MEM_FSPACE_SINFO
H5FD_MEM_OHDRH5FD_MEM_FHEAP_HDR, H5FD_MEM_FHEAP_IBLOCK, H5FD_MEM_FSPACE_HDR, H5FD_MEM_SOHM_TABLE
-
- -
-

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Allocation TypeDescription
H5FD_MEM_FHEAP_HDRFile space allocated for Fractal Heap Header.
H5FD_MEM_FHEAP_DBLOCKFile space allocated for Fractal Heap Direct Blocks.
H5FD_MEM_FHEAP_IBLOCKFile space allocated for Fractal Heap Indirect Blocks.
H5FD_MEM_FHEAP_HUGE_OBJFile space allocated for huge objects in the fractal heap.
H5FD_MEM_FSPACE_HDRFile space allocated for Free-space Manager Header.
H5FD_MEM_FSPACE_SINFOFile space allocated for Free-space Section List of the free-space manager.
H5FD_MEM_SOHM_TABLEFile space allocated for Shared Object Header Message Table.
H5FD_MEM_SOHM_INDEXFile space allocated for Shared Message Record List.
-
- -

VII. Appendix C: - Types of Indexes for Dataset Chunks

- -

For an HDF5 file without the latest format enabled, the library - uses the Version 1 B-tree to index dataset - chunks.

- -

For an HDF5 file with the latest format enabled, the library uses - one of the following five indexing types depending on a chunked - dataset’s dimension specification and the way it is extended. -

- - -

VII.A. The Single Chunk Index

 - -

The Single Chunk index can be used when the dataset fulfills - the following condition:

- -
    -
  • the current, maximum, and chunk dimension sizes are all the same
    • -
- -

The dataset has only one chunk, and the address of the single - chunk is stored in the version 4 Data Layout message. - See the Chunked Storage Property - Description layout and field description tables.

- - -

VII.B. The Implicit Index

 - -

The Implicit index can be used when the dataset fulfills - the following conditions:

- -
    -
  • fixed maximum dimension sizes
    • -
  • no filter applied to the dataset
    • -
  • the timing for the space allocation of the dataset chunks is - H5P_ALLOC_TIME_EARLY
    • -
- -

Since the dataset’s dimension sizes are known and storage space - is to be allocated early, an array of dataset chunks are allocated - based on the maximum dimension sizes when the dataset is created. - The base address of the array is stored in the version 4 - Data Layout message. See the - Chunked Storage Property - Description layout and field description tables. -

- -

When accessing a dataset chunk with a specified offset, the - address of the chunk in the array is computed as below:

- -

base address + (size of a chunk in bytes * chunk index - associated with the offset)

 - -

A chunk index starts at 0 and increases according to the - fastest changing dimension, then the next fastest, and so on. - - The chunk index for a dataset chunk offset is computed as below: -

    -
  1. Calculate the scaled offset for each dimension in - scaled_offset: -
    - 
    -        scaled_offset = chunk_offset/chunk_dims
-
    2. -
  2. Calculate the # of chunks for each dimension in - nchunks: -
    - 
    -        nchunks = (curr_dims + chunk_dims - 1)/chunk_dims
-
    3. - -
  3. Calculate the down chunks for each dimension in - down_chunks: -
    - 
    -        /* n is the # of dimensions */
-        for(i = (int)(n-1), acc = 1; i >= 0; i--) {
-        down_chunks[i] = acc;
-        acc *= nchunks[i];
-        }
-
    -
    4. - -
  4. Calculate the chunk index in chunk_index: -
    - 
    -        /* n is the # of dimensions */
-        for(u = 0, chunk_index = 0; u < n; u++)
-                                        chunk_index += down_chunks[u] * scaled_offset[u];
-
    -
    5. -
-

- For example, for a 2-dimensional dataset with - curr_dims[4,5] and chunk_dims[3,2], - there will be a total of 6 chunks, with 3 chunks in the fastest - changing dimension and 2 chunks in the slowest changing dimension. - See the figure below. - The chunk index for the chunk offset [3,4] - is computed as below: -

    - -
  1. scaled_offset[0] = 1, scaled_offset[1] = 2
    2. -
  2. nchunks[0] = 2, nchunks[1] = 3
    3. -
  3. down_chunks[0] = 3, down_chunks[1] = 1
    4. -
  4. chunk_index = 5
    5. -     -
- - - - - - - - - -
-
- Chunk Diagram
-
- Figure 3. Implicit index chunk diagram -
- - - - - - -

VII.C. The Fixed Array Index

 - -

The Fixed Array index can be used when the dataset fulfills - the following condition:

-
    -
  • fixed maximum dimension sizes
    • -
- -

Since the maximum number of chunks is known, an array of - in-file-on-disk addresses based on the maximum number of chunks is - allocated when data is written to the dataset. To access a dataset - chunk with a specified offset, the - chunk index associated with the offset -is calculated. The index is mapped into the array to locate the -disk address for the chunk.

- -

The Fixed Array (FA) index structure provides space and speed - improvements in locating chunks over index structures that handle - more dynamic data accesses like a - Version 2 B-tree index. - The entry into the Fixed Array is the Fixed Array header which - contains metadata about the entries stored in the array. The - header contains a pointer to a data block which stores the array - of entries that describe the dataset chunks. For greater efficiency, - the array will be divided into multiple pages if the number of - entries exceeds a threshold value. The space for the data block - and possibly data block pages are allocated as a single contiguous - block of space.

- -

The content of the data block depends on whether paging is - activated or not. When paging is not used, elements that describe - the chunks are stored in the data block. If paging is turned on, - the data block contains a bitmap indicating which pages are - initialized. Then subsequent data block pages will contain the - entries that describe the chunks.

- -

An entry describes either a filtered or non-filtered dataset - chunk. The formats for both element types are described below. -

-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Fixed Array Header -
bytebytebytebyte
Signature
VersionClient IDEntry SizePage Bits

Max Num - EntriesL


Data Block - AddressO

Checksum
- - - - - - - - -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Fixed Array Header -
Field NameDescription

Signature

 -

The ASCII character string “FAHD” - is used to indicate the beginning of a Fixed Array header. - This gives file consistency checking utilities a better - chance of reconstructing a damaged file. -

-

Version

 -

This document describes version 0.

-

Client ID

 -

The ID for identifying the client of the - Fixed Array: - - - - - - - - - - - - - - - - - - - -
IDDescription
0Non-filtered dataset chunks -
1Filtered dataset chunks -
2+Reserved -
-

-

Entry Size

 -

The size in bytes of an entry in the Fixed Array. -

-

Page Bits

 -

The number of bits needed to store the maximum - number of entries in a - data block page.

-

Max Num Entries

 -

The maximum number of entries in the Fixed - Array.

-

Data Block Address

 -

The address of the data block in the Fixed Array. -

-

Checksum

 -

The checksum for the header.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Fixed Array Data Block -
bytebytebytebyte
Signature
VersionClient IDThis space inserted - only to align table nicely

Header AddressO


Page Bitmap (variable size and - optional)


Elements (variable size and - optional)

Checksum
- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Fixed Array Data Block -
Field NameDescription

Signature

 -

The ASCII character string “FADB” - is used to indicate the beginning of a Fixed Array data - block. This gives file consistency checking utilities a - better chance of reconstructing a damaged file. -

-

Version

 -

This document describes version 0.

-

Client ID

 -

The ID for identifying the client of the - Fixed Array: - - - - - - - - - - - - - - - - - - - -
IDDescription
0Non-filtered dataset chunks -
1Filtered dataset chunks -
2+Reserved. -
-

-

Header Address

 -

The address of the Fixed Array header. Principally used - for file integrity checking. -

-

Page Bitmap

A bitmap indicating which data block pages are initialized.

-

Exists only if the data block is paged.

Elements

 -

Contains the elements stored in the data block - and exists only if the data block is not paged. - There are two element types: - - - - - - - - - - - - - - -
IDDescription
0Non-filtered - dataset chunks -
1Filtered dataset - chunks -
-

-

Checksum

 -

The checksum for the Fixed Array data block.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - -
- Layout: Fixed Array Data Block Page -
bytebytebytebyte

Elements (variable - size)

Checksum
-
- -
-
- - - - - - - - - - - - - - - - - -
- Fields: Fixed Array Data Block Page -
Field NameDescription

Elements

 -

Contains the elements stored in the data block page. - There are two element types: - - - - - - - - - - - - - - -
IDDescription
0Non-filtered dataset chunks -
1Filtered dataset chunks -
-

-

Checksum

 -

The checksum for a Fixed Array data block page.

-
-
- -
-
-
- -
- - - - - - - - - - - - - - -
- Layout: Data Block Element for Non-filtered Dataset Chunk -
bytebytebytebyte

AddressO

- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
-
- -
-
- - - - - - - - - - - - -
- Fields: Data Block Element for Non-filtered Dataset Chunk -
Field NameDescription

Address

The address of the dataset chunk in the file. -

-
-
- - -
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - -
- Layout: Data Block Element for Filtered Dataset Chunk -
bytebytebytebyte

AddressO


Chunk Size (variable size; at most - 8 bytes)

Filter Mask
- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Fields: Data Block Element for Filtered Dataset Chunk -
Field NameDescription

Address

The address of the dataset chunk in the file. -

-

Chunk Size

The size of the dataset chunk in bytes. -

-

Filter Mask

Indicates the filter to skip for the dataset chunk. Each - filter has an index number in the pipeline; if that filter is - skipped, the bit corresponding to its index is set. -

-
-
- - -

VII.D. The Extensible Array Index

 - -

The Extensible Array index can be used when the dataset - fulfills the following condition:

- -
    -
  • only one dimension of unlimited extent
    • -
- -

The Extensible Array (EA) is a data structure that is used as a - chunk index in datasets where the dataspace has a single - unlimited dimension. In other words, one dimension is set to - H5S_UNLIMITED, and the other dimensions are any number - of fixed-size dimensions. The idea behind the extensible array is - that a particular data object can be located via a lightweight - indexing structure of fixed depth for a given address space. This - indexing structure requires only a few (2-3) file operations per - element lookup and gives good cache performance. Unlike the B-tree - structure, the extensible array is optimized for appends. Where a - B-tree would always add at the rightmost node under these - circumstances, either creating a deep tree (version 1) or requiring - expensive rebalances to correct (version 2), the extensible array - has already mapped out a pre-balanced internal structure. This - optimized internal structure is instantiated as needed when chunk - records are inserted into the structure.

- - - - - - - -

An Extensible Array consists of a header, an index block, - secondary blocks, data blocks, and (optional) data block pages. The - general scheme is that the index block is used to reference a - secondary block, which is, in turn, used to reference the data block - page where the chunk information is stored. The data blocks will - be paged for efficiency when their size passes a threshold value. - These pages are laid out contiguously on the disk after the data - block, are initialized as needed, and are tracked via bitmaps - stored in the secondary block. The number of secondary and data - blocks/pages in a chunk index varies as they are allocated as - needed and the first few are (conceptually) stored in parent - elements as an optimization.

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Extensible Array Header -
bytebytebytebyte
Signature
VersionClient IDElement SizeMax Nelmts Bits
Index Blk ElmtsData Blk Min ElmtsSecondary Blk Min Data PtrsMax Data Blk Page Nelmts Bits

Num Secondary BlksL


Secondary Blk SizeL


Num Data BlksL


Data Blk SizeL


Max Index SetL


Num ElementsL


Index Block AddressO

Checksum
- - - - - - - - -
  - (Items marked with an ‘L’ in the above table are - of the size specified in the Size - of Lengths field in the superblock.) -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Extensible Array Header -
Field NameDescription

Signature

 -

The ASCII character string “EAHD” - is used to indicate the beginning of an Extensible Array - header. This gives file consistency checking utilities a - better chance of reconstructing a damaged file. -

-

Version

 -

This document describes version 0.

-

Client ID

 -

The ID for identifying the client of the - Fixed Array: - - - - - - - - - - - - - - - - - - - -
IDDescription
0Non-filtered dataset chunks -
1Filtered dataset chunks -
2+Reserved. -
-

-

Element Size

 -

The size in bytes of an element in the Extensible Array. -

-

Max Nelmts Bits

 -

The number of bits needed to store the - maximum number of elements in the Extensible Array.

-

Index Blk Elmts

 -

The number of elements to store in the index block. -

-

Data Blk Min Elmts

 -

The minimum number of elements per data block. -

-

Secondary Blk Min Data Ptrs

 -

The minimum number of data block pointers for a - secondary block. -

-

Max Dblk Page Nelmts Bits

 -

The number of bits needed to store the maximum number - of elements in a data block page. -

-

Num Secondary Blks

 -

The number of secondary blocks created. -

-

Secondary Blk Size

 -

The size of the secondary blocks created. -

-

Num Data Blks

 -

The number of data blocks created. -

-

Data Blk Size

 -

The size of the data blocks created. -

-

Max Index Set

 -

The maximum index set. -

-

Num Elmts

 -

The number of elements realized. -

-

Index Block Address

 -

The address of the index block. -

-

Checksum

 -

The checksum for the header.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Extensible Array Index Block -
bytebytebytebyte
Signature
VersionClient IDThis space inserted - only to align table nicely

Header AddressO


Elements (variable size and - optional)


Data Block Addresses (variable - size and optional)


Secondary Block Addresses (variable - size and optional)

Checksum
- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Extensible Array Index Block -
Field NameDescription

Signature

 -

The ASCII character string “EAIB” - is used to indicate the beginning of an Extensible Array - Index Block. This gives file consistency checking utilities - a better chance of reconstructing a damaged file. -

-

Version

 -

This document describes version 0.

-

Client ID

 -

The client ID for identifying the user of the - Extensible Array: - - - - - - - - - - - - - - - - - - - -
IDDescription
0Non-filtered dataset chunks -
1Filtered dataset chunks -
2+Reserved. -
-

-

Header Address

 -

The address of the Extensible Array header. Principally - used for file integrity checking.

-

Elements

 -

Contains the elements that are stored directly in - the index block. An optimization to avoid unnecessary - secondary blocks. -
-
- There are two element types: - - - - - - - - - - - - - - -
IDDescription
0Non-filtered dataset chunks -
1Filtered dataset chunks -
-

-

Data Block Addresses

 -

Contains the addresses of the data blocks - that are stored directly in the Index Block. An - optimization to avoid unnecessary secondary blocks.

-

Secondary Block Addresses

 -

Contains the addresses of the secondary - blocks.

-

Checksum

 -

The checksum for the Extensible Array Index Block.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Extensible Array Secondary Block -
bytebytebytebyte
Signature
VersionClient IDThis space inserted - only to align table nicely

Header AddressO


Block Offset (variable - size)


Page Bitmap (variable size and - optional)


Data Block Addresses (variable - size and optional)

Checksum
- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Extensible Array Secondary Block -
Field NameDescription

Signature

 -

The ASCII character string “EASB” - is used to indicate the beginning of an Extensible Array - Secondary Block. This gives file consistency checking utilities - a better chance of reconstructing a damaged file. -

-

Version

 -

This document describes version 0.

-

Client ID

 -

The ID for identifying the client of the - Extensible Array: - - - - - - - - - - - - - - - - - - - -
IDDescription
0Non-filtered dataset chunks -
1Filtered dataset chunks -
2+Reserved. -
-

-

Header Address

 -

The address of the Extensible Array header. Principally - used for file integrity checking.

-

Block Offset

 -

Stores the offset of the block in the array. -

-

Page Bitmap

 -

A bitmap indicating which - data block pages are initialized. -

- Exists only if the data block is paged. -

Data Block Addresses

 -

Contains the addresses of the data blocks - referenced by this secondary block.

-

Checksum

 -

The checksum for the Extensible Array - Secondary Block.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Extensible Array Data Block -
bytebytebytebyte
Signature
VersionClient IDThis space inserted - only to align table nicely

Header AddressO


Block Offset (variable - size)


Elements (variable size and - optional)

Checksum
- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Extensible Array Data Block -
Field NameDescription

Signature

 -

The ASCII character string “EADB” - is used to indicate the beginning of an Extensible Array - data block. This gives file consistency checking utilities - a better chance of reconstructing a damaged file. -

-

Version

 -

This document describes version 0.

-

Client ID

 -

The ID for identifying the client of the - Extensible Array: - - - - - - - - - - - - - - - - - - - -
IDDescription
0Non-filtered dataset chunks -
1Filtered dataset chunks -
2+Reserved. -
-

-

Header Address

 -

The address of the Extensible Array header. Principally - used for file integrity checking. -

-

Block Offset

 -

The offset of the block in the array. -

Elements

 -

Contains the elements stored in the data block and - exists only if the data block is not paged. -
-
- There are two element types: - - - - - - - - - - - - - - -
IDDescription
0Non-filtered dataset chunks -
1Filtered dataset chunks -
-

-

Checksum

 -

The checksum for the Extensible Array data block.

-
-
- -
-
-
-
- - - - - - - - - - - - - - - - - - -
- Layout: Extensible Array Data Block Page -
bytebytebytebyte

Elements (variable - size)

Checksum
-
- -
-
- - - - - - - - - - - - - - - - - -
- Fields: Extensible Array Data Block Page -
Field NameDescription

Elements

 -

Contains the elements stored in the data block - page.

-

- There are two element types: - - - - - - - - - - - - - - -
IDDescription
0Non-filtered dataset chunks -
1Filtered dataset chunks -
-

-

Checksum

 -

The checksum for an Extensible Array data block - page.

-
-
- -
-
-
- -
- - - - - - - - - - - - - - -
- Layout: Data Block Element for Non-filtered Dataset Chunk -
bytebytebytebyte

AddressO

- - - - - -
-
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
-
- -
-
- - - - - - - - - - - - -
- Fields: Data Block Element for Non-filtered Dataset Chunk -
Field NameDescription

Address

The address of the dataset chunk in the file. -

-
-
-

- -
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - -
- Layout: Data Block Element for Filtered Dataset Chunk -
bytebytebytebyte

AddressO


Chunk Size (variable size; at - most 8 bytes)

Filter Mask
- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - -
- Fields: Data Block Element for Filtered Dataset Chunk -
Field NameDescription

Address

The address of the dataset chunk in the file. -

-

Chunk Size

The size of the dataset chunk in bytes. -

-

Filter Mask

Indicates the filter to skip for the dataset chunk. - Each filter has an index number in the pipeline; if that - filter is skipped, the bit corresponding to its index is set. -

-
-
- - -

VII.E. The Version 2 B-trees Index

 - -

The Version 2 B-trees index can be used when the dataset - fulfills the following condition:

- -
    -
  • more than one dimension of unlimited extent
    • -
- -

Version 2 B-trees can be used to index various objects in the - library. See “Version 2 B-trees” - for more information. The B-tree types 10 - and 11 record layouts are for - indexing dataset chunks.

- -

VIII. Appendix D: - Encoding for dataspace and reference

- - -

VIII.A. Dataspace Encoding

-H5Sencode is a public routine that encodes a dataspace description into a buffer while -H5Sdecode is the corresponding routine that decodes the description encoded in the buffer. -

- See the reference manual description for these two public routines. - -
-
-
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Dataspace Description for H5Sencode/H5Sdecode -
bytebytebytebyte
Dataspace IDEncode VersionSize of SizeThis space inserted - only to align table nicely

Size of Extent -



Dataspace Message - (variable size) -



Dataspace Selection - (variable size) -

- -
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Dataspace Description for H5Sencode/H5Sdecode -
Field NameDescription

Dataspace ID

 -

The datspace message ID which is 1.

-

Encode Version

 -

H5S_ENCODE_VERSION which is 0. -

-

Size of Size

 -

The number of bytes used to store the size of an object. -

-

Size of Extent

 -

Size of the dataspace message. -

-

Dataspace Message

 -

The dataspace message information. See - Dataspace Message.

-

-

Dataspace Selection

 -

The dataspace selection information. See - Dataspace Selection.

-
-
- - -
-
-
- -
- - - - - - - - - - - - - - - - - -
- Layout: Dataspace Selection -
bytebytebytebyte
Selection Type

Selection Info (variable - size)

-
- -
-
-
- - - - - - - - - - - - - - - - - - -
- Fields: Dataspace Selection -
Field NameDescription

Selection Type

 -

There are 4 types of selection: - - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0H5S_SEL_NONE: Nothing selected -
1H5S_SEL_POINTS: Sequence of points selected -
2H5S_SEL_HYPER: Hyperslab selected -
3H5S_SEL_ALL: Entire extent selected -
-

Selection Info

 -

There are 4 types of selection info: - - - - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
0Selection info for H5S_SEL_NONE -
1Selection info for H5S_SEL_POINTS -
2Selection info for H5S_SEL_HYPER -
3Selection for H5S_SEL_ALL -
-

-
- - -
-
-
- -
- - - - - - - - - - - - - - - - - -
- Layout: Selection Info for H5S_SEL_NONE -
bytebytebytebyte
Version

Reserved (zero, 8 bytes)

-
- -
-
-
- - - - - - - - - - - -
- Fields: Selection Info for H5S_SEL_NONE -
Field NameDescription

Version

The version number for the H5S_SEL_NONE Selection Info. - The value is 1.

-
- - -
-
-
- -
- - - - - - - - - - - - - - - - - -
- Layout: Selection Info for H5S_SEL_POINTS -
bytebytebytebyte
Version


Points Selection Info (variable size) -


-
- -
-
-
- - - - - - - - - - - - - - - - - -
- Fields: Selection Info for H5S_SEL_POINTS -
Field NameDescription

Version

The version number for the H5S_SEL_POINTS Selection Info. - The value is either 1 or 2.

Points Selection Info

Depending on version: - - - - - - - - - - - - - - - - -
VersionDescription
1See Version 1 Points Selection Info -
2See Version 2 Points Selection Info -
-

-
- -
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 1 Points Selection Info -
bytebytebytebyte
Reserved (zero)
Length
Rank
Num Points
Point #1: coordinate #1
.
.
.
Point #1: coordinate #u
.
.
.
Point #n: coordinate #1
.
.
.
Point #n: coordinate #u
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 1 Points Selection Info -
Field NameDescription

Length

The size in bytes from Length to the end of the - selection info.

Rank

The number of dimensions.

Num Points

The number of points in the selection.

Point #n: coordinate #u

The array of points in the selection. -

The points selected are #1 to #n where n is Num Points. -

The list of coordinates for each point are #1 to #u where u is - Rank.

-
- - -
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 Points Selection Info -
bytebytebytebyte
Encode SizeThis space inserted only to align table nicely -
Rank
Num Points

(2, 4 or 8 bytes)

Point #1: coordinate #1

(2, 4 or 8 bytes)

.
.
.
Point #1: coordinate #u

(2, 4 or 8 bytes)

.
.
.
Point #n: coordinate #1

(2, 4 or 8 bytes)

.
.
.
Point #n: coordinate #u

(2, 4 or 8 bytes)

-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 Points Selection Info -
Field NameDescription

Encode Size

The size for encoding the points selection info which can be 2, 4 or 8 bytes. -

Rank

The number of dimensions.

Num Points

The number of points in the selection. -

The field Encode Size indicates the size of this field

Point #n: coordinate #u

The array of points in the selection. -

The points selected are #1 to #n where n is Num Points. -

The list of coordinates for each point are #1 to #u where u is - Rank. -

The field Encode Size indicates the size of this field

-
- - -
-
-
- -
- - - - - - - - - - - - - - - - - -
- Layout: Selection Info for H5S_SEL_HYPER -
bytebytebytebyte
Version

Hyperslab Selection Info - (variable size)

-
- -
-
-
- - - - - - - - - - - - - - - - - -
- Fields: Selection Info for H5S_SEL_HYPER -
Field NameDescription

Version

The version number for the H5S_SEL_HYPER selection info. - The value is 1, 2 or 3.

Hyperslab Selection Info

Depending on version: - - - - - - - - - - - - - - - - - - - - -
VersionDescription
1See Version 1 Hyperslab Selection Info. -
2See Version 2 Hyperslab Selection Info -
3See Version 3 Hyperslab Selection Info -
-

-
- -
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 1 Hyperslab Selection Info -
bytebytebytebyte
Reserved
Length
Rank
Num Blocks
Starting Offset #1 for Block #1
.
.
.
Starting Offset #n for Block #1
Ending Offset #1 for Block #1
.
.
.
Ending Offset #n for Block #1
.
.
.
.
.
.
.
.
.
Starting Offset #1 for Block #u
.
.
.
Starting Offset #n for Block #u
Ending Offset #1 for Block #u
.
.
.
Ending Offset #n for Block #u
-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 1 Hyperslab Selection Info -
Field NameDescription

Length

The size in bytes from the field Rank to the - end of the Selection Info.

Rank

The number of dimensions in the dataspace.

Num Blocks

The number of blocks in the selection.

Starting Offset #n for Block #u

The offset #n of the starting element in block #u. -

#n is from 1 to Rank. -

#u is from 1 to Num Blocks moving from the fastest - changing dimension to the slowest changing dimension. -

Ending Offset #n for Block #u

The offset #n of the ending element in block #u. -

#n is from 1 to Rank. -

#u is from 1 to Num Blocks moving from the fastest - changing dimension to the slowest changing dimension. -

-
- -
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 2 Hyperslab Selection Info -
bytebytebytebyte
FlagsThis space inserted - only to align table nicely
Length
Rank
Start #1 (8 bytes)

Stride #1 (8 bytes)

Count #1 (8 bytes)

Block #1 (8 bytes)

.
.
.
Start #n (8 bytes)

Stride #n (8 bytes)

Count #n (8 bytes)

Block #n (8 bytes)

-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 2 Hyperslab Selection Info -
Field NameDescription

Flags

This is a bit field with the following definition. - Currently, this is always set to 0x1. -

- - - - - - - - - - -
BitDescription
0If set, it is a regular hyperslab, otherwise, irregular. -
-

Length

The size in bytes from the field Rank to the - end of the Selection Info.

Rank

The number of dimensions in the dataspace.

Start #n

The offset of the starting element in the block. -

#n is from 1 to Rank. -

Stride #n

The number of elements to move in each dimension. -

#n is from 1 to Rank. -

Count #n

The number of blocks to select in each dimension. -

#n is from 1 to Rank. -

Block #n

The size (in elements) of each block in each dimension. -

#n is from 1 to Rank. -

-
- - - - -
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 3 Hyperslab Selection Info -
bytebytebytebyte
FlagsEncode SizeThis space inserted - only to align table nicely
Rank

Regular/Irregular Hyperslab Selection Info -

(variable size)

-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 3 Hyperslab Selection Info -
Field NameDescription

Flags

This is a bit field with the following definition: -

- - - - - - - - - - -
BitDescription
0If set, it is a regular hyperslab, otherwise, irregular. -
-

Encode Size

The size for encoding hyperslab selection info, which can 2, 4 or 8 bytes.

Rank

The number of dimensions in the dataspace.

Regular/Irregular Hyperslab Selection Info

This is the selection info for version 3 hyperslab which can be regular or irregular. -

If bit 0 of the field Flags is set, - See Version 3 Regular Hyperslab Selection Info -

Otherwise, see Version 3 Irregular Hyperslab Selection Info -

-
- - -
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 3 Regular Hyperslab Selection Info -
bytebytebytebyte
Start #1

(2, 4 or 8 bytes)

Stride #1

(2, 4 or 8 bytes)

Count #1

(2, 4 or 8 bytes)

Block #1

(2, 4 or 8 bytes)

.
.
.
Start #n

(2, 4 or 8 bytes)

Stride #n

(2, 4 or 8 bytes)

Count #n

(2, 4 or 8 bytes)

Block #n

(2, 4 or 8 bytes)

-
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Version 3 Regular Hyperslab Selection Info -
Field NameDescription

Start #n

The offset of the starting element in the block. -

#n is from 1 to Rank. -

The field Encode Size indicates the size of this field. -

Stride #n

The number of elements to move in each dimension. -

#n is from 1 to Rank. -

The field Encode Size indicates the size of this field. -

Count #n

The number of blocks to select in each dimension. -

#n is from 1 to Rank. -

The field Encode Size indicates the size of this field. -

Block #n

The size (in elements) of each block in each dimension. -

#n is from 1 to Rank. -

The field Encode Size indicates the size of this field. -

-
- -
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Version 3 Irregular Hyperslab Selection Info -
bytebytebytebyte
Num Blocks

(2, 4 or 8 bytes)

Starting Offset #1 for Block #1

(2, 4 or 8 bytes)

.
.
.
Starting Offset #n for Block #1

(2, 4 or 8 bytes)

Ending Offset #1 for Block #1

(2, 4 or 8 bytes)

.
.
.
Ending Offset #n for Block #1

(2, 4 or 8 bytes)

.
.
.
.
.
.
.
.
.
Starting Offset #1 for Block #u

(2, 4 or 8 bytes)

.
.
.
Starting Offset #n for Block #u

(2, 4 or 8 bytes)

Ending Offset #1 for Block #u

(2, 4 or 8 bytes)

.
.
.
Ending Offset #n for Block #u

(2, 4 or 8 bytes)

-
- -
-
-
- - - - - - - - - - - - - - - - - - -
- Fields: Version 3 Irregular Hyperslab Selection Info -

Num Blocks

The number of blocks in the selection. -

The field Encode Size indicates the size of this field

Starting Offset #n for Block #u

The offset #n of the starting element in block #u. -

#n is from 1 to Rank. -

#u is from 1 to Num Blocks moving from the fastest - changing dimension to the slowest changing dimension. -

The field Encode Size indicates the size of this field -

Ending Offset #n for Block #u

The offset #n of the ending element in block #u. -

#n is from 1 to Rank. -

#u is from 1 to Num Blocks moving from the fastest - changing dimension to the slowest changing dimension. -

The field Encode Size indicates the size of this field -

-
- - -
-
-
- -
- - - - - - - - - - - - - - - - - -
- Layout: Selection Info for H5S_SEL_ALL -
bytebytebytebyte
Version

Reserved (zero, - 8 bytes)

-
- -
-
-
- - - - - - - - - - - -
- Fields: Selection Info for H5S_SEL_ALL -
Field NameDescription

Version

The version number for the H5S_SEL_ALL Selection Info; - the value is 1.

-
- - -

VIII.B. Reference Encoding (Revised)

 -

-
- For the following reference type, - the Reference Header and Reference Block are stored together as the dataset's raw data: -

    -
  • Object Reference (H5R_OBJECT2) (without reference to an external file)
    • -
-

- For the following reference types, - the Reference Header plus the Global Heap ID are stored - as the dataset's raw data in the file. - The global heap ID is used to locate the Reference Block stored in the global heap: -

    -
  • Object Reference (H5R_OBJECT2) (with reference to an external file)
    • -
  • Dataset Region Reference (H5R_DATASET_REGION2) (with/without reference to an external file)
    • -
  • Attribute Reference (H5R_ATTR) (with/without reference to an external file)
    • -
-
-
- -
- - - - - - - - - - - - - - - - -
- Layout: Reference Header -
bytebytebytebyte
Reference TypeFlagsThis space inserted - only to align table nicely
- -
- -
-
-
- - - - - - - - - - - - - - - - - -
- Fields: Reference Header -
Field NameDescription

Reference Type

 -

There are 3 types of references: - - - - - - - - - - - - - - - - - - - - - -
ValueDescription
2H5R_OBJECT2: Object Reference -
3H5R_DATASET_REGION2: Dataset Region Reference -
4H5R_ATTR: Attribute Reference -
- -

Flags

This field describes the reference: - - - - - - - - - - - - - - -
BitDescription
0If set, the reference is to an external file. -
1-7Reserved

- -
-
- -
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Layout: Reference Block -
bytebytebytebyte
Token SizeThis space inserted - only to align table nicely


Token - (variable size)


Length of External File NameThis space inserted - only to align table nicely


External File Name - (variable size)


Size of Dataspace Selection
Rank of Dataspace Selection


Dataspace Selection Information - (variable size)


Length of Attribute Name This space inserted - only to align table nicely


Attribute Name - (variable size)


- -
- -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Fields: Reference Block -
Field NameDescription

Token size

This is the size of the token for the object. -

Token

 -

- This is the token for the object. -

-

Length of External File Name

This is the length for the external file name. -

This field exists if bit 0 of flags is set.

-

-

External File Name

This is the name of the external file being referenced.

-

-

This field exists if bit 0 of flags is set.

-

Dataspace Selection Information

See Dataspace Selection.

-

-

This field exists if the Reference Type is H5R_DATASET_REGION2.

-

Length of Attribute Name

This is the length of the attribute name. -

This field exists if the Reference Type is H5R_ATTRIBUTE.

-

Attribute Name

This is the name of the attribute being referenced. -

This field exists if the Reference Type is H5R_ATTRIBUTE.

-
-
- -
-
-
- - - -

VIII.C. Reference Encoding (Backward Compatibility)

 -

-
- The two references described below are maintained to preserve compatibility with previous versions of the library. -

- For the following reference type, - the reference encoding is stored as the dataset's raw data in the file: -

    -
  • Object Reference (H5R_OBJECT1)
    • -
-

- For the following reference type, - the Global Heap ID is stored as the dataset's raw data in the file. - The global heap ID is used to locate the reference encoding - stored in the global heap: -

    -
  • Dataset Region Reference (H5R_DATASET_REGION1)
    • -
- -
-
-
- - - - - - - - - - - - - - -
- Layout: Reference for H5R_OBJECT1 -
bytebytebytebyte

Object AddressO

- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
-
- - - - - - - - - - - - -
- Fields: Reference for H5R_OBJECT1 -
Field NameDescription

Object Address

 -

Address of the object being referenced -

-
- -
-
-
- -
- - - - - - - - - - - - - - - - - - -
- Layout: Reference for H5R_DATASET_REGION1 -
bytebytebytebyte

Object AddressO



Dataspace Selection Information - (variable size)


- - - - - -
  - (Items marked with an ‘O’ in the above table are - of the size specified in the Size - of Offsets field in the superblock.) -
- -
- -
-
-
- - - - - - - - - - - - - - - - - -
- Fields: Reference for H5R_DATASET_REGION1 -
Field NameDescription

Object Address

This is the address of the object being referenced. -

Dataspace Selection Information

This is the dataspace selection for the object being referenced. - See Dataspace Selection.

-

-
-
- -
-
-
- -
- - diff --git a/doxygen/examples/H5DS_Spec.pdf b/doxygen/examples/H5DS_Spec.pdf deleted file mode 100644 index 813f4ded3e1..00000000000 Binary files a/doxygen/examples/H5DS_Spec.pdf and /dev/null differ diff --git a/doxygen/examples/H5I_examples.c b/doxygen/examples/H5I_examples.c index 657e1b53d8d..6cf15b6d3bd 100644 --- a/doxygen/examples/H5I_examples.c +++ b/doxygen/examples/H5I_examples.c @@ -133,7 +133,7 @@ fail_dcpl:; hid_t obj_id; // register a new ID type - if ((type = H5Iregister_type(128, 1024, &free_func)) < 0) { + if ((type = H5Iregister_type2(1024, &free_func)) < 0) { ret_val = EXIT_FAILURE; goto fail_register; } @@ -167,7 +167,7 @@ fail_register:; hsize_t count; // register a new ID type - if ((type = H5Iregister_type(128, 1024, NULL)) < 0) { + if ((type = H5Iregister_type2(1024, NULL)) < 0) { ret_val = EXIT_FAILURE; goto fail_register; } @@ -194,7 +194,7 @@ fail_register:; hid_t obj_id; // register a new ID type - if ((type = H5Iregister_type(128, 1024, NULL)) < 0) { + if ((type = H5Iregister_type2(1024, NULL)) < 0) { ret_val = EXIT_FAILURE; goto fail_register; } @@ -224,7 +224,7 @@ fail_register:; H5I_type_t type; // register a new ID type - if ((type = H5Iregister_type(128, 1024, NULL)) < 0) { + if ((type = H5Iregister_type2(1024, NULL)) < 0) { ret_val = EXIT_FAILURE; goto fail_register; } diff --git a/doxygen/examples/ImageSpec.html b/doxygen/examples/ImageSpec.html deleted file mode 100644 index 130d86ecf6a..00000000000 --- a/doxygen/examples/ImageSpec.html +++ /dev/null @@ -1,1203 +0,0 @@ - - - - - - Image Specification - -The HDF5 specification defines the standard objects and storage for the -standard HDF5 objects. (For information about the HDF5 library, model and -specification, see the HDF documentation.)  This document is an additional -specification do define a standard profile for how to store image data -in HDF5. Image data in HDF5 is stored as HDF5 datasets with standard attributes -to define the properties of the image. -

This specification is primarily concerned with two dimensional raster -data similar to HDF4 Raster Images.  Specifications for storing other -types of imagery will be covered in other documents. -

This specification defines: -

    -
  • -Standard storage and attributes for an Image dataset (Section -1)
    • - -
  • -Standard storage and attributes for Palettes (Section -2)
    • - -
  • -Standard for associating Palettes with Images. (Section -3)
    • -
- -

-1. HDF5 Image Specification

- -

-1.1 Overview

-Image data is stored as an HDF5 dataset with values of HDF5 class Integer -or Float.  A common example would be a two dimensional dataset, with -elements of class Integer, e.g., a two dimensional array of unsigned 8 -bit integers.  However, this specification does not limit the dimensions -or number type that may be used for an Image. -

The dataset for an image is distinguished from other datasets by giving -it an attribute "CLASS=IMAGE".  In addition, the Image dataset may -have an optional attribute "PALETTE" that is an array of object references -for zero or more palettes. The Image dataset may have additional attributes -to describe the image data, as defined in Section 1.2. -

A Palette is an HDF5 dataset which contains color map information.  -A Pallet dataset has an attribute "CLASS=PALETTE" and other attributes -indicating the type and size of the palette, as defined in Section -2.1.  A Palette is an independent object, which can be shared -among several Image datasets. -

-1.2  Image Attributes

-The attributes for the Image are scalars unless otherwise noted.  -The length of String valued attributes should be at least the number of -characters. Optionally, String valued attributes may be stored in a String -longer than the minimum, in which case it must be zero terminated or null -padded.  "Required" attributes must always be used. "Optional" attributes -must be used when required. -
  -

-Attributes

- -
-
-Attribute name="CLASS" (Required)
- -
-This attribute is type H5T_C_S1, with size 5.
- -
-For all Images, the value of this attribute is "IMAGE".
- -
-
- -
-This attribute identifies this data set as intended to be interpreted as -an image that conforms to the specifications on this page.
-
- -
-Attribute name="PALETTE"
- -
-
-A Image dataset within an HDF5 file may optionally specify an array of -palettes to be viewed with. The dataset will have an attribute field called -"PALETTE" which contains a one-dimensional array of object reference -pointers (HDF5 datatype H5T_STD_REF_OBJ) which refer to palettes in the -file. The palette datasets must conform to the Palette specification in -section -2 below. The first palette in this array will be the default palette -that the data may be viewed with.
-
- -
-
-
- -
-Attribute name="IMAGE_SUBCLASS"
- -
-If present, the value of this attribute indicates the type of Palette that -should be used with the Image.  This attribute is a scalar of type -H5T_C_S1, with size according to the string plus one.  The values -are:
- -
-
-"IMAGE_GRAYSCALE" (length 15)
- -
-A grayscale image
- -
-"IMAGE_BITMAP" (length 12)
- -
-A bit map image
- -
-"IMAGE_TRUECOLOR" (length 15)
- -
-A truecolor image
- -
-"IMAGE_INDEXED" (length 13)
- -
-An indexed image
- -
-
-
- -
-Attribute name="INTERLACE_MODE"
- -
-For images with more than one component for each pixel, this optional attribute -specifies the layout of the data. The values are type H5T_C_S1 of length -15. See section 1.3 for information about the -storage layout for data.
- -
-"INTERLACE_PIXEL" (default): the component value for a pixel are contiguous.
- -
-"INTERLACE_PLANE": each component is stored as a plane.
- -
-
- -
-Attribute name="DISPLAY_ORIGIN"
- -
-This optional attribute indicates the intended orientation of the data -on a two-dimensional raster display.  The value indicates which corner -the pixel at (0, 0) should be viewed.  The values are type H5T_C_S1 -of length 2. If DISPLAY_ORIGIN is not set, the orientation is undefined.
- -
-"UL": (0,0) is at the upper left.
- -
-"LL": (0,0) is at the lower left.
- -
-"UR": (0,0) is at the upper right.
- -
-"LR": (0,0) is at the lower right.
-
- -
-Attribute name="IMAGE_WHITE_IS_ZERO"
- -
-
-This attribute is of type H5T_NATIVE_UCHAR.  0 = false, 1 = true .  -This is used for images with IMAGE_SUBCLASS="IMAGE_GRAYSCALE" or "IMAGE_BITMAP".
-
- -
-
-Attribute name="IMAGE_MINMAXRANGE"
- -
-If present, this attribute is an array of two numbers, of the same HDF5 -datatype as the data.  The first element is the minimum value of the -data, and the second is the maximum.  This is used for images with -IMAGE_SUBCLASS="IMAGE_GRAYSCALE", "IMAGE_BITMAP" or "IMAGE_INDEXED".
-
- -
-Attribute name="IMAGE_BACKGROUNDINDEX"
- -
-
-If set, this attribute indicates the index value that should be interpreted -as the "background color".  This attribute is HDF5 type H5T_NATIVE_UINT.
-
- -
-Attribute name="IMAGE_TRANSPARENCY"
- -
-
-If set, this attribute indicates the index value that should be interpreted -as the "transparent color".  This attribute is HDF5 type H5T_NATIVE_UINT.  -This attribute may not be used for IMAGE_SUBCLASS="IMAGE_TRUE_COLOR".
-
- -
-Attribute name="IMAGE_ASPECTRATIO"
- -
-
-If set, this attribute indicates the aspect ratio.
-
- -
-Attribute name="IMAGE_COLORMODEL"
- -
-
-If set, this attribute indicates the color model of Palette that should -be used with the Image.  This attribute is of type H5T_C_S1, with -size 3, 4, or 5.  The value is one of the color models described in -the Palette specification in section 2.2 below.  -This attribute may be used only for IMAGE_SUBCLASS="IMAGE_TRUECOLOR" or -"IMAGE_INDEXED".
-
- -
-Attribute name="IMAGE_GAMMACORRECTION"
- -
-
-If set, this attribute gives the Gamma correction.  The attribute -is type H5T_NATIVE_FLOAT.  This attribute may be used only for IMAGE_SUBCLASS="IMAGE_TRUECOLOR" -or "IMAGE_INDEXED".
-
-Attribute name="IMAGE_VERSION" (Required) -
-
-This attribute is of type H5T_C_S1, with size corresponding to the length -of the version string.  This attribute identifies the version number -of this specification to which it conforms.  The current version number -is "1.2".
- -
  -

  -
  -
  -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Table 1. Attributes of an Image Dataset
Attribute Name(R = Required -
O= Optional)		TypeString SizeValue
CLASSRString5"IMAGE"
PALETTEOArray Object References<references to Palette datasets>1
IMAGE_SUBCLASSO2String15,  -
12,  -
15, -
13		 -
-"IMAGE_GRAYSCALE",
- -
-"IMAGE_BITMAP",
- -
-"IMAGE_TRUECOLOR",
- -
-"IMAGE_INDEXED"
-
INTERLACE_MODEO3,6String15The layout of components if more than one component per pixel.
DISPLAY_ORIGINOString2If set, indicates the intended location of the pixel (0,0).
IMAGE_WHITE_IS_ZEROO3,4Unsigned Integer0 = false, 1 = true
IMAGE_MINMAXRANGEO3,5Array [2] <same datatype as data values>The (<minimum>, <maximum>) value of the data.
IMAGE_BACKGROUNDINDEXO3Unsigned IntegerThe index of the background color.
IMAGE_TRANSPARENCYO3,5Unsigned IntegerThe index of the transparent color.
IMAGE_ASPECTRATIOO3,4Unsigned IntegerThe aspect ratio.
IMAGE_COLORMODELO3,6String3, 4, or 5The color model, as defined below in the Palette specification for -attribute PAL_COLORMODEL.
IMAGE_GAMMACORRECTIONO3,6FloatThe gamma correction.
IMAGE_VERSIONRString3"1.2"
- -
1.  The first element of the array is the default -Palette. -
2.  This attribute is required for images -that use one of the standard color map types listed. -
3. This attribute is required if set for the source -image, in the case that the image is translated from another file into -HDF5. -
4.  This applies to:  IMAGE_SUBCLASS="IMAGE_GRAYSCALE" -or "IMAGE_BITMAP". -
5.  This applies to:  IMAGE_SUBCLASS="IMAGE_GRAYSCALE", -"IMAGE_BITMAP", or "IMAGE_INDEXED". -
6.  This applies to: IMAGE_SUBCLASS="IMAGE_TRUECOLOR", -or "IMAGE_INDEXED".
-
-Table 2 summarizes the standard attributes for an Image datasets using -the common sub-classes. R means that the attribute listed on the leftmost -column is Required for the image subclass on the first row, O means that -the attribute is Optional for that subclass and N that the attribute cannot -be applied to that subclass. The two first rows show the only required -attributes -for all subclasses. -
  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Table 2a. Applicability of Attributes to IMAGE sub-classes
IMAGE_SUBCLASS1IMAGE_GRAYSCALEIMAGE_BITMAP
CLASSRR
IMAGE_VERSIONRR
INTERLACE_MODENN
IMAGE_WHITE_IS_ZERORR
IMAGE_MINMAXRANGEOO
IMAGE_BACKGROUNDINDEXOO
IMAGE_TRANSPARENCYOO
IMAGE_ASPECTRATIOOO
IMAGE_COLORMODELNN
IMAGE_GAMMACORRECTIONNN
PALETTEOO
DISPLAY_ORIGINOO
- -
 
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Table 2b. Applicability of Attributes to IMAGE sub-classes
IMAGE_SUBCLASSIMAGE_TRUECOLORIMAGE_INDEXED
CLASSRR
IMAGE_VERSIONRR
INTERLACE_MODERN
IMAGE_WHITE_IS_ZERONN
IMAGE_MINMAXRANGENO
IMAGE_BACKGROUNDINDEXNO
IMAGE_TRANSPARENCYNO
IMAGE_ASPECTRATIOOO
IMAGE_COLORMODELOO
IMAGE_GAMMACORRECTIONOO
PALETTEOO
DISPLAY_ORIGINOO
- -

-1.3 Storage Layout and Properties for Images

-In the case of an image with more than one component per pixel (e.g., Red, -Green, and Blue), the data may be arranged in one of two ways.  Following -HDF4 terminology, the data may be interlaced by pixel or by plane, which -should be indicated by the INTERLACE_MODE  attribute.  In both -cases, the dataset will have a dataspace with three dimensions, height, -width, and components.  The interlace modes specify different orders -for the dimensions. -
  - - - - - - - - - - - - - - - - - - - - -
Table 3. Storage of multiple component image data.
Interlace ModeDimensions in the Dataspace
INTERLACE_PIXEL[height][width][pixel components]
INTERLACE_PLANE[pixel components][height][width]
- -

For example, consider a 5 (rows) by 10 (column) image, with Red, Green, -and Blue components.  Each component is an unsigned byte. In HDF5, -the datatype would be declared as an unsigned 8 bit integer.  For -pixel interlace, the dataspace would be a three dimensional array, with -dimensions: [10][5][3].  For plane interleave, the dataspace would -be three dimensions: [3][10][5]. -

In the case of images with only one component, the dataspace may be -either a two dimensional array, or a three dimensional array with the third -dimension of size 1.  For example, a 5 by 10 image with 8 bit color -indexes would be an HDF5 dataset with type unsigned 8 bit integer.  -The dataspace could be either a two dimensional array, with dimensions -[10][5], or three dimensions, with dimensions either [10][5][1] or [1][10][5]. -

Image datasets may be stored with any chunking or compression properties -supported by HDF5. -

A note concerning compatibility with HDF5 GR interface: An Image -dataset is stored as an HDF5 dataset.  It is important to note that -the order of the dimensions is the same as for any other HDF5 dataset.  -For a two dimensional image that is to be stored as a series of horizontal -scan lines, with the scan lines contiguous (i.e., the fastest changing -dimension is 'width'), the image will have a dataspace with dim[0] = -height and dim[1] = width.  This is completely consistent -with all other HDF5 datasets. -

Users familiar with HDF4 should be cautioned that this is not the -same as HDF4, and specifically is not consistent with what the HDF4 -GR interface does. -
  -

-2.  HDF5 Palette Specification

- -

-2.1 Overview

-A palette is the means by which color is applied to an image and is also -referred to as a color lookup table. It is a table in which every row contains -the numerical representation of a particular color. In the example of an -8 bit standard RGB color model palette, this numerical representation of -a color is presented as a triplet specifying the intensity of red, green, -and blue components that make up each color. -
-

- -

In this example, the color component numeric type is an 8 bit unsigned -integer. While this is most common and recommended for general use, other -component color numeric datatypes, such as a 16 bit unsigned integer , -may be used. This type is specified as the type attribute of the palette -dataset. (see H5Tget_type(), H5Tset_type()) -

The minimum and maximum values of the component color numeric are specified -as attribute of the palette dataset. See below (attribute PAL_MINMAXNUMERIC). -If these attributes do not exist, it is assumed that the range of values -will fill the space of the color numeric type. i.e. with an 8 bit unsigned -integer, the valid range would be 0 to 255 for each color component. -

The HDF5 palette specification additionally allows for color models -beyond RGB. YUV, HSV, CMY, CMYK, YCbCr color models are supported, and -may be specified as a color model attribute of the palette dataset. (see -"Palette Attributes" for details). -

In HDF 4 and earlier, palettes were limited to 256 colors. The HDF5 -palette specification allows for palettes of varying length. The length -is specified as the number of rows of the palette dataset. -
  -
  - - - - -
Important Note: The specification of the Indexed -Palette will change substantially in the next version.  The Palette -described here is denigrated and is not supported.
- -
  - - - - -
Denigrated -

In a standard palette, the color entries are indexed directly. HDF5 -supports the notion of a range index table. Such a table defines an ascending -ordered list of ranges that map dataset values to the palette. If a range -index table exists for the palette, the PAL_TYPE attribute will be set -to "RANGEINDEX", and the PAL_RANGEINDEX attribute will contain an object -reference to a range index table array. If not, the PAL_TYPE attribute -either does not exist, or will be set to "STANDARD". -

The range index table array consists of a one dimensional array with -the same length as the palette dataset - 1. Ideally, the range index would -be of the same type as the dataset it refers to, however this is not a -requirement. -

Example 2: A range index array of type floating point -

-

- -

The range index array attribute defines the "to" of the range. -Notice that the range index array attribute is one less entry in size than -the palette. The first entry of 0.1259, specifies that all values below -and up to 0.1259 inclusive, will map to the first palette entry. The second -entry signifies that all values greater than 0.1259 up to 0.3278 inclusive, -will map to the second palette entry, etc. All value greater than the last -range index array attribute (100000) map to the last entry in the palette.

- -

-2.2. Palette Attributes

-A palette exists in an HDF file as an independent data set with accompanying -attributes.  The Palette attributes are scalars except where noted -otherwise.  String values should have size the length of the string -value plus one.  "Required" attributes must be used.  "Optional" -attributes must be used when required. -

These attributes are defined as follows: -

-
-Attribute name="CLASS" (Required)
- -
-This attribute is of type H5T_C_S1, with size 7.
- -
-For all palettes, the value of this attribute is "PALETTE". This attribute -identifies this palette data set as a palette that conforms to the specifications -on this page.
- -
-Attribute name="PAL_COLORMODEL" (Required)
- -
-This attribute is of type H5T_C_S1, with size 3, 4, or 5.
- -
-Possible values for this are "RGB", "YUV", "CMY", "CMYK", "YCbCr", "HSV".
- -
-This defines the color model that the entries in the palette data set represent.
- -
-
-"RGB"
- -
-Each color index contains a triplet where the first value defines the -red component, second defines the green component, and the third the blue -component.
- -
-"CMY"
- -
-Each color index contains a triplet where the first value defines the -cyan component, second defines the magenta component, and the third the -yellow component.
- -
-"CMYK"
- -
-Each color index contains a quadruplet where the first value defines -the cyan component, second defines the magenta component, the third the -yellow component, and the forth the black component.
- -
-"YCbCr"
- -
-Class Y encoding model. Each color index contains a triplet where the -first value defines the luminance, second defines the Cb Chromonance, and -the third the Cr Chromonance.
- -
-"YUV"
- -
-Composite encoding color model. Each color index contains a triplet where -the first value defines the luminance component, second defines the -chromonance component, and the third the value component.
- -
-"HSV"
- -
-Each color index contains a triplet where the first value defines the -hue component, second defines the saturation component, and the third the -value component. The hue component defines the hue spectrum with a low -value representing magenta/red progressing to a high value which would -represent blue/magenta, passing through yellow, green, cyan. A low value -for the saturation component means less color saturation than a high value. -A low value for value will be darker than a high value.
- -
-
-
- -
-Attribute name="PAL_TYPE" (Required)
- -
-This attribute is of type H5T_C_S1, with size 9 or 10.
- -
-The current supported values for this attribute are : "STANDARD8" or "RANGEINDEX"
- -
-A PAL_TYPE of "STANDARD8" defines a palette dataset such that the first -entry defines index 0, the second entry defines index 1, etc. up until -the length of the palette - 1. This assumes an image dataset with direct -indexes into the palette.
-
- -
  - - - - -
Denigrated -

If the PAL_TYPE is set to "RANGEINDEX", there will be an additional -attribute with a name of "PAL_RANGEINDEX",  (See example 2 -for more details)

- - - - - -
-
-Attribute name="PAL_RANGEINDEX"   (Denigrated)
- -
-
-The PAL_RANGEINDEX attribute contains an HDF object reference (HDF5 -datatype H5T_STD_REF_OBJ) pointer which specifies a range index array in -the file to be used for color lookups for the palette.  (Only for -PAL_TYPE="RANGEINDEX")
-
-
- -
-Attribute name="PAL_MINMAXNUMERIC"
- -
-
-If present, this attribute is an array of two numbers, of the same HDF5 -datatype as the palette elements or color numerics.
- -
They specify the minimum and maximum values of the color numeric components. -For example, if the palette was an RGB of type Float, the color numeric -range for Red, Green, and Blue could be set to be between 0.0 and 1.0. -The intensity of the color guns would then be scaled accordingly to be -between this minimum and maximum attribute.
-Attribute name="PAL_VERSION"  (Required) -
This attribute is of type H5T_C_S1, with size corresponding to the -length of the version string.  This attribute identifies the version -number of this specification to which it conforms.  The current version -is "1.2".
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Table 4. Attributes of a Palette Dataset
Attribute Name(R = Required, -
O = Optional)		TypeString SizeValue
CLASSRString -
7
-		"PALETTE"
PAL_COLORMODELRString -
3, 4, or 5
-		Color Model:  "RGB", YUV", "CMY", "CMYK", "YCbCr", or "HSV"
PAL_TYPERString -
9
- -


- - - - -
or 10
-

"STANDARD8"  - - - - -
or "RANGEINDEX" (Denigrated)
-
- - - - -
Denigrated -
RANGE_INDEX
-		 - - - - -
Object Reference 
-		 - - - - -
<Object Reference to Dataset of range index values>
-
PAL_MINMAXNUMERICOArray[2] of <same datatype as palette>The first value is the <Minimum value for color values>, the second -value is <Maximum value for color values>2
PAL_VERSIONRString4"1.2"
- -
  - - - - -
1.  The RANGE_INDEX attribute is required if the -PAL_TYPE is "RANGEINDEX".  Otherwise, the RANGE_INDEX attribute should -be omitted. (Range index is denigrated.)
-2.  The minimum and maximum are optional.  If not -set, the range is assumed to the maximum range of the number type.  -If one of these attributes is set, then both should be set.  The value -of the minimum must be less than or equal to the value of the maximum.
-
-Table 5 summarized the uses of the standard attributes for a palette dataset. -R means that the attribute listed on the leftmost column is Required for -the palette type on the first row, O means that the attribute is Optional -for that type and N that the attribute cannot be applied to that type. -The four first rows show the attributes that are always required  -for the two palette types. -
  -
  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Table 5. Applicability of Attributes
PAL_TYPESTANDARD8RANGEINDEX
CLASSRR
PAL_VERSIONRR
PAL_COLORMODELRR
RANGE_INDEXNR
PAL_MINMAXNUMERICOO
- -

-2.3. Storage Layout for Palettes

-The values of the Palette are stored as a dataset.  The datatype can -be any HDF 5 atomic numeric type.  The dataset will have dimensions -(nentries  by  ncomponents), where 'nentries' -is the number of colors (usually 256) and 'ncomponents' is the -number of values per color (3 for RGB, 4 for CMYK, etc.) -
  -

-3.  Consistency and Correlation of Image and Palette -Attributes

-The objects in this specification are an extension to the base HDF5 specification -and library.  They are accessible with the standard HDF5 library, -but the semantics of the objects are not enforced by the base library.  -For example, it is perfectly possible to add an attribute called IMAGE -to any dataset, or to include an object reference to any -HDF5 dataset in a PALETTE attribute.  This would be a valid -HDF5 file, but not conformant to this specification.  The rules defined -in this specification must be implemented with appropriate software, and -applications must use conforming software to assure correctness. -

The Image and Palette specifications include several redundant standard -attributes, such as the IMAGE_COLORMODEL and the PAL_COLORMODEL.  -These attributes are informative not normative, in that it is acceptable -to attach a Palette to an Image dataset even if their attributes do not -match.  Software is not required to enforce consistency, and files -may contain mismatched associations of Images and Palettes.  In all -cases, it is up to applications to determine what kinds of images and color -models can be supported. -

For example, an Image that was created from a file with an "RGB" may -have a "YUV" Palette in its PALETTE attribute array.  This -would be a legal HDF5 file and also conforms to this specification, although -it may or may not be correct for a given application.

- - - diff --git a/doxygen/examples/LibraryReleaseVersionNumbers.html b/doxygen/examples/LibraryReleaseVersionNumbers.html deleted file mode 100644 index dedbece0c11..00000000000 --- a/doxygen/examples/LibraryReleaseVersionNumbers.html +++ /dev/null @@ -1,318 +0,0 @@ - - -HDF5 Library Release Version Numbers - - - - -
-

Contents

- - - -

Introduction

- -

HDF5 software is updated on a regular basis. These updates, known -as releases, range in scope and size from small to large. Some updates -may only fix bugs, and some updates may require a change in the format -of the data file. The version numbers that are applied to updates give -information about the kinds of changes made in the updates. This Tech -Note describes what the version numbers mean.

- -

Note that this document describes release version numbers for the -HDF5 Library. For more information, see the -Shared Library Version Numbers section at the end of this document.

- -

Definitions

- -

Each software release of the HDF5 Library is labeled with a version - number. The version number is a set of three integers written as HDF5-1.2.3, - HDF5 version 1.2 release 3, or HDF5 Release 1.2.3. The version number - might also include text. A patch version might be labeled HDF5-1.2.3-patch1. - The '5' in "HDF5" is part of the product name and will not change during - the life of the project.

- -

The key components in HDF5 Library version numbers are the major version - number, the minor version number, the release number, and an optional text - string.

- -

The first integer in a version number is the major version - number. This integer increments when there is an extensive change - to the file format or library API. Such a change may require files to - be translated and will likely require applications to be modified.

- -

The second integer, 2 in the examples above, is the minor version - number. This number is incremented when there are new features that - require a change in the file format. For example, a change in file format - was required during the change from version 1.6 to version 1.8. Stable - released versions of the library are given even minor version - numbers such as 1.6 and 1.8 while odd minor version numbers such - as 1.7 and 1.9 are used on the trunk for major development. See the - section below for more information.

- -

The third integer, 3 in the examples above, is the release - number. A change in this number indicates that the library has - been updated. The updates might include bug fixes, performance - improvements, and new features that do not require a file format - change.

- -

A version number might also include some text. The two current - possibilities are patch and snap. A patch version might - be made to a released version to make available a feature or a bug - fix. In the figure below, a patch to the 1.8.5 release is labeled - 1.8.5-patch1. A snapshot is an intermediate posting of the software - in a branch or in the trunk. Snapshots are made available so that users - may begin to test changes in the software that affect their software. - The changes may range from bug fixes to new features. Snapshots are made - and released regularly. How regularly depends on whether the software - passes the tests done on each build. A possible version number for a - snapshot might be 1.9-snap81. This version would hold the 81st snapshot - off the 1.9 development branch (the current trunk). For the - snapshots are available at - https://github.com/HDFGroup/hdf5/releases/tag/snapshot.

- -

The Trunk, Release Branches, and Feature Branches

- -

The HDF Group uses a version control system to manage the HDF5 - project. Within the system, a trunk and branches are used to track - changes. The version numbers described above identify where a given - piece of software was produced in the system. The figure below shows - the general scheme.

- - - - - - - - -
-
- The trunk, release branches, and feature branches
-
- Figure 1. The trunk, release branches, and feature branches -
- -

The trunk is the center of the system. New features are - implemented in feature branches and aggregated in the trunk. - Release branches are then created from the trunk.

- -

The minor version number of the trunk is always an odd number. From - the time of Release 1.8.0 to the first 1.10 release, the trunk will be - version 1.9. The trunk was version 1.7 from the time of release 1.6.0 - until the first 1.8 release.

- -

Projects that add new features, bug fixes, and performance improvements - are developed on feature branches. When a project is completed, - its feature branch is merged into the trunk. In the figure above, the - merging of a feature branch is represented by a dashed arrow from the - feature branch to the trunk. If a feature requires a file format change, - then the feature will stay in the trunk until the next significant - release. This would mean in the figure above that the new feature would - be released in a future 1.10 release branch. If a feature does not - require a file format change, then it might be merged into one or more - release branches. This would mean in the figure above that the new - feature could be merged into the 1.8 branch and could be included in - the 1.8.6 release. If the feature was added to the 1.8.5 branch, then a - patch version might be released.

- -

Release branches hold software that is distributed to general - users. In the figure above, a few release branches are shown below the - trunk. Work is done in release branches for a period of time. Branches - further from the trunk have less work done in them. For example, a patch - branch such as 1.8.5-patch1 may contain only one or two changes. A release - branch such as 1.8.5 may contain a number of bug fixes and new functions, - but these changes are small in number compared to the number of changes in - the 1.8 branch.

- -

We aim to make available to the public two maintenance releases a year. - The releases occur usually in the spring near May 15 and in the fall near - November 15. If two release branches are being maintained, then - maintenance releases may be made for each release branch. For example, - there was a time when both the 1.6 and 1.8 branches were actively - maintained. In one maintenance release, the 1.6.10 and 1.8.4 versions were - released at the same time. The 1.6 and 1.8 branches were both actively - maintained to give early adopters access to new features and to give most - users plenty of time to make the change to 1.8 software from 1.6.

- -

As we improve any branch, we consider the effect of any change on the - readability of objects. Applications built, for example, with version - 1.8.5 will be able to read data files written with any prior version - of the library. So, a 1.8.5 application will be able to read a dataset - written with 1.4.5. A 1.8.5 application may be able to read a dataset - written under the 1.8.7 library if no new features, features not known - to 1.8.5, were used. - -

Version Support from the Library

- -

The library provides macros and functions to query and check - version numbers.

- -

The following constants are defined in the file H5public.h - and determine the version of the include files.

- -
    -
  • H5_VERS_MAJOR
    • -
  • H5_VERS_MINOR
    • -
  • H5_VERS_RELEASE
    • -
  • H5_VERS_SUBRELEASE
    • -
  • H5_VERS_INFO
    • -
- -

The table below describes some of the function calls and macros - that can be used to query and check version numbers.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Table 1. Version function calls and macros
- Function Call or Macro - Comments
H5get_libversionThis function returns through its arguments the version - numbers for the library to which the application is linked.
H5checkThis macro uses the H5check_version function - to verify that the version number of the HDF5 include file used - to compile the application matches the version number of the - library to which the application is linked. This check occurs - automatically when the first HDF5 file is created or opened and - is important because a mismatch between the include files and the - library is likely to result in corrupted data and/or segmentation - faults. If a mismatch is detected, the library issues an error - message on the standard error stream and aborts.
H5check_version

This function is called by the H5check macro - with the include file version constants. The function compares - its arguments to the result returned by H5get_libversion. - If a mismatch is detected, it prints an error message on the standard - error stream and aborts.

- -

The behavior of this function can be modified by the - HDF5_DISABLE_VERSION_CHECK environment variable. Setting - the environment variable to a value of =911=92 will issue a - warning but continue without aborting. Setting the environment - variable to a value of =912=92 will suppress the warning - and continue silently without aborting.

H5_VERSION_GE and - H5_VERSION_LEThese macros compare the version of the HDF5 library being used - against the version number specified in the parameters. At compile - time, they can be used to conditionally include or exclude code - based on the library's version.
H5Pset_libver_boundsThis function can be used to control the versions of the object - formats that will be used when creating objects in a file.
- -

For more information on these and other function calls and macros, - see the HDF5 Reference Manual.

- -

Use Cases

- -

The purpose of this section is to describe how some of the version - functions, macros, and constants might be used.

- -

Application Version Checking

- -

Suppose first that a developer builds an application that will read - from and write to an HDF5 file. When the application is compiled, a - version of the HDF5 Library such as 1.8.6 will be used. The version - constants (H5_VERS_MAJOR, H5_VERS_MINOR, - and H5_VERS_RELEASE) are included in the application when - it is compiled.

- -

Suppose next that a user gets a copy of the application and starts it - up on a workstation. The executable is put into memory along with the - HDF5 Library. However, an application may only work successfully with - the version of the library with which the application was compiled. In - other words, the version of the library that is loaded when the applicati= -on - is started must be the same version as the version of the library with - which the application was compiled. This is verified by the library when - the first HDF5 API routine is called. If an application wants to confirm - early in its startup procedure that the version of the library that will - be loaded into memory at the workstation will work with the application, - then it can use the H5get_libversion and - H5check_version function calls.

- -

Conditional Inclusions or Exclusions Based on the Version

- -

The H5_VERSION_GE and H5_VERSION_LE version - macros compare the version of the HDF5 Library being used against the - version number specified in the parameters. At compile time, they can be - used to conditionally include or exclude code based on the library's - version. For example, the link functions, H5Lxxx, are - new in version 1.8, and some group functions, H5Gxxx, - are deprecated in 1.8. With the H5_VERSION_GE macro, an - application could use H5Ldelete if the library version is - 1.8.0 or greater, or it could use H5Gunlink if the library - version is less than 1.8.0.

- -

Specifying a Format

-

Suppose a data file has three datasets. It is possible that the three - datasets were added to the data file with applications using different - versions of HDF5. The different versions could be 1.4.5, 1.6.10, and - 1.8.6. If another dataset is written to the data file, then it will be - written by default in the oldest format possible that has all of the - features needed to successfully write the dataset. If a newer feature - such as compact storage, a new parameter for a function, or a partially - compressed dataset is used, then a newer format will be used. - H5Pset_libver_bounds could be used to specify the oldest - format used. In the situation above, the owners of the data file might - want all data written to the file in the future to be in a 1.8 format - rather than 1.6 or 1.4.

- -

Shared Library Version Numbers

- -

HDF5 shared libraries utilize the - -libtool versioning system in order to indicate interface -compatibility between maintenance releases of HDF5. While we always -attempt to maintain interface compatibility between minor maintenance -release versions of HDF5, if we are forced to break interface -compatibility in order to resolve a critical defect within the -library, then the library interface version attached to the shared -libraries for a given release will be incremented accordingly.

- -

Please note that this libtool version number for interface -compatibility is unrelated to the HDF5 release version for a given -release.

- - - diff --git a/doxygen/img/H5DS_fig1.png b/doxygen/img/H5DS_fig1.png new file mode 100644 index 00000000000..f47619eb6db Binary files /dev/null and b/doxygen/img/H5DS_fig1.png differ diff --git a/doxygen/img/H5DS_fig2.png b/doxygen/img/H5DS_fig2.png new file mode 100644 index 00000000000..6eace5a9d05 Binary files /dev/null and b/doxygen/img/H5DS_fig2.png differ diff --git a/doxygen/img/H5DS_fig3.png b/doxygen/img/H5DS_fig3.png new file mode 100644 index 00000000000..d0771268fdc Binary files /dev/null and b/doxygen/img/H5DS_fig3.png differ diff --git a/doxygen/img/H5DS_fig4.png b/doxygen/img/H5DS_fig4.png new file mode 100644 index 00000000000..6e1de71c73f Binary files /dev/null and b/doxygen/img/H5DS_fig4.png differ diff --git a/doxygen/img/H5DS_fig5.png b/doxygen/img/H5DS_fig5.png new file mode 100644 index 00000000000..3d1e7ecf241 Binary files /dev/null and b/doxygen/img/H5DS_fig5.png differ diff --git a/fortran/src/H5Pf.c b/fortran/src/H5Pf.c index 079d0277a43..31cb0f714f4 100644 --- a/fortran/src/H5Pf.c +++ b/fortran/src/H5Pf.c @@ -3504,37 +3504,6 @@ h5pget_attr_creation_order_c(hid_t_f *ocpl_id, int_f *crt_order_flags) ret_value = 0; return ret_value; } -/****if* H5Pf/h5pset_link_creation_order_c - * NAME - * h5pset_link_creation_order_c - * PURPOSE - * Calls H5Pset_link_creation_order - * - * INPUTS - * gcpl_id - Group creation property list identifier - * crt_order_flags - Creation order flag(s) - * OUTPUTS - * - * RETURNS - * 0 on success, -1 on failure - * SOURCE - */ -int_f -h5pset_link_creation_order_c(hid_t_f *gcpl_id, int_f *crt_order_flags) -/******/ -{ - int ret_value = -1; - herr_t ret; - /* - * Call H5Pset_link_creation_order function. - */ - ret = H5Pset_link_creation_order((hid_t)*gcpl_id, (unsigned)*crt_order_flags); - if (ret < 0) - return ret_value; - - ret_value = 0; - return ret_value; -} /****if* H5Pf/h5pget_link_phase_change_c * NAME @@ -3682,45 +3651,6 @@ h5pset_create_inter_group_c(hid_t_f *lcpl_id, int_f *crt_intermed_group) return ret_value; } -/****if* H5Pf/h5pget_link_creation_order_c - * NAME - * h5pget_link_creation_order_c - * PURPOSE - * Calls H5Pget_link_creation_order - * - * INPUTS - * - * gcpl_id - Group creation property list identifier - * OUTPUTS - * - * crt_order_flags - Creation order flag(s) - * - * RETURNS - * 0 on success, -1 on failure - * SOURCE - */ -int_f -h5pget_link_creation_order_c(hid_t_f *gcpl_id, int_f *crt_order_flags) -/******/ -{ - int ret_value = -1; - herr_t ret; - - unsigned c_crt_order_flags; - /* - * Call h5pget_link_creation_order function. - */ - - ret = H5Pget_link_creation_order((hid_t)*gcpl_id, &c_crt_order_flags); - if (ret < 0) - return ret_value; - - *crt_order_flags = (int_f)c_crt_order_flags; - - ret_value = 0; - return ret_value; -} - /****if* H5Pf/h5pset_char_encoding_c * NAME * h5pset_char_encoding_c diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index 7e73ac2255a..41529a17180 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -3470,17 +3470,17 @@ SUBROUTINE h5pset_link_creation_order_f(gcpl_id, crt_order_flags, hdferr) INTEGER, INTENT(IN) :: crt_order_flags INTEGER, INTENT(OUT) :: hdferr INTERFACE - INTEGER FUNCTION h5pset_link_creation_order_c(gcpl_id, crt_order_flags) & - BIND(C,NAME='h5pset_link_creation_order_c') - IMPORT :: HID_T + INTEGER FUNCTION h5pset_link_creation_order(gcpl_id, crt_order_flags) & + BIND(C,NAME='H5Pset_link_creation_order') + IMPORT :: HID_T, C_INT IMPLICIT NONE - INTEGER(HID_T), INTENT(IN) :: gcpl_id - INTEGER, INTENT(IN) :: crt_order_flags + INTEGER(HID_T), VALUE :: gcpl_id + INTEGER(C_INT), VALUE :: crt_order_flags - END FUNCTION H5pset_link_creation_order_c + END FUNCTION H5pset_link_creation_order END INTERFACE - hdferr = h5pset_link_creation_order_c(gcpl_id, crt_order_flags) + hdferr = h5pset_link_creation_order(gcpl_id, INT(crt_order_flags, C_INT)) END SUBROUTINE h5pset_link_creation_order_f @@ -3634,18 +3634,21 @@ SUBROUTINE h5pget_link_creation_order_f(gcpl_id, crt_order_flags, hdferr) INTEGER(HID_T), INTENT(IN) :: gcpl_id INTEGER, INTENT(OUT) :: crt_order_flags INTEGER, INTENT(OUT) :: hdferr + + INTEGER(C_INT) :: c_crt_order_flags INTERFACE - INTEGER FUNCTION h5pget_link_creation_order_c(gcpl_id, crt_order_flags) & - BIND(C,NAME='h5pget_link_creation_order_c') - IMPORT :: HID_T + INTEGER FUNCTION h5pget_link_creation_order(gcpl_id, crt_order_flags) & + BIND(C,NAME='H5Pget_link_creation_order') + IMPORT :: HID_T, C_INT IMPLICIT NONE - INTEGER(HID_T), INTENT(IN) :: gcpl_id - INTEGER, INTENT(OUT) :: crt_order_flags + INTEGER(HID_T), VALUE :: gcpl_id + INTEGER(C_INT) :: crt_order_flags - END FUNCTION H5pget_link_creation_order_c + END FUNCTION H5pget_link_creation_order END INTERFACE - hdferr = h5pget_link_creation_order_c(gcpl_id, crt_order_flags) + hdferr = h5pget_link_creation_order(gcpl_id, c_crt_order_flags) + crt_order_flags = INT(c_crt_order_flags) END SUBROUTINE h5pget_link_creation_order_f diff --git a/hl/src/H5DSpublic.h b/hl/src/H5DSpublic.h index a926035a4c8..f38f6fef0ba 100644 --- a/hl/src/H5DSpublic.h +++ b/hl/src/H5DSpublic.h @@ -31,6 +31,725 @@ extern "C" { #endif /** \page H5DS_UG HDF5 High Level Dimension Scales + * + * \section sec_dim_scales_stand HDF5 Standard for Dimension Scales + * Dimension scales are stored as datasets, with additional metadata indicating that they are to + * be treated as dimension scales. Each dimension scale has an optional name. There is no requirement + * as to where dimension scales should be stored in the file. Dimension Scale names are not required + * to be unique within a file. (The name of a dimension scale does not have to be the same as the HDF5 + * path name for the dataset representing the scale.) + * + * Datasets are linked to dimension scales. Each dimension of a Dataset may optionally have one or + * more associated Dimension Scales, as well as a label for the dimension. A Dimension Scale can be + * shared by two or more dimensions, including dimensions in the same or different dataset. + * Relationships between dataset dimensions and their corresponding dimension scales are not be directly + * maintained or enforced by the HDF5 library. For instance, a dimension scale would not be automatically + * deleted when all datasets that refer to it are deleted. + * + * Functions for creating and using Dimension Scales are implemented as high level functions, see \ref H5DS. + * + * A frequently requested feature is for Dimension Scales to be represented as functions, rather than a + * stored array of precomputed values. To meet this requirement, it is recommended that the dataset + * model be expanded in the future to allow datasets to be represented by a generating function. + * + * \section sec_dim_scales_concept Conceptual model + * Our study of dimension scale use cases has revealed an enormous variety of ways that dimension + * scales can be used. We recognize the importance of having a model that will be easy to understand + * and use for the vast majority of applications. It is our sense that those applications will need + * either no scale, a single 1-D array of floats or integers, or a simple function that provides a + * scale and offset. + * + * At the same time, we want to place as few restrictions as possible on other uses of dimension + * scales. For instance, we don’t want to require dimension scales to be 1-D arrays, or to allow + * only one scale per dimension. + * + * So our goal is to provide a model that serves the needs of two communities. We want to keep + * the dimension scale model conceptually simple for the majority of applications, but also to + * place as few restrictions as possible how dimension scales are interpreted and used. With + * this approach, it becomes the responsibility of applications to make sure that dimension scales + * satisfy the constraints of the model that they are assuming, such as constraints on the size + * of dimension scales and valid range of indices. + * + * \subsection subsec_dim_scales_concept_defs Definitions + * Dimension Scales are implemented as an extension of these objects. In the HDF5 Abstract Data + * Model, a Dataset has a Dataspace, which defines a multi dimensional array of elements. Conceptually, + * a Dataspace has N dimension objects, which define the current and maximum size of the array in + * that dimension. + * + * It is important to emphasize that the Dataspace of a Dataset has no intrinsic meaning except + * to define the layout in computer storage. Dimension Scales may be used to store application + * specific labels to the positions in the stored data array, i.e., to add application specific + * meaning to the dimensions of the dataspace. + * + * A Dimension Scale is an object associated with one dimension of a Dataspace. The meaning of + * the association is left to applications. The values of the Dimension Scale are set by the + * application to reflect semantics of the data, for example, to associate coordinates of a + * reference system with positions on the dimension. + * + * In general, these associations define a mapping between values of a dimension index and + * values of the Dimension Scale dataset. A simple case is where the Dimension Scale s is + * a (one dimensional) sequence of labels for the dimension ix of Dataset d. In this case, + * Dimension Scale is an array indexed by the same index as in the dimension of the Dataspace. + * For example, for the Dimension Scale s, associated with dimension ix, the ith position of + * ix is associated with the value s[i], so s[i] is taken as a label for ix[i]. + * + * \subsection subsec_dim_scales_concept_rel Entity Relationship Diagrams + * Figure 1 shows UML to illustrate the relationship between a Dimension and a Dimension + * Scale object. Conceptually, each Dimension of a Dataspace may have zero or more Dimension + * Scales associated with it. In turn, a Dimension Scale object may be associated with zero + * or more Dimensions (in zero or more Dataspaces). + * + * Figure 2 illustrates the abstract model for a Dimension Scale object. A Dimension Scale + * is represented as a sub-class of a Dataset: a Dimension Scale has all the properties of + * a Dataset, with some specializations. A Dimension Scale dataset has an attribute “CLASS” + * with the value “DIMENSION_SCALE”. (This is analogous to the Table, Image, and Palette + * objects.) The Dimension Scale dataset has other attributes, including an optional + * NAME and references to any associated Dataset, as discussed below. + * + * When the Dimension Scale is associated with a dimension of a Dataset, the association + * is represented by attributes of the two datasets. In the Dataset, the DIMENSION_LIST + * is an array of object references to scales (Dimension Scale Datasets) (Figure 1), and + * in the Dimension Scale Dataset the REFERENCE_LIST is an array of object references + * to Datasets (Figure 2). + * + * + * + * + *
+ * \image html H5DS_fig1.png "Figure 1. The relationship between a Dimension and a Dimension Scale." + *
+ * + * + * + * + * + *
+ * \image html H5DS_fig2.png "Figure 2. The definition of a Dimension Scale and its attributes." + *
+ * + * \subsection subsec_dim_scales_concept_types What types of scales should be implemented? + * There seems to be good agreement that the model should accommodate scales that consist of a + * stored 1-D list of values, certain simple functions, and “no scale.” This specification also + * includes scales that are higher dimensional arrays, as well. + * + * The four types of scales are: + * \li No scale. Frequently no scale is needed, so it should not be required. + * In some of these cases, an axis label may still be needed, and should be available. + * In this case, the Datase * t defines a Dimension Scale label for a dimension, with no + * Dimension Scale dataset. + * \li 1-D array. Both fixed length and extendable arrays should be available. + * The size of the Dimension Scale is not required by HDF5 to conform to the size of the + * corresponding dimension(s), so that the number of scale values could be less than, + * equal to, or greater than the corresponding dimension size. + * \li Simple function. At a minimum, a linear scale of the form A + Bx + * should be available. This will be discussed in a future proposal. + * \li Higher dimensional arrays. This specification allows Dimension Scales + * to have any number of dimensions. + * + * A number of use cases have been proposed in which more than one scale is needed for a given dimension. + * This specification places no restrictions on the number of scales that can be associated with a dimension, + * nor on the number or identities of Dimensions that may share the same Dimension Scale. + * + * There are use cases for storing many types of data in a scale, including, + * but not limited to integers, floats, and strings. Therefore, this specification places no restrictions + * on the datatypes of scale values: a Dimension Scale can have any HDF5 Datatype. The + * interpretation of dimension scale values is left to applications. + * + * \subsection subsec_dim_scales_concept_limits Limitations of this Specification + * One-to-many mapping. When there are fewer values in a dimension scale than in the corresponding + * dimension, it is useful to have a mapping between the two. For example, mappings are used by + * HDF-EOS to map geolocation information to dimension, in order that, for example, every second + * point may have geolocation. On the other hand, the way that mappings are defined can be very + * idiosyncratic, and it would seem to be challenging to provide a mapping model that satisfied a + * large number of cases. These mappings are not included in the model specified here. + * + * Visibility and Integrity. Since Dimension Scales are a specialization of a Dataset, it is + * “visible” and accessible as a regular Dataset through the HDF5 API. This means that an application + * program could alter the values of or delete a Dimension Scale object or required attributes without + * regard to any of the semantics defined in this document. + * + * One advantage it that the implementation requires no changes to the base library, which reduces + * the complexity of the code and the risk of side-effects. The implementation builds on existing + * functions, which should improve the quality and reliability of the code. Also, this approach leaves + * most of the semantics of dimension scales to applications and communities, who can use the + * specification in any way they need. + * + * An important disadvantage is that the core HDF5 library will not manage the semantics of + * Dimension Scales. In particular, applications or other software must implement: + * \li Naming – the HDF5 library will impose no rules on the names of Dimension Scales + * \li Consistency of references – e.g., if a Dataset (Dimension Scale) is deleted (e.g., + * with #H5Ldelete, any Dimension Scales (Datasets) that it refers to (refer to it) + * will not be updated by the HDF5 library. + * \li Consistency of extents – the HDF5 library will not assure that a Dimension and + * associated Dimension Scale have the same extent (number of elements), nor that shared + * objects are consistent with each other. As in the case of delete, if a Dimension or + * Dimension Scale is extended (e.g., H5S…), any associated objects will not be + * automatically extended. + * + * These are briefly summarized here. + * \li Naming and Name Spaces. There are many potential schemes for naming dimensions, each + * suited for different uses. This specification does not impose any specific approach, + * so it may be used by different applications. However, the lack of restrictions has + * disadvantages as well.
For some purposes, it will be important to iterate through + * all the Dimension Scale objects in a file. The iterate operation is difficult to + * implement with the design specified here. This will be left to other software. For example, + * the HDF-EOS library has its own mechanism for managing a set of dimensions, and the + * netCDF4 library will implement this if it needs to. + * \li Automatically extending dataset dimensions. When a dimension of a dataset is + * extended, should the library automatically extend the corresponding dimension scale, or + * should this be left to the application? Since a dimension scale can be shared among many + * datasets, this raises a number of issues that are difficult to address in a general way. + * For instance, which dimension scale should be extended when only one dataset is extended, + * and what values are to be added? We have seen no compelling reason to implement an automatic + * extension of dimension scales when dataset dimensions are extended, so we suggest letting + * applications be responsible for this operation. + * \li Automatically deleting dimension scales. Should a dimension scale be deleted + * when all datasets that use it have been deleted? This is another case where different applications + * might have different requirements, so a general policy would be difficult to devise. Furthermore, + * enforcing a deletion policy, even a simple one, adds complexity to the library, and could also + * affect performance. Deletion policies seem best left to applications. + * + * Section \ref sec_dim_scales_api presents an API and programming model that implements some of these + * features. However, applications + * may ignore or bypass these APIs, to write or read the attributes directly. + * + * \section sec_dim_scales_spec The HDF5 Dimension Scale Specification + * + * \subsection subsec_dim_scales_spec_summary Brief Summary + * A Dimension Scale is stored as an HDF5 Dataset. + * \li A Dimension Scale is an object that is associated with a dimension of a Dataset. + * \li A Dimension Scale can have at most one name. + * \li A Dimension Scale may be associated with zero, one, or many different dimensions in any number of + * Datasets. + * \li Unless otherwise specified, a Dimension Scale inherits the properties of an HDF5 Dataset. + * \li There are no restrictions on the size, shape, or datatype of a Dimension Scale. + * + * A Dimension Scale can be associated with a dimension of an HDF5 dataset + * \li A dimension of a Dataset may have zero, one, or more Dimension Scales associated with it. + * \li Each scale is identified by an index value. + * + * A dimension may have a label without a scale, and may have a scale with no label. + * \li The label need not be the same as the name of any associated Dimension Scales. + * + * The implementation has two parts: + * \li A storage profile + * \li An API and programming model + * + * \subsection subsec_dim_scales_spec_store + * This section specifies the storage profile for Dimension Scale objects and the association between + * Dimensions and Dimension Scales. + * + * This profile is compatible with an earlier netcdf prototype and the HDF4 to HDF5 Mapping. This + * profile is also compatible with the netCDF4 proposal. This profile may be used to augment the + * HDF-EOS5 profile. + * + * See Appendix 2 for a discussion of how to store converted HDF4 objects. See Appendix 3 for a + * discussion of netCDF4 issues. See Appendix 4 for a discussion of HDF-EOS5. + * + * \subsubsection subsubsec_dim_scales_spec_store_dset + * A Dimension Scale dataset is stored as an HDF5 dataset. Table 1 summarizes the stored data, i.e., + * the values of the scale. There are no restrictions on the dataspace or datatype, or storage properties of + * the dataset. + * + * The scale may have any HDF5 datatype, and does not have to be the same as the datatype of + * the Dataset(s) that use the scale. E.g., an integer dataset might have dimension scales + * that are string or float values. + * + * The dataspace of the scale can be any rank and shape. A scale is not limited to one dimension, + * and is not restricted by the size of any dimension(s) associated with it. When a dimension is + * associated with a one dimensional scale, the scale may be a different size from the dimension. + * In this case, it is up to the application to interpret or resolve the difference. When a + * dimension is associated with a scale with a rank higher than 1, the interpretation of the + * association is up to the application. + * + * The Dimension Scale dataset can use any storage properties (including fill values, filters, + * and storage layout), not limited by the properties of any datasets that refer to it. When + * the Dimension Scale is extendible, it must be chunked. + * + * Table 2 defines the required and optional attributes of the Dimension Scale Dataset. The + * attribute REFERENCE_LIST is a list of (dataset, index) pairs. Each pair represents an + * association defined by ‘attach_scale’. These pairs are stored as an array of compound data. + * Table 3 defines this datatype. + * + * The Dimension Scale Dataset has an attribute called SUB_CLASS. This string is intended to + * be used to document particular specializations of this profile, e.g., a Dimension Scale + * created by netCDF4. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Table 1. The properties of the Dimension Scale dataset
FieldDatatypeDataspaceStorage PropertiesNotes
<data>AnyAnyAnyThese are the values of the Dimension Scale.
+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Table 2. Standard Attributes for a stored Dimension Scale dataset.
Attribute NameDatatype and DimensionsValueRequired / OptionalNotes
CLASS#H5T_STRING length = 16“DIMENSION_SCALE”RequiredThis attribute distinguishes the dataset as a Dimension scale object.
This is set by + * #H5DSset_scale
NAME#H5T_STRING length = <user defined><user defined>
The name does not have to be the same as the HDF5 path name for the dataset. + * The name does not have to be related to any labels. Several Dimension Scales may have the same name.		Optional, (Maximum of 1)The user defined label of the Dimension Scale.
This is set by #H5DSset_label
REFERENCE_LISTArray of Dataset Reference Type (Compound Datatype), variable length.[ {dataset1, ind1 }, …] [,…] ….]Optional, required when scale is attachedSee Table 3. This is set by #H5DSattach_scale.
SUB_CLASS#H5T_STRING length = <profile defined>“HDF4_DIMENSION”,
“NC4_DIMENSION”,		Optional, defined by other profilesThis is used to indicate a specific profile was used.
<Other attributes>OptionalFor example, UNITS.
+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Table 3. Dataset Reference Type. + * This is a pair, <dataset_ref, index>. This is created when the Dimension Scale is attached to a + * Dataset.
FieldDatatypeValueNotes
DATASETObject Reference.Pointer to a Dataset that refers to the scaleSet by #H5DSattach_scale.
Removed by #H5DSdetach_scale.
INDEX#H5T_NATIVE_INTIndex of the dimension the dataset pointed to by DATASETSet by #H5DSattach_scale.
Removed by #H5DSdetach_scale.
+ * + * \subsubsection subsubsec_dim_scales_spec_store_attr Attributes of a Dataset with a Dimension Scale + * A Dataset may have zero or more Dimension Scales associated with its dataspace. When present, + * these associations are represented by two attributes of the Dataset. Table 4 defines these attributes. + * + * The DIMENSION_LIST is a two dimensional array with one row for each dimension of the Dataset, and + * a variable number of entries in each row, one for each associated scale. This is stored as a one + * dimensional array, with the HDF5 Datatype variable length array of object references. + * + * When a dimension has more than one scale, the order of the scales in the DIMENSION_LIST attribute + * is not defined. A given Dimension Scale should appear in the list only once. + * (I.E., the DIMENSION_LIST is a “set”.) + * + * When a scale is shared by more than one dimension (of one or more Dataset), the order of + * the records in REFERENCE_LIST is not defined. The Dataset and Dimension should appear in the list only + * once. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Table 4. Standard Attributes of a Dataset with associated Dimension Scale.
Attribute NameDatatype and DimensionsValueRequired / OptionalNotes
DIMENSION_LISTThe HDF5 datatype is ARRAY of Variable Length #H5T_STD_REF_OBJ with rank of the dataspace.[[{object__ref1, object__ref2, … object__refn}, …] […] ..]Optional, required if scales are attachedSet by #H5DSattach_scale.
Entries removed by #H5DSdetach_scale.
DIMENSION_LABELLISTThe HDF5 datatype is ARRAY of #H5T_STRING with rank of the dataspace.[ <Label1>, <Label2>, …, <Label3>] Optional, required for scales with a labelSet by #H5DSset_label.
+ * + * \subsection subsec_dim_scales_spec_lab Dimension Scale Names and Labels + * Dimension scales are often referred to by name, so Dimension Scales may have names. + * Since some applications do not wish to apply names to dimension scales, Dimension Scale + * names be optional. In addition, some applications will have a name but no associated data + * values for a dimension (i.e., just a label). To support this, each dimension may have a + * label, which may be but need not be the same as the name of an associated Dimension Scale. + * + * Dimension Scale Name. Associated with the Dimension Scale object. A Dimension Scale may + * have no name, or one name. + * + * Dimension Label. A optional label associated with a dimension of a Dataset. + * + * How is a name represented? Three options seem reasonable: + * \li Last link in the pathname. The h4toh5 mapping uses this approach [6], but there could be more + * than one path to a dataset, leading to ambiguities. This could be overcome by enforcing conventions. + * \li Attribute. This exposes this information at the top level, making it accessible to any viewer that + * can see attributes. It also makes it easy for applications to change the name, which could be + * dangerous, or valuable. + * \li Header message. This approach makes the name a little less available at the top level, + * but firmly pushes the concept into the base format and library. Since it also requires + * applications to change the name through a function call, it leaves open the possibility + * that the form of the name could be altered later without requiring application codes + * to change. On the other hand, if we treat names this way, it means that the “name” + * attribute is being treated differently from the “class” attribute, which could be confusing. + * + * Dimension Scale names are stored in attributes of the Dimension Scale or the Dataset that refers + * to a Dimension Scale. + * + * Should dimension scale names be unique among dimension scales within a file? + * We have seen a number of cases in which applications need more than one dimension scale with + * the same name. We have also seen applications where the opposite is true: dimension scale + * names are assumed to be unique within a file. This specification leaves it to applications + * to enforce a policy of uniqueness when they need it. + * + * Can a dimension have a label, without having an associated scale? Some + * applications may wish to name dimensions without having an associated scale. Therefore, + * a dataset may have a label for a dimension without having an associated Dimension Scale dataset. + * + * Can a dimension have a scale, without having an associated label? Some + * applications may wish to assign a dimension scale with no label. Therefore, a dataset + * may have one or more Dimension Scales for a dimension without having an associated label. + * + * Anonymous Dimensions. It is possible to have a Dimension Scale dataset + * with no name, and associate it with a dimension of a dataset with no label. This case + * associates an array of data values to the dimension, but no identifier. + * + * A dimension with a label and a name. A dimension of a dataset can be + * associated with a Dimension Scale that has a name, and assigned a label. In this case, + * the association has two “names”, the label and the dimension scale name. It is up to + * applications to interpret these names. + * + * Table 5 summarizes the six possible combinations of label and name. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Table 5. Labels and scales of a dimension.
No scaleScale with no nameScale with name
No labelDimension has no label or scale (default)Dimension has an anonymous scaleDimension has scale, the scale is called “name”
LabelDimension has labelDimension has scale with a label.Dimension has scale with both a label and name. A shared dimension has one name, + * but may have several labels
+ * + * \subsection subsec_dim_scales_spec_shared Shared Dimension Scales + * Given the design described above, datasets can share dimension scales. The following + * additional capabilities would seem to be useful. + * \li When a dimension scale is deleted, remove the reference to the dimension scale in + * all datasets that refer to it. + * \li Determine how many datasets are attached to a given dimension scale + * \li Determine what datasets are attached to a given dimension scale + * + * These capabilities can be provided in several ways: + * \li Back pointers. If every dimension scale contained a list of back pointers to all + * datasets that referenced it, then it would be relatively easy to open all of these + * datasets and remove the references, as well as to answer questions #2 and #3. + * This would require the library to update the back pointer list every time a link + * was made. + * \li Alternatively, such lists could be maintained in a separate table. Such a table + * could contain all information about a number of dimension scales, which might provide + * a convenient way for applications to gain information about a set of dimension scales. + * For instance, this table might correspond to the coordinate variable definitions in a netCDF file. + * \li If no such list were available, an HDF5 function could be available to search all datasets + * to determine which ones referenced a given dimension scale. This would be straightforward, + * but in some cases could be very time consuming. + * + * This specification defines attributes that maintain back pointers, which enable + * these kinds of cross referencing. Other software, such as NetCDF4, may well need a global table to + * track a set of dimensions. Such a table can be done in addition to the attributes defined here. + * + * \subsection subsec_dim_scales_spec_ex Example + * This section presents an example to illustrate the data structures defined above. + * + * Figure 3 shows a Dataset with a four dimensional Dataspace. The file also contains six Dimension + * Scale datasets. The Dimension Scale datasets are HDF5 objects, with path names such as “/DS1”. + * + * Figure 4 illustrates the use of dimension scales in this example. Each Dimension Scale Dataset + * has an optional NAME. For example, “/DS3” has been assigned the name “Scale3”. + * + * The dimensions of dataset D have been assigned zero or more scales and labels. Dimension 0 + * has two scales, Dimension 1 has one scale, and so on. Dimension 2 has no scale associated with it. + * + * Some of the dimensions have labels as well. Note that dimension 2 has a label but no scale, and + * dimension 3 has scales but no label. + * + * Some of the Dimension Scales are shared. Dimension Scale DS1 is referenced by dimension 0 of D + * and by another unspecified dataset. Dimension Scale DS3 is referenced by dimension 1 and 3 of Dataset D. + * + * These relationships are represented in the file by attributes of the Dataset D and the Dimension + * Scale Datasets. Figure 5 shows the values that are stored for the DIMENSION_LIST attribute of + * Dataset D. This + * + * + * + * + *
+ * \image html UML_Attribute.jpg "The UML model for an HDF5 attribute" + *
+ * attribute is a one-dimensional array with the HDF5 datatype variable length + * #H5T_STD_REF_OBJ. Each row of the array is zero or more object references for Dimension Scale datasets. + * + * Table 6 shows the DIMENSION_LABELLIST for Dataset D. This is a one dimensional array with some empty + * values. + * + * Each of the Dimension Scale Datasets has a name and other attributes. The references are + * represented in the REFERENCE_LIST attributes. Table 7 – Table 10 show the values for these + * tables. Note that Dimension Scale DS4 and DS6 have no references to them in this diagram. + * + * The tables are stored as attributes of the Dimension Scale Dataset and the Datasets that + * refer to scales. Essentially, the association between a dimension of a Dataset and a Dimension + * Scale is represented by “pointers” (i.e., HDF5 Object References) in both of the associated + * objects. Since there can be multiple associations, there can be multiple pointers stored at + * each object, representing the endpoints of the associations. These will be stored in tables, + * i.e., as an attribute with an array of values. + * + * When dimension scales are attached or detached, the tables in the Dataset and the Dimension + * Scale must be updated. The arrays in the attributes can grow, and items can be deleted. + * + * The associations are identified by the object reference and dimension which is stored in a + * back pointer and returned from an API. The detach function needs to be careful how it deletes + * an item from the table, because the entries at both ends of the association must be updated + * at the same time. + * + * + * + * + * + *
+ * \image html H5DS_fig3.png "Figure 3. Example dataset and scales." + *
+ * + * + * + * + * + *
+ * \image html H5DS_fig4.png "Figure 4. Example labels, names, and attached scales." + *
+ * + * + * + * + * + *
+ * \image html H5DS_fig5.png "Figure 5. The table of dimension references, stored as an attribute of the + * Dataset." + *
+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Table 6. The table of dimension labels.
Dataset DimensionLabel
0“LX”
1“LZ”
2“LQ”
3“”
+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Table 7. The reference list for DS1.
ReferenceDataset Reference Record
0{Object reference to Dataset D, 0}
1{Object reference to other Dataset, ?}
+ * + * + * + * + * + * + * + * + * + * + *
Table 8. Reference list for DS2
ReferenceDataset Reference Record
0{Object reference to Dataset D, 0}
+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Table 9. Reference list for DS3
ReferenceDataset Reference Record
0{Object reference to Dataset D, 1}
1{Object reference to Dataset D, 3}
+ * + * + * + * + * + * + * + * + * + * + *
Table 10. Reference List for DS5
ReferenceDataset Reference Record
0{Object reference to Dataset D, 3}
+ * + * \section sec_dim_scales_api Programming Model and API + * + * \subsection subsec_dim_scales_api_model Programming Model + * Dimension Scales are HDF5 Datasets, so they may be created and accesses through any + * HDF5 API for datasets [10]. The HDF5 Dimension Scale API implements the specification + * defined in this document. The operations include: + * \li Convert dataset to scale (D) – convert dataset D to a dimension scale. D may be + * specified by an id or by a path name. + * \li Attach scale (D, S, i) – attach dimension scale S to the ith dimension of D. D + * and S may be specified by an id or path name + * \li Detach scale (D, i, scale) – detach scale from the ith dimension of D. + * \li Iterate through scales of (D, i) – get each scale attached. + * \li Get the number of scales of (D, i) + * \li Get the ith scale of D + * \li Set/Get name (S) – set/get the name about dimension scale S. + * + * The API also defines operations for dimension labels: + * \li Set/Get label (D, i) – set/get the label for ith dimension of D. + * + * \subsubsection subsubsec_dim_scales_api_model_create Create new Dimension Scale with Initial Values + * 1. Create dataset with for the Dimension Scale with H5Dcreate and other standard HDF5 calls. + * 2. Initialize the values of the Dimension Scale with H5Dwrite and other calls. + * 3. Convert the dataset to a Dimension Scale with H5DSmake_scale. + * 4. Close the Dimension Scale when finished with H5Dclose. + * + * \subsubsection subsubsec_dim_scales_api_model_attach Attach Dimension Scale to Dataset + * 1. Create or open the Dataset, D, with H5Dopen, etc. + * 2. Create or open the Dimension Scale dataset, S, with H5Dopen or as above. + * 3. Attach the Dimension Scale S to dimension j of Dataset D with H5DSattach_scale + * 4. When finished, close the Dimension Scale and Dataset with H5Dclose. + * + * \subsubsection subsubsec_dim_scales_api_model_read Read Dimension Scale values + * 1. Open the Dataset D, with H5Dopen + * 2. Get the number of dimensions of D with H5Dget_space. + * 3. Iterate through the scales of dimension i, locate the target scale, S (e.g., by its name). + * 4. Get the datatype, dataspace, etc. of S with H5Dget_space, H5Dget_type, H5Sget_ndims, etc. + * 5. Read the values of S into memory with H5Dread, e.g. into dscalebuff. + * 6. When finished, close the S and other objects with H5Dclose etc. + * 7. When finished, close the Dataset D with H5Dclose. + * + * \subsubsection subsubsec_dim_scales_api_model_write Write or Update Dimension Scale values + * 1. Open the Dimension Scale Dataset S with H5open + * 2. Get the datatype, dataspace, etc. of S with H5Dget_space, H5Dget_type, H5Sget_ndims, etc. + * 3. If needed, read the values of S into memory with H5Dread. Note, may read selected values using a + * selection. + * 4. Write updated values to S with H5Dwrite. Note, may write selected values using a selection. + * 5. When finished, close S and other objects with H5Dclose etc. + * + * \subsubsection subsubsec_dim_scales_api_model_label Create a label for a dimension + * 1. Open the Dataset D with H5open + * 2. Add write a label for dimension i of D, with H5DSset_label. + * 3. When finished, close the Dimension Scale Dataset and other objects with H5Dclose etc. + * + * \subsubsection subsubsec_dim_scales_api_model_extend Extending a Dimension with a Dimension Scale attached + * When an extendible Dataset has Dimension Scales, it is necessary to coordinate when the dimensions change + * size. + * 1. Open the Dataset to be extended, with H5Dopen. + * 2. Extend the dimension(s) with H5Dextend + * 3. Iterate through the scales of each extended dimension. For each scale + * a. Extend the scale to the new size of the dimension with H5Dextend + * b. Write new values to the extended scale with H5Dwrite, etc. + * c. Close the Dimension Scale Dataset with H5Dclose if necessary. + * 4. When finished, close the Dataset with H5Dclose + * + * \subsubsection subsubsec_dim_scales_api_model_detach Detach Dimension Scale from Dataset + * The detach operation removes an association between a dimension and a scale. It does not delete the + * Dimension Scale Dataset. + * 1. Open the Dataset, D, with H5Dopen. + * 2. Iterate through the scales of dimension i, locate the target scale, S (e.g., by its name). + * 3. Detach the Dimension Scale S to dimension j of Dataset D with H5DSdetach_scale + * 4. When finished, close the Dimension Scale and Dataset with H5Dclose. + * + * \subsubsection subsubsec_dim_scales_api_model_del Delete a Dimension Scale Dataset + * When it is necessary to delete a Dimension Scale Dataset, it is necessary to detach it from all dataset. + * This section outlines the necessary steps. + * 1. Open the Dimension Scale to be deleted. + * 2. Read the REFERENCE_LIST attribute into memory with H5Aread etc. + * 3. For each entry in the list: + * a. Dereference the dataset reference + * b. Detach the scale with H5DSdetach_scale + * c. Close the dataset reference + * 4. Delete the Dimension Scale Dataset + * + * \subsubsection subsubsec_dim_scales_api_model_clean Clean up Dimension Scales when deleting a Dataset + * When it is necessary to delete a dataset that has scales attached, it is necessary to delete all the + * scales before deleting the dataset. Here is a sketch of the steps. + * 1. Open the Dataset to be deleted, with H5Dopen. + * 2. Iterate through the scales of each dimension of D + * 3. For each scale, detach the Dimension Scale S from dimension j of Dataset D with H5DSdetach_scale + * 4. Delete the Dataset, with H5Gunlink. + * + * \subsection subsec_dim_scales_api_func Programming API: H5DS + * @see H5DS Reference Manual + * * @todo Under Construction */ diff --git a/hl/src/H5PT.c b/hl/src/H5PT.c index c8cefd3a324..89e6a20d69d 100644 --- a/hl/src/H5PT.c +++ b/hl/src/H5PT.c @@ -25,8 +25,6 @@ typedef struct { static hsize_t H5PT_ptable_count = 0; static H5I_type_t H5PT_ptable_id_type = H5I_UNINIT; -#define H5PT_HASH_TABLE_SIZE 64 - /* Packet Table private functions */ static herr_t H5PT_free_id(void *id, void **_ctx); static herr_t H5PT_close(htbl_t *table); @@ -73,8 +71,7 @@ H5PTcreate(hid_t loc_id, const char *dset_name, hid_t dtype_id, hsize_t chunk_si /* Register the packet table ID type if this is the first table created */ if (H5PT_ptable_id_type < 0) - if ((H5PT_ptable_id_type = - H5Iregister_type((size_t)H5PT_HASH_TABLE_SIZE, 0, (H5I_free_t)H5PT_free_id)) < 0) + if ((H5PT_ptable_id_type = H5Iregister_type2(0, (H5I_free_t)H5PT_free_id)) < 0) goto error; /* Get memory for the table identifier */ @@ -187,8 +184,7 @@ H5PTcreate_fl(hid_t loc_id, const char *dset_name, hid_t dtype_id, hsize_t chunk /* Register the packet table ID type if this is the first table created */ if (H5PT_ptable_id_type < 0) - if ((H5PT_ptable_id_type = - H5Iregister_type((size_t)H5PT_HASH_TABLE_SIZE, 0, (H5I_free_t)H5PT_free_id)) < 0) + if ((H5PT_ptable_id_type = H5Iregister_type2(0, (H5I_free_t)H5PT_free_id)) < 0) goto error; /* Get memory for the table identifier */ @@ -287,8 +283,7 @@ H5PTopen(hid_t loc_id, const char *dset_name) /* Register the packet table ID type if this is the first table created */ if (H5PT_ptable_id_type < 0) - if ((H5PT_ptable_id_type = - H5Iregister_type((size_t)H5PT_HASH_TABLE_SIZE, 0, (H5I_free_t)H5PT_free_id)) < 0) + if ((H5PT_ptable_id_type = H5Iregister_type2(0, (H5I_free_t)H5PT_free_id)) < 0) goto error; table = (htbl_t *)malloc(sizeof(htbl_t)); diff --git a/java/src/Makefile.am b/java/src/Makefile.am index e194a0fce5d..7fac1f2eb98 100644 --- a/java/src/Makefile.am +++ b/java/src/Makefile.am @@ -41,7 +41,7 @@ CLASSPATH_ENV=CLASSPATH=.:$(top_srcdir)/java/lib/slf4j-api-2.0.6.jar:$$CLASSPATH AM_JAVACFLAGS = $(H5_JAVACFLAGS) -deprecation hdf5_java_JAVA = \ - ${pkgpath}/callbacks/Callbacks.java \ + ${pkgpath}/callbacks/H5Callbacks.java \ ${pkgpath}/callbacks/H5A_iterate_cb.java \ ${pkgpath}/callbacks/H5A_iterate_t.java \ ${pkgpath}/callbacks/H5D_append_cb.java \ diff --git a/java/src/hdf/hdf5lib/CMakeLists.txt b/java/src/hdf/hdf5lib/CMakeLists.txt index 69b5a93b664..8c0b6daa48c 100644 --- a/java/src/hdf/hdf5lib/CMakeLists.txt +++ b/java/src/hdf/hdf5lib/CMakeLists.txt @@ -10,7 +10,7 @@ SET_GLOBAL_VARIABLE (HDF5_JAVA_SOURCE_PACKAGES ) set (HDF5_JAVA_HDF_HDF5_CALLBACKS_SOURCES - callbacks/Callbacks.java + callbacks/H5Callbacks.java callbacks/H5A_iterate_cb.java callbacks/H5A_iterate_t.java callbacks/H5D_append_cb.java diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index fe475611bc8..f630faba8e5 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -5348,7 +5348,7 @@ public synchronized static native void H5Fset_libver_bounds(long file_id, int lo // herr_t H5Fget_mpi_atomicity(hid_t file_id, hbool_t *flag); // #endif /* H5_HAVE_PARALLEL */ - // /** + // /* // * H5Fget_vfd_handle returns a pointer to the file handle from the // low-level file driver // * currently being used by the HDF5 library for file I/O. @@ -5365,7 +5365,7 @@ public synchronized static native void H5Fset_libver_bounds(long file_id, int lo // H5Fget_vfd_handle(int file_id, int fapl) // throws HDF5LibraryException; - // /** + // /* // * H5Fget_mdc_config loads the current metadata cache configuration into // * the instance of H5AC_cache_config_t pointed to by the config_ptr // parameter. @@ -5382,7 +5382,7 @@ public synchronized static native void H5Fset_libver_bounds(long file_id, int lo // public synchronized static native void H5Fget_mdc_config(int file_id, H5AC_cache_config_t config_ptr) // throws HDF5LibraryException, NullPointerException; - // /** + // /* // * H5Fset_mdc_config attempts to configure the file's metadata cache // according to the configuration supplied. // * @@ -6306,7 +6306,7 @@ public synchronized static native void H5Iclear_type(int type_id, boolean force) // hid_t H5Iregister(H5I_type_t type, const void *object); // typedef herr_t (*H5I_free_t)(void *); - // H5I_type_t H5Iregister_type(size_t hash_size, unsigned reserved, H5I_free_t free_func); + // H5I_type_t H5Iregister_type2(unsigned reserved, H5I_free_t free_func); // void *H5Iremove_verify(hid_t id, H5I_type_t id_type); @@ -10028,7 +10028,7 @@ public synchronized static native String H5Pget_virtual_dsetname(long dcpl_id, l throws HDF5LibraryException, IllegalArgumentException; // ///// unimplemented ///// - // /** + // /* // * H5Pget_vds_file_cache_size retrieves the size of the vds link open file cache. // * // * @param fapl_id @@ -10043,7 +10043,7 @@ public synchronized static native String H5Pget_virtual_dsetname(long dcpl_id, l // public synchronized static native int H5Pget_vds_file_cache_size(long fapl_id) throws // HDF5LibraryException; // - // /** + // /* // * H5Pset_vds_file_cache_size sets the number of files that can be held open in an vds link open // * file cache. // * @@ -10075,10 +10075,9 @@ public synchronized static native String H5Pget_virtual_dsetname(long dcpl_id, l * @param size * OUT: the offset value and the size of the external file data. * - * 
-     *      size[0] = offset // a location to return an offset value
-     *      size[1] = size // a location to return the size of
-     *                // the external file data.
+     * 
+     *    size[0] = offset // a location to return an offset value
+     *    size[1] = size // a location to return the size of the external file data.
      * 

      *
      * @return a non-negative value if successful
@@ -13929,7 +13928,7 @@ public synchronized static native boolean H5Tdetect_class(long type_id, int cls)
     public synchronized static native int H5Tencode(long obj_id, byte[] buf, long nalloc)
         throws HDF5LibraryException, NullPointerException;
 
-    // /**
+    // /*
     //  * @ingroup JH5T
     //  *
     //  * H5Tencode converts a data type description into binary form in a buffer.
diff --git a/java/src/hdf/hdf5lib/callbacks/H5A_iterate_cb.java b/java/src/hdf/hdf5lib/callbacks/H5A_iterate_cb.java
index b7471805d68..bdef19f2160 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5A_iterate_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5A_iterate_cb.java
@@ -18,7 +18,7 @@
  * Information class for link callback for H5Aiterate.
  *
  */
-public interface H5A_iterate_cb extends Callbacks {
+public interface H5A_iterate_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/Callbacks.java b/java/src/hdf/hdf5lib/callbacks/H5Callbacks.java
similarity index 88%
rename from java/src/hdf/hdf5lib/callbacks/Callbacks.java
rename to java/src/hdf/hdf5lib/callbacks/H5Callbacks.java
index a106ab67625..d0af5c81e32 100644
--- a/java/src/hdf/hdf5lib/callbacks/Callbacks.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5Callbacks.java
@@ -13,11 +13,11 @@
 package hdf.hdf5lib.callbacks;
 
 /**
- * @page CALLBACKS HDF5 Java Callbacks Interface
+ * @page CALLBACKS HDF5 Java H5Callbacks Interface
  * All callback definitions must derive from this interface.  Any
  * derived interfaces must define a single public method  named "callback".
  * You are responsible for deregistering your callback (if necessary)
- * in its {@link Object#finalize} method.  If native code attempts to call
+ * in its Object finalize method.  If native code attempts to call
  * a callback which has been GC'd, you will likely crash the VM.  If
  * there is no method to deregister the callback (e.g. atexit
  * in the C library), you must ensure that you always keep a live reference
@@ -28,7 +28,7 @@
  * exceptions thrown will be passed to the default callback exception
  * handler.
  *
- * @defgroup JCALLBK HDF5 Library Java Callbacks
+ * @defgroup JCALLBK HDF5 Library Java H5Callbacks
  */
-public interface Callbacks {
+public interface H5Callbacks {
 }
diff --git a/java/src/hdf/hdf5lib/callbacks/H5D_append_cb.java b/java/src/hdf/hdf5lib/callbacks/H5D_append_cb.java
index d29a4199fa0..7059f88d903 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5D_append_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5D_append_cb.java
@@ -16,7 +16,7 @@
  * Information class for link callback for H5Pset/get_append_flush.
  *
  */
-public interface H5D_append_cb extends Callbacks {
+public interface H5D_append_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5D_iterate_cb.java b/java/src/hdf/hdf5lib/callbacks/H5D_iterate_cb.java
index 02f3069572d..4ecb45ac2d1 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5D_iterate_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5D_iterate_cb.java
@@ -16,7 +16,7 @@
  * Information class for link callback for H5Diterate.
  *
  */
-public interface H5D_iterate_cb extends Callbacks {
+public interface H5D_iterate_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5E_walk_cb.java b/java/src/hdf/hdf5lib/callbacks/H5E_walk_cb.java
index b4e9eb61592..a48f9b03090 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5E_walk_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5E_walk_cb.java
@@ -18,7 +18,7 @@
  * Information class for link callback for H5Ewalk.
  *
  */
-public interface H5E_walk_cb extends Callbacks {
+public interface H5E_walk_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5L_iterate_t.java b/java/src/hdf/hdf5lib/callbacks/H5L_iterate_t.java
index 1aaaabea5d6..771556e6372 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5L_iterate_t.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5L_iterate_t.java
@@ -18,7 +18,7 @@
  * Information class for link callback for H5Lvisit/H5Lvisit_by_name.
  *
  */
-public interface H5L_iterate_t extends Callbacks {
+public interface H5L_iterate_t extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5O_iterate_t.java b/java/src/hdf/hdf5lib/callbacks/H5O_iterate_t.java
index 74836a2eb5c..bb556310ec8 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5O_iterate_t.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5O_iterate_t.java
@@ -18,7 +18,7 @@
  * Information class for link callback for H5Ovisit/H5Ovisit_by_name.
  *
  */
-public interface H5O_iterate_t extends Callbacks {
+public interface H5O_iterate_t extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5P_cls_close_func_cb.java b/java/src/hdf/hdf5lib/callbacks/H5P_cls_close_func_cb.java
index 4eccb24a003..f96eaabf0ac 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5P_cls_close_func_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5P_cls_close_func_cb.java
@@ -16,7 +16,7 @@
  * Information class for link callback for H5Pcreate_class.
  *
  */
-public interface H5P_cls_close_func_cb extends Callbacks {
+public interface H5P_cls_close_func_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5P_cls_copy_func_cb.java b/java/src/hdf/hdf5lib/callbacks/H5P_cls_copy_func_cb.java
index 332181079d8..f9fc94cb962 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5P_cls_copy_func_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5P_cls_copy_func_cb.java
@@ -16,7 +16,7 @@
  * Information class for link callback for H5Pcreate_class
  *
  */
-public interface H5P_cls_copy_func_cb extends Callbacks {
+public interface H5P_cls_copy_func_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5P_cls_create_func_cb.java b/java/src/hdf/hdf5lib/callbacks/H5P_cls_create_func_cb.java
index 43f915a4943..b16d9cdd249 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5P_cls_create_func_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5P_cls_create_func_cb.java
@@ -16,7 +16,7 @@
  * Information class for link callback for H5Pcreate_class.
  *
  */
-public interface H5P_cls_create_func_cb extends Callbacks {
+public interface H5P_cls_create_func_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5P_iterate_cb.java b/java/src/hdf/hdf5lib/callbacks/H5P_iterate_cb.java
index 6586a6aca8f..6fd4ecfd396 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5P_iterate_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5P_iterate_cb.java
@@ -16,7 +16,7 @@
  * Information class for link callback for H5Piterate.
  *
  */
-public interface H5P_iterate_cb extends Callbacks {
+public interface H5P_iterate_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5P_prp_close_func_cb.java b/java/src/hdf/hdf5lib/callbacks/H5P_prp_close_func_cb.java
index 4eb3b33ba19..a1cdcd7f7e0 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5P_prp_close_func_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5P_prp_close_func_cb.java
@@ -16,7 +16,7 @@
  * Information class for link callback for H5Pregister2.
  *
  */
-public interface H5P_prp_close_func_cb extends Callbacks {
+public interface H5P_prp_close_func_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5P_prp_compare_func_cb.java b/java/src/hdf/hdf5lib/callbacks/H5P_prp_compare_func_cb.java
index 91c2863c95a..f85935837e5 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5P_prp_compare_func_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5P_prp_compare_func_cb.java
@@ -16,7 +16,7 @@
  * Information class for link callback for H5Pregister2.
  *
  */
-public interface H5P_prp_compare_func_cb extends Callbacks {
+public interface H5P_prp_compare_func_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5P_prp_copy_func_cb.java b/java/src/hdf/hdf5lib/callbacks/H5P_prp_copy_func_cb.java
index 49827d1666b..4f5f7d16762 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5P_prp_copy_func_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5P_prp_copy_func_cb.java
@@ -16,7 +16,7 @@
  * Information class for link callback for H5Pregister2.
  *
  */
-public interface H5P_prp_copy_func_cb extends Callbacks {
+public interface H5P_prp_copy_func_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5P_prp_create_func_cb.java b/java/src/hdf/hdf5lib/callbacks/H5P_prp_create_func_cb.java
index 0391596994f..963ce4c0276 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5P_prp_create_func_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5P_prp_create_func_cb.java
@@ -16,7 +16,7 @@
  * Information class for link callback for H5Pregister2.
  *
  */
-public interface H5P_prp_create_func_cb extends Callbacks {
+public interface H5P_prp_create_func_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5P_prp_delete_func_cb.java b/java/src/hdf/hdf5lib/callbacks/H5P_prp_delete_func_cb.java
index 477612ff23b..2340de94b99 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5P_prp_delete_func_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5P_prp_delete_func_cb.java
@@ -16,7 +16,7 @@
  * Information class for link callback for H5Pregister2.
  *
  */
-public interface H5P_prp_delete_func_cb extends Callbacks {
+public interface H5P_prp_delete_func_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5P_prp_get_func_cb.java b/java/src/hdf/hdf5lib/callbacks/H5P_prp_get_func_cb.java
index 2bf08ce4322..7b4c3aca4ca 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5P_prp_get_func_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5P_prp_get_func_cb.java
@@ -16,7 +16,7 @@
  * Information class for link callback for H5Pregister2.
  *
  */
-public interface H5P_prp_get_func_cb extends Callbacks {
+public interface H5P_prp_get_func_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/H5P_prp_set_func_cb.java b/java/src/hdf/hdf5lib/callbacks/H5P_prp_set_func_cb.java
index 11f00d65fad..cffeaefcede 100644
--- a/java/src/hdf/hdf5lib/callbacks/H5P_prp_set_func_cb.java
+++ b/java/src/hdf/hdf5lib/callbacks/H5P_prp_set_func_cb.java
@@ -16,7 +16,7 @@
  * Information class for link callback for H5Pregister2.
  *
  */
-public interface H5P_prp_set_func_cb extends Callbacks {
+public interface H5P_prp_set_func_cb extends H5Callbacks {
     /**
      * @ingroup JCALLBK
      *
diff --git a/java/src/hdf/hdf5lib/callbacks/package-info.java b/java/src/hdf/hdf5lib/callbacks/package-info.java
index 323442c79b1..f0cefdfca78 100644
--- a/java/src/hdf/hdf5lib/callbacks/package-info.java
+++ b/java/src/hdf/hdf5lib/callbacks/package-info.java
@@ -11,8 +11,8 @@
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 /**
- * @page CALLBACKS_UG HDF5 Java Callbacks Interface
- * All callback definitions must derive from the Callbacks interface.  Any
+ * @page CALLBACKS_UG HDF5 Java H5Callbacks Interface
+ * All callback definitions must derive from the H5Callbacks interface.  Any
  * derived interfaces must define a single public method named "callback".
  * You are responsible for deregistering your callback (if necessary)
  * in its {@link Object#finalize} method.  If native code attempts to call
diff --git a/release_docs/INSTALL_Autotools.txt b/release_docs/INSTALL_Autotools.txt
index 07c3b7f0182..7335ce5b156 100644
--- a/release_docs/INSTALL_Autotools.txt
+++ b/release_docs/INSTALL_Autotools.txt
@@ -320,14 +320,14 @@ III. Full installation instructions for source distributions
         parallelism on a distributed multi-processor system.  Read the
         file INSTALL_parallel for detailed information.
 
-        The threadsafe, C++ and Java interfaces are not compatible
-        with the parallel option.
+        The threadsafe and multi-threaded concurrency options and the C++ and
+        Java interfaces are not compatible with the parallel option.
         Unless --enable-unsupported has been specified on the configure line,
         the following options must be disabled:
-            --enable-threadsafe, --enable-cxx, --enable-java
+            --enable-threadsafe, --enable-concurrency, --enable-cxx, --enable-java
 
 
-   3.10. Threadsafe capability
+   3.10. Threadsafe and multi-threaded concurrency capabilities
         The HDF5 library can be configured to be thread-safe (on a very
         large scale) with the `--enable-threadsafe' flag to the configure
         script.  Some platforms may also require the '-with-pthread=INC,LIB'
@@ -343,6 +343,21 @@ III. Full installation instructions for source distributions
         the following options must be disabled:
             --enable-hl, --enable-cxx, --enable-fortran, --enable-java
 
+        The multi-threaded concurrency option improves upon the basic
+        thread-safe capability by providing both threadsafety for API calls
+        and allowing multi-threaded concurrent execution within the library.
+        This may be enabled with the '--enable-concurrency' configure option,
+        which is mutually exclusive with the '--enable-threadsafe' option.
+
+        The current list of API routines that are enabled for concurrent
+        execution by multiple threads is: 
+
+        The high-level, C++, Fortran, and Java interfaces are not compatible
+        with the multi-threaded concurrency option because the lock is not
+        hoisted into the higher-level API calls.
+        Unless --enable-unsupported has been specified on the configure line,
+        the following options must be disabled:
+            --enable-hl, --enable-cxx, --enable-fortran, --enable-java
 
    3.11. Backward compatibility
         The 2.0.0 version of the HDF5 library can be configured to operate
diff --git a/release_docs/INSTALL_CMake.txt b/release_docs/INSTALL_CMake.txt
index 28b7195695b..41d99fe3e59 100644
--- a/release_docs/INSTALL_CMake.txt
+++ b/release_docs/INSTALL_CMake.txt
@@ -285,20 +285,26 @@ IV. Further considerations
       web site. The HDF5 2."X"."Y" product requires a minimum CMake version 3.18.
       If you are using VS2022, the CMake minimum version is 3.21.
 
-   2. If you plan to use Zlib or Szip:
+   2. If you plan to use Zlib or Szip (aka libaec):
       A. Download the binary packages and install them in a central location.
          For example on Windows, create a folder extlibs and install the
          packages there. Add the following CMake options:
-            -DH5_ZLIB_LIBRARY:FILEPATH=some_location/lib/zlib.lib
-            -DH5_ZLIB_INCLUDE_DIR:PATH=some_location/include
+            -DZLIB_LIBRARY:FILEPATH=some_location/lib/zlib.lib
+            -DZLIB_INCLUDE_DIR:PATH=some_location/include
             -DZLIB_USE_EXTERNAL:BOOL=OFF
-            -DH5_SZIP_LIBRARY:FILEPATH=some_location/lib/szlib.lib
-            -DH5_SZIP_INCLUDE_DIR:PATH=some_location/include
+            -DSZIP_LIBRARY:FILEPATH=some_location/lib/libszaec.lib
+            -DSZIP_INCLUDE_DIR:PATH=some_location/include
+            -Dlibaec_LIBRARY:FILEPATH=some_location/lib/libaec.lib
+            -Dlibaec_INCLUDE_DIR:PATH=some_location/include
             -DSZIP_USE_EXTERNAL:BOOL=OFF
          where "some_location" is the full path to the extlibs folder.
-         Also the appropriate environment variable must be set;
+         Also if the appropriate environment variable is set, the above options are not required;
             set(ENV{ZLIB_ROOT} "some_location")
             set(ENV{SZIP_ROOT} "some_location")
+            set(ENV{libaec_ROOT} "some_location")
+
+        Note that if there is a problem finding the libraries, try adding the
+        CMake variable CMAKE_FIND_DEBUG_MODE:BOOL=ON to the command line.
 
       B. Use source packages from an GIT server by adding the following CMake
          options:
@@ -307,6 +313,8 @@ IV. Further considerations
             ZLIB_GIT_BRANCH="some_branch"
             SZIP_GIT_URL:STRING="https://some_location/szip"
             SZIP_GIT_BRANCH="some_branch"
+            LIBAEC_GIT_URL:STRING="https://some_location/libaec"
+            LIBAEC_GIT_BRANCH="some_branch"
          where "some_location" is the URL to the GIT repository and "some_branch" is
          a branch in the repository, usually the default. Also set
          CMAKE_BUILD_TYPE to the configuration type.
@@ -423,7 +431,7 @@ Notes: CMake and HDF5
    2. CMake support for HDF5 development should be usable on any
       system where CMake is supported. Please send us any comments on
       how CMake support can be improved on any system. Visit the
-      KitWare site for more information about CMake.
+      Kitware site for more information about CMake.
 
    3. Build and test results can be submitted to our CDash server:
       The CDash server for community submissions of hdf5 is at
@@ -443,7 +451,7 @@ Notes: CMake and HDF5
 
 Notes: CMake in General
 
-   1. More information about using CMake can be found at the KitWare site at
+   1. More information about using CMake can be found at the Kitware site at
          www.cmake.org.
 
    2. CMake uses the command line; however, the visual CMake tool is
@@ -503,10 +511,12 @@ These five steps are described in detail below.
             * Visual Studio 17 2022
 
          is:
-            * H5_SZIP_INCLUDE_DIR:PATH=
-            * H5_SZIP_LIBRARY:FILEPATH=
-            * H5_ZLIB_INCLUDE_DIR:PATH=
-            * H5_ZLIB_LIBRARY:FILEPATH=
+            * SZIP_INCLUDE_DIR:PATH=
+            * SZIP_LIBRARY:FILEPATH=
+            * libaec_INCLUDE_DIR:PATH=
+            * libaec_LIBRARY:FILEPATH=
+            * ZLIB_INCLUDE_DIR:PATH=
+            * ZLIB_LIBRARY:FILEPATH=
             * :BOOL=[ON | OFF]
 
          is:
@@ -556,7 +566,7 @@ These five steps are described in detail below.
             ########################
             # filter plugin options
             ########################
-            set (PLUGIN_TGZ_ORIGPATH "https://github.com/HDFGroup/hdf5_plugins/releases/download/snapshots" CACHE STRING "Use PLUGINS from original location" FORCE)
+            set (PLUGIN_TGZ_ORIGPATH "https://github.com/HDFGroup/hdf5_plugins/releases/download/snapshot" CACHE STRING "Use PLUGINS from original location" FORCE)
             set (PLUGIN_TGZ_NAME "hdf5_plugins-master.tar.gz" CACHE STRING "Use PLUGINS from compressed file" FORCE)
             set (PLUGIN_USE_LOCALCONTENT ON CACHE BOOL "Use local file for PLUGIN FetchContent" FORCE)
             set (PLUGIN_PACKAGE_NAME "pl" CACHE STRING "Name of PLUGIN package" FORCE)
@@ -729,6 +739,8 @@ These five steps are described in detail below.
                ZLIB_GIT_BRANCH="${git_branch}"
                SZIP_GIT_URL:STRING="https://${git_url}/szip"
                SZIP_GIT_BRANCH="${git_branch}"
+               LIBAEC_GIT_URL:STRING="https://${git_url}/libaec"
+               LIBAEC_GIT_BRANCH="${git_branch}"
                PLUGIN_GIT_URL:STRING="https://${git_url}/plugin"
                PLUGIN_GIT_BRANCH="${git_branch}"
            ${git_url} should be changed to your location and ${git_branch} is
@@ -787,7 +799,7 @@ These five steps are described in detail below.
       The HDF Group for daily testing. It should be altered/ignored for the user's
       installation and needs.
 
-   7. More information about using CMake can be found at the KitWare site,
+   7. More information about using CMake can be found at the Kitware site,
          www.cmake.org.
 
    8. Nullsoft Scriptable Install System
@@ -858,6 +870,7 @@ HDF5_ONLY_SHARED_LIBS          "Only Build Shared Libraries"
 HDF5_ALLOW_UNSUPPORTED         "Allow unsupported combinations of configure options"             OFF
 HDF5_ENABLE_PARALLEL           "Enable parallel build (requires MPI)"                            OFF
 HDF5_ENABLE_THREADSAFE         "Enable Threadsafety"                                             OFF
+HDF5_ENABLE_CONCURRENCY        "Enable multi-threaded concurrency"                                             OFF
 HDF5_DIMENSION_SCALES_NEW_REF  "Use new-style references with dimension scale APIs"              OFF
 HDF5_EXTERNAL_LIB_PREFIX       "Use prefix for custom library naming."                           ""
 HDF5_EXTERNAL_LIB_SUFFIX       "Use suffix for custom library naming."                           ""
@@ -939,7 +952,7 @@ if (BUILD_TESTING)
 HDF5_ALLOW_EXTERNAL_SUPPORT "Allow External Library Building (NO GIT TGZ)"  "NO"
 HDF5_ENABLE_PLUGIN_SUPPORT  "Enable PLUGIN Filters"                         OFF
 HDF5_ENABLE_SZIP_SUPPORT    "Use SZip Filter"                               OFF
-HDF5_ENABLE_ZLIB_SUPPORT   "Enable Zlib Filters"                           ON
+HDF5_ENABLE_ZLIB_SUPPORT    "Enable Zlib Filters"                           OFF
 
 if (HDF5_USE_ZLIB_NG)
     ZLIBNG_USE_EXTERNAL     "Use External Library Building for ZLIBNG"      OFF
@@ -949,7 +962,7 @@ else
     ZLIB_USE_EXTERNAL       "Use External Library Building for ZLIB"        OFF
     ZLIB_TGZ_ORIGPATH       "Use ZLIB from original location"               "https://github.com/madler/zlib/releases/download/v1.3.1"
     ZLIB_TGZ_NAME           "Use ZLIB from original compressed file"        "zlib-1.3.1.tar.gz"
-ZLIB_USE_LOCALCONTENT       "Use local file for ZLIB FetchContent"          ON
+ZLIB_USE_LOCALCONTENT       "Use local file for ZLIB FetchContent"          OFF
 HDF5_USE_ZLIB_STATIC        "Find static zlib library"                      OFF
 
 SZIP_USE_EXTERNAL           "Use External Library Building for SZIP else search"        OFF
@@ -957,10 +970,13 @@ if (HDF5_ENABLE_SZIP_SUPPORT)
     HDF5_ENABLE_SZIP_ENCODING "Use SZip Encoding"                           ON
 LIBAEC_TGZ_ORIGPATH         "Use LIBAEC from original location"             "https://github.com/MathisRosenhauer/libaec/releases/download/v1.1.3"
 LIBAEC_TGZ_NAME             "Use LIBAEC from original compressed file"      "libaec-1.1.3.tar.gz"
-LIBAEC_USE_LOCALCONTENT     "Use local file for LIBAEC FetchContent"        ON
+LIBAEC_USE_LOCALCONTENT     "Use local file for LIBAEC FetchContent"        OFF
 HDF5_USE_LIBAEC_STATIC      "Find static AEC library"                       OFF
 
 PLUGIN_USE_EXTERNAL         "Use External Library Building for PLUGINS else search"     OFF
+PLUGIN_TGZ_ORIGPATH         "Use PLUGIN from original location"             "https://github.com/HDFGroup/hdf5_plugins/releases/download/snapshot"
+PLUGIN_TGZ_NAME             "Use PLUGIN from original compressed file"      "hdf5_plugins-master.tar.gz"
+PLUGIN_USE_LOCALCONTENT     "Use local file for PLUGIN FetchContent"        OFF
 if (WINDOWS)
     H5_DEFAULT_PLUGINDIR    "%ALLUSERSPROFILE%/hdf5/lib/plugin"
 else ()
@@ -986,11 +1002,24 @@ NOTE:
 
     The high-level, C++, Fortran and Java interfaces are not compatible
     with the HDF5_ENABLE_THREADSAFE option because the lock is not hoisted
-    into the higher-level API calls.
-    Unless HDF5_ALLOW_UNSUPPORTED has been specified,
-    the following options must be disabled:
+    into the higher-level API calls.  Unless HDF5_ALLOW_UNSUPPORTED has been
+    specified, the following options must be disabled:
        HDF5_BUILD_HL_LIB, HDF5_BUILD_CPP_LIB, HDF5_BUILD_FORTRAN, HDF5_BUILD_JAVA
 
+    The multi-threaded concurrency and threadsafe options are mutually
+    exclusive, only one or the other may be enabled.
+
+    The multi-threaded concurrency, C++, and Java interfaces are not compatible
+    with the HDF5_ENABLE_PARALLEL option.
+    Unless ALLOW_UNSUPPORTED has been specified,
+    the following options must be disabled:
+        HDF5_ENABLE_CONCURRENCY, HDF5_BUILD_CPP_LIB, HDF5_BUILD_JAVA
+
+    The high-level, C++, Fortran, and Java interfaces are not compatible
+    with the HDF5_ENABLE_CONCURRENCY option because the lock is not hoisted
+    into the higher-level API calls.  Unless HDF5_ALLOW_UNSUPPORTED has been
+    specified, the following options must be disabled:
+       HDF5_BUILD_HL_LIB, HDF5_BUILD_CPP_LIB, HDF5_BUILD_FORTRAN, HDF5_BUILD_JAVA
 
 ========================================================================
 VII. User Defined Options for HDF5 Libraries with CMake
diff --git a/release_docs/NEWSLETTER.txt b/release_docs/NEWSLETTER.txt
index ede8f8c5428..e60e816a935 100644
--- a/release_docs/NEWSLETTER.txt
+++ b/release_docs/NEWSLETTER.txt
@@ -3,7 +3,13 @@ HDF5 version 2.0.0 currently under development
 Features included for the next major release:
 ----------------------------------------------------------------------------
 
-*
+      ************ Renamed the option: HDF5_ENABLE_Z_LIB_SUPPORT ************
+
+      The option has been renamed to HDF5_ENABLE_ZLIB_SUPPORT to be consistent
+      with the naming of other options.
+      *** Also, the option defaults to OFF. This requires the user to explicitly ***
+      *** enable zlib support when configuring the library. ***
+
 
 ----------------------------------------------------------------------------
 Please see the full release notes for detailed information regarding this release,
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index a0b2c2d6210..eed9e4a880a 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -47,10 +47,27 @@ New Features
 
     Configuration:
     -------------
-    - Renamed the option: HDF5_ENABLE_Z_LIB_SUPPORT
+    - Added configuration option for API concurrency support:
+
+          CMake:      HDF5_ENABLE_CONCURRENCY (ON/OFF) (Default: OFF)
+          Autotools:  --enable-concurrency    (yes/no) (Default: no)
+
+      This option enables support for concurrent multithreaded operation
+      of supported API routines.  This option also provides threadsafe
+      execution of all other, non-concurrent operations.  The 'concurrency'
+      option thus is a superset of the existing 'threadsafe' option.  Both
+      options are currently available, although mutually exclusive.  As the
+      'concurrency' code becomes more stable over time, the 'threadsafe' option
+      may be deprecated in favor of the new 'concurrency' option.
+      The following API routines support concurrent multithreaded operation:
+      
+
+    - ************ Renamed the option: HDF5_ENABLE_Z_LIB_SUPPORT ************
 
       The option has been renamed to HDF5_ENABLE_ZLIB_SUPPORT to be consistent
       with the naming of other options.
+      *** Also, the option defaults to OFF. This requires the user to explicitly ***
+      *** enable zlib support when configuring the library. ***
 
     - Added support for MinGW + MSYS2 when building with CMake
 
@@ -82,18 +99,6 @@ New Features
       Intel, GNU and Clang compilers are now in separate files included from
       the current compiler flags files; HDFCompilerFlags.cmake.
 
-    - Added a configuration option for internal threading/concurrency support:
-
-          CMake:      HDF5_ENABLE_THREADS (ON/OFF) (Default: ON)
-          Autotools:  --enable-threads    (yes/no) (Default: yes)
-
-      This option enables support for threading and concurrency algorithms
-      within the HDF5 library.  It is required for, but separate from, the
-      'threadsafe' configure option, which makes the HDF5 API safe to call from
-      multiple threads.  It is possible to enable the 'threads' option and
-      disable the 'threadsafe' option, but not vice versa.  The 'threads' option
-      must be on to enable the subfiling VFD.
-
     - Added support for native zlib-ng compression.
 
       Changed the zlib-ng CMake logic to prefer the native zlib-ng library. Added
@@ -153,9 +158,26 @@ New Features
       some platforms use gnu11 to get some GNU things to work.
 
 
-
     Library:
     --------
+    - The H5Iregister_type() signature has changed
+
+      The hash_size parameter has not been used since early versions of HDF5
+      1.8, so it has been removed and the API call has been versioned.
+
+      The old signature has been renamed to H5Iregister_type1() and is considered
+      deprecated:
+
+        H5I_type_t H5Iregister_type1(size_t hash_size, unsigned reserved, H5I_free_t free_func);
+
+      The new signature is H5Iregister_type2(). New code should use this
+      version:
+
+        H5I_type_t H5Iregister_type2(unsigned reserved, H5I_free_t free_func);
+
+      H5Iregister_type() will map to the new signature unless the library is
+      explicitly configured to use an older version of the API.
+
     - H5F_LIBVER_LATEST is now an enum value
 
       This was previously #defined to the latest H5F_libver_t API version, but
@@ -386,7 +408,17 @@ New Features
       There is no API compatibility wrapper for this change.
 
       Fixes GitHub issue #3506
+    
+    - H5Pset* routines now fail when used on default property lists
+
+      Modifying default property lists was never fully supported and could produce
+      inconsistent and unexpected behavior. 
+
+    - H5Pset_vol() now fails when used on a non-file-access property list
 
+      Similar to the above. Setting the connector on a non-FAPL had no effect on
+      library behavior, and the connector ID and information could not be read back
+      from that plist later.
 
     Parallel Library:
     -----------------
@@ -505,11 +537,44 @@ Bug Fixes since HDF5-2.0.0 release
 
     Java Library
     ------------
-    -
+    - Renamed the Callbacks.java file to H5Callbacks.java
+
+      The Callbacks.java file was renamed to H5Callbacks.java to match the file
+      pattern used by doxygen. This change only affects the Java filenames and
+      does not change the classname or the package name.
 
 
     Configuration
     -------------
+    - Use pre-installed libaec compression library
+
+      The CMake logic for finding the libaec compression library has been
+      modified for a system-installed version of the library. Two options
+      must be set;
+          HDF5_ALLOW_EXTERNAL_SUPPORT:STRING=NO
+          _USE_EXTERNAL:BOOL=OFF
+      where  is one of ZLIB, ZLIBNG, SZIP, PLUGIN.
+      Note that HDF5_ALLOW_EXTERNAL_SUPPORT:STRING=NO disables building all plugins
+      and external libraries in-line with the HDF5 library.
+
+      In addition, the _ROOT environment variables must be set,
+      where  is one of ZLIB, ZLIBNG, SZIP, libaec, PLUGIN.
+      Note that libaec is the expected name for using the libaec library in place of original szip.
+
+      See INSTALL_CMake.txt for more detailed information.
+
+    - Changed the zlib/szip compression find message to FATAL ERROR
+
+      The message was changed to indicate that zlib/szip compression was requested and
+      that it was not found. If an option is requested, not finding it should always
+      be an error.
+
+    - Removed the module search find_package for szip library
+    
+      There is not a szip module file to use, so the find_package only uses
+      find_package in config mode. The choice then is to either build szip, with libaec,
+      inline or find a system installed szip library, built with CMake.
+
     - Changed name of libhdf5hl_fortran installed by autotools to libhdf5_hl_fortran.  The 
       new name is consistent with the name of the lib when installed by CMake and with the 
       other hl libs.  
diff --git a/release_docs/RELEASE_PROCESS.md b/release_docs/RELEASE_PROCESS.md
index a57a1dbe904..4bf8c3601cc 100644
--- a/release_docs/RELEASE_PROCESS.md
+++ b/release_docs/RELEASE_PROCESS.md
@@ -197,7 +197,7 @@ For more information on the HDF5 versioning and backward and forward compatibili
     - Change 'HDF5 target bucket directory' to 'vX_Y/vX_Y_Z'
     - Press "Run Workflow"
 
-### 10. Add the contents of the RELEASE.txt file in the release code to the HISTORY- file in the **support** branch, just below the introductory lines at the top of the HISTORY file.
+### 10. Add the contents of the RELEASE.txt file in the release code to the HISTORY-X_Y file in the **support** branch, just below the introductory lines at the top of the HISTORY file.
 
 ### 11. Conduct Release Retrospective (Release Manager)
 1. Schedule time and solicit comments from retrospective
@@ -216,5 +216,5 @@ For more information on the HDF5 versioning and backward and forward compatibili
 [u11]: https://github.com/HDFGroup/hdf5/blob/develop/src/CMakeLists.txt
 [u12]: https://github.com/HDFGroup/hdf5/blob/develop/configure.ac
 [u13]: https://hdfgroup.github.io/hdf5/develop/api-compat-macros.html
-[u14]: https://github.com/HDFGroup/hdf5/releases/tag/snapshot-2.0.0
+[u14]: https://github.com/HDFGroup/hdf5/releases/tag/snapshot-1.14
 [u15]: https://github.com/HDFGroup/hdf5/releases/tag/snapshot
diff --git a/release_docs/USING_CMake_Examples.txt b/release_docs/USING_CMake_Examples.txt
index 514e300b393..ca87906cad2 100644
--- a/release_docs/USING_CMake_Examples.txt
+++ b/release_docs/USING_CMake_Examples.txt
@@ -9,7 +9,7 @@ Notes: This short instruction is written for users who want to quickly
        installations.
        For more information, see the USING_HDF5_CMake.txt file.
 
-       More information about using CMake can be found at the KitWare
+       More information about using CMake can be found at the Kitware
        site, www.cmake.org.
 
        CMake uses the command line and these instructions use the script
diff --git a/release_docs/USING_HDF5_CMake.txt b/release_docs/USING_HDF5_CMake.txt
index 12fdabdd5f0..2201ad7868e 100644
--- a/release_docs/USING_HDF5_CMake.txt
+++ b/release_docs/USING_HDF5_CMake.txt
@@ -7,7 +7,7 @@ Notes: This short instruction is written for users who want to quickly
        these instructions for their own applications. For more information,
        see the "Minimum C Project Files for CMake" section.
 
-       More information about using CMake can be found at the KitWare
+       More information about using CMake can be found at the Kitware
        site, www.cmake.org.
 
        CMake uses the command line; however, the visual CMake tool is
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 6250c58b3c9..9655962e65c 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -387,6 +387,7 @@ IDE_GENERATED_PROPERTIES ("H5HL" "${H5HL_HDRS}" "${H5HL_SOURCES}" )
 set (H5I_SOURCES
     ${HDF5_SRC_DIR}/H5I.c
     ${HDF5_SRC_DIR}/H5Idbg.c
+    ${HDF5_SRC_DIR}/H5Ideprec.c
     ${HDF5_SRC_DIR}/H5Iint.c
     ${HDF5_SRC_DIR}/H5Itest.c
 )
@@ -739,7 +740,7 @@ set (H5Z_SOURCES
     ${HDF5_SRC_DIR}/H5Ztrans.c
 )
 if (H5_ZLIB_HEADER)
-  message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
+  message(VERBOSE "H5_ZLIB_HEADER for library=${H5_ZLIB_HEADER}")
   set_source_files_properties(${HDF5_SRC_DIR}/H5Zdeflate.c
                PROPERTIES COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}"
   )
diff --git a/src/H5.c b/src/H5.c
index b246763b490..63d7aa15a26 100644
--- a/src/H5.c
+++ b/src/H5.c
@@ -31,6 +31,7 @@
 #include "H5PLprivate.h" /* Plugins                                  */
 #include "H5SLprivate.h" /* Skip lists                               */
 #include "H5Tprivate.h"  /* Datatypes                                */
+#include "H5TSprivate.h" /* Threadsafety                             */
 
 /****************/
 /* Local Macros */
@@ -214,14 +215,14 @@ H5_init_library(void)
      */
     if (!H5_dont_atexit_g) {
 
-#if defined(H5_HAVE_THREADSAFE)
+#ifdef H5_HAVE_THREADSAFE_API
         /* Clean up thread resources.
          *
          * This must be pushed before the library cleanup code so it's
          * executed in LIFO order (i.e., last).
          */
         (void)atexit(H5TS_term_package);
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
         /* Normal library termination code */
         (void)atexit(H5_term_library);
@@ -321,7 +322,7 @@ H5_term_library(void)
     H5CX_push(&api_ctx);
 
     /* Check if we should display error output */
-    (void)H5Eget_auto2(H5E_DEFAULT, &func, NULL);
+    (void)H5E_get_default_auto_func(&func);
 
     /* Iterate over the list of 'atclose' callbacks that have been registered */
     if (H5_atclose_head) {
@@ -332,8 +333,13 @@ H5_term_library(void)
         while (curr_atclose) {
             H5_atclose_node_t *tmp_atclose; /* Temporary pointer to 'atclose' node */
 
-            /* Invoke callback, providing context */
-            (*curr_atclose->func)(curr_atclose->ctx);
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB_NOCHECK
+                {
+                    /* Invoke callback, providing context */
+                    (*curr_atclose->func)(curr_atclose->ctx);
+                }
+            H5_AFTER_USER_CB_NOCHECK
 
             /* Advance to next node and free this one */
             tmp_atclose  = curr_atclose;
@@ -1057,11 +1063,11 @@ H5close(void)
      * whole library just to release it all right away.  It is safe to call
      * this function for an uninitialized library.
      */
-    FUNC_ENTER_API_NOINIT_NOERR
+    FUNC_ENTER_API_NAMECHECK_ONLY
 
     H5_term_library();
 
-    FUNC_LEAVE_API_NOERR(SUCCEED)
+    FUNC_LEAVE_API_NAMECHECK_ONLY(SUCCEED)
 } /* end H5close() */
 
 /*-------------------------------------------------------------------------
@@ -1097,13 +1103,14 @@ H5allocate_memory(size_t size, bool clear)
     FUNC_ENTER_API_NOINIT
 
     if (0 == size)
-        return NULL;
+        HGOTO_DONE(NULL);
 
     if (clear)
         ret_value = H5MM_calloc(size);
     else
         ret_value = H5MM_malloc(size);
 
+done:
     FUNC_LEAVE_API_NOINIT(ret_value)
 } /* end H5allocate_memory() */
 
@@ -1184,11 +1191,11 @@ H5is_library_threadsafe(bool *is_ts /*out*/)
     FUNC_ENTER_API_NOINIT
 
     if (is_ts) {
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
         *is_ts = true;
-#else  /* H5_HAVE_THREADSAFE */
+#else  /* H5_HAVE_THREADSAFE_API */
         *is_ts = false;
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
     }
     else
         ret_value = FAIL;
@@ -1226,3 +1233,63 @@ H5is_library_terminating(bool *is_terminating /*out*/)
 
     FUNC_LEAVE_API_NOINIT(ret_value)
 } /* end H5is_library_terminating() */
+
+/*-------------------------------------------------------------------------
+ * Function:    H5_user_cb_prepare
+ *
+ * Purpose:     Prepares library before a user callback
+ *
+ * Return:      SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5_user_cb_prepare(H5_user_cb_state_t *state)
+{
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Prepare H5E package for user callback */
+    if (H5E_user_cb_prepare(&state->h5e_state) < 0)
+        HGOTO_ERROR(H5E_LIB, H5E_CANTSET, FAIL, "unable to prepare H5E package for user callback");
+
+#ifdef H5_HAVE_CONCURRENCY
+    /* Prepare H5TS package for user callback */
+    if (H5TS_user_cb_prepare() < 0)
+        HGOTO_ERROR(H5E_LIB, H5E_CANTSET, FAIL, "unable to prepare H5TS package for user callback");
+#endif /* H5_HAVE_THREADSAFE_API */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5_user_cb_prepare() */
+
+/*-------------------------------------------------------------------------
+ * Function:    H5_user_cb_restore
+ *
+ * Purpose:     Restores library after a user callback
+ *
+ * Return:      SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5_user_cb_restore(const H5_user_cb_state_t *state)
+{
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Restore H5E package after user callback */
+    if (H5E_user_cb_restore(&state->h5e_state) < 0)
+        HGOTO_ERROR(H5E_LIB, H5E_CANTRESTORE, FAIL, "unable to restore H5E package after user callback");
+
+#ifdef H5_HAVE_CONCURRENCY
+    /* Restore H5TS package after user callback */
+    if (H5TS_user_cb_restore() < 0)
+        HGOTO_ERROR(H5E_LIB, H5E_CANTRESTORE, FAIL, "unable to restore H5TS package after user callback");
+#endif /* H5_HAVE_THREADSAFE_API */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5_user_cb_restore() */
diff --git a/src/H5ACpublic.h b/src/H5ACpublic.h
index 3e989394788..eb0f3da7490 100644
--- a/src/H5ACpublic.h
+++ b/src/H5ACpublic.h
@@ -719,8 +719,18 @@ typedef struct H5AC_cache_config_t {
 
 #define H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION 1
 
+/**
+ * No limit on number of times a prefetched entry can appear in subsequent
+ * cache images
+ * \since 1.10.1
+ */
 #define H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE -1
-#define H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX  100
+/**
+ * Limit on number of times a prefetched entry can appear in subsequent
+ * cache images
+ * \since 1.10.1
+ */
+#define H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX 100
 
 //! 
 /**
diff --git a/src/H5Adense.c b/src/H5Adense.c
index 11bb5479e93..f9ba92d1aa9 100644
--- a/src/H5Adense.c
+++ b/src/H5Adense.c
@@ -1053,17 +1053,27 @@ H5A__dense_iterate_bt2_cb(const void *_record, void *_bt2_udata)
                 if (H5A__get_info(fh_udata.attr, &ainfo) < 0)
                     HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, H5_ITER_ERROR, "unable to get attribute info");
 
-                /* Make the application callback */
-                ret_value = (bt2_udata->attr_op->u.app_op2)(bt2_udata->loc_id, fh_udata.attr->shared->name,
-                                                            &ainfo, bt2_udata->op_data);
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(H5_ITER_ERROR)
+                    {
+                        /* Make the application callback */
+                        ret_value = (bt2_udata->attr_op->u.app_op2)(
+                            bt2_udata->loc_id, fh_udata.attr->shared->name, &ainfo, bt2_udata->op_data);
+                    }
+                H5_AFTER_USER_CB(H5_ITER_ERROR)
                 break;
             }
 
 #ifndef H5_NO_DEPRECATED_SYMBOLS
             case H5A_ATTR_OP_APP:
-                /* Make the application callback */
-                ret_value = (bt2_udata->attr_op->u.app_op)(bt2_udata->loc_id, fh_udata.attr->shared->name,
-                                                           bt2_udata->op_data);
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(H5_ITER_ERROR)
+                    {
+                        /* Make the application callback */
+                        ret_value = (bt2_udata->attr_op->u.app_op)(
+                            bt2_udata->loc_id, fh_udata.attr->shared->name, bt2_udata->op_data);
+                    }
+                H5_AFTER_USER_CB(H5_ITER_ERROR)
                 break;
 #endif /* H5_NO_DEPRECATED_SYMBOLS */
 
diff --git a/src/H5Aint.c b/src/H5Aint.c
index f06c5ea0ede..88e26790cd0 100644
--- a/src/H5Aint.c
+++ b/src/H5Aint.c
@@ -1892,15 +1892,26 @@ H5A__attr_iterate_table(const H5A_attr_table_t *atable, hsize_t skip, hsize_t *l
                 if (H5A__get_info(atable->attrs[u], &ainfo) < 0)
                     HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, H5_ITER_ERROR, "unable to get attribute info");
 
-                /* Make the application callback */
-                ret_value = (attr_op->u.app_op2)(loc_id, ((atable->attrs[u])->shared)->name, &ainfo, op_data);
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(H5_ITER_ERROR)
+                    {
+                        /* Make the application callback */
+                        ret_value =
+                            (attr_op->u.app_op2)(loc_id, ((atable->attrs[u])->shared)->name, &ainfo, op_data);
+                    }
+                H5_AFTER_USER_CB(H5_ITER_ERROR)
                 break;
             }
 
 #ifndef H5_NO_DEPRECATED_SYMBOLS
             case H5A_ATTR_OP_APP:
-                /* Make the application callback */
-                ret_value = (attr_op->u.app_op)(loc_id, ((atable->attrs[u])->shared)->name, op_data);
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(H5_ITER_ERROR)
+                    {
+                        /* Make the application callback */
+                        ret_value = (attr_op->u.app_op)(loc_id, ((atable->attrs[u])->shared)->name, op_data);
+                    }
+                H5_AFTER_USER_CB(H5_ITER_ERROR)
                 break;
 #endif /* H5_NO_DEPRECATED_SYMBOLS */
 
diff --git a/src/H5CX.c b/src/H5CX.c
index 78892aacbd4..c6a43801d0b 100644
--- a/src/H5CX.c
+++ b/src/H5CX.c
@@ -40,7 +40,7 @@
 /* Local Macros */
 /****************/
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 /*
  * The per-thread API context.
  *
@@ -48,12 +48,12 @@
  * by "H5CX_node_t **ctx =".
  */
 #define H5CX_get_my_context() H5TS_get_api_ctx_ptr()
-#else /* H5_HAVE_THREADSAFE */
+#else /* H5_HAVE_THREADSAFE_API */
 /*
  * The current API context.
  */
 #define H5CX_get_my_context() (&H5CX_head_g)
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
 /* Common macro for the retrieving the pointer to a property list */
 #define H5CX_RETRIEVE_PLIST(PL, FAILVAL)                                                                     \
@@ -226,9 +226,9 @@ bool H5_PKG_INIT_VAR = false;
 /* Local Variables */
 /*******************/
 
-#ifndef H5_HAVE_THREADSAFE
+#ifndef H5_HAVE_THREADSAFE_API
 static H5CX_node_t *H5CX_head_g = NULL; /* Pointer to head of context stack */
-#endif                                  /* H5_HAVE_THREADSAFE */
+#endif                                  /* H5_HAVE_THREADSAFE_API */
 
 /* Define a "default" dataset transfer property list cache structure to use for default DXPLs */
 static H5CX_dxpl_cache_t H5CX_def_dxpl_cache;
diff --git a/src/H5Cimage.c b/src/H5Cimage.c
index d626640dbdd..24fcafaabb3 100644
--- a/src/H5Cimage.c
+++ b/src/H5Cimage.c
@@ -116,7 +116,8 @@
 /* Helper routines */
 static size_t H5C__cache_image_block_entry_header_size(const H5F_t *f);
 static size_t H5C__cache_image_block_header_size(const H5F_t *f);
-static herr_t H5C__decode_cache_image_header(const H5F_t *f, H5C_t *cache_ptr, const uint8_t **buf);
+static herr_t H5C__decode_cache_image_header(const H5F_t *f, H5C_t *cache_ptr, const uint8_t **buf,
+                                             size_t buf_size);
 #ifndef NDEBUG /* only used in assertions */
 static herr_t H5C__decode_cache_image_entry(const H5F_t *f, const H5C_t *cache_ptr, const uint8_t **buf,
                                             unsigned entry_num);
@@ -297,7 +298,7 @@ H5C__construct_cache_image_buffer(H5F_t *f, H5C_t *cache_ptr)
         /* needed for sanity checks */
         fake_cache_ptr->image_len = cache_ptr->image_len;
         q                         = (const uint8_t *)cache_ptr->image_buffer;
-        status                    = H5C__decode_cache_image_header(f, fake_cache_ptr, &q);
+        status = H5C__decode_cache_image_header(f, fake_cache_ptr, &q, cache_ptr->image_len + 1);
         assert(status >= 0);
 
         assert(NULL != p);
@@ -1267,7 +1268,7 @@ H5C__cache_image_block_header_size(const H5F_t *f)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5C__decode_cache_image_header(const H5F_t *f, H5C_t *cache_ptr, const uint8_t **buf)
+H5C__decode_cache_image_header(const H5F_t *f, H5C_t *cache_ptr, const uint8_t **buf, size_t buf_size)
 {
     uint8_t        version;
     uint8_t        flags;
@@ -1287,6 +1288,10 @@ H5C__decode_cache_image_header(const H5F_t *f, H5C_t *cache_ptr, const uint8_t *
     /* Point to buffer to decode */
     p = *buf;
 
+    /* Ensure buffer has enough data for signature comparison */
+    if (H5_IS_BUFFER_OVERFLOW(p, H5C__MDCI_BLOCK_SIGNATURE_LEN, *buf + buf_size - 1))
+        HGOTO_ERROR(H5E_CACHE, H5E_OVERFLOW, FAIL, "Insufficient buffer size for signature");
+
     /* Check signature */
     if (memcmp(p, H5C__MDCI_BLOCK_SIGNATURE, (size_t)H5C__MDCI_BLOCK_SIGNATURE_LEN) != 0)
         HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Bad metadata cache image header signature");
@@ -2386,7 +2391,7 @@ H5C__reconstruct_cache_contents(H5F_t *f, H5C_t *cache_ptr)
 
     /* Decode metadata cache image header */
     p = (uint8_t *)cache_ptr->image_buffer;
-    if (H5C__decode_cache_image_header(f, cache_ptr, &p) < 0)
+    if (H5C__decode_cache_image_header(f, cache_ptr, &p, cache_ptr->image_len + 1) < 0)
         HGOTO_ERROR(H5E_CACHE, H5E_CANTDECODE, FAIL, "cache image header decode failed");
     assert((size_t)(p - (uint8_t *)cache_ptr->image_buffer) < cache_ptr->image_len);
 
diff --git a/src/H5D.c b/src/H5D.c
index 0e47235ee2f..e3b132e5de3 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -1592,8 +1592,14 @@ H5Dscatter(H5D_scatter_func_t op, void *op_data, hid_t type_id, hid_t dst_space_
 
     /* Loop until all data has been scattered */
     while (nelmts > 0) {
-        /* Make callback to retrieve data */
-        if (op(&src_buf, &src_buf_nbytes, op_data) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                /* Make callback to retrieve data */
+                ret_value = op(&src_buf, &src_buf_nbytes, op_data);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_DATASET, H5E_CALLBACK, FAIL, "callback operator returned failure");
 
         /* Calculate number of elements */
@@ -1704,8 +1710,16 @@ H5Dgather(hid_t src_space_id, const void *src_buf, hid_t type_id, size_t dst_buf
         assert(nelmts_gathered == MIN(dst_buf_nelmts, (size_t)nelmts));
 
         /* Make callback to process dst_buf */
-        if (op && op(dst_buf, nelmts_gathered * type_size, op_data) < 0)
-            HGOTO_ERROR(H5E_DATASET, H5E_CALLBACK, FAIL, "callback operator returned failure");
+        if (op) {
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    ret_value = op(dst_buf, nelmts_gathered * type_size, op_data);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
+                HGOTO_ERROR(H5E_DATASET, H5E_CALLBACK, FAIL, "callback operator returned failure");
+        }
 
         nelmts -= (hssize_t)nelmts_gathered;
         assert(op || (nelmts == 0));
diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index f07d9b227d8..a9b4817d4db 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -1436,7 +1436,11 @@ H5D__chunk_mem_xfree(void *chk, const void *pline)
 void
 H5D__chunk_mem_free(void *chk, void *pline)
 {
-    (void)H5D__chunk_mem_xfree(chk, pline);
+    FUNC_ENTER_PACKAGE_NAMECHECK_ONLY
+
+    H5D__chunk_mem_xfree(chk, pline);
+
+    FUNC_LEAVE_NOAPI_VOID_NAMECHECK_ONLY
 }
 
 /*-------------------------------------------------------------------------
@@ -8134,16 +8138,23 @@ H5D__chunk_iter_cb(const H5D_chunk_rec_t *chunk_rec, void *udata)
     hsize_t                    offset[H5O_LAYOUT_NDIMS];
     int                        ret_value = H5_ITER_CONT;
 
+    FUNC_ENTER_PACKAGE_NOERR
+
     /* Similar to H5D__get_chunk_info */
     for (unsigned i = 0; i < chunk->ndims; i++)
         offset[i] = chunk_rec->scaled[i] * chunk->dim[i];
 
-    FUNC_ENTER_PACKAGE_NOERR
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB_NOERR(FAIL)
+        {
+            ret_value =
+                (data->op)(offset, (unsigned)chunk_rec->filter_mask, data->base_addr + chunk_rec->chunk_addr,
+                           (hsize_t)chunk_rec->nbytes, data->op_data);
+        }
+    H5_AFTER_USER_CB_NOERR(FAIL)
 
     /* Check for callback failure and pass along return value */
-    if ((ret_value =
-             (data->op)(offset, (unsigned)chunk_rec->filter_mask, data->base_addr + chunk_rec->chunk_addr,
-                        (hsize_t)chunk_rec->nbytes, data->op_data)) < 0)
+    if (ret_value < 0)
         HERROR(H5E_DATASET, H5E_CANTNEXT, "iteration operator failed");
 
     FUNC_LEAVE_NOAPI(ret_value)
diff --git a/src/H5Dfill.c b/src/H5Dfill.c
index e4703b8af74..093039987b8 100644
--- a/src/H5Dfill.c
+++ b/src/H5Dfill.c
@@ -144,7 +144,7 @@ H5D__fill(const void *fill, const H5T_t *fill_type, void *buf, const H5T_t *buf_
 
         /* Get a pointer to a buffer that's large enough for element */
         if (NULL == (elem_ptr = H5WB_actual_clear(elem_wb, dst_type_size)))
-            HGOTO_ERROR(H5E_DATASET, H5E_NOSPACE, FAIL, "can't get actual buffer");
+            HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't get actual buffer");
 
         /* Fill the selection in the memory buffer */
         if (H5S_select_fill(elem_ptr, dst_type_size, space, buf) < 0)
@@ -179,12 +179,12 @@ H5D__fill(const void *fill, const H5T_t *fill_type, void *buf, const H5T_t *buf_
 
             /* Allocate a temporary buffer */
             if (NULL == (tmp_buf = H5FL_BLK_MALLOC(type_conv, (size_t)nelmts * buf_size)))
-                HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed");
+                HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "memory allocation failed");
 
             /* Allocate a background buffer, if necessary */
             if (H5T_path_bkg(tpath) &&
                 NULL == (bkg_buf = H5FL_BLK_CALLOC(type_conv, (size_t)nelmts * buf_size)))
-                HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed");
+                HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "memory allocation failed");
 
             /* Replicate the file's fill value into the temporary buffer */
             H5VM_array_fill(tmp_buf, fill, src_type_size, (size_t)nelmts);
@@ -222,7 +222,7 @@ H5D__fill(const void *fill, const H5T_t *fill_type, void *buf, const H5T_t *buf_
 
                 /* Get a pointer to a buffer that's large enough for element */
                 if (NULL == (elem_ptr = H5WB_actual(elem_wb, buf_size)))
-                    HGOTO_ERROR(H5E_DATASET, H5E_NOSPACE, FAIL, "can't get actual buffer");
+                    HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't get actual buffer");
 
                 /* Copy the user's data into the buffer for conversion */
                 H5MM_memcpy(elem_ptr, fill, src_type_size);
@@ -236,7 +236,7 @@ H5D__fill(const void *fill, const H5T_t *fill_type, void *buf, const H5T_t *buf_
 
                     /* Get a pointer to a buffer that's large enough for element */
                     if (NULL == (bkg_ptr = H5WB_actual_clear(bkg_elem_wb, buf_size)))
-                        HGOTO_ERROR(H5E_DATASET, H5E_NOSPACE, FAIL, "can't get actual buffer");
+                        HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't get actual buffer");
                 } /* end if */
 
                 /* Perform datatype conversion */
@@ -264,9 +264,9 @@ H5D__fill(const void *fill, const H5T_t *fill_type, void *buf, const H5T_t *buf_
     if (tmp_buf)
         tmp_buf = H5FL_BLK_FREE(type_conv, tmp_buf);
     if (elem_wb && H5WB_unwrap(elem_wb) < 0)
-        HDONE_ERROR(H5E_ATTR, H5E_CLOSEERROR, FAIL, "can't close wrapped buffer");
+        HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close wrapped buffer");
     if (bkg_elem_wb && H5WB_unwrap(bkg_elem_wb) < 0)
-        HDONE_ERROR(H5E_ATTR, H5E_CLOSEERROR, FAIL, "can't close wrapped buffer");
+        HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close wrapped buffer");
     if (bkg_buf)
         bkg_buf = H5FL_BLK_FREE(type_conv, bkg_buf);
 
@@ -347,22 +347,28 @@ H5D__fill_init(H5D_fill_buf_info_t *fb_info, void *caller_fill_buf, H5MM_allocat
                 fb_info->use_caller_fill_buf = true;
             } /* end if */
             else {
-                if (alloc_func)
-                    fb_info->fill_buf = alloc_func(fb_info->fill_buf_size, alloc_info);
+                if (alloc_func) {
+                    /* Prepare & restore library for user callback */
+                    H5_BEFORE_USER_CB(FAIL)
+                        {
+                            fb_info->fill_buf = alloc_func(fb_info->fill_buf_size, alloc_info);
+                        }
+                    H5_AFTER_USER_CB(FAIL)
+                }
                 else
                     fb_info->fill_buf = H5FL_BLK_MALLOC(non_zero_fill, fb_info->fill_buf_size);
                 if (NULL == fb_info->fill_buf)
-                    HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for fill buffer");
+                    HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "memory allocation failed for fill buffer");
             } /* end else */
 
             /* Get the datatype conversion path for this operation */
             if (NULL == (fb_info->fill_to_mem_tpath = H5T_path_find(dset_type, fb_info->mem_type)))
-                HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL,
+                HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
                             "unable to convert between src and dst datatypes");
 
             /* Get the inverse datatype conversion path for this operation */
             if (NULL == (fb_info->mem_to_dset_tpath = H5T_path_find(fb_info->mem_type, dset_type)))
-                HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL,
+                HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
                             "unable to convert between src and dst datatypes");
 
             /* Check if we need to allocate a background buffer */
@@ -376,7 +382,7 @@ H5D__fill_init(H5D_fill_buf_info_t *fb_info, void *caller_fill_buf, H5MM_allocat
 
                 /* Allocate the background buffer */
                 if (NULL == (fb_info->bkg_buf = H5FL_BLK_MALLOC(type_conv, fb_info->bkg_buf_size)))
-                    HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed");
+                    HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "memory allocation failed");
             } /* end if */
         }     /* end if */
         else {
@@ -400,12 +406,18 @@ H5D__fill_init(H5D_fill_buf_info_t *fb_info, void *caller_fill_buf, H5MM_allocat
                 fb_info->use_caller_fill_buf = true;
             } /* end if */
             else {
-                if (alloc_func)
-                    fb_info->fill_buf = alloc_func(fb_info->fill_buf_size, alloc_info);
+                if (alloc_func) {
+                    /* Prepare & restore library for user callback */
+                    H5_BEFORE_USER_CB(FAIL)
+                        {
+                            fb_info->fill_buf = alloc_func(fb_info->fill_buf_size, alloc_info);
+                        }
+                    H5_AFTER_USER_CB(FAIL)
+                }
                 else
                     fb_info->fill_buf = H5FL_BLK_MALLOC(non_zero_fill, fb_info->fill_buf_size);
                 if (NULL == fb_info->fill_buf)
-                    HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for fill buffer");
+                    HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "memory allocation failed for fill buffer");
             } /* end else */
 
             /* Replicate the fill value into the cached buffer */
@@ -436,7 +448,14 @@ H5D__fill_init(H5D_fill_buf_info_t *fb_info, void *caller_fill_buf, H5MM_allocat
         } /* end if */
         else {
             if (alloc_func) {
-                fb_info->fill_buf = alloc_func(fb_info->fill_buf_size, alloc_info);
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        fb_info->fill_buf = alloc_func(fb_info->fill_buf_size, alloc_info);
+                    }
+                H5_AFTER_USER_CB(FAIL)
+                if (NULL == fb_info->fill_buf)
+                    HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "memory allocation failed for fill buffer");
 
                 memset(fb_info->fill_buf, 0, fb_info->fill_buf_size);
             } /* end if */
@@ -453,7 +472,7 @@ H5D__fill_init(H5D_fill_buf_info_t *fb_info, void *caller_fill_buf, H5MM_allocat
                     fb_info->fill_buf = H5FL_BLK_MALLOC(zero_fill, fb_info->fill_buf_size);
             } /* end else */
             if (fb_info->fill_buf == NULL)
-                HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for fill buffer");
+                HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "memory allocation failed for fill buffer");
         } /* end else */
     }     /* end else */
 
@@ -510,8 +529,14 @@ H5D__fill_refill_vl(H5D_fill_buf_info_t *fb_info, size_t nelmts)
         memset(fb_info->bkg_buf, 0, fb_info->bkg_buf_size);
 
     /* Make a copy of the fill buffer so we can free dynamic elements after conversion */
-    if (fb_info->fill_alloc_func)
-        buf = fb_info->fill_alloc_func(fb_info->fill_buf_size, fb_info->fill_alloc_info);
+    if (fb_info->fill_alloc_func) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                buf = fb_info->fill_alloc_func(fb_info->fill_buf_size, fb_info->fill_alloc_info);
+            }
+        H5_AFTER_USER_CB(FAIL)
+    }
     else
         buf = H5FL_BLK_MALLOC(non_zero_fill, fb_info->fill_buf_size);
     if (!buf)
@@ -537,8 +562,14 @@ H5D__fill_refill_vl(H5D_fill_buf_info_t *fb_info, size_t nelmts)
         } /* end else */
 
         /* Free temporary fill buffer */
-        if (fb_info->fill_free_func)
-            fb_info->fill_free_func(buf, fb_info->fill_free_info);
+        if (fb_info->fill_free_func) {
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB_NOERR(FAIL)
+                {
+                    fb_info->fill_free_func(buf, fb_info->fill_free_info);
+                }
+            H5_AFTER_USER_CB_NOERR(FAIL)
+        }
         else
             buf = H5FL_BLK_FREE(non_zero_fill, buf);
     } /* end if */
@@ -558,6 +589,8 @@ H5D__fill_refill_vl(H5D_fill_buf_info_t *fb_info, size_t nelmts)
 static herr_t
 H5D__fill_release(H5D_fill_buf_info_t *fb_info)
 {
+    herr_t ret_value = SUCCEED; /* Return value */
+
     FUNC_ENTER_PACKAGE_NOERR
 
     /* Check args */
@@ -566,8 +599,14 @@ H5D__fill_release(H5D_fill_buf_info_t *fb_info)
 
     /* Free the buffer for fill values */
     if (!fb_info->use_caller_fill_buf && fb_info->fill_buf) {
-        if (fb_info->fill_free_func)
-            fb_info->fill_free_func(fb_info->fill_buf, fb_info->fill_free_info);
+        if (fb_info->fill_free_func) {
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB_NOERR(FAIL)
+                {
+                    fb_info->fill_free_func(fb_info->fill_buf, fb_info->fill_free_info);
+                }
+            H5_AFTER_USER_CB_NOERR(FAIL)
+        }
         else {
             if (fb_info->fill->buf)
                 fb_info->fill_buf = H5FL_BLK_FREE(non_zero_fill, fb_info->fill_buf);
@@ -577,7 +616,7 @@ H5D__fill_release(H5D_fill_buf_info_t *fb_info)
         fb_info->fill_buf = NULL;
     } /* end if */
 
-    FUNC_LEAVE_NOAPI(SUCCEED)
+    FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5D__fill_release() */
 
 /*-------------------------------------------------------------------------
diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h
index c8635fdab72..a3257c99192 100644
--- a/src/H5Dprivate.h
+++ b/src/H5Dprivate.h
@@ -135,10 +135,10 @@
 /****************************/
 
 /* Typedef for dataset in memory (defined in H5Dpkg.h) */
-typedef struct H5D_t            H5D_t;
-typedef struct H5D_obj_create_t H5D_obj_create_t;
+typedef struct H5D_t H5D_t;
 
 /* Other forward declarations of structs needed by this file */
+typedef struct H5D_obj_create_t   H5D_obj_create_t;
 typedef struct H5D_io_type_info_t H5D_io_type_info_t;
 typedef struct H5D_dset_io_info_t H5D_dset_io_info_t;
 
diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h
index 67cea7a66b2..d61b8bf94de 100644
--- a/src/H5Dpublic.h
+++ b/src/H5Dpublic.h
@@ -28,9 +28,7 @@
 #define H5D_CHUNK_CACHE_NBYTES_DEFAULT SIZE_MAX
 #define H5D_CHUNK_CACHE_W0_DEFAULT     (-1.0)
 
-/**
- * Bit flags for the H5Pset_chunk_opts() and H5Pget_chunk_opts()
- */
+/** Bit flags for the H5Pset/get_chunk_opts() \since 1.10.0 */
 #define H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS (0x0002u)
 
 /*******************/
@@ -2064,7 +2062,7 @@ H5_DLL herr_t H5Dget_chunk_index_type(hid_t did, H5D_chunk_index_t *idx_type);
  */
 #ifndef H5_NO_DEPRECATED_SYMBOLS
 
-/* Macros */
+/** v1 B-tree index \since 1.10.0 */
 #define H5D_CHUNK_BTREE H5D_CHUNK_IDX_BTREE
 
 /* Formerly used to support the H5DOread/write_chunk() API calls.
diff --git a/src/H5ESint.c b/src/H5ESint.c
index 6a72d911440..655fb818c04 100644
--- a/src/H5ESint.c
+++ b/src/H5ESint.c
@@ -298,9 +298,18 @@ H5ES__insert(H5ES_t *es, H5VL_connector_t *connector, void *request_token, const
     ev_inserted = true;
 
     /* Invoke the event set's 'insert' callback, if present */
-    if (es->ins_func)
-        if ((es->ins_func)(&ev->op_info, es->ins_ctx) < 0)
+    if (es->ins_func) {
+        int status = -1;
+
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                status = (es->ins_func)(&ev->op_info, es->ins_ctx);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (status < 0)
             HGOTO_ERROR(H5E_EVENTSET, H5E_CALLBACK, FAIL, "'insert' callback for event set failed");
+    }
 
 done:
     /* Release resources on error */
@@ -557,6 +566,7 @@ H5ES__op_complete(H5ES_t *es, H5ES_event_t *ev, H5VL_request_status_t ev_status)
         /* Invoke the event set's 'complete' callback, if present */
         if (es->comp_func) {
             H5ES_status_t op_status; /* Status for complete callback */
+            int           status = -1;
 
             /* Set appropriate info for callback */
             if (H5VL_REQUEST_STATUS_SUCCEED == ev_status) {
@@ -577,7 +587,13 @@ H5ES__op_complete(H5ES_t *es, H5ES_event_t *ev, H5VL_request_status_t ev_status)
                 /* Translate status */
                 op_status = H5ES_STATUS_CANCELED;
 
-            if ((es->comp_func)(&ev->op_info, op_status, H5I_INVALID_HID, es->comp_ctx) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    status = (es->comp_func)(&ev->op_info, op_status, H5I_INVALID_HID, es->comp_ctx);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (status < 0)
                 HGOTO_ERROR(H5E_EVENTSET, H5E_CALLBACK, FAIL, "'complete' callback for event set failed");
         } /* end if */
 
@@ -591,6 +607,7 @@ H5ES__op_complete(H5ES_t *es, H5ES_event_t *ev, H5VL_request_status_t ev_status)
             /* Set up VOL callback arguments */
             vol_cb_args.op_type                         = H5VL_REQUEST_GET_ERR_STACK;
             vol_cb_args.args.get_err_stack.err_stack_id = H5I_INVALID_HID;
+            int status                                  = -1;
 
             /* Retrieve the error stack for the operation */
             if (H5VL_request_specific(ev->request, &vol_cb_args) < 0)
@@ -599,7 +616,13 @@ H5ES__op_complete(H5ES_t *es, H5ES_event_t *ev, H5VL_request_status_t ev_status)
             /* Set values */
             err_stack_id = vol_cb_args.args.get_err_stack.err_stack_id;
 
-            if ((es->comp_func)(&ev->op_info, H5ES_STATUS_FAIL, err_stack_id, es->comp_ctx) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    status = (es->comp_func)(&ev->op_info, H5ES_STATUS_FAIL, err_stack_id, es->comp_ctx);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (status < 0)
                 HGOTO_ERROR(H5E_EVENTSET, H5E_CALLBACK, FAIL, "'complete' callback for event set failed");
         } /* end if */
 
diff --git a/src/H5Eint.c b/src/H5Eint.c
index 8227e7cd421..271eb7639e0 100644
--- a/src/H5Eint.c
+++ b/src/H5Eint.c
@@ -78,12 +78,12 @@ static herr_t H5E__close_stack(H5E_stack_t *err_stack, void **request);
 /* Package Variables */
 /*********************/
 
-#ifndef H5_HAVE_THREADSAFE
+#ifndef H5_HAVE_THREADSAFE_API
 /*
  * The current error stack.
  */
 H5E_stack_t H5E_stack_g[1];
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
 /* Declare a free list to manage the H5E_stack_t struct */
 H5FL_DEFINE(H5E_stack_t);
@@ -112,14 +112,6 @@ hid_t H5E_ERR_CLS_g = FAIL;
 /* Local Variables */
 /*******************/
 
-#ifdef H5_HAVE_PARALLEL
-/*
- * variables used for MPI error reporting
- */
-char H5E_mpi_error_str[MPI_MAX_ERROR_STRING];
-int  H5E_mpi_error_str_len;
-#endif /* H5_HAVE_PARALLEL */
-
 /* Default value to initialize error stacks */
 static const H5E_stack_t H5E_err_stack_def = {
     0, /* nused */
@@ -265,9 +257,9 @@ H5E__init_package(void)
     if (H5I_register_type(H5I_ERRSTK_CLS) < 0)
         HGOTO_ERROR(H5E_ID, H5E_CANTINIT, FAIL, "unable to initialize ID group");
 
-#ifndef H5_HAVE_THREADSAFE
+#ifndef H5_HAVE_THREADSAFE_API
     H5E__set_default_auto(H5E_stack_g);
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
     /* Register the HDF5 error class */
     if ((H5E_ERR_CLS_g = H5I_register(H5I_ERROR_CLASS, &H5E_err_cls_s, false)) < 0)
@@ -363,6 +355,82 @@ H5E_term_package(void)
     FUNC_LEAVE_NOAPI(n)
 } /* end H5E_term_package() */
 
+/*-------------------------------------------------------------------------
+ * Function:    H5E_user_cb_prepare
+ *
+ * Purpose:     Prepare the H5E package before a user callback
+ *
+ * Return:      SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5E_user_cb_prepare(H5E_user_cb_state_t *state)
+{
+    H5E_stack_t *stack;               /* Pointer to the current error stack */
+    herr_t       ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Get a pointer to the current error stack */
+    if (NULL == (stack = H5E__get_my_stack()))
+        HGOTO_ERROR(H5E_ERROR, H5E_CANTGET, FAIL, "can't get current error stack");
+
+        /* Save state for current error stack */
+#ifndef H5_NO_DEPRECATED_SYMBOLS
+    assert(1 == stack->auto_op.vers || 2 == stack->auto_op.vers);
+
+    state->vers = stack->auto_op.vers;
+    if (1 == stack->auto_op.vers)
+        state->u.func1 = stack->auto_op.func1;
+    else
+        state->u.func2 = stack->auto_op.func2;
+#else  /* H5_NO_DEPRECATED_SYMBOLS */
+    state->func2 = stack->auto_op.func2;
+#endif /* H5_NO_DEPRECATED_SYMBOLS */
+    state->data = stack->auto_data;
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5E_user_cb_prepare() */
+
+/*-------------------------------------------------------------------------
+ * Function:    H5E_user_cb_restore
+ *
+ * Purpose:     Restores the state of the H5E package after a user callback
+ *
+ * Return:      SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5E_user_cb_restore(const H5E_user_cb_state_t *state)
+{
+    H5E_stack_t *stack;               /* Pointer to the current error stack */
+    herr_t       ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Get a pointer to the current error stack */
+    if (NULL == (stack = H5E__get_my_stack()))
+        HGOTO_ERROR(H5E_ERROR, H5E_CANTGET, FAIL, "can't get current error stack");
+
+        /* Restore state for current error stack */
+#ifndef H5_NO_DEPRECATED_SYMBOLS
+    stack->auto_op.vers = state->vers;
+    if (1 == state->vers)
+        stack->auto_op.func1 = state->u.func1;
+    else
+        stack->auto_op.func2 = state->u.func2;
+#else  /* H5_NO_DEPRECATED_SYMBOLS */
+    stack->auto_op.func2 = state->func2;
+#endif /* H5_NO_DEPRECATED_SYMBOLS */
+    stack->auto_data = state->data;
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5E_user_cb_restore() */
+
 /*-------------------------------------------------------------------------
  * Function:    H5E__free_class
  *
@@ -950,7 +1018,7 @@ H5E__walk1_cb(int n, H5E_error1_t *err_desc, void *client_data)
     const char      *maj_str   = "No major description"; /* Major error description */
     const char      *min_str   = "No minor description"; /* Minor error description */
     bool             have_desc = true; /* Flag to indicate whether the error has a "real" description */
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
     uint64_t thread_id = 0; /* ID of thread */
 #endif
     herr_t ret_value = SUCCEED;
@@ -982,7 +1050,7 @@ H5E__walk1_cb(int n, H5E_error1_t *err_desc, void *client_data)
     /* Get error class info */
     cls_ptr = maj_ptr->cls;
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
     if (H5TS_thread_id(&thread_id) < 0)
         HGOTO_DONE(FAIL);
 #endif
@@ -1014,13 +1082,13 @@ H5E__walk1_cb(int n, H5E_error1_t *err_desc, void *client_data)
                 MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
                 fprintf(stream, " MPI-process %d", mpi_rank);
             } /* end if */
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
             else
                 fprintf(stream, " thread %" PRIu64, thread_id);
 #endif
         } /* end block */
 #else
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
         fprintf(stream, " thread %" PRIu64, thread_id);
 #endif
 #endif
@@ -1081,7 +1149,7 @@ H5E__walk2_cb(unsigned n, const H5E_error2_t *err_desc, void *client_data)
     const char  *maj_str   = "No major description"; /* Major error description */
     const char  *min_str   = "No minor description"; /* Minor error description */
     bool         have_desc = true; /* Flag to indicate whether the error has a "real" description */
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
     uint64_t thread_id = 0; /* ID of thread */
 #endif
     herr_t ret_value = SUCCEED;
@@ -1118,7 +1186,7 @@ H5E__walk2_cb(unsigned n, const H5E_error2_t *err_desc, void *client_data)
     if (!cls_ptr)
         HGOTO_DONE(FAIL);
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
     if (H5TS_thread_id(&thread_id) < 0)
         HGOTO_DONE(FAIL);
 #endif
@@ -1150,13 +1218,13 @@ H5E__walk2_cb(unsigned n, const H5E_error2_t *err_desc, void *client_data)
                 MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
                 fprintf(stream, " MPI-process %d", mpi_rank);
             } /* end if */
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
             else
                 fprintf(stream, " thread %" PRIu64, thread_id);
 #endif
         } /* end block */
 #else
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
         fprintf(stream, " thread %" PRIu64, thread_id);
 #endif
 #endif
@@ -1293,7 +1361,12 @@ H5E__walk(const H5E_stack_t *estack, H5E_direction_t direction, const H5E_walk_o
                     old_err.line      = estack->entries[i].err.line;
                     old_err.desc      = estack->entries[i].err.desc;
 
-                    ret_value = (op->u.func1)(i, &old_err, client_data);
+                    /* Prepare & restore library for user callback */
+                    H5_BEFORE_USER_CB_NOERR(H5_ITER_ERROR)
+                        {
+                            ret_value = (op->u.func1)(i, &old_err, client_data);
+                        }
+                    H5_AFTER_USER_CB_NOERR(H5_ITER_ERROR)
                 } /* end for */
             }     /* end if */
             else {
@@ -1307,7 +1380,13 @@ H5E__walk(const H5E_stack_t *estack, H5E_direction_t direction, const H5E_walk_o
                     old_err.line      = estack->entries[i].err.line;
                     old_err.desc      = estack->entries[i].err.desc;
 
-                    ret_value = (op->u.func1)((int)(estack->nused - (size_t)(i + 1)), &old_err, client_data);
+                    /* Prepare & restore library for user callback */
+                    H5_BEFORE_USER_CB_NOERR(H5_ITER_ERROR)
+                        {
+                            ret_value =
+                                (op->u.func1)((int)(estack->nused - (size_t)(i + 1)), &old_err, client_data);
+                        }
+                    H5_AFTER_USER_CB_NOERR(H5_ITER_ERROR)
                 } /* end for */
             }     /* end else */
 
@@ -1323,14 +1402,26 @@ H5E__walk(const H5E_stack_t *estack, H5E_direction_t direction, const H5E_walk_o
         if (op->u.func2) {
             ret_value = SUCCEED;
             if (H5E_WALK_UPWARD == direction) {
-                for (i = 0; i < (int)estack->nused && ret_value == H5_ITER_CONT; i++)
-                    ret_value = (op->u.func2)((unsigned)i, &estack->entries[i].err, client_data);
+                for (i = 0; i < (int)estack->nused && ret_value == H5_ITER_CONT; i++) {
+                    /* Prepare & restore library for user callback */
+                    H5_BEFORE_USER_CB_NOERR(H5_ITER_ERROR)
+                        {
+                            ret_value = (op->u.func2)((unsigned)i, &estack->entries[i].err, client_data);
+                        }
+                    H5_AFTER_USER_CB_NOERR(H5_ITER_ERROR)
+                }
             } /* end if */
             else {
                 H5_CHECK_OVERFLOW(estack->nused - 1, size_t, int);
-                for (i = (int)(estack->nused - 1); i >= 0 && ret_value == H5_ITER_CONT; i--)
-                    ret_value = (op->u.func2)((unsigned)(estack->nused - (size_t)(i + 1)),
-                                              &estack->entries[i].err, client_data);
+                for (i = (int)(estack->nused - 1); i >= 0 && ret_value == H5_ITER_CONT; i--) {
+                    /* Prepare & restore library for user callback */
+                    H5_BEFORE_USER_CB_NOERR(H5_ITER_ERROR)
+                        {
+                            ret_value = (op->u.func2)((unsigned)(estack->nused - (size_t)(i + 1)),
+                                                      &estack->entries[i].err, client_data);
+                        }
+                    H5_AFTER_USER_CB_NOERR(H5_ITER_ERROR)
+                }
             } /* end else */
 
             if (ret_value < 0)
@@ -1369,6 +1460,40 @@ H5E__get_auto(const H5E_stack_t *estack, H5E_auto_op_t *op, void **client_data)
     FUNC_LEAVE_NOAPI(SUCCEED)
 } /* end H5E__get_auto() */
 
+/*-------------------------------------------------------------------------
+ * Function:    H5E_get_default_auto_func
+ *
+ * Purpose:     Private function to retrieve the default error stack's
+ *              reporting function.
+ *
+ * Return:      SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5E_get_default_auto_func(H5E_auto2_t *func)
+{
+    H5E_stack_t  *estack;              /* Error stack to operate on */
+    H5E_auto_op_t op;                  /* Error stack function */
+    herr_t        ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Retrieve default error stack */
+    if (NULL == (estack = H5E__get_my_stack()))
+        HGOTO_ERROR(H5E_ERROR, H5E_CANTGET, FAIL, "can't get current error stack");
+
+    /* Get the automatic error reporting information */
+    if (H5E__get_auto(estack, &op, NULL) < 0)
+        HGOTO_ERROR(H5E_ERROR, H5E_CANTGET, FAIL, "can't get automatic error info");
+
+    /* Retrieve error output function */
+    *func = op.func2;
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5E_get_default_auto_func() */
+
 /*-------------------------------------------------------------------------
  * Function:    H5E__set_auto
  *
@@ -1821,16 +1946,34 @@ H5E_dump_api_stack(void)
     assert(estack);
 
 #ifdef H5_NO_DEPRECATED_SYMBOLS
-    if (estack->auto_op.func2)
-        (void)((estack->auto_op.func2)(H5E_DEFAULT, estack->auto_data));
+    if (estack->auto_op.func2) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB_NOERR(H5_ITER_ERROR)
+            {
+                (void)((estack->auto_op.func2)(H5E_DEFAULT, estack->auto_data));
+            }
+        H5_AFTER_USER_CB_NOERR(H5_ITER_ERROR)
+    }
 #else  /* H5_NO_DEPRECATED_SYMBOLS */
     if (estack->auto_op.vers == 1) {
-        if (estack->auto_op.func1)
-            (void)((estack->auto_op.func1)(estack->auto_data));
+        if (estack->auto_op.func1) {
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB_NOERR(H5_ITER_ERROR)
+                {
+                    (void)((estack->auto_op.func1)(estack->auto_data));
+                }
+            H5_AFTER_USER_CB_NOERR(H5_ITER_ERROR)
+        }
     } /* end if */
     else {
-        if (estack->auto_op.func2)
-            (void)((estack->auto_op.func2)(H5E_DEFAULT, estack->auto_data));
+        if (estack->auto_op.func2) {
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB_NOERR(H5_ITER_ERROR)
+                {
+                    (void)((estack->auto_op.func2)(H5E_DEFAULT, estack->auto_data));
+                }
+            H5_AFTER_USER_CB_NOERR(H5_ITER_ERROR)
+        }
     } /* end else */
 #endif /* H5_NO_DEPRECATED_SYMBOLS */
 
diff --git a/src/H5Epkg.h b/src/H5Epkg.h
index 4eaad76fb6f..666f42e7d9f 100644
--- a/src/H5Epkg.h
+++ b/src/H5Epkg.h
@@ -37,7 +37,7 @@
 /* Number of entries in an error stack */
 #define H5E_MAX_ENTRIES 32
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 /*
  * The per-thread error stack.
  *
@@ -45,12 +45,12 @@
  * by "H5E_stack_t *estack =".
  */
 #define H5E__get_my_stack() H5TS_get_err_stack()
-#else /* H5_HAVE_THREADSAFE */
+#else /* H5_HAVE_THREADSAFE_API */
 /*
  * The current error stack.
  */
 #define H5E__get_my_stack() (H5E_stack_g + 0)
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
 /****************************/
 /* Package Private Typedefs */
@@ -118,7 +118,7 @@ typedef struct H5E_stack_t {
 /* Package Private Variables */
 /*****************************/
 
-#ifndef H5_HAVE_THREADSAFE
+#ifndef H5_HAVE_THREADSAFE_API
 /*
  * The current error stack.
  */
diff --git a/src/H5Eprivate.h b/src/H5Eprivate.h
index 1bb6d714eea..5b2b28c4173 100644
--- a/src/H5Eprivate.h
+++ b/src/H5Eprivate.h
@@ -16,11 +16,16 @@
 #ifndef H5Eprivate_H
 #define H5Eprivate_H
 
+/* Include package's public header */
 #include "H5Epublic.h"
 
 /* Private headers needed by this file */
 #include "H5private.h"
 
+/**************************/
+/* Library Private Macros */
+/**************************/
+
 /*
  * When one needs to temporarily disable recording errors while trying
  * something that's likely or expected to fail.  The code to try can be nested
@@ -181,29 +186,58 @@
 /*
  * MPI error handling macros.
  */
-
-extern char H5E_mpi_error_str[MPI_MAX_ERROR_STRING];
-extern int  H5E_mpi_error_str_len;
-
 #define HMPI_DONE_ERROR(retcode, str, mpierr)                                                                \
     {                                                                                                        \
+        char H5E_mpi_error_str[MPI_MAX_ERROR_STRING];                                                        \
+        int  H5E_mpi_error_str_len;                                                                          \
+                                                                                                             \
         MPI_Error_string(mpierr, H5E_mpi_error_str, &H5E_mpi_error_str_len);                                 \
         HDONE_ERROR(H5E_INTERNAL, H5E_MPI, retcode, "%s: MPI error string is '%s'", str, H5E_mpi_error_str); \
     }
 #define HMPI_GOTO_ERROR(retcode, str, mpierr)                                                                \
     {                                                                                                        \
+        char H5E_mpi_error_str[MPI_MAX_ERROR_STRING];                                                        \
+        int  H5E_mpi_error_str_len;                                                                          \
+                                                                                                             \
         MPI_Error_string(mpierr, H5E_mpi_error_str, &H5E_mpi_error_str_len);                                 \
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, retcode, "%s: MPI error string is '%s'", str, H5E_mpi_error_str); \
     }
 #endif /* H5_HAVE_PARALLEL */
 
-/* Library-private functions defined in H5E package */
+/****************************/
+/* Library Private Typedefs */
+/****************************/
+
+/* State to preserve across user callbacks */
+typedef struct H5E_user_cb_state_t {
+#ifndef H5_NO_DEPRECATED_SYMBOLS
+    unsigned vers; /* Which version callback to use */
+    union {
+        H5E_auto1_t func1;
+        H5E_auto2_t func2;
+    } u;
+#else           /* H5_NO_DEPRECATED_SYMBOLS */
+    H5E_auto2_t func2;
+#endif          /* H5_NO_DEPRECATED_SYMBOLS */
+    void *data; /* Callback data for 'automatic error reporting */
+} H5E_user_cb_state_t;
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+/******************************/
+/* Library Private Prototypes */
+/******************************/
 H5_DLL herr_t H5E_init(void);
+H5_DLL herr_t H5E_get_default_auto_func(H5E_auto2_t *func);
 H5_DLL herr_t H5E_printf_stack(const char *file, const char *func, unsigned line, hid_t maj_idx,
                                hid_t min_idx, const char *fmt, ...) H5_ATTR_FORMAT(printf, 6, 7);
 H5_DLL herr_t H5E_clear_stack(void);
 H5_DLL herr_t H5E_dump_api_stack(void);
 H5_DLL void   H5E_pause_stack(void);
 H5_DLL void   H5E_resume_stack(void);
+H5_DLL herr_t H5E_user_cb_prepare(H5E_user_cb_state_t *state);
+H5_DLL herr_t H5E_user_cb_restore(const H5E_user_cb_state_t *state);
 
 #endif /* H5Eprivate_H */
diff --git a/src/H5FD.c b/src/H5FD.c
index 567eb2c809f..cd971011b19 100644
--- a/src/H5FD.c
+++ b/src/H5FD.c
@@ -296,9 +296,17 @@ H5FD__free_cls(H5FD_class_t *cls, void H5_ATTR_UNUSED **request)
      * driver a chance to free singletons or other resources which will become
      * invalid once the class structure is freed.
      */
-    if (cls->terminate && cls->terminate() < 0)
-        HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEOBJ, FAIL, "virtual file driver '%s' did not terminate cleanly",
-                    cls->name);
+    if (cls->terminate) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = cls->terminate();
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEOBJ, FAIL, "virtual file driver '%s' did not terminate cleanly",
+                        cls->name);
+    }
 
     H5MM_xfree(cls);
 
@@ -390,13 +398,12 @@ H5FD_register(const void *_cls, size_t size, bool app_ref)
     assert(cls->get_eoa && cls->set_eoa);
     assert(cls->get_eof);
     assert(cls->read && cls->write);
-    for (type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; type++) {
+    for (type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; type++)
         assert(cls->fl_map[type] >= H5FD_MEM_NOLIST && cls->fl_map[type] < H5FD_MEM_NTYPES);
-    }
 
     /* Copy the class structure so the caller can reuse or free it */
     if (NULL == (saved = (H5FD_class_t *)H5MM_malloc(size)))
-        HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, H5I_INVALID_HID,
+        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, H5I_INVALID_HID,
                     "memory allocation failed for file driver class struct");
     H5MM_memcpy(saved, cls, size);
 
@@ -570,8 +577,14 @@ H5FD_sb_size(H5FD_t *file)
     assert(file->cls);
 
     /* Dispatch to driver */
-    if (file->cls->sb_size)
-        ret_value = (file->cls->sb_size)(file);
+    if (file->cls->sb_size) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB_NOERR(0)
+            {
+                ret_value = (file->cls->sb_size)(file);
+            }
+        H5_AFTER_USER_CB_NOERR(0)
+    }
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -603,8 +616,16 @@ H5FD_sb_encode(H5FD_t *file, char *name /*out*/, uint8_t *buf)
     assert(file->cls);
 
     /* Dispatch to driver */
-    if (file->cls->sb_encode && (file->cls->sb_encode)(file, name /*out*/, buf /*out*/) < 0)
-        HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_encode request failed");
+    if (file->cls->sb_encode) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->sb_encode)(file, name /*out*/, buf /*out*/);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_encode request failed");
+    }
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -631,8 +652,16 @@ H5FD__sb_decode(H5FD_t *file, const char *name, const uint8_t *buf)
     assert(file->cls);
 
     /* Dispatch to driver */
-    if (file->cls->sb_decode && (file->cls->sb_decode)(file, name, buf) < 0)
-        HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_decode request failed");
+    if (file->cls->sb_decode) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->sb_decode)(file, name, buf);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_decode request failed");
+    }
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -708,8 +737,14 @@ H5FD_fapl_get(H5FD_t *file)
     assert(file->cls);
 
     /* Dispatch to driver */
-    if (file->cls->fapl_get)
-        ret_value = (file->cls->fapl_get)(file);
+    if (file->cls->fapl_get) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB_NOERR(NULL)
+            {
+                ret_value = (file->cls->fapl_get)(file);
+            }
+        H5_AFTER_USER_CB_NOERR(NULL)
+    }
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -740,9 +775,15 @@ H5FD_free_driver_info(hid_t driver_id, const void *driver_info)
 
         /* Allow driver to free info or do it ourselves */
         if (driver->fapl_free) {
-            /* Free the const pointer */
-            /* Cast through uintptr_t to de-const memory */
-            if ((driver->fapl_free)((void *)(uintptr_t)driver_info) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    /* Free the const pointer */
+                    /* (Cast through uintptr_t to de-const memory) */
+                    ret_value = (driver->fapl_free)((void *)(uintptr_t)driver_info);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "driver free request failed");
         }
         else
@@ -895,20 +936,35 @@ H5FD_open(bool try, H5FD_t **_file, const char *name, unsigned flags, hid_t fapl
     if (HADDR_UNDEF == maxaddr)
         maxaddr = driver->maxaddr;
 
+    /* clang-format off */
+
     /* Try dispatching to file driver */
     if (try) {
         H5E_PAUSE_ERRORS
-        {
-            file = (driver->open)(name, flags, fapl_id, maxaddr);
-        }
+            {/* Prepare & restore library for user callback */
+                 H5_BEFORE_USER_CB(FAIL)
+                    {
+                        file = (driver->open)(name, flags, fapl_id, maxaddr);
+                    }
+                H5_AFTER_USER_CB(FAIL)
+            }
         H5E_RESUME_ERRORS
 
         /* Check if file was not opened */
         if (NULL == file)
             HGOTO_DONE(SUCCEED);
     }
-    else if (NULL == (file = (driver->open)(name, flags, fapl_id, maxaddr)))
-        HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "can't open file");
+    else
+    {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                file = (driver->open)(name, flags, fapl_id, maxaddr);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (NULL == file)
+            HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "can't open file");
+    }
 
     /* Set the file access flags */
     file->access_flags = flags;
@@ -931,10 +987,9 @@ H5FD_open(bool try, H5FD_t **_file, const char *name, unsigned flags, hid_t fapl
         HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to query file driver");
 
     /* Increment the global serial number & assign it to this H5FD_t object */
-    if (++H5FD_file_serial_no_g == 0) {
+    if (++H5FD_file_serial_no_g == 0)
         /* (Just error out if we wrap around for now...) */
         HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to get file serial number");
-    } /* end if */
     file->fileno = H5FD_file_serial_no_g;
 
     /* Start with base address set to 0 */
@@ -944,7 +999,9 @@ H5FD_open(bool try, H5FD_t **_file, const char *name, unsigned flags, hid_t fapl
     /* Set 'out' parameter */
     *_file = file;
 
-done:
+/* clang-format on */
+
+done :
     /* Can't cleanup 'file' information, since we don't know what type it is */
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD_open() */
@@ -1009,11 +1066,17 @@ H5FD_close(H5FD_t *file)
     if (H5I_dec_ref(file->driver_id) < 0)
         HGOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "can't close driver ID");
 
-    /* Dispatch to the driver for actual close. If the driver fails to
-     * close the file then the file will be in an unusable state.
-     */
-    assert(driver->close);
-    if ((driver->close)(file) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Dispatch to the driver for actual close. If the driver fails to
+             * close the file then the file will be in an unusable state.
+             */
+            assert(driver->close);
+            ret_value = (driver->close)(file);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "close failed");
 
 done:
@@ -1091,8 +1154,13 @@ H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2)
         HGOTO_DONE(0);
     }
 
-    /* Dispatch to driver */
-    ret_value = (f1->cls->cmp)(f1, f2);
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB_NOCHECK
+        {
+            /* Dispatch to driver */
+            ret_value = (f1->cls->cmp)(f1, f2);
+        }
+    H5_AFTER_USER_CB_NOCHECK
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -1153,7 +1221,13 @@ H5FD__query(const H5FD_t *file, unsigned long *flags /*out*/)
 
     /* Dispatch to driver (if available) */
     if (file->cls->query) {
-        if ((file->cls->query)(file, flags) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->query)(file, flags);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "unable to query feature flags");
     }
     else
@@ -1505,8 +1579,14 @@ H5FD_get_fs_type_map(const H5FD_t *file, H5FD_mem_t *type_map)
 
     /* Check for VFD class providing a type map retrieval routine */
     if (file->cls->get_type_map) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->get_type_map)(file, type_map);
+            }
+        H5_AFTER_USER_CB(FAIL)
         /* Retrieve type mapping for this file */
-        if ((file->cls->get_type_map)(file, type_map) < 0)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "driver get type map failed");
     } /* end if */
     else
@@ -2400,8 +2480,16 @@ H5FD_flush(H5FD_t *file, bool closing)
     assert(file->cls);
 
     /* Dispatch to driver */
-    if (file->cls->flush && (file->cls->flush)(file, H5CX_get_dxpl(), closing) < 0)
-        HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver flush request failed");
+    if (file->cls->flush) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->flush)(file, H5CX_get_dxpl(), closing);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver flush request failed");
+    }
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -2465,8 +2553,16 @@ H5FD_truncate(H5FD_t *file, bool closing)
     assert(file->cls);
 
     /* Dispatch to driver */
-    if (file->cls->truncate && (file->cls->truncate)(file, H5CX_get_dxpl(), closing) < 0)
-        HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "driver truncate request failed");
+    if (file->cls->truncate) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->truncate)(file, H5CX_get_dxpl(), closing);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "driver truncate request failed");
+    }
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -2523,8 +2619,16 @@ H5FD_lock(H5FD_t *file, bool rw)
     assert(file->cls);
 
     /* Dispatch to driver */
-    if (file->cls->lock && (file->cls->lock)(file, rw) < 0)
-        HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "driver lock request failed");
+    if (file->cls->lock) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->lock)(file, rw);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "driver lock request failed");
+    }
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -2581,8 +2685,16 @@ H5FD_unlock(H5FD_t *file)
     assert(file->cls);
 
     /* Dispatch to driver */
-    if (file->cls->unlock && (file->cls->unlock)(file) < 0)
-        HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "driver unlock request failed");
+    if (file->cls->unlock) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->unlock)(file);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "driver unlock request failed");
+    }
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -2671,16 +2783,18 @@ H5FD_ctl(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void
      * Otherwise, report success.
      */
     if (file->cls->ctl) {
-
-        if ((file->cls->ctl)(file, op_code, flags, input, output) < 0)
-
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->ctl)(file, op_code, flags, input, output);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "VFD ctl request failed");
     }
-    else if (flags & H5FD_CTL_FAIL_IF_UNKNOWN_FLAG) {
-
+    else if (flags & H5FD_CTL_FAIL_IF_UNKNOWN_FLAG)
         HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL,
                     "VFD ctl request failed (no ctl callback and fail if unknown flag is set)");
-    }
 
 done:
 
@@ -2778,7 +2892,14 @@ H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl_id, void **file_handle)
     /* Dispatch to driver */
     if (NULL == file->cls->get_handle)
         HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "file driver has no `get_vfd_handle' method");
-    if ((file->cls->get_handle)(file, fapl_id, file_handle) < 0)
+
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            ret_value = (file->cls->get_handle)(file, fapl_id, file_handle);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get file handle for file driver");
 
 done:
diff --git a/src/H5FDcore.c b/src/H5FDcore.c
index 408b7c57f59..f2c2a820e54 100644
--- a/src/H5FDcore.c
+++ b/src/H5FDcore.c
@@ -508,7 +508,7 @@ H5Pset_core_write_tracking(hid_t plist_id, hbool_t is_enabled, size_t page_size)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "page_size cannot be zero");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADID, FAIL, "can't find object for ID");
     if (H5FD_CORE != H5P_peek_driver(plist))
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver");
@@ -550,7 +550,7 @@ H5Pget_core_write_tracking(hid_t plist_id, hbool_t *is_enabled /*out*/, size_t *
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADID, FAIL, "can't find object for ID");
     if (H5FD_CORE != H5P_peek_driver(plist))
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver");
@@ -588,7 +588,7 @@ H5Pset_fapl_core(hid_t fapl_id, size_t increment, hbool_t backing_store)
     FUNC_ENTER_API(FAIL)
 
     /* Check argument */
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
 
     /* Set VFD info values */
@@ -624,7 +624,7 @@ H5Pget_fapl_core(hid_t fapl_id, size_t *increment /*out*/, hbool_t *backing_stor
 
     FUNC_ENTER_API(FAIL)
 
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
     if (H5FD_CORE != H5P_peek_driver(plist))
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver");
@@ -824,13 +824,19 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
         if (size) {
             /* Allocate memory for the file's data, using the file image callback if available. */
             if (file->fi_callbacks.image_malloc) {
-                if (NULL == (file->mem = (unsigned char *)file->fi_callbacks.image_malloc(
-                                 size, H5FD_FILE_IMAGE_OP_FILE_OPEN, file->fi_callbacks.udata)))
-                    HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "image malloc callback failed");
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(NULL)
+                    {
+                        file->mem = file->fi_callbacks.image_malloc(size, H5FD_FILE_IMAGE_OP_FILE_OPEN,
+                                                                    file->fi_callbacks.udata);
+                    }
+                H5_AFTER_USER_CB(NULL)
+                if (NULL == file->mem)
+                    HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "image malloc callback failed");
             } /* end if */
             else {
-                if (NULL == (file->mem = (unsigned char *)H5MM_malloc(size)))
-                    HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "unable to allocate memory block");
+                if (NULL == (file->mem = H5MM_malloc(size)))
+                    HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate memory block");
             } /* end else */
 
             /* Set up data structures */
@@ -839,10 +845,18 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr
             /* If there is an initial file image, copy it, using the callback if possible */
             if (file_image_info.buffer && file_image_info.size > 0) {
                 if (file->fi_callbacks.image_memcpy) {
-                    if (file->mem != file->fi_callbacks.image_memcpy(file->mem, file_image_info.buffer, size,
-                                                                     H5FD_FILE_IMAGE_OP_FILE_OPEN,
-                                                                     file->fi_callbacks.udata))
-                        HGOTO_ERROR(H5E_FILE, H5E_CANTCOPY, NULL, "image_memcpy callback failed");
+                    void *tmp;
+
+                    /* Prepare & restore library for user callback */
+                    H5_BEFORE_USER_CB(NULL)
+                        {
+                            tmp = file->fi_callbacks.image_memcpy(file->mem, file_image_info.buffer, size,
+                                                                  H5FD_FILE_IMAGE_OP_FILE_OPEN,
+                                                                  file->fi_callbacks.udata);
+                        }
+                    H5_AFTER_USER_CB(NULL)
+                    if (file->mem != tmp)
+                        HGOTO_ERROR(H5E_VFL, H5E_CANTCOPY, NULL, "image_memcpy callback failed");
                 } /* end if */
                 else
                     H5MM_memcpy(file->mem, file_image_info.buffer, size);
@@ -977,9 +991,15 @@ H5FD__core_close(H5FD_t *_file)
     if (file->mem) {
         /* Use image callback if available */
         if (file->fi_callbacks.image_free) {
-            if (file->fi_callbacks.image_free(file->mem, H5FD_FILE_IMAGE_OP_FILE_CLOSE,
-                                              file->fi_callbacks.udata) < 0)
-                HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "image_free callback failed");
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    ret_value = file->fi_callbacks.image_free(file->mem, H5FD_FILE_IMAGE_OP_FILE_CLOSE,
+                                                              file->fi_callbacks.udata);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
+                HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "image_free callback failed");
         } /* end if */
         else
             H5MM_xfree(file->mem);
@@ -1346,16 +1366,22 @@ H5FD__core_write(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UN
 
         /* (Re)allocate memory for the file buffer, using callbacks if available */
         if (file->fi_callbacks.image_realloc) {
-            if (NULL == (x = (unsigned char *)file->fi_callbacks.image_realloc(
-                             file->mem, new_eof, H5FD_FILE_IMAGE_OP_FILE_RESIZE, file->fi_callbacks.udata)))
-                HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL,
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    x = file->fi_callbacks.image_realloc(file->mem, new_eof, H5FD_FILE_IMAGE_OP_FILE_RESIZE,
+                                                         file->fi_callbacks.udata);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (NULL == x)
+                HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL,
                             "unable to allocate memory block of %llu bytes with callback",
                             (unsigned long long)new_eof);
         } /* end if */
         else {
-            if (NULL == (x = (unsigned char *)H5MM_realloc(file->mem, new_eof)))
-                HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL,
-                            "unable to allocate memory block of %llu bytes", (unsigned long long)new_eof);
+            if (NULL == (x = H5MM_realloc(file->mem, new_eof)))
+                HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate memory block of %llu bytes",
+                            (unsigned long long)new_eof);
         } /* end else */
 
         memset(x + file->eof, 0, (size_t)(new_eof - file->eof));
@@ -1504,15 +1530,20 @@ H5FD__core_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, bool closing)
 
             /* (Re)allocate memory for the file buffer, using callback if available */
             if (file->fi_callbacks.image_realloc) {
-                if (NULL ==
-                    (x = (unsigned char *)file->fi_callbacks.image_realloc(
-                         file->mem, new_eof, H5FD_FILE_IMAGE_OP_FILE_RESIZE, file->fi_callbacks.udata)))
-                    HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL,
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        x = file->fi_callbacks.image_realloc(
+                            file->mem, new_eof, H5FD_FILE_IMAGE_OP_FILE_RESIZE, file->fi_callbacks.udata);
+                    }
+                H5_AFTER_USER_CB(FAIL)
+                if (NULL == x)
+                    HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL,
                                 "unable to allocate memory block with callback");
             } /* end if */
             else {
-                if (NULL == (x = (unsigned char *)H5MM_realloc(file->mem, new_eof)))
-                    HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate memory block");
+                if (NULL == (x = H5MM_realloc(file->mem, new_eof)))
+                    HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate memory block");
             } /* end else */
 
             if (file->eof < new_eof)
diff --git a/src/H5FDdevelop.h b/src/H5FDdevelop.h
index f6915ef3c73..0933b827767 100644
--- a/src/H5FDdevelop.h
+++ b/src/H5FDdevelop.h
@@ -92,41 +92,49 @@
 /**
  * Extensible array header block; it is mapped to 'ohdr' type file memory to
  * benefit from their similarity.
+ * \since 1.10.0
  */
 #define H5FD_MEM_EARRAY_HDR H5FD_MEM_OHDR
 /**
  * Extensible array index block; it is mapped to 'ohdr' type file memory because
  * these index blocks are similar to extensible array header blocks.
+ * \since 1.10.0
  */
 #define H5FD_MEM_EARRAY_IBLOCK H5FD_MEM_OHDR
 /**
  * Extensible array super block; it is mappend to 'btree' type file memory
  * because the indices are similar enough to B-tree nodes.
+ * \since 1.10.0
  */
 #define H5FD_MEM_EARRAY_SBLOCK H5FD_MEM_BTREE
 /**
  * Extensible array data block; it is mapped to 'lheap' type file memory
  * because it is similar enough to local heap info.
+ * \since 1.10.0
  */
 #define H5FD_MEM_EARRAY_DBLOCK H5FD_MEM_LHEAP
 /**
  * Extensible array data block & page; it is mapped to 'lheap' type file memory
  * because it is similar enough to local heap info.
+ * \since 1.10.0
  */
 #define H5FD_MEM_EARRAY_DBLK_PAGE H5FD_MEM_LHEAP
 /**
  * Fixed array header block; it is mapped to 'ohdr' type file memory to
  * benefit their similarity.
+ * \since 1.10.0
  */
 #define H5FD_MEM_FARRAY_HDR H5FD_MEM_OHDR
 /**
  * Fixed array data block; it is mapped to 'lheap' type file memory
  * because it is similar enough to local heap info.
+ * \since 1.10.0
  */
 #define H5FD_MEM_FARRAY_DBLOCK H5FD_MEM_LHEAP
 /**
  * Fixed array data block & page; it is mapped to 'lheap' type file memory
  * because it is similar enough to local heap info.
+ * \since 1.10.0
  */
 #define H5FD_MEM_FARRAY_DBLK_PAGE H5FD_MEM_LHEAP
 
diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c
index ba287b9d4a4..0dfb2e7cbf0 100644
--- a/src/H5FDdirect.c
+++ b/src/H5FDdirect.c
@@ -222,7 +222,7 @@ H5Pset_fapl_direct(hid_t fapl_id, size_t boundary, size_t block_size, size_t cbu
 
     FUNC_ENTER_API(FAIL)
 
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
 
     if (H5FD__direct_populate_config(boundary, block_size, cbuf_size, &fa) < 0)
@@ -256,7 +256,7 @@ H5Pget_fapl_direct(hid_t fapl_id, size_t *boundary /*out*/, size_t *block_size /
 
     FUNC_ENTER_API(FAIL)
 
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list");
     if (H5FD_DIRECT != H5P_peek_driver(plist))
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver");
@@ -443,7 +443,7 @@ H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad
         HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct");
 
     /* Get the driver specific information */
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list");
     if (NULL == (fa = (const H5FD_direct_fapl_t *)H5P_peek_driver_info(plist))) {
         if (H5FD__direct_populate_config(0, 0, 0, &default_fa) < 0)
diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c
index e186c06017e..939a3ac5c4a 100644
--- a/src/H5FDfamily.c
+++ b/src/H5FDfamily.c
@@ -378,7 +378,7 @@ H5Pget_fapl_family(hid_t fapl_id, hsize_t *msize /*out*/, hid_t *memb_fapl_id /*
 
     FUNC_ENTER_API(FAIL)
 
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list");
     if (H5FD_FAMILY != H5P_peek_driver(plist))
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver");
@@ -1118,7 +1118,7 @@ H5FD__family_get_handle(H5FD_t *_file, hid_t fapl, void **file_handle)
     FUNC_ENTER_PACKAGE
 
     /* Get the plist structure and family offset */
-    if (NULL == (plist = H5P_object_verify(fapl, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_VFL, H5E_BADID, FAIL, "can't find object for ID");
     if (H5P_get(plist, H5F_ACS_FAMILY_OFFSET_NAME, &offset) < 0)
         HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get offset for family driver");
@@ -1488,9 +1488,9 @@ H5FD__family_delete(const char *filename, hid_t fapl_id)
             herr_t delete_error;
 
             H5E_PAUSE_ERRORS
-            {
-                delete_error = H5FD_delete(member_name, memb_fapl_id);
-            }
+                {
+                    delete_error = H5FD_delete(member_name, memb_fapl_id);
+                }
             H5E_RESUME_ERRORS
             if (delete_error < 0)
                 break;
diff --git a/src/H5FDhdfs.c b/src/H5FDhdfs.c
index 615067d1e22..f1e3837a2d3 100644
--- a/src/H5FDhdfs.c
+++ b/src/H5FDhdfs.c
@@ -582,7 +582,7 @@ H5Pset_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa)
     fprintf(stdout, "called %s.\n", __func__);
 #endif
 
-    plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS);
+    plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false);
     if (plist == NULL)
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
     if (FAIL == H5FD__hdfs_validate_config(fa))
@@ -621,7 +621,7 @@ H5Pget_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa_dst /*out*/)
 
     if (fa_dst == NULL)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_dst ptr is NULL");
-    plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS);
+    plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true);
     if (plist == NULL)
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list");
 
diff --git a/src/H5FDhdfs.h b/src/H5FDhdfs.h
index cc667ac1d81..bf18e4fa59a 100644
--- a/src/H5FDhdfs.h
+++ b/src/H5FDhdfs.h
@@ -31,7 +31,7 @@
 
 #else
 
-/** Initializer for the hdfs VFD (disabled) */
+/** Initializer for the hdfs VFD (disabled) \since 1.8.22 */
 #define H5FD_HDFS       (H5I_INVALID_HID)
 
 /** Identifier for the hdfs VFD (disabled) */
@@ -47,11 +47,11 @@
  */
 #define H5FD__CURR_HDFS_FAPL_T_VERSION 1
 
-/** Max size of the node name */
+/** Max size of the node name \since 1.8.22 1.10.6 */
 #define H5FD__HDFS_NODE_NAME_SPACE 128
-/** Max size of the user name */
+/** Max size of the user name \since 1.8.22 1.10.6 */
 #define H5FD__HDFS_USER_NAME_SPACE 128
-/** Max size of the kerberos cache path */
+/** Max size of the kerberos cache path \since 1.8.22 1.10.6 */
 #define H5FD__HDFS_KERB_CACHE_PATH_SPACE 128
 
 /**
diff --git a/src/H5FDint.c b/src/H5FDint.c
index 07761742c13..bcb79e3fee6 100644
--- a/src/H5FDint.c
+++ b/src/H5FDint.c
@@ -245,7 +245,13 @@ H5FD_read(H5FD_t *file, H5FD_mem_t type, haddr_t addr, size_t size, void *buf /*
     if (!(file->access_flags & H5F_ACC_SWMR_READ)) {
         haddr_t eoa;
 
-        if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type)))
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                eoa = (file->cls->get_eoa)(file, type);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (!H5_addr_defined(eoa))
             HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed");
 
         if ((addr + file->base_addr + size) > eoa)
@@ -254,8 +260,14 @@ H5FD_read(H5FD_t *file, H5FD_mem_t type, haddr_t addr, size_t size, void *buf /*
                         (unsigned long long)eoa);
     }
 
-    /* Dispatch to driver */
-    if ((file->cls->read)(file, type, dxpl_id, addr + file->base_addr, size, buf) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Dispatch to driver */
+            ret_value = (file->cls->read)(file, type, dxpl_id, addr + file->base_addr, size, buf);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver read request failed");
 
     /* Set actual selection I/O, if this is a raw data operation */
@@ -306,15 +318,27 @@ H5FD_write(H5FD_t *file, H5FD_mem_t type, haddr_t addr, size_t size, const void
         HGOTO_DONE(SUCCEED);
 #endif /* H5_HAVE_PARALLEL */
 
-    if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type)))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            eoa = (file->cls->get_eoa)(file, type);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (!H5_addr_defined(eoa))
         HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed");
     if ((addr + file->base_addr + size) > eoa)
         HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addr = %llu, size=%llu, eoa=%llu",
                     (unsigned long long)(addr + file->base_addr), (unsigned long long)size,
                     (unsigned long long)eoa);
 
-    /* Dispatch to driver */
-    if ((file->cls->write)(file, type, dxpl_id, addr + file->base_addr, size, buf) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Dispatch to driver */
+            ret_value = (file->cls->write)(file, type, dxpl_id, addr + file->base_addr, size, buf);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "driver write request failed");
 
     /* Set actual selection I/O, if this is a raw data operation */
@@ -404,20 +428,16 @@ H5FD_read_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addrs
      * Do not return early for Parallel mode since the I/O could be a
      * collective transfer.
      */
-    if (0 == count) {
+    if (0 == count)
         HGOTO_DONE(SUCCEED);
-    }
 #endif /* H5_HAVE_PARALLEL */
 
     if (file->base_addr > 0) {
-
         /* apply the base_addr offset to the addrs array.  Must undo before
          * we return.
          */
-        for (i = 0; i < count; i++) {
-
+        for (i = 0; i < count; i++)
             addrs[i] += file->base_addr;
-        }
         addrs_cooked = true;
     }
 
@@ -434,29 +454,21 @@ H5FD_read_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addrs
         extend_types = false;
 
         for (i = 0; i < count; i++) {
-
             if (!extend_sizes) {
-
                 if (sizes[i] == 0) {
-
                     extend_sizes = true;
                     size         = sizes[i - 1];
                 }
-                else {
-
+                else
                     size = sizes[i];
-                }
             }
 
             if (!extend_types) {
-
                 if (types[i] == H5FD_MEM_NOLIST) {
-
                     extend_types = true;
                     type         = types[i - 1];
                 }
                 else {
-
                     type = types[i];
 
                     /* Check for raw data operation */
@@ -465,11 +477,16 @@ H5FD_read_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addrs
                 }
             }
 
-            if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type)))
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    eoa = (file->cls->get_eoa)(file, type);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (!H5_addr_defined(eoa))
                 HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed");
 
             if ((addrs[i] + size) > eoa)
-
                 HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL,
                             "addr overflow, addrs[%d] = %llu, sizes[%d] = %llu, eoa = %llu", (int)i,
                             (unsigned long long)(addrs[i]), (int)i, (unsigned long long)size,
@@ -486,7 +503,13 @@ H5FD_read_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addrs
 
     /* if the underlying VFD supports vector read, make the call */
     if (file->cls->read_vector) {
-        if ((file->cls->read_vector)(file, dxpl_id, count, types, addrs, sizes, bufs) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->read_vector)(file, dxpl_id, count, types, addrs, sizes, bufs);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver read vector request failed");
 
         /* Set actual selection I/O mode, if this is a raw data operation */
@@ -499,7 +522,6 @@ H5FD_read_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addrs
         }
     }
     else {
-
         /* otherwise, implement the vector read as a sequence of regular
          * read calls.
          */
@@ -509,38 +531,34 @@ H5FD_read_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addrs
         uint32_t actual_selection_io_mode;
 
         for (i = 0; i < count; i++) {
-
             /* we have already verified that sizes[0] != 0 and
              * types[0] != H5FD_MEM_NOLIST
              */
-
             if (!extend_sizes) {
-
                 if (sizes[i] == 0) {
-
                     extend_sizes = true;
                     size         = sizes[i - 1];
                 }
-                else {
-
+                else
                     size = sizes[i];
-                }
             }
 
             if (!extend_types) {
-
                 if (types[i] == H5FD_MEM_NOLIST) {
-
                     extend_types = true;
                     type         = types[i - 1];
                 }
-                else {
-
+                else
                     type = types[i];
-                }
             }
 
-            if ((file->cls->read)(file, type, dxpl_id, addrs[i], size, bufs[i]) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    ret_value = (file->cls->read)(file, type, dxpl_id, addrs[i], size, bufs[i]);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver read request failed");
         }
 
@@ -560,14 +578,11 @@ H5FD_read_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addrs
 done:
     /* undo the base addr offset to the addrs array if necessary */
     if (addrs_cooked) {
-
         assert(file->base_addr > 0);
-
-        for (i = 0; i < count; i++) {
-
+        for (i = 0; i < count; i++)
             addrs[i] -= file->base_addr;
-        }
     }
+
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD_read_vector() */
 
@@ -651,14 +666,11 @@ H5FD_write_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addr
 #endif /* H5_HAVE_PARALLEL */
 
     if (file->base_addr > 0) {
-
         /* apply the base_addr offset to the addrs array.  Must undo before
          * we return.
          */
-        for (i = 0; i < count; i++) {
-
+        for (i = 0; i < count; i++)
             addrs[i] += file->base_addr;
-        }
         addrs_cooked = true;
     }
 
@@ -666,29 +678,21 @@ H5FD_write_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addr
     extend_types = false;
 
     for (i = 0; i < count; i++) {
-
         if (!extend_sizes) {
-
             if (sizes[i] == 0) {
-
                 extend_sizes = true;
                 size         = sizes[i - 1];
             }
-            else {
-
+            else
                 size = sizes[i];
-            }
         }
 
         if (!extend_types) {
-
             if (types[i] == H5FD_MEM_NOLIST) {
-
                 extend_types = true;
                 type         = types[i - 1];
             }
             else {
-
                 type = types[i];
 
                 /* Check for raw data operation */
@@ -697,21 +701,31 @@ H5FD_write_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addr
             }
         }
 
-        if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type)))
-
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                eoa = (file->cls->get_eoa)(file, type);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (!H5_addr_defined(eoa))
             HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed");
 
         if ((addrs[i] + size) > eoa)
-
-            HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addrs[%d] = %llu, sizes[%d] = %llu, \
-                        eoa = %llu",
-                        (int)i, (unsigned long long)(addrs[i]), (int)i, (unsigned long long)size,
+            HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL,
+                        "addr overflow, addrs[%d] = %llu, sizes[%d] = %llu, eoa = %llu", (int)i,
+                        (unsigned long long)(addrs[i]), (int)i, (unsigned long long)size,
                         (unsigned long long)eoa);
     }
 
     /* if the underlying VFD supports vector write, make the call */
     if (file->cls->write_vector) {
-        if ((file->cls->write_vector)(file, dxpl_id, count, types, addrs, sizes, bufs) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->write_vector)(file, dxpl_id, count, types, addrs, sizes, bufs);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "driver write vector request failed");
 
         /* Set actual selection I/O mode, if this is a raw data operation */
@@ -733,38 +747,34 @@ H5FD_write_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addr
         uint32_t actual_selection_io_mode;
 
         for (i = 0; i < count; i++) {
-
             /* we have already verified that sizes[0] != 0 and
              * types[0] != H5FD_MEM_NOLIST
              */
-
             if (!extend_sizes) {
-
                 if (sizes[i] == 0) {
-
                     extend_sizes = true;
                     size         = sizes[i - 1];
                 }
-                else {
-
+                else
                     size = sizes[i];
-                }
             }
 
             if (!extend_types) {
-
                 if (types[i] == H5FD_MEM_NOLIST) {
-
                     extend_types = true;
                     type         = types[i - 1];
                 }
-                else {
-
+                else
                     type = types[i];
-                }
             }
 
-            if ((file->cls->write)(file, type, dxpl_id, addrs[i], size, bufs[i]) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    ret_value = (file->cls->write)(file, type, dxpl_id, addrs[i], size, bufs[i]);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver write request failed");
         }
 
@@ -784,14 +794,11 @@ H5FD_write_vector(H5FD_t *file, uint32_t count, H5FD_mem_t types[], haddr_t addr
 done:
     /* undo the base addr offset to the addrs array if necessary */
     if (addrs_cooked) {
-
         assert(file->base_addr > 0);
-
-        for (i = 0; i < count; i++) {
-
+        for (i = 0; i < count; i++)
             addrs[i] -= file->base_addr;
-        }
     }
+
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD_write_vector() */
 
@@ -1020,11 +1027,18 @@ H5FD__read_selection_translate(uint32_t skip_vector_cb, H5FD_t *file, H5FD_mem_t
                 vec_bufs[vec_arr_nused] = (void *)((uint8_t *)buf + mem_off[mem_seq_i]);
                 vec_arr_nused++;
             }
-            else
-                /* Issue scalar read call */
-                if ((file->cls->read)(file, type, dxpl_id, offsets[i] + file_off[file_seq_i], io_len,
-                                      (void *)((uint8_t *)buf + mem_off[mem_seq_i])) < 0)
+            else {
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        /* Issue scalar read call */
+                        ret_value = (file->cls->read)(file, type, dxpl_id, offsets[i] + file_off[file_seq_i],
+                                                      io_len, (void *)((uint8_t *)buf + mem_off[mem_seq_i]));
+                    }
+                H5_AFTER_USER_CB(FAIL)
+                if (ret_value < 0)
                     HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver read request failed");
+            }
 
             /* Update file sequence */
             if (io_len == file_len[file_seq_i])
@@ -1062,8 +1076,14 @@ H5FD__read_selection_translate(uint32_t skip_vector_cb, H5FD_t *file, H5FD_mem_t
         uint32_t actual_selection_io_mode;
 
         H5_CHECK_OVERFLOW(vec_arr_nused, size_t, uint32_t);
-        if ((file->cls->read_vector)(file, dxpl_id, (uint32_t)vec_arr_nused, types, addrs, sizes, vec_bufs) <
-            0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->read_vector)(file, dxpl_id, (uint32_t)vec_arr_nused, types, addrs,
+                                                     sizes, vec_bufs);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver read vector request failed");
 
         /* Set actual selection I/O, if this is a raw data operation */
@@ -1197,20 +1217,16 @@ H5FD_read_selection(H5FD_t *file, H5FD_mem_t type, uint32_t count, H5S_t **mem_s
      * Do not return early for Parallel mode since the I/O could be a
      * collective transfer.
      */
-    if (0 == count) {
+    if (0 == count)
         HGOTO_DONE(SUCCEED);
-    }
 #endif /* H5_HAVE_PARALLEL */
 
     if (file->base_addr > 0) {
-
         /* apply the base_addr offset to the offsets array.  Must undo before
          * we return.
          */
-        for (i = 0; i < count; i++) {
-
+        for (i = 0; i < count; i++)
             offsets[i] += file->base_addr;
-        }
         offsets_cooked = true;
     }
 
@@ -1227,16 +1243,19 @@ H5FD_read_selection(H5FD_t *file, H5FD_mem_t type, uint32_t count, H5S_t **mem_s
     if (!(file->access_flags & H5F_ACC_SWMR_READ)) {
         haddr_t eoa;
 
-        if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type)))
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                eoa = (file->cls->get_eoa)(file, type);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (!H5_addr_defined(eoa))
             HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed");
 
-        for (i = 0; i < count; i++) {
-
+        for (i = 0; i < count; i++)
             if ((offsets[i]) > eoa)
-
                 HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, offsets[%d] = %llu, eoa = %llu",
                             (int)i, (unsigned long long)(offsets[i]), (unsigned long long)eoa);
-        }
     }
 
     /* if the underlying VFD supports selection read, make the call */
@@ -1265,8 +1284,14 @@ H5FD_read_selection(H5FD_t *file, H5FD_mem_t type, uint32_t count, H5S_t **mem_s
             }
         }
 
-        if ((file->cls->read_selection)(file, type, dxpl_id, count, mem_space_ids, file_space_ids, offsets,
-                                        element_sizes, bufs) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->read_selection)(file, type, dxpl_id, count, mem_space_ids,
+                                                        file_space_ids, offsets, element_sizes, bufs);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver read selection request failed");
 
         /* Set actual selection I/O, if this is a raw data operation */
@@ -1287,13 +1312,9 @@ H5FD_read_selection(H5FD_t *file, H5FD_mem_t type, uint32_t count, H5S_t **mem_s
 done:
     /* undo the base addr offset to the offsets array if necessary */
     if (offsets_cooked) {
-
         assert(file->base_addr > 0);
-
-        for (i = 0; i < count; i++) {
-
+        for (i = 0; i < count; i++)
             offsets[i] -= file->base_addr;
-        }
     }
 
     /* Cleanup dataspace arrays.  Use H5I_remove() so we only close the IDs and
@@ -1379,23 +1400,19 @@ H5FD_read_selection_id(uint32_t skip_cb, H5FD_t *file, H5FD_mem_t type, uint32_t
      * Do not return early for Parallel mode since the I/O could be a
      * collective transfer.
      */
-    if (0 == count) {
+    if (0 == count)
         HGOTO_DONE(SUCCEED);
-    }
 #endif /* H5_HAVE_PARALLEL */
 
     skip_selection_cb = skip_cb & SKIP_SELECTION_CB;
     skip_vector_cb    = skip_cb & SKIP_VECTOR_CB;
 
     if (file->base_addr > 0) {
-
         /* apply the base_addr offset to the offsets array.  Must undo before
          * we return.
          */
-        for (i = 0; i < count; i++) {
-
+        for (i = 0; i < count; i++)
             offsets[i] += file->base_addr;
-        }
         offsets_cooked = true;
     }
 
@@ -1412,24 +1429,33 @@ H5FD_read_selection_id(uint32_t skip_cb, H5FD_t *file, H5FD_mem_t type, uint32_t
     if (!(file->access_flags & H5F_ACC_SWMR_READ)) {
         haddr_t eoa;
 
-        if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type)))
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                eoa = (file->cls->get_eoa)(file, type);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (!H5_addr_defined(eoa))
             HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed");
 
-        for (i = 0; i < count; i++) {
-
+        for (i = 0; i < count; i++)
             if ((offsets[i]) > eoa)
-
                 HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, offsets[%d] = %llu, eoa = %llu",
                             (int)i, (unsigned long long)(offsets[i]), (unsigned long long)eoa);
-        }
     }
 
     /* if the underlying VFD supports selection read, make the call */
     if (!skip_selection_cb && file->cls->read_selection) {
         uint32_t actual_selection_io_mode;
 
-        if ((file->cls->read_selection)(file, type, dxpl_id, count, mem_space_ids, file_space_ids, offsets,
-                                        element_sizes, bufs) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->read_selection)(file, type, dxpl_id, count, mem_space_ids,
+                                                        file_space_ids, offsets, element_sizes, bufs);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver read selection request failed");
 
         /* Set actual selection I/O, if this is a raw data operation */
@@ -1471,13 +1497,9 @@ H5FD_read_selection_id(uint32_t skip_cb, H5FD_t *file, H5FD_mem_t type, uint32_t
 done:
     /* undo the base addr offset to the offsets array if necessary */
     if (offsets_cooked) {
-
         assert(file->base_addr > 0);
-
-        for (i = 0; i < count; i++) {
-
+        for (i = 0; i < count; i++)
             offsets[i] -= file->base_addr;
-        }
     }
 
     /* Cleanup dataspace arrays */
@@ -1571,34 +1593,24 @@ H5FD__write_selection_translate(uint32_t skip_vector_cb, H5FD_t *file, H5FD_mem_
 
     /* Loop over dataspaces */
     for (i = 0; i < count; i++) {
-
         /* we have already verified that element_sizes[0] != 0 and bufs[0]
          * != NULL */
-
         if (!extend_sizes) {
-
             if (element_sizes[i] == 0) {
-
                 extend_sizes = true;
                 element_size = element_sizes[i - 1];
             }
-            else {
-
+            else
                 element_size = element_sizes[i];
-            }
         }
 
         if (!extend_bufs) {
-
             if (bufs[i] == NULL) {
-
                 extend_bufs = true;
                 buf         = bufs[i - 1];
             }
-            else {
-
+            else
                 buf = bufs[i];
-            }
         }
 
         /* Initialize sequence lists for memory and file spaces */
@@ -1712,11 +1724,19 @@ H5FD__write_selection_translate(uint32_t skip_vector_cb, H5FD_t *file, H5FD_mem_
                 vec_bufs[vec_arr_nused] = (const void *)((const uint8_t *)buf + mem_off[mem_seq_i]);
                 vec_arr_nused++;
             }
-            else
-                /* Issue scalar write call */
-                if ((file->cls->write)(file, type, dxpl_id, offsets[i] + file_off[file_seq_i], io_len,
-                                       (const void *)((const uint8_t *)buf + mem_off[mem_seq_i])) < 0)
+            else {
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        /* Issue scalar write call */
+                        ret_value =
+                            (file->cls->write)(file, type, dxpl_id, offsets[i] + file_off[file_seq_i], io_len,
+                                               (const void *)((const uint8_t *)buf + mem_off[mem_seq_i]));
+                    }
+                H5_AFTER_USER_CB(FAIL)
+                if (ret_value < 0)
                     HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "driver write request failed");
+            }
 
             /* Update file sequence */
             if (io_len == file_len[file_seq_i])
@@ -1754,8 +1774,14 @@ H5FD__write_selection_translate(uint32_t skip_vector_cb, H5FD_t *file, H5FD_mem_
         uint32_t actual_selection_io_mode;
 
         H5_CHECK_OVERFLOW(vec_arr_nused, size_t, uint32_t);
-        if ((file->cls->write_vector)(file, dxpl_id, (uint32_t)vec_arr_nused, types, addrs, sizes, vec_bufs) <
-            0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->write_vector)(file, dxpl_id, (uint32_t)vec_arr_nused, types, addrs,
+                                                      sizes, vec_bufs);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "driver write vector request failed");
 
         /* Set actual selection I/O, if this is a raw data operation */
@@ -1857,8 +1883,9 @@ H5FD_write_selection(H5FD_t *file, H5FD_mem_t type, uint32_t count, H5S_t **mem_
     hid_t   *mem_space_ids = mem_space_ids_local;
     hid_t    file_space_ids_local[H5FD_LOCAL_SEL_ARR_LEN];
     hid_t   *file_space_ids = file_space_ids_local;
-    uint32_t num_spaces     = 0;
-    hid_t    dxpl_id        = H5I_INVALID_HID; /* DXPL for operation */
+    haddr_t  eoa;
+    uint32_t num_spaces = 0;
+    hid_t    dxpl_id    = H5I_INVALID_HID; /* DXPL for operation */
     uint32_t i;
     herr_t   ret_value = SUCCEED; /* Return value */
 
@@ -1887,20 +1914,16 @@ H5FD_write_selection(H5FD_t *file, H5FD_mem_t type, uint32_t count, H5S_t **mem_
      * Do not return early for Parallel mode since the I/O could be a
      * collective transfer.
      */
-    if (0 == count) {
+    if (0 == count)
         HGOTO_DONE(SUCCEED);
-    }
 #endif /* H5_HAVE_PARALLEL */
 
     if (file->base_addr > 0) {
-
         /* apply the base_addr offset to the offsets array.  Must undo before
          * we return.
          */
-        for (i = 0; i < count; i++) {
-
+        for (i = 0; i < count; i++)
             offsets[i] += file->base_addr;
-        }
         offsets_cooked = true;
     }
 
@@ -1908,20 +1931,19 @@ H5FD_write_selection(H5FD_t *file, H5FD_mem_t type, uint32_t count, H5S_t **mem_
      * looking into the highest offset in the selection (different from the
      * bounds) is potentially expensive.
      */
-    {
-        haddr_t eoa;
-
-        if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type)))
-            HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed");
-
-        for (i = 0; i < count; i++) {
-
-            if ((offsets[i]) > eoa)
-
-                HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, offsets[%d] = %llu, eoa = %llu",
-                            (int)i, (unsigned long long)(offsets[i]), (unsigned long long)eoa);
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            eoa = (file->cls->get_eoa)(file, type);
         }
-    }
+    H5_AFTER_USER_CB(FAIL)
+    if (!H5_addr_defined(eoa))
+        HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed");
+
+    for (i = 0; i < count; i++)
+        if ((offsets[i]) > eoa)
+            HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, offsets[%d] = %llu, eoa = %llu", (int)i,
+                        (unsigned long long)(offsets[i]), (unsigned long long)eoa);
 
     /* if the underlying VFD supports selection write, make the call */
     if (file->cls->write_selection) {
@@ -1949,8 +1971,14 @@ H5FD_write_selection(H5FD_t *file, H5FD_mem_t type, uint32_t count, H5S_t **mem_
             }
         }
 
-        if ((file->cls->write_selection)(file, type, dxpl_id, count, mem_space_ids, file_space_ids, offsets,
-                                         element_sizes, bufs) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->write_selection)(file, type, dxpl_id, count, mem_space_ids,
+                                                         file_space_ids, offsets, element_sizes, bufs);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "driver write selection request failed");
 
         /* Set actual selection I/O, if this is a raw data operation */
@@ -1972,13 +2000,9 @@ H5FD_write_selection(H5FD_t *file, H5FD_mem_t type, uint32_t count, H5S_t **mem_
 done:
     /* undo the base addr offset to the offsets array if necessary */
     if (offsets_cooked) {
-
         assert(file->base_addr > 0);
-
-        for (i = 0; i < count; i++) {
-
+        for (i = 0; i < count; i++)
             offsets[i] -= file->base_addr;
-        }
     }
 
     /* Cleanup dataspace arrays.  Use H5I_remove() so we only close the IDs and
@@ -2031,7 +2055,8 @@ H5FD_write_selection_id(uint32_t skip_cb, H5FD_t *file, H5FD_mem_t type, uint32_
     H5S_t  **mem_spaces = mem_spaces_local;
     H5S_t   *file_spaces_local[H5FD_LOCAL_SEL_ARR_LEN];
     H5S_t  **file_spaces = file_spaces_local;
-    hid_t    dxpl_id     = H5I_INVALID_HID; /* DXPL for operation */
+    haddr_t  eoa;
+    hid_t    dxpl_id = H5I_INVALID_HID; /* DXPL for operation */
     uint32_t i;
     uint32_t skip_selection_cb;
     uint32_t skip_vector_cb;
@@ -2062,23 +2087,19 @@ H5FD_write_selection_id(uint32_t skip_cb, H5FD_t *file, H5FD_mem_t type, uint32_
      * Do not return early for Parallel mode since the I/O could be a
      * collective transfer.
      */
-    if (0 == count) {
+    if (0 == count)
         HGOTO_DONE(SUCCEED);
-    }
 #endif /* H5_HAVE_PARALLEL */
 
     skip_selection_cb = skip_cb & SKIP_SELECTION_CB;
     skip_vector_cb    = skip_cb & SKIP_VECTOR_CB;
 
     if (file->base_addr > 0) {
-
         /* apply the base_addr offset to the offsets array.  Must undo before
          * we return.
          */
-        for (i = 0; i < count; i++) {
-
+        for (i = 0; i < count; i++)
             offsets[i] += file->base_addr;
-        }
         offsets_cooked = true;
     }
 
@@ -2086,27 +2107,32 @@ H5FD_write_selection_id(uint32_t skip_cb, H5FD_t *file, H5FD_mem_t type, uint32_
      * looking into the highest offset in the selection (different from the
      * bounds) is potentially expensive.
      */
-    {
-        haddr_t eoa;
-
-        if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type)))
-            HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed");
-
-        for (i = 0; i < count; i++) {
-
-            if ((offsets[i]) > eoa)
-
-                HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, offsets[%d] = %llu, eoa = %llu",
-                            (int)i, (unsigned long long)(offsets[i]), (unsigned long long)eoa);
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            eoa = (file->cls->get_eoa)(file, type);
         }
-    }
+    H5_AFTER_USER_CB(FAIL)
+    if (!H5_addr_defined(eoa))
+        HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed");
+
+    for (i = 0; i < count; i++)
+        if ((offsets[i]) > eoa)
+            HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, offsets[%d] = %llu, eoa = %llu", (int)i,
+                        (unsigned long long)(offsets[i]), (unsigned long long)eoa);
 
     /* if the underlying VFD supports selection write, make the call */
     if (!skip_selection_cb && file->cls->write_selection) {
         uint32_t actual_selection_io_mode;
 
-        if ((file->cls->write_selection)(file, type, dxpl_id, count, mem_space_ids, file_space_ids, offsets,
-                                         element_sizes, bufs) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (file->cls->write_selection)(file, type, dxpl_id, count, mem_space_ids,
+                                                         file_space_ids, offsets, element_sizes, bufs);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "driver write selection request failed");
 
         /* Set actual selection I/O, if this is a raw data operation */
@@ -2139,7 +2165,6 @@ H5FD_write_selection_id(uint32_t skip_cb, H5FD_t *file, H5FD_mem_t type, uint32_
         }
 
         /* Translate to vector or scalar I/O */
-
         if (H5FD__write_selection_translate(skip_vector_cb, file, type, dxpl_id, count, mem_spaces,
                                             file_spaces, offsets, element_sizes, bufs) < 0)
             HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "translation to vector or scalar write failed");
@@ -2148,13 +2173,9 @@ H5FD_write_selection_id(uint32_t skip_cb, H5FD_t *file, H5FD_mem_t type, uint32_
 done:
     /* undo the base addr offset to the offsets array if necessary */
     if (offsets_cooked) {
-
         assert(file->base_addr > 0);
-
-        for (i = 0; i < count; i++) {
-
+        for (i = 0; i < count; i++)
             offsets[i] -= file->base_addr;
-        }
     }
 
     /* Cleanup dataspace arrays */
@@ -2393,8 +2414,14 @@ H5FD_set_eoa(H5FD_t *file, H5FD_mem_t type, haddr_t addr)
     assert(file && file->cls);
     assert(H5_addr_defined(addr) && addr <= file->maxaddr);
 
-    /* Dispatch to driver, convert to absolute address */
-    if ((file->cls->set_eoa)(file, type, addr + file->base_addr) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Dispatch to driver, convert to absolute address */
+            ret_value = (file->cls->set_eoa)(file, type, addr + file->base_addr);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver set_eoa request failed");
 
 done:
@@ -2426,8 +2453,14 @@ H5FD_get_eoa(const H5FD_t *file, H5FD_mem_t type)
 
     assert(file && file->cls);
 
-    /* Dispatch to driver */
-    if (HADDR_UNDEF == (ret_value = (file->cls->get_eoa)(file, type)))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(HADDR_UNDEF)
+        {
+            /* Dispatch to driver */
+            ret_value = (file->cls->get_eoa)(file, type);
+        }
+    H5_AFTER_USER_CB(HADDR_UNDEF)
+    if (!H5_addr_defined(ret_value))
         HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "driver get_eoa request failed");
 
     /* Adjust for base address in file (convert to relative address) */
@@ -2464,7 +2497,13 @@ H5FD_get_eof(const H5FD_t *file, H5FD_mem_t type)
 
     /* Dispatch to driver */
     if (file->cls->get_eof) {
-        if (HADDR_UNDEF == (ret_value = (file->cls->get_eof)(file, type)))
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(HADDR_UNDEF)
+            {
+                ret_value = (file->cls->get_eof)(file, type);
+            }
+        H5_AFTER_USER_CB(HADDR_UNDEF)
+        if (!H5_addr_defined(ret_value))
             HGOTO_ERROR(H5E_VFL, H5E_CANTGET, HADDR_UNDEF, "driver get_eof request failed");
     }
     else
@@ -2500,8 +2539,14 @@ H5FD_driver_query(const H5FD_class_t *driver, unsigned long *flags /*out*/)
     assert(flags);
 
     /* Check for the driver to query and then query it */
-    if (driver->query)
-        ret_value = (driver->query)(NULL, flags);
+    if (driver->query) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB_NOERR(FAIL)
+            {
+                ret_value = (driver->query)(NULL, flags);
+            }
+        H5_AFTER_USER_CB_NOERR(FAIL)
+    }
     else
         *flags = 0;
 
@@ -3014,8 +3059,14 @@ H5FD_delete(const char *filename, hid_t fapl_id)
     if (NULL == driver->del)
         HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "file driver has no 'del' method");
 
-    /* Dispatch to file driver */
-    if ((driver->del)(filename, fapl_id))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Dispatch to file driver */
+            ret_value = (driver->del)(filename, fapl_id);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VFL, H5E_CANTDELETEFILE, FAIL, "delete failed");
 
 done:
diff --git a/src/H5FDlog.c b/src/H5FDlog.c
index f3877f684ed..cd55ded0aca 100644
--- a/src/H5FDlog.c
+++ b/src/H5FDlog.c
@@ -270,7 +270,7 @@ H5Pset_fapl_log(hid_t fapl_id, const char *logfile, unsigned long long flags, si
     memset(&fa, 0, sizeof(H5FD_log_fapl_t));
 
     /* Check arguments */
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
 
     /* Duplicate the log file string
@@ -446,7 +446,7 @@ H5FD__log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
         o_flags |= O_EXCL;
 
     /* Get the driver specific information */
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list");
     if (NULL == (fa = (const H5FD_log_fapl_t *)H5P_peek_driver_info(plist))) {
         /* Use default driver configuration*/
diff --git a/src/H5FDlog.h b/src/H5FDlog.h
index 871845cf189..55ca106bdd5 100644
--- a/src/H5FDlog.h
+++ b/src/H5FDlog.h
@@ -27,40 +27,60 @@
 #define H5FD_LOG_VALUE H5_VFD_LOG
 
 /* Flags for H5Pset_fapl_log() */
-/* Flags for tracking 'meta' operations (truncate) */
+
+/** Flag for tracking truncate operation \since 1.10.1 */
 #define H5FD_LOG_TRUNCATE 0x00000001
-#define H5FD_LOG_META_IO  (H5FD_LOG_TRUNCATE)
-/* Flags for tracking where reads/writes/seeks occur */
-#define H5FD_LOG_LOC_READ  0x00000002
+/** Flag for tracking meta IO operations \since 1.10.1 */
+#define H5FD_LOG_META_IO (H5FD_LOG_TRUNCATE)
+/** Flag for tracking where reads occur \since 1.6.0 */
+#define H5FD_LOG_LOC_READ 0x00000002
+/** Flag for tracking where writes occur \since 1.6.0 */
 #define H5FD_LOG_LOC_WRITE 0x00000004
-#define H5FD_LOG_LOC_SEEK  0x00000008
-#define H5FD_LOG_LOC_IO    (H5FD_LOG_LOC_READ | H5FD_LOG_LOC_WRITE | H5FD_LOG_LOC_SEEK)
-/* Flags for tracking number of times each byte is read/written */
-#define H5FD_LOG_FILE_READ  0x00000010
+/** Flag for tracking where seeks occur \since 1.6.0 */
+#define H5FD_LOG_LOC_SEEK 0x00000008
+/** Flag for tracking where IO operations occur \since 1.6.0 */
+#define H5FD_LOG_LOC_IO (H5FD_LOG_LOC_READ | H5FD_LOG_LOC_WRITE | H5FD_LOG_LOC_SEEK)
+/** Flag for tracking number of times each byte is read \since 1.6.0 */
+#define H5FD_LOG_FILE_READ 0x00000010
+/** Flag for tracking number of times each byte is written \since 1.6.0 */
 #define H5FD_LOG_FILE_WRITE 0x00000020
-#define H5FD_LOG_FILE_IO    (H5FD_LOG_FILE_READ | H5FD_LOG_FILE_WRITE)
-/* Flag for tracking "flavor" (type) of information stored at each byte */
+/** Flag for tracking number of times each byte is read/written \since 1.6.0 */
+#define H5FD_LOG_FILE_IO (H5FD_LOG_FILE_READ | H5FD_LOG_FILE_WRITE)
+/** Flag for tracking "flavor" (type) of information stored at each byte \since 1.6.0 */
 #define H5FD_LOG_FLAVOR 0x00000040
-/* Flags for tracking total number of reads/writes/seeks/truncates */
-#define H5FD_LOG_NUM_READ     0x00000080
-#define H5FD_LOG_NUM_WRITE    0x00000100
-#define H5FD_LOG_NUM_SEEK     0x00000200
+/** Flag for tracking total number of reads \since 1.6.0 */
+#define H5FD_LOG_NUM_READ 0x00000080
+/** Flag for tracking total number of writes \since 1.6.0 */
+#define H5FD_LOG_NUM_WRITE 0x00000100
+/** Flag for tracking total number of seeks \since 1.6.0 */
+#define H5FD_LOG_NUM_SEEK 0x00000200
+/** Flag for tracking total number of truncates \since 1.8.7 */
 #define H5FD_LOG_NUM_TRUNCATE 0x00000400
-#define H5FD_LOG_NUM_IO       (H5FD_LOG_NUM_READ | H5FD_LOG_NUM_WRITE | H5FD_LOG_NUM_SEEK | H5FD_LOG_NUM_TRUNCATE)
-/* Flags for tracking time spent in open/stat/read/write/seek/truncate/close */
-#define H5FD_LOG_TIME_OPEN     0x00000800
-#define H5FD_LOG_TIME_STAT     0x00001000
-#define H5FD_LOG_TIME_READ     0x00002000
-#define H5FD_LOG_TIME_WRITE    0x00004000
-#define H5FD_LOG_TIME_SEEK     0x00008000
+/** Flag for tracking total number of IO operations \since 1.6.0 */
+#define H5FD_LOG_NUM_IO (H5FD_LOG_NUM_READ | H5FD_LOG_NUM_WRITE | H5FD_LOG_NUM_SEEK | H5FD_LOG_NUM_TRUNCATE)
+/** Flag for tracking time spent in open \since 1.8.7 */
+#define H5FD_LOG_TIME_OPEN 0x00000800
+/** Flag for tracking time spent in stat \since 1.8.7 */
+#define H5FD_LOG_TIME_STAT 0x00001000
+/** Flag for tracking time spent in read \since 1.8.7 */
+#define H5FD_LOG_TIME_READ 0x00002000
+/** Flag for tracking time spent in write \since 1.6.0 */
+#define H5FD_LOG_TIME_WRITE 0x00004000
+/** Flag for tracking time spent in seek \since 1.6.0 */
+#define H5FD_LOG_TIME_SEEK 0x00008000
+/** Flag for tracking time spent in truncate \since 1.10.1 */
 #define H5FD_LOG_TIME_TRUNCATE 0x00010000
-#define H5FD_LOG_TIME_CLOSE    0x00020000
+/** Flag for tracking time spent in close \since 1.6.0 */
+#define H5FD_LOG_TIME_CLOSE 0x00020000
+/** Flag for tracking time spent in IO operations \since 1.6.0 */
 #define H5FD_LOG_TIME_IO                                                                                     \
     (H5FD_LOG_TIME_OPEN | H5FD_LOG_TIME_STAT | H5FD_LOG_TIME_READ | H5FD_LOG_TIME_WRITE |                    \
      H5FD_LOG_TIME_SEEK | H5FD_LOG_TIME_TRUNCATE | H5FD_LOG_TIME_CLOSE)
-/* Flags for tracking allocation/release of space in file */
+/** Flag for tracking allocation of space in file \since 1.6.0 */
 #define H5FD_LOG_ALLOC 0x00040000
-#define H5FD_LOG_FREE  0x00080000
+/** Flag for tracking release of space in file \since 1.10.1 */
+#define H5FD_LOG_FREE 0x00080000
+/** Flag for tracking all info \since 1.6.0 */
 #define H5FD_LOG_ALL                                                                                         \
     (H5FD_LOG_FREE | H5FD_LOG_ALLOC | H5FD_LOG_TIME_IO | H5FD_LOG_NUM_IO | H5FD_LOG_FLAVOR |                 \
      H5FD_LOG_FILE_IO | H5FD_LOG_LOC_IO | H5FD_LOG_META_IO)
diff --git a/src/H5FDmirror.c b/src/H5FDmirror.c
index d566eafe080..2cc8f005eeb 100644
--- a/src/H5FDmirror.c
+++ b/src/H5FDmirror.c
@@ -1247,7 +1247,7 @@ H5Pget_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa_dst /*out*/)
     if (NULL == fa_dst)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_dst is NULL");
 
-    plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS);
+    plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true);
     if (NULL == plist)
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
     if (H5P_peek_driver(plist) != H5FD_MIRROR)
@@ -1284,7 +1284,7 @@ H5Pset_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa)
 
     LOG_OP_CALL(__func__);
 
-    plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS);
+    plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false);
     if (NULL == plist)
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
     if (NULL == fa)
diff --git a/src/H5FDmirror.h b/src/H5FDmirror.h
index 3758dc083fa..74db0fe1a1f 100644
--- a/src/H5FDmirror.h
+++ b/src/H5FDmirror.h
@@ -22,7 +22,7 @@
 
 #ifdef H5_HAVE_MIRROR_VFD
 
-/** ID for the mirror VFD */
+/** ID for the mirror VFD \since 1.10.7 */
 #define H5FD_MIRROR (H5OPEN H5FD_MIRROR_id_g)
 
 /** Identifier for the mirror VFD */
diff --git a/src/H5FDmodule.h b/src/H5FDmodule.h
index d692b8fef0d..13b8dbd69d8 100644
--- a/src/H5FDmodule.h
+++ b/src/H5FDmodule.h
@@ -25,5 +25,9 @@
 #define H5_MY_PKG      H5FD
 #define H5_MY_PKG_ERR  H5E_VFL
 #define H5_MY_PKG_INIT YES
+/**
+ * \defgroup H5VFD Virtual File Driver Features
+ *
+ */
 
 #endif /* H5FDmodule_H */
diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c
index cce5d2ee1ff..793678ca470 100644
--- a/src/H5FDmpio.c
+++ b/src/H5FDmpio.c
@@ -427,7 +427,7 @@ H5Pset_fapl_mpio(hid_t fapl_id, MPI_Comm comm, MPI_Info info)
     /* Check arguments */
     if (fapl_id == H5P_DEFAULT)
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list");
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access list");
     if (MPI_COMM_NULL == comm)
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "MPI_COMM_NULL is not a valid communicator");
@@ -485,7 +485,7 @@ H5Pget_fapl_mpio(hid_t fapl_id, MPI_Comm *comm /*out*/, MPI_Info *info /*out*/)
         *info = MPI_INFO_NULL;
 
     /* Check arguments */
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access list");
     if (H5FD_MPIO != H5P_peek_driver(plist))
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "VFL driver is not MPI-I/O");
@@ -549,7 +549,7 @@ H5Pset_dxpl_mpio(hid_t dxpl_id, H5FD_mpio_xfer_t xfer_mode)
     /* Check arguments */
     if (dxpl_id == H5P_DEFAULT)
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list");
-    if (NULL == (plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl");
     if (H5FD_MPIO_INDEPENDENT != xfer_mode && H5FD_MPIO_COLLECTIVE != xfer_mode)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "incorrect xfer_mode");
@@ -589,7 +589,7 @@ H5Pget_dxpl_mpio(hid_t dxpl_id, H5FD_mpio_xfer_t *xfer_mode /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Check arguments */
-    if (NULL == (plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl");
 
     /* Initialize driver, if it's not yet */
@@ -633,7 +633,7 @@ H5Pset_dxpl_mpio_collective_opt(hid_t dxpl_id, H5FD_mpio_collective_opt_t opt_mo
     /* Check arguments */
     if (dxpl_id == H5P_DEFAULT)
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list");
-    if (NULL == (plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl");
 
     /* Initialize driver, if it's not yet */
@@ -676,7 +676,7 @@ H5Pset_dxpl_mpio_chunk_opt(hid_t dxpl_id, H5FD_mpio_chunk_opt_t opt_mode)
     /* Check arguments */
     if (dxpl_id == H5P_DEFAULT)
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list");
-    if (NULL == (plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl");
 
     /* Initialize driver, if it's not yet */
@@ -717,7 +717,7 @@ H5Pset_dxpl_mpio_chunk_opt_num(hid_t dxpl_id, unsigned num_chunk_per_proc)
     /* Check arguments */
     if (dxpl_id == H5P_DEFAULT)
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list");
-    if (NULL == (plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl");
 
     /* Initialize driver, if it's not yet */
@@ -761,7 +761,7 @@ H5Pset_dxpl_mpio_chunk_opt_ratio(hid_t dxpl_id, unsigned percent_num_proc_per_ch
     /* Check arguments */
     if (dxpl_id == H5P_DEFAULT)
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list");
-    if (NULL == (plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl");
 
     /* Initialize driver, if it's not yet */
@@ -904,7 +904,7 @@ H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t H5_ATTR
             HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "can't initialize driver");
 
     /* Get a pointer to the fapl */
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list");
 
     if (H5FD_mpi_self_initialized_s) {
@@ -3822,7 +3822,7 @@ H5FD__mpio_delete(const char *filename, hid_t fapl_id)
         if (H5FD__mpio_init() < 0)
             HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "can't initialize driver");
 
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
     assert(H5FD_MPIO == H5P_peek_driver(plist));
 
diff --git a/src/H5FDonion.c b/src/H5FDonion.c
index ddbc25bc04c..cb118c3f881 100644
--- a/src/H5FDonion.c
+++ b/src/H5FDonion.c
@@ -278,7 +278,7 @@ H5Pget_fapl_onion(hid_t fapl_id, H5FD_onion_fapl_info_t *fa_out)
     if (NULL == fa_out)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL info-out pointer");
 
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Not a valid FAPL ID");
 
     if (H5FD_ONION != H5P_peek_driver(plist))
@@ -316,7 +316,7 @@ H5Pset_fapl_onion(hid_t fapl_id, const H5FD_onion_fapl_info_t *fa)
 
     FUNC_ENTER_API(FAIL)
 
-    if (NULL == (fapl = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (fapl = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Not a valid FAPL ID");
     if (NULL == fa)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL info pointer");
@@ -328,11 +328,11 @@ H5Pset_fapl_onion(hid_t fapl_id, const H5FD_onion_fapl_info_t *fa)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid info page size");
 
     if (H5P_DEFAULT == fa->backing_fapl_id) {
-        if (NULL == (backing_fapl = H5P_object_verify(H5P_FILE_ACCESS_DEFAULT, H5P_FILE_ACCESS)))
+        if (NULL == (backing_fapl = H5P_object_verify(H5P_FILE_ACCESS_DEFAULT, H5P_FILE_ACCESS, true)))
             HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid backing fapl id");
     }
     else {
-        if (NULL == (backing_fapl = H5P_object_verify(fa->backing_fapl_id, H5P_FILE_ACCESS)))
+        if (NULL == (backing_fapl = H5P_object_verify(fa->backing_fapl_id, H5P_FILE_ACCESS, true)))
             HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid backing fapl id");
     }
 
@@ -1635,7 +1635,7 @@ H5FDonion_get_revision_count(const char *filename, hid_t fapl_id, uint64_t *revi
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "revision count can't be null");
 
     /* Make sure using the correct driver */
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid FAPL ID");
     if (H5FD_ONION != H5P_peek_driver(plist))
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a Onion VFL driver");
diff --git a/src/H5FDonion.h b/src/H5FDonion.h
index 5c374bb16dd..080970297a5 100644
--- a/src/H5FDonion.h
+++ b/src/H5FDonion.h
@@ -162,7 +162,7 @@ H5_DLL herr_t H5Pset_fapl_onion(hid_t fapl_id, const H5FD_onion_fapl_info_t *fa)
 
 /**
  * --------------------------------------------------------------------------
- * \ingroup H5FD
+ * \ingroup H5VFD
  *
  * \brief get the number of revisions
  *
diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h
index d8e080edab9..40c64c760a4 100644
--- a/src/H5FDpublic.h
+++ b/src/H5FDpublic.h
@@ -94,11 +94,13 @@
  * from the file when it is flushed/closed, if the file is opened R/W).
  */
 #define H5FD_FEAT_IGNORE_DRVRINFO 0x00000020
-/*
+/**
  * Defining the H5FD_FEAT_DIRTY_DRVRINFO_LOAD for a VFL driver means that
  * the library will mark the driver info dirty when the file is opened
  * R/W.  This will cause the driver info to be re-encoded when the file
  * is flushed/closed.
+ *
+ * \since 1.10.0
  */
 #define H5FD_FEAT_DIRTY_DRVRINFO_LOAD 0x00000040
 /*
@@ -107,18 +109,24 @@
  * of type 'int' and is compatible with POSIX I/O calls.
  */
 #define H5FD_FEAT_POSIX_COMPAT_HANDLE 0x00000080
-/*
+
+/**
  * Defining H5FD_FEAT_HAS_MPI for a VFL driver means that
  * the driver makes use of MPI communication and code may retrieve
  * communicator/rank information from it
+ *
+ * \since 1.8.15
  */
 #define H5FD_FEAT_HAS_MPI 0x00000100
-/*
- * Defining the H5FD_FEAT_ALLOCATE_EARLY for a VFL driver will force
- * the library to use the H5D_ALLOC_TIME_EARLY on dataset create
- * instead of the default H5D_ALLOC_TIME_LATE
- */
+
 #define H5FD_FEAT_ALLOCATE_EARLY 0x00000200
+/**< Defining the H5FD_FEAT_ALLOCATE_EARLY for a VFL driver will force
+ *   the library to use the H5D_ALLOC_TIME_EARLY on dataset create
+ *   instead of the default H5D_ALLOC_TIME_LATE
+ *
+ * \since 1.8.15
+ */
+
 /*
  * Defining H5FD_FEAT_ALLOW_FILE_IMAGE for a VFL driver means that
  * the driver is able to use a file image in the fapl as the initial
@@ -131,31 +139,39 @@
  * image to store in memory.
  */
 #define H5FD_FEAT_CAN_USE_FILE_IMAGE_CALLBACKS 0x00000800
-/*
+/**
  * Defining H5FD_FEAT_SUPPORTS_SWMR_IO for a VFL driver means that the
  * driver supports the single-writer/multiple-readers I/O pattern.
+ *
+ * \since 1.10.0
  */
 #define H5FD_FEAT_SUPPORTS_SWMR_IO 0x00001000
-/*
+/**
  * Defining H5FD_FEAT_USE_ALLOC_SIZE for a VFL driver
  * means that the library will just pass the allocation size to the
  * the driver's allocation callback which will eventually handle alignment.
  * This is specifically used for the multi/split driver.
+ *
+ * \since 1.10.1
  */
 #define H5FD_FEAT_USE_ALLOC_SIZE 0x00002000
-/*
+/**
  * Defining H5FD_FEAT_PAGED_AGGR for a VFL driver
  * means that the driver needs special file space mapping for paged aggregation.
  * This is specifically used for the multi/split driver.
+ *
+ * \since 1.10.1
  */
 #define H5FD_FEAT_PAGED_AGGR 0x00004000
-/*
+/**
  * Defining H5FD_FEAT_DEFAULT_VFD_COMPATIBLE for a VFL driver
  * that creates a file which is compatible with the default VFD.
  * Generally, this means that the VFD creates a single file that follows
  * the canonical HDF5 file format.
  * Regarding the Splitter VFD specifically, only drivers with this flag
  * enabled may be used as the Write-Only (W/O) channel driver.
+ *
+ * \since 1.10.2
  */
 #define H5FD_FEAT_DEFAULT_VFD_COMPATIBLE 0x00008000
 /*
@@ -402,7 +418,7 @@ extern "C" {
 /* Function prototypes */
 
 /**
- * \ingroup H5FD
+ * \ingroup H5VFD
  *
  * \brief Allows querying a VFD ID for features before the file is opened
  *
diff --git a/src/H5FDros3.c b/src/H5FDros3.c
index 02698c59399..56321730c12 100644
--- a/src/H5FDros3.c
+++ b/src/H5FDros3.c
@@ -77,28 +77,35 @@ typedef struct H5FD_ros3_stats_bin {
  * Stores all information needed to maintain access to a single HDF5 file
  * that has been stored as a S3 object.
  *
- * `pub` (H5FD_t)
+ * pub
  *
  *     Instance of H5FD_t which contains all fields common to all VFDs.
  *     It must be the first item in this structure, since at higher levels,
  *     this structure will be treated as an instance of H5FD_t.
  *
- * `fa` (H5FD_ros3_fapl_t)
+ * fa
  *
  *     Instance of `H5FD_ros3_fapl_t` containing the S3 configuration data
  *     needed to "open" the HDF5 file.
  *
- * `eoa` (haddr_t)
+ * eoa
  *
  *     End of addressed space in file. After open, it should always
  *     equal the file size.
  *
- * `s3r_handle` (s3r_t *)
+ * s3r_handle
  *
  *     Instance of S3 Request handle associated with the target resource.
  *     Responsible for communicating with remote host and presenting file
  *     contents as indistinguishable from a file on the local filesystem.
  *
+ * cache
+ * cache_size (in bytes)
+ *
+ *     A simple cache of the first N bytes of the file. Especially useful
+ *     at file open, when we perform several reads that would otherwise
+ *     be uncached.
+ *
  * *** present only if ROS3_SATS is set to enable stats collection ***
  *
  * `meta` (H5FD_ros3_stats_bin_t[])
@@ -118,8 +125,8 @@ typedef struct H5FD_ros3_stats_bin {
  ***************************************************************************/
 typedef struct H5FD_ros3_t {
     H5FD_t           pub;
-    H5FD_ros3_fapl_t fa;
     haddr_t          eoa;
+    H5FD_ros3_fapl_t fa;
     s3r_t           *s3r_handle;
     uint8_t         *cache;
     size_t           cache_size;
@@ -295,11 +302,11 @@ H5Pset_fapl_ros3(hid_t fapl_id, const H5FD_ros3_fapl_t *fa)
 
     assert(fa != NULL);
 
-    plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS);
+    plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false);
     if (plist == NULL)
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
 
-    if (FAIL == H5FD__ros3_validate_config(fa))
+    if (H5FD__ros3_validate_config(fa) < 0)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid ros3 config");
 
     ret_value = H5P_set_driver(plist, H5FD_ROS3, (const void *)fa, NULL);
@@ -357,7 +364,7 @@ H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_dst /*out*/)
 
     if (fa_dst == NULL)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_dst is NULL");
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list");
     if (H5FD_ROS3 != H5P_peek_driver(plist))
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fapl not set to use the ros3 VFD");
@@ -489,7 +496,7 @@ H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token_dst /*out*/)
     if (token_dst == NULL)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "token_dst is NULL");
 
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list");
     if (H5FD_ROS3 != H5P_peek_driver(plist))
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver");
@@ -641,7 +648,7 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token)
 
     if (fapl_id == H5P_DEFAULT)
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list");
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list");
     if (H5FD_ROS3 != H5P_peek_driver(plist))
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver");
@@ -676,9 +683,9 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token)
 /*-------------------------------------------------------------------------
  * Function:    H5FD__ros3_open
  *
- * Purpose:     Create and/or open a file as an HDF5 file.
+ * Purpose:     Create and/or open a file as an HDF5 file
  *
- *     Any flag except H5F_ACC_RDONLY will cause an error.
+ *     Any flag except H5F_ACC_RDONLY will cause an error
  *
  *     `url` param (as received from `H5FD_open()`) must conform to web url:
  *         NAME   :: HTTP "://" DOMAIN [PORT] ["/" [URI] [QUERY] ]
@@ -695,16 +702,12 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token)
 static H5FD_t *
 H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
 {
-    H5FD_ros3_t            *file = NULL;
-    struct tm              *now  = NULL;
-    char                    iso8601now[ISO8601_SIZE];
-    unsigned char           signing_key[SHA256_DIGEST_LENGTH];
-    s3r_t                  *handle = NULL;
-    const H5FD_ros3_fapl_t *fa     = NULL;
-    H5P_genplist_t         *plist  = NULL;
-    htri_t                  token_exists;
-    char                   *token;
-    H5FD_t                 *ret_value = NULL;
+    H5FD_ros3_t            *file       = NULL;
+    s3r_t                  *handle     = NULL;
+    const H5FD_ros3_fapl_t *fa         = NULL;
+    H5P_genplist_t         *plist      = NULL;
+    char                   *fapl_token = NULL;
+    H5FD_t                 *ret_value  = NULL;
 
     FUNC_ENTER_PACKAGE
 
@@ -717,7 +720,7 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
         HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr");
     if (flags != H5F_ACC_RDONLY)
         HGOTO_ERROR(H5E_ARGS, H5E_UNSUPPORTED, NULL, "only Read-Only access allowed");
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list");
 
     /* Initialize driver, if it's not yet */
@@ -733,47 +736,26 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
     if (NULL == (fa = (const H5FD_ros3_fapl_t *)H5P_peek_driver_info(plist)))
         HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "could not get ros3 VFL driver info");
 
-    /* Session/security token */
-    if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0)
-        HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "failed check for property token in plist");
-    if (token_exists) {
-        if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token) < 0)
-            HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "unable to get token value");
+    /* Get the token, if it exists */
+    if (fa->authenticate) {
+        htri_t token_exists;
+
+        /* Does the token exist in the fapl? */
+        if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "failed check for property token in plist");
+
+        /* If so, get it */
+        if (token_exists) {
+            if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &fapl_token) < 0)
+                HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "unable to get token value");
+        }
     }
 
     /* Open file; procedure depends on whether or not the fapl instructs to
      * authenticate requests or not.
      */
-    if (fa->authenticate == true) {
-        /* Compute signing key (part of AWS/S3 REST API). Can be re-used by
-         * user/key for 7 days after creation.
-         *
-         * TODO: Find way to reuse/share?
-         */
-        now = gmnow();
-        assert(now != NULL);
-        if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1))
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while writing iso8601 timestamp");
-        if (FAIL == H5FD_s3comms_signing_key(signing_key, (const char *)fa->secret_key,
-                                             (const char *)fa->aws_region, (const char *)iso8601now))
-            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "problem while computing signing key");
-
-        if (token_exists)
-            handle = H5FD_s3comms_s3r_open(url, (const char *)fa->aws_region, (const char *)fa->secret_id,
-                                           (const unsigned char *)signing_key, (const char *)token);
-        else
-            handle = H5FD_s3comms_s3r_open(url, (const char *)fa->aws_region, (const char *)fa->secret_id,
-                                           (const unsigned char *)signing_key, "");
-    }
-    else
-        handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL, NULL);
-
-    if (handle == NULL)
-        /* If we want to check CURL's say on the matter in a controlled
-         * fashion, this is the place to do it, but would need to make a
-         * few minor changes to s3comms `s3r_t` and `s3r_read()`.
-         */
-        HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "could not open");
+    if (NULL == (handle = H5FD__s3comms_s3r_open(url, fa, fapl_token)))
+        HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "s3r_open failed");
 
     /* Create new file struct */
     if (NULL == (file = H5FL_CALLOC(H5FD_ros3_t)))
@@ -783,19 +765,19 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
     H5MM_memcpy(&(file->fa), fa, sizeof(H5FD_ros3_fapl_t));
 
 #ifdef ROS3_STATS
-    if (FAIL == H5FD__ros3_reset_stats(file))
+    if (H5FD__ros3_reset_stats(file) < 0)
         HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics");
 #endif
 
     /* Cache the initial bytes of the file */
     {
-        size_t filesize = H5FD_s3comms_s3r_get_filesize(file->s3r_handle);
+        size_t filesize = H5FD__s3comms_s3r_get_filesize(file->s3r_handle);
 
         file->cache_size = (filesize < ROS3_MAX_CACHE_SIZE) ? filesize : ROS3_MAX_CACHE_SIZE;
 
         if (NULL == (file->cache = (uint8_t *)H5MM_calloc(file->cache_size)))
             HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, NULL, "unable to allocate cache memory");
-        if (H5FD_s3comms_s3r_read(file->s3r_handle, 0, file->cache_size, file->cache) < 0)
+        if (H5FD__s3comms_s3r_read(file->s3r_handle, 0, file->cache_size, file->cache) < 0)
             HGOTO_ERROR(H5E_VFL, H5E_READERROR, NULL, "unable to execute read");
     }
 
@@ -804,7 +786,7 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
 done:
     if (ret_value == NULL) {
         if (handle != NULL)
-            if (FAIL == H5FD_s3comms_s3r_close(handle))
+            if (H5FD__s3comms_s3r_close(handle) < 0)
                 HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close s3 file handle");
         if (file != NULL) {
             H5MM_xfree(file->cache);
@@ -836,12 +818,12 @@ H5FD__ros3_close(H5FD_t H5_ATTR_UNUSED *_file)
     assert(file->s3r_handle != NULL);
 
 #ifdef ROS3_STATS
-    if (H5FD__ros3_print_stats(stdout, file) == FAIL)
+    if (H5FD__ros3_print_stats(stdout, file) < 0)
         HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, "problem while writing file statistics");
 #endif
 
     /* Close the underlying request handle */
-    if (FAIL == H5FD_s3comms_s3r_close(file->s3r_handle))
+    if (H5FD__s3comms_s3r_close(file->s3r_handle) < 0)
         HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close S3 request handle");
 
     /* Release the file info */
@@ -1054,7 +1036,7 @@ H5FD__ros3_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type)
 
     FUNC_ENTER_PACKAGE_NOERR
 
-    FUNC_LEAVE_NOAPI(H5FD_s3comms_s3r_get_filesize(file->s3r_handle))
+    FUNC_LEAVE_NOAPI(H5FD__s3comms_s3r_get_filesize(file->s3r_handle))
 } /* end H5FD__ros3_get_eof() */
 
 /*-------------------------------------------------------------------------
@@ -1107,7 +1089,7 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU
     assert(file->s3r_handle);
     assert(buf);
 
-    filesize = H5FD_s3comms_s3r_get_filesize(file->s3r_handle);
+    filesize = H5FD__s3comms_s3r_get_filesize(file->s3r_handle);
 
     if ((addr > filesize) || ((addr + size) > filesize))
         HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "range exceeds file address");
@@ -1119,7 +1101,7 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU
         memcpy(buf, file->cache + addr, size);
     }
     else {
-        if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) == FAIL)
+        if (H5FD__s3comms_s3r_read(file->s3r_handle, addr, size, buf) < 0)
             HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read");
 
 #ifdef ROS3_STATS
diff --git a/src/H5FDros3.h b/src/H5FDros3.h
index 8117ad8a1e5..31a91ff08bc 100644
--- a/src/H5FDros3.h
+++ b/src/H5FDros3.h
@@ -21,17 +21,17 @@
 
 #ifdef H5_HAVE_ROS3_VFD
 
-/** ID for the ros3 VFD */
+/** Initializer for the ros3 VFD \since 1.8.22 */
 #define H5FD_ROS3 (H5OPEN H5FD_ROS3_id_g)
 
-/** Identifier for the ros3 VFD */
+/** Identifier for the ros3 VFD \since 1.14.0 */
 #define H5FD_ROS3_VALUE H5_VFD_ROS3
 
 #else
-/** Initializer for the ros3 VFD (disabled) */
+/** Initializer for the ros3 VFD (disabled) \since 1.8.22 */
 #define H5FD_ROS3       (H5I_INVALID_HID)
 
-/** Identifier for the ros3 VFD (disabled) */
+/** Identifier for the ros3 VFD (disabled) \since 1.14.0 */
 #define H5FD_ROS3_VALUE H5_VFD_INVALID
 #endif
 
@@ -47,16 +47,22 @@
 /**
  * \def H5FD_ROS3_MAX_REGION_LEN
  * Maximum string length for specifying the region of the S3 bucket.
+ *
+ * \since 1.10.6
  */
 #define H5FD_ROS3_MAX_REGION_LEN 32
 /**
  * \def H5FD_ROS3_MAX_SECRET_ID_LEN
  * Maximum string length for specifying the security ID.
+ *
+ * \since 1.10.6
  */
 #define H5FD_ROS3_MAX_SECRET_ID_LEN 128
 /**
  * \def H5FD_ROS3_MAX_SECRET_KEY_LEN
  * Maximum string length for specifying the security key.
+ *
+ * \since 1.10.6
  */
 #define H5FD_ROS3_MAX_SECRET_KEY_LEN 128
 /**
@@ -66,7 +72,7 @@
 #define H5FD_ROS3_MAX_SECRET_TOK_LEN 4096
 
 /**
- *\struct H5FD_ros3_fapl_t
+ * \struct H5FD_ros3_fapl_t
  * \brief Configuration structure for H5Pset_fapl_ros3() / H5Pget_fapl_ros3().
  *
  * \details H5FD_ros_fapl_t is a public structure that is used to pass
@@ -94,7 +100,6 @@
  *
  * \var char H5FD_ros3_fapl_t::secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1]
  *      A string which specifies the security key.
- *
  */
 typedef struct H5FD_ros3_fapl_t {
     int32_t version;
diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c
index c5de645ddb8..fdd93af6821 100644
--- a/src/H5FDs3comms.c
+++ b/src/H5FDs3comms.c
@@ -35,18 +35,22 @@
 /* Headers */
 /***********/
 
+/* There's no H5FDs3comms_test.c file, so the test functions are located here */
+#define H5FD_S3COMMS_TESTING
+
 #include "H5private.h"   /* generic functions */
 #include "H5Eprivate.h"  /* error handling    */
 #include "H5MMprivate.h" /* memory management */
 #include "H5FDs3comms.h" /* S3 Communications */
 #include "H5FDros3.h"    /* ros3 file driver  */
 
+#ifdef H5_HAVE_ROS3_VFD
+#include 
+
 /****************/
 /* Local Macros */
 /****************/
 
-#ifdef H5_HAVE_ROS3_VFD
-
 /* manipulate verbosity of CURL output
  *
  * 0 -> no explicit curl output
@@ -56,10 +60,12 @@
  */
 #define S3COMMS_CURL_VERBOSITY 0
 
-/* size to allocate for "bytes=[-]" HTTP Range value
- */
+/* size to allocate for "bytes=[-]" HTTP Range value */
 #define S3COMMS_MAX_RANGE_STRING_SIZE 128
 
+/* Size of buffers that hold hex representations of SHA256 digests */
+#define S3COMMS_SHA256_HEXSTR_LENGTH (SHA256_DIGEST_LENGTH * 2 + 1)
+
 /******************/
 /* Local Typedefs */
 /******************/
@@ -68,9 +74,9 @@
 /* Local Structures */
 /********************/
 
-/* struct s3r_datastruct
- * Structure passed to curl write callback
- * pointer to data region and record of bytes written (offset)
+/* Structure passed to curl write callback
+ *
+ * Pointer to data region and record of bytes written (offset)
  */
 struct s3r_datastruct {
     char  *data;
@@ -81,9 +87,24 @@ struct s3r_datastruct {
 /* Local Prototypes */
 /********************/
 
-size_t curlwritecallback(char *ptr, size_t size, size_t nmemb, void *userdata);
+static size_t H5FD__s3comms_curl_write_callback(char *ptr, size_t size, size_t nmemb, void *userdata);
+
+static herr_t H5FD__s3comms_s3r_configure_aws(s3r_t *handle, const H5FD_ros3_fapl_t *fa,
+                                              const char *fapl_token);
+
+static herr_t H5FD__s3comms_s3r_getsize(s3r_t *handle);
+
+static herr_t H5FD__s3comms_bytes_to_hex(char *dest, size_t dest_len, const unsigned char *msg,
+                                         size_t msg_len);
 
-herr_t H5FD_s3comms_s3r_getsize(s3r_t *handle);
+static herr_t H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, char *key_id,
+                                                     char *access_key, char *aws_region);
+
+static herr_t H5FD__s3comms_make_iso_8661_string(time_t time, char iso8601[ISO8601_SIZE]);
+
+static parsed_url_t *H5FD__s3comms_parse_url(const char *url);
+
+static herr_t H5FD__s3comms_free_purl(parsed_url_t *purl);
 
 /*********************/
 /* Package Variables */
@@ -102,50 +123,36 @@ herr_t H5FD_s3comms_s3r_getsize(s3r_t *handle);
 /*************/
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_curl_write_callback
  *
- * Function: curlwritecallback()
- *
- * Purpose:
- *
- *     Function called by CURL to write received data.
- *
- *     Writes bytes to `userdata`.
- *
- *     Internally manages number of bytes processed.
- *
- * Return:
- *
- *     - Number of bytes processed.
- *         - Should equal number of bytes passed to callback.
- *         - Failure will result in curl error: CURLE_WRITE_ERROR.
+ * Purpose:     Function called by CURL to write received data
  *
+ * Return:      Number of bytes processed
  *----------------------------------------------------------------------------
  */
-size_t
-curlwritecallback(char *ptr, size_t size, size_t nmemb, void *userdata)
+static size_t
+H5FD__s3comms_curl_write_callback(char *ptr, size_t size, size_t nmemb, void *userdata)
 {
-    struct s3r_datastruct *sds     = (struct s3r_datastruct *)userdata;
-    size_t                 product = (size * nmemb);
-    size_t                 written = 0;
+    struct s3r_datastruct *sds    = (struct s3r_datastruct *)userdata;
+    size_t                 nbytes = size * nmemb;
 
+    /* Write bytes and size to userdata/sds struct */
     if (size > 0) {
-        H5MM_memcpy(&(sds->data[sds->size]), ptr, product);
-        sds->size += product;
-        written = product;
+        H5MM_memcpy(&(sds->data[sds->size]), ptr, nbytes);
+        sds->size += nbytes;
     }
 
-    return written;
-} /* end curlwritecallback() */
+    return nbytes;
+} /* end H5FD__s3comms_curl_write_callback() */
 
 /*----------------------------------------------------------------------------
- *
- * Function: H5FD_s3comms_hrb_node_set()
+ * Function: H5FD__s3comms_hrb_node_set
  *
  * Purpose:
  *
  *     Create, insert, modify, and remove elements in a field node list.
  *
- *     `name` cannot be null; will return FAIL and list will be unaltered.
+ *     `name` cannot be NULL; will return FAIL and list will be unaltered.
  *
  *     Entries are accessed via the lowercase representation of their name:
  *     "Host", "host", and "hOSt" would all access the same node,
@@ -188,7 +195,7 @@ curlwritecallback(char *ptr, size_t size, size_t nmemb, void *userdata)
  *----------------------------------------------------------------------------
  */
 herr_t
-H5FD_s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value)
+H5FD__s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value)
 {
     size_t      i          = 0;
     char       *valuecpy   = NULL;
@@ -201,28 +208,25 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value)
     bool        is_looking = true;
     herr_t      ret_value  = SUCCEED;
 
-    FUNC_ENTER_NOAPI_NOINIT
+    FUNC_ENTER_PACKAGE
 
     if (name == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to operate on null name");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to operate on NULL name");
     namelen = strlen(name);
 
     /***********************
      * PREPARE ALL STRINGS *
      **********************/
 
-    /* copy and lowercase name
-     */
-    lowername = (char *)H5MM_malloc(sizeof(char) * (namelen + 1));
-    if (lowername == NULL)
-        HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for lowercase name copy.");
+    /* Copy and lowercase name */
+    if (NULL == (lowername = strdup(name)))
+        HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "cannot make space for lowercase name copy");
     for (i = 0; i < namelen; i++)
-        lowername[i] = (char)tolower((int)name[i]);
-    lowername[namelen] = 0;
+        lowername[i] = (char)tolower((int)lowername[i]);
 
     /* If value supplied, copy name, value, and concatenated "name: value".
      * If NULL, we will be removing a node or doing nothing, so no need for
-     * copies
+     * copies.
      */
     if (value != NULL) {
         int    ret      = 0;
@@ -230,35 +234,20 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value)
         size_t catlen   = namelen + valuelen + 2; /* +2 from ": " */
         size_t catwrite = catlen + 3;             /* 3 not 1 to quiet compiler warning */
 
-        namecpy = (char *)H5MM_malloc(sizeof(char) * (namelen + 1));
-        if (namecpy == NULL)
-            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for name copy.");
-        H5MM_memcpy(namecpy, name, (namelen + 1));
-
-        valuecpy = (char *)H5MM_malloc(sizeof(char) * (valuelen + 1));
-        if (valuecpy == NULL)
-            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for value copy.");
-        H5MM_memcpy(valuecpy, value, (valuelen + 1));
+        if (NULL == (namecpy = strdup(name)))
+            HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "cannot copy name");
+        if (NULL == (valuecpy = strdup(value)))
+            HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "cannot copy value");
 
-        nvcat = (char *)H5MM_malloc(sizeof(char) * catwrite);
-        if (nvcat == NULL)
-            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for concatenated string.");
+        if (NULL == (nvcat = (char *)H5MM_malloc(sizeof(char) * catwrite)))
+            HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "cannot make space for concatenated string");
         ret = snprintf(nvcat, catwrite, "%s: %s", name, value);
         if (ret < 0 || (size_t)ret > catlen)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot concatenate `%s: %s", name, value);
-        assert(catlen == strlen(nvcat));
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "cannot concatenate `%s: %s", name, value);
 
-        /* create new_node, should we need it
-         */
-        new_node = (hrb_node_t *)H5MM_malloc(sizeof(hrb_node_t));
-        if (new_node == NULL)
-            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for new set.");
-
-        new_node->name      = NULL;
-        new_node->value     = NULL;
-        new_node->cat       = NULL;
-        new_node->lowername = NULL;
-        new_node->next      = NULL;
+        /* Create new_node, should we need it */
+        if (NULL == (new_node = (hrb_node_t *)H5MM_calloc(sizeof(hrb_node_t))))
+            HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "cannot make space for new_node");
     }
 
     /***************
@@ -267,7 +256,7 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value)
 
     if (*L == NULL) {
         if (value == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "trying to remove node from empty list");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "trying to remove node from empty list");
         else {
             /*******************
              * CREATE NEW LIST *
@@ -283,13 +272,9 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value)
         }
     }
 
-    /* sanity-check pointer passed in
-     */
-    assert((*L) != NULL);
     node_ptr = (*L);
 
-    /* Check whether to modify/remove first node in list
-     */
+    /* Check whether to modify/remove first node in list */
     if (strcmp(lowername, node_ptr->lowername) == 0) {
 
         is_looking = false;
@@ -334,7 +319,7 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value)
         is_looking = false;
 
         if (value == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "trying to remove a node 'before' head");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "trying to remove a node 'before' head");
         else {
             /*******************
              * INSERT NEW HEAD *
@@ -359,7 +344,7 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value)
             is_looking = false;
 
             if (value == NULL)
-                HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "trying to remove absent node");
+                HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "trying to remove absent node");
             else {
                 /*******************
                  * APPEND NEW NODE *
@@ -378,7 +363,7 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value)
             is_looking = false;
 
             if (value == NULL)
-                HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "trying to remove absent node");
+                HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "trying to remove absent node");
             else {
                 /*******************
                  * INSERT NEW NODE *
@@ -445,159 +430,104 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value)
     } /* end while is_looking */
 
 done:
-    if (ret_value == FAIL) {
-        /* clean up */
-        if (nvcat != NULL)
-            H5MM_xfree(nvcat);
-        if (namecpy != NULL)
-            H5MM_xfree(namecpy);
-        if (lowername != NULL)
-            H5MM_xfree(lowername);
-        if (valuecpy != NULL)
-            H5MM_xfree(valuecpy);
-        if (new_node != NULL) {
-            H5MM_xfree(new_node);
-        }
+    if (ret_value < 0) {
+        H5MM_xfree(nvcat);
+        H5MM_xfree(namecpy);
+        H5MM_xfree(lowername);
+        H5MM_xfree(valuecpy);
+        H5MM_xfree(new_node);
     }
 
     FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_s3comms_hrb_node_set() */
+} /* end H5FD__s3comms_hrb_node_set() */
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_hrb_destroy
  *
- * Function: H5FD_s3comms_hrb_destroy()
- *
- * Purpose:
- *
- *    Destroy and free resources _directly_ associated with an HTTP Buffer.
- *
- *    Takes a pointer to pointer to the buffer structure.
- *    This allows for the pointer itself to be NULLed from within the call.
+ * Purpose:     Destroy and free resources associated with an HTTP buffer
  *
- *    If buffer or buffer pointer is NULL, there is no effect.
+ *              buf can be NULL
  *
- *    Headers list at `first_header` is not touched.
+ *              NOTE: The hrb_node_t list is not destroyed by this function
  *
- *    - Programmer should reuse or destroy `first_header` pointer
- *      (hrb_node_t *) as suits their purposes.
- *    - Recommend fetching prior to destroy()
- *      e.g., `reuse_node = hrb_to_die->first_header; destroy(hrb_to_die);`
- *      or maintaining an external reference.
- *    - Destroy node/list separately as appropriate
- *    - Failure to account for this will result in a memory leak.
- *
- * Return:  SUCCEED (can't fail)
+ * Return:      SUCCEED (can't fail)
  *----------------------------------------------------------------------------
  */
 herr_t
-H5FD_s3comms_hrb_destroy(hrb_t **_buf)
+H5FD__s3comms_hrb_destroy(hrb_t *buf)
 {
-    hrb_t *buf       = NULL;
-    herr_t ret_value = SUCCEED;
-
-    FUNC_ENTER_NOAPI_NOINIT_NOERR
-
-    if (_buf != NULL && *_buf != NULL) {
-        buf = *_buf;
+    FUNC_ENTER_PACKAGE_NOERR
 
+    if (buf != NULL) {
         H5MM_xfree(buf->verb);
         H5MM_xfree(buf->version);
         H5MM_xfree(buf->resource);
         H5MM_xfree(buf);
-        *_buf = NULL;
     }
 
-    FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_s3comms_hrb_destroy() */
+    FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5FD__s3comms_hrb_destroy() */
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_hrb_init_request
  *
- * Function: H5FD_s3comms_hrb_init_request()
- *
- * Purpose:
- *
- *     Create a new HTTP Request Buffer
- *
- *     All non-null arguments should be null-terminated strings.
+ * Purpose:     Create a new HTTP Request Buffer
  *
- *     If `verb` is NULL, defaults to "GET".
- *     If `http_version` is NULL, defaults to "HTTP/1.1".
+ *              All non-NULL arguments should be NUL-terminated strings.
  *
- *     `resource` cannot be NULL; should be string beginning with slash
- *     character ('/').
+ *              If `verb` is NULL, defaults to "GET".
+ *              If `http_version` is NULL, defaults to "HTTP/1.1".
  *
- *     All strings are copied into the structure, making them safe from
- *     modification in source strings.
- *
- * Return:
- *
- *     - SUCCESS: pointer to new `hrb_t`
- *     - FAILURE: `NULL`
+ *              `resource` cannot be NULL
  *
+ * Return:      SUCCESS:    pointer to new hrb_t
+ *              FAILURE:    NULL
  *----------------------------------------------------------------------------
  */
 hrb_t *
-H5FD_s3comms_hrb_init_request(const char *_verb, const char *_resource, const char *_http_version)
+H5FD__s3comms_hrb_init_request(const char *_verb, const char *_resource, const char *_http_version)
 {
     hrb_t *request   = NULL;
     char  *res       = NULL;
-    size_t reslen    = 0;
-    hrb_t *ret_value = NULL;
     char  *verb      = NULL;
-    size_t verblen   = 0;
     char  *vrsn      = NULL;
-    size_t vrsnlen   = 0;
+    hrb_t *ret_value = NULL;
 
-    FUNC_ENTER_NOAPI_NOINIT
+    FUNC_ENTER_PACKAGE
 
     if (_resource == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "resource string cannot be null.");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "resource string cannot be NULL");
 
-    /* populate valid NULLs with defaults */
+    /* Populate valid NULLs with defaults */
     if (_verb == NULL)
         _verb = "GET";
     if (_http_version == NULL)
         _http_version = "HTTP/1.1";
 
-    /* malloc space for and prepare structure */
-    request = (hrb_t *)H5MM_malloc(sizeof(hrb_t));
-    if (request == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, "no space for request structure");
-    request->body         = NULL;
-    request->body_len     = 0;
-    request->first_header = NULL;
-
-    /* malloc and copy strings for the structure */
-    reslen = strlen(_resource);
+    /* Allocate space for HTTP request buffer */
+    if (NULL == (request = (hrb_t *)H5MM_calloc(sizeof(hrb_t))))
+        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "no space for request structure");
 
+    /* Ensure the resource string starts with '/' */
     if (_resource[0] == '/') {
-        res = (char *)H5MM_malloc(sizeof(char) * (reslen + 1));
-        if (res == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, "no space for resource string");
-        H5MM_memcpy(res, _resource, (reslen + 1));
+        if (NULL == (res = strdup(_resource)))
+            HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "cannot copy resource string");
     }
     else {
-        res = (char *)H5MM_malloc(sizeof(char) * (reslen + 2));
-        if (res == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, "no space for resource string");
+        size_t reslen = strlen(_resource);
+
+        if (NULL == (res = (char *)H5MM_malloc(sizeof(char) * (reslen + 2))))
+            HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "no space for resource string");
         *res = '/';
         H5MM_memcpy((&res[1]), _resource, (reslen + 1));
-        assert((reslen + 1) == strlen(res));
-    } /* end if (else resource string not starting with '/') */
-
-    verblen = strlen(_verb) + 1;
-    verb    = (char *)H5MM_malloc(sizeof(char) * verblen);
-    if (verb == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "no space for verb string");
-    strncpy(verb, _verb, verblen);
-
-    vrsnlen = strlen(_http_version) + 1;
-    vrsn    = (char *)H5MM_malloc(sizeof(char) * vrsnlen);
-    if (vrsn == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "no space for http-version string");
-    strncpy(vrsn, _http_version, vrsnlen);
-
-    /* place new copies into structure */
+    }
+
+    if (NULL == (verb = strdup(_verb)))
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "cannot copy verb string");
+
+    if (NULL == (vrsn = strdup(_http_version)))
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "cannot copy http-version string");
+
     request->resource = res;
     request->verb     = verb;
     request->version  = vrsn;
@@ -605,208 +535,165 @@ H5FD_s3comms_hrb_init_request(const char *_verb, const char *_resource, const ch
     ret_value = request;
 
 done:
-    /* if there is an error, clean up after ourselves */
     if (ret_value == NULL) {
-        if (request != NULL)
-            H5MM_xfree(request);
-        if (vrsn != NULL)
-            H5MM_xfree(vrsn);
-        if (verb != NULL)
-            H5MM_xfree(verb);
-        if (res != NULL)
-            H5MM_xfree(res);
+        H5MM_xfree(request);
+        H5MM_xfree(vrsn);
+        H5MM_xfree(verb);
+        H5MM_xfree(res);
     }
 
     FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_s3comms_hrb_init_request() */
+} /* end H5FD__s3comms_hrb_init_request() */
 
 /****************************************************************************
  * S3R FUNCTIONS
  ****************************************************************************/
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_s3r_close
  *
- * Function: H5FD_s3comms_s3r_close()
- *
- * Purpose:
+ * Purpose:     Close communications through given S3 Request Handle (s3r_t)
+ *              and clean up associated resources
  *
- *     Close communications through given S3 Request Handle (`s3r_t`)
- *     and clean up associated resources.
- *
- * Return:  SUCCEED/FAIL
+ * Return:      SUCCEED/FAIL
  *----------------------------------------------------------------------------
  */
 herr_t
-H5FD_s3comms_s3r_close(s3r_t *handle)
+H5FD__s3comms_s3r_close(s3r_t *handle)
 {
     herr_t ret_value = SUCCEED;
 
-    FUNC_ENTER_NOAPI_NOINIT
+    FUNC_ENTER_PACKAGE
 
     if (handle == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle cannot be null.");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle cannot be NULL");
 
     curl_easy_cleanup(handle->curlhandle);
 
     H5MM_xfree(handle->secret_id);
-    H5MM_xfree(handle->region);
+    H5MM_xfree(handle->aws_region);
     H5MM_xfree(handle->signing_key);
     H5MM_xfree(handle->token);
+    H5MM_xfree(handle->http_verb);
 
-    assert(handle->httpverb != NULL);
-    H5MM_xfree(handle->httpverb);
-
-    if (FAIL == H5FD_s3comms_free_purl(handle->purl))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to release parsed url structure");
+    if (H5FD__s3comms_free_purl(handle->purl) < 0)
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to release parsed url structure");
 
     H5MM_xfree(handle);
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
-} /* H5FD_s3comms_s3r_close */
+} /* H5FD__s3comms_s3r_close */
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_s3r_get_filesize
  *
- * Function: H5FD_s3comms_s3r_get_filesize()
- *
- * Purpose:
- *
- *     Retrieve the filesize of an open request handle.
- *
- *     Wrapper "getter" to hide implementation details.
- *
- *
- * Return:
- *
- *     - SUCCESS: size of file, in bytes, if handle is valid.
- *     - FAILURE: 0, if handle is NULL or undefined.
+ * Purpose:     Retrieve the filesize of an open request handle
  *
+ * Return:      SUCCEED/FAIL
  *----------------------------------------------------------------------------
  */
 size_t
-H5FD_s3comms_s3r_get_filesize(s3r_t *handle)
+H5FD__s3comms_s3r_get_filesize(s3r_t *handle)
 {
     size_t ret_value = 0;
 
-    FUNC_ENTER_NOAPI_NOINIT_NOERR
+    FUNC_ENTER_PACKAGE_NOERR
 
     if (handle != NULL)
         ret_value = handle->filesize;
 
     FUNC_LEAVE_NOAPI(ret_value)
-} /* H5FD_s3comms_s3r_get_filesize */
+} /* H5FD__s3comms_s3r_get_filesize */
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_s3r_getsize
  *
- * Function: H5FD_s3comms_s3r_getsize()
- *
- * Purpose:
- *
- *    Get the number of bytes of handle's target resource.
- *
- *    Sets handle and curlhandle with to enact an HTTP HEAD request on file,
- *    and parses received headers to extract "Content-Length" from response
- *    headers, storing file size at `handle->filesize`.
- *
- *    Critical step in opening (initiating) an `s3r_t` handle.
- *
- *    Wraps `s3r_read()`.
- *    Sets curlhandle to write headers to a temporary buffer (using extant
- *    write callback) and provides no buffer for body.
- *
- *    Upon exit, unsets HTTP HEAD settings from curl handle, returning to
- *    initial state. In event of error, curl handle state is undefined and is
- *    not to be trusted.
- *
- * Return:
- *
- *     - SUCCESS: `SUCCEED`
- *     - FAILURE: `FAIL`
+ * Purpose:     Get the number of bytes of handle's target resource
  *
+ * Return:      SUCCEED/FAIL
  *----------------------------------------------------------------------------
  */
-herr_t
-H5FD_s3comms_s3r_getsize(s3r_t *handle)
+static herr_t
+H5FD__s3comms_s3r_getsize(s3r_t *handle)
 {
-    uintmax_t             content_length = 0;
-    CURL                 *curlh          = NULL;
-    char                 *end            = NULL;
-    char                 *headerresponse = NULL;
-    struct s3r_datastruct sds            = {NULL, 0};
-    char                 *start          = NULL;
-    herr_t                ret_value      = SUCCEED;
+    uintmax_t             content_length  = 0;
+    CURL                 *curlh           = NULL;
+    char                 *end             = NULL;
+    char                 *header_response = NULL;
+    struct s3r_datastruct sds             = {NULL, 0};
+    char                 *start           = NULL;
+    herr_t                ret_value       = SUCCEED;
 
-    FUNC_ENTER_NOAPI_NOINIT
+    FUNC_ENTER_PACKAGE
 
-    if (handle == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle cannot be null.");
-    if (handle->curlhandle == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle has bad (null) curlhandle.");
+    assert(handle);
+    assert(handle->curlhandle);
+    assert(handle->http_verb);
 
     /********************
      * PREPARE FOR HEAD *
      ********************/
 
+    /* Set handle and curlhandle to perform an HTTP HEAD request on file */
+
     curlh = handle->curlhandle;
-    if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_NOBODY, 1L))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "error while setting CURL option (CURLOPT_NOBODY).");
+    if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_NOBODY, 1))
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "error while setting CURL option (CURLOPT_NOBODY)");
 
     if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HEADERDATA, &sds))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "error while setting CURL option (CURLOPT_HEADERDATA).");
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "error while setting CURL option (CURLOPT_HEADERDATA)");
 
-    assert(handle->httpverb == NULL);
-    handle->httpverb = (char *)H5MM_malloc(sizeof(char) * 16);
-    if (handle->httpverb == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "unable to allocate space for S3 request HTTP verb");
-    H5MM_memcpy(handle->httpverb, "HEAD", 5);
+    strcpy(handle->http_verb, "HEAD");
 
-    headerresponse = (char *)H5MM_malloc(sizeof(char) * CURL_MAX_HTTP_HEADER);
-    if (headerresponse == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "unable to allocate space for curl header response");
-    sds.data = headerresponse;
+    if (NULL == (header_response = (char *)H5MM_malloc(sizeof(char) * CURL_MAX_HTTP_HEADER)))
+        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate space for curl header response");
+    sds.data = header_response;
 
     /*******************
      * PERFORM REQUEST *
      *******************/
 
-    /* these parameters fetch the entire file,
-     * but, with a NULL destination and NOBODY and HEADERDATA supplied above,
-     * only http metadata will be sent by server and recorded by s3comms
+    /* These parameters fetch the entire file, but, with a NULL destination and
+     * NOBODY and HEADERDATA supplied above, only http metadata will be sent by
+     * the server and recorded by s3comms
      */
-    if (FAIL == H5FD_s3comms_s3r_read(handle, 0, 0, NULL))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem in reading during getsize.");
+    if (H5FD__s3comms_s3r_read(handle, 0, 0, NULL) < 0)
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem in reading during getsize");
 
     if (sds.size > CURL_MAX_HTTP_HEADER)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "HTTP metadata buffer overrun");
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "HTTP metadata buffer overrun");
     else if (sds.size == 0)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "No HTTP metadata");
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "No HTTP metadata");
 
     /******************
      * PARSE RESPONSE *
      ******************/
 
-    start = HDstrcasestr(headerresponse, "\r\nContent-Length: ");
-    if (start == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not find \"Content-Length\" in response.");
+    /* Parse received headers to extract "Content-Length" from response
+     * headers, storing file size at handle->filesize.
+     */
+
+    if (NULL == (start = HDstrcasestr(header_response, "\r\nContent-Length: ")))
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not find \"Content-Length\" in response");
 
     /* move "start" to beginning of value in line; find end of line */
     start = start + strlen("\r\nContent-Length: ");
     end   = strstr(start, "\r\n");
     if (end == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not find end of content length line");
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not find end of content length line");
 
-    /* place null terminator at end of numbers
-     */
+    /* place NUL terminator at end of numbers */
     *end = '\0';
 
     content_length = strtoumax((const char *)start, NULL, 0);
     if (UINTMAX_MAX > SIZE_MAX && content_length > SIZE_MAX)
-        HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "content_length overflows size_t");
+        HGOTO_ERROR(H5E_VFL, H5E_OVERFLOW, FAIL, "content_length overflows size_t");
 
-    if (content_length == 0 || errno == ERANGE) /* errno set by strtoumax*/
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
-                    "could not convert found \"Content-Length\" response (\"%s\")",
-                    start); /* range is null-terminated, remember */
+    /* errno set by strtoumax */
+    if (content_length == 0 || errno == ERANGE)
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL,
+                    "could not convert found \"Content-Length\" response (\"%s\")", start);
 
     handle->filesize = (size_t)content_length;
 
@@ -819,207 +706,194 @@ H5FD_s3comms_s3r_getsize(s3r_t *handle)
      * UNDO HEAD SETTINGS *
      **********************/
 
-    if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_NOBODY, NULL))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "error while setting CURL option (CURLOPT_NOBODY).");
+    if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_NOBODY, 0))
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "error while setting CURL option (CURLOPT_NOBODY)");
 
+    /* Unset HTTP HEAD settings from curl handle, returning to initial state */
     if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HEADERDATA, NULL))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "error while setting CURL option (CURLOPT_HEADERDATA).");
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "error while setting CURL option (CURLOPT_HEADERDATA)");
 
+    strcpy(handle->http_verb, "GET");
 done:
-    H5MM_xfree(headerresponse);
+    H5MM_xfree(header_response);
 
     FUNC_LEAVE_NOAPI(ret_value)
-} /* H5FD_s3comms_s3r_getsize */
+} /* H5FD__s3comms_s3r_getsize */
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_s3r_open
  *
- * Function: H5FD_s3comms_s3r_open()
- *
- * Purpose:
+ * Purpose:     Logically open a file hosted on S3
  *
- *     Logically 'open' a file hosted on S3.
- *
- *     - create new Request Handle
- *     - copy supplied url
- *     - copy authentication info if supplied
- *     - create CURL handle
- *     - fetch size of file
- *         - connect with server and execute HEAD request
- *     - return request handle ready for reads
- *
- *     To use 'default' port to connect, `port` should be 0.
- *
- *     To prevent AWS4 authentication, pass null pointer to `region`, `id`,
- *     and `signing_key`.
- *
- *     Uses `H5FD_s3comms_parse_url()` to validate and parse url input.
- *
- * Return:
- *
- *     - SUCCESS: Pointer to new request handle.
- *     - FAILURE: NULL
- *         - occurs if:
- *             - authentication strings are inconsistent
- *             - must _all_ be null, or have at least `region` and `id`
- *             - url is NULL (no filename)
- *             - unable to parse url (malformed?)
- *             - error while performing `getsize()`
+ *              fa can be NULL (implies no authentication)
+ *              fapl_token can be NULL
  *
+ * Return:      SUCCESS:    Pointer to new request handle.
+ *              FAILURE:    NULL
  *----------------------------------------------------------------------------
  */
 s3r_t *
-H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const unsigned char *signing_key,
-                      const char *token)
+H5FD__s3comms_s3r_open(const char *url, const H5FD_ros3_fapl_t *fa, const char *fapl_token)
 {
-    size_t        tmplen    = 0;
-    CURL         *curlh     = NULL;
-    s3r_t        *handle    = NULL;
-    parsed_url_t *purl      = NULL;
-    s3r_t        *ret_value = NULL;
+    CURL  *curlh     = NULL;
+    s3r_t *handle    = NULL;
+    s3r_t *ret_value = NULL;
 
-    FUNC_ENTER_NOAPI_NOINIT
-
-    if (url == NULL || url[0] == '\0')
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "url cannot be null.");
+    FUNC_ENTER_PACKAGE
 
-    if (FAIL == H5FD_s3comms_parse_url(url, &purl))
-        /* probably a malformed url, but could be internal error */
-        HGOTO_ERROR(H5E_ARGS, H5E_CANTCREATE, NULL, "unable to create parsed url structure");
+    if (url == NULL)
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "url cannot be NULL");
+    if (url[0] == '\0')
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "url cannot be an empty string");
 
-    assert(purl != NULL); /* if above passes, this must be true */
+    /* Create handle and set fields */
+    if (NULL == (handle = (s3r_t *)H5MM_calloc(sizeof(s3r_t))))
+        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "could not allocate space for handle");
 
-    handle = (s3r_t *)H5MM_malloc(sizeof(s3r_t));
-    if (handle == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, "could not malloc space for handle.");
+    if (NULL == (handle->http_verb = (char *)H5MM_calloc(sizeof(char) * 16)))
+        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate space for S3 request HTTP verb");
 
-    handle->purl        = purl;
-    handle->filesize    = 0;
-    handle->region      = NULL;
-    handle->secret_id   = NULL;
-    handle->signing_key = NULL;
-    handle->token       = NULL;
-    handle->httpverb    = NULL;
+    /* Parse URL */
+    if (NULL == (handle->purl = H5FD__s3comms_parse_url(url)))
+        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "could not allocate and create parsed URL");
 
     /*************************************
      * RECORD AUTHENTICATION INFORMATION *
      *************************************/
 
-    if ((region != NULL && *region != '\0') || (id != NULL && *id != '\0') || (signing_key != NULL) ||
-        (token != NULL)) {
-
-        /* if one exists, all three must exist */
-        if (region == NULL || region[0] == '\0')
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "region cannot be null.");
-        if (id == NULL || id[0] == '\0')
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "secret id cannot be null.");
-        if (signing_key == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "signing key cannot be null.");
-        if (token == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "token cannot be null.");
-
-        /* copy strings */
-        tmplen         = strlen(region) + 1;
-        handle->region = (char *)H5MM_malloc(sizeof(char) * tmplen);
-        if (handle->region == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle region copy.");
-        H5MM_memcpy(handle->region, region, tmplen);
-
-        tmplen            = strlen(id) + 1;
-        handle->secret_id = (char *)H5MM_malloc(sizeof(char) * tmplen);
-        if (handle->secret_id == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle ID copy.");
-        H5MM_memcpy(handle->secret_id, id, tmplen);
-
-        tmplen              = SHA256_DIGEST_LENGTH;
-        handle->signing_key = (unsigned char *)H5MM_malloc(sizeof(unsigned char) * tmplen);
-        if (handle->signing_key == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle key copy.");
-        H5MM_memcpy(handle->signing_key, signing_key, tmplen);
-
-        tmplen        = strlen(token) + 1;
-        handle->token = (char *)H5MM_malloc(sizeof(char) * tmplen);
-        if (handle->token == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle token copy.");
-        H5MM_memcpy(handle->token, token, tmplen);
-    } /* if authentication information provided */
+    if (fa && fa->authenticate)
+        if (H5FD__s3comms_s3r_configure_aws(handle, fa, fapl_token) < 0)
+            HGOTO_ERROR(H5E_ARGS, H5E_CANTINIT, NULL, "failure to configure AWS");
 
-    /************************
-     * INITIATE CURL HANDLE *
-     ************************/
+    /**************************
+     * INITIALIZE CURL HANDLE *
+     **************************/
 
-    curlh = curl_easy_init();
-    if (curlh == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem creating curl easy handle!");
+    if (NULL == (curlh = curl_easy_init()))
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "problem creating curl easy handle!");
 
     if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPGET, 1L))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "error while setting CURL option (CURLOPT_HTTPGET).");
-
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "error while setting CURL option (CURLOPT_HTTPGET)");
     if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "error while setting CURL option (CURLOPT_HTTP_VERSION).");
-
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "error while setting CURL option (CURLOPT_HTTP_VERSION)");
     if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_FAILONERROR, 1L))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "error while setting CURL option (CURLOPT_FAILONERROR).");
-
-    if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_WRITEFUNCTION, curlwritecallback))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "error while setting CURL option (CURLOPT_WRITEFUNCTION).");
-
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "error while setting CURL option (CURLOPT_FAILONERROR)");
+    if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_WRITEFUNCTION, H5FD__s3comms_curl_write_callback))
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "error while setting CURL option (CURLOPT_WRITEFUNCTION)");
     if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_URL, url))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "error while setting CURL option (CURLOPT_URL).");
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "error while setting CURL option (CURLOPT_URL)");
 
 #if S3COMMS_CURL_VERBOSITY > 1
-    /* CURL will print (to stdout) information for each operation
-     */
+    /* CURL will print (to stdout) information for each operation */
     curl_easy_setopt(curlh, CURLOPT_VERBOSE, 1L);
 #endif
 
     handle->curlhandle = curlh;
 
-    /*******************
-     * OPEN CONNECTION *
-     * * * * * * * * * *
-     *  GET FILE SIZE  *
-     *******************/
-
-    if (FAIL == H5FD_s3comms_s3r_getsize(handle))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem in H5FD_s3comms_s3r_getsize.");
-
-    /*********************
-     * FINAL PREPARATION *
-     *********************/
+    /***************
+     *  FINISH UP  *
+     ***************/
 
-    assert(handle->httpverb != NULL);
-    H5MM_memcpy(handle->httpverb, "GET", 4);
+    /* Get the S3 object's size. This is the only time we touch the S3 object
+     * (and thus ensure it exists) during the VFD's open callback.
+     */
+    if (H5FD__s3comms_s3r_getsize(handle) < 0)
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "problem in H5FD__s3comms_s3r_getsize");
 
     ret_value = handle;
 
 done:
     if (ret_value == NULL) {
-        if (curlh != NULL)
-            curl_easy_cleanup(curlh);
-        if (FAIL == H5FD_s3comms_free_purl(purl))
-            HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "unable to free parsed url structure");
+        curl_easy_cleanup(curlh);
+
         if (handle != NULL) {
-            H5MM_xfree(handle->region);
+            if (H5FD__s3comms_free_purl(handle->purl) < 0)
+                HDONE_ERROR(H5E_VFL, H5E_CANTFREE, NULL, "unable to free parsed url structure");
+
+            H5MM_xfree(handle->aws_region);
             H5MM_xfree(handle->secret_id);
             H5MM_xfree(handle->signing_key);
             H5MM_xfree(handle->token);
-            if (handle->httpverb != NULL)
-                H5MM_xfree(handle->httpverb);
+            H5MM_xfree(handle->http_verb);
             H5MM_xfree(handle);
         }
     }
 
     FUNC_LEAVE_NOAPI(ret_value)
-} /* H5FD_s3comms_s3r_open */
+} /* H5FD__s3comms_s3r_open */
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_s3r_configure_aws
  *
- * Function: H5FD_s3comms_s3r_read()
+ * Purpose:     Add AWS configuration and authentication info to an s3r_t
+ *              handle
  *
- * Purpose:
+ * Return:      SUCCEED/FAIL
+ *----------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__s3comms_s3r_configure_aws(s3r_t *handle, const H5FD_ros3_fapl_t *fa, const char *fapl_token)
+{
+    uint8_t signing_key[SHA256_DIGEST_LENGTH];
+    char    iso8601[ISO8601_SIZE]; /* ISO-8601 time string */
+
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_PACKAGE
+
+    /* These all need to exist to authenticate */
+    if (fa->aws_region[0] == '\0')
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "region cannot be NULL");
+    if (fa->secret_id[0] == '\0')
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "secret id cannot be NULL");
+    if (fa->secret_key[0] == '\0')
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "signing key cannot be NULL");
+
+    /* Copy strings into the s3r_t handle */
+    if (NULL == (handle->aws_region = strdup(fa->aws_region)))
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not copy AWS region");
+    if (NULL == (handle->secret_id = strdup(fa->secret_id)))
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not copy secret_id");
+
+    /* SIGNING KEY */
+
+    /* Get the current time in ISO-8601 format */
+    if (H5FD__s3comms_make_iso_8661_string(time(NULL), iso8601) < 0)
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not construct ISO-8601 string");
+
+    /* Compute signing key (part of AWS/S3 REST API). Can be re-used by
+     * user/key for 7 days after creation.
+     */
+    if (H5FD__s3comms_make_aws_signing_key(signing_key, (const char *)fa->secret_key,
+                                           (const char *)fa->aws_region, (const char *)iso8601) < 0)
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem while computing signing key");
+
+    /* Copy signing key (not a string) */
+    if (NULL == (handle->signing_key = (uint8_t *)H5MM_malloc(sizeof(uint8_t) * SHA256_DIGEST_LENGTH)))
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not allocate space for handle key");
+    H5MM_memcpy(handle->signing_key, signing_key, SHA256_DIGEST_LENGTH);
+
+    /* TOKEN */
+
+    if (fapl_token) {
+        if (NULL == (handle->token = strdup(fapl_token)))
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not copy token");
+    }
+    else {
+        if (NULL == (handle->token = strdup("")))
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not copy empty token");
+    }
+
+done:
+    /* Cleanup is handled when the s3r_t handle is cleaned up */
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD__s3comms_s3r_configure_aws() */
+
+/*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_s3r_read
  *
- *     Read file pointed to by request handle, writing specified
- *     `offset` .. `offset + len` bytes to buffer `dest`.
+ * Purpose:     Read file pointed to by request handle, writing specified
+ *              offset .. (offset + len) bytes to buffer dest
  *
  *     If `len` is 0, reads entirety of file starting at `offset`.
  *     If `offset` and `len` are both 0, reads entire file.
@@ -1043,46 +917,40 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const
  *       conjunction with CURLOPT_NOBODY to preempt transmission of file data
  *       from server.
  *
- * Return:
- *
- *     - SUCCESS: `SUCCEED`
- *     - FAILURE: `FAIL`
- *
+ * Return:      SUCCEED/FAIL
  *----------------------------------------------------------------------------
  */
 herr_t
-H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
+H5FD__s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
 {
-    CURL              *curlh         = NULL;
-    CURLcode           p_status      = CURLE_OK;
-    struct curl_slist *curlheaders   = NULL;
-    hrb_node_t        *headers       = NULL;
-    hrb_node_t        *node          = NULL;
-    struct tm         *now           = NULL;
-    char              *rangebytesstr = NULL;
-    hrb_t             *request       = NULL;
-    int                ret           = 0; /* working variable to check  */
-                                          /* return value of snprintf  */
+    CURL                  *curlh          = NULL;
+    CURLcode               p_status       = CURLE_OK;
+    struct curl_slist     *curlheaders    = NULL;
+    hrb_node_t            *headers        = NULL;
+    hrb_node_t            *node           = NULL;
+    char                  *rangebytesstr  = NULL;
+    hrb_t                 *request        = NULL;
     char                  *authorization  = NULL;
     char                  *buffer1        = NULL;
     char                  *signed_headers = NULL;
     struct s3r_datastruct *sds            = NULL;
+    int                    ret            = 0;
     herr_t                 ret_value      = SUCCEED;
 
-    FUNC_ENTER_NOAPI_NOINIT
+    FUNC_ENTER_PACKAGE
 
     /**************************************
      * ABSOLUTELY NECESSARY SANITY-CHECKS *
      **************************************/
 
     if (handle == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle cannot be null.");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle cannot be NULL");
     if (handle->curlhandle == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle has bad (null) curlhandle.");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle has bad (NULL) curlhandle");
     if (handle->purl == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle has bad (null) url.");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle has bad (NULL) url");
     if (offset > handle->filesize || (len + offset) > handle->filesize)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to read past EoF");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to read past EOF");
 
     curlh = handle->curlhandle;
 
@@ -1091,15 +959,14 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
      *********************/
 
     if (dest != NULL) {
-        sds = (struct s3r_datastruct *)H5MM_malloc(sizeof(struct s3r_datastruct));
-        if (sds == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "could not malloc destination datastructure.");
+        if (NULL == (sds = (struct s3r_datastruct *)H5MM_malloc(sizeof(struct s3r_datastruct))))
+            HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "could not malloc destination datastructure");
 
         sds->data = (char *)dest;
         sds->size = 0;
         if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_WRITEDATA, sds))
-            HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL,
-                        "error while setting CURL option (CURLOPT_WRITEDATA).");
+            HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL,
+                        "error while setting CURL option (CURLOPT_WRITEDATA)");
     }
 
     /*********************
@@ -1107,25 +974,23 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
      *********************/
 
     if (len > 0) {
-        rangebytesstr = (char *)H5MM_malloc(sizeof(char) * (S3COMMS_MAX_RANGE_STRING_SIZE + 1));
-        if (rangebytesstr == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "could not malloc range format string.");
+        if (NULL == (rangebytesstr = (char *)H5MM_malloc(sizeof(char) * (S3COMMS_MAX_RANGE_STRING_SIZE + 1))))
+            HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "could not malloc range format string");
         ret = snprintf(rangebytesstr, (S3COMMS_MAX_RANGE_STRING_SIZE), "bytes=%" PRIuHADDR "-%" PRIuHADDR,
                        offset, offset + len - 1);
         if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format HTTP Range value");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to format HTTP Range value");
     }
     else if (offset > 0) {
-        rangebytesstr = (char *)H5MM_malloc(sizeof(char) * (S3COMMS_MAX_RANGE_STRING_SIZE + 1));
-        if (rangebytesstr == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "could not malloc range format string.");
+        if (NULL == (rangebytesstr = (char *)H5MM_malloc(sizeof(char) * (S3COMMS_MAX_RANGE_STRING_SIZE + 1))))
+            HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "could not malloc range format string.");
         ret = snprintf(rangebytesstr, (S3COMMS_MAX_RANGE_STRING_SIZE), "bytes=%" PRIuHADDR "-", offset);
         if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format HTTP Range value");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to format HTTP Range value");
     }
 
 #if S3COMMS_CURL_VERBOSITY > 0
-    fprintf(stdout, "%s: Bytes %" PRIuHADDR " - %" PRIuHADDR ", Request Size: %zu\n", handle->httpverb,
+    fprintf(stdout, "%s: Bytes %" PRIuHADDR " - %" PRIuHADDR ", Request Size: %zu\n", handle->http_verb,
             offset, offset + len - 1, len);
     fflush(stdout);
 #endif
@@ -1147,15 +1012,13 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
 
             if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_RANGE, bytesrange_ptr))
                 HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL,
-                            "error while setting CURL option (CURLOPT_RANGE). ");
+                            "error while setting CURL option (CURLOPT_RANGE)");
         }
     }
     else {
-        /* authenticate request
-         */
-        authorization = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + 1);
-        if (authorization == NULL)
-            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for authorization variable.");
+        unsigned char md[SHA256_DIGEST_LENGTH];
+        unsigned int  md_len = SHA256_DIGEST_LENGTH;
+
         /*   4608 := approximate max length...
          *     67 
@@ -1167,120 +1030,131 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
          * + 4096 
          */
         char buffer2[256 + 1]; /* -> String To Sign -> Credential */
-        char iso8601now[ISO8601_SIZE];
-        buffer1 = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN +
-                                      1); /* -> Canonical Request -> Signature */
+        char iso8601[ISO8601_SIZE];
+
+        /* Authenticate request */
+        authorization = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + 1);
+        if (authorization == NULL)
+            HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "cannot make space for authorization variable");
+
+        /* -> Canonical Request -> Signature */
+        buffer1 = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + 1);
         if (buffer1 == NULL)
-            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for buffer1 variable.");
+            HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "cannot make space for buffer1 variable");
         signed_headers = (char *)H5MM_malloc(48 + H5FD_ROS3_MAX_SECRET_KEY_LEN + 1);
         if (signed_headers == NULL)
-            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for signed_headers variable.");
-        /* should be large enough for nominal listing:
+            HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "cannot make space for signed_headers variable");
+
+        /* Should be large enough for nominal listing:
          * "host;range;x-amz-content-sha256;x-amz-date;x-amz-security-token"
          * + '\0', with "range;" and/or "x-amz-security-token" possibly absent
          */
 
-        /* zero start of strings */
-        authorization[0]  = 0;
-        buffer1[0]        = 0;
-        buffer2[0]        = 0;
-        iso8601now[0]     = 0;
-        signed_headers[0] = 0;
+        /* Zero start of strings */
+        authorization[0]  = '\0';
+        buffer1[0]        = '\0';
+        buffer2[0]        = '\0';
+        iso8601[0]        = '\0';
+        signed_headers[0] = '\0';
 
         /**** VERIFY INFORMATION EXISTS ****/
 
-        if (handle->region == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null region.");
+        if (handle->aws_region == NULL)
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "handle must have non-NULL region");
         if (handle->secret_id == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null secret_id.");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "handle must have non-NULL secret_id");
         if (handle->signing_key == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null signing_key.");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "handle must have non-NULL signing_key");
         if (handle->token == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null token.");
-        if (handle->httpverb == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null httpverb.");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "handle must have non-NULL token");
+        if (handle->http_verb == NULL)
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "handle must have non-NULL http_verb");
         if (handle->purl->host == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null host.");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "handle must have non-NULL host");
         if (handle->purl->path == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null resource.");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "handle must have non-NULL resource");
 
         /**** CREATE HTTP REQUEST STRUCTURE (hrb_t) ****/
 
-        request = H5FD_s3comms_hrb_init_request((const char *)handle->httpverb,
-                                                (const char *)handle->purl->path, "HTTP/1.1");
+        request = H5FD__s3comms_hrb_init_request((const char *)handle->http_verb,
+                                                 (const char *)handle->purl->path, "HTTP/1.1");
         if (request == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not allocate hrb_t request.");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not allocate hrb_t request");
 
-        now = gmnow();
-        if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1))
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not format ISO8601 time.");
+        /* Get a time string for the current time in ISO-8601 format */
+        if (H5FD__s3comms_make_iso_8661_string(time(NULL), iso8601) < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not format ISO-8601 time");
 
-        if (FAIL == H5FD_s3comms_hrb_node_set(&headers, "x-amz-date", (const char *)iso8601now))
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set x-amz-date header");
+        if (H5FD__s3comms_hrb_node_set(&headers, "x-amz-date", (const char *)iso8601) < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to set x-amz-date header");
         if (headers == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list.");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem building headers list");
 
-        if (FAIL == H5FD_s3comms_hrb_node_set(&headers, "x-amz-content-sha256", (const char *)EMPTY_SHA256))
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set x-amz-content-sha256 header");
+        if (H5FD__s3comms_hrb_node_set(&headers, "x-amz-content-sha256", (const char *)EMPTY_SHA256) < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to set x-amz-content-sha256 header");
         if (headers == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list.");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem building headers list");
 
         if (strlen((const char *)handle->token) > 0) {
-            if (FAIL ==
-                H5FD_s3comms_hrb_node_set(&headers, "x-amz-security-token", (const char *)handle->token))
-                HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set x-amz-security-token header");
+            if (H5FD__s3comms_hrb_node_set(&headers, "x-amz-security-token", (const char *)handle->token) < 0)
+                HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to set x-amz-security-token header");
             if (headers == NULL)
-                HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list.");
+                HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem building headers list");
         }
 
         if (rangebytesstr != NULL) {
-            if (FAIL == H5FD_s3comms_hrb_node_set(&headers, "Range", rangebytesstr))
-                HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set range header");
+            if (H5FD__s3comms_hrb_node_set(&headers, "Range", rangebytesstr) < 0)
+                HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to set range header");
             if (headers == NULL)
-                HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list.");
+                HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem building headers list");
         }
 
-        if (FAIL == H5FD_s3comms_hrb_node_set(&headers, "Host", handle->purl->host))
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set host header");
+        if (H5FD__s3comms_hrb_node_set(&headers, "Host", handle->purl->host) < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to set host header");
         if (headers == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list.");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem building headers list");
 
         request->first_header = headers;
 
         /**** COMPUTE AUTHORIZATION ****/
 
         /* buffer1 -> canonical request */
-        if (FAIL == H5FD_s3comms_aws_canonical_request(buffer1, 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN,
-                                                       signed_headers, 48 + H5FD_ROS3_MAX_SECRET_TOK_LEN,
-                                                       request)) {
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad canonical request");
+        if (H5FD__s3comms_make_aws_canonical_request(buffer1, 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN,
+                                                     signed_headers, 48 + H5FD_ROS3_MAX_SECRET_TOK_LEN,
+                                                     request) < 0) {
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "bad canonical request");
         }
         /* buffer2->string-to-sign */
-        if (FAIL == H5FD_s3comms_tostringtosign(buffer2, buffer1, iso8601now, handle->region))
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad string-to-sign");
+        if (H5FD__s3comms_make_aws_stringtosign(buffer2, buffer1, iso8601, handle->aws_region) < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "bad string-to-sign");
+
         /* buffer1 -> signature */
-        if (FAIL == H5FD_s3comms_HMAC_SHA256(handle->signing_key, SHA256_DIGEST_LENGTH, buffer2,
-                                             strlen(buffer2), buffer1))
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad signature");
+        HMAC(EVP_sha256(), handle->signing_key, SHA256_DIGEST_LENGTH, (const unsigned char *)buffer2,
+             strlen(buffer2), md, &md_len);
+        if (H5FD__s3comms_bytes_to_hex(buffer1, 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + 1,
+                                       (const unsigned char *)md, (size_t)md_len) < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not convert to hex string.");
+
+        /* Trim to yyyyMMDD */
+        iso8601[8] = 0;
 
-        iso8601now[8] = 0; /* trim to yyyyMMDD */
-        ret = S3COMMS_FORMAT_CREDENTIAL(buffer2, handle->secret_id, iso8601now, handle->region, "s3");
+        ret = S3COMMS_FORMAT_CREDENTIAL(buffer2, handle->secret_id, iso8601, handle->aws_region, "s3");
         if (ret == 0 || ret >= S3COMMS_MAX_CREDENTIAL_SIZE)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 credential string");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to format aws4 credential string");
 
         ret = snprintf(authorization, 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN,
                        "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", buffer2,
                        signed_headers, buffer1);
         if (ret <= 0 || ret >= 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 authorization string");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to format aws4 authorization string");
 
-        /* append authorization header to http request buffer */
-        if (H5FD_s3comms_hrb_node_set(&headers, "Authorization", (const char *)authorization) == FAIL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set Authorization header");
+        /* Append authorization header to http request buffer */
+        if (H5FD__s3comms_hrb_node_set(&headers, "Authorization", (const char *)authorization) < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to set Authorization header");
         if (headers == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list.");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem building headers list");
 
-        /* update hrb's "first header" pointer */
+        /* Update hrb's "first header" pointer */
         request->first_header = headers;
 
         /**** SET CURLHANDLE HTTP HEADERS FROM GENERATED DATA ****/
@@ -1289,19 +1163,18 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
         while (node != NULL) {
             curlheaders = curl_slist_append(curlheaders, (const char *)node->cat);
             if (curlheaders == NULL)
-                HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not append header to curl slist.");
+                HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not append header to curl slist");
             node = node->next;
         }
 
-        /* sanity-check */
+        /* Sanity-check */
         if (curlheaders == NULL)
             /* above loop was probably never run */
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "curlheaders was never populated.");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "curlheaders was never populated");
 
-        /* finally, set http headers in curl handle */
+        /* Finally, set http headers in curl handle */
         if (curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, curlheaders) != CURLE_OK)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
-                        "error while setting CURL option (CURLOPT_HTTPHEADER).");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "error while setting CURL option (CURLOPT_HTTPHEADER)");
     } /* end if should authenticate (info provided) */
 
     /*******************
@@ -1318,21 +1191,21 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
         curlerrbuf[0] = '\0';
 
         if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, curlerrbuf))
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem setting error buffer");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem setting error buffer");
 
         p_status = curl_easy_perform(curlh);
 
         if (p_status != CURLE_OK) {
             if (CURLE_OK != curl_easy_getinfo(curlh, CURLINFO_RESPONSE_CODE, &httpcode))
-                HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem getting response code");
+                HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem getting response code");
             fprintf(stdout, "CURL ERROR CODE: %d\nHTTP CODE: %ld\n", p_status, httpcode);
             fprintf(stdout, "%s\n", curl_easy_strerror(p_status));
 
-            HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "problem while performing request.");
+            HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "problem while performing request");
         }
         if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, NULL))
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem unsetting error buffer");
-    } /* verbose error reporting */
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem unsetting error buffer");
+    }
 #else
     p_status = curl_easy_perform(curlh);
 
@@ -1341,133 +1214,95 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
 #endif
 
 done:
-    /* clean any malloc'd resources
-     */
-    if (authorization != NULL) {
-        H5MM_xfree(authorization);
-        authorization = NULL;
-    }
-    if (buffer1 != NULL) {
-        H5MM_xfree(buffer1);
-        buffer1 = NULL;
-    }
-    if (signed_headers != NULL) {
-        H5MM_xfree(signed_headers);
-        signed_headers = NULL;
-    }
-    if (curlheaders != NULL) {
+    H5MM_xfree(authorization);
+    H5MM_xfree(buffer1);
+    H5MM_xfree(signed_headers);
+    H5MM_xfree(rangebytesstr);
+    H5MM_xfree(sds);
+
+    if (curlheaders != NULL)
         curl_slist_free_all(curlheaders);
-        curlheaders = NULL;
-    }
-    if (rangebytesstr != NULL) {
-        H5MM_xfree(rangebytesstr);
-        rangebytesstr = NULL;
-    }
-    if (sds != NULL) {
-        H5MM_xfree(sds);
-        sds = NULL;
-    }
     if (request != NULL) {
         while (headers != NULL)
-            if (FAIL == H5FD_s3comms_hrb_node_set(&headers, headers->name, NULL))
-                HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot release header node");
+            if (H5FD__s3comms_hrb_node_set(&headers, headers->name, NULL) < 0)
+                HDONE_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "cannot release header node");
         assert(NULL == headers);
-        if (FAIL == H5FD_s3comms_hrb_destroy(&request))
-            HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot release header request structure");
-        assert(NULL == request);
+        if (H5FD__s3comms_hrb_destroy(request) < 0)
+            HDONE_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "cannot release header request structure");
     }
 
     if (curlh != NULL) {
-        /* clear any Range */
+        /* Clear any Range */
         if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_RANGE, NULL))
-            HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot unset CURLOPT_RANGE");
+            HDONE_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "cannot unset CURLOPT_RANGE");
 
-        /* clear headers */
+        /* Clear headers */
         if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, NULL))
-            HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot unset CURLOPT_HTTPHEADER");
+            HDONE_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "cannot unset CURLOPT_HTTPHEADER");
     }
 
     FUNC_LEAVE_NOAPI(ret_value)
-} /* H5FD_s3comms_s3r_read */
+} /* H5FD__s3comms_s3r_read */
 
 /****************************************************************************
  * MISCELLANEOUS FUNCTIONS
  ****************************************************************************/
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_make_iso_8661_string
  *
- * Function: gmnow()
- *
- * Purpose:
- *
- *    Get the output of `time.h`'s `gmtime()` call while minimizing setup
- *    clutter where important.
- *
- * Return:
- *
- *    Pointer to resulting `struct tm`,as created by gmtime(time_t * T).
+ * Purpose:     Create an ISO-8601 string from a time_t
  *
+ * Return:      SUCCEED/FAIL
  *----------------------------------------------------------------------------
  */
-struct tm *
-gmnow(void)
+static herr_t
+H5FD__s3comms_make_iso_8661_string(time_t time, char iso8601[ISO8601_SIZE])
 {
-    time_t     now;
-    time_t    *now_ptr   = &now;
-    struct tm *ret_value = NULL;
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_PACKAGE
 
-    /* Doctor assert, checks against error in time() */
-    if ((time_t)(-1) != time(now_ptr))
-        ret_value = gmtime(now_ptr);
+    assert(iso8601);
 
-    assert(ret_value != NULL);
+    if (strftime(iso8601, ISO8601_SIZE, "%Y%m%dT%H%M%SZ", gmtime(&time)) != (ISO8601_SIZE - 1))
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not construct ISO-8601 string");
 
-    return ret_value;
-} /* end gmnow() */
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD__s3comms_make_iso_8661_string() */
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_make_aws_canonical_request
  *
- * Function: H5FD_s3comms_aws_canonical_request()
- *
- * Purpose:
- *
- *     Compose AWS "Canonical Request" (and signed headers string)
- *     as defined in the REST API documentation.
+ * Purpose:     Compose AWS "Canonical Request" (and signed headers string)
+ *              as defined in the REST API documentation.
  *
- *     Both destination strings are null-terminated.
+ *              NOTE: Destination string arguments must be provided with
+ *                    adequate space
  *
- *     Destination string arguments must be provided with adequate space.
+ *              Canonical Request format:
  *
- *     Canonical Request format:
- *
- *      "\n"
- *      "\n"
- *      "\n"
- *      "\n" (`lowercase(name)`":"`trim(value)`)
- *      "\n"
- *      ... (headers sorted by name)
- *      "\n"
- *      "\n"
- *      "\n" (`lowercase(header 1 name)`";"`header 2 name`;...)
- *       ("e3b0c4429...", e.g.)
- *
- * Return:
- *
- *     - SUCCESS: `SUCCEED`
- *         - writes canonical request to respective `...dest` strings
- *     - FAILURE: `FAIL`
- *         - one or more input argument was NULL
- *         - internal error
+ *              "\n"
+ *              "\n"
+ *              "\n"
+ *              "\n" (`lowercase(name)`":"`trim(value)`)
+ *              "\n"
+ *              ... (headers sorted by name)
+ *              "\n"
+ *              "\n"
+ *              "\n" (`lowercase(header 1 name)`";"`header 2 name`;...)
+ *               ("e3b0c4429...", e.g.)
  *
+ * Return:      SUCCEED/FAIL
  *----------------------------------------------------------------------------
  */
 herr_t
-H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, char *signed_headers_dest,
-                                   int _sh_size, hrb_t *http_request)
+H5FD__s3comms_make_aws_canonical_request(char *canonical_request_dest, int _cr_size,
+                                         char *signed_headers_dest, int _sh_size, hrb_t *http_request)
 {
     hrb_node_t  *node         = NULL;
     const char  *query_params = ""; /* unused at present */
-    herr_t       ret_value    = SUCCEED;
     int          ret          = 0;
     size_t       cr_size      = (size_t)_cr_size;
     size_t       sh_size      = (size_t)_sh_size;
@@ -1475,6 +1310,7 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c
     size_t       sh_len       = 0; /* working length of signed headers str */
     char        *tmpstr       = NULL;
     const size_t TMP_STR_SIZE = sizeof(char) * H5FD_ROS3_MAX_SECRET_TOK_LEN;
+    herr_t       ret_value    = SUCCEED;
 
     /* "query params" refers to the optional element in the URL, e.g.
      *     http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J
@@ -1486,63 +1322,60 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c
      * VFD use-cases.
      */
 
-    FUNC_ENTER_NOAPI_NOINIT
+    FUNC_ENTER_PACKAGE
 
     if (http_request == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "hrb object cannot be null.");
-
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "hrb object cannot be NULL");
     if (canonical_request_dest == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "canonical request destination cannot be null.");
-
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "canonical request destination cannot be NULL");
     if (signed_headers_dest == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "signed headers destination cannot be null.");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "signed headers destination cannot be NULL");
 
     /* HTTP verb, resource path, and query string lines */
     cr_len = (strlen(http_request->verb) + strlen(http_request->resource) + strlen(query_params) +
               (size_t)3); /* three newline chars */
     if (cr_len >= cr_size)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not enough space in canonical request");
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "not enough space in canonical request");
 
     ret = snprintf(canonical_request_dest, (cr_size - 1), "%s\n%s\n%s\n", http_request->verb,
                    http_request->resource, query_params);
     if (ret < 0 || (size_t)ret >= cr_size)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to compose canonical request first line");
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to compose canonical request first line");
 
     if (NULL == (tmpstr = (char *)H5MM_calloc(TMP_STR_SIZE)))
         HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, FAIL, "unable to allocate space for temp string");
 
-    /* write in canonical headers, building signed headers concurrently */
-    node = http_request->first_header; /* assumed sorted */
+    /* Write in canonical headers, building signed headers concurrently */
+    node = http_request->first_header; /* Assumed sorted */
     while (node != NULL) {
 
         ret = snprintf(tmpstr, TMP_STR_SIZE, "%s:%s\n", node->lowername, node->value);
         if (ret < 0 || ret >= (int)TMP_STR_SIZE)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to concatenate HTTP header %s:%s",
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to concatenate HTTP header %s:%s",
                         node->lowername, node->value);
         cr_len += strlen(tmpstr);
         if (cr_len + 1 > cr_size)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not enough space in canonical request");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "not enough space in canonical request");
         strcat(canonical_request_dest, tmpstr);
 
         ret = snprintf(tmpstr, TMP_STR_SIZE, "%s;", node->lowername);
         if (ret < 0 || ret >= (int)TMP_STR_SIZE)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to append semicolon to lowername %s",
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to append semicolon to lowername %s",
                         node->lowername);
         sh_len += strlen(tmpstr);
         if (sh_len + 1 > sh_size)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not enough space in signed headers");
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "not enough space in signed headers");
         strcat(signed_headers_dest, tmpstr);
 
         node = node->next;
-    } /* end while node is not NULL */
+    }
 
-    /* remove trailing ';' from signed headers sequence */
+    /* Remove trailing ';' from signed headers sequence */
     if (*signed_headers_dest != '\0')
         signed_headers_dest[strlen(signed_headers_dest) - 1] = '\0';
 
-    /* append signed headers and payload hash
-     * NOTE: at present, no HTTP body is handled, per the nature of
-     *       requests/range-gets
+    /* Append signed headers and payload hash
+     * (no HTTP body is handled, per the nature of requests/range-gets)
      */
     strcat(canonical_request_dest, "\n");
     strcat(canonical_request_dest, signed_headers_dest);
@@ -1553,194 +1386,161 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c
     free(tmpstr);
 
     FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_s3comms_aws_canonical_request() */
+} /* end H5FD__s3comms_make_aws_canonical_request() */
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_bytes_to_hex
  *
- * Function: H5FD_s3comms_bytes_to_hex()
- *
- * Purpose:
- *
- *     Produce human-readable hex string [0-9A-F] from sequence of bytes.
- *
- *     For each byte (char), writes two-character hexadecimal representation.
- *
- *     No null-terminator appended.
- *
- *     Assumes `dest` is allocated to enough size (msg_len * 2).
- *
- *     Fails if either `dest` or `msg` are null.
- *
- *     `msg_len` message length of 0 has no effect.
- *
- * Return:
- *
- *     - SUCCESS: `SUCCEED`
- *         - hex string written to `dest` (not null-terminated)
- *     - FAILURE: `FAIL`
- *         - `dest == NULL`
- *         - `msg == NULL`
+ * Purpose:     Create a NUL-terminated hex string from a byte array
  *
+ * Return:      SUCCEED/FAIL
  *----------------------------------------------------------------------------
  */
-herr_t
-H5FD_s3comms_bytes_to_hex(char *dest, const unsigned char *msg, size_t msg_len, bool lowercase)
+static herr_t
+H5FD__s3comms_bytes_to_hex(char *dest, size_t dest_len, const unsigned char *msg, size_t msg_len)
 {
-    size_t i         = 0;
     herr_t ret_value = SUCCEED;
 
-    FUNC_ENTER_NOAPI_NOINIT
+    FUNC_ENTER_PACKAGE
 
-    if (dest == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "hex destination cannot be null.");
-    if (msg == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bytes sequence cannot be null.");
-
-    for (i = 0; i < msg_len; i++) {
-        int chars_written = snprintf(&(dest[i * 2]), 3, /* 'X', 'X', '\n' */
-                                     (lowercase == true) ? "%02x" : "%02X", msg[i]);
-        if (chars_written != 2)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem while writing hex chars for %c", msg[i]);
-    }
+    assert(dest);
+    assert(msg);
+
+    memset(dest, 0, dest_len);
+
+    if (0 == (OPENSSL_buf2hexstr_ex(dest, dest_len, NULL, msg, msg_len, '\0')))
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not create hex string");
+
+    /* AWS demands lower-case and buf2hexstr returns an upper-case hex string */
+    for (size_t i = 0; i < dest_len; i++)
+        dest[i] = (char)tolower(dest[i]);
+
+    dest[dest_len - 1] = '\0';
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_s3comms_bytes_to_hex() */
+} /* end H5FD__s3comms_bytes_to_hex() */
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_parse_url
  *
- * Function: H5FD_s3comms_free_purl()
- *
- * Purpose:
- *
- *     Release resources from a parsed_url_t pointer.
- *
- *     If pointer is null, nothing happens.
- *
- * Return:
- *
- *     `SUCCEED` (never fails)
+ * Purpose:     Release resources from a parsed_url_t pointer
  *
+ * Return:      Success:    A pointer to a parsed_url_t
+ *              Failure:    NULL
  *----------------------------------------------------------------------------
  */
-herr_t
-H5FD_s3comms_free_purl(parsed_url_t *purl)
+static parsed_url_t *
+H5FD__s3comms_parse_url(const char *url)
 {
-    FUNC_ENTER_NOAPI_NOINIT_NOERR
-
-    if (purl != NULL) {
-        if (purl->scheme != NULL)
-            H5MM_xfree(purl->scheme);
-        if (purl->host != NULL)
-            H5MM_xfree(purl->host);
-        if (purl->port != NULL)
-            H5MM_xfree(purl->port);
-        if (purl->path != NULL)
-            H5MM_xfree(purl->path);
-        if (purl->query != NULL)
-            H5MM_xfree(purl->query);
+    CURLUcode     rc;
+    CURLU        *curlurl   = NULL;
+    parsed_url_t *purl      = NULL;
+    parsed_url_t *ret_value = NULL;
+
+    FUNC_ENTER_PACKAGE
+
+    assert(url);
+
+    /* Get a curl URL handle */
+    if (NULL == (curlurl = curl_url()))
+        HGOTO_ERROR(H5E_VFL, H5E_CANTCREATE, NULL, "unable to get curl url");
+
+    /* Separate the URL into parts using libcurl */
+    if (CURLUE_OK != curl_url_set(curlurl, CURLUPART_URL, url, 0))
+        HGOTO_ERROR(H5E_VFL, H5E_CANTCREATE, NULL, "unable to parse url");
+
+    /* Allocate memory for the parsed URL to return */
+    if (NULL == (purl = (parsed_url_t *)H5MM_calloc(sizeof(parsed_url_t))))
+        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "can't allocate space for parsed_url_t");
+
+    /* Extract the URL components using libcurl */
+
+    /* scheme */
+    rc = curl_url_get(curlurl, CURLUPART_SCHEME, &(purl->scheme), 0);
+    if (CURLUE_OK != rc)
+        HGOTO_ERROR(H5E_VFL, H5E_CANTCREATE, NULL, "unable to get url scheme");
+    /* host */
+    rc = curl_url_get(curlurl, CURLUPART_HOST, &(purl->host), 0);
+    if (CURLUE_OK != rc)
+        HGOTO_ERROR(H5E_VFL, H5E_CANTCREATE, NULL, "unable to get url host");
+    /* port - okay to not exist */
+    rc = curl_url_get(curlurl, CURLUPART_PORT, &(purl->port), 0);
+    if (CURLUE_OK != rc && CURLUE_NO_PORT != rc)
+        HGOTO_ERROR(H5E_VFL, H5E_CANTCREATE, NULL, "unable to get url port");
+    /* path */
+    rc = curl_url_get(curlurl, CURLUPART_PATH, &(purl->path), 0);
+    if (CURLUE_OK != rc)
+        HGOTO_ERROR(H5E_VFL, H5E_CANTCREATE, NULL, "unable to get url path");
+    /* query - okay to not exist */
+    rc = curl_url_get(curlurl, CURLUPART_QUERY, &(purl->query), 0);
+    if (CURLUE_OK != rc && CURLUE_NO_QUERY != rc)
+        HGOTO_ERROR(H5E_VFL, H5E_CANTCREATE, NULL, "unable to get url query");
+
+    ret_value = purl;
+
+done:
+    curl_url_cleanup(curlurl);
+
+    if (ret_value == NULL) {
+        if (H5FD__s3comms_free_purl(purl) < 0)
+            HDONE_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "unable to free parsed url structure");
         H5MM_xfree(purl);
     }
 
-    FUNC_LEAVE_NOAPI(SUCCEED)
-} /* end H5FD_s3comms_free_purl() */
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD__s3comms_parse_url() */
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_free_purl
  *
- * Function: H5FD_s3comms_HMAC_SHA256()
- *
- * Purpose:
- *
- *     Generate Hash-based Message Authentication Checksum using the SHA-256
- *     hashing algorithm.
- *
- *     Given a key, message, and respective lengths (to accommodate null
- *     characters in either), generate _hex string_ of authentication checksum
- *     and write to `dest`.
- *
- *     `dest` must be at least `SHA256_DIGEST_LENGTH * 2` characters in size.
- *     Not enforceable by this function.
- *     `dest` will _not_ be null-terminated by this function.
- *
- * Return:
- *
- *     - SUCCESS: `SUCCEED`
- *         - hex string written to `dest` (not null-terminated)
- *     - FAILURE: `FAIL`
- *         - `dest == NULL`
- *         - error while generating hex string output
+ * Purpose:     Release resources from a parsed_url_t pointer
  *
+ * Return:      SUCCEED (Can't fail - passing NULL is okay)
  *----------------------------------------------------------------------------
  */
-herr_t
-H5FD_s3comms_HMAC_SHA256(const unsigned char *key, size_t key_len, const char *msg, size_t msg_len,
-                         char *dest)
+static herr_t
+H5FD__s3comms_free_purl(parsed_url_t *purl)
 {
-    unsigned char md[SHA256_DIGEST_LENGTH];
-    unsigned int  md_len    = SHA256_DIGEST_LENGTH;
-    herr_t        ret_value = SUCCEED;
+    herr_t ret_value = SUCCEED;
 
-    FUNC_ENTER_NOAPI_NOINIT
+    FUNC_ENTER_PACKAGE_NOERR
 
-    if (!key)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "signing key not provided");
+    if (NULL == purl)
+        HGOTO_DONE(SUCCEED);
 
-    if (dest == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "destination cannot be null.");
+    curl_free(purl->scheme);
+    curl_free(purl->host);
+    curl_free(purl->port);
+    curl_free(purl->path);
+    curl_free(purl->query);
 
-    HMAC(EVP_sha256(), key, (int)key_len, (const unsigned char *)msg, msg_len, md, &md_len);
-
-    if (H5FD_s3comms_bytes_to_hex(dest, (const unsigned char *)md, (size_t)md_len, true) == FAIL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not convert to hex string.");
+    H5MM_xfree(purl);
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
-} /* H5FD_s3comms_HMAC_SHA256 */
+} /* end H5FD__s3comms_free_purl() */
 
 /*-----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_load_aws_creds_from_file
  *
- * Function: H5FD__s3comms_load_aws_creds_from_file()
- *
- * Purpose:
- *
- *     Extract AWS configuration information from a target file.
+ * Purpose:     Extract AWS configuration information from a target file
  *
  *     Given a file and a profile name, e.g. "ros3_vfd_test", attempt to locate
  *     that region in the file. If not found, returns in error and output
  *     pointers are not modified.
  *
- *     If the profile label is found, attempts to locate and parse configuration
- *     data, stopping at the first line where:
- *     + reached end of file
- *     + line does not start with a recognized setting name
- *
  *     Following AWS documentation, looks for any of:
  *     + aws_access_key_id
  *     + aws_secret_access_key
  *     + region
  *
- *     To be valid, the setting must begin the line with one of the keywords,
- *     followed immediately by an equals sign '=', and have some data before
- *     newline at end of line.
- *     + `spam=eggs` would be INVALID because name is unrecognized
- *     + `region = us-east-2` would be INVALID because of spaces
- *     + `region=` would be INVALID because no data.
- *
  *     Upon successful parsing of a setting line, will store the result in the
  *     corresponding output pointer. If the output pointer is NULL, will skip
  *     any matching setting line while parsing -- useful to prevent overwrite
  *     when reading from multiple files.
  *
- * Return:
- *
- *     + SUCCESS: `SUCCEED`
- *         + no error. settings may or may not have been loaded.
- *     + FAILURE: `FAIL`
- *         + internal error occurred.
- *         + -1 :: unable to format profile label
- *         + -2 :: profile name/label not found in file
- *         + -3 :: some other error
- *
+ * Return:      SUCCEED/FAIL
  *-----------------------------------------------------------------------------
  */
 static herr_t
@@ -1768,32 +1568,29 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha
 
     FUNC_ENTER_PACKAGE
 
-    /* format target line for start of profile */
+    /* Format target line for start of profile */
     if (32 < snprintf(profile_line, 32, "[%s]", profile_name))
-        HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format profile label");
+        HGOTO_ERROR(H5E_VFL, H5E_CANTCOPY, FAIL, "unable to format profile label");
 
-    /* look for start of profile */
+    /* Look for start of profile */
     do {
-        /* clear buffer */
         memset(buffer, 0, 128);
 
         line_buffer = fgets(line_buffer, 128, file);
-        if (line_buffer == NULL) /* reached end of file */
+        if (line_buffer == NULL)
             goto done;
     } while (strncmp(line_buffer, profile_line, strlen(profile_line)));
 
-    /* extract credentials from lines */
+    /* Extract credentials from lines */
     do {
-        /* clear buffer and flag */
         memset(buffer, 0, 128);
         found_setting = 0;
 
-        /* collect a line from file */
         line_buffer = fgets(line_buffer, 128, file);
         if (line_buffer == NULL)
             goto done; /* end of file */
 
-        /* loop over names to see if line looks like assignment */
+        /* Loop over names to see if line looks like assignment */
         for (setting_i = 0; setting_i < setting_count; setting_i++) {
             size_t      setting_name_len = 0;
             const char *setting_name     = NULL;
@@ -1802,29 +1599,29 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha
             setting_name     = setting_names[setting_i];
             setting_name_len = strlen(setting_name);
             if (snprintf(line_prefix, 128, "%s=", setting_name) < 0)
-                HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format line prefix");
+                HGOTO_ERROR(H5E_VFL, H5E_CANTCOPY, FAIL, "unable to format line prefix");
 
-            /* found a matching name? */
+            /* Found a matching name? */
             if (!strncmp(line_buffer, line_prefix, setting_name_len + 1)) {
                 found_setting = 1;
 
-                /* skip NULL destination buffer */
+                /* Skip NULL destination buffer */
                 if (setting_pointers[setting_i] == NULL)
                     break;
 
-                /* advance to end of name in string */
+                /* Advance to end of name in string */
                 do {
                     line_buffer++;
                 } while (*line_buffer != 0 && *line_buffer != '=');
 
                 if (*line_buffer == 0 || *(line_buffer + 1) == 0)
-                    HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "incomplete assignment in file");
-                line_buffer++; /* was pointing at '='; advance */
+                    HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "incomplete assignment in file");
+                line_buffer++; /* Was pointing at '='; advance */
 
-                /* copy line buffer into out pointer */
+                /* Copy line buffer into out pointer */
                 strncpy(setting_pointers[setting_i], (const char *)line_buffer, strlen(line_buffer));
 
-                /* "trim" tailing whitespace by replacing with null terminator*/
+                /* "Trim" tailing whitespace by replacing with NUL terminator*/
                 end = strlen(line_buffer) - 1;
                 while (end > 0 && isspace((int)setting_pointers[setting_i][end])) {
                     setting_pointers[setting_i][end] = '\0';
@@ -1841,37 +1638,16 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha
 } /* end H5FD__s3comms_load_aws_creds_from_file() */
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_load_aws_profile
  *
- * Function: H5FD_s3comms_load_aws_profile()
- *
- * Purpose :
- *
- *     Read aws profile elements from standard location on system and store
- *     settings in memory.
- *
- *     Looks for both `~/.aws/config` and `~/.aws/credentials`, the standard
- *     files for AWS tools. If a file exists (can be opened), looks for the
- *     given profile name and reads the settings into the relevant buffer.
- *
- *     Any setting duplicated in both files will be set to that from
- *     `credentials`.
- *
- *     Settings are stored in the supplied buffers as null-terminated strings.
- *
- * Return:
- *
- *     + SUCCESS: `SUCCEED` (0)
- *         + no error occurred and all settings were populated
- *     + FAILURE: `FAIL` (-1)
- *         + internal error occurred
- *         + unable to locate profile
- *         + region, key id, and secret key were not all found and set
+ * Purpose:     Read AWS profile elements from ~/.aws/config and credentials
  *
+ * Return:      SUCCEED/FAIL
  *----------------------------------------------------------------------------
  */
 herr_t
-H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char *secret_access_key_out,
-                              char *aws_region_out)
+H5FD__s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char *secret_access_key_out,
+                               char *aws_region_out)
 {
     herr_t ret_value = SUCCEED;
     FILE  *credfile  = NULL;
@@ -1879,7 +1655,7 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char *
     char   filepath[128];
     int    ret = 0;
 
-    FUNC_ENTER_NOAPI_NOINIT
+    FUNC_ENTER_PACKAGE
 
 #ifdef H5_HAVE_WIN32_API
     ret = snprintf(awspath, 117, "%s/.aws/", getenv("USERPROFILE"));
@@ -1887,285 +1663,82 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char *
     ret = snprintf(awspath, 117, "%s/.aws/", getenv("HOME"));
 #endif
     if (ret < 0 || (size_t)ret >= 117)
-        HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format home-aws path");
+        HGOTO_ERROR(H5E_VFL, H5E_CANTCOPY, FAIL, "unable to format home-aws path");
     ret = snprintf(filepath, 128, "%s%s", awspath, "credentials");
     if (ret < 0 || (size_t)ret >= 128)
-        HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format credentials path");
+        HGOTO_ERROR(H5E_VFL, H5E_CANTCOPY, FAIL, "unable to format credentials path");
+
+    /* Looks for both `~/.aws/config` and `~/.aws/credentials`, the standard
+     * files for AWS tools. If a file exists (can be opened), looks for the
+     * given profile name and reads the settings into the relevant buffer.
+     *
+     * Any setting duplicated in both files will be set to that from
+     * credentials
+     */
 
     credfile = fopen(filepath, "r");
     if (credfile != NULL) {
         if (H5FD__s3comms_load_aws_creds_from_file(credfile, profile_name, key_id_out, secret_access_key_out,
-                                                   aws_region_out) == FAIL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to load from aws credentials");
+                                                   aws_region_out) < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to load from aws credentials");
         if (fclose(credfile) == EOF)
             HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close credentials file");
         credfile = NULL;
-    } /* end if credential file opened */
+    }
 
     ret = snprintf(filepath, 128, "%s%s", awspath, "config");
     if (ret < 0 || (size_t)ret >= 128)
-        HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format config path");
+        HGOTO_ERROR(H5E_VFL, H5E_CANTCOPY, FAIL, "unable to format config path");
+
     credfile = fopen(filepath, "r");
     if (credfile != NULL) {
         if (H5FD__s3comms_load_aws_creds_from_file(
                 credfile, profile_name, (*key_id_out == 0) ? key_id_out : NULL,
                 (*secret_access_key_out == 0) ? secret_access_key_out : NULL,
-                (*aws_region_out == 0) ? aws_region_out : NULL) == FAIL)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to load from aws config");
+                (*aws_region_out == 0) ? aws_region_out : NULL) < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to load from aws config");
         if (fclose(credfile) == EOF)
             HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close config file");
         credfile = NULL;
-    } /* end if credential file opened */
+    }
 
-    /* fail if not all three settings were loaded */
+    /* Fail if not all three settings were loaded */
     if (*key_id_out == 0 || *secret_access_key_out == 0 || *aws_region_out == 0)
         ret_value = FAIL;
 
 done:
     if (credfile != NULL)
         if (fclose(credfile) == EOF)
-            HDONE_ERROR(H5E_ARGS, H5E_ARGS, FAIL, "problem error-closing aws configuration file");
+            HDONE_ERROR(H5E_VFL, H5E_VFL, FAIL, "problem error-closing aws configuration file");
 
     FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_s3comms_load_aws_profile() */
+} /* end H5FD__s3comms_load_aws_profile() */
 
 /*----------------------------------------------------------------------------
+ * Function:    H5FD__s3comms_make_aws_signing_key
  *
- * Function: H5FD_s3comms_parse_url()
- *
- * Purpose:
+ * Purpose:     Create AWS4 "Signing Key" from secret key, AWS region, and
+ *              timestamp
  *
- *     Parse URL-like string and stuff URL components into
- *     `parsed_url` structure, if possible.
+ *              `secret` is `access key id` for targeted service/bucket/resource
  *
- *     Expects null-terminated string of format:
- *     SCHEME "://" HOST [":" PORT ] ["/" [ PATH ] ] ["?" QUERY]
- *     where SCHEME :: "[a-zA-Z/.-]+"
- *           PORT   :: "[0-9]"
+ *              `region` should be one of AWS service region names, e.g. "us-east-1"
  *
- *     Stores resulting structure in argument pointer `purl`, if successful,
- *     creating and populating new `parsed_url_t` structure pointer.
- *     Empty or absent elements are NULL in new purl structure.
+ *              `iso8601` is an ISO-8601 time string with no punctuation
+ *              (e.g.: "20170713T145903Z", so... YYYYmmdd'T'HHMMSSZ).
+ *              This can be constructed using H5FD__s3comms_make_iso_8661_string().
  *
- * Return:
+ *              Hard-coded "service" algorithm requirement to "s3"
  *
- *     - SUCCESS: `SUCCEED`
- *         - `purl` pointer is populated
- *     - FAILURE: `FAIL`
- *         - unable to parse
- *             - `purl` is unaltered (probably NULL)
+ *              Writes to `md` the raw byte data, length of `SHA256_DIGEST_LENGTH`.
+ *              Programmer must ensure that `md` is appropriately allocated.
  *
+ * Return:      SUCCEED/FAIL
  *----------------------------------------------------------------------------
  */
 herr_t
-H5FD_s3comms_parse_url(const char *str, parsed_url_t **_purl)
-{
-    parsed_url_t *purl      = NULL; /* pointer to new structure */
-    const char   *tmpstr    = NULL; /* working pointer in string */
-    const char   *curstr    = str;  /* "start" pointer in string */
-    long int      len       = 0;    /* substring length */
-    long int      urllen    = 0;    /* length of passed-in url string */
-    unsigned int  i         = 0;
-    herr_t        ret_value = FAIL;
-
-    FUNC_ENTER_NOAPI_NOINIT
-
-    if (str == NULL || *str == '\0')
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid url string");
-
-    urllen = (long int)strlen(str);
-
-    purl = (parsed_url_t *)H5MM_malloc(sizeof(parsed_url_t));
-    if (purl == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "can't allocate space for parsed_url_t");
-    purl->scheme = NULL;
-    purl->host   = NULL;
-    purl->port   = NULL;
-    purl->path   = NULL;
-    purl->query  = NULL;
-
-    /***************
-     * READ SCHEME *
-     ***************/
-
-    tmpstr = strchr(curstr, ':');
-    if (tmpstr == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid SCHEME construction: probably not URL");
-    len = tmpstr - curstr;
-    assert((0 <= len) && (len < urllen));
-
-    /* check for restrictions */
-    for (i = 0; i < len; i++) {
-        /* scheme = [a-zA-Z+-.]+ (terminated by ":") */
-        if (!isalpha(curstr[i]) && '+' != curstr[i] && '-' != curstr[i] && '.' != curstr[i])
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid SCHEME construction");
-    }
-
-    /* copy lowercased scheme to structure */
-    purl->scheme = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
-    if (purl->scheme == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "can't allocate space for SCHEME");
-    strncpy(purl->scheme, curstr, (size_t)len);
-    purl->scheme[len] = '\0';
-    for (i = 0; i < len; i++)
-        purl->scheme[i] = (char)tolower(purl->scheme[i]);
-
-    /* Skip "://" */
-    tmpstr += 3;
-    curstr = tmpstr;
-
-    /*************
-     * READ HOST *
-     *************/
-
-    if (*curstr == '[') {
-        /* IPv6 */
-        while (']' != *tmpstr) {
-            /* end of string reached! */
-            if (tmpstr == 0)
-                HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "reached end of URL: incomplete IPv6 HOST");
-            tmpstr++;
-        }
-        tmpstr++;
-    } /* end if (IPv6) */
-    else {
-        while (0 != *tmpstr) {
-            if (':' == *tmpstr || '/' == *tmpstr || '?' == *tmpstr)
-                break;
-            tmpstr++;
-        }
-    } /* end else (IPv4) */
-    len = tmpstr - curstr;
-    if (len == 0)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "HOST substring cannot be empty");
-    else if (len > urllen)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem with length of HOST substring");
-
-    /* copy host */
-    purl->host = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
-    if (purl->host == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "can't allocate space for HOST");
-    strncpy(purl->host, curstr, (size_t)len);
-    purl->host[len] = 0;
-
-    /*************
-     * READ PORT *
-     *************/
-
-    if (':' == *tmpstr) {
-        tmpstr += 1; /* advance past ':' */
-        curstr = tmpstr;
-        while ((0 != *tmpstr) && ('/' != *tmpstr) && ('?' != *tmpstr))
-            tmpstr++;
-        len = tmpstr - curstr;
-        if (len == 0)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "PORT element cannot be empty");
-        else if (len > urllen)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem with length of PORT substring");
-        for (i = 0; i < len; i++)
-            if (!isdigit(curstr[i]))
-                HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "PORT is not a decimal string");
-
-        /* copy port */
-        purl->port = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
-        if (purl->port == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "can't allocate space for PORT");
-        strncpy(purl->port, curstr, (size_t)len);
-        purl->port[len] = 0;
-    } /* end if PORT element */
-
-    /*************
-     * READ PATH *
-     *************/
-
-    if ('/' == *tmpstr) {
-        /* advance past '/' */
-        tmpstr += 1;
-        curstr = tmpstr;
-
-        /* seek end of PATH */
-        while ((0 != *tmpstr) && ('?' != *tmpstr))
-            tmpstr++;
-        len = tmpstr - curstr;
-        if (len > urllen)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem with length of PATH substring");
-        if (len > 0) {
-            purl->path = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
-            if (purl->path == NULL)
-                HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "can't allocate space for PATH");
-            strncpy(purl->path, curstr, (size_t)len);
-            purl->path[len] = 0;
-        }
-    } /* end if PATH element */
-
-    /**************
-     * READ QUERY *
-     **************/
-
-    if ('?' == *tmpstr) {
-        tmpstr += 1;
-        curstr = tmpstr;
-        while (0 != *tmpstr)
-            tmpstr++;
-        len = tmpstr - curstr;
-        if (len == 0)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "QUERY cannot be empty");
-        else if (len > urllen)
-            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem with length of QUERY substring");
-        purl->query = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
-        if (purl->query == NULL)
-            HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "can't allocate space for QUERY");
-        strncpy(purl->query, curstr, (size_t)len);
-        purl->query[len] = 0;
-    } /* end if QUERY exists */
-
-    *_purl    = purl;
-    ret_value = SUCCEED;
-
-done:
-    if (ret_value == FAIL)
-        H5FD_s3comms_free_purl(purl);
-
-    FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_s3comms_parse_url() */
-
-/*----------------------------------------------------------------------------
- *
- * Function: H5FD_s3comms_signing_key()
- *
- * Purpose:
- *
- *     Create AWS4 "Signing Key" from secret key, AWS region, and timestamp.
- *
- *     Sequentially runs HMAC_SHA256 on strings in specified order,
- *     generating reusable checksum (according to documentation, valid for
- *     7 days from time given).
- *
- *     `secret` is `access key id` for targeted service/bucket/resource.
- *
- *     `iso8601now` must conform to format, yyyyMMDD'T'hhmmss'Z'
- *     e.g. "19690720T201740Z".
- *
- *     `region` should be one of AWS service region names, e.g. "us-east-1".
- *
- *     Hard-coded "service" algorithm requirement to "s3".
- *
- *     Inputs must be null-terminated strings.
- *
- *     Writes to `md` the raw byte data, length of `SHA256_DIGEST_LENGTH`.
- *     Programmer must ensure that `md` is appropriately allocated.
- *
- * Return:
- *
- *     - SUCCESS: `SUCCEED`
- *         - raw byte data of signing key written to `md`
- *     - FAILURE: `FAIL`
- *         - if any input arguments was NULL
- *
- *----------------------------------------------------------------------------
- */
-herr_t
-H5FD_s3comms_signing_key(unsigned char *md, const char *secret, const char *region, const char *iso8601now)
+H5FD__s3comms_make_aws_signing_key(unsigned char *md, const char *secret, const char *region,
+                                   const char *iso8601)
 {
     char         *AWS4_secret     = NULL;
     size_t        AWS4_secret_len = 0;
@@ -2175,32 +1748,34 @@ H5FD_s3comms_signing_key(unsigned char *md, const char *secret, const char *regi
     int           ret       = 0; /* return value of snprintf */
     herr_t        ret_value = SUCCEED;
 
-    FUNC_ENTER_NOAPI_NOINIT
+    FUNC_ENTER_PACKAGE
 
     if (md == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Destination `md` cannot be NULL.");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Destination `md` cannot be NULL");
     if (secret == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "`secret` cannot be NULL.");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "`secret` cannot be NULL");
     if (region == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "`region` cannot be NULL.");
-    if (iso8601now == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "`iso8601now` cannot be NULL.");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "`region` cannot be NULL");
 
     AWS4_secret_len = 4 + strlen(secret) + 1;
     AWS4_secret     = (char *)H5MM_malloc(AWS4_secret_len);
     if (AWS4_secret == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Could not allocate space.");
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "Could not allocate space");
 
-    /* prepend "AWS4" to start of the secret key */
+    /* Prepend "AWS4" to start of the secret key */
     ret = snprintf(AWS4_secret, AWS4_secret_len, "%s%s", "AWS4", secret);
     if ((size_t)ret != (AWS4_secret_len - 1))
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem writing AWS4+secret `%s`", secret);
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem writing AWS4+secret `%s`", secret);
 
-    /* hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest
+    /* Sequentially runs HMAC_SHA256 on strings in specified order,
+     * generating reusable checksum (according to documentation, valid for
+     * 7 days from time given)
+     *
+     * hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest
      * we know digest length, so ignore via NULL
      */
     HMAC(EVP_sha256(), (const unsigned char *)AWS4_secret, (int)strlen(AWS4_secret),
-         (const unsigned char *)iso8601now, 8, /* 8 --> length of 8 --> "yyyyMMDD"  */
+         (const unsigned char *)iso8601, 8, /* 8 --> length of 8 --> "yyyyMMDD"  */
          datekey, NULL);
 
     HMAC(EVP_sha256(), (const unsigned char *)datekey, SHA256_DIGEST_LENGTH, (const unsigned char *)region,
@@ -2216,11 +1791,11 @@ H5FD_s3comms_signing_key(unsigned char *md, const char *secret, const char *regi
     H5MM_xfree(AWS4_secret);
 
     FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_s3comms_signing_key() */
+} /* end H5FD__s3comms_make_aws_signing_key() */
 
 /*----------------------------------------------------------------------------
  *
- * Function: H5FD_s3comms_tostringtosign()
+ * Function: H5FD__s3comms_make_aws_stringtosign()
  *
  * Purpose:
  *
@@ -2236,15 +1811,15 @@ H5FD_s3comms_signing_key(unsigned char *md, const char *secret, const char *regi
  *
  *     Inputs `creq` (canonical request string), `now` (ISO8601 format),
  *     and `region` (s3 region designator string) must all be
- *     null-terminated strings.
+ *     NUL-terminated strings.
  *
- *     Result is written to `dest` with null-terminator.
+ *     Result is written to `dest` with NUL-terminator.
  *     It is left to programmer to ensure `dest` has adequate space.
  *
  * Return:
  *
  *     - SUCCESS: `SUCCEED`
- *         - "string to sign" written to `dest` and null-terminated
+ *         - "string to sign" written to `dest` and NUL-terminated
  *     - FAILURE: `FAIL`
  *         - if any of the inputs are NULL
  *         - if an error is encountered while computing checksum
@@ -2252,31 +1827,31 @@ H5FD_s3comms_signing_key(unsigned char *md, const char *secret, const char *regi
  *----------------------------------------------------------------------------
  */
 herr_t
-H5FD_s3comms_tostringtosign(char *dest, const char *req, const char *now, const char *region)
+H5FD__s3comms_make_aws_stringtosign(char *dest, const char *req, const char *now, const char *region)
 {
-    unsigned char checksum[SHA256_DIGEST_LENGTH * 2 + 1];
+    unsigned char checksum[S3COMMS_SHA256_HEXSTR_LENGTH];
     size_t        d = 0;
     char          day[9];
-    char          hexsum[SHA256_DIGEST_LENGTH * 2 + 1];
-    size_t        i         = 0;
-    int           ret       = 0; /* snprintf return value */
-    herr_t        ret_value = SUCCEED;
+    char          hexsum[S3COMMS_SHA256_HEXSTR_LENGTH];
+    size_t        i   = 0;
+    int           ret = 0; /* snprintf return value */
     char          tmp[128];
+    herr_t        ret_value = SUCCEED;
 
-    FUNC_ENTER_NOAPI_NOINIT
+    FUNC_ENTER_PACKAGE
 
     if (dest == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "destination buffer cannot be null.");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "destination buffer cannot be NULL");
     if (req == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "canonical request cannot be null.");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "canonical request cannot be NULL");
     if (now == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Timestring cannot be NULL.");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Timestring cannot be NULL");
     if (region == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Region cannot be NULL.");
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Region cannot be NULL");
 
     for (i = 0; i < 128; i++)
         tmp[i] = '\0';
-    for (i = 0; i < SHA256_DIGEST_LENGTH * 2 + 1; i++) {
+    for (i = 0; i < S3COMMS_SHA256_HEXSTR_LENGTH; i++) {
         checksum[i] = '\0';
         hexsum[i]   = '\0';
     }
@@ -2284,7 +1859,7 @@ H5FD_s3comms_tostringtosign(char *dest, const char *req, const char *now, const
     day[8] = '\0';
     ret    = snprintf(tmp, 127, "%s/%s/s3/aws4_request", day, region);
     if (ret <= 0 || ret >= 127)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem adding day and region to string");
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem adding day and region to string");
 
     H5MM_memcpy((dest + d), "AWS4-HMAC-SHA256\n", 17);
     d = 17;
@@ -2299,17 +1874,17 @@ H5FD_s3comms_tostringtosign(char *dest, const char *req, const char *now, const
 
     SHA256((const unsigned char *)req, strlen(req), checksum);
 
-    if (H5FD_s3comms_bytes_to_hex(hexsum, (const unsigned char *)checksum, SHA256_DIGEST_LENGTH, true) ==
-        FAIL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not create hex string");
+    if (H5FD__s3comms_bytes_to_hex(hexsum, S3COMMS_SHA256_HEXSTR_LENGTH, (const unsigned char *)checksum,
+                                   SHA256_DIGEST_LENGTH) < 0)
+        HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "could not create hex string");
 
-    for (i = 0; i < SHA256_DIGEST_LENGTH * 2; i++)
+    for (i = 0; i < S3COMMS_SHA256_HEXSTR_LENGTH - 1; i++)
         dest[d++] = hexsum[i];
 
     dest[d] = '\0';
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5ros3_tostringtosign() */
+} /* end H5FD__s3comms_make_aws_stringtosign() */
 
 #endif /* H5_HAVE_ROS3_VFD */
diff --git a/src/H5FDs3comms.h b/src/H5FDs3comms.h
index fdf13ac6b99..93433f17054 100644
--- a/src/H5FDs3comms.h
+++ b/src/H5FDs3comms.h
@@ -24,7 +24,6 @@
  *     - Abstract away the REST API (HTTP,
  *       networked communications) behind a series of uniform function calls.
  *     - Handle AWS4 authentication, if appropriate.
- *     - Fail predictably in event of errors.
  *     - Eventually, support more S3 operations, such as creating, writing to,
  *       and removing Objects remotely.
  *
@@ -48,6 +47,7 @@
  *****************************************************************************/
 
 #include "H5private.h" /* Generic Functions        */
+#include "H5FDros3.h"  /* ros3 VFD                 */
 
 #ifdef H5_HAVE_ROS3_VFD
 
@@ -57,9 +57,9 @@
 #include 
 #include 
 
-/*****************
- * PUBLIC MACROS *
- *****************/
+/**********
+ * MACROS *
+ **********/
 
 /* hexadecimal string of pre-computed sha256 checksum of the empty string
  * hex(sha256sum(""))
@@ -71,47 +71,6 @@
  */
 #define ISO8601_SIZE 17
 
-/* string length (plus null terminator)
- * example RFC7231-format string: "Fri, 30 Jun 2017 20:41:55 GMT"
- */
-#define RFC7231_SIZE 30
-
-/*---------------------------------------------------------------------------
- *
- * Macro: ISO8601NOW()
- *
- * Purpose:
- *
- *     write "YYYYmmdd'T'HHMMSS'Z'" (less single-quotes) to dest
- *     e.g., "20170630T204155Z"
- *
- *     wrapper for strftime()
- *
- *     It is left to the programmer to check return value of
- *     ISO8601NOW (should equal ISO8601_SIZE - 1).
- *
- *---------------------------------------------------------------------------
- */
-#define ISO8601NOW(dest, now_gm) strftime((dest), ISO8601_SIZE, "%Y%m%dT%H%M%SZ", (now_gm))
-
-/*---------------------------------------------------------------------------
- *
- * Macro: RFC7231NOW()
- *
- * Purpose:
- *
- *     write "Day, dd Mmm YYYY HH:MM:SS GMT" to dest
- *     e.g., "Fri, 30 Jun 2017 20:41:55 GMT"
- *
- *     wrapper for strftime()
- *
- *     It is left to the programmer to check return value of
- *     RFC7231NOW (should equal RFC7231_SIZE - 1).
- *
- *---------------------------------------------------------------------------
- */
-#define RFC7231NOW(dest, now_gm) strftime((dest), RFC7231_SIZE, "%a, %d %b %Y %H:%M:%S GMT", (now_gm))
-
 /* Reasonable maximum length of a credential string.
  * Provided for error-checking S3COMMS_FORMAT_CREDENTIAL (below).
  *  17 <- "////aws4_request\0"
@@ -123,7 +82,6 @@
 #define S3COMMS_MAX_CREDENTIAL_SIZE 155
 
 /*---------------------------------------------------------------------------
- *
  * Macro: H5FD_S3COMMS_FORMAT_CREDENTIAL()
  *
  * Purpose:
@@ -149,7 +107,6 @@
  *     `date` must be of format "YYYYmmdd".
  *     `region` should be relevant AWS region, i.e. "us-east-1".
  *     `service` should be "s3".
- *
  *---------------------------------------------------------------------------
  */
 #define S3COMMS_FORMAT_CREDENTIAL(dest, access, iso8601_date, region, service)                               \
@@ -161,13 +118,10 @@
  *********************/
 
 /*----------------------------------------------------------------------------
- *
  * Structure: hrb_node_t
  *
  * HTTP Header Field Node
  *
- *
- *
  * Maintain a ordered (linked) list of HTTP Header fields.
  *
  * Provides efficient access and manipulation of a logical sequence of
@@ -233,7 +187,6 @@
  *
  *     Pointers to next node in the list, or NULL sentinel as end of list.
  *     Next node must have a greater `lowername` as determined by strcmp().
- *
  *----------------------------------------------------------------------------
  */
 typedef struct hrb_node_t {
@@ -245,13 +198,10 @@ typedef struct hrb_node_t {
 } hrb_node_t;
 
 /*----------------------------------------------------------------------------
- *
  * Structure: hrb_t
  *
  * HTTP Request Buffer structure
  *
- *
- *
  * Logically represent an HTTP request
  *
  *     GET /myplace/myfile.h5 HTTP/1.1
@@ -299,7 +249,6 @@ typedef struct hrb_node_t {
  * `version` (char *) :
  *
  *     Pointer to HTTP version string, e.g., "HTTP/1.1".
- *
  *----------------------------------------------------------------------------
  */
 typedef struct {
@@ -312,10 +261,8 @@ typedef struct {
 } hrb_t;
 
 /*----------------------------------------------------------------------------
- *
  * Structure: parsed_url_t
  *
- *
  * Represent a URL with easily-accessed pointers to logical elements within.
  * These elements (components) are stored as null-terminated strings (or just
  * NULLs). These components should be allocated for the structure, making the
@@ -354,7 +301,6 @@ typedef struct {
  *
  *     Single string of all query parameters in url (if any).
  *     "arg1=value1&arg2=value2"
- *
  *----------------------------------------------------------------------------
  */
 typedef struct {
@@ -366,39 +312,36 @@ typedef struct {
 } parsed_url_t;
 
 /*----------------------------------------------------------------------------
- *
  * Structure: s3r_t
  *
- *
- *
  * S3 request structure "handle".
  *
  * Holds persistent information for Amazon S3 requests.
  *
- * Instantiated through `H5FD_s3comms_s3r_open()`, copies data into self.
+ * Instantiated through `H5FD__s3comms_s3r_open()`, copies data into self.
  *
  * Intended to be re-used for operations on a remote object.
  *
- * Cleaned up through `H5FD_s3comms_s3r_close()`.
+ * Cleaned up through `H5FD__s3comms_s3r_close()`.
  *
  * _DO NOT_ share handle between threads: curl easy handle `curlhandle` has
  * undefined behavior if called to perform in multiple threads.
  *
  *
- * `curlhandle` (CURL)
+ * curlhandle
  *
- *     Pointer to the curl_easy handle generated for the request.
+ *     Pointer to the curl_easy handle generated for the request
  *
- * `httpverb` (char *)
+ * http_verb
  *
  *     Pointer to NULL-terminated string. HTTP verb,
  *     e.g. "GET", "HEAD", "PUT", etc.
  *
- *     Default is NULL, resulting in a "GET" request.
+ *     Default is NULL, resulting in a "GET" request
  *
- * `purl` (parsed_url_t *)
+ * purl ("parsed url")
  *
- *     Pointer to structure holding the elements of URL for file open.
+ *     Pointer to structure holding the elements of URL for file open
  *
  *     e.g., "http://bucket.aws.com:8080/myfile.dat?q1=v1&q2=v2"
  *     parsed into...
@@ -409,102 +352,75 @@ typedef struct {
  *         query:  "q1=v1&q2=v2"
  *     }
  *
- *     Cannot be NULL.
+ *     Cannot be NULL
  *
- * `region` (char *)
+ * aws_region
  *
- *     Pointer to NULL-terminated string, specifying S3 "region",
+ *     Pointer to NULL-terminated string, specifying S3 "region"
  *     e.g., "us-east-1".
  *
- *     Required to authenticate.
+ *     Required to authenticate
  *
- * `secret_id` (char *)
+ * secret_id
  *
- *     Pointer to NULL-terminated string for "secret" access id to S3 resource.
+ *     Pointer to NULL-terminated string for "secret" access id to S3 resource
  *
- *     Required to authenticate.
+ *     Required to authenticate
  *
- * `signing_key` (unsigned char *)
+ * signing_key
  *
- *     Pointer to `SHA256_DIGEST_LENGTH`-long string for "reusable" signing
+ *     Pointer to `SHA256_DIGEST_LENGTH`-long buffer for "reusable" signing
  *     key, generated via
  *     `HMAC-SHA256(HMAC-SHA256(HMAC-SHA256(HMAC-SHA256("AWS4",
  *         ""), ""), "aws4_request")`
- *     which may be re-used for several (up to seven (7)) days from creation?
- *     Computed once upon file open.
+ *     which may be re-used for several (up to seven (7)) days from creation
  *
- *     Required to authenticate.
+ *     Computed once upon file open from the secret key string in the fapl
  *
+ *     Required to authenticate
  *----------------------------------------------------------------------------
  */
 typedef struct {
-    CURL          *curlhandle;
-    size_t         filesize;
-    char          *httpverb;
-    parsed_url_t  *purl;
-    char          *region;
-    char          *secret_id;
-    unsigned char *signing_key;
-    char          *token;
+    CURL         *curlhandle;
+    size_t        filesize;
+    char         *http_verb;
+    parsed_url_t *purl;
+    char         *aws_region;
+    char         *secret_id;
+    uint8_t      *signing_key;
+    char         *token;
 } s3r_t;
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-/*******************************************
- * DECLARATION OF HTTP FIELD LIST ROUTINES *
- *******************************************/
-
-H5_DLL herr_t H5FD_s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value);
-
-/***********************************************
- * DECLARATION OF HTTP REQUEST BUFFER ROUTINES *
- ***********************************************/
-
-H5_DLL herr_t H5FD_s3comms_hrb_destroy(hrb_t **buf);
-
-H5_DLL hrb_t *H5FD_s3comms_hrb_init_request(const char *verb, const char *resource, const char *host);
-
-/*************************************
- * DECLARATION OF S3REQUEST ROUTINES *
- *************************************/
-
-H5_DLL herr_t H5FD_s3comms_s3r_close(s3r_t *handle);
-
-H5_DLL size_t H5FD_s3comms_s3r_get_filesize(s3r_t *handle);
-
-H5_DLL s3r_t *H5FD_s3comms_s3r_open(const char url[], const char region[], const char id[],
-                                    const unsigned char signing_key[], const char token[]);
-
-H5_DLL herr_t H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest);
-
-/*********************************
- * DECLARATION OF OTHER ROUTINES *
- *********************************/
-
-H5_DLL struct tm *gmnow(void);
-
-H5_DLL herr_t H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int cr_size,
-                                                 char *signed_headers_dest, int sh_size, hrb_t *http_request);
-
-H5_DLL herr_t H5FD_s3comms_bytes_to_hex(char *dest, const unsigned char *msg, size_t msg_len, bool lowercase);
-
-H5_DLL herr_t H5FD_s3comms_free_purl(parsed_url_t *purl);
-
-H5_DLL herr_t H5FD_s3comms_HMAC_SHA256(const unsigned char *key, size_t key_len, const char *msg,
-                                       size_t msg_len, char *dest);
-
-H5_DLL herr_t H5FD_s3comms_load_aws_profile(const char *name, char *key_id_out, char *secret_access_key_out,
-                                            char *aws_region_out);
-
-H5_DLL herr_t H5FD_s3comms_parse_url(const char *str, parsed_url_t **purl);
-
-H5_DLL herr_t H5FD_s3comms_signing_key(unsigned char *md, const char *secret, const char *region,
-                                       const char *iso8601now);
+/* HTTP request buffer routines */
+H5_DLL hrb_t *H5FD__s3comms_hrb_init_request(const char *verb, const char *resource, const char *host);
+H5_DLL herr_t H5FD__s3comms_hrb_destroy(hrb_t *buf);
+H5_DLL herr_t H5FD__s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value);
+
+/* S3 request buffer routines */
+H5_DLL s3r_t *H5FD__s3comms_s3r_open(const char *url, const H5FD_ros3_fapl_t *fa, const char *fapl_token);
+H5_DLL herr_t H5FD__s3comms_s3r_close(s3r_t *handle);
+H5_DLL size_t H5FD__s3comms_s3r_get_filesize(s3r_t *handle);
+H5_DLL herr_t H5FD__s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest);
+
+/* Functions that construct AWS things */
+H5_DLL herr_t H5FD__s3comms_make_aws_canonical_request(char *canonical_request_dest, int cr_size,
+                                                       char *signed_headers_dest, int sh_size,
+                                                       hrb_t *http_request);
+H5_DLL herr_t H5FD__s3comms_make_aws_signing_key(unsigned char *md, const char *secret, const char *region,
+                                                 const char *iso8601);
+H5_DLL herr_t H5FD__s3comms_make_aws_stringtosign(char *dest, const char *req_str, const char *now,
+                                                  const char *region);
+
+/* Testing routines */
+#ifdef H5FD_S3COMMS_TESTING
+H5_DLL herr_t H5FD__s3comms_load_aws_profile(const char *name, char *key_id_out, char *secret_access_key_out,
+                                             char *aws_region_out);
+#endif /* H5FD_S3COMMS_TESTING */
 
-H5_DLL herr_t H5FD_s3comms_tostringtosign(char *dest, const char *req_str, const char *now,
-                                          const char *region);
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c
index 0475f9a7647..778910915c6 100644
--- a/src/H5FDsec2.c
+++ b/src/H5FDsec2.c
@@ -223,7 +223,7 @@ H5Pset_fapl_sec2(hid_t fapl_id)
 
     FUNC_ENTER_API(FAIL)
 
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
 
     ret_value = H5P_set_driver(plist, H5FD_SEC2, NULL, NULL);
diff --git a/src/H5FDspace.c b/src/H5FDspace.c
index 3bf233d56e5..cdd379e632e 100644
--- a/src/H5FDspace.c
+++ b/src/H5FDspace.c
@@ -86,6 +86,7 @@ static haddr_t
 H5FD__extend(H5FD_t *file, H5FD_mem_t type, hsize_t size)
 {
     haddr_t eoa;                     /* Address of end-of-allocated space */
+    herr_t  status;                  /* Generic status return */
     haddr_t ret_value = HADDR_UNDEF; /* Return value */
 
     FUNC_ENTER_PACKAGE
@@ -96,8 +97,13 @@ H5FD__extend(H5FD_t *file, H5FD_mem_t type, hsize_t size)
     assert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES);
     assert(size > 0);
 
-    /* Get current end-of-allocated space address */
-    eoa = file->cls->get_eoa(file, type);
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(HADDR_UNDEF)
+        {
+            /* Get current end-of-allocated space address */
+            eoa = (file->cls->get_eoa)(file, type);
+        }
+    H5_AFTER_USER_CB(HADDR_UNDEF)
 
     /* Check for overflow when extending */
     if (H5_addr_overflow(eoa, size) || (eoa + size) > file->maxaddr)
@@ -106,9 +112,15 @@ H5FD__extend(H5FD_t *file, H5FD_mem_t type, hsize_t size)
     /* Set the [NOT aligned] address to return */
     ret_value = eoa;
 
-    /* Extend the end-of-allocated space address */
-    eoa += size;
-    if (file->cls->set_eoa(file, type, eoa) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(HADDR_UNDEF)
+        {
+            /* Extend the end-of-allocated space address */
+            eoa += size;
+            status = (file->cls->set_eoa)(file, type, eoa);
+        }
+    H5_AFTER_USER_CB(HADDR_UNDEF)
+    if (status < 0)
         HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "file allocation request failed");
 
 done:
@@ -135,6 +147,7 @@ H5FD__alloc_real(H5FD_t *file, H5FD_mem_t type, hsize_t size, haddr_t *frag_addr
     hsize_t       extra;                   /* Extra space to allocate, to align request */
     unsigned long flags = 0;               /* Driver feature flags */
     bool          use_alloc_size;          /* Just pass alloc size to the driver */
+    herr_t        status;                  /* Generic status return */
     haddr_t       ret_value = HADDR_UNDEF; /* Return value */
 
     FUNC_ENTER_PACKAGE
@@ -149,14 +162,27 @@ H5FD__alloc_real(H5FD_t *file, H5FD_mem_t type, hsize_t size, haddr_t *frag_addr
     assert(size > 0);
 
     /* Check for query driver and call it */
-    if (file->cls->query)
-        (file->cls->query)(file, &flags);
+    if (file->cls->query) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(HADDR_UNDEF)
+            {
+                status = (file->cls->query)(file, &flags);
+            }
+        H5_AFTER_USER_CB(HADDR_UNDEF)
+        if (status < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_CANTGET, HADDR_UNDEF, "driver query request failed");
+    }
 
     /* Check for the driver feature flag */
     use_alloc_size = flags & H5FD_FEAT_USE_ALLOC_SIZE;
 
-    /* Get current end-of-allocated space address */
-    eoa = file->cls->get_eoa(file, type);
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(HADDR_UNDEF)
+        {
+            /* Get current end-of-allocated space address */
+            eoa = (file->cls->get_eoa)(file, type);
+        }
+    H5_AFTER_USER_CB(HADDR_UNDEF)
 
     /* Compute extra space to allocate, if this should be aligned */
     extra = 0;
@@ -179,7 +205,13 @@ H5FD__alloc_real(H5FD_t *file, H5FD_mem_t type, hsize_t size, haddr_t *frag_addr
     /* For all other drivers: the size passed down to the alloc callback is the size + [possibly] alignment
      * size */
     if (file->cls->alloc) {
-        ret_value = (file->cls->alloc)(file, type, H5CX_get_dxpl(), use_alloc_size ? size : size + extra);
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(HADDR_UNDEF)
+            {
+                ret_value =
+                    (file->cls->alloc)(file, type, H5CX_get_dxpl(), use_alloc_size ? size : size + extra);
+            }
+        H5_AFTER_USER_CB(HADDR_UNDEF)
         if (!H5_addr_defined(ret_value))
             HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "driver allocation request failed");
     } /* end if */
@@ -292,7 +324,14 @@ H5FD__free_real(H5FD_t *file, H5FD_mem_t type, haddr_t addr, hsize_t size)
 #ifdef H5FD_ALLOC_DEBUG
         fprintf(stderr, "%s: Letting VFD free space\n", __func__);
 #endif /* H5FD_ALLOC_DEBUG */
-        if ((file->cls->free)(file, type, H5CX_get_dxpl(), addr, size) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                /* Dispatch to driver */
+                ret_value = (file->cls->free)(file, type, H5CX_get_dxpl(), addr, size);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "driver free request failed");
     } /* end if */
     /* Check if this free block is at the end of file allocated space.
@@ -301,7 +340,13 @@ H5FD__free_real(H5FD_t *file, H5FD_mem_t type, haddr_t addr, hsize_t size)
     else if (file->cls->get_eoa) {
         haddr_t eoa;
 
-        eoa = file->cls->get_eoa(file, type);
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                /* Dispatch to driver */
+                eoa = (file->cls->get_eoa)(file, type);
+            }
+        H5_AFTER_USER_CB(FAIL)
 #ifdef H5FD_ALLOC_DEBUG
         fprintf(stderr, "%s: eoa = %" PRIuHADDR "\n", __func__, eoa);
 #endif /* H5FD_ALLOC_DEBUG */
@@ -309,7 +354,14 @@ H5FD__free_real(H5FD_t *file, H5FD_mem_t type, haddr_t addr, hsize_t size)
 #ifdef H5FD_ALLOC_DEBUG
             fprintf(stderr, "%s: Reducing file size to = %" PRIuHADDR "\n", __func__, addr);
 #endif /* H5FD_ALLOC_DEBUG */
-            if (file->cls->set_eoa(file, type, addr) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    /* Dispatch to driver */
+                    ret_value = (file->cls->set_eoa)(file, type, addr);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "set end of space allocation request failed");
         } /* end if */
     }     /* end else-if */
@@ -395,8 +447,14 @@ H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, H5F_t *f, haddr_t blk_end, hsize_
     assert(extra_requested > 0);
     assert(f);
 
-    /* Retrieve the end of the address space */
-    if (HADDR_UNDEF == (eoa = file->cls->get_eoa(file, type)))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Retrieve the end of the address space */
+            eoa = (file->cls->get_eoa)(file, type);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (!H5_addr_defined(eoa))
         HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "driver get_eoa request failed");
 
     /* Adjust block end by base address of the file, to create absolute address */
diff --git a/src/H5FDsplitter.c b/src/H5FDsplitter.c
index 495328c40eb..0536ab29a4e 100644
--- a/src/H5FDsplitter.c
+++ b/src/H5FDsplitter.c
@@ -329,7 +329,7 @@ H5Pget_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config /*out*/)
     config->wo_fapl_id = H5I_INVALID_HID;
 
     /* Check and get the splitter fapl */
-    if (NULL == (plist_ptr = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist_ptr = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
     if (H5FD_SPLITTER != H5P_peek_driver(plist_ptr))
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver");
diff --git a/src/H5FDsplitter.h b/src/H5FDsplitter.h
index 66e3ebaf38f..48d7107c4ba 100644
--- a/src/H5FDsplitter.h
+++ b/src/H5FDsplitter.h
@@ -31,11 +31,11 @@
 
 /**
  * Maximum length of a filename/path string in the Write-Only channel,
- * including the NULL-terminator.
+ * including the NULL-terminator. \since 1.10.7
  */
 #define H5FD_SPLITTER_PATH_MAX 4096
 
-/** Semi-unique constant used to help identify structure pointers */
+/** Semi-unique constant used to help identify structure pointers \since 1.10.7 */
 #define H5FD_SPLITTER_MAGIC 0x2B916880
 
 //! 
diff --git a/src/H5FDsubfiling/H5FDioc.c b/src/H5FDsubfiling/H5FDioc.c
index e1c61883169..f178b45ac6f 100644
--- a/src/H5FDsubfiling/H5FDioc.c
+++ b/src/H5FDsubfiling/H5FDioc.c
@@ -319,7 +319,7 @@ H5Pset_fapl_ioc(hid_t fapl_id, H5FD_ioc_config_t *vfd_config)
 
     FUNC_ENTER_API(FAIL)
 
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
 
     /* Initialize driver, if it's not yet */
@@ -369,7 +369,7 @@ H5Pget_fapl_ioc(hid_t fapl_id, H5FD_ioc_config_t *config_out)
     /* Check arguments */
     if (config_out == NULL)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "config_out is NULL");
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
 
     /* Initialize driver, if it's not yet */
@@ -1249,7 +1249,7 @@ H5FD__ioc_delete(const char *name, hid_t fapl)
         if (H5FD__ioc_init() < 0)
             HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "can't initialize driver");
 
-    if (NULL == (plist = H5P_object_verify(fapl, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
     assert(H5FD_IOC == H5P_peek_driver(plist));
 
diff --git a/src/H5FDsubfiling/H5FDsubfiling.c b/src/H5FDsubfiling/H5FDsubfiling.c
index 8deb746cfa0..bafe0d65181 100644
--- a/src/H5FDsubfiling/H5FDsubfiling.c
+++ b/src/H5FDsubfiling/H5FDsubfiling.c
@@ -451,7 +451,7 @@ H5Pset_fapl_subfiling(hid_t fapl_id, const H5FD_subfiling_config_t *vfd_config)
 
     FUNC_ENTER_API(FAIL)
 
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
 
     /* Initialize driver, if it's not yet */
@@ -479,7 +479,7 @@ H5Pset_fapl_subfiling(hid_t fapl_id, const H5FD_subfiling_config_t *vfd_config)
         comm = MPI_COMM_WORLD;
 
     /* Set MPI parameters on IOC FAPL */
-    if (NULL == (ioc_plist = H5P_object_verify(vfd_config->ioc_fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (ioc_plist = H5P_object_verify(vfd_config->ioc_fapl_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, FAIL, "not a file access property list");
     if (H5P_set(ioc_plist, H5F_ACS_MPI_PARAMS_COMM_NAME, &comm) < 0)
         HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't set MPI communicator on plist");
@@ -532,7 +532,7 @@ H5Pget_fapl_subfiling(hid_t fapl_id, H5FD_subfiling_config_t *config_out)
 
     if (config_out == NULL)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "config_out is NULL");
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
 
     /* Initialize driver, if it's not yet */
@@ -1784,7 +1784,7 @@ H5FD__subfiling_delete(const char *name, hid_t fapl)
         if (H5FD__subfiling_init() < 0)
             HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "can't initialize driver");
 
-    if (NULL == (plist = H5P_object_verify(fapl, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
 
     if (H5FD_SUBFILING != H5P_peek_driver(plist))
diff --git a/src/H5FDsubfiling/H5subfiling_common.c b/src/H5FDsubfiling/H5subfiling_common.c
index 9b393412242..3e0fc3abf48 100644
--- a/src/H5FDsubfiling/H5subfiling_common.c
+++ b/src/H5FDsubfiling/H5subfiling_common.c
@@ -522,7 +522,7 @@ H5FD__subfiling_open_stub_file(const char *name, unsigned flags, MPI_Comm file_c
 
         if ((fapl_id = H5P_create_id(H5P_CLS_FILE_ACCESS_g, false)) < 0)
             HGOTO_ERROR(H5E_VFL, H5E_CANTREGISTER, FAIL, "can't create FAPL for stub file");
-        if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+        if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
             HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, FAIL, "not a file access property list");
 
         /* Use MPI I/O driver for stub file to allow access to vector I/O */
diff --git a/src/H5FDwindows.c b/src/H5FDwindows.c
index 182958fcb15..4852c90845f 100644
--- a/src/H5FDwindows.c
+++ b/src/H5FDwindows.c
@@ -47,7 +47,7 @@ H5Pset_fapl_windows(hid_t fapl_id)
 
     FUNC_ENTER_API(FAIL)
 
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
 
     ret_value = H5P_set_driver(plist, H5FD_WINDOWS, NULL, NULL);
diff --git a/src/H5FO.c b/src/H5FO.c
index a5c47fdb4bd..3a57a57b3cb 100644
--- a/src/H5FO.c
+++ b/src/H5FO.c
@@ -20,12 +20,13 @@
 
 #define H5F_FRIEND /*suppress error about including H5Fpkg	  */
 
-#include "H5Eprivate.h"  /* Error handling		  	*/
-#include "H5Fpkg.h"      /* File access                          */
-#include "H5FLprivate.h" /* Free lists                           */
-#include "H5FOprivate.h" /* File objects                         */
-#include "H5Oprivate.h"  /* Object headers		  	*/
-#include "H5SLprivate.h" /* Skip Lists                               */
+#include "H5private.h"   /* Generic Functions                     */
+#include "H5Eprivate.h"  /* Error handling		  	  */
+#include "H5Fpkg.h"      /* File access                           */
+#include "H5FLprivate.h" /* Free lists                            */
+#include "H5FOprivate.h" /* File objects                          */
+#include "H5Oprivate.h"  /* Object headers		  	  */
+#include "H5SLprivate.h" /* Skip Lists                            */
 
 /* Private typedefs */
 
diff --git a/src/H5Fint.c b/src/H5Fint.c
index d2ede2f978d..9f2b41f572f 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -3501,9 +3501,16 @@ H5F_object_flush_cb(H5F_t *f, hid_t obj_id)
     assert(f->shared);
 
     /* Invoke object flush callback if there is one */
-    if (f->shared->object_flush.func &&
-        f->shared->object_flush.func(obj_id, f->shared->object_flush.udata) < 0)
-        HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "object flush callback returns error");
+    if (f->shared->object_flush.func) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = f->shared->object_flush.func(obj_id, f->shared->object_flush.udata);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
+            HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "object flush callback returns error");
+    }
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
diff --git a/src/H5Fmodule.h b/src/H5Fmodule.h
index e83214bb40e..2551e13aaeb 100644
--- a/src/H5Fmodule.h
+++ b/src/H5Fmodule.h
@@ -658,7 +658,7 @@
  *
  * HDF5 employs an extremely flexible mechanism called the virtual file layer, or VFL, for file
  * I/O. A full understanding of the VFL is only necessary if you plan to write your own drivers
- * see \ref VFL in the HDF5 Technical Notes.
+ * see \ref VFLTN in the HDF5 \ref TN.
  *
  * For our
  * purposes here, it is sufficient to know that the low-level drivers used for file I/O reside in the
@@ -691,7 +691,7 @@
  *
  * If an application requires a special-purpose low-level driver, the VFL provides a public API for
  * creating one. For more information on how to create a driver,
- * see \ref VFL in the HDF5 Technical Notes.
+ * see \ref VFLTN in the HDF5 \ref TN.
  *
  * \subsubsection subsubsec_file_alternate_drivers_id Identifying the Previously‐used File Driver
  * When creating a new HDF5 file, no history exists, so the file driver must be specified if it is to be
@@ -1285,7 +1285,7 @@
  * that the Memory virtual file driver, #H5FD_CORE, is used. The Memory file driver is also known
  * as the Core file driver.
  *
- * Links to the \ref VFL and List of Functions documents can be found in the HDF5 \ref TN.
+ * Links to the \ref VFLTN and List of Functions documents can be found in the HDF5 \ref TN.
  *
  * \subsection subsec_file_image_api File Image C API Call Syntax
  * The C API function calls described in this chapter fall into two categories: low-level routines that are
@@ -2653,7 +2653,7 @@
  * of functions that deal with advanced file management tasks and use cases:
  * 1. The control of the HDF5 \ref MDC
  * 2. The use of (MPI-) \ref PH5F HDF5
- * 3. The \ref SWMR pattern
+ * 3. The \ref SWMRTN pattern
  *
  * \defgroup MDC Metadata Cache
  * \ingroup H5F
diff --git a/src/H5Fmpi.c b/src/H5Fmpi.c
index 78909aa1bf2..9b8039a20d6 100644
--- a/src/H5Fmpi.c
+++ b/src/H5Fmpi.c
@@ -395,7 +395,7 @@ H5F_mpi_retrieve_comm(hid_t loc_id, hid_t acspl_id, MPI_Comm *mpi_comm)
         unsigned long      driver_feat_flags;
         H5FD_class_t      *driver_class = NULL;
 
-        if (NULL == (plist = H5P_object_verify(acspl_id, H5P_FILE_ACCESS)))
+        if (NULL == (plist = H5P_object_verify(acspl_id, H5P_FILE_ACCESS, true)))
             HGOTO_ERROR(H5E_FILE, H5E_BADTYPE, FAIL, "not a file access list");
 
         if (H5P_peek(plist, H5F_ACS_FILE_DRV_NAME, &driver_prop) < 0)
diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h
index 1fb32c2c142..bacd6fa01a4 100644
--- a/src/H5Fpublic.h
+++ b/src/H5Fpublic.h
@@ -32,13 +32,13 @@
 /* NOTE: 0x0008u was H5F_ACC_DEBUG, now deprecated */
 #define H5F_ACC_CREAT (0x0010u) /**< Create non-existing files  */
 #define H5F_ACC_SWMR_WRITE                                                                                   \
-    (0x0020u) /**< Indicate that this file is open for writing in a                                          \
+    (0x0020u) /**< Indicates that this file is open for writing in a                                         \
                *   single-writer/multi-reader (SWMR)  scenario.                                              \
                *   Note that the process(es) opening the file for reading                                    \
                *   must open the file with #H5F_ACC_RDONLY and use the                                       \
                *   #H5F_ACC_SWMR_READ access flag. */
 #define H5F_ACC_SWMR_READ                                                                                    \
-    (0x0040u) /**< Indicate that this file is open for reading in a                                          \
+    (0x0040u) /**< Indicates that this file is open for reading in a                                         \
                * single-writer/multi-reader (SWMR) scenario. Note that                                       \
                * the process(es) opening the file for SWMR reading must                                      \
                * also open the file with the #H5F_ACC_RDONLY flag.  */
@@ -204,6 +204,7 @@ typedef enum H5F_file_space_type_t {
 } H5F_file_space_type_t;
 
 //! 
+/** Total number of metadata read retry types \since 1.10.0 */
 #define H5F_NUM_METADATA_READ_RETRY_TYPES 21
 
 /**
@@ -229,11 +230,12 @@ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata);
 #define H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS                                                             \
     (0x0001u) /**< Suppress errors for numeric datatypes with an unusually                                   \
                *   high number of unused bits.  See documentation for                                        \
-               *   H5Pset_relax_file_integrity_checks for details. */
+               *   H5Pset_relax_file_integrity_checks() for details. */
 #define H5F_RFIC_ALL                                                                                         \
-    (H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS) /**< Suppress all format integrity check errors.  See         \
-                                                *   documentation for H5Pset_relax_file_integrity_checks     \
-                                                *   for details. */
+    (H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS) /**< Suppress all format integrity check                      \
+                                                * errors.  See documentation for                             \
+                                                * H5Pset_relax_file_integrity_checks()                       \
+                                                * for details. */
 
 /*********************/
 /* Public Prototypes */
diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c
index 1a84c455f83..095f17024b6 100644
--- a/src/H5Fsuper.c
+++ b/src/H5Fsuper.c
@@ -364,9 +364,9 @@ H5F__super_read(H5F_t *f, H5P_genplist_t *fa_plist, bool initial_read)
             /* Try detecting file's signature */
             /* (Don't leave before Bcast, to avoid hang on error) */
             H5E_PAUSE_ERRORS
-            {
-                H5FD_locate_signature(file, &super_addr);
-            }
+                {
+                    H5FD_locate_signature(file, &super_addr);
+                }
             H5E_RESUME_ERRORS
         } /* end if */
 
diff --git a/src/H5Gint.c b/src/H5Gint.c
index c037b99de8b..34dbfb97331 100644
--- a/src/H5Gint.c
+++ b/src/H5Gint.c
@@ -857,8 +857,13 @@ H5G__iterate_cb(const H5O_link_t *lnk, void *_udata)
     switch (udata->lnk_op.op_type) {
 #ifndef H5_NO_DEPRECATED_SYMBOLS
         case H5G_LINK_OP_OLD:
-            /* Make the old-type application callback */
-            ret_value = (udata->lnk_op.op_func.op_old)(udata->gid, lnk->name, udata->op_data);
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(H5_ITER_ERROR)
+                {
+                    /* Make the old-type application callback */
+                    ret_value = (udata->lnk_op.op_func.op_old)(udata->gid, lnk->name, udata->op_data);
+                }
+            H5_AFTER_USER_CB(H5_ITER_ERROR)
             break;
 #endif /* H5_NO_DEPRECATED_SYMBOLS */
 
@@ -869,8 +874,13 @@ H5G__iterate_cb(const H5O_link_t *lnk, void *_udata)
             if (H5G_link_to_info(udata->link_loc, lnk, &info) < 0)
                 HGOTO_ERROR(H5E_SYM, H5E_CANTGET, H5_ITER_ERROR, "unable to get info for link");
 
-            /* Make the application callback */
-            ret_value = (udata->lnk_op.op_func.op_new)(udata->gid, lnk->name, &info, udata->op_data);
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(H5_ITER_ERROR)
+                {
+                    /* Make the application callback */
+                    ret_value = (udata->lnk_op.op_func.op_new)(udata->gid, lnk->name, &info, udata->op_data);
+                }
+            H5_AFTER_USER_CB(H5_ITER_ERROR)
         } break;
 
         default:
@@ -1010,8 +1020,13 @@ H5G__visit_cb(const H5O_link_t *lnk, void *_udata)
     if (H5G_link_to_info(udata->curr_loc->oloc, lnk, &info) < 0)
         HGOTO_ERROR(H5E_SYM, H5E_CANTGET, H5_ITER_ERROR, "unable to get info for link");
 
-    /* Make the application callback */
-    ret_value = (udata->op)(udata->gid, udata->path, &info, udata->op_data);
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(H5_ITER_ERROR)
+        {
+            /* Make the application callback */
+            ret_value = (udata->op)(udata->gid, udata->path, &info, udata->op_data);
+        }
+    H5_AFTER_USER_CB(H5_ITER_ERROR)
 
     /* Check for doing more work */
     if (ret_value == H5_ITER_CONT && lnk->type == H5L_TYPE_HARD) {
diff --git a/src/H5Glink.c b/src/H5Glink.c
index 4614d622a3c..2888481ae69 100644
--- a/src/H5Glink.c
+++ b/src/H5Glink.c
@@ -233,11 +233,17 @@ H5G_link_to_info(const H5O_loc_t *link_loc, const H5O_link_t *lnk, H5L_info2_t *
                 if (link_class != NULL && link_class->query_func != NULL) {
                     ssize_t cb_ret; /* Return value from UD callback */
 
-                    /* Call the link's query routine to retrieve the user-defined link's value size */
-                    /* (in case the query routine packs/unpacks the link value in some way that changes its
-                     * size) */
-                    if ((cb_ret = (link_class->query_func)(lnk->name, lnk->u.ud.udata, lnk->u.ud.size, NULL,
-                                                           (size_t)0)) < 0)
+                    /* Prepare & restore library for user callback */
+                    H5_BEFORE_USER_CB(FAIL)
+                        {
+                            /* Call the link's query routine to retrieve the user-defined link's value size */
+                            /* (in case the query routine packs/unpacks the link value in some way that
+                             * changes its size) */
+                            cb_ret = (link_class->query_func)(lnk->name, lnk->u.ud.udata, lnk->u.ud.size,
+                                                              NULL, (size_t)0);
+                        }
+                    H5_AFTER_USER_CB(FAIL)
+                    if (cb_ret < 0)
                         HGOTO_ERROR(H5E_LINK, H5E_CALLBACK, FAIL,
                                     "query buffer size callback returned failure");
 
diff --git a/src/H5Gnode.c b/src/H5Gnode.c
index d647096b18c..336eb76d6cc 100644
--- a/src/H5Gnode.c
+++ b/src/H5Gnode.c
@@ -1420,9 +1420,9 @@ H5G_node_debug(H5F_t *f, haddr_t addr, FILE *stream, int indent, int fwidth, had
 
     /* Try loading symbol table node */
     H5E_PAUSE_ERRORS
-    {
-        sn = (H5G_node_t *)H5AC_protect(f, H5AC_SNODE, addr, f, H5AC__READ_ONLY_FLAG);
-    }
+        {
+            sn = (H5G_node_t *)H5AC_protect(f, H5AC_SNODE, addr, f, H5AC__READ_ONLY_FLAG);
+        }
     H5E_RESUME_ERRORS
     if (sn) {
         unsigned u; /* Local index variable */
diff --git a/src/H5Gstab.c b/src/H5Gstab.c
index c86406422b7..5c071b25f9c 100644
--- a/src/H5Gstab.c
+++ b/src/H5Gstab.c
@@ -975,9 +975,9 @@ H5G__stab_valid(H5O_loc_t *grp_oloc, H5O_stab_t *alt_stab)
 
     /* Check if the symbol table message's b-tree address is valid */
     H5E_PAUSE_ERRORS
-    {
-        bt_status = H5B_valid(grp_oloc->file, H5B_SNODE, stab.btree_addr);
-    }
+        {
+            bt_status = H5B_valid(grp_oloc->file, H5B_SNODE, stab.btree_addr);
+        }
     H5E_RESUME_ERRORS
 
     if (bt_status < 0) {
@@ -994,9 +994,9 @@ H5G__stab_valid(H5O_loc_t *grp_oloc, H5O_stab_t *alt_stab)
 
     /* Check if the symbol table message's heap address is valid */
     H5E_PAUSE_ERRORS
-    {
-        heap = H5HL_protect(grp_oloc->file, stab.heap_addr, H5AC__READ_ONLY_FLAG);
-    }
+        {
+            heap = H5HL_protect(grp_oloc->file, stab.heap_addr, H5AC__READ_ONLY_FLAG);
+        }
     H5E_RESUME_ERRORS
 
     if (NULL == heap) {
diff --git a/src/H5Gtraverse.c b/src/H5Gtraverse.c
index 4f0e03fb849..89fac987c72 100644
--- a/src/H5Gtraverse.c
+++ b/src/H5Gtraverse.c
@@ -187,15 +187,32 @@ H5G__traverse_ud(const H5G_loc_t *grp_loc /*in,out*/, const H5O_link_t *lnk, H5G
         /* Invoke user-defined callback function */
 #ifndef H5_NO_DEPRECATED_SYMBOLS
     /* (Backwardly compatible with v0 H5L_class_t traversal callback) */
-    if (link_class->version == H5L_LINK_CLASS_T_VERS_0)
-        cb_return = (((const H5L_class_0_t *)link_class)->trav_func)(lnk->name, cur_grp, lnk->u.ud.udata,
-                                                                     lnk->u.ud.size, H5CX_get_lapl());
-    else
-        cb_return = (link_class->trav_func)(lnk->name, cur_grp, lnk->u.ud.udata, lnk->u.ud.size,
-                                            H5CX_get_lapl(), H5CX_get_dxpl());
+    if (link_class->version == H5L_LINK_CLASS_T_VERS_0) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                cb_return = (((const H5L_class_0_t *)link_class)->trav_func)(
+                    lnk->name, cur_grp, lnk->u.ud.udata, lnk->u.ud.size, H5CX_get_lapl());
+            }
+        H5_AFTER_USER_CB(FAIL)
+    }
+    else {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                cb_return = (link_class->trav_func)(lnk->name, cur_grp, lnk->u.ud.udata, lnk->u.ud.size,
+                                                    H5CX_get_lapl(), H5CX_get_dxpl());
+            }
+        H5_AFTER_USER_CB(FAIL)
+    }
 #else  /* H5_NO_DEPRECATED_SYMBOLS */
-    cb_return = (link_class->trav_func)(lnk->name, cur_grp, lnk->u.ud.udata, lnk->u.ud.size, H5CX_get_lapl(),
-                                        H5CX_get_dxpl());
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            cb_return = (link_class->trav_func)(lnk->name, cur_grp, lnk->u.ud.udata, lnk->u.ud.size,
+                                                H5CX_get_lapl(), H5CX_get_dxpl());
+        }
+    H5_AFTER_USER_CB(FAIL)
 #endif /* H5_NO_DEPRECATED_SYMBOLS */
 
     /* Resume recording errors, if we were just checking for object's existence */
diff --git a/src/H5I.c b/src/H5I.c
index 731f0f31c56..dab4eed59a7 100644
--- a/src/H5I.c
+++ b/src/H5I.c
@@ -73,16 +73,15 @@ static int H5I__iterate_pub_cb(void *obj, hid_t id, void *udata);
 /*******************/
 
 /*-------------------------------------------------------------------------
- * Function:    H5Iregister_type
+ * Function:    H5Iregister_type2
  *
- * Purpose:     Public interface to H5I_register_type.  Creates a new type
+ * Purpose:     Public interface to H5I_register_type2.  Creates a new type
  *              of ID's to give out.  A specific number (RESERVED) of type
  *              entries may be reserved to enable "constant" values to be handed
  *              out which are valid IDs in the type, but which do not map to any
- *              data structures and are not allocated dynamically later. HASH_SIZE is
- *              the minimum hash table size to use for the type. FREE_FUNC is
- *              called with an object pointer when the object is removed from
- *              the type.
+ *              data structures and are not allocated dynamically later.
+ *              FREE_FUNC is called with an object pointer when the object is
+ *              removed from the type.
  *
  * Return:      Success:    Type ID of the new type
  *              Failure:    H5I_BADID
@@ -90,65 +89,18 @@ static int H5I__iterate_pub_cb(void *obj, hid_t id, void *udata);
  *-------------------------------------------------------------------------
  */
 H5I_type_t
-H5Iregister_type(size_t H5_ATTR_UNUSED hash_size, unsigned reserved, H5I_free_t free_func)
+H5Iregister_type2(unsigned reserved, H5I_free_t free_func)
 {
-    H5I_class_t *cls       = NULL;      /* New ID class */
-    H5I_type_t   new_type  = H5I_BADID; /* New ID type value */
-    H5I_type_t   ret_value = H5I_BADID; /* Return value */
+    H5I_type_t ret_value = H5I_BADID;
 
     FUNC_ENTER_API(H5I_BADID)
 
-    /* Generate a new H5I_type_t value */
-
-    /* Increment the number of types */
-    if (H5I_next_type_g < H5I_MAX_NUM_TYPES) {
-        new_type = (H5I_type_t)H5I_next_type_g;
-        H5I_next_type_g++;
-    }
-    else {
-        bool done; /* Indicate that search was successful */
-        int  i;
-
-        /* Look for a free type to give out */
-        done = false;
-        for (i = H5I_NTYPES; i < H5I_MAX_NUM_TYPES && done == false; i++) {
-            if (NULL == H5I_type_info_array_g[i]) {
-                /* Found a free type ID */
-                new_type = (H5I_type_t)i;
-                done     = true;
-            }
-        }
-
-        /* Verify that we found a type to give out */
-        if (done == false)
-            HGOTO_ERROR(H5E_ID, H5E_NOSPACE, H5I_BADID, "Maximum number of ID types exceeded");
-    }
-
-    /* Allocate new ID class */
-    if (NULL == (cls = H5MM_calloc(sizeof(H5I_class_t))))
-        HGOTO_ERROR(H5E_ID, H5E_CANTALLOC, H5I_BADID, "ID class allocation failed");
-
-    /* Initialize class fields */
-    cls->type      = new_type;
-    cls->flags     = H5I_CLASS_IS_APPLICATION;
-    cls->reserved  = reserved;
-    cls->free_func = free_func;
-
-    /* Register the new ID class */
-    if (H5I_register_type(cls) < 0)
+    if (H5I_BADID == (ret_value = H5I__register_type_common(reserved, free_func)))
         HGOTO_ERROR(H5E_ID, H5E_CANTINIT, H5I_BADID, "can't initialize ID class");
 
-    /* Set return value */
-    ret_value = new_type;
-
 done:
-    /* Clean up on error */
-    if (ret_value < 0)
-        if (cls)
-            cls = H5MM_xfree(cls);
-
     FUNC_LEAVE_API(ret_value)
-} /* end H5Iregister_type() */
+} /* end H5Iregister_type2() */
 
 /*-------------------------------------------------------------------------
  * Function:    H5Itype_exists
@@ -668,13 +620,18 @@ H5Iis_valid(hid_t id)
 static int
 H5I__search_cb(void *obj, hid_t id, void *_udata)
 {
-    H5I_search_ud_t *udata = (H5I_search_ud_t *)_udata; /* User data for callback */
-    herr_t           cb_ret_val;                        /* User callback return value */
-    int              ret_value = H5_ITER_ERROR;         /* Callback return value */
+    H5I_search_ud_t *udata      = (H5I_search_ud_t *)_udata; /* User data for callback */
+    herr_t           cb_ret_val = FAIL;                      /* User callback return value */
+    int              ret_value  = H5_ITER_ERROR;             /* Callback return value */
 
     FUNC_ENTER_PACKAGE_NOERR
 
-    cb_ret_val = (*udata->app_cb)(obj, id, udata->app_key);
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB_NOERR(H5_ITER_ERROR)
+        {
+            cb_ret_val = (*udata->app_cb)(obj, id, udata->app_key);
+        }
+    H5_AFTER_USER_CB_NOERR(H5_ITER_ERROR)
 
     /* Set the return value based on the callback's return value */
     if (cb_ret_val > 0) {
@@ -758,8 +715,13 @@ H5I__iterate_pub_cb(void H5_ATTR_UNUSED *obj, hid_t id, void *_udata)
 
     FUNC_ENTER_PACKAGE_NOERR
 
-    /* Invoke the callback */
-    cb_ret_val = (*udata->op)(id, udata->op_data);
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB_NOERR(H5_ITER_ERROR)
+        {
+            /* Invoke the callback */
+            cb_ret_val = (*udata->op)(id, udata->op_data);
+        }
+    H5_AFTER_USER_CB_NOERR(H5_ITER_ERROR)
 
     /* Set the return value based on the callback's return value */
     if (cb_ret_val > 0)
diff --git a/src/H5Ideprec.c b/src/H5Ideprec.c
new file mode 100644
index 00000000000..0d172f668bd
--- /dev/null
+++ b/src/H5Ideprec.c
@@ -0,0 +1,97 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group.                                               *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of HDF5.  The full HDF5 copyright notice, including     *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the LICENSE file, which can be found at the root of the source code       *
+ * distribution tree, or in https://www.hdfgroup.org/licenses.               *
+ * If you do not have access to either file, you may request a copy from     *
+ * help@hdfgroup.org.                                                        *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*-------------------------------------------------------------------------
+ *
+ * Created:	H5Ideprec.c
+ *
+ * Purpose:	Deprecated functions from the H5I interface.  These
+ *          functions are here for compatibility purposes and may be
+ *          removed in the future.  Applications should switch to the
+ *          newer APIs.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+/****************/
+/* Module Setup */
+/****************/
+
+#include "H5Imodule.h" /* This source code file is part of the H5I module */
+
+/***********/
+/* Headers */
+/***********/
+#include "H5private.h"  /* Generic Functions                        */
+#include "H5Eprivate.h" /* Error handling                           */
+#include "H5Ipkg.h"     /* File access                              */
+
+/****************/
+/* Local Macros */
+/****************/
+
+/******************/
+/* Local Typedefs */
+/******************/
+
+/********************/
+/* Package Typedefs */
+/********************/
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+/*********************/
+/* Package Variables */
+/*********************/
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+#ifndef H5_NO_DEPRECATED_SYMBOLS
+/*-------------------------------------------------------------------------
+ * Function:    H5Iregister_type1
+ *
+ * Purpose:     Public interface to H5I_register_type.  Creates a new type
+ *              of ID's to give out.  A specific number (RESERVED) of type
+ *              entries may be reserved to enable "constant" values to be handed
+ *              out which are valid IDs in the type, but which do not map to any
+ *              data structures and are not allocated dynamically later. HASH_SIZE is
+ *              the minimum hash table size to use for the type. FREE_FUNC is
+ *              called with an object pointer when the object is removed from
+ *              the type.
+ *
+ * Return:      Success:    Type ID of the new type
+ *              Failure:    H5I_BADID
+ *
+ *-------------------------------------------------------------------------
+ */
+H5I_type_t
+H5Iregister_type1(size_t H5_ATTR_UNUSED hash_size, unsigned reserved, H5I_free_t free_func)
+{
+    H5I_type_t ret_value = H5I_BADID;
+
+    FUNC_ENTER_API(H5I_BADID)
+
+    if (H5I_BADID == (ret_value = H5I__register_type_common(reserved, free_func)))
+        HGOTO_ERROR(H5E_ID, H5E_CANTINIT, H5I_BADID, "can't initialize ID class");
+
+done:
+    FUNC_LEAVE_API(ret_value)
+} /* end H5Iregister_type1() */
+#endif /* H5_NO_DEPRECATED_SYMBOLS */
diff --git a/src/H5Iint.c b/src/H5Iint.c
index 785dd9f0e86..fc59b1583bd 100644
--- a/src/H5Iint.c
+++ b/src/H5Iint.c
@@ -156,6 +156,76 @@ H5I_term_package(void)
     FUNC_LEAVE_NOAPI(in_use)
 } /* end H5I_term_package() */
 
+/*-------------------------------------------------------------------------
+ * Function:    H5I__register_type_common
+ *
+ * Purpose:     Common functionality for H5Iregister_type(1|2)
+ *
+ * Return:      Success:    Type ID of the new type
+ *              Failure:    H5I_BADID
+ *-------------------------------------------------------------------------
+ */
+H5I_type_t
+H5I__register_type_common(unsigned reserved, H5I_free_t free_func)
+{
+    H5I_class_t *cls       = NULL;      /* New ID class */
+    H5I_type_t   new_type  = H5I_BADID; /* New ID type value */
+    H5I_type_t   ret_value = H5I_BADID; /* Return value */
+
+    FUNC_ENTER_PACKAGE
+
+    /* Generate a new H5I_type_t value */
+
+    /* Increment the number of types */
+    if (H5I_next_type_g < H5I_MAX_NUM_TYPES) {
+        new_type = (H5I_type_t)H5I_next_type_g;
+        H5I_next_type_g++;
+    }
+    else {
+        bool done; /* Indicate that search was successful */
+        int  i;
+
+        /* Look for a free type to give out */
+        done = false;
+        for (i = H5I_NTYPES; i < H5I_MAX_NUM_TYPES && done == false; i++) {
+            if (NULL == H5I_type_info_array_g[i]) {
+                /* Found a free type ID */
+                new_type = (H5I_type_t)i;
+                done     = true;
+            }
+        }
+
+        /* Verify that we found a type to give out */
+        if (done == false)
+            HGOTO_ERROR(H5E_ID, H5E_NOSPACE, H5I_BADID, "Maximum number of ID types exceeded");
+    }
+
+    /* Allocate new ID class */
+    if (NULL == (cls = H5MM_calloc(sizeof(H5I_class_t))))
+        HGOTO_ERROR(H5E_ID, H5E_CANTALLOC, H5I_BADID, "ID class allocation failed");
+
+    /* Initialize class fields */
+    cls->type      = new_type;
+    cls->flags     = H5I_CLASS_IS_APPLICATION;
+    cls->reserved  = reserved;
+    cls->free_func = free_func;
+
+    /* Register the new ID class */
+    if (H5I_register_type(cls) < 0)
+        HGOTO_ERROR(H5E_ID, H5E_CANTINIT, H5I_BADID, "can't initialize ID class");
+
+    /* Set return value */
+    ret_value = new_type;
+
+done:
+    /* Clean up on error */
+    if (ret_value == H5I_BADID)
+        if (cls)
+            cls = H5MM_xfree(cls);
+
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5I__register_type_common() */
+
 /*-------------------------------------------------------------------------
  * Function:    H5I_register_type
  *
@@ -379,8 +449,16 @@ H5I__mark_node(void *_info, void H5_ATTR_UNUSED *key, void *_udata)
     if (udata->force || (info->count - (!udata->app_ref * info->app_count)) <= 1) {
         /* Check if this is an un-realized future object */
         if (info->is_future) {
-            /* Discard the future object */
-            if ((info->discard_cb)(info->u.object) < 0) {
+            herr_t status;
+
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB_NOCHECK
+                {
+                    /* Discard the future object */
+                    status = (info->discard_cb)(info->u.object);
+                }
+            H5_AFTER_USER_CB_NOCHECK
+            if (status < 0) {
                 if (udata->force) {
                     /* Indicate node should be removed from list */
                     mark = true;
@@ -393,12 +471,24 @@ H5I__mark_node(void *_info, void H5_ATTR_UNUSED *key, void *_udata)
         }
         else {
             /* Check for a 'free' function and call it, if it exists */
-            if (udata->type_info->cls->free_func &&
-                (udata->type_info->cls->free_func)(info->u.object, H5_REQUEST_NULL) < 0) {
-                if (udata->force) {
+            if (udata->type_info->cls->free_func) {
+                herr_t status;
+
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB_NOCHECK
+                    {
+                        status = (udata->type_info->cls->free_func)(info->u.object, H5_REQUEST_NULL);
+                    }
+                H5_AFTER_USER_CB_NOCHECK
+                if (status < 0) {
+                    if (udata->force) {
+                        /* Indicate node should be removed from list */
+                        mark = true;
+                    }
+                }
+                else
                     /* Indicate node should be removed from list */
                     mark = true;
-                }
             }
             else {
                 /* Indicate node should be removed from list */
@@ -972,15 +1062,31 @@ H5I__dec_ref(hid_t id, void **request)
      */
     if (1 == info->count) {
         H5I_type_info_t *type_info; /*ptr to the type    */
+        bool             remove_node = false;
 
         /* Get the ID's type */
         type_info = H5I_type_info_array_g[H5I_TYPE(id)];
 
-        if (!type_info->cls->free_func || (type_info->cls->free_func)(info->u.object, request) >= 0) {
+        if (type_info->cls->free_func) {
+            herr_t status;
+
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB((-1))
+                {
+                    status = (type_info->cls->free_func)(info->u.object, request);
+                }
+            H5_AFTER_USER_CB((-1))
+
+            if (status >= 0)
+                remove_node = true;
+        }
+        else
+            remove_node = true;
+
+        if (remove_node) {
             /* Remove the node from the type */
             if (NULL == H5I__remove_common(type_info, id))
                 HGOTO_ERROR(H5E_ID, H5E_CANTDELETE, (-1), "can't remove ID node");
-            ret_value = 0;
         } /* end if */
         else
             ret_value = -1;
@@ -1451,7 +1557,7 @@ H5I__iterate_cb(void *_item, void H5_ATTR_UNUSED *_key, void *_udata)
     if ((!udata->app_ref) || (info->app_count > 0)) {
         H5I_type_t type = udata->obj_type;
         void      *object;
-        herr_t     cb_ret_val;
+        herr_t     cb_ret_val = FAIL;
 
         /* The stored object pointer might be an H5VL_object_t, in which
          * case we'll need to get the wrapped object struct (H5F_t *, etc.).
@@ -1459,7 +1565,12 @@ H5I__iterate_cb(void *_item, void H5_ATTR_UNUSED *_key, void *_udata)
         object = H5I__unwrap(info->u.object, type);
 
         /* Invoke callback function */
-        cb_ret_val = (*udata->user_func)((void *)object, info->id, udata->user_udata);
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB_NOERR(H5_ITER_ERROR)
+            {
+                cb_ret_val = (*udata->user_func)((void *)object, info->id, udata->user_udata);
+            }
+        H5_AFTER_USER_CB_NOERR(H5_ITER_ERROR)
 
         /* Set the return value based on the callback's return value */
         if (cb_ret_val > 0)
@@ -1580,12 +1691,19 @@ H5I__find_id(hid_t id)
 
     /* Check if this is a future ID */
     if (id_info && id_info->is_future) {
-        hid_t actual_id = H5I_INVALID_HID; /* ID for actual object */
-        void *future_object;               /* Pointer to the future object */
-        void *actual_object;               /* Pointer to the actual object */
-
-        /* Invoke the realize callback, to get the actual object */
-        if ((id_info->realize_cb)(id_info->u.object, &actual_id) < 0)
+        hid_t  actual_id = H5I_INVALID_HID; /* ID for actual object */
+        void  *future_object;               /* Pointer to the future object */
+        void  *actual_object;               /* Pointer to the actual object */
+        herr_t status = FAIL;
+
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB_NOERR(NULL)
+            {
+                /* Invoke the realize callback, to get the actual object */
+                status = (id_info->realize_cb)(id_info->u.object, &actual_id);
+            }
+        H5_AFTER_USER_CB_NOERR(NULL)
+        if (status < 0)
             HGOTO_DONE(NULL);
 
         /* Verify that we received a valid ID, of the same type */
@@ -1600,8 +1718,14 @@ H5I__find_id(hid_t id)
         assert(actual_object);
         id_info->u.object = actual_object;
 
-        /* Discard the future object */
-        if ((id_info->discard_cb)(future_object) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB_NOERR(NULL)
+            {
+                /* Discard the future object */
+                status = (id_info->discard_cb)(future_object);
+            }
+        H5_AFTER_USER_CB_NOERR(NULL)
+        if (status < 0)
             HGOTO_DONE(NULL);
         future_object = NULL;
 
diff --git a/src/H5Ipkg.h b/src/H5Ipkg.h
index 852ab151dc5..5393c3d2b39 100644
--- a/src/H5Ipkg.h
+++ b/src/H5Ipkg.h
@@ -106,6 +106,7 @@ H5_DLLVAR int H5I_next_type_g;
 
 H5_DLL hid_t          H5I__register(H5I_type_t type, const void *object, bool app_ref,
                                     H5I_future_realize_func_t realize_cb, H5I_future_discard_func_t discard_cb);
+H5_DLL H5I_type_t     H5I__register_type_common(unsigned reserved, H5I_free_t free_func);
 H5_DLL int            H5I__destroy_type(H5I_type_t type);
 H5_DLL void          *H5I__remove_verify(hid_t id, H5I_type_t type);
 H5_DLL int            H5I__inc_type_ref(H5I_type_t type);
diff --git a/src/H5Ipublic.h b/src/H5Ipublic.h
index 9a28dac9094..0ccebda6da5 100644
--- a/src/H5Ipublic.h
+++ b/src/H5Ipublic.h
@@ -394,20 +394,14 @@ H5_DLL int H5Iget_ref(hid_t id);
  *
  * \brief Creates and returns a new ID type
  *
- * \param[in] hash_size Minimum hash table size (in entries) used to store IDs
- *                      for the new type (unused in 1.8.13 and later)
  * \param[in] reserved Number of reserved IDs for the new type
  * \param[in] free_func Function used to deallocate space for a single ID
  *
  * \return Returns the type identifier on success, negative on failure.
  *
- * \details H5Iregister_type() allocates space for a new ID type and returns an
+ * \details H5Iregister_type2() allocates space for a new ID type and returns an
  *          identifier for it.
  *
- *          The \p hash_size parameter indicates the minimum size of the hash
- *          table used to store IDs in the new type. This parameter is unused
- *          in 1.8.13 and later, when the implementation of ID storage changed.
- *
  *          The \p reserved parameter indicates the number of IDs in this new
  *          type to be reserved. Reserved IDs are valid IDs which are not
  *          associated with any storage within the library.
@@ -420,10 +414,10 @@ H5_DLL int H5Iget_ref(hid_t id);
  *          pointer which was passed in to the H5Iregister() function. The \p
  *          free_func function should return 0 on success and -1 on failure.
  *
- * \since 1.8.0
+ * \since 2.0.0
  *
  */
-H5_DLL H5I_type_t H5Iregister_type(size_t hash_size, unsigned reserved, H5I_free_t free_func);
+H5_DLL H5I_type_t H5Iregister_type2(unsigned reserved, H5I_free_t free_func);
 /**
  * \ingroup H5IUD
  *
@@ -683,6 +677,51 @@ H5_DLL htri_t H5Itype_exists(H5I_type_t type);
  */
 H5_DLL htri_t H5Iis_valid(hid_t id);
 
+/* Symbols defined for compatibility with previous versions of the HDF5 API.
+ *
+ * Use of these symbols is deprecated.
+ */
+#ifndef H5_NO_DEPRECATED_SYMBOLS
+
+/**
+ * \ingroup H5IUD
+ *
+ * \brief Creates and returns a new ID type
+ *
+ * \param[in] hash_size Minimum hash table size (in entries) used to store IDs
+ *                      for the new type (unused in 1.8.13 and later)
+ * \param[in] reserved Number of reserved IDs for the new type
+ * \param[in] free_func Function used to deallocate space for a single ID
+ *
+ * \return Returns the type identifier on success, negative on failure.
+ *
+ * \details H5Iregister_type1() allocates space for a new ID type and returns an
+ *          identifier for it.
+ *
+ *          The \p hash_size parameter indicates the minimum size of the hash
+ *          table used to store IDs in the new type. This parameter is unused
+ *          in 1.8.13 and later, when the implementation of ID storage changed.
+ *
+ *          The \p reserved parameter indicates the number of IDs in this new
+ *          type to be reserved. Reserved IDs are valid IDs which are not
+ *          associated with any storage within the library.
+ *
+ *          The \p free_func parameter is a function pointer to a function
+ *          which returns an herr_t and accepts a \c void*. The purpose of this
+ *          function is to deallocate memory for a single ID. It will be called
+ *          by H5Iclear_type() and H5Idestroy_type() on each ID. This function
+ *          is NOT called by H5Iremove_verify(). The \c void* will be the same
+ *          pointer which was passed in to the H5Iregister() function. The \p
+ *          free_func function should return 0 on success and -1 on failure.
+ *
+ * \since 1.8.0
+ * \deprecated 2.0.0 Deprecated in favor of the function H5Iregister_type2()
+ *
+ */
+H5_DLL H5I_type_t H5Iregister_type1(size_t hash_size, unsigned reserved, H5I_free_t free_func);
+
+#endif /* H5_NO_DEPRECATED_SYMBOLS */
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/H5Ldevelop.h b/src/H5Ldevelop.h
index 21d7458d512..5a619c8c44a 100644
--- a/src/H5Ldevelop.h
+++ b/src/H5Ldevelop.h
@@ -299,7 +299,7 @@ H5_DLL herr_t H5Lunregister(H5L_type_t id);
  */
 #ifndef H5_NO_DEPRECATED_SYMBOLS
 
-/* Previous versions of the H5L_class_t struct */
+/** Previous versions of the H5L_class_t struct \since 1.10.3 */
 #define H5L_LINK_CLASS_T_VERS_0 0
 
 /** Callback during link traversal */
diff --git a/src/H5Lexternal.c b/src/H5Lexternal.c
index 888ff404679..793825d1c3d 100644
--- a/src/H5Lexternal.c
+++ b/src/H5Lexternal.c
@@ -134,7 +134,7 @@ H5L__extern_traverse(const char H5_ATTR_UNUSED *link_name, hid_t cur_group, cons
     obj_name  = (const char *)p + fname_len + 1;
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, H5I_INVALID_HID, "can't find object for ID");
 
     /* Get the fapl_id set for lapl_id if any */
@@ -162,7 +162,7 @@ H5L__extern_traverse(const char H5_ATTR_UNUSED *link_name, hid_t cur_group, cons
         HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, H5I_INVALID_HID, "can't get elink callback info");
 
     /* Get file access property list */
-    if (NULL == (fa_plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (fa_plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, H5I_INVALID_HID, "can't find object for ID");
 
     /* Make callback if it exists */
@@ -193,9 +193,14 @@ H5L__extern_traverse(const char H5_ATTR_UNUSED *link_name, hid_t cur_group, cons
         if (H5G_get_name(&loc, parent_group_name, group_name_len, NULL, NULL) < 0)
             HGOTO_ERROR(H5E_LINK, H5E_CANTGET, H5I_INVALID_HID, "unable to retrieve group name");
 
-        /* Make callback */
-        if ((cb_info.func)(parent_file_name, parent_group_name, file_name, obj_name, &intent, fapl_id,
-                           cb_info.user_data) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (cb_info.func)(parent_file_name, parent_group_name, file_name, obj_name, &intent,
+                                           fapl_id, cb_info.user_data);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_LINK, H5E_CALLBACK, H5I_INVALID_HID, "traversal operator failed");
 
         /* Check access flags */
diff --git a/src/H5Lint.c b/src/H5Lint.c
index ab066dd20fd..e8d5e150241 100644
--- a/src/H5Lint.c
+++ b/src/H5Lint.c
@@ -641,9 +641,15 @@ H5L__link_cb(H5G_loc_t *grp_loc /*in*/, const char *name, const H5O_link_t H5_AT
             if ((grp_id = H5VL_wrap_register(H5I_GROUP, grp, true)) < 0)
                 HGOTO_ERROR(H5E_LINK, H5E_CANTREGISTER, FAIL, "unable to register ID for group");
 
-            /* Make callback */
-            if ((link_class->create_func)(name, grp_id, udata->lnk->u.ud.udata, udata->lnk->u.ud.size,
-                                          H5P_DEFAULT) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    /* Make callback */
+                    ret_value = (link_class->create_func)(name, grp_id, udata->lnk->u.ud.udata,
+                                                          udata->lnk->u.ud.size, H5P_DEFAULT);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_LINK, H5E_CALLBACK, FAIL, "link creation callback failed");
         } /* end if */
     }     /* end if */
@@ -974,7 +980,15 @@ H5L__get_val_real(const H5O_link_t *lnk, void *buf, size_t size)
         link_class = H5L_find_class(lnk->type);
 
         if (link_class != NULL && link_class->query_func != NULL) {
-            if ((link_class->query_func)(lnk->name, lnk->u.ud.udata, lnk->u.ud.size, buf, size) < 0)
+            ssize_t len;
+
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    len = (link_class->query_func)(lnk->name, lnk->u.ud.udata, lnk->u.ud.size, buf, size);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (len < 0)
                 HGOTO_ERROR(H5E_LINK, H5E_CALLBACK, FAIL, "query callback returned failure");
         } /* end if */
         else if (buf && size > 0)
@@ -1378,13 +1392,25 @@ H5L__move_dest_cb(H5G_loc_t *grp_loc /*in*/, const char *name, const H5O_link_t
                 HGOTO_ERROR(H5E_LINK, H5E_CANTREGISTER, FAIL, "unable to register group ID");
 
             if (udata->copy) {
-                if ((link_class->copy_func)(udata->lnk->name, grp_id, udata->lnk->u.ud.udata,
-                                            udata->lnk->u.ud.size) < 0)
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        ret_value = (link_class->copy_func)(udata->lnk->name, grp_id, udata->lnk->u.ud.udata,
+                                                            udata->lnk->u.ud.size);
+                    }
+                H5_AFTER_USER_CB(FAIL)
+                if (ret_value < 0)
                     HGOTO_ERROR(H5E_LINK, H5E_CALLBACK, FAIL, "UD copy callback returned error");
             } /* end if */
             else {
-                if ((link_class->move_func)(udata->lnk->name, grp_id, udata->lnk->u.ud.udata,
-                                            udata->lnk->u.ud.size) < 0)
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        ret_value = (link_class->move_func)(udata->lnk->name, grp_id, udata->lnk->u.ud.udata,
+                                                            udata->lnk->u.ud.size);
+                    }
+                H5_AFTER_USER_CB(FAIL)
+                if (ret_value < 0)
                     HGOTO_ERROR(H5E_LINK, H5E_CALLBACK, FAIL, "UD move callback returned error");
             } /* end else */
         }     /* end if */
diff --git a/src/H5Mpublic.h b/src/H5Mpublic.h
index dd9def1939c..0e34bfe01bd 100644
--- a/src/H5Mpublic.h
+++ b/src/H5Mpublic.h
@@ -31,16 +31,26 @@
 /*****************/
 
 /* Macros defining operation IDs for map VOL callbacks (implemented using the
- * "optional" VOL callback) */
-#define H5VL_MAP_CREATE   1 /**< Callback operation ID for map create */
-#define H5VL_MAP_OPEN     2 /**< Callback operation ID for map open */
-#define H5VL_MAP_GET_VAL  3 /**< Callback operation ID for getting an associated value from a map */
-#define H5VL_MAP_EXISTS   4 /**< Callback operation ID for checking if a value exists in a map */
-#define H5VL_MAP_PUT      5 /**< Callback operation ID for putting a key-value pair to a map */
-#define H5VL_MAP_GET      6 /**< Callback operation ID for map get callback */
-#define H5VL_MAP_SPECIFIC 7 /**< Callback operation ID for map specific operation */
-#define H5VL_MAP_OPTIONAL 8 /**< Currently unused */
-#define H5VL_MAP_CLOSE    9 /**< Callback operation ID for terminating access to a map */
+ * "optional" VOL callback)
+ */
+/** Callback operation ID for map create \since 1.12.0 */
+#define H5VL_MAP_CREATE 1
+/** Callback operation ID for map open \since 1.12.0 */
+#define H5VL_MAP_OPEN 2
+/** Callback operation ID for getting an associated value from a map \since 1.12.0 */
+#define H5VL_MAP_GET_VAL 3
+/** Callback operation ID for checking if a value exists in a map \since 1.12.0 */
+#define H5VL_MAP_EXISTS 4
+/** Callback operation ID for putting a key-value pair to a map \since 1.12.0 */
+#define H5VL_MAP_PUT 5
+/** Callback operation ID for map get callback \since 1.12.0 */
+#define H5VL_MAP_GET 6
+/** Callback operation ID for map specific operation \since 1.12.0 */
+#define H5VL_MAP_SPECIFIC 7
+/** Currently unused \since 1.12.0 */
+#define H5VL_MAP_OPTIONAL 8
+/** Callback operation ID for terminating access to a map \since 1.12.0 */
+#define H5VL_MAP_CLOSE 9
 
 /*******************/
 /* Public Typedefs */
@@ -50,19 +60,23 @@
  * Types for map GET callback
  */
 typedef enum H5VL_map_get_t {
-    H5VL_MAP_GET_MAPL,     /**< Callback operation ID for getting map access property list   */
-    H5VL_MAP_GET_MCPL,     /**< Callback operation ID for getting map creation property list */
-    H5VL_MAP_GET_KEY_TYPE, /**< Callback operation ID for getting the key datatype for a map */
-    H5VL_MAP_GET_VAL_TYPE, /**< Callback operation ID for getting the value datatype for a map */
-    H5VL_MAP_GET_COUNT /**< Callback operation ID for getting the number of key-value pairs stored in a map */
+    H5VL_MAP_GET_MAPL,     /**< Callback operation ID for getting map access property list   \since 1.12.0 */
+    H5VL_MAP_GET_MCPL,     /**< Callback operation ID for getting map creation property list \since 1.12.0 */
+    H5VL_MAP_GET_KEY_TYPE, /**< Callback operation ID for getting the key datatype for a map \since 1.12.0 */
+    H5VL_MAP_GET_VAL_TYPE, /**< Callback operation ID for getting the value datatype for a map \since 1.12.0
+                            */
+    H5VL_MAP_GET_COUNT /**< Callback operation ID for getting the number of key-value pairs stored in a map
+                          \since 1.12.0 */
 } H5VL_map_get_t;
 
 /**
  * Types for map SPECIFIC callback
  */
 typedef enum H5VL_map_specific_t {
-    H5VL_MAP_ITER,  /**< Callback operation ID for iterating over all key-value pairs stored in the map */
-    H5VL_MAP_DELETE /**< Callback operation ID for deleting a key-value pair stored in the map */
+    H5VL_MAP_ITER,  /**< Callback operation ID for iterating over all key-value pairs stored in the map
+                       \since 1.12.0 */
+    H5VL_MAP_DELETE /**< Callback operation ID for deleting a key-value pair stored in the map \since 1.12.0
+                     */
 } H5VL_map_specific_t;
 
 //! 
diff --git a/src/H5Ocache.c b/src/H5Ocache.c
index a5bf35aa7ba..8d06f282077 100644
--- a/src/H5Ocache.c
+++ b/src/H5Ocache.c
@@ -781,6 +781,10 @@ H5O__cache_chk_serialize(const H5F_t *f, void *image, size_t len, void *_thing)
     /* copy the chunk into the image -- this is potentially expensive.
      * Can we rework things so that the chunk and the cache share a buffer?
      */
+    /* Ensure len does not exceed the size of the source buffer */
+    if (len > chk_proxy->oh->chunk[chk_proxy->chunkno].size)
+        HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "buffer overflow detected");
+
     H5MM_memcpy(image, chk_proxy->oh->chunk[chk_proxy->chunkno].image, len);
 
 done:
diff --git a/src/H5Ocopy.c b/src/H5Ocopy.c
index 9317be0b4f7..3917ad15989 100644
--- a/src/H5Ocopy.c
+++ b/src/H5Ocopy.c
@@ -1506,9 +1506,16 @@ H5O__copy_search_comm_dt(H5F_t *file_src, H5O_t *oh_src, H5O_loc_t *oloc_dst /*i
             H5O_mcdt_search_ret_t search_cb_ret = H5O_MCDT_SEARCH_CONT;
 
             /* Make callback to see if we should search destination file */
-            if (cpy_info->mcdt_cb)
-                if ((search_cb_ret = cpy_info->mcdt_cb(cpy_info->mcdt_ud)) == H5O_MCDT_SEARCH_ERROR)
+            if (cpy_info->mcdt_cb) {
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        search_cb_ret = cpy_info->mcdt_cb(cpy_info->mcdt_ud);
+                    }
+                H5_AFTER_USER_CB(FAIL)
+                if (H5O_MCDT_SEARCH_ERROR == search_cb_ret)
                     HGOTO_ERROR(H5E_OHDR, H5E_CALLBACK, FAIL, "callback returned error");
+            }
 
             if (search_cb_ret == H5O_MCDT_SEARCH_CONT) {
                 /* Build the complete dst dt list */
diff --git a/src/H5Odeprec.c b/src/H5Odeprec.c
index c4395c1c011..76adad4db42 100644
--- a/src/H5Odeprec.c
+++ b/src/H5Odeprec.c
@@ -132,17 +132,32 @@ H5O__iterate1_adapter(hid_t obj_id, const char *name, const H5O_info2_t *oinfo2,
     if (H5O__reset_info1(&oinfo) < 0)
         HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't reset object data struct");
 
+    /* Get the location object */
+    if (NULL == (vol_obj = H5VL_vol_object(obj_id)))
+        HGOTO_ERROR(H5E_OHDR, H5E_BADTYPE, H5_ITER_ERROR, "invalid location identifier");
+
     /* Check for retrieving data model information */
     dm_fields = shim_data->fields & (H5O_INFO_BASIC | H5O_INFO_TIME | H5O_INFO_NUM_ATTRS);
     if (dm_fields) {
         /* Set the data model fields */
         if (shim_data->fields & H5O_INFO_BASIC) {
+            H5I_type_t vol_obj_type = H5I_BADID; /* Object type of loc_id */
+            void      *vol_obj_data;
+
             oinfo.fileno = oinfo2->fileno;
             oinfo.type   = oinfo2->type;
             oinfo.rc     = oinfo2->rc;
 
+            /* Get object type */
+            if ((vol_obj_type = H5I_get_type(obj_id)) < 0)
+                HGOTO_ERROR(H5E_OHDR, H5E_BADTYPE, FAIL, "invalid location identifier");
+
+            /* Retrieve the underlying object */
+            if (NULL == (vol_obj_data = H5VL_object_data(vol_obj)))
+                HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't get underlying VOL object");
+
             /* Deserialize VOL object token into object address */
-            if (H5VLnative_token_to_addr(obj_id, oinfo2->token, &oinfo.addr) < 0)
+            if (H5VL_native_token_to_addr(vol_obj_data, vol_obj_type, oinfo2->token, &oinfo.addr) < 0)
                 HGOTO_ERROR(H5E_OHDR, H5E_CANTUNSERIALIZE, FAIL,
                             "can't deserialize object token into address");
         }
@@ -170,10 +185,6 @@ H5O__iterate1_adapter(hid_t obj_id, const char *name, const H5O_info2_t *oinfo2,
         loc_params.loc_data.loc_by_name.lapl_id = H5P_LINK_ACCESS_DEFAULT;
         loc_params.obj_type                     = H5I_get_type(obj_id);
 
-        /* Get the location object */
-        if (NULL == (vol_obj = H5VL_vol_object(obj_id)))
-            HGOTO_ERROR(H5E_OHDR, H5E_BADTYPE, H5_ITER_ERROR, "invalid location identifier");
-
         /* Set up VOL callback arguments */
         obj_opt_args.get_native_info.fields = nat_fields;
         obj_opt_args.get_native_info.ninfo  = &nat_info;
@@ -246,13 +257,14 @@ H5O__get_info_old(H5VL_object_t *vol_obj, H5VL_loc_params_t *loc_params, H5O_inf
         if (fields & H5O_INFO_BASIC) {
             void *vol_obj_data;
 
-            if (NULL == (vol_obj_data = H5VL_object_data(vol_obj)))
-                HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't get underlying VOL object");
-
             oinfo->fileno = dm_info.fileno;
             oinfo->type   = dm_info.type;
             oinfo->rc     = dm_info.rc;
 
+            /* Retrieve the underlying object */
+            if (NULL == (vol_obj_data = H5VL_object_data(vol_obj)))
+                HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't get underlying VOL object");
+
             /* Deserialize VOL object token into object address */
             if (H5VL_native_token_to_addr(vol_obj_data, loc_params->obj_type, dm_info.token, &oinfo->addr) <
                 0)
@@ -359,8 +371,14 @@ H5Oopen_by_addr(hid_t loc_id, haddr_t addr)
         HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, H5I_INVALID_HID,
                     "can't determine if VOL object is native connector object");
     if (is_native_vol_obj) {
+        void *vol_obj_data;
+
+        /* Retrieve the underlying object */
+        if (NULL == (vol_obj_data = H5VL_object_data(vol_obj)))
+            HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, H5I_INVALID_HID, "can't retrieve pointer to native object");
+
         /* This is a native-specific routine that requires serialization of the token */
-        if (H5VLnative_addr_to_token(loc_id, addr, &obj_token) < 0)
+        if (H5VL_native_addr_to_token(vol_obj_data, vol_obj_type, addr, &obj_token) < 0)
             HGOTO_ERROR(H5E_OHDR, H5E_CANTSERIALIZE, H5I_INVALID_HID,
                         "can't serialize address into object token");
     } /* end if */
@@ -423,7 +441,7 @@ H5Oget_info1(hid_t loc_id, H5O_info1_t *oinfo /*out*/)
 
     /* Must use native VOL connector for this operation */
     if (!is_native_vol_obj)
-        HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL,
+        HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL,
                     "Deprecated H5Oget_info1 is only meant to be used with the native VOL connector");
 
     /* Retrieve the object's information */
@@ -482,7 +500,7 @@ H5Oget_info_by_name1(hid_t loc_id, const char *name, H5O_info1_t *oinfo /*out*/,
 
     /* Must use native VOL connector for this operation */
     if (!is_native_vol_obj)
-        HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL,
+        HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL,
                     "Deprecated H5Oget_info_by_name1 is only meant to be used with the native VOL connector");
 
     /* Retrieve the object's information */
@@ -547,7 +565,7 @@ H5Oget_info_by_idx1(hid_t loc_id, const char *group_name, H5_index_t idx_type, H
 
     /* Must use native VOL connector for this operation */
     if (!is_native_vol_obj)
-        HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL,
+        HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL,
                     "Deprecated H5Oget_info_by_idx1 is only meant to be used with the native VOL connector");
 
     /* Retrieve the object's information */
@@ -798,7 +816,7 @@ H5Ovisit1(hid_t obj_id, H5_index_t idx_type, H5_iter_order_t order, H5O_iterate1
 
     /* Must use native VOL connector for this operation */
     if (!is_native_vol_obj)
-        HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL,
+        HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL,
                     "Deprecated H5Ovisit1 is only meant to be used with the native VOL connector");
 
     /* Set location parameters */
@@ -895,7 +913,7 @@ H5Ovisit_by_name1(hid_t loc_id, const char *obj_name, H5_index_t idx_type, H5_it
 
     /* Must use native VOL connector for this operation */
     if (!is_native_vol_obj)
-        HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL,
+        HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL,
                     "Deprecated H5Ovisit_by_name1 is only meant to be used with the native VOL connector");
 
     /* Set location parameters */
diff --git a/src/H5Oint.c b/src/H5Oint.c
index 7f7921aeaf8..bf62ad35a9e 100644
--- a/src/H5Oint.c
+++ b/src/H5Oint.c
@@ -2572,8 +2572,13 @@ H5O__visit_cb(hid_t H5_ATTR_UNUSED group, const char *name, const H5L_info2_t *l
             if (H5O_get_info(&obj_oloc, &oinfo, udata->fields) < 0)
                 HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, H5_ITER_ERROR, "unable to get object info");
 
-            /* Make the application callback */
-            ret_value = (udata->op)(udata->obj_id, name, &oinfo, udata->op_data);
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    /* Make the application callback */
+                    ret_value = (udata->op)(udata->obj_id, name, &oinfo, udata->op_data);
+                }
+            H5_AFTER_USER_CB(FAIL)
 
             /* Check for continuing to visit objects */
             if (ret_value == H5_ITER_CONT) {
diff --git a/src/H5Olink.c b/src/H5Olink.c
index b530e421cba..1aa5d14955c 100644
--- a/src/H5Olink.c
+++ b/src/H5Olink.c
@@ -636,8 +636,14 @@ H5O_link_delete(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, void *_mesg)
             if ((file_id = H5F_get_id(f)) < 0)
                 HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to get file ID");
 
-            /* Call user-defined link's 'delete' callback */
-            if ((link_class->del_func)(lnk->name, file_id, lnk->u.ud.udata, lnk->u.ud.size) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    /* Call user-defined link's 'delete' callback */
+                    ret_value = (link_class->del_func)(lnk->name, file_id, lnk->u.ud.udata, lnk->u.ud.size);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_OHDR, H5E_CALLBACK, FAIL, "link deletion callback returned failure");
         } /* end if */
     }     /* end if */
diff --git a/src/H5Opublic.h b/src/H5Opublic.h
index 8968ac1f1e2..6ffe96acf05 100644
--- a/src/H5Opublic.h
+++ b/src/H5Opublic.h
@@ -78,9 +78,9 @@
  * These flags determine which fields will be filled in the H5O_info_t
  * struct.
  */
-#define H5O_INFO_BASIC     0x0001u /**< Fill in the fileno, addr, type, and rc fields */
-#define H5O_INFO_TIME      0x0002u /**< Fill in the atime, mtime, ctime, and btime fields */
-#define H5O_INFO_NUM_ATTRS 0x0004u /**< Fill in the num_attrs field */
+#define H5O_INFO_BASIC     0x0001u /**< Fill in the fileno, addr, type, and rc fields \since 1.10.3 */
+#define H5O_INFO_TIME      0x0002u /**< Fill in the atime, mtime, ctime, and btime fields \since 1.10.3 */
+#define H5O_INFO_NUM_ATTRS 0x0004u /**< Fill in the num_attrs field \since 1.10.3 */
 #define H5O_INFO_ALL       (H5O_INFO_BASIC | H5O_INFO_TIME | H5O_INFO_NUM_ATTRS)
 
 //! 
@@ -88,8 +88,8 @@
  * Flags for H5Oget_native_info().  These flags determine which fields will be
  * filled in the \ref H5O_native_info_t struct.
  */
-#define H5O_NATIVE_INFO_HDR       0x0008u /**< Fill in the hdr field */
-#define H5O_NATIVE_INFO_META_SIZE 0x0010u /**< Fill in the meta_size field */
+#define H5O_NATIVE_INFO_HDR       0x0008u /**< Fill in the hdr field \since 1.12.0 */
+#define H5O_NATIVE_INFO_META_SIZE 0x0010u /**< Fill in the meta_size field \since 1.12.0 */
 #define H5O_NATIVE_INFO_ALL       (H5O_NATIVE_INFO_HDR | H5O_NATIVE_INFO_META_SIZE)
 //! 
 
diff --git a/src/H5PLpublic.h b/src/H5PLpublic.h
index 18ca6ec51ce..1c7bd911cd2 100644
--- a/src/H5PLpublic.h
+++ b/src/H5PLpublic.h
@@ -23,8 +23,7 @@
 /* Public Typedefs */
 /*******************/
 
-/* Special string to indicate no plugin loading.
- */
+/** Special string to indicate no plugin loading \since 1.10.2 */
 #define H5PL_NO_PLUGIN "::"
 
 //! 
@@ -41,10 +40,16 @@ typedef enum H5PL_type_t {
 //! 
 
 /* Common dynamic plugin type flags used by the set/get_loading_state functions */
+/** Flag for filter plugin \since 1.8.15 */
 #define H5PL_FILTER_PLUGIN 0x0001
-#define H5PL_VOL_PLUGIN    0x0002
-#define H5PL_VFD_PLUGIN    0x0004
-#define H5PL_ALL_PLUGIN    0xFFFF
+/** Flag for VOL plugin \since 1.12.0 */
+#define H5PL_VOL_PLUGIN 0x0002
+/** Flag for VFD plugin \since 1.14.0 */
+#define H5PL_VFD_PLUGIN 0x0004
+/** Flag for all plugin types \since 1.8.15 */
+#define H5PL_ALL_PLUGIN 0xFFFF
+
+#define H5F_ACC_DEBUG (0x0000u) /**< Print debug info \deprecated In which version? */
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/src/H5Pdapl.c b/src/H5Pdapl.c
index af123632963..586664547af 100644
--- a/src/H5Pdapl.c
+++ b/src/H5Pdapl.c
@@ -771,7 +771,7 @@ H5Pset_chunk_cache(hid_t dapl_id, size_t rdcc_nslots, size_t rdcc_nbytes, double
             "raw data cache w0 value must be between 0.0 and 1.0 inclusive, or H5D_CHUNK_CACHE_W0_DEFAULT");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(dapl_id, H5P_DATASET_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(dapl_id, H5P_DATASET_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set sizes */
@@ -811,7 +811,7 @@ H5Pget_chunk_cache(hid_t dapl_id, size_t *rdcc_nslots /*out*/, size_t *rdcc_nbyt
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(dapl_id, H5P_DATASET_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(dapl_id, H5P_DATASET_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get default file access plist */
@@ -1076,7 +1076,7 @@ H5Pset_virtual_view(hid_t plist_id, H5D_vds_view_t view)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid bounds option");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Update property list */
@@ -1107,7 +1107,7 @@ H5Pget_virtual_view(hid_t plist_id, H5D_vds_view_t *view /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get value from property list */
@@ -1216,7 +1216,7 @@ H5Pset_virtual_printf_gap(hid_t plist_id, hsize_t gap_size)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid printf gap size");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Update property list */
@@ -1248,7 +1248,7 @@ H5Pget_virtual_printf_gap(hid_t plist_id, hsize_t *gap_size /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get value from property list */
@@ -1300,7 +1300,7 @@ H5Pset_append_flush(hid_t plist_id, unsigned ndims, const hsize_t *boundary, H5D
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "callback is NULL while user data is not");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set up values */
@@ -1348,7 +1348,7 @@ H5Pget_append_flush(hid_t plist_id, unsigned ndims, hsize_t boundary[], H5D_appe
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Retrieve info for append flush */
@@ -1397,7 +1397,7 @@ H5Pset_efile_prefix(hid_t plist_id, const char *prefix)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set prefix */
@@ -1429,7 +1429,7 @@ H5Pget_efile_prefix(hid_t plist_id, char *prefix /*out*/, size_t size)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the current prefix */
@@ -1483,7 +1483,7 @@ H5Pset_virtual_prefix(hid_t plist_id, const char *prefix)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set prefix */
@@ -1517,7 +1517,7 @@ H5Pget_virtual_prefix(hid_t plist_id, char *prefix /*out*/, size_t size)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the current prefix */
diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c
index 5a10139ae48..689b17ace14 100644
--- a/src/H5Pdcpl.c
+++ b/src/H5Pdcpl.c
@@ -1862,7 +1862,7 @@ H5Pset_layout(hid_t plist_id, H5D_layout_t layout_type)
         HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "raw data layout method is not valid");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get pointer to correct default layout */
@@ -1922,7 +1922,7 @@ H5Pget_layout(hid_t plist_id)
     FUNC_ENTER_API(H5D_LAYOUT_ERROR)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, H5D_LAYOUT_ERROR, "can't find object for ID");
 
     /* Peek at layout property */
@@ -1985,7 +1985,7 @@ H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/])
     }                                                   /* end for */
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set chunk information in property list */
@@ -2021,7 +2021,7 @@ H5Pget_chunk(hid_t plist_id, int max_ndims, hsize_t dim[] /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Peek at the layout property */
@@ -2093,7 +2093,7 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, const
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "invalid mapping selections");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the current layout */
@@ -2243,7 +2243,7 @@ H5Pget_virtual_count(hid_t dcpl_id, size_t *count /*out*/)
 
     if (count) {
         /* Get the plist structure */
-        if (NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE)))
+        if (NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE, true)))
             HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
         /* Retrieve the layout property */
@@ -2284,7 +2284,7 @@ H5Pget_virtual_vspace(hid_t dcpl_id, size_t idx)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Retrieve the layout property */
@@ -2337,7 +2337,7 @@ H5Pget_virtual_srcspace(hid_t dcpl_id, size_t idx)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Retrieve the layout property */
@@ -2436,7 +2436,7 @@ H5Pget_virtual_filename(hid_t dcpl_id, size_t idx, char *name /*out*/, size_t si
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Retrieve the layout property */
@@ -2493,7 +2493,7 @@ H5Pget_virtual_dsetname(hid_t dcpl_id, size_t idx, char *name /*out*/, size_t si
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Retrieve the layout property */
@@ -2540,7 +2540,7 @@ H5Pset_chunk_opts(hid_t plist_id, unsigned options)
         HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "unknown chunk options");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Retrieve the layout property */
@@ -2587,7 +2587,7 @@ H5Pget_chunk_opts(hid_t plist_id, unsigned *options /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Retrieve the layout property */
@@ -2646,7 +2646,7 @@ H5Pset_external(hid_t plist_id, const char *name, HDoff_t offset, hsize_t size)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "negative external file offset");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     if (H5P_peek(plist, H5D_CRT_EXT_FILE_LIST_NAME, &efl) < 0)
@@ -2708,7 +2708,7 @@ H5Pget_external_count(hid_t plist_id)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get value */
@@ -2754,7 +2754,7 @@ H5Pget_external(hid_t plist_id, unsigned idx, size_t name_size, char *name /*out
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get value */
@@ -2813,7 +2813,7 @@ H5Pset_szip(hid_t plist_id, unsigned options_mask, unsigned pixels_per_block)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "pixels_per_block is too large");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Always set K13 compression (and un-set CHIP compression) */
@@ -3016,7 +3016,7 @@ H5Pset_fill_value(hid_t plist_id, hid_t type_id, const void *value)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the current fill value */
@@ -3199,7 +3199,7 @@ H5Pget_fill_value(hid_t plist_id, hid_t type_id, void *value /*out*/)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no fill value output buffer");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the fill value */
@@ -3298,7 +3298,7 @@ H5Pfill_value_defined(hid_t plist_id, H5D_fill_value_t *status)
     assert(status);
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the fill-value status */
@@ -3335,7 +3335,7 @@ H5Pset_alloc_time(hid_t plist_id, H5D_alloc_time_t alloc_time)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid allocation time setting");
 
     /* Get the property list structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Check for resetting to default for layout type */
@@ -3418,7 +3418,7 @@ H5Pget_alloc_time(hid_t plist_id, H5D_alloc_time_t *alloc_time /*out*/)
         H5O_fill_t      fill;  /* Fill value property to query */
 
         /* Get the property list structure */
-        if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+        if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, true)))
             HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
         /* Retrieve fill value settings */
@@ -3457,7 +3457,7 @@ H5Pset_fill_time(hid_t plist_id, H5D_fill_time_t fill_time)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid fill time setting");
 
     /* Get the property list structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Retrieve previous fill value settings */
@@ -3498,7 +3498,7 @@ H5Pget_fill_time(hid_t plist_id, H5D_fill_time_t *fill_time /*out*/)
         H5O_fill_t      fill;  /* Fill value property to query */
 
         /* Get the property list structure */
-        if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+        if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, true)))
             HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
         /* Retrieve fill value settings */
@@ -3540,7 +3540,7 @@ H5Pget_dset_no_attrs_hint(hid_t dcpl_id, hbool_t *minimize /*out*/)
     if (NULL == minimize)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "receiving pointer cannot be NULL");
 
-    plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE);
+    plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE, true);
     if (NULL == plist)
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
@@ -3577,7 +3577,7 @@ H5Pset_dset_no_attrs_hint(hid_t dcpl_id, hbool_t minimize)
 
     FUNC_ENTER_API(FAIL)
 
-    plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE);
+    plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE, false);
     if (NULL == plist)
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
@@ -3640,7 +3640,7 @@ H5Pset_struct_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/], uns
     /* TBD: should set fields u.struct_chunk e.g. struct_type to parameter flag */
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set chunk information in property list */
@@ -3678,7 +3678,7 @@ H5Pget_struct_chunk(hid_t plist_id, int max_ndims, hsize_t dim[] /*out*/, unsign
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Peek at the layout property */
diff --git a/src/H5Pdeprec.c b/src/H5Pdeprec.c
index c986eb30142..c0895b837d5 100644
--- a/src/H5Pdeprec.c
+++ b/src/H5Pdeprec.c
@@ -53,6 +53,7 @@
 /********************/
 /* Local Prototypes */
 /********************/
+static herr_t H5P__get_file_space(H5P_genplist_t *plist, H5F_file_space_type_t *strategy, hsize_t *threshold);
 
 /*********************/
 /* Package Variables */
@@ -447,7 +448,7 @@ H5Pget_version(hid_t plist_id, unsigned *super /*out*/, unsigned *freelist /*out
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get values */
@@ -514,9 +515,9 @@ H5Pencode1(hid_t plist_id, void *buf, size_t *nalloc)
 /*-------------------------------------------------------------------------
  * Function:	H5Pset_file_space
  *
- * Purpose:	    It is mapped to H5Pset_file_space_strategy().
+ * Purpose:	Mapped to H5Pset_file_space_strategy().
  *
- * Return:	    Non-negative on success/Negative on failure
+ * Return:	Non-negative on success/Negative on failure
  *
  *-------------------------------------------------------------------------
  */
@@ -524,6 +525,7 @@ herr_t
 H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold)
 {
 
+    H5P_genplist_t       *plist;                                        /* Property list pointer */
     H5F_fspace_strategy_t new_strategy;                                 /* File space strategy type */
     bool                  new_persist   = H5F_FREE_SPACE_PERSIST_DEF;   /* Persisting free-space or not */
     hsize_t               new_threshold = H5F_FREE_SPACE_THRESHOLD_DEF; /* Free-space section threshold */
@@ -533,8 +535,14 @@ H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t thresh
 
     FUNC_ENTER_API(FAIL)
 
+    /* Check args */
     if ((unsigned)in_strategy >= H5F_FILE_SPACE_NTYPES)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid strategy");
+
+    /* Get the plist structure */
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, false)))
+        HGOTO_ERROR(H5E_PLIST, H5E_BADID, FAIL, "can't find object for ID");
+
     /*
      *  For 1.10.0 H5Pset_file_space:
      *      If strategy is zero, the property is not changed;
@@ -543,9 +551,11 @@ H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t thresh
      *      the existing threshold is retained.
      */
     if (!in_strategy)
-        H5Pget_file_space(plist_id, &in_strategy, NULL);
+        if (H5P__get_file_space(plist, &in_strategy, NULL) < 0)
+            HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space strategy");
     if (!in_threshold)
-        H5Pget_file_space(plist_id, NULL, &in_threshold);
+        if (H5P__get_file_space(plist, NULL, &in_threshold) < 0)
+            HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get free-space threshold");
 
     switch (in_strategy) {
         case H5F_FILE_SPACE_ALL_PERSIST:
@@ -573,7 +583,7 @@ H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t thresh
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file space strategy");
     }
 
-    if (H5Pset_file_space_strategy(plist_id, new_strategy, new_persist, new_threshold) < 0)
+    if (H5P__set_file_space_strategy(plist, new_strategy, new_persist, new_threshold) < 0)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set file space strategy");
 
 done:
@@ -581,27 +591,27 @@ H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t thresh
 } /* H5Pset_file_space() */
 
 /*-------------------------------------------------------------------------
- * Function:	H5Pget_file_space
+ * Function:	H5P__get_file_space
  *
- * Purpose:	    It is mapped to H5Pget_file_space_strategy().
+ * Purpose:	Mapped to H5Pget_file_space_strategy().
  *
- * Return:	    Non-negative on success/Negative on failure
+ * Return:	Non-negative on success/Negative on failure
  *
  *-------------------------------------------------------------------------
  */
-herr_t
-H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy /*out*/, hsize_t *threshold /*out*/)
+static herr_t
+H5P__get_file_space(H5P_genplist_t *plist, H5F_file_space_type_t *strategy, hsize_t *threshold)
 {
     H5F_fspace_strategy_t new_strategy;        /* File space strategy type */
     bool                  new_persist;         /* Persisting free-space or not */
     hsize_t               new_threshold;       /* Free-space section threshold */
     herr_t                ret_value = SUCCEED; /* Return value */
 
-    FUNC_ENTER_API(FAIL)
+    FUNC_ENTER_PACKAGE
 
     /* Get current file space info */
-    if (H5Pget_file_space_strategy(plist_id, &new_strategy, &new_persist, &new_threshold) < 0)
-        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space strategy");
+    if (H5P__get_file_space_strategy(plist, &new_strategy, &new_persist, &new_threshold) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space strategy values");
 
     /* Get value(s) */
     if (strategy) {
@@ -632,6 +642,35 @@ H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy /*out*/, hsize
     if (threshold)
         *threshold = new_threshold;
 
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5P__get_file_space() */
+
+/*-------------------------------------------------------------------------
+ * Function:	H5Pget_file_space
+ *
+ * Purpose:	    It is mapped to H5Pget_file_space_strategy().
+ *
+ * Return:	    Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy /*out*/, hsize_t *threshold /*out*/)
+{
+    H5P_genplist_t *plist;               /* Property list pointer */
+    herr_t          ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_API(FAIL)
+
+    /* Get the plist structure */
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, true)))
+        HGOTO_ERROR(H5E_PLIST, H5E_BADID, FAIL, "can't find object for ID");
+
+    /* Get current file space info */
+    if (H5P__get_file_space(plist, strategy, threshold) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space strategy");
+
 done:
     FUNC_LEAVE_API(ret_value)
 } /* H5Pget_file_space() */
diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c
index 4fc27f6be96..28dbe54792b 100644
--- a/src/H5Pdxpl.c
+++ b/src/H5Pdxpl.c
@@ -974,7 +974,7 @@ H5Pset_data_transform(hid_t plist_id, const char *expression)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "expression cannot be NULL");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* See if a data transform is already set, and free it if it is */
@@ -1032,7 +1032,7 @@ H5Pget_data_transform(hid_t plist_id, char *expression /*out*/, size_t size)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     if (H5P_peek(plist, H5D_XFER_XFORM_NAME, &data_xform_prop) < 0)
@@ -1090,7 +1090,7 @@ H5Pset_buffer(hid_t plist_id, size_t size, void *tconv, void *bkg)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "buffer size must not be zero");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Update property list */
@@ -1126,7 +1126,7 @@ H5Pget_buffer(hid_t plist_id, void **tconv /*out*/, void **bkg /*out*/)
     FUNC_ENTER_API(0)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, 0, "can't find object for ID");
 
     /* Return values */
@@ -1171,7 +1171,7 @@ H5Pset_preserve(hid_t plist_id, hbool_t status)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Update property list */
@@ -1204,7 +1204,7 @@ H5Pget_preserve(hid_t plist_id)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get value */
@@ -1243,7 +1243,7 @@ H5Pset_edc_check(hid_t plist_id, H5Z_EDC_t check)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid value");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Update property list */
@@ -1275,7 +1275,7 @@ H5Pget_edc_check(hid_t plist_id)
     FUNC_ENTER_API(H5Z_ERROR_EDC)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, H5Z_ERROR_EDC, "can't find object for ID");
 
     /* Update property list */
@@ -1307,7 +1307,7 @@ H5Pset_filter_callback(hid_t plist_id, H5Z_filter_func_t func, void *op_data)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Update property list */
@@ -1342,7 +1342,7 @@ H5Pset_type_conv_cb(hid_t plist_id, H5T_conv_except_func_t op, void *operate_dat
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Update property list */
@@ -1377,7 +1377,7 @@ H5Pget_type_conv_cb(hid_t plist_id, H5T_conv_except_func_t *op /*out*/, void **o
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get property */
@@ -1414,7 +1414,7 @@ H5Pget_btree_ratios(hid_t plist_id, double *left /*out*/, double *middle /*out*/
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the split ratios */
@@ -1464,7 +1464,7 @@ H5Pset_btree_ratios(hid_t plist_id, double left, double middle, double right)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "split ratio must satisfy 0.0 <= X <= 1.0");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set values */
@@ -1546,7 +1546,7 @@ H5Pset_vlen_mem_manager(hid_t plist_id, H5MM_allocate_t alloc_func, void *alloc_
     FUNC_ENTER_API(FAIL)
 
     /* Check arguments */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset transfer property list");
 
     /* Update property list */
@@ -1576,7 +1576,7 @@ H5Pget_vlen_mem_manager(hid_t plist_id, H5MM_allocate_t *alloc_func /*out*/, voi
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     if (alloc_func)
@@ -1627,7 +1627,7 @@ H5Pset_hyper_vector_size(hid_t plist_id, size_t vector_size)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "vector size too small");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Update property list */
@@ -1656,7 +1656,7 @@ H5Pget_hyper_vector_size(hid_t plist_id, size_t *vector_size /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Return values */
@@ -1887,7 +1887,7 @@ H5Pget_mpio_actual_chunk_opt_mode(hid_t                             plist_id,
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Return values */
@@ -1918,7 +1918,7 @@ H5Pget_mpio_actual_io_mode(hid_t plist_id, H5D_mpio_actual_io_mode_t *actual_io_
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Return values */
@@ -1949,7 +1949,7 @@ H5Pget_mpio_no_collective_cause(hid_t plist_id, uint32_t *local_no_collective_ca
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Return values */
@@ -2269,7 +2269,7 @@ H5Pset_dataset_io_hyperslab_selection(hid_t plist_id, unsigned rank, H5S_seloper
     /* block is allowed to be NULL, and will be assumed to be all '1's when NULL */
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* See if a dataset I/O selection is already set, and free it if it is */
@@ -2365,7 +2365,7 @@ H5Pset_selection_io(hid_t plist_id, H5D_selection_io_mode_t selection_io_mode)
     if (plist_id == H5P_DEFAULT)
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list");
 
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl");
 
     /* Set the selection I/O mode */
@@ -2399,7 +2399,7 @@ H5Pget_selection_io(hid_t plist_id, H5D_selection_io_mode_t *selection_io_mode /
     FUNC_ENTER_API(FAIL)
 
     /* Check arguments */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl");
 
     /* Get the selection I/O mode */
@@ -2429,7 +2429,7 @@ H5Pget_no_selection_io_cause(hid_t plist_id, uint32_t *no_selection_io_cause /*o
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Return values */
@@ -2461,7 +2461,7 @@ H5Pget_actual_selection_io_mode(hid_t plist_id, uint32_t *actual_selection_io_mo
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Return values */
@@ -2561,7 +2561,7 @@ H5Pset_modify_write_buf(hid_t plist_id, hbool_t modify_write_buf)
     if (plist_id == H5P_DEFAULT)
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list");
 
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, false)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl");
 
     /* Set the selection I/O mode */
@@ -2591,7 +2591,7 @@ H5Pget_modify_write_buf(hid_t plist_id, hbool_t *modify_write_buf /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Check arguments */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_XFER, true)))
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl");
 
     /* Get the selection I/O mode */
diff --git a/src/H5Pencdec.c b/src/H5Pencdec.c
index 96cbfc3f018..9550a071be3 100644
--- a/src/H5Pencdec.c
+++ b/src/H5Pencdec.c
@@ -352,9 +352,15 @@ H5P__encode_cb(H5P_genprop_t *prop, void *_udata)
         } /* end if */
         *(udata->enc_size_ptr) += prop_name_len;
 
-        /* Encode (or not, if *(udata->pp) is NULL) the property value */
-        prop_value_len = 0;
-        if ((prop->encode)(prop->value, udata->pp, &prop_value_len) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(H5_ITER_ERROR)
+            {
+                /* Encode (or not, if *(udata->pp) is NULL) the property value */
+                prop_value_len = 0;
+                ret_value      = (prop->encode)(prop->value, udata->pp, &prop_value_len);
+            }
+        H5_AFTER_USER_CB(H5_ITER_ERROR)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTENCODE, H5_ITER_ERROR, "property encoding routine failed");
         *(udata->enc_size_ptr) += prop_value_len;
     } /* end if */
@@ -768,7 +774,13 @@ H5P__decode(const void *buf)
 
         /* Decode serialized value */
         if (prop->decode) {
-            if ((prop->decode)((const void **)&p, value_buf) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    ret_value = (prop->decode)((const void **)&p, value_buf);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_PLIST, H5E_CANTDECODE, FAIL,
                             "property decoding routine failed, property: '%s'", name);
         } /* end if */
diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c
index a498b99db0b..a07018a9214 100644
--- a/src/H5Pfapl.c
+++ b/src/H5Pfapl.c
@@ -1086,7 +1086,7 @@ H5Pset_alignment(hid_t fapl_id, hsize_t threshold, hsize_t alignment)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "alignment must be positive");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set values */
@@ -1119,7 +1119,7 @@ H5Pget_alignment(hid_t fapl_id, hsize_t *threshold /*out*/, hsize_t *alignment /
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get values */
@@ -1614,7 +1614,7 @@ H5Pget_driver_config_str(hid_t fapl_id, char *config_buf, size_t buf_size)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, (-1), "config_buf cannot be NULL if buf_size is non-zero");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, (-1), "can't find object for ID");
 
     /* Retrieve configuration string property */
@@ -2000,7 +2000,7 @@ H5Pset_family_offset(hid_t fapl_id, hsize_t offset)
     /* Get the plist structure */
     if (H5P_DEFAULT == fapl_id)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't modify default property list");
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set value */
@@ -2034,7 +2034,7 @@ H5Pget_family_offset(hid_t fapl_id, hsize_t *offset /*out*/)
     /* Get the plist structure */
     if (H5P_DEFAULT == fapl_id)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't modify default property list");
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get value */
@@ -2070,7 +2070,7 @@ H5Pset_multi_type(hid_t fapl_id, H5FD_mem_t type)
     /* Get the plist structure */
     if (H5P_DEFAULT == fapl_id)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't modify default property list");
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set value */
@@ -2104,7 +2104,7 @@ H5Pget_multi_type(hid_t fapl_id, H5FD_mem_t *type /*out*/)
     /* Get the plist structure */
     if (H5P_DEFAULT == fapl_id)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't modify default property list");
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get value */
@@ -2151,7 +2151,7 @@ H5Pset_cache(hid_t plist_id, int H5_ATTR_UNUSED mdc_nelmts, size_t rdcc_nslots,
                     "raw data cache w0 value must be between 0.0 and 1.0 inclusive");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set sizes */
@@ -2189,7 +2189,7 @@ H5Pget_cache(hid_t plist_id, int *mdc_nelmts, size_t *rdcc_nslots /*out*/, size_
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get sizes */
@@ -2231,7 +2231,7 @@ H5Pset_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* validate the new configuration */
@@ -2274,7 +2274,7 @@ H5Pget_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config /*out*
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* validate the config ptr */
@@ -2315,7 +2315,7 @@ H5Pset_mdc_config(hid_t plist_id, H5AC_cache_config_t *config_ptr)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* validate the new configuration */
@@ -2358,7 +2358,7 @@ H5Pget_mdc_config(hid_t plist_id, H5AC_cache_config_t *config /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* validate the config ptr */
@@ -2410,7 +2410,7 @@ H5Pset_gc_references(hid_t plist_id, unsigned gc_ref)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set values */
@@ -2440,7 +2440,7 @@ H5Pget_gc_references(hid_t plist_id, unsigned *gc_ref /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get values */
@@ -2470,7 +2470,7 @@ H5Pset_fclose_degree(hid_t plist_id, H5F_close_degree_t degree)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set values */
@@ -2499,7 +2499,7 @@ H5Pget_fclose_degree(hid_t plist_id, H5F_close_degree_t *degree /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     if (degree && H5P_get(plist, H5F_ACS_CLOSE_DEGREE_NAME, degree) < 0)
@@ -2537,7 +2537,7 @@ H5Pset_meta_block_size(hid_t plist_id, hsize_t size)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set values */
@@ -2567,7 +2567,7 @@ H5Pget_meta_block_size(hid_t plist_id, hsize_t *size /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get values */
@@ -2608,7 +2608,7 @@ H5Pset_sieve_buf_size(hid_t plist_id, size_t size)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set values */
@@ -2638,7 +2638,7 @@ H5Pget_sieve_buf_size(hid_t plist_id, size_t *size /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get values */
@@ -2678,7 +2678,7 @@ H5Pset_small_data_block_size(hid_t plist_id, hsize_t size)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set values */
@@ -2708,7 +2708,7 @@ H5Pget_small_data_block_size(hid_t plist_id, hsize_t *size /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get values */
@@ -2845,7 +2845,7 @@ H5Pset_libver_bounds(hid_t plist_id, H5F_libver_t low, H5F_libver_t high)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Invalid (low,high) combination of library version bound");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set values */
@@ -2876,7 +2876,7 @@ H5Pget_libver_bounds(hid_t plist_id, H5F_libver_t *low /*out*/, H5F_libver_t *hi
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get values */
@@ -2914,7 +2914,7 @@ H5Pset_elink_file_cache_size(hid_t plist_id, unsigned efc_size)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set value */
@@ -2947,7 +2947,7 @@ H5Pget_elink_file_cache_size(hid_t plist_id, unsigned *efc_size /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get value */
@@ -2983,8 +2983,8 @@ H5Pset_file_image(hid_t fapl_id, void *buf_ptr, size_t buf_len)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "inconsistent buf_ptr and buf_len");
 
     /* Get the plist structure */
-    if (NULL == (fapl = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
-        HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
+    if (NULL == (fapl = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
+        HGOTO_ERROR(H5E_PLIST, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get old image info */
     if (H5P_peek(fapl, H5F_ACS_FILE_IMAGE_INFO_NAME, &image_info) < 0)
@@ -2993,10 +2993,15 @@ H5Pset_file_image(hid_t fapl_id, void *buf_ptr, size_t buf_len)
     /* Release previous buffer, if it exists */
     if (image_info.buffer != NULL) {
         if (image_info.callbacks.image_free) {
-            if (SUCCEED != image_info.callbacks.image_free(image_info.buffer,
-                                                           H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET,
-                                                           image_info.callbacks.udata))
-                HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "image_free callback failed");
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    ret_value = image_info.callbacks.image_free(
+                        image_info.buffer, H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET, image_info.callbacks.udata);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
+                HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "image_free callback failed");
         } /* end if */
         else
             H5MM_xfree(image_info.buffer);
@@ -3006,19 +3011,33 @@ H5Pset_file_image(hid_t fapl_id, void *buf_ptr, size_t buf_len)
     if (buf_ptr) {
         /* Allocate memory */
         if (image_info.callbacks.image_malloc) {
-            if (NULL == (image_info.buffer = image_info.callbacks.image_malloc(
-                             buf_len, H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET, image_info.callbacks.udata)))
-                HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "image malloc callback failed");
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    image_info.buffer = image_info.callbacks.image_malloc(
+                        buf_len, H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET, image_info.callbacks.udata);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (NULL == image_info.buffer)
+                HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "image malloc callback failed");
         } /* end if */
         else if (NULL == (image_info.buffer = H5MM_malloc(buf_len)))
-            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate memory block");
+            HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "unable to allocate memory block");
 
         /* Copy data */
         if (image_info.callbacks.image_memcpy) {
-            if (image_info.buffer != image_info.callbacks.image_memcpy(image_info.buffer, buf_ptr, buf_len,
-                                                                       H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET,
-                                                                       image_info.callbacks.udata))
-                HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCOPY, FAIL, "image_memcpy callback failed");
+            void *tmp;
+
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    tmp = image_info.callbacks.image_memcpy(image_info.buffer, buf_ptr, buf_len,
+                                                            H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET,
+                                                            image_info.callbacks.udata);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (image_info.buffer != tmp)
+                HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "image_memcpy callback failed");
         } /* end if */
         else
             H5MM_memcpy(image_info.buffer, buf_ptr, buf_len);
@@ -3072,8 +3091,8 @@ H5Pget_file_image(hid_t fapl_id, void **buf /*out*/, size_t *buf_len /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (fapl = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
-        HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
+    if (NULL == (fapl = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
+        HGOTO_ERROR(H5E_PLIST, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get values */
     if (H5P_peek(fapl, H5F_ACS_FILE_IMAGE_INFO_NAME, &image_info) < 0)
@@ -3094,20 +3113,34 @@ H5Pget_file_image(hid_t fapl_id, void **buf /*out*/, size_t *buf_len /*out*/)
         if (image_info.buffer != NULL) {
             /* Allocate memory */
             if (image_info.callbacks.image_malloc) {
-                if (NULL ==
-                    (copy_ptr = image_info.callbacks.image_malloc(
-                         image_info.size, H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET, image_info.callbacks.udata)))
-                    HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "image malloc callback failed");
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        copy_ptr = image_info.callbacks.image_malloc(image_info.size,
+                                                                     H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET,
+                                                                     image_info.callbacks.udata);
+                    }
+                H5_AFTER_USER_CB(FAIL)
+                if (NULL == copy_ptr)
+                    HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "image malloc callback failed");
             } /* end if */
             else if (NULL == (copy_ptr = H5MM_malloc(image_info.size)))
-                HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate copy");
+                HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "unable to allocate copy");
 
             /* Copy data */
             if (image_info.callbacks.image_memcpy) {
-                if (copy_ptr != image_info.callbacks.image_memcpy(
-                                    copy_ptr, image_info.buffer, image_info.size,
-                                    H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET, image_info.callbacks.udata))
-                    HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCOPY, FAIL, "image_memcpy callback failed");
+                void *tmp;
+
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        tmp = image_info.callbacks.image_memcpy(copy_ptr, image_info.buffer, image_info.size,
+                                                                H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET,
+                                                                image_info.callbacks.udata);
+                    }
+                H5_AFTER_USER_CB(FAIL)
+                if (copy_ptr != tmp)
+                    HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "image_memcpy callback failed");
             } /* end if */
             else
                 H5MM_memcpy(copy_ptr, image_info.buffer, image_info.size);
@@ -3144,8 +3177,8 @@ H5Pset_file_image_callbacks(hid_t fapl_id, H5FD_file_image_callbacks_t *callback
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (fapl = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
-        HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
+    if (NULL == (fapl = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
+        HGOTO_ERROR(H5E_PLIST, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get old info */
     if (H5P_peek(fapl, H5F_ACS_FILE_IMAGE_INFO_NAME, &info) < 0)
@@ -3172,8 +3205,15 @@ H5Pset_file_image_callbacks(hid_t fapl_id, H5FD_file_image_callbacks_t *callback
     /* Release old udata if it exists */
     if (info.callbacks.udata != NULL) {
         assert(info.callbacks.udata_free);
-        if (info.callbacks.udata_free(info.callbacks.udata) < 0)
-            HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "udata_free callback failed");
+
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = info.callbacks.udata_free(info.callbacks.udata);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
+            HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "udata_free callback failed");
     } /* end if */
 
     /* Update struct */
@@ -3182,7 +3222,14 @@ H5Pset_file_image_callbacks(hid_t fapl_id, H5FD_file_image_callbacks_t *callback
     if (callbacks_ptr->udata) {
         assert(callbacks_ptr->udata_copy);
         assert(callbacks_ptr->udata_free);
-        if ((info.callbacks.udata = callbacks_ptr->udata_copy(callbacks_ptr->udata)) == NULL)
+
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                info.callbacks.udata = callbacks_ptr->udata_copy(callbacks_ptr->udata);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (NULL == info.callbacks.udata)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't copy the supplied udata");
     } /* end if */
 
@@ -3194,8 +3241,16 @@ H5Pset_file_image_callbacks(hid_t fapl_id, H5FD_file_image_callbacks_t *callback
 
 done:
     if (ret_value < 0) {
-        if (copied_udata && (callbacks_ptr->udata_free(info.callbacks.udata) < 0))
-            HDONE_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "udata_free callback failed");
+        if (copied_udata) {
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    ret_value = callbacks_ptr->udata_free(info.callbacks.udata);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
+                HDONE_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "udata_free callback failed");
+        }
     }
 
     FUNC_LEAVE_API(ret_value)
@@ -3223,8 +3278,8 @@ H5Pget_file_image_callbacks(hid_t fapl_id, H5FD_file_image_callbacks_t *callback
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (fapl = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
-        HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
+    if (NULL == (fapl = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
+        HGOTO_ERROR(H5E_PLIST, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get old info */
     if (H5P_peek(fapl, H5F_ACS_FILE_IMAGE_INFO_NAME, &info) < 0)
@@ -3243,7 +3298,14 @@ H5Pget_file_image_callbacks(hid_t fapl_id, H5FD_file_image_callbacks_t *callback
     /* Copy udata if it exists */
     if (info.callbacks.udata != NULL) {
         assert(info.callbacks.udata_copy);
-        if ((callbacks->udata = info.callbacks.udata_copy(info.callbacks.udata)) == 0)
+
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                callbacks->udata = info.callbacks.udata_copy(info.callbacks.udata);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (NULL == callbacks->udata)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't copy udata");
     } /* end if */
 
@@ -3290,20 +3352,32 @@ H5P__file_image_info_copy(void *value)
 
             /* Allocate new buffer */
             if (info->callbacks.image_malloc) {
-                if (NULL == (info->buffer = info->callbacks.image_malloc(
-                                 info->size, H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY, info->callbacks.udata)))
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        info->buffer = info->callbacks.image_malloc(
+                            info->size, H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY, info->callbacks.udata);
+                    }
+                H5_AFTER_USER_CB(FAIL)
+                if (NULL == info->buffer)
                     HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "image malloc callback failed");
             } /* end if */
-            else {
-                if (NULL == (info->buffer = H5MM_malloc(info->size)))
-                    HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "unable to allocate memory block");
-            } /* end else */
+            else if (NULL == (info->buffer = H5MM_malloc(info->size)))
+                HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "unable to allocate memory block");
 
             /* Copy data to new buffer */
             if (info->callbacks.image_memcpy) {
-                if (info->buffer != info->callbacks.image_memcpy(info->buffer, old_buffer, info->size,
-                                                                 H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY,
-                                                                 info->callbacks.udata))
+                void *tmp;
+
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        tmp = info->callbacks.image_memcpy(info->buffer, old_buffer, info->size,
+                                                           H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY,
+                                                           info->callbacks.udata);
+                    }
+                H5_AFTER_USER_CB(FAIL)
+                if (info->buffer != tmp)
                     HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "image_memcpy callback failed");
             } /* end if */
             else
@@ -3317,7 +3391,12 @@ H5P__file_image_info_copy(void *value)
             if (NULL == info->callbacks.udata_copy)
                 HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "udata_copy not defined");
 
-            info->callbacks.udata = info->callbacks.udata_copy(old_udata);
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    info->callbacks.udata = info->callbacks.udata_copy(old_udata);
+                }
+            H5_AFTER_USER_CB(FAIL)
         } /* end if */
     }     /* end if */
 
@@ -3355,8 +3434,14 @@ H5P__file_image_info_free(void *value)
         /* Free buffer */
         if (info->buffer != NULL && info->size > 0) {
             if (info->callbacks.image_free) {
-                if ((*info->callbacks.image_free)(info->buffer, H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE,
-                                                  info->callbacks.udata) < 0)
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        ret_value = (*info->callbacks.image_free)(
+                            info->buffer, H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE, info->callbacks.udata);
+                    }
+                H5_AFTER_USER_CB(FAIL)
+                if (ret_value < 0)
                     HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "image_free callback failed");
             } /* end if */
             else
@@ -3367,7 +3452,13 @@ H5P__file_image_info_free(void *value)
         if (info->callbacks.udata) {
             if (NULL == info->callbacks.udata_free)
                 HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "udata_free not defined");
-            if ((*info->callbacks.udata_free)(info->callbacks.udata) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    ret_value = (*info->callbacks.udata_free)(info->callbacks.udata);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "udata_free callback failed");
         } /* end if */
     }     /* end if */
@@ -3428,13 +3519,13 @@ H5P__facc_cache_image_config_cmp(const void *_config1, const void *_config2, siz
 } /* end H5P__facc_cache_image_config_cmp() */
 
 /*-------------------------------------------------------------------------
- * Function:       H5P__facc_cache_image_config_enc
+ * Function: H5P__facc_cache_image_config_enc
  *
- * Purpose:        Callback routine which is called whenever the default
- *                 cache image config property in the file creation
+ * Purpose:  Callback routine which is called whenever the default
+ *           cache image config property in the file creation
  *           property list is encoded.
  *
- * Return:       Success:    Non-negative
+ * Return:   Success:    Non-negative
  *           Failure:    Negative
  *
  *-------------------------------------------------------------------------
@@ -3456,11 +3547,8 @@ H5P__facc_cache_image_config_enc(const void *value, void **_pp, size_t *size)
         *(*pp)++ = (uint8_t)sizeof(unsigned);
 
         INT32ENCODE(*pp, (int32_t)config->version);
-
         H5_ENCODE_UNSIGNED(*pp, config->generate_image);
-
         H5_ENCODE_UNSIGNED(*pp, config->save_resize_status);
-
         INT32ENCODE(*pp, (int32_t)config->entry_ageout);
     } /* end if */
 
@@ -3471,13 +3559,13 @@ H5P__facc_cache_image_config_enc(const void *value, void **_pp, size_t *size)
 } /* end H5P__facc_cache_image_config_enc() */
 
 /*-------------------------------------------------------------------------
- * Function:       H5P__facc_cache_image_config_dec
+ * Function: H5P__facc_cache_image_config_dec
  *
- * Purpose:        Callback routine which is called whenever the default
- *                 cache image config property in the file creation property
+ * Purpose:  Callback routine which is called whenever the default
+ *           cache image config property in the file creation property
  *           list is  decoded.
  *
- * Return:       Success:    Non-negative
+ * Return:   Success:    Non-negative
  *           Failure:    Negative
  *
  *-------------------------------------------------------------------------
@@ -3507,11 +3595,8 @@ H5P__facc_cache_image_config_dec(const void **_pp, void *_value)
         HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "unsigned value can't be decoded");
 
     INT32DECODE(*pp, config->version);
-
     H5_DECODE_UNSIGNED(*pp, config->generate_image);
-
     H5_DECODE_UNSIGNED(*pp, config->save_resize_status);
-
     INT32DECODE(*pp, config->entry_ageout);
 
 done:
@@ -4387,7 +4472,7 @@ H5Pset_metadata_read_attempts(hid_t plist_id, unsigned attempts)
                     "number of metadatata read attempts must be greater than 0");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set values */
@@ -4419,7 +4504,7 @@ H5Pget_metadata_read_attempts(hid_t plist_id, unsigned *attempts /*out*/)
         H5P_genplist_t *plist; /* Property list pointer */
 
         /* Get the plist structure */
-        if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+        if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
             HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
         /* Get the # of read attempts set */
@@ -4460,7 +4545,7 @@ H5Pset_object_flush_cb(hid_t plist_id, H5F_flush_cb_t func, void *udata)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "callback is NULL while user data is not");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Update property list */
@@ -4495,7 +4580,7 @@ H5Pget_object_flush_cb(hid_t plist_id, H5F_flush_cb_t *func /*out*/, void **udat
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Retrieve the callback function and user data */
@@ -4537,7 +4622,7 @@ H5Pset_mdc_log_options(hid_t plist_id, hbool_t is_enabled, const char *location,
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "location cannot be NULL");
 
     /* Get the property list structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plist_id is not a file access property list");
 
     /* Make a copy of the passed-in location */
@@ -4576,7 +4661,7 @@ H5Pget_mdc_log_options(hid_t plist_id, hbool_t *is_enabled /*out*/, char *locati
     FUNC_ENTER_API(FAIL)
 
     /* Get the property list structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plist_id is not a file access property list");
 
     /* Get simple values */
@@ -5671,7 +5756,7 @@ H5Pset_page_buffer_size(hid_t plist_id, size_t buf_size, unsigned min_meta_perc,
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     if (min_meta_perc > 100)
@@ -5716,7 +5801,7 @@ H5Pget_page_buffer_size(hid_t plist_id, size_t *buf_size /*out*/, unsigned *min_
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get size */
@@ -5829,7 +5914,7 @@ H5Pset_vol(hid_t plist_id, hid_t new_vol_id, const void *new_vol_info)
     FUNC_ENTER_API(FAIL)
 
     /* Check arguments */
-    if (NULL == (plist = (H5P_genplist_t *)H5I_object_verify(plist_id, H5I_GENPROP_LST)))
+    if (NULL == (plist = (H5P_genplist_t *)H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list");
     if (NULL == (connector = H5I_object_verify(new_vol_id, H5I_VOL)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file VOL ID");
@@ -6217,7 +6302,7 @@ H5Pset_relax_file_integrity_checks(hid_t plist_id, uint64_t flags)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid flags");
 
     /* Get the property list structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, false)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plist_id is not a file access property list");
 
     /* Set value */
@@ -6249,7 +6334,7 @@ H5Pget_relax_file_integrity_checks(hid_t plist_id, uint64_t *flags /*out*/)
         plist_id = H5P_FILE_ACCESS_DEFAULT;
 
     /* Get the property list structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plist_id is not a file access property list");
 
     /* Get value */
diff --git a/src/H5Pfcpl.c b/src/H5Pfcpl.c
index c2f44dec615..401c550a7e9 100644
--- a/src/H5Pfcpl.c
+++ b/src/H5Pfcpl.c
@@ -330,7 +330,7 @@ H5Pset_userblock(hid_t plist_id, hsize_t size)
     } /* end if */
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set value */
@@ -362,7 +362,7 @@ H5Pget_userblock(hid_t plist_id, hsize_t *size /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get value */
@@ -404,7 +404,7 @@ H5Pset_sizes(hid_t plist_id, size_t sizeof_addr, size_t sizeof_size)
     } /* end if */
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set value */
@@ -446,7 +446,7 @@ H5Pget_sizes(hid_t plist_id, size_t *sizeof_addr /*out*/, size_t *sizeof_size /*
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get values */
@@ -501,7 +501,7 @@ H5Pset_sym_k(hid_t plist_id, unsigned ik, unsigned lk)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set values */
@@ -545,7 +545,7 @@ H5Pget_sym_k(hid_t plist_id, unsigned *ik /*out*/, unsigned *lk /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get values */
@@ -590,7 +590,7 @@ H5Pset_istore_k(hid_t plist_id, unsigned ik)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "istore IK value exceeds maximum B-tree entries");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set value */
@@ -627,7 +627,7 @@ H5Pget_istore_k(hid_t plist_id, unsigned *ik /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get value */
@@ -754,7 +754,7 @@ H5Pset_shared_mesg_nindexes(hid_t plist_id, unsigned nindexes)
                     "number of indexes is greater than H5O_SHMESG_MAX_NINDEXES");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     if (H5P_set(plist, H5F_CRT_SHMSG_NINDEXES_NAME, &nindexes) < 0)
@@ -783,7 +783,7 @@ H5Pget_shared_mesg_nindexes(hid_t plist_id, unsigned *nindexes /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     if (H5P_get(plist, H5F_CRT_SHMSG_NINDEXES_NAME, nindexes) < 0)
@@ -823,7 +823,7 @@ H5Pset_shared_mesg_index(hid_t plist_id, unsigned index_num, unsigned mesg_type_
         HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "unrecognized flags in mesg_type_flags");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Read the current number of indexes */
@@ -878,7 +878,7 @@ H5Pget_shared_mesg_index(hid_t plist_id, unsigned index_num, unsigned *mesg_type
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Read the current number of indexes */
@@ -1118,7 +1118,7 @@ H5Pset_shared_mesg_phase_change(hid_t plist_id, unsigned max_list, unsigned min_
         min_btree = 0;
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     if (H5P_set(plist, H5F_CRT_SHMSG_LIST_MAX_NAME, &max_list) < 0)
@@ -1149,7 +1149,7 @@ H5Pget_shared_mesg_phase_change(hid_t plist_id, unsigned *max_list /*out*/, unsi
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get value(s) */
@@ -1165,32 +1165,21 @@ H5Pget_shared_mesg_phase_change(hid_t plist_id, unsigned *max_list /*out*/, unsi
 } /* end H5Pget_shared_mesg_phase_change() */
 
 /*-------------------------------------------------------------------------
- * Function:	H5Pset_file_space_strategy
+ * Function:	H5P__set_file_space_strategy
  *
- * Purpose:	Sets the "strategy" that the library employs in managing file space
- *		    Sets the "persist" value as to persist free-space or not
- *		    Sets the "threshold" value that the free space manager(s) will use to track free space
- *sections. Ignore "persist" and "threshold" for strategies that do not use free-space managers
+ * Purpose:	Internal routine to set file space strategy properties.
  *
  * Return:	Non-negative on success/Negative on failure
  *
  *-------------------------------------------------------------------------
  */
 herr_t
-H5Pset_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t strategy, hbool_t persist, hsize_t threshold)
+H5P__set_file_space_strategy(H5P_genplist_t *plist, H5F_fspace_strategy_t strategy, hbool_t persist,
+                             hsize_t threshold)
 {
-    H5P_genplist_t *plist;               /* Property list pointer */
-    herr_t          ret_value = SUCCEED; /* Return value */
-
-    FUNC_ENTER_API(FAIL)
-
-    /* Check arguments */
-    if (strategy >= H5F_FSPACE_STRATEGY_NTYPES)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid strategy");
+    herr_t ret_value = SUCCEED; /* Return value */
 
-    /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
-        HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
+    FUNC_ENTER_PACKAGE
 
     /* Set value(s), if non-zero */
     if (H5P_set(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &strategy) < 0)
@@ -1206,32 +1195,64 @@ H5Pset_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t strategy, hbool
     } /* end if */
 
 done:
-    FUNC_LEAVE_API(ret_value)
-} /* H5Pset_file_space_strategy() */
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5P__set_file_space_strategy() */
 
 /*-------------------------------------------------------------------------
- * Function:	H5Pget_file_space_strategy
+ * Function:	H5Pset_file_space_strategy
  *
- * Purpose:	Retrieves the strategy, persist, and threshold that the library
- *          uses in managing file space.
+ * Purpose:	Sets the "strategy" that the library employs in managing file space
+ *		Sets the "persist" value as to persist free-space or not
+ *		Sets the "threshold" value that the free space manager(s) will
+ *              use to track free space sections. Ignore "persist" and
+ *              "threshold" for strategies that do not use free-space managers.
  *
  * Return:	Non-negative on success/Negative on failure
  *
  *-------------------------------------------------------------------------
  */
 herr_t
-H5Pget_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t *strategy /*out*/, hbool_t *persist /*out*/,
-                           hsize_t *threshold /*out*/)
+H5Pset_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t strategy, hbool_t persist, hsize_t threshold)
 {
     H5P_genplist_t *plist;               /* Property list pointer */
     herr_t          ret_value = SUCCEED; /* Return value */
 
     FUNC_ENTER_API(FAIL)
 
+    /* Check arguments */
+    if (strategy >= H5F_FSPACE_STRATEGY_NTYPES)
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid strategy");
+
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
+    /* Set value(s) */
+    if (H5P__set_file_space_strategy(plist, strategy, persist, threshold) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set file space strategy values");
+
+done:
+    FUNC_LEAVE_API(ret_value)
+} /* H5Pset_file_space_strategy() */
+
+/*-------------------------------------------------------------------------
+ * Function:	H5P__get_file_space_strategy
+ *
+ * Purpose:	Retrieves the strategy, persist, and threshold that the library
+ *          uses in managing file space.
+ *
+ * Return:	Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5P__get_file_space_strategy(H5P_genplist_t *plist, H5F_fspace_strategy_t *strategy /*out*/,
+                             hbool_t *persist /*out*/, hsize_t *threshold /*out*/)
+{
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_PACKAGE
+
     /* Get value(s) */
     if (strategy)
         if (H5P_get(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, strategy) < 0)
@@ -1243,6 +1264,37 @@ H5Pget_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t *strategy /*out
         if (H5P_get(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, threshold) < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get free-space threshold");
 
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5P__get_file_space_strategy() */
+
+/*-------------------------------------------------------------------------
+ * Function:	H5Pget_file_space_strategy
+ *
+ * Purpose:	Retrieves the strategy, persist, and threshold that the library
+ *          uses in managing file space.
+ *
+ * Return:	Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pget_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t *strategy /*out*/, hbool_t *persist /*out*/,
+                           hsize_t *threshold /*out*/)
+{
+    H5P_genplist_t *plist;               /* Property list pointer */
+    herr_t          ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_API(FAIL)
+
+    /* Get the plist structure */
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, true)))
+        HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
+
+    /* Get value(s) */
+    if (H5P__get_file_space_strategy(plist, strategy, persist, threshold) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space strategy values");
+
 done:
     FUNC_LEAVE_API(ret_value)
 } /* H5Pget_file_space_strategy() */
@@ -1331,7 +1383,7 @@ H5Pset_file_space_page_size(hid_t plist_id, hsize_t fsp_size)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     if (fsp_size < H5F_FILE_SPACE_PAGE_SIZE_MIN)
@@ -1367,7 +1419,7 @@ H5Pget_file_space_page_size(hid_t plist_id, hsize_t *fsp_size /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get value */
diff --git a/src/H5Pgcpl.c b/src/H5Pgcpl.c
index 657eecd95f6..3dcb4c6ebc3 100644
--- a/src/H5Pgcpl.c
+++ b/src/H5Pgcpl.c
@@ -151,7 +151,7 @@ H5Pset_local_heap_size_hint(hid_t plist_id, size_t size_hint)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get value */
@@ -191,7 +191,7 @@ H5Pget_local_heap_size_hint(hid_t plist_id, size_t *size_hint /*out*/)
         H5O_ginfo_t     ginfo; /* Group information structure */
 
         /* Get the plist structure */
-        if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE)))
+        if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE, true)))
             HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
         /* Get value */
@@ -239,7 +239,7 @@ H5Pset_link_phase_change(hid_t plist_id, unsigned max_compact, unsigned min_dens
         HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "min dense value must be < 65536");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get group info */
@@ -285,7 +285,7 @@ H5Pget_link_phase_change(hid_t plist_id, unsigned *max_compact /*out*/, unsigned
         H5O_ginfo_t     ginfo; /* Group information structure */
 
         /* Get the plist structure */
-        if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE)))
+        if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE, true)))
             HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
         /* Get group info */
@@ -335,7 +335,7 @@ H5Pset_est_link_info(hid_t plist_id, unsigned est_num_entries, unsigned est_name
         HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "est. name length must be < 65536");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get group info */
@@ -381,7 +381,7 @@ H5Pget_est_link_info(hid_t plist_id, unsigned *est_num_entries /*out*/, unsigned
         H5O_ginfo_t     ginfo; /* Group information structure */
 
         /* Get the plist structure */
-        if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE)))
+        if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE, true)))
             HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
         /* Get group info */
@@ -421,7 +421,7 @@ H5Pset_link_creation_order(hid_t plist_id, unsigned crt_order_flags)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "tracking creation order is required for index");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get link info */
@@ -466,7 +466,7 @@ H5Pget_link_creation_order(hid_t plist_id, unsigned *crt_order_flags /*out*/)
         *crt_order_flags = 0;
 
         /* Get the plist structure */
-        if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE)))
+        if (NULL == (plist = H5P_object_verify(plist_id, H5P_GROUP_CREATE, true)))
             HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
         /* Get link info */
diff --git a/src/H5Pint.c b/src/H5Pint.c
index 3ff94a63d3d..0cacaf79527 100644
--- a/src/H5Pint.c
+++ b/src/H5Pint.c
@@ -804,8 +804,14 @@ H5P__do_prop_cb1(H5SL_t *slist, H5P_genprop_t *prop, H5P_prp_cb1_t cb)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "memory allocation failed for temporary property value");
     H5MM_memcpy(tmp_value, prop->value, prop->size);
 
-    /* Call "type 1" callback ('create', 'copy' or 'close') */
-    if (cb(prop->name, prop->size, tmp_value) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call "type 1" callback ('create', 'copy' or 'close') */
+            ret_value = cb(prop->name, prop->size, tmp_value);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "Property callback failed");
 
     /* Make a copy of the class's property */
@@ -1016,7 +1022,15 @@ H5P_copy_plist(const H5P_genplist_t *old_plist, bool app_ref)
 
             /* Call property copy callback, if it exists */
             if (new_prop->copy) {
-                if ((new_prop->copy)(new_prop->name, new_prop->size, new_prop->value) < 0) {
+                herr_t status;
+
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(H5I_INVALID_HID)
+                    {
+                        status = (new_prop->copy)(new_prop->name, new_prop->size, new_prop->value);
+                    }
+                H5_AFTER_USER_CB(H5I_INVALID_HID)
+                if (status < 0) {
                     H5P__free_prop(new_prop);
                     HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, H5I_INVALID_HID, "Can't copy property");
                 } /* end if */
@@ -1104,7 +1118,16 @@ H5P_copy_plist(const H5P_genplist_t *old_plist, bool app_ref)
     tclass = new_plist->pclass;
     while (NULL != tclass) {
         if (NULL != tclass->copy_func) {
-            if ((tclass->copy_func)(new_plist_id, old_plist->plist_id, old_plist->pclass->copy_data) < 0) {
+            herr_t status;
+
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(H5I_INVALID_HID)
+                {
+                    status =
+                        (tclass->copy_func)(new_plist_id, old_plist->plist_id, old_plist->pclass->copy_data);
+                }
+            H5_AFTER_USER_CB(H5I_INVALID_HID)
+            if (status < 0) {
                 /* Delete ID, ignore return value */
                 H5I_remove(new_plist_id);
                 HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, H5I_INVALID_HID, "Can't initialize property");
@@ -1530,8 +1553,15 @@ H5P__free_prop_cb(void *item, void H5_ATTR_UNUSED *key, void *op_data)
     assert(tprop);
 
     /* Call the close callback and ignore the return value, there's nothing we can do about it */
-    if (make_cb && tprop->close != NULL)
-        (tprop->close)(tprop->name, tprop->size, tprop->value);
+    if (make_cb && tprop->close != NULL) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB_NOCHECK
+            {
+                /* Call user's callback */
+                (tprop->close)(tprop->name, tprop->size, tprop->value);
+            }
+        H5_AFTER_USER_CB_NOCHECK
+    }
 
     /* Free the property, ignoring return value, nothing we can do */
     H5P__free_prop(tprop);
@@ -2009,7 +2039,15 @@ H5P_create_id(H5P_genclass_t *pclass, bool app_ref)
     tclass = plist->pclass;
     while (NULL != tclass) {
         if (NULL != tclass->create_func) {
-            if ((tclass->create_func)(plist_id, tclass->create_data) < 0) {
+            herr_t status;
+
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    status = (tclass->create_func)(plist_id, tclass->create_data);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (status < 0) {
                 /* Delete ID, ignore return value */
                 H5I_remove(plist_id);
                 HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, H5I_INVALID_HID, "Can't initialize property");
@@ -3033,8 +3071,14 @@ H5P__set_plist_cb(H5P_genplist_t *plist, const char *name, H5P_genprop_t *prop,
             HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "memory allocation failed temporary property value");
         H5MM_memcpy(tmp_value, udata->value, prop->size);
 
-        /* Call user's callback */
-        if ((*(prop->set))(plist->plist_id, name, prop->size, tmp_value) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                /* Call user's callback */
+                ret_value = (*(prop->set))(plist->plist_id, name, prop->size, tmp_value);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't set property value");
 
         /* Set the pointer for copying */
@@ -3046,8 +3090,14 @@ H5P__set_plist_cb(H5P_genplist_t *plist, const char *name, H5P_genprop_t *prop,
 
     /* Free any previous value for the property */
     if (NULL != prop->del) {
-        /* Call user's 'delete' callback */
-        if ((*(prop->del))(plist->plist_id, name, prop->size, prop->value) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                /* Call user's callback */
+                ret_value = (*(prop->del))(plist->plist_id, name, prop->size, prop->value);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "can't release property value");
     } /* end if */
 
@@ -3111,8 +3161,14 @@ H5P__set_pclass_cb(H5P_genplist_t *plist, const char *name, H5P_genprop_t *prop,
             HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "memory allocation failed temporary property value");
         H5MM_memcpy(tmp_value, udata->value, prop->size);
 
-        /* Call user's callback */
-        if ((*(prop->set))(plist->plist_id, name, prop->size, tmp_value) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                /* Call user's callback */
+                ret_value = (*(prop->set))(plist->plist_id, name, prop->size, tmp_value);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't set property value");
 
         /* Set the pointer for copying */
@@ -3709,8 +3765,15 @@ H5P__cmp_prop(const H5P_genprop_t *prop1, const H5P_genprop_t *prop2)
     if (prop1->value != NULL && prop2->value == NULL)
         HGOTO_DONE(1);
     if (prop1->value != NULL) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB_NOCHECK
+            {
+                /* Call comparison routine */
+                cmp_value = prop1->cmp(prop1->value, prop2->value, prop1->size);
+            }
+        H5_AFTER_USER_CB_NOCHECK
         /* Call comparison routine */
-        if ((cmp_value = prop1->cmp(prop1->value, prop2->value, prop1->size)) != 0)
+        if (0 != cmp_value)
             HGOTO_DONE(cmp_value);
     } /* end if */
 
@@ -4084,6 +4147,40 @@ H5P_isa_class(hid_t plist_id, hid_t pclass_id)
     FUNC_LEAVE_NOAPI(ret_value)
 } /* H5P_isa_class() */
 
+/*-------------------------------------------------------------------------
+ * Function:    H5P_is_default_plist
+ *
+ * Purpose:     Determine if the provided ID refers to a default property list.
+ *
+ * Return:      True if the ID refers to a default property list, false otherwise.
+ *
+ *-------------------------------------------------------------------------
+ */
+bool
+H5P_is_default_plist(hid_t plist_id)
+{
+    hid_t H5I_def_plists[] = {
+        H5P_LST_FILE_CREATE_ID_g,      H5P_LST_FILE_ACCESS_ID_g,      H5P_LST_DATASET_CREATE_ID_g,
+        H5P_LST_DATASET_ACCESS_ID_g,   H5P_LST_DATASET_XFER_ID_g,     H5P_LST_FILE_MOUNT_ID_g,
+        H5P_LST_GROUP_CREATE_ID_g,     H5P_LST_GROUP_ACCESS_ID_g,     H5P_LST_DATATYPE_CREATE_ID_g,
+        H5P_LST_DATATYPE_ACCESS_ID_g,  H5P_LST_MAP_CREATE_ID_g,       H5P_LST_MAP_ACCESS_ID_g,
+        H5P_LST_ATTRIBUTE_CREATE_ID_g, H5P_LST_ATTRIBUTE_ACCESS_ID_g, H5P_LST_OBJECT_COPY_ID_g,
+        H5P_LST_LINK_CREATE_ID_g,      H5P_LST_LINK_ACCESS_ID_g,      H5P_LST_VOL_INITIALIZE_ID_g,
+        H5P_LST_REFERENCE_ACCESS_ID_g};
+
+    size_t num_default_plists = (size_t)(sizeof(H5I_def_plists) / sizeof(H5I_def_plists[0]));
+
+    if (plist_id == H5P_DEFAULT)
+        return true;
+
+    for (size_t i = 0; i < num_default_plists; i++) {
+        if (plist_id == H5I_def_plists[i])
+            return true;
+    }
+
+    return false;
+}
+
 /*--------------------------------------------------------------------------
  NAME
     H5P_object_verify
@@ -4091,9 +4188,10 @@ H5P_isa_class(hid_t plist_id, hid_t pclass_id)
     Internal routine to query whether a property list is a certain class and
         retrieve the property list object associated with it.
  USAGE
-    void *H5P_object_verify(plist_id, pclass_id)
+    void *H5P_object_verify(plist_id, pclass_id, allow_default)
         hid_t plist_id;         IN: Property list to query
         hid_t pclass_id;        IN: Property class to query
+        bool  allow_default;    IN: Whether to consider the default property lists valid
  RETURNS
     Success: valid pointer to a property list object
     Failure: NULL
@@ -4113,7 +4211,7 @@ H5P_isa_class(hid_t plist_id, hid_t pclass_id)
  REVISION LOG
 --------------------------------------------------------------------------*/
 H5P_genplist_t *
-H5P_object_verify(hid_t plist_id, hid_t pclass_id)
+H5P_object_verify(hid_t plist_id, hid_t pclass_id, bool allow_default)
 {
     H5P_genplist_t *ret_value = NULL; /* Return value */
 
@@ -4123,6 +4221,10 @@ H5P_object_verify(hid_t plist_id, hid_t pclass_id)
     if (H5P_isa_class(plist_id, pclass_id) != true)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTCOMPARE, NULL, "property list is not a member of the class");
 
+    if (!allow_default && H5P_is_default_plist(plist_id)) {
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTCOMPARE, NULL, "property list is a default list");
+    }
+
     /* Get the plist structure */
     if (NULL == (ret_value = (H5P_genplist_t *)H5I_object(plist_id)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, NULL, "can't find object for ID");
@@ -4167,8 +4269,13 @@ H5P__iterate_plist_cb(void *_item, void *_key, void *_udata)
 
     /* Check if we've found the correctly indexed property */
     if (*udata->curr_idx_ptr >= udata->prev_idx) {
-        /* Call the callback function */
-        ret_value = (*udata->cb_func)(item, udata->udata);
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB_NOERR(H5_ITER_ERROR)
+            {
+                /* Call the callback function */
+                ret_value = (*udata->cb_func)(item, udata->udata);
+            }
+        H5_AFTER_USER_CB_NOERR(H5_ITER_ERROR)
         if (ret_value != 0)
             HGOTO_DONE(ret_value);
     } /* end if */
@@ -4378,8 +4485,13 @@ H5P__iterate_pclass_cb(void *_item, void H5_ATTR_NDEBUG_UNUSED *_key, void *_uda
 
     /* Check if we've found the correctly indexed property */
     if (*udata->curr_idx_ptr >= udata->prev_idx) {
-        /* Call the callback function */
-        ret_value = (*udata->cb_func)(item, udata->udata);
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB_NOERR(H5_ITER_ERROR)
+            {
+                /* Call the callback function */
+                ret_value = (*udata->cb_func)(item, udata->udata);
+            }
+        H5_AFTER_USER_CB_NOERR(H5_ITER_ERROR)
         if (ret_value != 0)
             HGOTO_DONE(ret_value);
     } /* end if */
@@ -4614,8 +4726,14 @@ H5P__get_cb(H5P_genplist_t *plist, const char *name, H5P_genprop_t *prop, void *
             HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "memory allocation failed temporary property value");
         H5MM_memcpy(tmp_value, prop->value, prop->size);
 
-        /* Call user's callback */
-        if ((*(prop->get))(plist->plist_id, name, prop->size, tmp_value) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                /* Call user's callback */
+                ret_value = (*(prop->get))(plist->plist_id, name, prop->size, tmp_value);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't set property value");
 
         /* Copy new [possibly unchanged] value into return value */
@@ -4719,8 +4837,14 @@ H5P__del_plist_cb(H5P_genplist_t *plist, const char *name, H5P_genprop_t *prop,
 
     /* Pass value to 'close' callback, if it exists */
     if (NULL != prop->del) {
-        /* Call user's callback */
-        if ((*(prop->del))(plist->plist_id, name, prop->size, prop->value) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                /* Call user's callback */
+                ret_value = (*(prop->del))(plist->plist_id, name, prop->size, prop->value);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "can't release property value");
     } /* end if */
 
@@ -4794,8 +4918,14 @@ H5P__del_pclass_cb(H5P_genplist_t *plist, const char *name, H5P_genprop_t *prop,
                         "memory allocation failed for temporary property value");
         H5MM_memcpy(tmp_value, prop->value, prop->size);
 
-        /* Call user's callback */
-        if ((*(prop->del))(plist->plist_id, name, prop->size, tmp_value) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                /* Call user's callback */
+                ret_value = (*(prop->del))(plist->plist_id, name, prop->size, tmp_value);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't close property value");
     } /* end if */
 
@@ -4934,7 +5064,14 @@ H5P__copy_prop_plist(hid_t dst_id, hid_t src_id, const char *name)
 
         /* Call property copy callback, if it exists */
         if (new_prop->copy) {
-            if ((new_prop->copy)(new_prop->name, new_prop->size, new_prop->value) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    /* Call user's callback */
+                    ret_value = (new_prop->copy)(new_prop->name, new_prop->size, new_prop->value);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "Can't copy property");
         } /* end if */
     }     /* end if */
@@ -4951,7 +5088,13 @@ H5P__copy_prop_plist(hid_t dst_id, hid_t src_id, const char *name)
 
         /* Call property creation callback, if it exists */
         if (new_prop->create) {
-            if ((new_prop->create)(new_prop->name, new_prop->size, new_prop->value) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    ret_value = (new_prop->create)(new_prop->name, new_prop->size, new_prop->value);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "Can't initialize property");
         } /* end if */
     }     /* end else */
@@ -5161,8 +5304,13 @@ H5P_close(H5P_genplist_t *plist)
         tclass = plist->pclass;
         while (NULL != tclass) {
             if (NULL != tclass->close_func) {
-                /* Call user's "close" callback function, ignoring return value */
-                (tclass->close_func)(plist->plist_id, tclass->close_data);
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        /* Call user's "close" callback function, ignoring return value */
+                        (tclass->close_func)(plist->plist_id, tclass->close_data);
+                    }
+                H5_AFTER_USER_CB(FAIL)
             } /* end if */
 
             /* Go up to parent class */
@@ -5188,8 +5336,13 @@ H5P_close(H5P_genplist_t *plist)
 
             /* Call property close callback, if it exists */
             if (tmp->close) {
-                /* Call the 'close' callback */
-                (tmp->close)(tmp->name, tmp->size, tmp->value);
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        /* Call user's callback */
+                        (tmp->close)(tmp->name, tmp->size, tmp->value);
+                    }
+                H5_AFTER_USER_CB(FAIL)
             } /* end if */
 
             /* Add property name to "seen" list */
@@ -5235,8 +5388,13 @@ H5P_close(H5P_genplist_t *plist)
                                         "memory allocation failed for temporary property value");
                         H5MM_memcpy(tmp_value, tmp->value, tmp->size);
 
-                        /* Call the 'close' callback */
-                        (tmp->close)(tmp->name, tmp->size, tmp_value);
+                        /* Prepare & restore library for user callback */
+                        H5_BEFORE_USER_CB(FAIL)
+                            {
+                                /* Call user's callback */
+                                (tmp->close)(tmp->name, tmp->size, tmp_value);
+                            }
+                        H5_AFTER_USER_CB(FAIL)
 
                         /* Release the temporary value buffer */
                         H5MM_xfree(tmp_value);
diff --git a/src/H5Plapl.c b/src/H5Plapl.c
index c6ca851e62c..bdb13bb4ddc 100644
--- a/src/H5Plapl.c
+++ b/src/H5Plapl.c
@@ -259,7 +259,7 @@ H5P__lacc_elink_fapl_set(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED
     if (l_fapl_id != H5P_DEFAULT) {
         H5P_genplist_t *l_fapl_plist;
 
-        if (NULL == (l_fapl_plist = (H5P_genplist_t *)H5P_object_verify(l_fapl_id, H5P_FILE_ACCESS)))
+        if (NULL == (l_fapl_plist = (H5P_genplist_t *)H5P_object_verify(l_fapl_id, H5P_FILE_ACCESS, true)))
             HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "can't get property list");
         if (((*(hid_t *)value) = H5P_copy_plist(l_fapl_plist, false)) < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to copy file access property list");
@@ -298,7 +298,7 @@ H5P__lacc_elink_fapl_get(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED
     if (l_fapl_id != H5P_DEFAULT) {
         H5P_genplist_t *l_fapl_plist;
 
-        if (NULL == (l_fapl_plist = (H5P_genplist_t *)H5P_object_verify(l_fapl_id, H5P_FILE_ACCESS)))
+        if (NULL == (l_fapl_plist = (H5P_genplist_t *)H5P_object_verify(l_fapl_id, H5P_FILE_ACCESS, true)))
             HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "can't get property list");
         if (((*(hid_t *)value) = H5P_copy_plist(l_fapl_plist, false)) < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to copy file access property list");
@@ -334,7 +334,7 @@ H5P__lacc_elink_fapl_enc(const void *value, void **_pp, size_t *size)
 
     /* Check for non-default FAPL */
     if (*elink_fapl != H5P_DEFAULT) {
-        if (NULL == (fapl_plist = (H5P_genplist_t *)H5P_object_verify(*elink_fapl, H5P_FILE_ACCESS)))
+        if (NULL == (fapl_plist = (H5P_genplist_t *)H5P_object_verify(*elink_fapl, H5P_FILE_ACCESS, true)))
             HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get property list");
         non_default_fapl = true;
     } /* end if */
@@ -492,7 +492,7 @@ H5P__lacc_elink_fapl_copy(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED
     if (l_fapl_id != H5P_DEFAULT) {
         H5P_genplist_t *l_fapl_plist;
 
-        if (NULL == (l_fapl_plist = (H5P_genplist_t *)H5P_object_verify(l_fapl_id, H5P_FILE_ACCESS)))
+        if (NULL == (l_fapl_plist = (H5P_genplist_t *)H5P_object_verify(l_fapl_id, H5P_FILE_ACCESS, true)))
             HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "can't get property list");
         if (((*(hid_t *)value) = H5P_copy_plist(l_fapl_plist, false)) < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to copy file access property list");
@@ -861,7 +861,7 @@ H5Pset_nlinks(hid_t plist_id, size_t nlinks)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "number of links must be positive");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_LINK_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_LINK_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set number of links */
@@ -897,7 +897,7 @@ H5Pget_nlinks(hid_t plist_id, size_t *nlinks /*out*/)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid pointer passed in");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_LINK_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_LINK_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the current number of links */
@@ -928,7 +928,7 @@ H5Pset_elink_prefix(hid_t plist_id, const char *prefix)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_LINK_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_LINK_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set prefix */
@@ -963,7 +963,7 @@ H5Pget_elink_prefix(hid_t plist_id, char *prefix /*out*/, size_t size)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_LINK_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_LINK_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the current prefix */
@@ -1008,7 +1008,7 @@ H5Pset_elink_fapl(hid_t lapl_id, hid_t fapl_id)
     FUNC_ENTER_API(FAIL)
 
     /* Check arguments */
-    if (NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS, false)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a link access property list");
 
     /* Set the file access property list for the link access */
@@ -1038,7 +1038,7 @@ H5Pget_elink_fapl(hid_t lapl_id)
     FUNC_ENTER_API(H5I_INVALID_HID)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, H5I_INVALID_HID, "can't find object for ID");
 
     if (H5P_get(plist, H5L_ACS_ELINK_FAPL_NAME, &ret_value) < 0)
@@ -1074,7 +1074,7 @@ H5Pset_elink_acc_flags(hid_t lapl_id, unsigned flags)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file open flags");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set flags */
@@ -1104,7 +1104,7 @@ H5Pget_elink_acc_flags(hid_t lapl_id, unsigned *flags /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get flags */
@@ -1142,7 +1142,7 @@ H5Pset_elink_cb(hid_t lapl_id, H5L_elink_traverse_t func, void *op_data)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "callback is NULL while user data is not");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Populate the callback info struct */
@@ -1177,7 +1177,7 @@ H5Pget_elink_cb(hid_t lapl_id, H5L_elink_traverse_t *func /*out*/, void **op_dat
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get callback_info */
diff --git a/src/H5Plcpl.c b/src/H5Plcpl.c
index 9afad0a278a..a2bebc85fa6 100644
--- a/src/H5Plcpl.c
+++ b/src/H5Plcpl.c
@@ -141,7 +141,7 @@ H5Pset_create_intermediate_group(hid_t plist_id, unsigned crt_intmd_group)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_LINK_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_LINK_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set value */
@@ -172,7 +172,7 @@ H5Pget_create_intermediate_group(hid_t plist_id, unsigned *crt_intmd_group /*out
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_LINK_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_LINK_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get values */
diff --git a/src/H5Pmapl.c b/src/H5Pmapl.c
index b3b9b776075..c10161cdb8a 100644
--- a/src/H5Pmapl.c
+++ b/src/H5Pmapl.c
@@ -156,7 +156,7 @@ H5Pset_map_iterate_hints(hid_t mapl_id, size_t key_prefetch_size, size_t key_all
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(mapl_id, H5P_MAP_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(mapl_id, H5P_MAP_ACCESS, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set sizes */
@@ -187,7 +187,7 @@ H5Pget_map_iterate_hints(hid_t mapl_id, size_t *key_prefetch_size /*out*/, size_
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(mapl_id, H5P_MAP_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(mapl_id, H5P_MAP_ACCESS, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the properties */
diff --git a/src/H5Pmodule.h b/src/H5Pmodule.h
index 6a65f3a269b..cd747b5a8f9 100644
--- a/src/H5Pmodule.h
+++ b/src/H5Pmodule.h
@@ -1114,6 +1114,10 @@
  * TAPL isn't supported yet.
  *
  *
+ * \defgroup MAPL VOL Data Mapping Properties
+ * \ingroup H5P
+ * Empty property class.
+ *
  *
  */
 
diff --git a/src/H5Pocpl.c b/src/H5Pocpl.c
index 39c2081f6fc..dd2e6316cfc 100644
--- a/src/H5Pocpl.c
+++ b/src/H5Pocpl.c
@@ -216,7 +216,7 @@ H5Pset_attr_phase_change(hid_t plist_id, unsigned max_compact, unsigned min_dens
         HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "min dense value must be < 65536");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set property values */
@@ -247,7 +247,7 @@ H5Pget_attr_phase_change(hid_t plist_id, unsigned *max_compact /*out*/, unsigned
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get values */
@@ -287,7 +287,7 @@ H5Pset_attr_creation_order(hid_t plist_id, unsigned crt_order_flags)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "tracking creation order is required for index");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get object header flags */
@@ -337,7 +337,7 @@ H5Pget_attr_creation_order(hid_t plist_id, unsigned *crt_order_flags /*out*/)
         *crt_order_flags = 0;
 
         /* Get the plist structure */
-        if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+        if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, true)))
             HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
         /* Get object header flags */
@@ -386,7 +386,7 @@ H5Pset_obj_track_times(hid_t plist_id, hbool_t track_times)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get object header flags */
@@ -429,7 +429,7 @@ H5Pget_obj_track_times(hid_t plist_id, hbool_t *track_times /*out*/)
         uint8_t         ohdr_flags; /* Object header flags */
 
         /* Get the plist structure */
-        if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+        if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, true)))
             HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
         /* Get object header flags */
@@ -553,7 +553,7 @@ H5Pmodify_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int flags, size_t
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no client data values supplied");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Modify the filter parameters of the I/O pipeline */
@@ -621,7 +621,7 @@ H5Pmodify_filter2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_fi
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no client data values supplied");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
 #ifdef TBD
@@ -701,7 +701,7 @@ H5Pset_filter2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number, H5Z_filte
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no client data values supplied");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Call the private function */
@@ -836,7 +836,7 @@ H5Pset_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int flags, size_t cd
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no client data values supplied");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Call the private function */
@@ -936,7 +936,7 @@ H5Pget_nfilters(hid_t plist_id)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the pipeline property to query */
@@ -979,7 +979,7 @@ H5Pget_nfilters2(hid_t plist_id, uint64_t H5_ATTR_UNUSED section_number)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
 #ifdef TBD
@@ -1057,7 +1057,7 @@ H5Pget_filter2(hid_t plist_id, unsigned idx, unsigned int *flags /*out*/, size_t
     } /* end if */
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, H5Z_FILTER_ERROR, "can't find object for ID");
 
     /* Get the pipeline property to query */
@@ -1152,7 +1152,7 @@ H5Pget_filter3(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section_nu
     } /* end if */
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, H5Z_FILTER_ERROR, "can't find object for ID");
 
     /* Get the pipeline property to query */
@@ -1280,7 +1280,7 @@ H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t id, unsigned int *flags /*out*
     } /* end if */
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get filter information */
@@ -1360,7 +1360,7 @@ H5Pget_filter_by_id3(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED sect
     } /* end if */
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get filter information */
@@ -1395,7 +1395,7 @@ H5Pall_filters_avail(hid_t plist_id)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the pipeline property to query */
@@ -1462,7 +1462,7 @@ H5Premove_filter(hid_t plist_id, H5Z_filter_t filter)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the pipeline property to modify */
@@ -1505,7 +1505,7 @@ H5Premove_filter2(hid_t H5_ATTR_UNUSED plist_id, uint64_t H5_ATTR_UNUSED section
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
 #ifdef TBD
@@ -1563,7 +1563,7 @@ H5Pset_deflate(hid_t plist_id, unsigned level)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid deflate level");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the pipeline property to append to */
@@ -1602,7 +1602,7 @@ H5Pset_fletcher32(hid_t plist_id)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the pipeline property to append to */
@@ -2182,7 +2182,7 @@ H5Pget_filter1(hid_t plist_id, unsigned idx, unsigned int *flags /*out*/, size_t
     } /* end if */
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, H5Z_FILTER_ERROR, "can't find object for ID");
 
     /* Get pipeline info */
@@ -2259,7 +2259,7 @@ H5Pget_filter_by_id1(hid_t plist_id, H5Z_filter_t id, unsigned int *flags /*out*
     } /* end if */
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get filter info */
diff --git a/src/H5Pocpypl.c b/src/H5Pocpypl.c
index 622e35f416e..65aaca03ecb 100644
--- a/src/H5Pocpypl.c
+++ b/src/H5Pocpypl.c
@@ -616,7 +616,7 @@ H5Pset_copy_object(hid_t plist_id, unsigned cpy_option)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unknown option specified");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_COPY)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_COPY, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set value */
@@ -646,7 +646,7 @@ H5Pget_copy_object(hid_t plist_id, unsigned *cpy_option /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_COPY)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_COPY, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get values */
@@ -693,7 +693,7 @@ H5Padd_merge_committed_dtype_path(hid_t plist_id, const char *path)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "path is empty string");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_COPY)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_COPY, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get dtype list */
@@ -745,7 +745,7 @@ H5Pfree_merge_committed_dtype_paths(hid_t plist_id)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_COPY)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_COPY, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get dtype list */
@@ -795,7 +795,7 @@ H5Pset_mcdt_search_cb(hid_t plist_id, H5O_mcdt_search_cb_t func, void *op_data)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "callback is NULL while user data is not");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_COPY)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_COPY, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Populate the callback info struct */
@@ -835,7 +835,7 @@ H5Pget_mcdt_search_cb(hid_t plist_id, H5O_mcdt_search_cb_t *func /*out*/, void *
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_COPY)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_OBJECT_COPY, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get callback info */
diff --git a/src/H5Ppkg.h b/src/H5Ppkg.h
index a1f3e3a5233..c4abf101640 100644
--- a/src/H5Ppkg.h
+++ b/src/H5Ppkg.h
@@ -193,6 +193,12 @@ H5_DLL herr_t H5P__decode_coll_md_read_flag_t(const void **_pp, void *value);
 /* Private FAPL routines */
 H5_DLL herr_t H5P__facc_set_def_driver(void);
 
+/* Private FCPL routines */
+H5_DLL herr_t H5P__get_file_space_strategy(H5P_genplist_t *plist, H5F_fspace_strategy_t *strategy,
+                                           hbool_t *persist, hsize_t *threshold);
+H5_DLL herr_t H5P__set_file_space_strategy(H5P_genplist_t *plist, H5F_fspace_strategy_t strategy,
+                                           hbool_t persist, hsize_t threshold);
+
 /* Private OCPL routines */
 H5_DLL herr_t H5P__get_filter(const struct H5Z_filter_info_t *filter, unsigned int *flags, size_t *cd_nelmts,
                               unsigned cd_values[], size_t namelen, char name[], unsigned *filter_config);
diff --git a/src/H5Pprivate.h b/src/H5Pprivate.h
index b4abbc3b549..b471e6ce039 100644
--- a/src/H5Pprivate.h
+++ b/src/H5Pprivate.h
@@ -201,6 +201,7 @@ H5_DLL herr_t H5P_get_filter_by_id(H5P_genplist_t *plist, H5Z_filter_t id, unsig
                                    size_t *cd_nelmts, unsigned cd_values[], size_t namelen, char name[],
                                    unsigned *filter_config);
 H5_DLL htri_t H5P_filter_in_pline(H5P_genplist_t *plist, H5Z_filter_t id);
+H5_DLL bool   H5P_is_default_plist(hid_t plist_id);
 
 /* Query internal fields of the property list struct */
 H5_DLL hid_t           H5P_get_plist_id(const H5P_genplist_t *plist);
@@ -208,7 +209,7 @@ H5_DLL H5P_genclass_t *H5P_get_class(const H5P_genplist_t *plist);
 
 /* *SPECIAL* Don't make more of these! -QAK */
 H5_DLL htri_t          H5P_isa_class(hid_t plist_id, hid_t pclass_id);
-H5_DLL H5P_genplist_t *H5P_object_verify(hid_t plist_id, hid_t pclass_id);
+H5_DLL H5P_genplist_t *H5P_object_verify(hid_t plist_id, hid_t pclass_id, bool allow_default);
 
 /* Private DCPL routines */
 H5_DLL herr_t H5P_fill_value_defined(H5P_genplist_t *plist, H5D_fill_value_t *status);
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index cd89de52879..ee7d1d9b8bd 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -1012,7 +1012,7 @@ H5_DLL htri_t H5Pexist(hid_t plist_id, const char *name);
  *
  *          The property name must exist or this routine will fail.
  *
- *          If the \p get callback routine returns an error, \ value will
+ *          If the \p get callback routine returns an error, \p value will
  *          not be modified.
  *
  * \since 1.4.0
@@ -4252,7 +4252,7 @@ H5_DLL herr_t H5Pget_evict_on_close(hid_t fapl_id, hbool_t *evict_on_close);
  *          application can retrieve a file handle for low-level access to
  *          a particular member of a family of files. The file handle is
  *          retrieved with a separate call to H5Fget_vfd_handle() (or,
- *          in special circumstances, to H5FDget_vfd_handle(), see \ref VFL).
+ *          in special circumstances, to H5FDget_vfd_handle(), see \ref VFLTN).
  *
  * \since 1.6.0
  *
@@ -4536,8 +4536,7 @@ H5_DLL herr_t H5Pget_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t
  *          access property list, and H5Fget_mdc_logging_status() will return
  *          the current state of the logging flags.
  *
- *          The log format is described in the
- *           Metadata Cache Logging document.
+ *          The log format is described in the \ref_mdc_logging document.
  *
  * \since 1.10.0
  */
@@ -5200,7 +5199,7 @@ H5_DLL herr_t H5Pset_evict_on_close(hid_t fapl_id, hbool_t evict_on_close);
  *          retrieve a file handle for low-level access to a particular member
  *          of a family of files. The file handle is retrieved with a separate
  *          call to H5Fget_vfd_handle() (or, in special circumstances, to
- *          H5FDget_vfd_handle(); see \ref VFL).
+ *          H5FDget_vfd_handle(); see \ref VFLTN).
  *
  *          The value of \p offset is an offset in bytes from the beginning of
  *          the HDF5 file, identifying a user-determined location within the
@@ -5940,7 +5939,7 @@ H5_DLL herr_t H5Pset_metadata_read_attempts(hid_t plist_id, unsigned attempts);
  *          low-level access to the particular member of a set of \TText{MULTI}
  *          files in which that type of data is stored. The file handle is
  *          retrieved with a separate call to H5Fget_vfd_handle() (or, in special
- *          circumstances, to H5FDget_vfd_handle(); see \ref VFL.
+ *          circumstances, to H5FDget_vfd_handle(); see \ref VFLTN.
  *
  * The type of data specified in \p type may be one of the following:
  *
diff --git a/src/H5Pstrcpl.c b/src/H5Pstrcpl.c
index 57543444239..254ed170896 100644
--- a/src/H5Pstrcpl.c
+++ b/src/H5Pstrcpl.c
@@ -146,7 +146,7 @@ H5Pset_char_encoding(hid_t plist_id, H5T_cset_t encoding)
         HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "character encoding is not valid");
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_STRING_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_STRING_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Set the character encoding */
@@ -175,7 +175,7 @@ H5Pget_char_encoding(hid_t plist_id, H5T_cset_t *encoding /*out*/)
     FUNC_ENTER_API(FAIL)
 
     /* Get the plist structure */
-    if (NULL == (plist = H5P_object_verify(plist_id, H5P_STRING_CREATE)))
+    if (NULL == (plist = H5P_object_verify(plist_id, H5P_STRING_CREATE, true)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get value */
diff --git a/src/H5Rpublic.h b/src/H5Rpublic.h
index c7e03e126f2..93acee6a15a 100644
--- a/src/H5Rpublic.h
+++ b/src/H5Rpublic.h
@@ -49,13 +49,13 @@
  *           encoded into the datatype message header.
  */
 typedef enum {
-    H5R_BADTYPE         = (-1), /**< Invalid reference type           */
-    H5R_OBJECT1         = 0,    /**< Backward compatibility (object)  */
-    H5R_DATASET_REGION1 = 1,    /**< Backward compatibility (region)  */
-    H5R_OBJECT2         = 2,    /**< Object reference                 */
-    H5R_DATASET_REGION2 = 3,    /**< Region reference                 */
-    H5R_ATTR            = 4,    /**< Attribute Reference              */
-    H5R_MAXTYPE         = 5     /**< Highest type (invalid)           */
+    H5R_BADTYPE         = (-1), /**< Invalid reference type \since 1.0.0           */
+    H5R_OBJECT1         = 0,    /**< Backward compatibility (object) \since 1.12.0 */
+    H5R_DATASET_REGION1 = 1,    /**< Backward compatibility (region) \since 1.12.0 */
+    H5R_OBJECT2         = 2,    /**< Object reference \since 1.12.0                */
+    H5R_DATASET_REGION2 = 3,    /**< Region reference \since 1.12.0                */
+    H5R_ATTR            = 4,    /**< Attribute Reference \since 1.12.0             */
+    H5R_MAXTYPE         = 5     /**< Highest type (invalid) \since 1.0.0           */
 } H5R_type_t;
 //! 
 
diff --git a/src/H5SMcache.c b/src/H5SMcache.c
index 3be26d6945c..eb6d612a783 100644
--- a/src/H5SMcache.c
+++ b/src/H5SMcache.c
@@ -28,10 +28,11 @@
 /***********/
 /* Headers */
 /***********/
-#include "H5Eprivate.h"  /* Error handling		  	*/
+#include "H5private.h"   /* Generic Functions                    */
+#include "H5Eprivate.h"  /* Error handling		  	 */
 #include "H5Fprivate.h"  /* File access                          */
 #include "H5FLprivate.h" /* Free Lists                           */
-#include "H5MMprivate.h" /* Memory management			*/
+#include "H5MMprivate.h" /* Memory management			 */
 #include "H5SMpkg.h"     /* Shared object header messages        */
 
 /****************/
diff --git a/src/H5Spublic.h b/src/H5Spublic.h
index 654ab1c859c..17c0b597938 100644
--- a/src/H5Spublic.h
+++ b/src/H5Spublic.h
@@ -60,6 +60,8 @@
             * increasing offset order. Note that the order is only increasing                                \
             * for each call to H5Sget_seq_list(), the next set of sequences                                  \
             * could start with an earlier offset than the previous one.                                      \
+            *                                                                                                \
+            * \since 1.12.0                                                                                  \
             */
 #define H5S_SEL_ITER_SHARE_WITH_DATASPACE                                                                    \
     0x0002 /**< Don't copy the dataspace selection when creating the selection                               \
@@ -67,24 +69,26 @@
             * but the dataspace \Bold{MUST NOT} be modified or closed until the                              \
             * selection iterator is closed or the iterator's behavior will be                                \
             * undefined.                                                                                     \
+            *                                                                                                \
+            * \since 1.12.0                                                                                  \
             */
 
 /**
  * Types of dataspaces
  */
 typedef enum H5S_class_t {
-    H5S_NO_CLASS = -1, /**< Error                                      */
-    H5S_SCALAR   = 0,  /**< Singleton (scalar)                         */
-    H5S_SIMPLE   = 1,  /**< Regular grid                               */
-    H5S_NULL     = 2   /**< Empty set                                  */
+    H5S_NO_CLASS = -1, /**< Error \since 1.0.0                         */
+    H5S_SCALAR   = 0,  /**< Singleton (scalar) \since 1.0.0            */
+    H5S_SIMPLE   = 1,  /**< Regular grid \since 1.0.0                  */
+    H5S_NULL     = 2   /**< Empty set \since 1.8.0                     */
 } H5S_class_t;
 
 /**
  * Different ways of combining selections
  */
 typedef enum H5S_seloper_t {
-    H5S_SELECT_NOOP = -1, /**< Error                                     */
-    H5S_SELECT_SET  = 0,  /**< Select "set" operation 		             */
+    H5S_SELECT_NOOP = -1, /**< Error \since 1.0.0                        */
+    H5S_SELECT_SET  = 0,  /**< Select "set" operation \since 1.0.0       */
     H5S_SELECT_OR,        /**< Binary "or" operation for hyperslabs
                            * (add new selection to existing selection)
                            * \code
@@ -92,6 +96,8 @@ typedef enum H5S_seloper_t {
                            * New region:             BBBBBBBBBB
                            * A or B:           CCCCCCCCCCCCCCCC
                            * \endcode
+                           *
+                           * \since 1.0.0
                            */
     H5S_SELECT_AND,       /**< Binary "and" operation for hyperslabs
                            * (only leave overlapped regions in selection)
@@ -100,6 +106,8 @@ typedef enum H5S_seloper_t {
                            * New region:             BBBBBBBBBB
                            * A and B:                CCCC
                            * \endcode
+                           *
+                           * \since 1.6.0
                            */
     H5S_SELECT_XOR,       /**< Binary "xor" operation for hyperslabs
                            * (only leave non-overlapped regions in selection)
@@ -108,6 +116,8 @@ typedef enum H5S_seloper_t {
                            * New region:             BBBBBBBBBB
                            * A xor B:          CCCCCC    CCCCCC
                            * \endcode
+                           *
+                           * \since 1.6.0
                            */
     H5S_SELECT_NOTB,      /**< Binary "not" operation for hyperslabs
                            * (only leave non-overlapped regions in original selection)
@@ -116,6 +126,8 @@ typedef enum H5S_seloper_t {
                            * New region:             BBBBBBBBBB
                            * A not B:          CCCCCC
                            * \endcode
+                           *
+                           * \since 1.6.0
                            */
     H5S_SELECT_NOTA,      /**< Binary "not" operation for hyperslabs
                            * (only leave non-overlapped regions in new selection)
@@ -124,22 +136,24 @@ typedef enum H5S_seloper_t {
                            * New region:             BBBBBBBBBB
                            * B not A:                    CCCCCC
                            * \endcode
+                           *
+                           * \since 1.6.0
                            */
-    H5S_SELECT_APPEND,    /**< Append elements to end of point selection */
-    H5S_SELECT_PREPEND,   /**< Prepend elements to beginning of point selection */
-    H5S_SELECT_INVALID    /**< Invalid upper bound on selection operations */
+    H5S_SELECT_APPEND,    /**< Append elements to end of point selection \since 1.4.0 */
+    H5S_SELECT_PREPEND,   /**< Prepend elements to beginning of point selection \since 1.4.0 */
+    H5S_SELECT_INVALID    /**< Invalid upper bound on selection operations \since 1.0.0 */
 } H5S_seloper_t;
 
 /**
  * Selection type
  */
 typedef enum {
-    H5S_SEL_ERROR      = -1, /**< Error                                 */
-    H5S_SEL_NONE       = 0,  /**< Empty selection                       */
-    H5S_SEL_POINTS     = 1,  /**< Set of points                         */
-    H5S_SEL_HYPERSLABS = 2,  /**< Hyperslab                             */
-    H5S_SEL_ALL        = 3,  /**< Everything	                        */
-    H5S_SEL_N                /**< Sentinel \internal THIS MUST BE LAST	*/
+    H5S_SEL_ERROR      = -1, /**< Error \since 1.0.0                                */
+    H5S_SEL_NONE       = 0,  /**< Empty selection \since 1.0.0                      */
+    H5S_SEL_POINTS     = 1,  /**< Set of points \since 1.0.0                        */
+    H5S_SEL_HYPERSLABS = 2,  /**< Hyperslab \since 1.0.0                            */
+    H5S_SEL_ALL        = 3,  /**< Everything \since 1.0.0                           */
+    H5S_SEL_N                /**< Sentinel \internal THIS MUST BE LAST \since 1.0.0 */
 } H5S_sel_type;
 
 #ifdef __cplusplus
diff --git a/src/H5Sselect.c b/src/H5Sselect.c
index 88110be5f97..d40474a5ac0 100644
--- a/src/H5Sselect.c
+++ b/src/H5Sselect.c
@@ -1449,8 +1449,14 @@ H5S_select_iterate(void *buf, const H5T_t *type, H5S_t *space, const H5S_sel_ite
                 /* Check which type of callback to make */
                 switch (op->op_type) {
                     case H5S_SEL_ITER_OP_APP:
-                        /* Make the application callback */
-                        user_ret = (op->u.app_op.op)(loc, op->u.app_op.type_id, ndims, coords, op_data);
+                        /* Prepare & restore library for user callback */
+                        H5_BEFORE_USER_CB(H5_ITER_ERROR)
+                            {
+                                /* Make the application callback */
+                                user_ret =
+                                    (op->u.app_op.op)(loc, op->u.app_op.type_id, ndims, coords, op_data);
+                            }
+                        H5_AFTER_USER_CB(H5_ITER_ERROR)
                         break;
 
                     case H5S_SEL_ITER_OP_LIB:
diff --git a/src/H5T.c b/src/H5T.c
index 1999141cfc1..4ad4f2dbf97 100644
--- a/src/H5T.c
+++ b/src/H5T.c
@@ -3198,7 +3198,14 @@ H5T__register(H5T_pers_t pers, const char *name, H5T_t *src, H5T_t *dst, H5T_con
                     HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL,
                                 "unable to register ID for destination datatype");
 
-                if ((conv->u.app_func)(tmp_sid, tmp_did, &cdata, 0, 0, 0, NULL, NULL, H5CX_get_dxpl()) < 0) {
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        ret_value = (conv->u.app_func)(tmp_sid, tmp_did, &cdata, 0, 0, 0, NULL, NULL,
+                                                       H5CX_get_dxpl());
+                    }
+                H5_AFTER_USER_CB(FAIL)
+                if (ret_value < 0) {
                     if (H5I_dec_ref(tmp_sid) < 0)
                         HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDEC, FAIL,
                                     "unable to decrement reference count on temporary ID");
@@ -5870,7 +5877,13 @@ H5T__path_find_init_new_path(H5T_path_t *path, const H5T_t *src, const H5T_t *ds
                 HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL,
                             "unable to register ID for destination datatype");
 
-            status = (conv->u.app_func)(src_id, dst_id, &(path->cdata), 0, 0, 0, NULL, NULL, H5CX_get_dxpl());
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    status = (conv->u.app_func)(src_id, dst_id, &(path->cdata), 0, 0, 0, NULL, NULL,
+                                                H5CX_get_dxpl());
+                }
+            H5_AFTER_USER_CB(FAIL)
         }
         else
             status = (conv->u.lib_func)(path->src, path->dst, &(path->cdata), conv_ctx, 0, 0, 0, NULL, NULL);
@@ -5924,8 +5937,13 @@ H5T__path_find_init_new_path(H5T_path_t *path, const H5T_t *src, const H5T_t *ds
                 HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL,
                             "unable to register ID for destination datatype");
 
-            status = (H5T_g.soft[i].conv.u.app_func)(src_id, dst_id, &(path->cdata), 0, 0, 0, NULL, NULL,
-                                                     H5CX_get_dxpl());
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    status = (H5T_g.soft[i].conv.u.app_func)(src_id, dst_id, &(path->cdata), 0, 0, 0, NULL,
+                                                             NULL, H5CX_get_dxpl());
+                }
+            H5_AFTER_USER_CB(FAIL)
         }
         else
             status = (H5T_g.soft[i].conv.u.lib_func)(path->src, path->dst, &(path->cdata), conv_ctx, 0, 0, 0,
@@ -6010,9 +6028,16 @@ H5T__path_free(H5T_path_t *path, H5T_conv_ctx_t *conv_ctx)
 
         path->cdata.command = H5T_CONV_FREE;
 
-        if (path->conv.is_app)
-            status = (path->conv.u.app_func)(conv_ctx->u.free.src_type_id, conv_ctx->u.free.dst_type_id,
-                                             &(path->cdata), 0, 0, 0, NULL, NULL, H5CX_get_dxpl());
+        if (path->conv.is_app) {
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB_NOERR(FAIL)
+                {
+                    status =
+                        (path->conv.u.app_func)(conv_ctx->u.free.src_type_id, conv_ctx->u.free.dst_type_id,
+                                                &(path->cdata), 0, 0, 0, NULL, NULL, H5CX_get_dxpl());
+                }
+            H5_AFTER_USER_CB_NOERR(FAIL)
+        }
         else
             status =
                 (path->conv.u.lib_func)(path->src, path->dst, &(path->cdata), conv_ctx, 0, 0, 0, NULL, NULL);
@@ -6426,9 +6451,15 @@ H5T_convert_with_ctx(H5T_path_t *tpath, const H5T_t *src_type, const H5T_t *dst_
     /* Call the appropriate conversion callback */
     tpath->cdata.command = H5T_CONV_CONV;
     if (tpath->conv.is_app) {
-        if ((tpath->conv.u.app_func)(conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id,
-                                     &(tpath->cdata), nelmts, buf_stride, bkg_stride, buf, bkg,
-                                     conv_ctx->u.conv.dxpl_id) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (tpath->conv.u.app_func)(
+                    conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, &(tpath->cdata), nelmts,
+                    buf_stride, bkg_stride, buf, bkg, conv_ctx->u.conv.dxpl_id);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "datatype conversion failed");
     } /* end if */
     else if ((tpath->conv.u.lib_func)(src_type, dst_type, &(tpath->cdata), conv_ctx, nelmts, buf_stride,
diff --git a/src/H5TS.c b/src/H5TS.c
index 2b025724cc4..469d492acf3 100644
--- a/src/H5TS.c
+++ b/src/H5TS.c
@@ -35,7 +35,7 @@
 #include "H5Eprivate.h" /* Error handling                      */
 #include "H5TSpkg.h"    /* Threadsafety                        */
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 
 /****************/
 /* Local Macros */
@@ -147,4 +147,4 @@ H5TSmutex_release(unsigned *lock_count)
 
     FUNC_LEAVE_API_NAMECHECK_ONLY(ret_value)
 } /* end H5TSmutex_release() */
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
diff --git a/src/H5TSc11.c b/src/H5TSc11.c
index 016a066c1dd..6ea9caefb6f 100644
--- a/src/H5TSc11.c
+++ b/src/H5TSc11.c
@@ -58,7 +58,7 @@
 /* Local Variables */
 /*******************/
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 /*--------------------------------------------------------------------------
  * Function:    H5TS__c11_first_thread_init
  *
@@ -76,10 +76,10 @@ H5TS__c11_first_thread_init(void)
     FUNC_ENTER_PACKAGE_NAMECHECK_ONLY
 
     /* Initialize H5TS package */
-    H5TS__init();
+    H5TS__init_package();
 
     FUNC_LEAVE_NOAPI_VOID_NAMECHECK_ONLY
 } /* end H5TS__c11_first_thread_init() */
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
 #endif /* H5_HAVE_C11_THREADS */
diff --git a/src/H5TSint.c b/src/H5TSint.c
index 503e21cb1e8..7edc46f5ada 100644
--- a/src/H5TSint.c
+++ b/src/H5TSint.c
@@ -37,7 +37,7 @@
 #include "H5Epkg.h"      /* Error handling                      */
 #include "H5TSpkg.h"     /* Threadsafety                        */
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 
 /****************/
 /* Local Macros */
@@ -62,6 +62,9 @@ typedef struct H5TS_thread_info_t {
     uint64_t            id;               /* Unique ID for each thread */
     struct H5CX_node_t *api_ctx_node_ptr; /* Pointer to an API context node */
     H5E_stack_t         err_stack;        /* Error stack */
+#ifdef H5_HAVE_CONCURRENCY
+    unsigned dlftt; /* Whether locking is disabled for this thread */
+#endif              /* H5_HAVE_CONCURRENCY */
 } H5TS_thread_info_t;
 
 /* An H5TS_tinfo_node_t is a thread info that is available for reuse */
@@ -74,6 +77,12 @@ typedef struct H5TS_tinfo_node_t {
 /* Local Prototypes */
 /********************/
 static H5TS_tinfo_node_t *H5TS__tinfo_create(void);
+#ifdef H5_HAVE_CONCURRENCY
+static herr_t H5TS__get_dlftt(unsigned *dlftt);
+static herr_t H5TS__set_dlftt(unsigned dlftt);
+static herr_t H5TS__inc_dlftt(void);
+static herr_t H5TS__dec_dlftt(void);
+#endif /* H5_HAVE_CONCURRENCY */
 
 /*********************/
 /* Package Variables */
@@ -103,26 +112,33 @@ static uint64_t           H5TS_next_thrd_id_s    = 0;
 /* Mutex for access to H5TS_tinfo_next_free_s and H5TS_next_thrd_id_s */
 static H5TS_mutex_t H5TS_tinfo_mtx_s;
 
-/*-------------------------------------------------------------------------
- * Function: H5TS__init
- *
- * Purpose:  Initialize the H5TS interface
- *
- * Return:   Non-negative on success / Negative on failure
- *
- *-------------------------------------------------------------------------
- */
+/*--------------------------------------------------------------------------
+NAME
+   H5TS__init_package -- Initialize interface-specific information
+USAGE
+    herr_t H5TS__init_package()
+RETURNS
+    Non-negative on success/Negative on failure
+DESCRIPTION
+    Initializes any interface-specific data or routines.
+
+--------------------------------------------------------------------------*/
 herr_t
-H5TS__init(void)
+H5TS__init_package(void)
 {
     herr_t ret_value = SUCCEED; /* Return value */
 
-    FUNC_ENTER_PACKAGE_NAMECHECK_ONLY
+    FUNC_ENTER_PACKAGE_NOERR
 
     /* Initialize the global API lock info */
+#ifdef H5_HAVE_THREADSAFE
     if (H5_UNLIKELY(H5TS_mutex_init(&H5TS_api_info_p.api_mutex, H5TS_MUTEX_TYPE_RECURSIVE) < 0))
         HGOTO_DONE(FAIL);
     H5TS_api_info_p.lock_count = 0;
+#else /* H5_HAVE_CONCURRENCY */
+    if (H5_UNLIKELY(H5TS_rwlock_init(&H5TS_api_info_p.api_lock) < 0))
+        HGOTO_DONE(FAIL);
+#endif
     H5TS_atomic_init_uint(&H5TS_api_info_p.attempt_lock_count, 0);
 
     /* Initialize per-thread library info */
@@ -130,8 +146,8 @@ H5TS__init(void)
         HGOTO_DONE(FAIL);
 
 done:
-    FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
-} /* end H5TS__init() */
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5TS__init_package() */
 
 /*-------------------------------------------------------------------------
  * Function: H5TS_term_package
@@ -153,12 +169,66 @@ H5TS_term_package(void)
     FUNC_ENTER_NOAPI_NOINIT_NOERR
 
     /* Reset global API lock info */
+#ifdef H5_HAVE_THREADSAFE
     H5TS_mutex_destroy(&H5TS_api_info_p.api_mutex);
+#else /* H5_HAVE_CONCURRENCY */
+    H5TS_rwlock_destroy(&H5TS_api_info_p.api_lock);
+#endif
     H5TS_atomic_destroy_uint(&H5TS_api_info_p.attempt_lock_count);
 
     FUNC_LEAVE_NOAPI_VOID
 } /* end H5TS_term_package() */
 
+#ifdef H5_HAVE_CONCURRENCY
+/*-------------------------------------------------------------------------
+ * Function:    H5TS_user_cb_prepare
+ *
+ * Purpose:     Prepare the H5E package before a user callback
+ *
+ * Return:      SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5TS_user_cb_prepare(void)
+{
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Increment the 'disable locking for this thread' (DLFTT) value */
+    if (H5TS__inc_dlftt() < 0)
+        HGOTO_ERROR(H5E_LIB, H5E_CANTINC, FAIL, "unable to increment DLFTT value");
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5TS_user_cb_prepare() */
+
+/*-------------------------------------------------------------------------
+ * Function:    H5TS_user_cb_restore
+ *
+ * Purpose:     Restores the state of the H5TS package after a user callback
+ *
+ * Return:      SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5TS_user_cb_restore(void)
+{
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Decrement the 'disable locking for this thread' (DLFTT) value */
+    if (H5TS__dec_dlftt() < 0)
+        HGOTO_ERROR(H5E_LIB, H5E_CANTDEC, FAIL, "unable to decrement DLFTT value");
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5TS_user_cb_restore() */
+#endif /* H5_HAVE_CONCURRENCY */
+
 /*--------------------------------------------------------------------------
  * Function:    H5TS__api_mutex_acquire
  *
@@ -174,21 +244,45 @@ H5TS_term_package(void)
 herr_t
 H5TS__api_mutex_acquire(unsigned lock_count, bool *acquired)
 {
+#ifdef H5_HAVE_CONCURRENCY
+    unsigned dlftt = 0;
+#endif
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_PACKAGE_NAMECHECK_ONLY
 
+#ifdef H5_HAVE_THREADSAFE
     /* Attempt to acquire the lock */
     if (H5_UNLIKELY(H5TS_mutex_trylock(&H5TS_api_info_p.api_mutex, acquired) < 0))
         HGOTO_DONE(FAIL);
 
-    /* If acquired, increment the levels of recursion by 'lock_count' - 1 */
+    /* If acquired, acquire the mutex ('lock_count' - 1) more times */
     if (*acquired) {
         for (unsigned u = 0; u < (lock_count - 1); u++)
             if (H5_UNLIKELY(H5TS_mutex_lock(&H5TS_api_info_p.api_mutex) < 0))
                 HGOTO_DONE(FAIL);
         H5TS_api_info_p.lock_count += lock_count;
     }
+#else /* H5_HAVE_CONCURRENCY */
+    /* Query the DLFTT value */
+    if (H5_UNLIKELY(H5TS__get_dlftt(&dlftt) < 0))
+        HGOTO_DONE(FAIL);
+
+    /* Check if we haven't acquired the lock */
+    if (0 == dlftt) {
+        /* Attempt to acquire the lock */
+        if (H5_UNLIKELY(H5TS_rwlock_trywrlock(&H5TS_api_info_p.api_lock, acquired) < 0))
+            HGOTO_DONE(FAIL);
+    }
+    else
+        *acquired = true;
+
+    /* If acquired, increment the DLFTT count by 'lock_count' */
+    if (*acquired)
+        /* Set the DLFTT value */
+        if (H5_UNLIKELY(H5TS__set_dlftt(dlftt + lock_count) < 0))
+            HGOTO_DONE(FAIL);
+#endif
 
 done:
     FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
@@ -209,6 +303,7 @@ H5TS__api_mutex_acquire(unsigned lock_count, bool *acquired)
  *
  *--------------------------------------------------------------------------
  */
+#ifdef H5_HAVE_THREADSAFE
 herr_t
 H5TS_api_lock(void)
 {
@@ -234,6 +329,40 @@ H5TS_api_lock(void)
 done:
     FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
 } /* end H5TS_api_lock() */
+#else
+#ifdef H5_HAVE_CONCURRENCY
+herr_t
+H5TS_api_lock(unsigned *dlftt)
+{
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI_NAMECHECK_ONLY
+
+    /* Initialize the thread-safety code, once */
+    if (H5_UNLIKELY(!H5_INIT_GLOBAL))
+        if (H5_UNLIKELY(H5TS_once(&H5TS_first_init_s, H5TS_ONCE_INIT_FUNC) < 0))
+            HGOTO_DONE(FAIL);
+
+    /* Increment the attempt lock count */
+    H5TS_atomic_fetch_add_uint(&H5TS_api_info_p.attempt_lock_count, 1);
+
+    /* Query the DLFTT value */
+    if (H5_UNLIKELY(H5TS__get_dlftt(dlftt) < 0))
+        HGOTO_DONE(FAIL);
+
+    /* Don't acquire the API lock if locking is disabled */
+    if (0 == *dlftt)
+        /* Acquire the library's API lock */
+        if (H5_UNLIKELY(H5TS_rwlock_wrlock(&H5TS_api_info_p.api_lock) < 0))
+            HGOTO_DONE(FAIL);
+
+done:
+    FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
+} /* end H5TS_api_lock() */
+#else
+#error "Unknown multithreading mode"
+#endif
+#endif
 
 /*--------------------------------------------------------------------------
  * Function:    H5TS__api_mutex_release
@@ -253,6 +382,7 @@ H5TS__api_mutex_release(unsigned *lock_count)
 
     FUNC_ENTER_PACKAGE_NAMECHECK_ONLY
 
+#ifdef H5_HAVE_THREADSAFE
     /* Return the current lock count */
     *lock_count = H5TS_api_info_p.lock_count;
 
@@ -263,6 +393,19 @@ H5TS__api_mutex_release(unsigned *lock_count)
     for (unsigned u = 0; u < *lock_count; u++)
         if (H5_UNLIKELY(H5TS_mutex_unlock(&H5TS_api_info_p.api_mutex) < 0))
             HGOTO_DONE(FAIL);
+#else /* H5_HAVE_CONCURRENCY */
+    /* Query the DLFTT value */
+    if (H5_UNLIKELY(H5TS__get_dlftt(lock_count) < 0))
+        HGOTO_DONE(FAIL);
+
+    /* Reset the DLFTT value */
+    if (H5_UNLIKELY(H5TS__set_dlftt(0) < 0))
+        HGOTO_DONE(FAIL);
+
+    /* Release the library's API lock */
+    if (H5_UNLIKELY(H5TS_rwlock_wrunlock(&H5TS_api_info_p.api_lock) < 0))
+        HGOTO_DONE(FAIL);
+#endif
 
 done:
     FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
@@ -286,12 +429,18 @@ H5TS_api_unlock(void)
 
     FUNC_ENTER_NOAPI_NAMECHECK_ONLY
 
+#ifdef H5_HAVE_THREADSAFE
     /* Decrement the lock count for this thread */
     H5TS_api_info_p.lock_count--;
 
     /* Release the library's API lock */
     if (H5_UNLIKELY(H5TS_mutex_unlock(&H5TS_api_info_p.api_mutex) < 0))
         HGOTO_DONE(FAIL);
+#else /* H5_HAVE_CONCURRENCY */
+    /* Release the library's API lock */
+    if (H5_UNLIKELY(H5TS_rwlock_wrunlock(&H5TS_api_info_p.api_lock) < 0))
+        HGOTO_DONE(FAIL);
+#endif
 
 done:
     FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
@@ -506,6 +655,136 @@ H5TS_get_err_stack(void)
     FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
 } /* H5TS_get_err_stack() */
 
+#ifdef H5_HAVE_CONCURRENCY
+/*--------------------------------------------------------------------------
+ * Function:    H5TS__get_dlftt
+ *
+ * Purpose:     Retrieve the DLFTT value for this thread.
+ *
+ * Return:      Non-negative on success / Negative on failure
+ *
+ *--------------------------------------------------------------------------
+ */
+static herr_t
+H5TS__get_dlftt(unsigned *dlftt)
+{
+    H5TS_tinfo_node_t *tinfo_node;
+    herr_t             ret_value = SUCCEED;
+
+    FUNC_ENTER_PACKAGE_NAMECHECK_ONLY
+
+    /* Check if info for thread has been created */
+    if (H5_UNLIKELY(H5TS_key_get_value(H5TS_thrd_info_key_g, (void **)&tinfo_node) < 0))
+        HGOTO_DONE(FAIL);
+    if (H5_UNLIKELY(NULL == tinfo_node))
+        /* Create thread info for this thread */
+        if (H5_UNLIKELY(NULL == (tinfo_node = H5TS__tinfo_create())))
+            HGOTO_DONE(FAIL);
+
+    /* Get value */
+    *dlftt = tinfo_node->info.dlftt;
+
+done:
+    FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
+} /* H5TS__get_dlftt() */
+
+/*--------------------------------------------------------------------------
+ * Function:    H5TS__set_dlftt
+ *
+ * Purpose:     Set the DLFTT value for this thread.
+ *
+ * Return:      Non-negative on success / Negative on failure
+ *
+ *--------------------------------------------------------------------------
+ */
+static herr_t
+H5TS__set_dlftt(unsigned dlftt)
+{
+    H5TS_tinfo_node_t *tinfo_node;
+    herr_t             ret_value = SUCCEED;
+
+    FUNC_ENTER_PACKAGE_NAMECHECK_ONLY
+
+    /* Check if info for thread has been created */
+    if (H5_UNLIKELY(H5TS_key_get_value(H5TS_thrd_info_key_g, (void **)&tinfo_node) < 0))
+        HGOTO_DONE(FAIL);
+    if (H5_UNLIKELY(NULL == tinfo_node))
+        /* Create thread info for this thread */
+        if (H5_UNLIKELY(NULL == (tinfo_node = H5TS__tinfo_create())))
+            HGOTO_DONE(FAIL);
+
+    /* Set value */
+    tinfo_node->info.dlftt = dlftt;
+
+done:
+    FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
+} /* H5TS__set_dlftt() */
+
+/*--------------------------------------------------------------------------
+ * Function:    H5TS__inc_dlftt
+ *
+ * Purpose:     Increment the DLFTT value for this thread.
+ *
+ * Return:      Non-negative on success / Negative on failure
+ *
+ *--------------------------------------------------------------------------
+ */
+static herr_t
+H5TS__inc_dlftt(void)
+{
+    H5TS_tinfo_node_t *tinfo_node;
+    herr_t             ret_value = SUCCEED;
+
+    FUNC_ENTER_PACKAGE_NAMECHECK_ONLY
+
+    /* Check if info for thread has been created */
+    if (H5_UNLIKELY(H5TS_key_get_value(H5TS_thrd_info_key_g, (void **)&tinfo_node) < 0))
+        HGOTO_DONE(FAIL);
+    if (H5_UNLIKELY(NULL == tinfo_node))
+        /* Create thread info for this thread */
+        if (H5_UNLIKELY(NULL == (tinfo_node = H5TS__tinfo_create())))
+            HGOTO_DONE(FAIL);
+
+    /* Increment value */
+    tinfo_node->info.dlftt++;
+
+done:
+    FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
+} /* H5TS__inc_dlftt() */
+
+/*--------------------------------------------------------------------------
+ * Function:    H5TS__dec_dlftt
+ *
+ * Purpose:     Decrement the DLFTT value for this thread.
+ *
+ * Return:      Non-negative on success / Negative on failure
+ *
+ *--------------------------------------------------------------------------
+ */
+static herr_t
+H5TS__dec_dlftt(void)
+{
+    H5TS_tinfo_node_t *tinfo_node;
+    herr_t             ret_value = SUCCEED;
+
+    FUNC_ENTER_PACKAGE_NAMECHECK_ONLY
+
+    /* Check if info for thread has been created */
+    if (H5_UNLIKELY(H5TS_key_get_value(H5TS_thrd_info_key_g, (void **)&tinfo_node) < 0))
+        HGOTO_DONE(FAIL);
+    if (H5_UNLIKELY(NULL == tinfo_node))
+        /* Create thread info for this thread */
+        if (H5_UNLIKELY(NULL == (tinfo_node = H5TS__tinfo_create())))
+            HGOTO_DONE(FAIL);
+
+    /* Decrement value */
+    tinfo_node->info.dlftt--;
+
+done:
+    FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
+} /* H5TS__dec_dlftt() */
+#endif /* H5_HAVE_CONCURRENCY */
+
 /*--------------------------------------------------------------------------
  * Function:    H5TS__tinfo_destroy
  *
@@ -528,11 +807,14 @@ H5TS__tinfo_destroy(void *_tinfo_node)
 
     if (tinfo_node) {
         H5TS_mutex_lock(&H5TS_tinfo_mtx_s);
+
         /* Add thread info node to the free list */
         tinfo_node->next       = H5TS_tinfo_next_free_s;
         H5TS_tinfo_next_free_s = tinfo_node;
+
         /* Release resources held by error records in thread-local error stack */
         H5E__destroy_stack(&tinfo_node->info.err_stack);
+
         H5TS_mutex_unlock(&H5TS_tinfo_mtx_s);
     }
 
@@ -590,15 +872,16 @@ H5TS__tinfo_term(void)
     if (H5_UNLIKELY(H5TS_mutex_unlock(&H5TS_tinfo_mtx_s) < 0))
         HGOTO_DONE(FAIL);
 
+    /* Release critical section / mutex for modifying the thread info globals */
+    if (H5_UNLIKELY(H5TS_mutex_destroy(&H5TS_tinfo_mtx_s) < 0))
+        HGOTO_DONE(FAIL);
+
     /* Release key for thread-specific API contexts */
     if (H5_UNLIKELY(H5TS_key_delete(H5TS_thrd_info_key_g) < 0))
         HGOTO_DONE(FAIL);
 
-    /* Release critical section / mutex for modifying the thread info globals */
-    if (H5_UNLIKELY(H5TS_mutex_destroy(&H5TS_tinfo_mtx_s) < 0))
-        HGOTO_DONE(FAIL);
 done:
     FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
 } /* end H5TS__tinfo_term() */
 
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
diff --git a/src/H5TSmodule.h b/src/H5TSmodule.h
index 71d0de45349..67648d0fdc3 100644
--- a/src/H5TSmodule.h
+++ b/src/H5TSmodule.h
@@ -24,6 +24,6 @@
 #define H5TS_MODULE
 #define H5_MY_PKG      H5TS
 #define H5_MY_PKG_ERR  H5E_THREADSAFE
-#define H5_MY_PKG_INIT YES
+#define H5_MY_PKG_INIT NO
 
 #endif /* H5TSmodule_H */
diff --git a/src/H5TSpkg.h b/src/H5TSpkg.h
index 47ab8cb7720..0b5ab28558c 100644
--- a/src/H5TSpkg.h
+++ b/src/H5TSpkg.h
@@ -43,19 +43,24 @@
 /* Package Private Typedefs */
 /****************************/
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 /* Info for the global API lock */
 typedef struct H5TS_api_info_t {
+#ifdef H5_HAVE_THREADSAFE
     /* API lock */
     H5TS_mutex_t api_mutex;
 
     /* Count of recursive API calls by the same thread */
     unsigned lock_count;
+#else /* H5_HAVE_CONCURRENCY */
+    /* API lock */
+    H5TS_rwlock_t api_lock;
+#endif
 
     /* Count of # of attempts to acquire API lock */
     H5TS_atomic_uint_t attempt_lock_count;
 } H5TS_api_info_t;
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
 #if H5TS_ENABLE_REC_RWLOCK_STATS
 /******************************************************************************
@@ -214,25 +219,25 @@ typedef struct H5TS_rec_rwlock_t {
 /* Package Private Variables */
 /*****************************/
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 /* API threadsafety info */
 extern H5TS_api_info_t H5TS_api_info_p;
 
 /* Per-thread info */
 extern H5TS_key_t H5TS_thrd_info_key_g;
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
 /******************************/
 /* Package Private Prototypes */
 /******************************/
-#ifdef H5_HAVE_THREADSAFE
-H5_DLL herr_t H5TS__init(void);
+#ifdef H5_HAVE_THREADSAFE_API
+H5_DLL herr_t H5TS__init_package(void);
 H5_DLL herr_t H5TS__api_mutex_acquire(unsigned lock_count, bool *acquired);
 H5_DLL herr_t H5TS__api_mutex_release(unsigned *lock_count);
 H5_DLL herr_t H5TS__tinfo_init(void);
 H5_DLL void   H5TS__tinfo_destroy(void *tinfo_node);
 H5_DLL herr_t H5TS__tinfo_term(void);
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
 /* Recursive R/W lock related function declarations */
 H5_DLL herr_t H5TS__rec_rwlock_init(H5TS_rec_rwlock_t *lock);
@@ -243,7 +248,7 @@ H5_DLL herr_t H5TS__rec_rwlock_wrunlock(H5TS_rec_rwlock_t *lock);
 H5_DLL herr_t H5TS__rec_rwlock_destroy(H5TS_rec_rwlock_t *lock);
 
 /* 'once' callbacks */
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 #ifdef H5_HAVE_C11_THREADS
 H5_DLL void H5TS__c11_first_thread_init(void);
 #else
@@ -253,7 +258,7 @@ H5_DLL BOOL CALLBACK H5TS__win32_process_enter(PINIT_ONCE InitOnce, PVOID Parame
 H5_DLL void H5TS__pthread_first_thread_init(void);
 #endif /* H5_HAVE_WIN_THREADS */
 #endif
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
 #if H5TS_ENABLE_REC_RWLOCK_STATS
 H5_DLL herr_t H5TS__rec_rwlock_get_stats(H5TS_rec_rwlock_t *lock, H5TS_rec_rwlock_stats_t *stats);
diff --git a/src/H5TSprivate.h b/src/H5TSprivate.h
index 1e62a4ec9af..f36f9a02dde 100644
--- a/src/H5TSprivate.h
+++ b/src/H5TSprivate.h
@@ -23,10 +23,10 @@
 
 #ifdef H5_HAVE_THREADS
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 /* Include package's public headers */
 #include "H5TSdevelop.h"
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
 /**************************/
 /* Library Private Macros */
@@ -178,14 +178,40 @@ typedef CONDITION_VARIABLE     H5TS_cond_t;
 typedef INIT_ONCE              H5TS_once_t;
 typedef PINIT_ONCE_FN          H5TS_once_init_func_t;
 #else
+
+/* Non-recursive readers/writer lock */
+#if defined(__MACH__)
+/*
+ * Emulated pthread rwlock for MacOS
+ *
+ * Can't use pthread rwlock on MacOS due to: "The results [of calling
+ *      pthread_rwlock_wrlock] are undefined if the calling thread already
+ *      holds the lock at the time the call is made."
+ *  but the pthread standard says: "If a deadlock condition occurs or the
+ *      calling thread already owns the read-write lock for writing or reading,
+ *      the call shall either deadlock or return [EDEADLK]."
+ *
+ * The net result of this is that the current version of MacOS (v15.x) allows
+ * the same thread to recursively acquire a write lock, violating the pthread
+ * guarantee of deadlocking or failing.
+ *
+ */
+typedef struct H5TS_rwlock_t {
+    pthread_mutex_t mutex;
+    pthread_cond_t  read_cv, write_cv;
+    unsigned        readers, writers, read_waiters, write_waiters;
+} H5TS_rwlock_t;
+#else
+typedef pthread_rwlock_t H5TS_rwlock_t;
+#endif
+
 typedef pthread_t H5TS_thread_t;
 typedef void *(*H5TS_thread_start_func_t)(void *);
-typedef void            *H5TS_thread_ret_t;
-typedef pthread_key_t    H5TS_key_t;
-typedef pthread_mutex_t  H5TS_CAPABILITY("mutex") H5TS_mutex_t;
-typedef pthread_rwlock_t H5TS_rwlock_t;
-typedef pthread_cond_t   H5TS_cond_t;
-typedef pthread_once_t   H5TS_once_t;
+typedef void           *H5TS_thread_ret_t;
+typedef pthread_key_t   H5TS_key_t;
+typedef pthread_mutex_t H5TS_CAPABILITY("mutex") H5TS_mutex_t;
+typedef pthread_cond_t  H5TS_cond_t;
+typedef pthread_once_t  H5TS_once_t;
 typedef void (*H5TS_once_init_func_t)(void);
 #endif
 #endif
@@ -273,20 +299,30 @@ typedef atomic_flag H5TS_spinlock_t;
 /* Library-private Function Prototypes */
 /***************************************/
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 /* Library/thread init/term operations */
 H5_DLL void H5TS_term_package(void);
 H5_DLL int  H5TS_top_term_package(void);
 
+/* Prepare for / restore after user callback */
+#ifdef H5_HAVE_CONCURRENCY
+H5_DLL herr_t H5TS_user_cb_prepare(void);
+H5_DLL herr_t H5TS_user_cb_restore(void);
+#endif /* H5_HAVE_CONCURRENCY */
+
 /* API locking */
+#ifdef H5_HAVE_THREADSAFE
 H5_DLL herr_t H5TS_api_lock(void);
+#else /* H5_HAVE_CONCURRENCY */
+H5_DLL herr_t H5TS_api_lock(unsigned *dlftt);
+#endif
 H5_DLL herr_t H5TS_api_unlock(void);
 
 /* Retrieve per-thread info */
 H5_DLL herr_t               H5TS_thread_id(uint64_t *id);
 H5_DLL struct H5CX_node_t **H5TS_get_api_ctx_ptr(void);
 H5_DLL struct H5E_stack_t  *H5TS_get_err_stack(void);
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
 /* 'Once' operationss */
 H5_DLL herr_t H5TS_once(H5TS_once_t *once, H5TS_once_init_func_t func);
@@ -304,6 +340,8 @@ H5_DLL herr_t H5TS_rwlock_init(H5TS_rwlock_t *lock);
 static inline herr_t H5TS_rwlock_rdlock(H5TS_rwlock_t *lock);
 static inline herr_t H5TS_rwlock_rdunlock(H5TS_rwlock_t *lock);
 static inline herr_t H5TS_rwlock_wrlock(H5TS_rwlock_t *lock);
+static inline herr_t H5TS_rwlock_trywrlock(H5TS_rwlock_t *lock, bool *acquired)
+    H5TS_TRY_ACQUIRE(SUCCEED, *lock);
 static inline herr_t H5TS_rwlock_wrunlock(H5TS_rwlock_t *lock);
 #endif
 H5_DLL herr_t H5TS_rwlock_destroy(H5TS_rwlock_t *lock);
@@ -378,18 +416,18 @@ static inline herr_t H5TS_semaphore_wait(H5TS_semaphore_t *sem);
 H5_DLL herr_t H5TS_semaphore_destroy(H5TS_semaphore_t *sem);
 
 /* Headers with inlined routines */
+#ifndef __cplusplus
 #include "H5TScond.h"
 #include "H5TSmutex.h"
 #include "H5TSkey.h"
-#if !defined(__cplusplus)
-#if !defined(H5_HAVE_STDATOMIC_H)
+#ifndef H5_HAVE_STDATOMIC_H
 #include "H5TSatomic.h"
 #endif /* H5_HAVE_STDATOMIC_H */
 #include "H5TSbarrier.h"
 #include "H5TSrwlock.h"
 #include "H5TSsemaphore.h"
 #include "H5TSpool.h"
-#endif
+#endif /* __cplusplus */
 
 #endif /* H5_HAVE_THREADS */
 
diff --git a/src/H5TSpthread.c b/src/H5TSpthread.c
index 56715654694..f9de512218b 100644
--- a/src/H5TSpthread.c
+++ b/src/H5TSpthread.c
@@ -58,7 +58,7 @@
 /* Local Variables */
 /*******************/
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 /*--------------------------------------------------------------------------
  * Function:    H5TS__pthread_first_thread_init
  *
@@ -76,10 +76,10 @@ H5TS__pthread_first_thread_init(void)
     FUNC_ENTER_PACKAGE_NAMECHECK_ONLY
 
     /* Initialize H5TS package */
-    H5TS__init();
+    H5TS__init_package();
 
     FUNC_LEAVE_NOAPI_VOID_NAMECHECK_ONLY
 } /* end H5TS__pthread_first_thread_init() */
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
 #endif /* H5_HAVE_PTHREAD_H */
diff --git a/src/H5TSrwlock.c b/src/H5TSrwlock.c
index 70b7c27687f..8f9e73ca8df 100644
--- a/src/H5TSrwlock.c
+++ b/src/H5TSrwlock.c
@@ -178,6 +178,89 @@ H5TS_rwlock_destroy(H5TS_rwlock_t *lock)
     /* Destroy synchronization primitives */
     /* SRWLOCKs don't have to be destroyed */
 
+done:
+    FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
+} /* end H5TS_rwlock_destroy() */
+#elif defined(__MACH__)
+/*-------------------------------------------------------------------------
+ * Function: H5TS_rwlock_init
+ *
+ * Purpose:  Initialize a H5TS_rwlock_t (does not allocate it)
+ *
+ * Return:   Non-negative on success / Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5TS_rwlock_init(H5TS_rwlock_t *lock)
+{
+    pthread_mutexattr_t  _attr;
+    pthread_mutexattr_t *attr      = NULL;
+    herr_t               ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI_NAMECHECK_ONLY
+
+    /* Check argument */
+    if (H5_UNLIKELY(NULL == lock))
+        HGOTO_DONE(FAIL);
+
+    /* Create mutex attribute */
+    if (H5_UNLIKELY(pthread_mutexattr_init(&_attr)))
+        HGOTO_DONE(FAIL);
+    attr = &_attr;
+
+    /* Use "normal" mutex, without error checking */
+    if (H5_UNLIKELY(pthread_mutexattr_settype(attr, PTHREAD_MUTEX_NORMAL)))
+        HGOTO_DONE(FAIL);
+
+    /* Initialize the mutex */
+    if (H5_UNLIKELY(pthread_mutex_init(&lock->mutex, attr)))
+        HGOTO_DONE(FAIL);
+
+    /* Initialize the condition variables */
+    if (H5_UNLIKELY(pthread_cond_init(&lock->read_cv, NULL)))
+        HGOTO_DONE(FAIL);
+    if (H5_UNLIKELY(pthread_cond_init(&lock->write_cv, NULL)))
+        HGOTO_DONE(FAIL);
+
+    /* Initialize scalar fields */
+    lock->readers       = 0;
+    lock->writers       = 0;
+    lock->read_waiters  = 0;
+    lock->write_waiters = 0;
+
+done:
+    FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
+} /* end H5TS_rwlock_init() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5TS_rwlock_destroy
+ *
+ * Purpose:  Destroy a H5TS_rwlock_t (does not free it)
+ *
+ * Return:   Non-negative on success / Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5TS_rwlock_destroy(H5TS_rwlock_t *lock)
+{
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI_NAMECHECK_ONLY
+
+    /* Check argument */
+    if (H5_UNLIKELY(NULL == lock))
+        HGOTO_DONE(FAIL);
+
+    /* Destroy synchronization primitives */
+    if (H5_UNLIKELY(pthread_mutex_destroy(&lock->mutex)))
+        HGOTO_DONE(FAIL);
+    if (H5_UNLIKELY(pthread_cond_destroy(&lock->read_cv)))
+        HGOTO_DONE(FAIL);
+    if (H5_UNLIKELY(pthread_cond_destroy(&lock->write_cv)))
+        HGOTO_DONE(FAIL);
+
 done:
     FUNC_LEAVE_NOAPI_NAMECHECK_ONLY(ret_value)
 } /* end H5TS_rwlock_destroy() */
diff --git a/src/H5TSrwlock.h b/src/H5TSrwlock.h
index 951f4569b3d..c05fb5b7e11 100644
--- a/src/H5TSrwlock.h
+++ b/src/H5TSrwlock.h
@@ -184,6 +184,52 @@ H5TS_rwlock_wrlock(H5TS_rwlock_t *lock)
     return SUCCEED;
 } /* end H5TS_rwlock_wrlock() */
 
+/*-------------------------------------------------------------------------
+ * Function: H5TS_rwlock_trywrlock
+ *
+ * Purpose:  Attempt to acquire a write lock
+ *
+ * Return:   Non-negative on success / Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static inline herr_t
+H5TS_rwlock_trywrlock(H5TS_rwlock_t *lock, bool *acquired)
+{
+    int ret;
+
+    /* Check argument */
+    if (H5_UNLIKELY(NULL == lock || NULL == acquired))
+        return FAIL;
+
+    /* Acquire the lock's mutex */
+    if (H5_UNLIKELY(thrd_error == (ret = mtx_lock(&lock->mutex))))
+        return FAIL;
+    if (thrd_busy == ret) {
+        /* We did not acquire the lock */
+        *acquired = false;
+        return SUCCEED;
+    }
+
+    /* Check for readers or other writers */
+    if (lock->readers || lock->writers)
+        /* We did not acquire the lock */
+        *acquired = false;
+    else {
+        /* Increment # of writers */
+        lock->writers++;
+
+        /* We acquired the lock */
+        *acquired = true;
+    }
+
+    /* Release mutex */
+    if (H5_UNLIKELY(mtx_unlock(&lock->mutex) != thrd_success))
+        return FAIL;
+
+    return SUCCEED;
+} /* end H5TS_rwlock_trywrlock() */
+
 /*-------------------------------------------------------------------------
  * Function: H5TS_rwlock_wrunlock
  *
@@ -292,6 +338,30 @@ H5TS_rwlock_wrlock(H5TS_rwlock_t *lock)
     return SUCCEED;
 } /* end H5TS_rwlock_wrlock() */
 
+/*-------------------------------------------------------------------------
+ * Function: H5TS_rwlock_trywrlock
+ *
+ * Purpose:  Attempt to acquire a write lock
+ *
+ * Return:   Non-negative on success / Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static inline herr_t
+H5TS_rwlock_trywrlock(H5TS_rwlock_t *lock, bool *acquired)
+{
+    /* Check argument */
+    if (H5_UNLIKELY(NULL == lock || NULL == acquired))
+        return FAIL;
+
+    if (TryAcquireSRWLockExclusive(lock))
+        *acquired = true;
+    else
+        *acquired = false;
+
+    return SUCCEED;
+} /* end H5TS_rwlock_trywrlock() */
+
 /*-------------------------------------------------------------------------
  * Function: H5TS_rwlock_wrunlock
  *
@@ -313,6 +383,228 @@ H5TS_rwlock_wrunlock(H5TS_rwlock_t *lock)
     return SUCCEED;
 } /* end H5TS_rwlock_wrunlock() */
 
+#elif defined(__MACH__)
+/*-------------------------------------------------------------------------
+ * Function: H5TS_rwlock_rdlock
+ *
+ * Purpose:  Acquire a read lock
+ *
+ * Return:   Non-negative on success / Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static inline herr_t
+H5TS_rwlock_rdlock(H5TS_rwlock_t *lock)
+{
+    /* Check argument */
+    if (H5_UNLIKELY(NULL == lock))
+        return FAIL;
+
+    /* Acquire the lock's mutex */
+    if (H5_UNLIKELY(pthread_mutex_lock(&lock->mutex)))
+        return FAIL;
+
+    /* Check for writers */
+    if (lock->writers || lock->write_waiters) {
+        /* Read waiting */
+        lock->read_waiters++;
+
+        /* Wait for writers */
+        do {
+            if (H5_UNLIKELY(pthread_cond_wait(&lock->read_cv, &lock->mutex))) {
+                pthread_mutex_unlock(&lock->mutex);
+                return FAIL;
+            }
+        } while (lock->writers || lock->write_waiters);
+
+        /* Read not waiting any longer */
+        lock->read_waiters--;
+    }
+
+    /* Increment # of readers */
+    lock->readers++;
+
+    /* Release mutex */
+    if (H5_UNLIKELY(pthread_mutex_unlock(&lock->mutex)))
+        return FAIL;
+
+    return SUCCEED;
+} /* end H5TS_rwlock_rdlock() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5TS_rwlock_rdunlock
+ *
+ * Purpose:  Release a read lock
+ *
+ * Return:   Non-negative on success / Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static inline herr_t
+H5TS_rwlock_rdunlock(H5TS_rwlock_t *lock)
+{
+    /* Check argument */
+    if (H5_UNLIKELY(NULL == lock))
+        return FAIL;
+
+    /* Acquire the lock's mutex */
+    if (H5_UNLIKELY(pthread_mutex_lock(&lock->mutex)))
+        return FAIL;
+
+    /* Decrement # of readers */
+    lock->readers--;
+
+    /* Check for waiting writers when last readers */
+    if (lock->write_waiters && 0 == lock->readers)
+        if (H5_UNLIKELY(pthread_cond_signal(&lock->write_cv))) {
+            pthread_mutex_unlock(&lock->mutex);
+            return FAIL;
+        }
+
+    /* Release mutex */
+    if (H5_UNLIKELY(pthread_mutex_unlock(&lock->mutex)))
+        return FAIL;
+
+    return SUCCEED;
+} /* end H5TS_rwlock_rdunlock() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5TS_rwlock_wrlock
+ *
+ * Purpose:  Acquire a write lock
+ *
+ * Return:   Non-negative on success / Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static inline herr_t
+H5TS_rwlock_wrlock(H5TS_rwlock_t *lock)
+{
+    /* Check argument */
+    if (H5_UNLIKELY(NULL == lock))
+        return FAIL;
+
+    /* Acquire the lock's mutex */
+    if (H5_UNLIKELY(pthread_mutex_lock(&lock->mutex)))
+        return FAIL;
+
+    /* Check for readers or other writers */
+    if (lock->readers || lock->writers) {
+        /* Write waiting */
+        lock->write_waiters++;
+
+        /* Wait for mutex */
+        do {
+            if (H5_UNLIKELY(pthread_cond_wait(&lock->write_cv, &lock->mutex))) {
+                pthread_mutex_unlock(&lock->mutex);
+                return FAIL;
+            }
+        } while (lock->readers || lock->writers);
+
+        /* Write not waiting any longer */
+        lock->write_waiters--;
+    }
+
+    /* Increment # of writers */
+    lock->writers++;
+
+    /* Release mutex */
+    if (H5_UNLIKELY(pthread_mutex_unlock(&lock->mutex)))
+        return FAIL;
+
+    return SUCCEED;
+} /* end H5TS_rwlock_wrlock() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5TS_rwlock_trywrlock
+ *
+ * Purpose:  Attempt to acquire a write lock
+ *
+ * Return:   Non-negative on success / Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static inline herr_t
+H5TS_rwlock_trywrlock(H5TS_rwlock_t *lock, bool *acquired)
+{
+    int rc;
+
+    /* Check argument */
+    if (H5_UNLIKELY(NULL == lock || NULL == acquired))
+        return FAIL;
+
+    /* Acquire the lock's mutex */
+    rc = pthread_mutex_trylock(&lock->mutex);
+    if (EBUSY == rc) {
+        /* We did not acquire the lock */
+        *acquired = false;
+        return SUCCEED;
+    }
+    else if (0 != rc)
+        return FAIL;
+
+    /* Check for readers or other writers */
+    if (lock->readers || lock->writers)
+        /* We did not acquire the lock */
+        *acquired = false;
+    else {
+        /* Increment # of writers */
+        lock->writers++;
+
+        /* We acquired the lock */
+        *acquired = true;
+    }
+
+    /* Release mutex */
+    if (H5_UNLIKELY(pthread_mutex_unlock(&lock->mutex)))
+        return FAIL;
+
+    return SUCCEED;
+} /* end H5TS_rwlock_trywrlock() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5TS_rwlock_wrunlock
+ *
+ * Purpose:  Release a write lock
+ *
+ * Return:   Non-negative on success / Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static inline herr_t
+H5TS_rwlock_wrunlock(H5TS_rwlock_t *lock)
+{
+    /* Check argument */
+    if (H5_UNLIKELY(NULL == lock))
+        return FAIL;
+
+    /* Acquire the lock's mutex */
+    if (H5_UNLIKELY(pthread_mutex_lock(&lock->mutex)))
+        return FAIL;
+
+    /* Decrement # of writers */
+    lock->writers--;
+
+    /* Check for waiting writers */
+    if (lock->write_waiters) {
+        if (H5_UNLIKELY(pthread_cond_signal(&lock->write_cv))) {
+            pthread_mutex_unlock(&lock->mutex);
+            return FAIL;
+        }
+    }
+    else if (lock->read_waiters)
+        if (H5_UNLIKELY(pthread_cond_broadcast(&lock->read_cv))) {
+            pthread_mutex_unlock(&lock->mutex);
+            return FAIL;
+        }
+
+    /* Release mutex */
+    if (H5_UNLIKELY(pthread_mutex_unlock(&lock->mutex)))
+        return FAIL;
+
+    return SUCCEED;
+} /* end H5TS_rwlock_wrunlock() */
+
 #else
 /*-------------------------------------------------------------------------
  * Function: H5TS_rwlock_rdlock
@@ -380,6 +672,35 @@ H5TS_rwlock_wrlock(H5TS_rwlock_t *lock)
     return SUCCEED;
 } /* end H5TS_rwlock_wrlock() */
 
+/*-------------------------------------------------------------------------
+ * Function: H5TS_rwlock_trywrlock
+ *
+ * Purpose:  Attempt to acquire a write lock
+ *
+ * Return:   Non-negative on success / Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5TS_rwlock_trywrlock(H5TS_rwlock_t *lock, bool *acquired)
+{
+    int ret;
+
+    /* Check argument */
+    if (H5_UNLIKELY(NULL == lock || NULL == acquired))
+        return FAIL;
+
+    ret = pthread_rwlock_trywrlock(lock);
+    if (EBUSY == ret)
+        *acquired = false; /* We did not acquire the lock */
+    else if (H5_UNLIKELY(0 != ret))
+        return FAIL;
+    else
+        *acquired = true; /* We acquired the lock */
+
+    return SUCCEED;
+} /* end H5TS_rwlock_trywrlock() */
+
 /*-------------------------------------------------------------------------
  * Function: H5TS_rwlock_rdunlock
  *
diff --git a/src/H5TSwin.c b/src/H5TSwin.c
index b82f364bdd2..59f5db42d57 100644
--- a/src/H5TSwin.c
+++ b/src/H5TSwin.c
@@ -64,7 +64,7 @@ static herr_t H5TS__win32_thread_exit(void);
 /* Local Variables */
 /*******************/
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 /*--------------------------------------------------------------------------
  * Function:    H5TS__win32_process_enter
  *
@@ -82,7 +82,7 @@ H5TS__win32_process_enter(PINIT_ONCE InitOnce, PVOID Parameter, PVOID *lpContex)
     FUNC_ENTER_PACKAGE_NAMECHECK_ONLY
 
     /* Initialize H5TS package */
-    if (H5_UNLIKELY(H5TS__init() < 0))
+    if (H5_UNLIKELY(H5TS__init_package() < 0))
         HGOTO_DONE(FALSE);
 
 done:
@@ -206,6 +206,6 @@ DllMain(_In_ HINSTANCE hinstDLL, _In_ DWORD fdwReason, _In_ LPVOID lpvReserved)
     return fOkay;
 }
 #endif /* H5_HAVE_WIN32_API && H5_BUILT_AS_DYNAMIC_LIB */
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
 #endif /* H5_HAVE_WIN_THREADS */
diff --git a/src/H5Tconv_bitfield.c b/src/H5Tconv_bitfield.c
index 8b46c75c5c5..cc2ad9eb483 100644
--- a/src/H5Tconv_bitfield.c
+++ b/src/H5Tconv_bitfield.c
@@ -164,10 +164,18 @@ H5T__conv_b_b(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T_
                 if (src->shared->u.atomic.prec > dst->shared->u.atomic.prec) {
                     /*overflow*/
                     if (conv_ctx->u.conv.cb_struct.func) { /*If user's exception handler is present, use it*/
-                        H5T__reverse_order(src_rev, s, src); /*reverse order first*/
-                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                            H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
-                            conv_ctx->u.conv.dst_type_id, src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                        /* Reverse order first */
+                        H5T__reverse_order(src_rev, s, src);
+
+                        /* Prepare & restore library for user callback */
+                        H5_BEFORE_USER_CB(FAIL)
+                            {
+                                except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                    H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
+                                    conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                    conv_ctx->u.conv.cb_struct.user_data);
+                            }
+                        H5_AFTER_USER_CB(FAIL)
                     } /* end if */
 
                     if (except_ret == H5T_CONV_UNHANDLED) {
diff --git a/src/H5Tconv_complex.c b/src/H5Tconv_complex.c
index cdd2402cb0f..9b4197c0f87 100644
--- a/src/H5Tconv_complex.c
+++ b/src/H5Tconv_complex.c
@@ -22,8 +22,9 @@
 /***********/
 /* Headers */
 /***********/
-#include "H5Eprivate.h"
-#include "H5Tconv.h"
+#include "H5private.h"  /*  Generic Functions                    */
+#include "H5Eprivate.h" /* Error handling                       */
+#include "H5Tconv.h"    /* Datatype conversions                 */
 #include "H5Tconv_macros.h"
 #include "H5Tconv_complex.h"
 #include "H5Tconv_integer.h"
@@ -280,7 +281,7 @@ H5T__conv_complex_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ct
             if (conv_ctx->u.conv.cb_struct.func) {
                 H5T_conv_except_t except_type; /* type of conversion exception that occurred */
 
-                /* reverse source buffer order first */
+                /* Reverse source buffer order first */
                 H5T__reverse_order(src_rev, s, src_p);
 
                 /*
@@ -308,9 +309,14 @@ H5T__conv_complex_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ct
                     except_type = H5T_CONV_EXCEPT_NAN;
                 }
 
-                except_ret = (conv_ctx->u.conv.cb_struct.func)(except_type, conv_ctx->u.conv.src_type_id,
-                                                               conv_ctx->u.conv.dst_type_id, src_rev, d,
-                                                               conv_ctx->u.conv.cb_struct.user_data);
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                            except_type, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, src_rev,
+                            d, conv_ctx->u.conv.cb_struct.user_data);
+                    }
+                H5_AFTER_USER_CB(FAIL)
             }
 
             if (except_ret == H5T_CONV_UNHANDLED) {
@@ -619,12 +625,17 @@ H5T__conv_complex_part(const H5T_t *src_p, const H5T_t *dst_p, uint8_t *s, uint8
          * original byte order.
          */
         if (conv_ctx->u.conv.cb_struct.func) { /* If user's exception handler is present, use it */
-            /* reverse source buffer order first */
+            /* Reverse source buffer order first */
             H5T__reverse_order(src_rev, s, src_p);
 
-            except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, src_rev,
-                d, conv_ctx->u.conv.cb_struct.user_data);
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id,
+                        src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                }
+            H5_AFTER_USER_CB(FAIL)
         }
 
         if (except_ret == H5T_CONV_UNHANDLED) {
@@ -711,12 +722,17 @@ H5T__conv_complex_part(const H5T_t *src_p, const H5T_t *dst_p, uint8_t *s, uint8
              * hand it is in the original byte order.
              */
             if (conv_ctx->u.conv.cb_struct.func) { /* If user's exception handler is present, use it */
-                /* reverse source buffer order first */
+                /* Reverse source buffer order first */
                 H5T__reverse_order(src_rev, s, src_p);
 
-                except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                    H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id,
-                    src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                            H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
+                            conv_ctx->u.conv.dst_type_id, src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                    }
+                H5_AFTER_USER_CB(FAIL)
             }
 
             if (except_ret == H5T_CONV_UNHANDLED) {
@@ -1075,7 +1091,7 @@ H5T__conv_complex_f_matched(const H5T_t *src_p, const H5T_t *dst_p, const H5T_co
             if (conv_ctx->u.conv.cb_struct.func) {
                 H5T_conv_except_t except_type; /* type of conversion exception that occurred */
 
-                /* reverse source buffer order first */
+                /* Reverse source buffer order first */
                 H5T__reverse_order(src_rev, s, src_p);
 
                 if (specval_type == H5T_CONV_FLOAT_SPECVAL_POSINF)
@@ -1085,9 +1101,14 @@ H5T__conv_complex_f_matched(const H5T_t *src_p, const H5T_t *dst_p, const H5T_co
                 else
                     except_type = H5T_CONV_EXCEPT_NAN;
 
-                except_ret = (conv_ctx->u.conv.cb_struct.func)(except_type, conv_ctx->u.conv.src_type_id,
-                                                               conv_ctx->u.conv.dst_type_id, src_rev, d,
-                                                               conv_ctx->u.conv.cb_struct.user_data);
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                            except_type, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, src_rev,
+                            d, conv_ctx->u.conv.cb_struct.user_data);
+                    }
+                H5_AFTER_USER_CB(FAIL)
             }
 
             if (except_ret == H5T_CONV_UNHANDLED) {
diff --git a/src/H5Tconv_enum.c b/src/H5Tconv_enum.c
index 744f7b9920c..6460f6a8291 100644
--- a/src/H5Tconv_enum.c
+++ b/src/H5Tconv_enum.c
@@ -401,11 +401,19 @@ H5T__conv_enum(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T
                     if (n < 0 || (unsigned)n >= priv->length || priv->src2dst[n] < 0) {
                         /*overflow*/
                         except_ret = H5T_CONV_UNHANDLED;
-                        /*If user's exception handler is present, use it*/
-                        if (conv_ctx->u.conv.cb_struct.func)
-                            except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
-                                conv_ctx->u.conv.dst_type_id, s, d, conv_ctx->u.conv.cb_struct.user_data);
+
+                        /* If user's exception handler is present, use it*/
+                        if (conv_ctx->u.conv.cb_struct.func) {
+                            /* Prepare & restore library for user callback */
+                            H5_BEFORE_USER_CB(FAIL)
+                                {
+                                    except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
+                                        conv_ctx->u.conv.dst_type_id, s, d,
+                                        conv_ctx->u.conv.cb_struct.user_data);
+                                }
+                            H5_AFTER_USER_CB(FAIL)
+                        }
 
                         if (except_ret == H5T_CONV_UNHANDLED)
                             memset(d, 0xff, dst_sh->size);
@@ -441,11 +449,19 @@ H5T__conv_enum(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T
                     } /* end while */
                     if (lt >= rt) {
                         except_ret = H5T_CONV_UNHANDLED;
-                        /*If user's exception handler is present, use it*/
-                        if (conv_ctx->u.conv.cb_struct.func)
-                            except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
-                                conv_ctx->u.conv.dst_type_id, s, d, conv_ctx->u.conv.cb_struct.user_data);
+
+                        /* If user's exception handler is present, use it*/
+                        if (conv_ctx->u.conv.cb_struct.func) {
+                            /* Prepare & restore library for user callback */
+                            H5_BEFORE_USER_CB(FAIL)
+                                {
+                                    except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
+                                        conv_ctx->u.conv.dst_type_id, s, d,
+                                        conv_ctx->u.conv.cb_struct.user_data);
+                                }
+                            H5_AFTER_USER_CB(FAIL)
+                        }
 
                         if (except_ret == H5T_CONV_UNHANDLED)
                             memset(d, 0xff, dst_sh->size);
diff --git a/src/H5Tconv_float.c b/src/H5Tconv_float.c
index b7231657281..e5767bb3f62 100644
--- a/src/H5Tconv_float.c
+++ b/src/H5Tconv_float.c
@@ -82,10 +82,8 @@ H5T__conv_f_f(const H5T_t *src_p, const H5T_t *dst_p, H5T_cdata_t *cdata, const
                 HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype");
             if (NULL == conv_ctx)
                 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid datatype conversion context pointer");
-
             if (H5T__conv_f_f_loop(src_p, dst_p, conv_ctx, nelmts, buf_stride, buf) < 0)
                 HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "unable to convert data values");
-
             break;
 
         default:
@@ -294,7 +292,7 @@ H5T__conv_f_f_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
             if (conv_ctx->u.conv.cb_struct.func) {
                 H5T_conv_except_t except_type; /* type of conversion exception that occurred */
 
-                /* reverse source buffer order first */
+                /* Reverse source buffer order first */
                 H5T__reverse_order(src_rev, s, src_p);
 
                 if (specval_type == H5T_CONV_FLOAT_SPECVAL_POSINF)
@@ -304,9 +302,14 @@ H5T__conv_f_f_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
                 else
                     except_type = H5T_CONV_EXCEPT_NAN;
 
-                except_ret = (conv_ctx->u.conv.cb_struct.func)(except_type, conv_ctx->u.conv.src_type_id,
-                                                               conv_ctx->u.conv.dst_type_id, src_rev, d,
-                                                               conv_ctx->u.conv.cb_struct.user_data);
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                            except_type, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, src_rev,
+                            d, conv_ctx->u.conv.cb_struct.user_data);
+                    }
+                H5_AFTER_USER_CB(FAIL)
             }
 
             if (except_ret == H5T_CONV_UNHANDLED) {
@@ -437,12 +440,17 @@ H5T__conv_f_f_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
              * original byte order.
              */
             if (conv_ctx->u.conv.cb_struct.func) { /* If user's exception handler is present, use it */
-                /* reverse source buffer order first */
+                /* Reverse source buffer order first */
                 H5T__reverse_order(src_rev, s, src_p);
 
-                except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                    H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id,
-                    src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                            H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
+                            conv_ctx->u.conv.dst_type_id, src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                    }
+                H5_AFTER_USER_CB(FAIL)
             }
 
             if (except_ret == H5T_CONV_UNHANDLED) {
@@ -529,12 +537,18 @@ H5T__conv_f_f_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
                  * hand it is in the original byte order.
                  */
                 if (conv_ctx->u.conv.cb_struct.func) { /* If user's exception handler is present, use it */
-                    /* reverse source buffer order first */
+                    /* Reverse source buffer order first */
                     H5T__reverse_order(src_rev, s, src_p);
 
-                    except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id,
-                        src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                    /* Prepare & restore library for user callback */
+                    H5_BEFORE_USER_CB(FAIL)
+                        {
+                            except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
+                                conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                conv_ctx->u.conv.cb_struct.user_data);
+                        }
+                    H5_AFTER_USER_CB(FAIL)
                 }
 
                 if (except_ret == H5T_CONV_UNHANDLED) {
@@ -762,7 +776,6 @@ H5T__conv_f_i(const H5T_t *src_p, const H5T_t *dst_p, H5T_cdata_t *cdata, const
 
             if (H5T__conv_f_i_loop(src_p, dst_p, conv_ctx, nelmts, buf_stride, buf) < 0)
                 HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "unable to convert data values");
-
             break;
 
         default:
@@ -962,7 +975,7 @@ H5T__conv_f_i_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
             if (conv_ctx->u.conv.cb_struct.func) {
                 H5T_conv_except_t except_type; /* type of conversion exception that occurred */
 
-                /* reverse source buffer order first */
+                /* Reverse source buffer order first */
                 H5T__reverse_order(src_rev, s, src_p);
 
                 if (specval_type == H5T_CONV_FLOAT_SPECVAL_POSINF)
@@ -972,9 +985,14 @@ H5T__conv_f_i_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
                 else
                     except_type = H5T_CONV_EXCEPT_NAN;
 
-                except_ret = (conv_ctx->u.conv.cb_struct.func)(except_type, conv_ctx->u.conv.src_type_id,
-                                                               conv_ctx->u.conv.dst_type_id, src_rev, d,
-                                                               conv_ctx->u.conv.cb_struct.user_data);
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                            except_type, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, src_rev,
+                            d, conv_ctx->u.conv.cb_struct.user_data);
+                    }
+                H5_AFTER_USER_CB(FAIL)
             }
 
             if (except_ret == H5T_CONV_UNHANDLED) {
@@ -1088,12 +1106,18 @@ H5T__conv_f_i_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
             if (sign) { /* source is negative */
                 /* If user's exception handler is present, use it */
                 if (conv_ctx->u.conv.cb_struct.func) {
-                    /* reverse source buffer order first */
+                    /* Reverse source buffer order first */
                     H5T__reverse_order(src_rev, s, src_p);
 
-                    except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                        H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id,
-                        src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                    /* Prepare & restore library for user callback */
+                    H5_BEFORE_USER_CB(FAIL)
+                        {
+                            except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,
+                                conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                conv_ctx->u.conv.cb_struct.user_data);
+                        }
+                    H5_AFTER_USER_CB(FAIL)
 
                     if (except_ret == H5T_CONV_HANDLED) {
                         /* No need to reverse the order of destination because user handles it */
@@ -1108,12 +1132,18 @@ H5T__conv_f_i_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
                 if (new_msb_pos >= (ssize_t)dst_atomic.prec) {
                     /* overflow - if user's exception handler is present, use it */
                     if (conv_ctx->u.conv.cb_struct.func) {
-                        /* reverse source buffer order first */
+                        /* Reverse source buffer order first */
                         H5T__reverse_order(src_rev, s, src_p);
 
-                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                            H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
-                            conv_ctx->u.conv.dst_type_id, src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                        /* Prepare & restore library for user callback */
+                        H5_BEFORE_USER_CB(FAIL)
+                            {
+                                except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                    H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
+                                    conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                    conv_ctx->u.conv.cb_struct.user_data);
+                            }
+                        H5_AFTER_USER_CB(FAIL)
                     }
 
                     if (except_ret == H5T_CONV_UNHANDLED)
@@ -1129,12 +1159,18 @@ H5T__conv_f_i_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
                 else {
                     /* If user's exception handler is present, use it */
                     if (truncated && conv_ctx->u.conv.cb_struct.func) {
-                        /* reverse source buffer order first */
+                        /* Reverse source buffer order first */
                         H5T__reverse_order(src_rev, s, src_p);
 
-                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                            H5T_CONV_EXCEPT_TRUNCATE, conv_ctx->u.conv.src_type_id,
-                            conv_ctx->u.conv.dst_type_id, src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                        /* Prepare & restore library for user callback */
+                        H5_BEFORE_USER_CB(FAIL)
+                            {
+                                except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                    H5T_CONV_EXCEPT_TRUNCATE, conv_ctx->u.conv.src_type_id,
+                                    conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                    conv_ctx->u.conv.cb_struct.user_data);
+                            }
+                        H5_AFTER_USER_CB(FAIL)
                     }
 
                     if (except_ret == H5T_CONV_UNHANDLED) {
@@ -1157,12 +1193,18 @@ H5T__conv_f_i_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
                 if ((new_msb_pos >= 0) && ((size_t)new_msb_pos < dst_atomic.prec - 1)) {
                     /* If user's exception handler is present, use it */
                     if (truncated && conv_ctx->u.conv.cb_struct.func) {
-                        /* reverse source buffer order first */
+                        /* Reverse source buffer order first */
                         H5T__reverse_order(src_rev, s, src_p);
 
-                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                            H5T_CONV_EXCEPT_TRUNCATE, conv_ctx->u.conv.src_type_id,
-                            conv_ctx->u.conv.dst_type_id, src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                        /* Prepare & restore library for user callback */
+                        H5_BEFORE_USER_CB(FAIL)
+                            {
+                                except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                    H5T_CONV_EXCEPT_TRUNCATE, conv_ctx->u.conv.src_type_id,
+                                    conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                    conv_ctx->u.conv.cb_struct.user_data);
+                            }
+                        H5_AFTER_USER_CB(FAIL)
                     }
 
                     if (except_ret == H5T_CONV_UNHANDLED) { /* If this case ignored by user handler */
@@ -1188,12 +1230,18 @@ H5T__conv_f_i_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
                      * If user's exception handler is present, use it
                      */
                     if (conv_ctx->u.conv.cb_struct.func) {
-                        /* reverse source buffer order first */
+                        /* Reverse source buffer order first */
                         H5T__reverse_order(src_rev, s, src_p);
 
-                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                            H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,
-                            conv_ctx->u.conv.dst_type_id, src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                        /* Prepare & restore library for user callback */
+                        H5_BEFORE_USER_CB(FAIL)
+                            {
+                                except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                    H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,
+                                    conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                    conv_ctx->u.conv.cb_struct.user_data);
+                            }
+                        H5_AFTER_USER_CB(FAIL)
                     }
 
                     if (except_ret == H5T_CONV_UNHANDLED)
@@ -1211,12 +1259,18 @@ H5T__conv_f_i_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
                 if (new_msb_pos >= (ssize_t)dst_atomic.prec - 1) {
                     /* overflow - if user's exception handler is present, use it */
                     if (conv_ctx->u.conv.cb_struct.func) {
-                        /* reverse source buffer order first */
+                        /* Reverse source buffer order first */
                         H5T__reverse_order(src_rev, s, src_p);
 
-                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                            H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
-                            conv_ctx->u.conv.dst_type_id, src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                        /* Prepare & restore library for user callback */
+                        H5_BEFORE_USER_CB(FAIL)
+                            {
+                                except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                    H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
+                                    conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                    conv_ctx->u.conv.cb_struct.user_data);
+                            }
+                        H5_AFTER_USER_CB(FAIL)
                     }
 
                     if (except_ret == H5T_CONV_UNHANDLED)
@@ -1232,12 +1286,18 @@ H5T__conv_f_i_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
                 else if (new_msb_pos < (ssize_t)dst_atomic.prec - 1) {
                     /* If user's exception handler is present, use it */
                     if (truncated && conv_ctx->u.conv.cb_struct.func) {
-                        /* reverse source buffer order first */
+                        /* Reverse source buffer order first */
                         H5T__reverse_order(src_rev, s, src_p);
 
-                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                            H5T_CONV_EXCEPT_TRUNCATE, conv_ctx->u.conv.src_type_id,
-                            conv_ctx->u.conv.dst_type_id, src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                        /* Prepare & restore library for user callback */
+                        H5_BEFORE_USER_CB(FAIL)
+                            {
+                                except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                    H5T_CONV_EXCEPT_TRUNCATE, conv_ctx->u.conv.src_type_id,
+                                    conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                    conv_ctx->u.conv.cb_struct.user_data);
+                            }
+                        H5_AFTER_USER_CB(FAIL)
                     }
 
                     if (except_ret == H5T_CONV_UNHANDLED) {
diff --git a/src/H5Tconv_integer.c b/src/H5Tconv_integer.c
index b2af085c064..db61986c409 100644
--- a/src/H5Tconv_integer.c
+++ b/src/H5Tconv_integer.c
@@ -201,12 +201,18 @@ H5T__conv_i_i(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T_
                         /*overflow*/
                         if (conv_ctx->u.conv.cb_struct
                                 .func) { /*If user's exception handler is present, use it*/
-                            /* reverse source buffer order first */
+                            /* Reverse source buffer order first */
                             H5T__reverse_order(src_rev, s, src);
-                            except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
-                                conv_ctx->u.conv.dst_type_id, src_rev, d,
-                                conv_ctx->u.conv.cb_struct.user_data);
+
+                            /* Prepare & restore library for user callback */
+                            H5_BEFORE_USER_CB(FAIL)
+                                {
+                                    except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
+                                        conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                        conv_ctx->u.conv.cb_struct.user_data);
+                                }
+                            H5_AFTER_USER_CB(FAIL)
                         }
 
                         if (except_ret == H5T_CONV_UNHANDLED) {
@@ -237,12 +243,18 @@ H5T__conv_i_i(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T_
                         /*overflow - source is negative*/
                         if (conv_ctx->u.conv.cb_struct
                                 .func) { /*If user's exception handler is present, use it*/
-                            /* reverse source buffer order first */
+                            /* Reverse source buffer order first */
                             H5T__reverse_order(src_rev, s, src);
-                            except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                                H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,
-                                conv_ctx->u.conv.dst_type_id, src_rev, d,
-                                conv_ctx->u.conv.cb_struct.user_data);
+
+                            /* Prepare & restore library for user callback */
+                            H5_BEFORE_USER_CB(FAIL)
+                                {
+                                    except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                        H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,
+                                        conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                        conv_ctx->u.conv.cb_struct.user_data);
+                                }
+                            H5_AFTER_USER_CB(FAIL)
                         }
 
                         if (except_ret == H5T_CONV_UNHANDLED) {
@@ -265,12 +277,18 @@ H5T__conv_i_i(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T_
                         /*overflow - source is positive*/
                         if (conv_ctx->u.conv.cb_struct
                                 .func) { /*If user's exception handler is present, use it*/
-                            /* reverse source buffer order first */
+                            /* Reverse source buffer order first */
                             H5T__reverse_order(src_rev, s, src);
-                            except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
-                                conv_ctx->u.conv.dst_type_id, src_rev, d,
-                                conv_ctx->u.conv.cb_struct.user_data);
+
+                            /* Prepare & restore library for user callback */
+                            H5_BEFORE_USER_CB(FAIL)
+                                {
+                                    except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
+                                        conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                        conv_ctx->u.conv.cb_struct.user_data);
+                                }
+                            H5_AFTER_USER_CB(FAIL)
                         }
 
                         if (except_ret == H5T_CONV_UNHANDLED)
@@ -298,12 +316,18 @@ H5T__conv_i_i(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T_
                         /*overflow*/
                         if (conv_ctx->u.conv.cb_struct
                                 .func) { /*If user's exception handler is present, use it*/
-                            /* reverse source buffer order first */
+                            /* Reverse source buffer order first */
                             H5T__reverse_order(src_rev, s, src);
-                            except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
-                                conv_ctx->u.conv.dst_type_id, src_rev, d,
-                                conv_ctx->u.conv.cb_struct.user_data);
+
+                            /* Prepare & restore library for user callback */
+                            H5_BEFORE_USER_CB(FAIL)
+                                {
+                                    except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
+                                        conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                        conv_ctx->u.conv.cb_struct.user_data);
+                                }
+                            H5_AFTER_USER_CB(FAIL)
                         }
 
                         if (except_ret == H5T_CONV_UNHANDLED) {
@@ -346,12 +370,18 @@ H5T__conv_i_i(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T_
                         /*overflow*/
                         if (conv_ctx->u.conv.cb_struct
                                 .func) { /*If user's exception handler is present, use it*/
-                            /* reverse source buffer order first */
+                            /* Reverse source buffer order first */
                             H5T__reverse_order(src_rev, s, src);
-                            except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                                H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,
-                                conv_ctx->u.conv.dst_type_id, src_rev, d,
-                                conv_ctx->u.conv.cb_struct.user_data);
+
+                            /* Prepare & restore library for user callback */
+                            H5_BEFORE_USER_CB(FAIL)
+                                {
+                                    except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                        H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,
+                                        conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                        conv_ctx->u.conv.cb_struct.user_data);
+                                }
+                            H5_AFTER_USER_CB(FAIL)
                         }
 
                         if (except_ret == H5T_CONV_UNHANDLED) {
@@ -389,12 +419,18 @@ H5T__conv_i_i(const H5T_t *src, const H5T_t *dst, H5T_cdata_t *cdata, const H5T_
                         /*overflow*/
                         if (conv_ctx->u.conv.cb_struct
                                 .func) { /*If user's exception handler is present, use it*/
-                            /* reverse source buffer order first */
+                            /* Reverse source buffer order first */
                             H5T__reverse_order(src_rev, s, src);
-                            except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
-                                conv_ctx->u.conv.dst_type_id, src_rev, d,
-                                conv_ctx->u.conv.cb_struct.user_data);
+
+                            /* Prepare & restore library for user callback */
+                            H5_BEFORE_USER_CB(FAIL)
+                                {
+                                    except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
+                                        conv_ctx->u.conv.dst_type_id, src_rev, d,
+                                        conv_ctx->u.conv.cb_struct.user_data);
+                                }
+                            H5_AFTER_USER_CB(FAIL)
                         }
 
                         if (except_ret == H5T_CONV_UNHANDLED) {
@@ -687,11 +723,17 @@ H5T__conv_i_f_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
              * precision loss. Let user's handler deal with the case if it's present
              */
             if (conv_ctx->u.conv.cb_struct.func) {
-                /* reverse source buffer order first */
+                /* Reverse source buffer order first */
                 H5T__reverse_order(src_rev, s, src_p);
-                except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                    H5T_CONV_EXCEPT_PRECISION, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id,
-                    src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                            H5T_CONV_EXCEPT_PRECISION, conv_ctx->u.conv.src_type_id,
+                            conv_ctx->u.conv.dst_type_id, src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                    }
+                H5_AFTER_USER_CB(FAIL)
             }
 
             if (except_ret == H5T_CONV_HANDLED) {
@@ -757,13 +799,19 @@ H5T__conv_i_f_loop(const H5T_t *src_p, const H5T_t *dst_p, const H5T_conv_ctx_t
         /* Check if the exponent is too big */
         expo_max = (hsize_t)(pow(2.0, (double)dst_atomic.u.f.esize) - 1);
 
-        if (expo > expo_max) { /* overflows */
-            if (conv_ctx->u.conv.cb_struct.func) {
-                /* user's exception handler. Reverse back source order */
+        if (expo > expo_max) {                     /* overflows */
+            if (conv_ctx->u.conv.cb_struct.func) { /* user's exception handler */
+                /* Reverse back source order */
                 H5T__reverse_order(src_rev, s, src_p);
-                except_ret = (conv_ctx->u.conv.cb_struct.func)(
-                    H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id,
-                    src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+
+                /* Prepare & restore library for user callback */
+                H5_BEFORE_USER_CB(FAIL)
+                    {
+                        except_ret = (conv_ctx->u.conv.cb_struct.func)(
+                            H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,
+                            conv_ctx->u.conv.dst_type_id, src_rev, d, conv_ctx->u.conv.cb_struct.user_data);
+                    }
+                H5_AFTER_USER_CB(FAIL)
 
                 if (except_ret == H5T_CONV_ABORT)
                     HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "can't handle conversion exception");
diff --git a/src/H5Tconv_macros.h b/src/H5Tconv_macros.h
index 843255e51cc..80a9369fc70 100644
--- a/src/H5Tconv_macros.h
+++ b/src/H5Tconv_macros.h
@@ -239,10 +239,16 @@ typedef struct H5T_conv_hw_t {
  */
 #define H5T_CONV_Xx_CORE(STYPE, DTYPE, S, D, ST, DT, D_MIN, D_MAX)                                           \
     {                                                                                                        \
+        H5T_conv_ret_t except_ret;                                                                           \
         if (*(S) > (ST)(D_MAX)) {                                                                            \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D,  \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,                              \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = (DT)(D_MAX);                                                                          \
@@ -251,9 +257,14 @@ typedef struct H5T_conv_hw_t {
             /* if(except_ret==H5T_CONV_HANDLED): Fall through, user handled it */                            \
         }                                                                                                    \
         else if (*(S) < (ST)(D_MIN)) {                                                                       \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D, \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,                             \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = (DT)(D_MIN);                                                                          \
@@ -279,9 +290,16 @@ typedef struct H5T_conv_hw_t {
 #define H5T_CONV_Ux_CORE(STYPE, DTYPE, S, D, ST, DT, D_MIN, D_MAX)                                           \
     {                                                                                                        \
         if (*(S) > (ST)(D_MAX)) {                                                                            \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D,  \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            H5T_conv_ret_t except_ret;                                                                       \
+                                                                                                             \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,                              \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = (DT)(D_MAX);                                                                          \
@@ -310,9 +328,16 @@ typedef struct H5T_conv_hw_t {
 #define H5T_CONV_sU_CORE(STYPE, DTYPE, S, D, ST, DT, D_MIN, D_MAX)                                           \
     {                                                                                                        \
         if (*(S) < 0) {                                                                                      \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D, \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            H5T_conv_ret_t except_ret;                                                                       \
+                                                                                                             \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,                             \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = 0;                                                                                    \
@@ -372,9 +397,16 @@ typedef struct H5T_conv_hw_t {
 /* Called if overflow is possible */
 #define H5T_CONV_uS_CORE_1(S, D, ST, DT, D_MIN, D_MAX)                                                       \
     if (*(S) > (DT)(D_MAX)) {                                                                                \
-        H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                       \
-            H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D,      \
-            conv_ctx->u.conv.cb_struct.user_data);                                                           \
+        H5T_conv_ret_t except_ret;                                                                           \
+                                                                                                             \
+        /* Prepare & restore library for user callback */                                                    \
+        H5_BEFORE_USER_CB(FAIL)                                                                              \
+            {                                                                                                \
+                except_ret = (conv_ctx->u.conv.cb_struct.func)(                                              \
+                    H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, \
+                    D, conv_ctx->u.conv.cb_struct.user_data);                                                \
+            }                                                                                                \
+        H5_AFTER_USER_CB(FAIL)                                                                               \
         if (except_ret == H5T_CONV_UNHANDLED)                                                                \
             /* Let compiler convert if case is ignored by user handler */                                    \
             *(D) = (DT)(D_MAX);                                                                              \
@@ -434,10 +466,16 @@ typedef struct H5T_conv_hw_t {
 
 #define H5T_CONV_Su_CORE(STYPE, DTYPE, S, D, ST, DT, D_MIN, D_MAX)                                           \
     {                                                                                                        \
+        H5T_conv_ret_t except_ret;                                                                           \
         if (*(S) < 0) {                                                                                      \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D, \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,                             \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = 0;                                                                                    \
@@ -446,9 +484,14 @@ typedef struct H5T_conv_hw_t {
             /* if(except_ret==H5T_CONV_HANDLED): Fall through, user handled it */                            \
         }                                                                                                    \
         else if (sizeof(ST) > sizeof(DT) && *(S) > (ST)(D_MAX)) {                                            \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D,  \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,                              \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = (DT)(D_MAX);                                                                          \
@@ -491,9 +534,16 @@ typedef struct H5T_conv_hw_t {
     {                                                                                                        \
         /* Assumes memory format of unsigned & signed integers is same */                                    \
         if (*(S) < 0) {                                                                                      \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D, \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            H5T_conv_ret_t except_ret;                                                                       \
+                                                                                                             \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,                             \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = 0;                                                                                    \
@@ -523,9 +573,16 @@ typedef struct H5T_conv_hw_t {
     {                                                                                                        \
         /* Assumes memory format of unsigned & signed integers is same */                                    \
         if (*(S) > (ST)(D_MAX)) {                                                                            \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D,  \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            H5T_conv_ret_t except_ret;                                                                       \
+                                                                                                             \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,                              \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = (DT)(D_MAX);                                                                          \
@@ -562,10 +619,16 @@ typedef struct H5T_conv_hw_t {
  */
 #define H5T_CONV_Ff_CORE(STYPE, DTYPE, S, D, ST, DT, D_MIN, D_MAX)                                           \
     {                                                                                                        \
+        H5T_conv_ret_t except_ret;                                                                           \
         if (*(S) > (ST)(D_MAX)) {                                                                            \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D,  \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,                              \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = H5_GLUE3(H5T_NATIVE_, DTYPE, _POS_INF_g);                                             \
@@ -574,9 +637,14 @@ typedef struct H5T_conv_hw_t {
             /* if(except_ret==H5T_CONV_HANDLED): Fall through, user handled it */                            \
         }                                                                                                    \
         else if (*(S) < (ST)(D_MIN)) {                                                                       \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D, \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,                             \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = H5_GLUE3(H5T_NATIVE_, DTYPE, _NEG_INF_g);                                             \
@@ -673,9 +741,16 @@ typedef struct H5T_conv_hw_t {
                                                                                                              \
             /* Check for more bits of precision in src than available in dst */                              \
             if ((high_bit_pos - low_bit_pos) >= dprec) {                                                     \
-                H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                               \
-                    H5T_CONV_EXCEPT_PRECISION, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id,   \
-                    S, D, conv_ctx->u.conv.cb_struct.user_data);                                             \
+                H5T_conv_ret_t except_ret;                                                                   \
+                                                                                                             \
+                /* Prepare & restore library for user callback */                                            \
+                H5_BEFORE_USER_CB(FAIL)                                                                      \
+                    {                                                                                        \
+                        except_ret = (conv_ctx->u.conv.cb_struct.func)(                                      \
+                            H5T_CONV_EXCEPT_PRECISION, conv_ctx->u.conv.src_type_id,                         \
+                            conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);       \
+                    }                                                                                        \
+                H5_AFTER_USER_CB(FAIL)                                                                       \
                 if (except_ret == H5T_CONV_UNHANDLED)                                                        \
                     /* Let compiler convert if case is ignored by user handler*/                             \
                     *(D) = (DT)(*(S));                                                                       \
@@ -709,10 +784,16 @@ typedef struct H5T_conv_hw_t {
  */
 #define H5T_CONV_Fx_CORE(STYPE, DTYPE, S, D, ST, DT, D_MIN, D_MAX)                                           \
     {                                                                                                        \
+        H5T_conv_ret_t except_ret;                                                                           \
         if (*(S) > (ST)(D_MAX) || (sprec < dprec && *(S) == (ST)(D_MAX))) {                                  \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D,  \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,                              \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = (DT)(D_MAX);                                                                          \
@@ -721,9 +802,14 @@ typedef struct H5T_conv_hw_t {
             /* if(except_ret==H5T_CONV_HANDLED): Fall through, user handled it */                            \
         }                                                                                                    \
         else if (*(S) < (ST)(D_MIN)) {                                                                       \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D, \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,                             \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = (DT)(D_MIN);                                                                          \
@@ -732,9 +818,14 @@ typedef struct H5T_conv_hw_t {
             /* if(except_ret==H5T_CONV_HANDLED): Fall through, user handled it */                            \
         }                                                                                                    \
         else if (*(S) != (ST)((DT)(*(S)))) {                                                                 \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_TRUNCATE, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D,  \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_TRUNCATE, conv_ctx->u.conv.src_type_id,                              \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = (DT)(*(S));                                                                           \
@@ -768,10 +859,16 @@ typedef struct H5T_conv_hw_t {
 
 #define H5T_CONV_Xf_CORE(STYPE, DTYPE, S, D, ST, DT, D_MIN, D_MAX)                                           \
     {                                                                                                        \
+        H5T_conv_ret_t except_ret;                                                                           \
         if (*(S) > (ST)(D_MAX) || (sprec < dprec && *(S) == (ST)(D_MAX))) {                                  \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D,  \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,                              \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = H5_GLUE3(H5T_NATIVE_, DTYPE, _POS_INF_g);                                             \
@@ -780,9 +877,14 @@ typedef struct H5T_conv_hw_t {
             /* if(except_ret==H5T_CONV_HANDLED): Fall through, user handled it */                            \
         }                                                                                                    \
         else if (*(S) < (ST)(D_MIN)) {                                                                       \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D, \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,                             \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = H5_GLUE3(H5T_NATIVE_, DTYPE, _NEG_INF_g);                                             \
@@ -798,9 +900,14 @@ typedef struct H5T_conv_hw_t {
                                                                                                              \
             /* Check for more bits of precision in src than available in dst */                              \
             if ((high_bit_pos - low_bit_pos) >= dprec) {                                                     \
-                H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                               \
-                    H5T_CONV_EXCEPT_PRECISION, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id,   \
-                    S, D, conv_ctx->u.conv.cb_struct.user_data);                                             \
+                /* Prepare & restore library for user callback */                                            \
+                H5_BEFORE_USER_CB(FAIL)                                                                      \
+                    {                                                                                        \
+                        except_ret = (conv_ctx->u.conv.cb_struct.func)(                                      \
+                            H5T_CONV_EXCEPT_PRECISION, conv_ctx->u.conv.src_type_id,                         \
+                            conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);       \
+                    }                                                                                        \
+                H5_AFTER_USER_CB(FAIL)                                                                       \
                 if (except_ret == H5T_CONV_UNHANDLED)                                                        \
                     /* Let compiler convert if case is ignored by user handler*/                             \
                     *(D) = (DT)(*(S));                                                                       \
@@ -988,14 +1095,24 @@ typedef struct H5T_conv_hw_t {
              * number.                                                                                       \
              */                                                                                              \
             if (sr_over || si_over) {                                                                        \
-                except_ret = (conv_ctx->u.conv.cb_struct.func)(                                              \
-                    H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, \
-                    D, conv_ctx->u.conv.cb_struct.user_data);                                                \
+                /* Prepare & restore library for user callback */                                            \
+                H5_BEFORE_USER_CB(FAIL)                                                                      \
+                    {                                                                                        \
+                        except_ret = (conv_ctx->u.conv.cb_struct.func)(                                      \
+                            H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,                          \
+                            conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);       \
+                    }                                                                                        \
+                H5_AFTER_USER_CB(FAIL)                                                                       \
             }                                                                                                \
             else if (sr_under || si_under) {                                                                 \
-                except_ret = (conv_ctx->u.conv.cb_struct.func)(                                              \
-                    H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id,   \
-                    S, D, conv_ctx->u.conv.cb_struct.user_data);                                             \
+                /* Prepare & restore library for user callback */                                            \
+                H5_BEFORE_USER_CB(FAIL)                                                                      \
+                    {                                                                                        \
+                        except_ret = (conv_ctx->u.conv.cb_struct.func)(                                      \
+                            H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,                         \
+                            conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);       \
+                    }                                                                                        \
+                H5_AFTER_USER_CB(FAIL)                                                                       \
             }                                                                                                \
                                                                                                              \
             /* If user conversion exception function handled the exception, do nothing.                      \
@@ -1117,9 +1234,16 @@ typedef struct H5T_conv_hw_t {
     {                                                                                                        \
         H5T_CONV_##STYPE##_REALVAL(S);                                                                       \
         if ((sr_val) > (SBT)(D_MAX)) {                                                                       \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D,  \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            H5T_conv_ret_t except_ret;                                                                       \
+                                                                                                             \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,                              \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = H5_GLUE3(H5T_NATIVE_, DTYPE, _POS_INF_g);                                             \
@@ -1128,9 +1252,16 @@ typedef struct H5T_conv_hw_t {
             /* if(except_ret==H5T_CONV_HANDLED): Fall through, user handled it */                            \
         }                                                                                                    \
         else if ((sr_val) < (SBT)(D_MIN)) {                                                                  \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D, \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            H5T_conv_ret_t except_ret;                                                                       \
+                                                                                                             \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,                             \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = H5_GLUE3(H5T_NATIVE_, DTYPE, _NEG_INF_g);                                             \
@@ -1196,9 +1327,16 @@ typedef struct H5T_conv_hw_t {
 #define H5T_CONV_Fz_CORE(STYPE, DTYPE, S, D, ST, DT, D_MIN, D_MAX)                                           \
     {                                                                                                        \
         if (*(S) > (ST)(D_MAX)) {                                                                            \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D,  \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            H5T_conv_ret_t except_ret;                                                                       \
+                                                                                                             \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,                              \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = H5_GLUE3(H5T_NATIVE_, DTYPE, _POS_INF_g);                                             \
@@ -1207,9 +1345,16 @@ typedef struct H5T_conv_hw_t {
             /* if(except_ret==H5T_CONV_HANDLED): Fall through, user handled it */                            \
         }                                                                                                    \
         else if (*(S) < (ST)(D_MIN)) {                                                                       \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D, \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            H5T_conv_ret_t except_ret;                                                                       \
+                                                                                                             \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,                             \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = H5_GLUE3(H5T_NATIVE_, DTYPE, _NEG_INF_g);                                             \
@@ -1285,9 +1430,16 @@ typedef struct H5T_conv_hw_t {
                                                                                                              \
             /* Check for more bits of precision in src than available in dst */                              \
             if ((high_bit_pos - low_bit_pos) >= dprec) {                                                     \
-                H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                               \
-                    H5T_CONV_EXCEPT_PRECISION, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id,   \
-                    S, D, conv_ctx->u.conv.cb_struct.user_data);                                             \
+                H5T_conv_ret_t except_ret;                                                                   \
+                                                                                                             \
+                /* Prepare & restore library for user callback */                                            \
+                H5_BEFORE_USER_CB(FAIL)                                                                      \
+                    {                                                                                        \
+                        except_ret = (conv_ctx->u.conv.cb_struct.func)(                                      \
+                            H5T_CONV_EXCEPT_PRECISION, conv_ctx->u.conv.src_type_id,                         \
+                            conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);       \
+                    }                                                                                        \
+                H5_AFTER_USER_CB(FAIL)                                                                       \
                 if (except_ret == H5T_CONV_UNHANDLED)                                                        \
                     /* Let compiler convert if case is ignored by user handler*/                             \
                     H5T_CONV_CAST_Z(xZ, STYPE, DTYPE, S, -, -, D, ST, DT);                                   \
@@ -1316,9 +1468,16 @@ typedef struct H5T_conv_hw_t {
     {                                                                                                        \
         H5T_CONV_##STYPE##_REALVAL(S); /* Extract "real" part of complex number */                           \
         if ((sr_val) > (SBT)(D_MAX) || (sprec < dprec && (sr_val) == (SBT)(D_MAX))) {                        \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D,  \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            H5T_conv_ret_t except_ret;                                                                       \
+                                                                                                             \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_HI, conv_ctx->u.conv.src_type_id,                              \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = (DT)(D_MAX);                                                                          \
@@ -1327,9 +1486,16 @@ typedef struct H5T_conv_hw_t {
             /* if(except_ret==H5T_CONV_HANDLED): Fall through, user handled it */                            \
         }                                                                                                    \
         else if ((sr_val) < (SBT)(D_MIN)) {                                                                  \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D, \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            H5T_conv_ret_t except_ret;                                                                       \
+                                                                                                             \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_RANGE_LOW, conv_ctx->u.conv.src_type_id,                             \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = (DT)(D_MIN);                                                                          \
@@ -1338,9 +1504,16 @@ typedef struct H5T_conv_hw_t {
             /* if(except_ret==H5T_CONV_HANDLED): Fall through, user handled it */                            \
         }                                                                                                    \
         else if ((sr_val) != (SBT)((DT)((sr_val)))) {                                                        \
-            H5T_conv_ret_t except_ret = (conv_ctx->u.conv.cb_struct.func)(                                   \
-                H5T_CONV_EXCEPT_TRUNCATE, conv_ctx->u.conv.src_type_id, conv_ctx->u.conv.dst_type_id, S, D,  \
-                conv_ctx->u.conv.cb_struct.user_data);                                                       \
+            H5T_conv_ret_t except_ret;                                                                       \
+                                                                                                             \
+            /* Prepare & restore library for user callback */                                                \
+            H5_BEFORE_USER_CB(FAIL)                                                                          \
+                {                                                                                            \
+                    except_ret = (conv_ctx->u.conv.cb_struct.func)(                                          \
+                        H5T_CONV_EXCEPT_TRUNCATE, conv_ctx->u.conv.src_type_id,                              \
+                        conv_ctx->u.conv.dst_type_id, S, D, conv_ctx->u.conv.cb_struct.user_data);           \
+                }                                                                                            \
+            H5_AFTER_USER_CB(FAIL)                                                                           \
             if (except_ret == H5T_CONV_UNHANDLED)                                                            \
                 /* Let compiler convert if case is ignored by user handler*/                                 \
                 *(D) = (DT)((sr_val));                                                                       \
diff --git a/src/H5Tmodule.h b/src/H5Tmodule.h
index 44aba1cb7fa..5d9ee022d8c 100644
--- a/src/H5Tmodule.h
+++ b/src/H5Tmodule.h
@@ -4087,6 +4087,8 @@ filled according to the value of this property. The padding can be:
  * \ingroup H5T
  * \defgroup ATOM Atomic Datatypes
  * \ingroup H5T
+ * \defgroup COMPLEX Complex Datatypes
+ * \ingroup H5T
  * \defgroup CONV Conversion Function
  * \ingroup H5T
  * \defgroup OPAQUE Opaque Datatypes
diff --git a/src/H5Tvlen.c b/src/H5Tvlen.c
index f98d9287968..fc0ff4b18a8 100644
--- a/src/H5Tvlen.c
+++ b/src/H5Tvlen.c
@@ -500,7 +500,13 @@ H5T__vlen_mem_seq_write(H5VL_object_t H5_ATTR_UNUSED *file, const H5T_vlen_alloc
 
         /* Use the user's memory allocation routine if one is defined */
         if (vl_alloc_info->alloc_func != NULL) {
-            if (NULL == (vl.p = (vl_alloc_info->alloc_func)(len, vl_alloc_info->alloc_info)))
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    vl.p = (vl_alloc_info->alloc_func)(len, vl_alloc_info->alloc_info);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (NULL == vl.p)
                 HGOTO_ERROR(H5E_DATATYPE, H5E_CANTALLOC, FAIL,
                             "application memory allocation routine failed for VL data");
         }    /* end if */
@@ -672,8 +678,13 @@ H5T__vlen_mem_str_write(H5VL_object_t H5_ATTR_UNUSED *file, const H5T_vlen_alloc
 
     /* Use the user's memory allocation routine if one is defined */
     if (vl_alloc_info->alloc_func != NULL) {
-        if (NULL ==
-            (t = (char *)(vl_alloc_info->alloc_func)((seq_len + 1) * base_size, vl_alloc_info->alloc_info)))
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                t = (vl_alloc_info->alloc_func)((seq_len + 1) * base_size, vl_alloc_info->alloc_info);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (NULL == t)
             HGOTO_ERROR(H5E_DATATYPE, H5E_CANTALLOC, FAIL,
                         "application memory allocation routine failed for VL data");
     }    /* end if */
diff --git a/src/H5UC.c b/src/H5UC.c
index cab195eb6c0..212b7fe117f 100644
--- a/src/H5UC.c
+++ b/src/H5UC.c
@@ -20,7 +20,8 @@
  *
  */
 
-#include "H5Eprivate.h"  /* Error handling		  	*/
+#include "H5private.h"   /* Generic Functions                    */
+#include "H5Eprivate.h"  /* Error handling		  	 */
 #include "H5FLprivate.h" /* Free lists                           */
 #include "H5UCprivate.h" /* Reference-counted buffers            */
 
diff --git a/src/H5VLcallback.c b/src/H5VLcallback.c
index 830a61fc945..71ad2035565 100644
--- a/src/H5VLcallback.c
+++ b/src/H5VLcallback.c
@@ -233,8 +233,16 @@ H5VLinitialize(hid_t connector_id, hid_t vipl_id)
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a VOL connector ID");
 
     /* Invoke class' callback, if there is one */
-    if (connector->cls->initialize && connector->cls->initialize(vipl_id) < 0)
-        HGOTO_ERROR(H5E_VOL, H5E_CANTCLOSEOBJ, FAIL, "VOL connector did not initialize");
+    if (connector->cls->initialize) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = connector->cls->initialize(vipl_id);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
+            HGOTO_ERROR(H5E_VOL, H5E_CANTCLOSEOBJ, FAIL, "VOL connector did not initialize");
+    }
 
 done:
     FUNC_LEAVE_API_NOINIT(ret_value)
@@ -263,8 +271,16 @@ H5VLterminate(hid_t connector_id)
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a VOL connector ID");
 
     /* Invoke class' callback, if there is one */
-    if (connector->cls->terminate && connector->cls->terminate() < 0)
-        HGOTO_ERROR(H5E_VOL, H5E_CANTCLOSEOBJ, FAIL, "VOL connector did not terminate cleanly");
+    if (connector->cls->terminate) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = connector->cls->terminate();
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
+            HGOTO_ERROR(H5E_VOL, H5E_CANTCLOSEOBJ, FAIL, "VOL connector did not terminate cleanly");
+    }
 
 done:
     FUNC_LEAVE_API_NOINIT(ret_value)
@@ -399,7 +415,13 @@ H5VL_copy_connector_info(const H5VL_connector_t *connector, void **dst_info, con
     if (src_info) {
         /* Allow the connector to copy or do it ourselves */
         if (connector->cls->info_cls.copy) {
-            if (NULL == (new_connector_info = (connector->cls->info_cls.copy)(src_info)))
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    new_connector_info = (connector->cls->info_cls.copy)(src_info);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (NULL == new_connector_info)
                 HGOTO_ERROR(H5E_VOL, H5E_CANTCOPY, FAIL, "connector info copy callback failed");
         } /* end if */
         else if (connector->cls->info_cls.size > 0) {
@@ -492,7 +514,13 @@ H5VL_cmp_connector_info(const H5VL_connector_t *connector, int *cmp_value, const
      * memory buffers
      */
     if (connector->cls->info_cls.cmp) {
-        if ((connector->cls->info_cls.cmp)(cmp_value, info1, info2) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (connector->cls->info_cls.cmp)(cmp_value, info1, info2);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VOL, H5E_CANTCOMPARE, FAIL, "can't compare connector info");
     } /* end if */
     else {
@@ -563,8 +591,14 @@ H5VL_free_connector_info(const H5VL_connector_t *connector, const void *info)
     if (info) {
         /* Allow the connector to free info or do it ourselves */
         if (connector->cls->info_cls.free) {
-            /* Cast through uintptr_t to de-const memory */
-            if ((connector->cls->info_cls.free)((void *)(uintptr_t)info) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    /* Cast through uintptr_t to de-const memory */
+                    ret_value = (connector->cls->info_cls.free)((void *)(uintptr_t)info);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_VOL, H5E_CANTRELEASE, FAIL, "connector info free request failed");
         }
         else
@@ -632,7 +666,13 @@ H5VLconnector_info_to_str(const void *info, hid_t connector_id, char **str)
 
         /* Allow the connector to serialize info */
         if (connector->cls->info_cls.to_str) {
-            if ((connector->cls->info_cls.to_str)(info, str) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    ret_value = (connector->cls->info_cls.to_str)(info, str);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_VOL, H5E_CANTSERIALIZE, FAIL, "can't serialize connector info");
         } /* end if */
         else
@@ -666,7 +706,13 @@ H5VL__connector_str_to_info(const char *str, H5VL_connector_t *connector, void *
     if (str) {
         /* Allow the connector to deserialize info */
         if (connector->cls->info_cls.from_str) {
-            if ((connector->cls->info_cls.from_str)(str, info) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    ret_value = (connector->cls->info_cls.from_str)(str, info);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_VOL, H5E_CANTUNSERIALIZE, FAIL, "can't deserialize connector info");
         } /* end if */
         else
@@ -734,8 +780,14 @@ H5VLget_object(void *obj, hid_t connector_id)
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a VOL connector ID");
 
     /* Check for 'get_object' callback in connector */
-    if (connector->cls->wrap_cls.get_object)
-        ret_value = (connector->cls->wrap_cls.get_object)(obj);
+    if (connector->cls->wrap_cls.get_object) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(NULL)
+            {
+                ret_value = (connector->cls->wrap_cls.get_object)(obj);
+            }
+        H5_AFTER_USER_CB(NULL)
+    }
     else
         ret_value = obj;
 
@@ -773,8 +825,14 @@ H5VLget_wrap_ctx(void *obj, hid_t connector_id, void **wrap_ctx /*out*/)
         /* Sanity check */
         assert(connector->cls->wrap_cls.free_wrap_ctx);
 
-        /* Invoke cls's callback */
-        if ((connector->cls->wrap_cls.get_wrap_ctx)(obj, wrap_ctx) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                /* Invoke connector's callback */
+                ret_value = (connector->cls->wrap_cls.get_wrap_ctx)(obj, wrap_ctx);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "connector wrap context callback failed");
     } /* end if */
     else
@@ -807,8 +865,14 @@ H5VL_wrap_object(const H5VL_class_t *cls, void *wrap_ctx, void *obj, H5I_type_t
 
     /* Only wrap object if there's a wrap context */
     if (wrap_ctx) {
-        /* Ask the connector to wrap the object */
-        if (NULL == (ret_value = (cls->wrap_cls.wrap_object)(obj, obj_type, wrap_ctx)))
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(NULL)
+            {
+                /* Ask the connector to wrap the object */
+                ret_value = (cls->wrap_cls.wrap_object)(obj, obj_type, wrap_ctx);
+            }
+        H5_AFTER_USER_CB(NULL)
+        if (NULL == ret_value)
             HGOTO_ERROR(H5E_VOL, H5E_CANTGET, NULL, "can't wrap object");
     } /* end if */
     else
@@ -873,8 +937,14 @@ H5VL_unwrap_object(const H5VL_class_t *cls, void *obj)
 
     /* Only unwrap object if there's an unwrap callback */
     if (cls->wrap_cls.wrap_object) {
-        /* Ask the connector to unwrap the object */
-        if (NULL == (ret_value = (cls->wrap_cls.unwrap_object)(obj)))
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(NULL)
+            {
+                /* Ask the connector to unwrap the object */
+                ret_value = (cls->wrap_cls.unwrap_object)(obj);
+            }
+        H5_AFTER_USER_CB(NULL)
+        if (NULL == ret_value)
             HGOTO_ERROR(H5E_VOL, H5E_CANTGET, NULL, "can't unwrap object");
     } /* end if */
     else
@@ -939,10 +1009,17 @@ H5VLfree_wrap_ctx(void *wrap_ctx, hid_t connector_id)
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a VOL connector ID");
 
     /* Only free wrap context, if it's non-NULL */
-    if (wrap_ctx)
-        /* Free the connector's object wrapping context */
-        if ((connector->cls->wrap_cls.free_wrap_ctx)(wrap_ctx) < 0)
+    if (wrap_ctx) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                /* Free the connector's object wrapping context */
+                ret_value = (connector->cls->wrap_cls.free_wrap_ctx)(wrap_ctx);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VOL, H5E_CANTRELEASE, FAIL, "connector wrap context free request failed");
+    } /* end if */
 
 done:
     FUNC_LEAVE_API_NOINIT(ret_value)
@@ -970,9 +1047,15 @@ H5VL__attr_create(void *obj, const H5VL_loc_params_t *loc_params, const H5VL_cla
     if (NULL == cls->attr_cls.create)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, NULL, "VOL connector has no 'attr create' method");
 
-    /* Call the corresponding VOL callback */
-    if (NULL == (ret_value = (cls->attr_cls.create)(obj, loc_params, name, type_id, space_id, acpl_id,
-                                                    aapl_id, dxpl_id, req)))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(NULL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->attr_cls.create)(obj, loc_params, name, type_id, space_id, acpl_id, aapl_id,
+                                               dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(NULL)
+    if (NULL == ret_value)
         HGOTO_ERROR(H5E_VOL, H5E_CANTCREATE, NULL, "attribute create failed");
 
 done:
@@ -1073,8 +1156,14 @@ H5VL__attr_open(void *obj, const H5VL_loc_params_t *loc_params, const H5VL_class
     if (NULL == cls->attr_cls.open)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, NULL, "VOL connector has no 'attr open' method");
 
-    /* Call the corresponding VOL open callback */
-    if (NULL == (ret_value = (cls->attr_cls.open)(obj, loc_params, name, aapl_id, dxpl_id, req)))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(NULL)
+        {
+            /* Call the corresponding VOL open callback */
+            ret_value = (cls->attr_cls.open)(obj, loc_params, name, aapl_id, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(NULL)
+    if (NULL == ret_value)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPENOBJ, NULL, "attribute open failed");
 
 done:
@@ -1172,8 +1261,14 @@ H5VL__attr_read(void *obj, const H5VL_class_t *cls, hid_t mem_type_id, void *buf
     if (NULL == cls->attr_cls.read)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'attr read' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->attr_cls.read)(obj, mem_type_id, buf, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->attr_cls.read)(obj, mem_type_id, buf, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_READERROR, FAIL, "attribute read failed");
 
 done:
@@ -1269,8 +1364,14 @@ H5VL__attr_write(void *obj, const H5VL_class_t *cls, hid_t mem_type_id, const vo
     if (NULL == cls->attr_cls.write)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'attr write' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->attr_cls.write)(obj, mem_type_id, buf, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->attr_cls.write)(obj, mem_type_id, buf, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_WRITEERROR, FAIL, "write failed");
 
 done:
@@ -1366,8 +1467,14 @@ H5VL__attr_get(void *obj, const H5VL_class_t *cls, H5VL_attr_get_args_t *args, h
     if (NULL == cls->attr_cls.get)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'attr get' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->attr_cls.get)(obj, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->attr_cls.get)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "attribute get failed");
 
 done:
@@ -1465,9 +1572,15 @@ H5VL__attr_specific(void *obj, const H5VL_loc_params_t *loc_params, const H5VL_c
     if (NULL == cls->attr_cls.specific)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'attr specific' method");
 
-    /* Call the corresponding VOL callback */
-    /* (Must return value from callback, for iterators) */
-    if ((ret_value = (cls->attr_cls.specific)(obj, loc_params, args, dxpl_id, req)) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            /* (Must return value from callback, for iterators) */
+            ret_value = (cls->attr_cls.specific)(obj, loc_params, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HERROR(H5E_VOL, H5E_CANTOPERATE, "unable to execute attribute 'specific' callback");
 
 done:
@@ -1567,9 +1680,15 @@ H5VL__attr_optional(void *obj, const H5VL_class_t *cls, H5VL_optional_args_t *ar
     if (NULL == cls->attr_cls.optional)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'attr optional' method");
 
-    /* Call the corresponding VOL callback */
-    /* (Must return value from callback, for iterators) */
-    if ((ret_value = (cls->attr_cls.optional)(obj, args, dxpl_id, req)) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            /* (Must return value from callback, for iterators) */
+            ret_value = (cls->attr_cls.optional)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HERROR(H5E_VOL, H5E_CANTOPERATE, "unable to execute attribute optional callback");
 
 done:
@@ -1709,8 +1828,14 @@ H5VL__attr_close(void *obj, const H5VL_class_t *cls, hid_t dxpl_id, void **req)
     if (NULL == cls->attr_cls.close)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'attr close' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->attr_cls.close)(obj, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->attr_cls.close)(obj, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTCLOSEOBJ, FAIL, "attribute close failed");
 
 done:
@@ -1810,9 +1935,15 @@ H5VL__dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const H5VL_
     if (NULL == cls->dataset_cls.create)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, NULL, "VOL connector has no 'dataset create' method");
 
-    /* Call the corresponding VOL callback */
-    if (NULL == (ret_value = (cls->dataset_cls.create)(obj, loc_params, name, lcpl_id, type_id, space_id,
-                                                       dcpl_id, dapl_id, dxpl_id, req)))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(NULL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->dataset_cls.create)(obj, loc_params, name, lcpl_id, type_id, space_id, dcpl_id,
+                                                  dapl_id, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(NULL)
+    if (NULL == ret_value)
         HGOTO_ERROR(H5E_VOL, H5E_CANTCREATE, NULL, "dataset create failed");
 
 done:
@@ -1915,8 +2046,14 @@ H5VL__dataset_open(void *obj, const H5VL_loc_params_t *loc_params, const H5VL_cl
     if (NULL == cls->dataset_cls.open)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, NULL, "VOL connector has no 'dataset open' method");
 
-    /* Call the corresponding VOL callback */
-    if (NULL == (ret_value = (cls->dataset_cls.open)(obj, loc_params, name, dapl_id, dxpl_id, req)))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(NULL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->dataset_cls.open)(obj, loc_params, name, dapl_id, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(NULL)
+    if (NULL == ret_value)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPENOBJ, NULL, "dataset open failed");
 
 done:
@@ -2016,8 +2153,15 @@ H5VL__dataset_read(size_t count, void *obj[], const H5VL_class_t *cls, hid_t mem
     if (NULL == cls->dataset_cls.read)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'dataset read' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->dataset_cls.read)(count, obj, mem_type_id, mem_space_id, file_space_id, dxpl_id, buf, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->dataset_cls.read)(count, obj, mem_type_id, mem_space_id, file_space_id, dxpl_id,
+                                                buf, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_READERROR, FAIL, "dataset read failed");
 
 done:
@@ -2140,8 +2284,15 @@ H5VL__dataset_write(size_t count, void *obj[], const H5VL_class_t *cls, hid_t me
     if (NULL == cls->dataset_cls.write)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'dataset write' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->dataset_cls.write)(count, obj, mem_type_id, mem_space_id, file_space_id, dxpl_id, buf, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->dataset_cls.write)(count, obj, mem_type_id, mem_space_id, file_space_id,
+                                                 dxpl_id, buf, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_WRITEERROR, FAIL, "dataset write failed");
 
 done:
@@ -2264,8 +2415,14 @@ H5VL__dataset_get(void *obj, const H5VL_class_t *cls, H5VL_dataset_get_args_t *a
     if (NULL == cls->dataset_cls.get)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'dataset get' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->dataset_cls.get)(obj, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->dataset_cls.get)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "dataset get failed");
 
 done:
@@ -2362,8 +2519,14 @@ H5VL__dataset_specific(void *obj, const H5VL_class_t *cls, H5VL_dataset_specific
     if (NULL == cls->dataset_cls.specific)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'dataset specific' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->dataset_cls.specific)(obj, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->dataset_cls.specific)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL, "unable to execute dataset specific callback");
 
 done:
@@ -2461,8 +2624,14 @@ H5VL__dataset_optional(void *obj, const H5VL_class_t *cls, H5VL_optional_args_t
     if (NULL == cls->dataset_cls.optional)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'dataset optional' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->dataset_cls.optional)(obj, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->dataset_cls.optional)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL, "unable to execute dataset optional callback");
 
 done:
@@ -2604,8 +2773,14 @@ H5VL__dataset_close(void *obj, const H5VL_class_t *cls, hid_t dxpl_id, void **re
     if (NULL == cls->dataset_cls.close)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'dataset close' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->dataset_cls.close)(obj, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->dataset_cls.close)(obj, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTCLOSEOBJ, FAIL, "dataset close failed");
 
 done:
@@ -2708,9 +2883,15 @@ H5VL__datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const H5VL
     if (NULL == cls->datatype_cls.commit)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, NULL, "VOL connector has no 'datatype commit' method");
 
-    /* Call the corresponding VOL callback */
-    if (NULL == (ret_value = (cls->datatype_cls.commit)(obj, loc_params, name, type_id, lcpl_id, tcpl_id,
-                                                        tapl_id, dxpl_id, req)))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(NULL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->datatype_cls.commit)(obj, loc_params, name, type_id, lcpl_id, tcpl_id, tapl_id,
+                                                   dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(NULL)
+    if (NULL == ret_value)
         HGOTO_ERROR(H5E_VOL, H5E_CANTCREATE, NULL, "datatype commit failed");
 
 done:
@@ -2811,8 +2992,14 @@ H5VL__datatype_open(void *obj, const H5VL_loc_params_t *loc_params, const H5VL_c
     if (NULL == cls->datatype_cls.open)
         HGOTO_ERROR(H5E_VOL, H5E_CANTINIT, NULL, "no datatype open callback");
 
-    /* Call the corresponding VOL callback */
-    if (NULL == (ret_value = (cls->datatype_cls.open)(obj, loc_params, name, tapl_id, dxpl_id, req)))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(NULL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->datatype_cls.open)(obj, loc_params, name, tapl_id, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(NULL)
+    if (NULL == ret_value)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPENOBJ, NULL, "datatype open failed");
 
 done:
@@ -2912,8 +3099,14 @@ H5VL__datatype_get(void *obj, const H5VL_class_t *cls, H5VL_datatype_get_args_t
     if (NULL == cls->datatype_cls.get)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'datatype get' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->datatype_cls.get)(obj, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->datatype_cls.get)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "datatype 'get' failed");
 
 done:
@@ -3010,8 +3203,14 @@ H5VL__datatype_specific(void *obj, const H5VL_class_t *cls, H5VL_datatype_specif
     if (NULL == cls->datatype_cls.specific)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'datatype specific' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->datatype_cls.specific)(obj, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->datatype_cls.specific)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL, "unable to execute datatype specific callback");
 
 done:
@@ -3109,8 +3308,14 @@ H5VL__datatype_optional(void *obj, const H5VL_class_t *cls, H5VL_optional_args_t
     if (NULL == cls->datatype_cls.optional)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'datatype optional' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->datatype_cls.optional)(obj, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->datatype_cls.optional)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL, "unable to execute datatype optional callback");
 
 done:
@@ -3296,8 +3501,14 @@ H5VL__datatype_close(void *obj, const H5VL_class_t *cls, hid_t dxpl_id, void **r
     if (NULL == cls->datatype_cls.close)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'datatype close' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->datatype_cls.close)(obj, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->datatype_cls.close)(obj, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTCLOSEOBJ, FAIL, "datatype close failed");
 
 done:
@@ -3396,8 +3607,14 @@ H5VL__file_create(const H5VL_class_t *cls, const char *name, unsigned flags, hid
     if (NULL == cls->file_cls.create)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, NULL, "VOL connector has no 'file create' method");
 
-    /* Call the corresponding VOL callback */
-    if (NULL == (ret_value = (cls->file_cls.create)(name, flags, fcpl_id, fapl_id, dxpl_id, req)))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(NULL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->file_cls.create)(name, flags, fcpl_id, fapl_id, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(NULL)
+    if (NULL == ret_value)
         HGOTO_ERROR(H5E_VOL, H5E_CANTCREATE, NULL, "file create failed");
 
 done:
@@ -3490,8 +3707,14 @@ H5VL__file_open(const H5VL_class_t *cls, const char *name, unsigned flags, hid_t
     if (NULL == cls->file_cls.open)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, NULL, "VOL connector has no 'file open' method");
 
-    /* Call the corresponding VOL callback */
-    if (NULL == (ret_value = (cls->file_cls.open)(name, flags, fapl_id, dxpl_id, req)))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(NULL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->file_cls.open)(name, flags, fapl_id, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(NULL)
+    if (NULL == ret_value)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPENOBJ, NULL, "open failed");
 
 done:
@@ -3559,9 +3782,9 @@ H5VL__file_open_find_connector_cb(H5PL_type_t H5_ATTR_UNUSED plugin_type,
     vol_cb_args.args.is_accessible.accessible = &is_accessible;
 
     H5E_PAUSE_ERRORS
-    {
-        status = H5VL_file_specific(NULL, &vol_cb_args, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL);
-    }
+        {
+            status = H5VL_file_specific(NULL, &vol_cb_args, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL);
+        }
     H5E_RESUME_ERRORS
 
     if (status >= 0 && is_accessible) {
@@ -3708,8 +3931,14 @@ H5VL__file_get(void *obj, const H5VL_class_t *cls, H5VL_file_get_args_t *args, h
     if (NULL == cls->file_cls.get)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'file get' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->file_cls.get)(obj, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->file_cls.get)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "file get failed");
 
 done:
@@ -3805,8 +4034,14 @@ H5VL__file_specific(void *obj, const H5VL_class_t *cls, H5VL_file_specific_args_
     if (NULL == cls->file_cls.specific)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'file specific' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->file_cls.specific)(obj, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->file_cls.specific)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL, "file specific failed");
 
 done:
@@ -3935,8 +4170,14 @@ H5VL__file_optional(void *obj, const H5VL_class_t *cls, H5VL_optional_args_t *ar
     if (NULL == cls->file_cls.optional)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'file optional' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->file_cls.optional)(obj, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->file_cls.optional)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL, "file optional failed");
 
 done:
@@ -4078,8 +4319,14 @@ H5VL__file_close(void *obj, const H5VL_class_t *cls, hid_t dxpl_id, void **req)
     if (NULL == cls->file_cls.close)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'file close' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->file_cls.close)(obj, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->file_cls.close)(obj, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTCLOSEFILE, FAIL, "file close failed");
 
 done:
@@ -4175,9 +4422,15 @@ H5VL__group_create(void *obj, const H5VL_loc_params_t *loc_params, const H5VL_cl
     if (NULL == cls->group_cls.create)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, NULL, "VOL connector has no 'group create' method");
 
-    /* Call the corresponding VOL callback */
-    if (NULL ==
-        (ret_value = (cls->group_cls.create)(obj, loc_params, name, lcpl_id, gcpl_id, gapl_id, dxpl_id, req)))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(NULL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value =
+                (cls->group_cls.create)(obj, loc_params, name, lcpl_id, gcpl_id, gapl_id, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(NULL)
+    if (NULL == ret_value)
         HGOTO_ERROR(H5E_VOL, H5E_CANTCREATE, NULL, "group create failed");
 
 done:
@@ -4277,8 +4530,14 @@ H5VL__group_open(void *obj, const H5VL_loc_params_t *loc_params, const H5VL_clas
     if (NULL == cls->group_cls.open)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, NULL, "VOL connector has no 'group open' method");
 
-    /* Call the corresponding VOL callback */
-    if (NULL == (ret_value = (cls->group_cls.open)(obj, loc_params, name, gapl_id, dxpl_id, req)))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(NULL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->group_cls.open)(obj, loc_params, name, gapl_id, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(NULL)
+    if (NULL == ret_value)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPENOBJ, NULL, "group open failed");
 
 done:
@@ -4376,8 +4635,14 @@ H5VL__group_get(void *obj, const H5VL_class_t *cls, H5VL_group_get_args_t *args,
     if (NULL == cls->group_cls.get)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'group get' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->group_cls.get)(obj, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->group_cls.get)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "group get failed");
 
 done:
@@ -4473,8 +4738,14 @@ H5VL__group_specific(void *obj, const H5VL_class_t *cls, H5VL_group_specific_arg
     if (NULL == cls->group_cls.specific)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'group specific' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->group_cls.specific)(obj, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->group_cls.specific)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL, "unable to execute group specific callback");
 
 done:
@@ -4571,9 +4842,15 @@ H5VL__group_optional(void *obj, const H5VL_class_t *cls, H5VL_optional_args_t *a
     if (NULL == cls->group_cls.optional)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'group optional' method");
 
-    /* Call the corresponding VOL callback */
-    /* (Must return value from callback, for iterators) */
-    if ((ret_value = (cls->group_cls.optional)(obj, args, dxpl_id, req)) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            /* (Must return value from callback, for iterators) */
+            ret_value = (cls->group_cls.optional)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HERROR(H5E_VOL, H5E_CANTOPERATE, "unable to execute group optional callback");
 
 done:
@@ -4717,8 +4994,14 @@ H5VL__group_close(void *obj, const H5VL_class_t *cls, hid_t dxpl_id, void **req)
     if (NULL == cls->group_cls.close)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'group close' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->group_cls.close)(obj, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->group_cls.close)(obj, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTCLOSEOBJ, FAIL, "group close failed");
 
 done:
@@ -4816,8 +5099,14 @@ H5VL__link_create(H5VL_link_create_args_t *args, void *obj, const H5VL_loc_param
     if (NULL == cls->link_cls.create)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'link create' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->link_cls.create)(args, obj, loc_params, lcpl_id, lapl_id, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->link_cls.create)(args, obj, loc_params, lcpl_id, lapl_id, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTCREATE, FAIL, "link create failed");
 
 done:
@@ -4927,8 +5216,15 @@ H5VL__link_copy(void *src_obj, const H5VL_loc_params_t *loc_params1, void *dst_o
     if (NULL == cls->link_cls.copy)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'link copy' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->link_cls.copy)(src_obj, loc_params1, dst_obj, loc_params2, lcpl_id, lapl_id, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->link_cls.copy)(src_obj, loc_params1, dst_obj, loc_params2, lcpl_id, lapl_id,
+                                             dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTCOPY, FAIL, "link copy failed");
 
 done:
@@ -5033,8 +5329,15 @@ H5VL__link_move(void *src_obj, const H5VL_loc_params_t *loc_params1, void *dst_o
     if (NULL == cls->link_cls.move)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'link move' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->link_cls.move)(src_obj, loc_params1, dst_obj, loc_params2, lcpl_id, lapl_id, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->link_cls.move)(src_obj, loc_params1, dst_obj, loc_params2, lcpl_id, lapl_id,
+                                             dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTMOVE, FAIL, "link move failed");
 
 done:
@@ -5142,8 +5445,14 @@ H5VL__link_get(void *obj, const H5VL_loc_params_t *loc_params, const H5VL_class_
     if (NULL == cls->link_cls.get)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'link get' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->link_cls.get)(obj, loc_params, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->link_cls.get)(obj, loc_params, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "link get failed");
 
 done:
@@ -5241,9 +5550,15 @@ H5VL__link_specific(void *obj, const H5VL_loc_params_t *loc_params, const H5VL_c
     if (NULL == cls->link_cls.specific)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'link specific' method");
 
-    /* Call the corresponding VOL callback */
-    /* (Must return value from callback, for iterators) */
-    if ((ret_value = (cls->link_cls.specific)(obj, loc_params, args, dxpl_id, req)) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            /* (Must return value from callback, for iterators) */
+            ret_value = (cls->link_cls.specific)(obj, loc_params, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HERROR(H5E_VOL, H5E_CANTOPERATE, "unable to execute link specific callback");
 
 done:
@@ -5344,8 +5659,14 @@ H5VL__link_optional(void *obj, const H5VL_loc_params_t *loc_params, const H5VL_c
     if (NULL == cls->link_cls.optional)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'link optional' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->link_cls.optional)(obj, loc_params, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->link_cls.optional)(obj, loc_params, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL, "unable to execute link optional callback");
 
 done:
@@ -5502,8 +5823,14 @@ H5VL__object_open(void *obj, const H5VL_loc_params_t *params, const H5VL_class_t
     if (NULL == cls->object_cls.open)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, NULL, "VOL connector has no 'object open' method");
 
-    /* Call the corresponding VOL callback */
-    if (NULL == (ret_value = (cls->object_cls.open)(obj, params, opened_type, dxpl_id, req)))
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(NULL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->object_cls.open)(obj, params, opened_type, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(NULL)
+    if (NULL == ret_value)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPENOBJ, NULL, "object open failed");
 
 done:
@@ -5603,9 +5930,15 @@ H5VL__object_copy(void *src_obj, const H5VL_loc_params_t *src_loc_params, const
     if (NULL == cls->object_cls.copy)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'object copy' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->object_cls.copy)(src_obj, src_loc_params, src_name, dst_obj, dst_loc_params, dst_name,
-                               ocpypl_id, lcpl_id, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->object_cls.copy)(src_obj, src_loc_params, src_name, dst_obj, dst_loc_params,
+                                               dst_name, ocpypl_id, lcpl_id, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTCOPY, FAIL, "object copy failed");
 
 done:
@@ -5712,8 +6045,14 @@ H5VL__object_get(void *obj, const H5VL_loc_params_t *loc_params, const H5VL_clas
     if (NULL == cls->object_cls.get)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'object get' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->object_cls.get)(obj, loc_params, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->object_cls.get)(obj, loc_params, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "get failed");
 
 done:
@@ -5811,9 +6150,15 @@ H5VL__object_specific(void *obj, const H5VL_loc_params_t *loc_params, const H5VL
     if (NULL == cls->object_cls.specific)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'object specific' method");
 
-    /* Call the corresponding VOL callback */
-    /* (Must return value from callback, for iterators) */
-    if ((ret_value = (cls->object_cls.specific)(obj, loc_params, args, dxpl_id, req)) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            /* (Must return value from callback, for iterators) */
+            ret_value = (cls->object_cls.specific)(obj, loc_params, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HERROR(H5E_VOL, H5E_CANTOPERATE, "object specific failed");
 
 done:
@@ -5914,8 +6259,14 @@ H5VL__object_optional(void *obj, const H5VL_loc_params_t *loc_params, const H5VL
     if (NULL == cls->object_cls.optional)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'object optional' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->object_cls.optional)(obj, loc_params, args, dxpl_id, req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->object_cls.optional)(obj, loc_params, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL, "unable to execute object optional callback");
 
 done:
@@ -6080,8 +6431,14 @@ H5VL__introspect_get_conn_cls(void *obj, const H5VL_class_t *cls, H5VL_get_conn_
     if (NULL == cls->introspect_cls.get_conn_cls)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'get_conn_cls' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->introspect_cls.get_conn_cls)(obj, lvl, conn_cls) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->introspect_cls.get_conn_cls)(obj, lvl, conn_cls);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "can't query connector class");
 
 done:
@@ -6189,8 +6546,14 @@ H5VL_introspect_get_cap_flags(const void *info, const H5VL_class_t *cls, uint64_
     if (NULL == cls->introspect_cls.get_cap_flags)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'get_cap_flags' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->introspect_cls.get_cap_flags)(info, cap_flags) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->introspect_cls.get_cap_flags)(info, cap_flags);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "can't query connector capability flags");
 
 done:
@@ -6255,8 +6618,14 @@ H5VL__introspect_opt_query(void *obj, const H5VL_class_t *cls, H5VL_subclass_t s
     if (NULL == cls->introspect_cls.opt_query)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'opt_query' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->introspect_cls.opt_query)(obj, subcls, opt_type, flags) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->introspect_cls.opt_query)(obj, subcls, opt_type, flags);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "can't query optional operation support");
 
 done:
@@ -6357,8 +6726,14 @@ H5VL__request_wait(void *req, const H5VL_class_t *cls, uint64_t timeout, H5VL_re
     if (NULL == cls->request_cls.wait)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'async wait' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->request_cls.wait)(req, timeout, status) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->request_cls.wait)(req, timeout, status);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTRELEASE, FAIL, "request wait failed");
 
 done:
@@ -6459,8 +6834,14 @@ H5VL__request_notify(void *req, const H5VL_class_t *cls, H5VL_request_notify_t c
     if (NULL == cls->request_cls.notify)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'async notify' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->request_cls.notify)(req, cb, ctx) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->request_cls.notify)(req, cb, ctx);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTRELEASE, FAIL, "request notify failed");
 
 done:
@@ -6562,8 +6943,14 @@ H5VL__request_cancel(void *req, const H5VL_class_t *cls, H5VL_request_status_t *
     if (NULL == cls->request_cls.cancel)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'async cancel' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->request_cls.cancel)(req, status) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->request_cls.cancel)(req, status);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTRELEASE, FAIL, "request cancel failed");
 
 done:
@@ -6663,8 +7050,14 @@ H5VL__request_specific(void *req, const H5VL_class_t *cls, H5VL_request_specific
     if (NULL == cls->request_cls.specific)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'async specific' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->request_cls.specific)(req, args) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->request_cls.specific)(req, args);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL,
                     "unable to execute asynchronous request specific callback");
 
@@ -6767,8 +7160,14 @@ H5VL__request_optional(void *req, const H5VL_class_t *cls, H5VL_optional_args_t
     if (NULL == cls->request_cls.optional)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'async optional' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->request_cls.optional)(req, args) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->request_cls.optional)(req, args);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL,
                     "unable to execute asynchronous request optional callback");
 
@@ -6907,8 +7306,14 @@ H5VL__request_free(void *req, const H5VL_class_t *cls)
     if (NULL == cls->request_cls.free)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'async free' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->request_cls.free)(req) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->request_cls.free)(req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTRELEASE, FAIL, "request free failed");
 
 done:
@@ -7009,8 +7414,14 @@ H5VL__blob_put(void *obj, const H5VL_class_t *cls, const void *buf, size_t size,
     if (NULL == cls->blob_cls.put)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'blob put' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->blob_cls.put)(obj, buf, size, blob_id, ctx) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->blob_cls.put)(obj, buf, size, blob_id, ctx);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTSET, FAIL, "blob put callback failed");
 
 done:
@@ -7103,8 +7514,14 @@ H5VL__blob_get(void *obj, const H5VL_class_t *cls, const void *blob_id, void *bu
     if (NULL == cls->blob_cls.get)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'blob get' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->blob_cls.get)(obj, blob_id, buf, size, ctx) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->blob_cls.get)(obj, blob_id, buf, size, ctx);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "blob get callback failed");
 
 done:
@@ -7196,8 +7613,14 @@ H5VL__blob_specific(void *obj, const H5VL_class_t *cls, void *blob_id, H5VL_blob
     if (NULL == cls->blob_cls.specific)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'blob specific' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->blob_cls.specific)(obj, blob_id, args) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->blob_cls.specific)(obj, blob_id, args);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL, "unable to execute blob specific callback");
 
 done:
@@ -7289,8 +7712,14 @@ H5VL__blob_optional(void *obj, const H5VL_class_t *cls, void *blob_id, H5VL_opti
     if (NULL == cls->blob_cls.optional)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'blob optional' method");
 
-    /* Call the corresponding VOL callback */
-    if ((cls->blob_cls.optional)(obj, blob_id, args) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->blob_cls.optional)(obj, blob_id, args);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL, "unable to execute blob optional callback");
 
 done:
@@ -7396,7 +7825,13 @@ H5VL__token_cmp(void *obj, const H5VL_class_t *cls, const H5O_token_t *token1, c
          * memory buffers.
          */
         if (cls->token_cls.cmp) {
-            if ((cls->token_cls.cmp)(obj, token1, token2, cmp_value) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    ret_value = (cls->token_cls.cmp)(obj, token1, token2, cmp_value);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_VOL, H5E_CANTCOMPARE, FAIL, "can't compare object tokens");
         } /* end if */
         else
@@ -7508,7 +7943,13 @@ H5VL__token_to_str(void *obj, H5I_type_t obj_type, const H5VL_class_t *cls, cons
      *  callback, otherwise just set the token_str to NULL.
      */
     if (cls->token_cls.to_str) {
-        if ((cls->token_cls.to_str)(obj, obj_type, token, token_str) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (cls->token_cls.to_str)(obj, obj_type, token, token_str);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VOL, H5E_CANTSERIALIZE, FAIL, "can't serialize object token");
     } /* end if */
     else
@@ -7614,7 +8055,13 @@ H5VL__token_from_str(void *obj, H5I_type_t obj_type, const H5VL_class_t *cls, co
      *  callback, otherwise just set the token to H5_TOKEN_UNDEF.
      */
     if (cls->token_cls.from_str) {
-        if ((cls->token_cls.from_str)(obj, obj_type, token_str, token) < 0)
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = (cls->token_cls.from_str)(obj, obj_type, token_str, token);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VOL, H5E_CANTUNSERIALIZE, FAIL, "can't deserialize object token string");
     } /* end if */
     else
@@ -7713,8 +8160,14 @@ H5VL__optional(void *obj, const H5VL_class_t *cls, H5VL_optional_args_t *args, h
     if (NULL == cls->optional)
         HGOTO_ERROR(H5E_VOL, H5E_UNSUPPORTED, FAIL, "VOL connector has no 'optional' method");
 
-    /* Call the corresponding VOL callback */
-    if ((ret_value = (cls->optional)(obj, args, dxpl_id, req)) < 0)
+    /* Prepare & restore library for user callback */
+    H5_BEFORE_USER_CB(FAIL)
+        {
+            /* Call the corresponding VOL callback */
+            ret_value = (cls->optional)(obj, args, dxpl_id, req);
+        }
+    H5_AFTER_USER_CB(FAIL)
+    if (ret_value < 0)
         HERROR(H5E_VOL, H5E_CANTOPERATE, "unable to execute optional callback");
 
 done:
diff --git a/src/H5VLint.c b/src/H5VLint.c
index 6d2864e2d91..477f5bc319a 100644
--- a/src/H5VLint.c
+++ b/src/H5VLint.c
@@ -328,8 +328,16 @@ H5VL__free_cls(H5VL_class_t *cls)
     assert(cls);
 
     /* Shut down the VOL connector */
-    if (cls->terminate && cls->terminate() < 0)
-        HGOTO_ERROR(H5E_VOL, H5E_CANTCLOSEOBJ, FAIL, "VOL connector did not terminate cleanly");
+    if (cls->terminate) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                ret_value = cls->terminate();
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
+            HGOTO_ERROR(H5E_VOL, H5E_CANTCLOSEOBJ, FAIL, "VOL connector did not terminate cleanly");
+    }
 
     /* Release the class */
     H5MM_xfree_const(cls->name);
@@ -1320,8 +1328,18 @@ H5VL__register_connector(const H5VL_class_t *cls, hid_t vipl_id)
         HGOTO_ERROR(H5E_VOL, H5E_CANTALLOC, NULL, "memory allocation failed for VOL connector name");
 
     /* Initialize the VOL connector */
-    if (cls->initialize && cls->initialize(vipl_id) < 0)
-        HGOTO_ERROR(H5E_VOL, H5E_CANTINIT, NULL, "unable to init VOL connector");
+    if (cls->initialize) {
+        herr_t status;
+
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(NULL)
+            {
+                status = cls->initialize(vipl_id);
+            }
+        H5_AFTER_USER_CB(NULL)
+        if (status < 0)
+            HGOTO_ERROR(H5E_VOL, H5E_CANTINIT, NULL, "unable to init VOL connector");
+    }
     init_done = true;
 
     /* Create new connector for the class */
@@ -1787,8 +1805,14 @@ H5VL_object_data(const H5VL_object_t *vol_obj)
     FUNC_ENTER_NOAPI_NOINIT_NOERR
 
     /* Check for 'get_object' callback in connector */
-    if (vol_obj->connector->cls->wrap_cls.get_object)
-        ret_value = (vol_obj->connector->cls->wrap_cls.get_object)(vol_obj->data);
+    if (vol_obj->connector->cls->wrap_cls.get_object) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB_NOERR(NULL)
+            {
+                ret_value = (vol_obj->connector->cls->wrap_cls.get_object)(vol_obj->data);
+            }
+        H5_AFTER_USER_CB_NOERR(NULL)
+    }
     else
         ret_value = vol_obj->data;
 
@@ -2221,11 +2245,19 @@ H5VL__free_vol_wrapper(H5VL_wrap_ctx_t *vol_wrap_ctx)
     assert(vol_wrap_ctx->connector->cls);
 
     /* If there is a VOL connector object wrapping context, release it */
-    if (vol_wrap_ctx->obj_wrap_ctx)
-        /* Release the VOL connector's object wrapping context */
-        if ((*vol_wrap_ctx->connector->cls->wrap_cls.free_wrap_ctx)(vol_wrap_ctx->obj_wrap_ctx) < 0)
+    if (vol_wrap_ctx->obj_wrap_ctx) {
+        /* Prepare & restore library for user callback */
+        H5_BEFORE_USER_CB(FAIL)
+            {
+                /* Release the VOL connector's object wrapping context */
+                ret_value =
+                    (*vol_wrap_ctx->connector->cls->wrap_cls.free_wrap_ctx)(vol_wrap_ctx->obj_wrap_ctx);
+            }
+        H5_AFTER_USER_CB(FAIL)
+        if (ret_value < 0)
             HGOTO_ERROR(H5E_VOL, H5E_CANTRELEASE, FAIL,
                         "unable to release connector's object wrapping context");
+    }
 
     /* Decrement refcount on connector */
     if (H5VL_conn_dec_rc(vol_wrap_ctx->connector) < 0)
@@ -2275,8 +2307,15 @@ H5VL_set_vol_wrapper(const H5VL_object_t *vol_obj)
             /* Sanity check */
             assert(vol_obj->connector->cls->wrap_cls.free_wrap_ctx);
 
-            /* Get the wrap context from the connector */
-            if ((vol_obj->connector->cls->wrap_cls.get_wrap_ctx)(vol_obj->data, &obj_wrap_ctx) < 0)
+            /* Prepare & restore library for user callback */
+            H5_BEFORE_USER_CB(FAIL)
+                {
+                    /* Get the wrap context from the connector */
+                    ret_value =
+                        (vol_obj->connector->cls->wrap_cls.get_wrap_ctx)(vol_obj->data, &obj_wrap_ctx);
+                }
+            H5_AFTER_USER_CB(FAIL)
+            if (ret_value < 0)
                 HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "can't retrieve VOL connector's object wrap context");
         } /* end if */
 
diff --git a/src/H5VLnative_attr.c b/src/H5VLnative_attr.c
index 7cc9c6a0f4b..c8514cf0d60 100644
--- a/src/H5VLnative_attr.c
+++ b/src/H5VLnative_attr.c
@@ -93,7 +93,7 @@ H5VL__native_attr_create(void *obj, const H5VL_loc_params_t *loc_params, const c
     if (0 == (H5F_INTENT(loc.oloc->file) & H5F_ACC_RDWR))
         HGOTO_ERROR(H5E_ARGS, H5E_WRITEERROR, NULL, "no write intent on file");
 
-    if (NULL == (plist = H5P_object_verify(aapl_id, H5P_ATTRIBUTE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(aapl_id, H5P_ATTRIBUTE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "AAPL is not an attribute access property list");
 
     if (NULL == (dt = (H5T_t *)H5I_object_verify(type_id, H5I_DATATYPE)))
@@ -154,7 +154,7 @@ H5VL__native_attr_open(void *obj, const H5VL_loc_params_t *loc_params, const cha
     if (H5G_loc_real(obj, loc_params->obj_type, &loc) < 0)
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file or file object");
 
-    if (NULL == (plist = H5P_object_verify(aapl_id, H5P_ATTRIBUTE_ACCESS)))
+    if (NULL == (plist = H5P_object_verify(aapl_id, H5P_ATTRIBUTE_ACCESS, true)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "AAPL is not an attribute access property list");
 
     if (loc_params->type == H5VL_OBJECT_BY_SELF) {
diff --git a/src/H5VLnative_dataset.c b/src/H5VLnative_dataset.c
index bff2d54d5ac..11ac2b48216 100644
--- a/src/H5VLnative_dataset.c
+++ b/src/H5VLnative_dataset.c
@@ -133,7 +133,7 @@ H5VL__native_dataset_io_setup(size_t count, void *obj[], hid_t mem_type_id[], hi
             H5S_t          *space; /* Dataspace to hold selection */
 
             /* Get the plist structure */
-            if (NULL == (plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER)))
+            if (NULL == (plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER, true)))
                 HGOTO_ERROR(H5E_DATASET, H5E_BADID, FAIL, "bad dataset transfer property list");
 
             /* Get a pointer to the file space in the property list */
@@ -205,7 +205,7 @@ H5VL__native_dataset_io_setup(size_t count, void *obj[], hid_t mem_type_id[], hi
  * Purpose:     Frees memory allocated by H5VL__native_dataset_io_setup()
  *
  * Return:      SUCCEED/FAIL
-
+ *
  *-------------------------------------------------------------------------
  */
 static herr_t
diff --git a/src/H5VLnative_private.h b/src/H5VLnative_private.h
index 15c9ae94e59..8d49a8ca871 100644
--- a/src/H5VLnative_private.h
+++ b/src/H5VLnative_private.h
@@ -47,7 +47,7 @@ H5_DLLVAR H5VL_connector_t *H5VL_NATIVE_conn_g;
 H5_DLL void  *H5VL__native_attr_create(void *obj, const H5VL_loc_params_t *loc_params, const char *attr_name,
                                        hid_t type_id, hid_t space_id, hid_t acpl_id, hid_t aapl_id,
                                        hid_t dxpl_id, void **req);
-void         *H5VL__native_attr_open(void *obj, const H5VL_loc_params_t *loc_params, const char *attr_name,
+H5_DLL void  *H5VL__native_attr_open(void *obj, const H5VL_loc_params_t *loc_params, const char *attr_name,
                                      hid_t aapl_id, hid_t dxpl_id, void **req);
 H5_DLL herr_t H5VL__native_attr_read(void *attr, hid_t dtype_id, void *buf, hid_t dxpl_id, void **req);
 H5_DLL herr_t H5VL__native_attr_write(void *attr, hid_t dtype_id, const void *buf, hid_t dxpl_id, void **req);
diff --git a/src/H5VLpublic.h b/src/H5VLpublic.h
index b114f333c6b..d5844bf2d7f 100644
--- a/src/H5VLpublic.h
+++ b/src/H5VLpublic.h
@@ -38,70 +38,94 @@
 /* VOL connector identifier values
  * These are H5VL_class_value_t values, NOT hid_t values!
  */
-/**
- * \ingroup H5VLDEF
- * Invalid ID for VOL connector ID
- */
+/** Invalid ID for VOL connector ID \since 1.12.0 */
 #define H5_VOL_INVALID (-1)
-/**
- * \ingroup H5VLDEF
- * Native HDF5 file format VOL connector
- */
+/** Native HDF5 file format VOL connector \since 1.12.0 */
 #define H5_VOL_NATIVE 0
-/**
- * \ingroup H5VLDEF
- * VOL connector IDs below this value are reserved for library use
- */
+/** VOL connector IDs below this value are reserved for library use \since 1.12.0 */
 #define H5_VOL_RESERVED 256
-/**
- * \ingroup H5VLDEF
- * Maximum VOL connector ID
- */
+/** Maximum VOL connector ID \since 1.12.0 */
 #define H5_VOL_MAX 65535
 
 /*
  * Capability flags for VOL connectors
  */
-#define H5VL_CAP_FLAG_NONE          0x0000000000000000 /**< No special connector capabilities */
-#define H5VL_CAP_FLAG_THREADSAFE    0x0000000000000001 /**< Connector is threadsafe */
-#define H5VL_CAP_FLAG_ASYNC         0x0000000000000002 /**< Connector performs operations asynchronously*/
-#define H5VL_CAP_FLAG_NATIVE_FILES  0x0000000000000004 /**< Connector produces native file format */
-#define H5VL_CAP_FLAG_ATTR_BASIC    0x0000000000000008 /**< H5A create/delete/exists/open/close/read/write */
-#define H5VL_CAP_FLAG_ATTR_MORE     0x0000000000000010 /**< All other H5A API calls */
-#define H5VL_CAP_FLAG_DATASET_BASIC 0x0000000000000020 /**< H5D create/open/close/read/write */
-#define H5VL_CAP_FLAG_DATASET_MORE  0x0000000000000040 /**< All other H5D API calls */
-#define H5VL_CAP_FLAG_FILE_BASIC    0x0000000000000080 /**< H5F create/open/close/read/write */
-#define H5VL_CAP_FLAG_FILE_MORE     0x0000000000000100 /**< All other H5F API calls */
-#define H5VL_CAP_FLAG_GROUP_BASIC   0x0000000000000200 /**< H5G create/open/close */
-#define H5VL_CAP_FLAG_GROUP_MORE    0x0000000000000400 /**< All other H5G API calls*/
-#define H5VL_CAP_FLAG_LINK_BASIC    0x0000000000000800 /**< H5L exists/delete */
-#define H5VL_CAP_FLAG_LINK_MORE     0x0000000000001000 /**< All other H5L API calls */
-#define H5VL_CAP_FLAG_MAP_BASIC                                                                              \
-    0x0000000000002000 /**< H5M create/open/close/get*type/get_count/put/get/exists/delete */
-#define H5VL_CAP_FLAG_MAP_MORE         0x0000000000004000 /**< All other H5M API calls */
-#define H5VL_CAP_FLAG_OBJECT_BASIC     0x0000000000008000 /**< H5O open/close/exists */
-#define H5VL_CAP_FLAG_OBJECT_MORE      0x0000000000010000 /**< All other H5O API calls */
-#define H5VL_CAP_FLAG_REF_BASIC        0x0000000000020000 /**< H5Rdestroy */
-#define H5VL_CAP_FLAG_REF_MORE         0x0000000000040000 /**< All other H5R API calls */
-#define H5VL_CAP_FLAG_OBJ_REF          0x0000000000080000 /**< Connector supports object references */
-#define H5VL_CAP_FLAG_REG_REF          0x0000000000100000 /**< Connector supports regional references */
-#define H5VL_CAP_FLAG_ATTR_REF         0x0000000000200000 /**< Connector supports attribute references */
-#define H5VL_CAP_FLAG_STORED_DATATYPES 0x0000000000400000 /**< Connector supports stored datatypes */
-#define H5VL_CAP_FLAG_CREATION_ORDER   0x0000000000800000 /**< Connector tracks creation order */
-#define H5VL_CAP_FLAG_ITERATE          0x0000000001000000 /**< Connector supports iteration functions */
-#define H5VL_CAP_FLAG_STORAGE_SIZE     0x0000000002000000 /**< Connector can return a meaningful storage size */
-#define H5VL_CAP_FLAG_BY_IDX           0x0000000004000000 /**< "by index" API calls are supported */
-#define H5VL_CAP_FLAG_GET_PLIST                                                                              \
-    0x0000000008000000 /**< Connector can return the property lists used to create an object */
-#define H5VL_CAP_FLAG_FLUSH_REFRESH  0x0000000010000000 /**< flush/refresh calls are supported */
-#define H5VL_CAP_FLAG_EXTERNAL_LINKS 0x0000000020000000 /**< External links are supported */
-#define H5VL_CAP_FLAG_HARD_LINKS     0x0000000040000000 /**< Hard links are supported */
-#define H5VL_CAP_FLAG_SOFT_LINKS     0x0000000080000000 /**< Soft links are supported */
-#define H5VL_CAP_FLAG_UD_LINKS       0x0000000100000000 /**< User-defined links are supported */
-#define H5VL_CAP_FLAG_TRACK_TIMES    0x0000000200000000 /**< Connector tracks creation, etc. times */
-#define H5VL_CAP_FLAG_MOUNT          0x0000000400000000 /**< H5Fmount/unmount supported */
-#define H5VL_CAP_FLAG_FILTERS        0x0000000800000000 /**< Connector implements a filter pipeline */
-#define H5VL_CAP_FLAG_FILL_VALUES    0x0000001000000000 /**< Connector allows fill values to be set */
+/** No special connector capabilities \since 1.12.0 */
+#define H5VL_CAP_FLAG_NONE 0x0000000000000000
+/** Connector is threadsafe \since 1.12.0 */
+#define H5VL_CAP_FLAG_THREADSAFE 0x0000000000000001
+/** Connector performs operations asynchronously\since 1.12.0 */
+#define H5VL_CAP_FLAG_ASYNC 0x0000000000000002
+/** Connector produces native file format \since 1.12.0 */
+#define H5VL_CAP_FLAG_NATIVE_FILES 0x0000000000000004
+/** H5A create/delete/exists/open/close/read/write \since 1.12.0 */
+#define H5VL_CAP_FLAG_ATTR_BASIC 0x0000000000000008
+/** All other H5A API calls \since 1.12.0 */
+#define H5VL_CAP_FLAG_ATTR_MORE 0x0000000000000010
+/** H5D create/open/close/read/write \since 1.12.0 */
+#define H5VL_CAP_FLAG_DATASET_BASIC 0x0000000000000020
+/** All other H5D API calls \since 1.12.0 */
+#define H5VL_CAP_FLAG_DATASET_MORE 0x0000000000000040
+/** H5F create/open/close/read/write \since 1.12.0 */
+#define H5VL_CAP_FLAG_FILE_BASIC 0x0000000000000080
+/** All other H5F API calls \since 1.12.0 */
+#define H5VL_CAP_FLAG_FILE_MORE 0x0000000000000100
+/** H5G create/open/close \since 1.12.0 */
+#define H5VL_CAP_FLAG_GROUP_BASIC 0x0000000000000200
+/** All other H5G API calls\since 1.12.0 */
+#define H5VL_CAP_FLAG_GROUP_MORE 0x0000000000000400
+/** H5L exists/delete \since 1.12.0 */
+#define H5VL_CAP_FLAG_LINK_BASIC 0x0000000000000800
+/** All other H5L API calls \since 1.12.0 */
+#define H5VL_CAP_FLAG_LINK_MORE 0x0000000000001000
+/** H5M create/open/close/get*type/get_count/put/get/exists/delete \since 1.12.0 */
+#define H5VL_CAP_FLAG_MAP_BASIC 0x0000000000002000
+/** All other H5M API calls \since 1.12.0 */
+#define H5VL_CAP_FLAG_MAP_MORE 0x0000000000004000
+/** H5O open/close/exists \since 1.12.0 */
+#define H5VL_CAP_FLAG_OBJECT_BASIC 0x0000000000008000
+/** All other H5O API calls \since 1.12.0 */
+#define H5VL_CAP_FLAG_OBJECT_MORE 0x0000000000010000
+/** H5Rdestroy \since 1.12.0 */
+#define H5VL_CAP_FLAG_REF_BASIC 0x0000000000020000
+/** All other H5R API calls \since 1.12.0 */
+#define H5VL_CAP_FLAG_REF_MORE 0x0000000000040000
+/** Connector supports object references \since 1.12.0 */
+#define H5VL_CAP_FLAG_OBJ_REF 0x0000000000080000
+/** Connector supports regional references \since 1.12.0 */
+#define H5VL_CAP_FLAG_REG_REF 0x0000000000100000
+/** Connector supports attribute references \since 1.12.0 */
+#define H5VL_CAP_FLAG_ATTR_REF 0x0000000000200000
+/** Connector supports stored datatypes \since 1.12.0 */
+#define H5VL_CAP_FLAG_STORED_DATATYPES 0x0000000000400000
+/** Connector tracks creation order \since 1.12.0 */
+#define H5VL_CAP_FLAG_CREATION_ORDER 0x0000000000800000
+/** Connector supports iteration functions \since 1.12.0 */
+#define H5VL_CAP_FLAG_ITERATE 0x0000000001000000
+/** Connector can return a meaningful storage size \since 1.12.0 */
+#define H5VL_CAP_FLAG_STORAGE_SIZE 0x0000000002000000
+/** "by index" API calls are supported \since 1.12.0 */
+#define H5VL_CAP_FLAG_BY_IDX 0x0000000004000000
+/** Connector can return the property lists used to create an object \since 1.12.0 */
+#define H5VL_CAP_FLAG_GET_PLIST 0x0000000008000000
+/** flush/refresh calls are supported \since 1.12.0 */
+#define H5VL_CAP_FLAG_FLUSH_REFRESH 0x0000000010000000
+/** External links are supported \since 1.12.0 */
+#define H5VL_CAP_FLAG_EXTERNAL_LINKS 0x0000000020000000
+/** Hard links are supported \since 1.12.0 */
+#define H5VL_CAP_FLAG_HARD_LINKS 0x0000000040000000
+/** Soft links are supported \since 1.12.0 */
+#define H5VL_CAP_FLAG_SOFT_LINKS 0x0000000080000000
+/** User-defined links are supported \since 1.12.0 */
+#define H5VL_CAP_FLAG_UD_LINKS 0x0000000100000000
+/** Connector tracks creation, etc. times \since 1.12.0 */
+#define H5VL_CAP_FLAG_TRACK_TIMES 0x0000000200000000
+/** H5Fmount/unmount supported \since 1.12.0 */
+#define H5VL_CAP_FLAG_MOUNT 0x0000000400000000
+/** Connector implements a filter pipeline \since 1.12.0 */
+#define H5VL_CAP_FLAG_FILTERS 0x0000000800000000
+/** Connector allows fill values to be set \since 1.12.0 */
+#define H5VL_CAP_FLAG_FILL_VALUES 0x0000001000000000
 
 /**
  * \ingroup H5VLDEF
@@ -116,15 +140,22 @@
  *          the connector that registered it should assume that dataset elements
  *          for _any_ dataset in the file could be written to)
  */
-#define H5VL_OPT_QUERY_SUPPORTED       0x0001 /**< VOL connector supports this operation */
-#define H5VL_OPT_QUERY_READ_DATA       0x0002 /**< Operation reads data for object */
-#define H5VL_OPT_QUERY_WRITE_DATA      0x0004 /**< Operation writes data for object */
-#define H5VL_OPT_QUERY_QUERY_METADATA  0x0008 /**< Operation reads metadata for object */
-#define H5VL_OPT_QUERY_MODIFY_METADATA 0x0010 /**< Operation modifies metadata for object */
-#define H5VL_OPT_QUERY_COLLECTIVE                                                                            \
-    0x0020 /**< Operation is collective (operations without this flag are assumed to be independent) */
-#define H5VL_OPT_QUERY_NO_ASYNC  0x0040 /**< Operation may NOT be executed asynchronously */
-#define H5VL_OPT_QUERY_MULTI_OBJ 0x0080 /**< Operation involves multiple objects */
+/** VOL connector supports this operation \since 1.14.0 */
+#define H5VL_OPT_QUERY_SUPPORTED 0x0001
+/** Operation reads data for object \since 1.14.0 */
+#define H5VL_OPT_QUERY_READ_DATA 0x0002
+/** Operation writes data for object \since 1.14.0 */
+#define H5VL_OPT_QUERY_WRITE_DATA 0x0004
+/** Operation reads metadata for object \since 1.14.0 */
+#define H5VL_OPT_QUERY_QUERY_METADATA 0x0008
+/** Operation modifies metadata for object \since 1.14.0 */
+#define H5VL_OPT_QUERY_MODIFY_METADATA 0x0010
+/** Operation is collective (operations without this flag are assumed to be independent) \since 1.14.0 */
+#define H5VL_OPT_QUERY_COLLECTIVE 0x0020
+/** Operation may NOT be executed asynchronously \since 1.14.0 */
+#define H5VL_OPT_QUERY_NO_ASYNC 0x0040
+/** Operation involves multiple objects \since 1.14.0 */
+#define H5VL_OPT_QUERY_MULTI_OBJ 0x0080
 
 /*******************/
 /* Public Typedefs */
diff --git a/src/H5Z.c b/src/H5Z.c
index 4ad75a6668c..749652ce451 100644
--- a/src/H5Z.c
+++ b/src/H5Z.c
@@ -469,7 +469,7 @@ H5Z__check_unregister(hid_t ocpl_id, H5Z_filter_t filter_id)
     FUNC_ENTER_PACKAGE
 
     /* Get the plist structure of object creation */
-    if (NULL == (plist = H5P_object_verify(ocpl_id, H5P_OBJECT_CREATE)))
+    if (NULL == (plist = H5P_object_verify(ocpl_id, H5P_OBJECT_CREATE, true)))
         HGOTO_ERROR(H5E_PLINE, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Check if the object creation property list uses the filter */
@@ -797,8 +797,15 @@ H5Z__prelude_callback(const H5O_pline_t *pline, hid_t dcpl_id, hid_t type_id, hi
 
                     /* Check if there is a "can apply" callback */
                     if (fclass->can_apply) {
-                        /* Make callback to filter's "can apply" function */
-                        htri_t status = (fclass->can_apply)(dcpl_id, type_id, space_id);
+                        htri_t status;
+
+                        /* Prepare & restore library for user callback */
+                        H5_BEFORE_USER_CB(FAIL)
+                            {
+                                /* Make callback to filter's "can apply" function */
+                                status = (fclass->can_apply)(dcpl_id, type_id, space_id);
+                            }
+                        H5_AFTER_USER_CB(FAIL)
 
                         /* Indicate error during filter callback */
                         if (status < 0)
@@ -814,9 +821,18 @@ H5Z__prelude_callback(const H5O_pline_t *pline, hid_t dcpl_id, hid_t type_id, hi
                 case H5Z_PRELUDE_SET_LOCAL:
                     /* Check if there is a "set local" callback */
                     if (fclass->set_local) {
-                        /* Make callback to filter's "set local" function */
-                        if ((fclass->set_local)(dcpl_id, type_id, space_id) < 0)
-                            /* Indicate error during filter callback */
+                        herr_t status;
+
+                        /* Prepare & restore library for user callback */
+                        H5_BEFORE_USER_CB(FAIL)
+                            {
+                                /* Make callback to filter's "set local" function */
+                                status = (fclass->set_local)(dcpl_id, type_id, space_id);
+                            }
+                        H5_AFTER_USER_CB(FAIL)
+
+                        /* Indicate error during filter callback */
+                        if (status < 0)
                             HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "error during user callback");
                     } /* end if */
                     break;
@@ -1373,6 +1389,8 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, unsigned *filter_mask /*i
     assert(buf && *buf);
     assert(!pline || pline->nused < H5Z_MAX_NFILTERS);
 
+    /* clang-format off */
+
 #ifdef H5Z_DEBUG
     H5_timer_init(&timer);
 #endif
@@ -1430,11 +1448,16 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, unsigned *filter_mask /*i
 
             tmp_flags = flags | (pline->filter[idx].flags);
             tmp_flags |= (edc_read == H5Z_DISABLE_EDC) ? H5Z_FLAG_SKIP_EDC : 0;
+
             H5E_PAUSE_ERRORS
-            {
-                new_nbytes = (fclass->filter)(tmp_flags, pline->filter[idx].cd_nelmts,
-                                              pline->filter[idx].cd_values, *nbytes, buf_size, buf);
-            }
+                {/* Prepare & restore library for user callback */
+                    H5_BEFORE_USER_CB(FAIL)
+                        {
+                            new_nbytes = (fclass->filter)(tmp_flags, pline->filter[idx].cd_nelmts,
+                                                          pline->filter[idx].cd_values, *nbytes, buf_size, buf);
+                        }
+                    H5_AFTER_USER_CB(FAIL)
+                }
             H5E_RESUME_ERRORS
 
 #ifdef H5Z_DEBUG
@@ -1450,9 +1473,19 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, unsigned *filter_mask /*i
 #endif
 
             if (0 == new_nbytes) {
-                if ((cb_struct.func && (H5Z_CB_FAIL == cb_struct.func(pline->filter[idx].id, *buf, *buf_size,
-                                                                      cb_struct.op_data))) ||
-                    !cb_struct.func)
+                if (cb_struct.func) {
+                    H5Z_cb_return_t status;
+
+                    /* Prepare & restore library for user callback */
+                    H5_BEFORE_USER_CB(FAIL)
+                        {
+                            status = cb_struct.func(pline->filter[idx].id, *buf, *buf_size, cb_struct.op_data);
+                        }
+                    H5_AFTER_USER_CB(FAIL)
+                    if (H5Z_CB_FAIL == status)
+                        HGOTO_ERROR(H5E_PLINE, H5E_READERROR, FAIL, "filter returned failure during read");
+                }
+                else
                     HGOTO_ERROR(H5E_PLINE, H5E_READERROR, FAIL, "filter returned failure during read");
 
                 *nbytes = *buf_size;
@@ -1462,7 +1495,8 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, unsigned *filter_mask /*i
                 *nbytes = new_nbytes;
         }
     }
-    else if (pline) { /* Write */
+    else if (pline)
+    { /* Write */
         for (idx = 0; idx < pline->nused; idx++) {
             if (*filter_mask & ((unsigned)1 << idx)) {
                 failed |= (unsigned)1 << idx;
@@ -1484,10 +1518,14 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, unsigned *filter_mask /*i
 #endif
 
             H5E_PAUSE_ERRORS
-            {
-                new_nbytes = (fclass->filter)(flags | pline->filter[idx].flags, pline->filter[idx].cd_nelmts,
-                                              pline->filter[idx].cd_values, *nbytes, buf_size, buf);
-            }
+                {/* Prepare & restore library for user callback */
+                    H5_BEFORE_USER_CB(FAIL)
+                        {
+                            new_nbytes = (fclass->filter)(flags | (pline->filter[idx].flags), pline->filter[idx].cd_nelmts,
+                                                          pline->filter[idx].cd_values, *nbytes, buf_size, buf);
+                        }
+                    H5_AFTER_USER_CB(FAIL)
+                }
             H5E_RESUME_ERRORS
 
 #ifdef H5Z_DEBUG
@@ -1504,9 +1542,19 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, unsigned *filter_mask /*i
 
             if (0 == new_nbytes) {
                 if (0 == (pline->filter[idx].flags & H5Z_FLAG_OPTIONAL)) {
-                    if ((cb_struct.func && (H5Z_CB_FAIL == cb_struct.func(pline->filter[idx].id, *buf,
-                                                                          *nbytes, cb_struct.op_data))) ||
-                        !cb_struct.func)
+                    if (cb_struct.func) {
+                        H5Z_cb_return_t status;
+
+                        /* Prepare & restore library for user callback */
+                        H5_BEFORE_USER_CB(FAIL)
+                            {
+                                status = cb_struct.func(pline->filter[idx].id, *buf, *nbytes, cb_struct.op_data);
+                            }
+                        H5_AFTER_USER_CB(FAIL)
+                        if (H5Z_CB_FAIL == status)
+                            HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "filter returned failure");
+                    }
+                    else
                         HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "filter returned failure");
 
                     *nbytes = *buf_size;
@@ -1522,6 +1570,8 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, unsigned *filter_mask /*i
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
+
+/* clang-format on */
 }
 
 /*-------------------------------------------------------------------------
diff --git a/src/H5Zmodule.h b/src/H5Zmodule.h
index d8e01676d6f..4458481d36f 100644
--- a/src/H5Zmodule.h
+++ b/src/H5Zmodule.h
@@ -130,6 +130,8 @@
  * \ingroup H5ZPRE
  * \defgroup SZIP Szip Filter
  * \ingroup H5ZPRE
+ * \defgroup NBIT N-bit Filter
+ * \ingroup H5ZPRE
  *
  */
 
diff --git a/src/H5Znbit.c b/src/H5Znbit.c
index f1ffd2480ec..fed86a1a3bd 100644
--- a/src/H5Znbit.c
+++ b/src/H5Znbit.c
@@ -825,7 +825,7 @@ H5Z__set_local_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id)
         HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for cd_values[]");
 
     /* Get the plist structure */
-    if (NULL == (dcpl_plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE)))
+    if (NULL == (dcpl_plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get the filter's current parameters */
diff --git a/src/H5Zscaleoffset.c b/src/H5Zscaleoffset.c
index 4330b4a6cb1..8121b3baf99 100644
--- a/src/H5Zscaleoffset.c
+++ b/src/H5Zscaleoffset.c
@@ -938,7 +938,7 @@ H5Z__set_local_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id)
     FUNC_ENTER_PACKAGE
 
     /* Get the plist structure */
-    if (NULL == (dcpl_plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE)))
+    if (NULL == (dcpl_plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get datatype */
diff --git a/src/H5Zshuffle.c b/src/H5Zshuffle.c
index acc864f774c..8ee35f67afb 100644
--- a/src/H5Zshuffle.c
+++ b/src/H5Zshuffle.c
@@ -64,7 +64,7 @@ H5Z__set_local_shuffle(hid_t dcpl_id, hid_t type_id, hid_t H5_ATTR_UNUSED space_
     FUNC_ENTER_PACKAGE
 
     /* Get the plist structure */
-    if (NULL == (dcpl_plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE)))
+    if (NULL == (dcpl_plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get datatype */
diff --git a/src/H5Zszip.c b/src/H5Zszip.c
index a5e961d3b6f..9b37035374f 100644
--- a/src/H5Zszip.c
+++ b/src/H5Zszip.c
@@ -130,7 +130,7 @@ H5Z__set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id)
     FUNC_ENTER_PACKAGE
 
     /* Get the plist structure */
-    if (NULL == (dcpl_plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE)))
+    if (NULL == (dcpl_plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE, false)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID");
 
     /* Get datatype */
diff --git a/src/H5build_settings.autotools.c.in b/src/H5build_settings.autotools.c.in
index 8fd583de62f..36547212e54 100644
--- a/src/H5build_settings.autotools.c.in
+++ b/src/H5build_settings.autotools.c.in
@@ -95,6 +95,7 @@ const char H5build_settings[]=
     "                  Build HDF5 Tools: @HDF5_TOOLS@\n"
     "                           Threads: @THREADS@\n"
     "                      Threadsafety: @THREADSAFE@\n"
+    "                       Concurrency: @CONCURRENCY@\n"
     "               Default API mapping: @DEFAULT_API_VERSION@\n"
     "    With deprecated public symbols: @DEPRECATED_SYMBOLS@\n"
     "            I/O filters (external): @EXTERNAL_FILTERS@\n"
diff --git a/src/H5build_settings.cmake.c.in b/src/H5build_settings.cmake.c.in
index 9530fbc822e..ecbd08eb666 100644
--- a/src/H5build_settings.cmake.c.in
+++ b/src/H5build_settings.cmake.c.in
@@ -94,6 +94,7 @@ const char H5build_settings[]=
     "                  Build HDF5 Tests: @BUILD_TESTING@\n"
     "                  Build HDF5 Tools: @HDF5_BUILD_TOOLS@\n"
     "                      Threadsafety: @HDF5_ENABLE_THREADSAFE@\n"
+    "                       Concurrency: @HDF5_ENABLE_CONCURRENCY@\n"
     "               Default API mapping: @HDF5_DEFAULT_API_VERSION@\n"
     "    With deprecated public symbols: @HDF5_ENABLE_DEPRECATED_SYMBOLS@\n"
     "            I/O filters (external): @EXTERNAL_FILTERS@\n"
diff --git a/src/H5module.h b/src/H5module.h
index 35429b57a15..a5a448af6bf 100644
--- a/src/H5module.h
+++ b/src/H5module.h
@@ -593,7 +593,7 @@
  * 
  * 
  * 
  * 
  * 

- * \image html Dmodel_fig14_c.gif " Another HDF5 file structure with groups and datasets"
+ * \image html Dmodel_fig14_d.gif " Another HDF5 file structure with groups and datasets"
  * 

diff --git a/src/H5private.h b/src/H5private.h
index cff6b37390c..8950b541bed 100644
--- a/src/H5private.h
+++ b/src/H5private.h
@@ -1000,6 +1000,48 @@ extern H5_debug_t H5_debug_g;
 /* Embedded build information */
 extern const char H5build_settings[];
 
+/* Prepare to call / return from user callback */
+#include "H5Eprivate.h"
+typedef struct H5_user_cb_state_t {
+    H5E_user_cb_state_t h5e_state; /* State for H5E package */
+} H5_user_cb_state_t;
+
+#define H5_BEFORE_USER_CB(err)                                                                               \
+    {                                                                                                        \
+        H5_user_cb_state_t state;                                                                            \
+                                                                                                             \
+        if (H5_user_cb_prepare(&state) < 0)                                                                  \
+            HGOTO_ERROR(H5E_LIB, H5E_CANTSET, (err), "preparation for user callback failed");
+
+#define H5_AFTER_USER_CB(err)                                                                                \
+    if (H5_user_cb_restore(&state) < 0)                                                                      \
+        HGOTO_ERROR(H5E_LIB, H5E_CANTRESTORE, (err), "preparation for user callback failed");                \
+    }
+
+#define H5_BEFORE_USER_CB_NOERR(err)                                                                         \
+    {                                                                                                        \
+        H5_user_cb_state_t state;                                                                            \
+                                                                                                             \
+        if (H5_user_cb_prepare(&state) < 0)                                                                  \
+            ret_value = (err);                                                                               \
+        else {
+
+#define H5_AFTER_USER_CB_NOERR(err)                                                                          \
+    if (H5_user_cb_restore(&state) < 0)                                                                      \
+        ret_value = (err);                                                                                   \
+    } /* end else */                                                                                         \
+    }
+
+#define H5_BEFORE_USER_CB_NOCHECK                                                                            \
+    {                                                                                                        \
+        H5_user_cb_state_t state;                                                                            \
+                                                                                                             \
+        H5_user_cb_prepare(&state);
+
+#define H5_AFTER_USER_CB_NOCHECK                                                                             \
+    H5_user_cb_restore(&state);                                                                              \
+    }
+
 /*-------------------------------------------------------------------------
  * Purpose: These macros are used to track arguments in event sets and are
  *          inserted automatically into H5ES_insert() by the bin/trace script
@@ -1056,7 +1098,14 @@ H5_DLL herr_t H5_trace_args(struct H5RS_str_t *rs, const char *type, va_list ap)
 /* global library version information string */
 extern char H5_lib_vers_info_g[];
 
-#ifdef H5_HAVE_THREADSAFE
+/* Both the 'threadsafe' and 'concurrency' options provide threadsafely for
+ * API calls.
+ */
+#if defined(H5_HAVE_THREADSAFE) || defined(H5_HAVE_CONCURRENCY)
+#define H5_HAVE_THREADSAFE_API
+#endif
+
+#ifdef H5_HAVE_THREADSAFE_API
 
 /* Lock headers */
 #include "H5TSprivate.h"
@@ -1078,13 +1127,21 @@ extern char H5_lib_vers_info_g[];
     } while (0)
 #else
 /* Local variable for saving cancellation state */
-#define H5CANCEL_DECL       /* */
+#define H5CANCEL_DECL /* */
 
 /* Disable & restore canceling the thread */
-#define H5TS_DISABLE_CANCEL /* */
-#define H5TS_RESTORE_CANCEL /* */
+#define H5TS_DISABLE_CANCEL                                                                                  \
+    do {                                                                                                     \
+    } while (0) /* no-op */
+#define H5TS_RESTORE_CANCEL                                                                                  \
+    do {                                                                                                     \
+    } while (0) /* no-op */
 #endif
 
+#ifdef H5_HAVE_THREADSAFE
+/* Local variable for 'disable locking for this thread' (DLFTT) state */
+#define H5DLFTT_DECL /* */
+
 /* Macros for entering & leaving an API routine in a threadsafe manner */
 #define H5_API_LOCK                                                                                          \
     /* Acquire the API lock */                                                                               \
@@ -1098,16 +1155,40 @@ extern char H5_lib_vers_info_g[];
                                                                                                              \
     /* Restore previous thread cancellation state */                                                         \
     H5TS_RESTORE_CANCEL;
-#else                 /* H5_HAVE_THREADSAFE */
+#else /* H5_HAVE_CONCURRENCY */
+/* Local variable for 'disable locking for this thread' (DLFTT) state */
+#define H5DLFTT_DECL unsigned dlftt = 0;
+
+/* Macros for entering & leaving an API routine in a threadsafe manner */
+#define H5_API_LOCK                                                                                          \
+    /* Acquire the API lock */                                                                               \
+    H5TS_api_lock(&dlftt);                                                                                   \
+                                                                                                             \
+    /* Set thread cancellation state to 'disable', and remember previous state */                            \
+    if (0 == dlftt)                                                                                          \
+        H5TS_DISABLE_CANCEL;
+#define H5_API_UNLOCK                                                                                        \
+    if (0 == dlftt) {                                                                                        \
+        /* Release the API lock */                                                                           \
+        H5TS_api_unlock();                                                                                   \
+                                                                                                             \
+        /* Restore previous thread cancellation state */                                                     \
+        H5TS_RESTORE_CANCEL;                                                                                 \
+    }
+#endif
+#else                 /* H5_HAVE_THREADSAFE_API */
 
 /* Local variable for saving cancellation state */
 #define H5CANCEL_DECL /* */
 
+/* Local variable for 'disable locking for this thread' (DLFTT) state */
+#define H5DLFTT_DECL  /* */
+
 /* No locks (non-threadsafe builds) */
-#define H5_API_LOCK   /* */
-#define H5_API_UNLOCK /* */
+#define H5_API_LOCK   /* no-op */
+#define H5_API_UNLOCK /* no-op */
 
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
 /* Macros for accessing the global variables */
 #define H5_INIT_GLOBAL (H5_libinit_g)
@@ -1239,7 +1320,8 @@ extern char H5_lib_vers_info_g[];
 
 /* Entry setup for public API call variables */
 #define H5_API_SETUP_PUBLIC_API_VARS                                                                         \
-    H5CANCEL_DECL /* thread cancellation */
+    H5CANCEL_DECL /* thread cancellation */                 \
+    H5DLFTT_DECL  /* user callback protection */
 
 /* Macro to initialize the library, if some other package hasn't already done that */
 #define H5_API_SETUP_INIT_LIBRARY(err)                                                                       \
@@ -1329,7 +1411,7 @@ extern char H5_lib_vers_info_g[];
 /*
  * Use this macro for public API functions that shouldn't perform _any_
  * initialization of the library or an interface or push themselves on the
- * function stack, just perform tracing, etc. Examples are: H5close,
+ * function stack, just perform tracing, etc. Examples are: H5dont_atexit,
  * H5check_version, etc.
  */
 #define FUNC_ENTER_API_NOINIT_NOERR                                                                          \
@@ -1791,4 +1873,7 @@ H5_DLL herr_t  H5_mpio_get_file_sync_required(MPI_File fh, bool *file_sync_requi
 H5_DLL herr_t H5_buffer_dump(FILE *stream, int indent, const uint8_t *buf, const uint8_t *marker,
                              size_t buf_offset, size_t buf_size);
 
+/* Functions for preparing for / returning from user callbacks */
+H5_DLL herr_t H5_user_cb_prepare(H5_user_cb_state_t *state);
+H5_DLL herr_t H5_user_cb_restore(const H5_user_cb_state_t *state);
 #endif /* H5private_H */
diff --git a/src/H5public.h b/src/H5public.h
index 62e0db9764b..a49a4b34ad1 100644
--- a/src/H5public.h
+++ b/src/H5public.h
@@ -327,15 +327,22 @@ typedef off_t HDoff_t;
  * should be discouraged in new code.
  */
 typedef int64_t hssize_t;
-#define PRIdHSIZE          PRId64
-#define PRIiHSIZE          PRIi64
-#define PRIoHSIZE          PRIo64
-#define PRIuHSIZE          PRIu64
-#define PRIxHSIZE          PRIx64
+/** d print conversion specifier for signed integer type \since 1.10.8 */
+#define PRIdHSIZE PRId64
+/** i print conversion specifier for signed integer type \since 1.10.8 */
+#define PRIiHSIZE PRIi64
+/** o print conversion specifier for signed integer type \since 1.10.8 */
+#define PRIoHSIZE PRIo64
+/** u print conversion specifier for signed integer type \since 1.10.8 */
+#define PRIuHSIZE PRIu64
+/** x print conversion specifier for signed integer type \since 1.10.8 */
+#define PRIxHSIZE PRIx64
+/** X print conversion specifier for signed integer type \since 1.10.8 */
 #define PRIXHSIZE          PRIX64
 #define H5_SIZEOF_HSIZE_T  8
 #define H5_SIZEOF_HSSIZE_T 8
-#define HSIZE_UNDEF        UINT64_MAX
+/** Represents the largest possible value of uint64_t \since 1.10.0 */
+#define HSIZE_UNDEF UINT64_MAX
 
 /**
  * The address of an object in the file.
@@ -343,10 +350,15 @@ typedef int64_t hssize_t;
  * \internal Defined as a (minimum) 64-bit unsigned integer type.
  */
 typedef uint64_t haddr_t;
-#define PRIdHADDR           PRId64
-#define PRIoHADDR           PRIo64
-#define PRIuHADDR           PRIu64
-#define PRIxHADDR           PRIx64
+/** d print conversion specifier for unsigned integer type \since 1.8.23 */
+#define PRIdHADDR PRId64
+/** o print conversion specifier for unsigned integer type \since 1.8.23 */
+#define PRIoHADDR PRIo64
+/** u print conversion specifier for unsigned integer type \since 1.8.23 */
+#define PRIuHADDR PRIu64
+/** x print conversion specifier for unsigned integer type \since 1.8.23 */
+#define PRIxHADDR PRIx64
+/** X print conversion specifier for unsigned integer type \since 1.8.23 */
 #define PRIXHADDR           PRIX64
 #define H5_SIZEOF_HADDR_T   8
 #define HADDR_UNDEF         UINT64_MAX
@@ -407,6 +419,8 @@ typedef struct H5_ih_info_t {
  * \details Tokens are unique and permanent identifiers that are
  *          used to reference HDF5 objects in a container. This allows
  *          for 128-bit tokens
+ *
+ * \since 1.12.0
  */
 #define H5O_MAX_TOKEN_SIZE (16)
 
diff --git a/src/H5vers.txt b/src/H5vers.txt
index c2b0a45886b..ce17f889c7b 100644
--- a/src/H5vers.txt
+++ b/src/H5vers.txt
@@ -39,7 +39,7 @@
 
 # API function names
 # (although not required, it's easier to compare this file with the headers
-#       generated if the list below is in alphanumeric sort order - QAK)
+#       generated if the list below is in alphanumeric sort order)
 FUNCTION: H5Acreate; ; v10, v18
 FUNCTION: H5Aiterate; H5A_operator; v10, v18
 FUNCTION: H5Dcreate; ; v10, v18
@@ -53,6 +53,7 @@ FUNCTION: H5Ewalk; H5E_walk, H5E_error; v10, v18
 FUNCTION: H5Fget_info; H5F_info; v18, v110
 FUNCTION: H5Gcreate; ; v10, v18
 FUNCTION: H5Gopen; ; v10, v18
+FUNCTION: H5Iregister_type; ; v18, v200
 FUNCTION: H5Lget_info; H5L_info; v18, v112
 FUNCTION: H5Lget_info_by_idx; H5L_info; v18, v112
 FUNCTION: H5Literate; H5L_iterate; v18, v112
@@ -88,7 +89,7 @@ FUNCTION: H5Topen; ; v10, v18
 
 # API typedefs
 # (although not required, it's easier to compare this file with the headers
-#       generated if the list below is in alphanumeric sort order - QAK)
+#       generated if the list below is in alphanumeric sort order)
 TYPEDEF: H5E_auto; v10, v18
 TYPEDEF: H5O_info; v18, v112
 TYPEDEF: H5O_iterate; v18, v112
diff --git a/src/H5version.h b/src/H5version.h
index 089e6e9f2de..9fe80159695 100644
--- a/src/H5version.h
+++ b/src/H5version.h
@@ -43,6 +43,10 @@
   #define H5_USE_114_API 1
 #endif /* H5_USE_114_API_DEFAULT && !H5_USE_114_API */
 
+#if defined(H5_USE_200_API_DEFAULT) && !defined(H5_USE_200_API)
+  #define H5_USE_200_API 1
+#endif /* H5_USE_200_API_DEFAULT && !H5_USE_200_API */
+
 
 /* Issue error if contradicting macros have been defined. */
 /* (Can't use an older (deprecated) API version if deprecated symbols have been disabled) */
@@ -224,6 +228,10 @@
   #define H5Gopen_vers 2
 #endif /* !defined(H5Gopen_vers) */
 
+#if !defined(H5Iregister_type_vers)
+  #define H5Iregister_type_vers 1
+#endif /* !defined(H5Iregister_type_vers) */
+
 #if !defined(H5Lget_info_vers)
   #define H5Lget_info_vers 1
 #endif /* !defined(H5Lget_info_vers) */
@@ -392,6 +400,10 @@
   #define H5Gopen_vers 2
 #endif /* !defined(H5Gopen_vers) */
 
+#if !defined(H5Iregister_type_vers)
+  #define H5Iregister_type_vers 1
+#endif /* !defined(H5Iregister_type_vers) */
+
 #if !defined(H5Lget_info_vers)
   #define H5Lget_info_vers 1
 #endif /* !defined(H5Lget_info_vers) */
@@ -564,6 +576,10 @@
   #define H5Gopen_vers 2
 #endif /* !defined(H5Gopen_vers) */
 
+#if !defined(H5Iregister_type_vers)
+  #define H5Iregister_type_vers 1
+#endif /* !defined(H5Iregister_type_vers) */
+
 #if !defined(H5Lget_info_vers)
   #define H5Lget_info_vers 2
 #endif /* !defined(H5Lget_info_vers) */
@@ -736,6 +752,10 @@
   #define H5Gopen_vers 2
 #endif /* !defined(H5Gopen_vers) */
 
+#if !defined(H5Iregister_type_vers)
+  #define H5Iregister_type_vers 1
+#endif /* !defined(H5Iregister_type_vers) */
+
 #if !defined(H5Lget_info_vers)
   #define H5Lget_info_vers 2
 #endif /* !defined(H5Lget_info_vers) */
@@ -850,6 +870,182 @@
 
 #endif /* H5_USE_114_API */
 
+#ifdef H5_USE_200_API
+
+/*************/
+/* Functions */
+/*************/
+
+#if !defined(H5Acreate_vers)
+  #define H5Acreate_vers 2
+#endif /* !defined(H5Acreate_vers) */
+
+#if !defined(H5Aiterate_vers)
+  #define H5Aiterate_vers 2
+#endif /* !defined(H5Aiterate_vers) */
+
+#if !defined(H5Dcreate_vers)
+  #define H5Dcreate_vers 2
+#endif /* !defined(H5Dcreate_vers) */
+
+#if !defined(H5Dopen_vers)
+  #define H5Dopen_vers 2
+#endif /* !defined(H5Dopen_vers) */
+
+#if !defined(H5Eclear_vers)
+  #define H5Eclear_vers 2
+#endif /* !defined(H5Eclear_vers) */
+
+#if !defined(H5Eget_auto_vers)
+  #define H5Eget_auto_vers 2
+#endif /* !defined(H5Eget_auto_vers) */
+
+#if !defined(H5Eprint_vers)
+  #define H5Eprint_vers 2
+#endif /* !defined(H5Eprint_vers) */
+
+#if !defined(H5Epush_vers)
+  #define H5Epush_vers 2
+#endif /* !defined(H5Epush_vers) */
+
+#if !defined(H5Eset_auto_vers)
+  #define H5Eset_auto_vers 2
+#endif /* !defined(H5Eset_auto_vers) */
+
+#if !defined(H5Ewalk_vers)
+  #define H5Ewalk_vers 2
+#endif /* !defined(H5Ewalk_vers) */
+
+#if !defined(H5Fget_info_vers)
+  #define H5Fget_info_vers 2
+#endif /* !defined(H5Fget_info_vers) */
+
+#if !defined(H5Gcreate_vers)
+  #define H5Gcreate_vers 2
+#endif /* !defined(H5Gcreate_vers) */
+
+#if !defined(H5Gopen_vers)
+  #define H5Gopen_vers 2
+#endif /* !defined(H5Gopen_vers) */
+
+#if !defined(H5Iregister_type_vers)
+  #define H5Iregister_type_vers 2
+#endif /* !defined(H5Iregister_type_vers) */
+
+#if !defined(H5Lget_info_vers)
+  #define H5Lget_info_vers 2
+#endif /* !defined(H5Lget_info_vers) */
+
+#if !defined(H5Lget_info_by_idx_vers)
+  #define H5Lget_info_by_idx_vers 2
+#endif /* !defined(H5Lget_info_by_idx_vers) */
+
+#if !defined(H5Literate_vers)
+  #define H5Literate_vers 2
+#endif /* !defined(H5Literate_vers) */
+
+#if !defined(H5Literate_by_name_vers)
+  #define H5Literate_by_name_vers 2
+#endif /* !defined(H5Literate_by_name_vers) */
+
+#if !defined(H5Lvisit_vers)
+  #define H5Lvisit_vers 2
+#endif /* !defined(H5Lvisit_vers) */
+
+#if !defined(H5Lvisit_by_name_vers)
+  #define H5Lvisit_by_name_vers 2
+#endif /* !defined(H5Lvisit_by_name_vers) */
+
+#if !defined(H5Oget_info_vers)
+  #define H5Oget_info_vers 3
+#endif /* !defined(H5Oget_info_vers) */
+
+#if !defined(H5Oget_info_by_idx_vers)
+  #define H5Oget_info_by_idx_vers 3
+#endif /* !defined(H5Oget_info_by_idx_vers) */
+
+#if !defined(H5Oget_info_by_name_vers)
+  #define H5Oget_info_by_name_vers 3
+#endif /* !defined(H5Oget_info_by_name_vers) */
+
+#if !defined(H5Ovisit_vers)
+  #define H5Ovisit_vers 3
+#endif /* !defined(H5Ovisit_vers) */
+
+#if !defined(H5Ovisit_by_name_vers)
+  #define H5Ovisit_by_name_vers 3
+#endif /* !defined(H5Ovisit_by_name_vers) */
+
+#if !defined(H5Pencode_vers)
+  #define H5Pencode_vers 2
+#endif /* !defined(H5Pencode_vers) */
+
+#if !defined(H5Pget_filter_vers)
+  #define H5Pget_filter_vers 2
+#endif /* !defined(H5Pget_filter_vers) */
+
+#if !defined(H5Pget_filter_by_id_vers)
+  #define H5Pget_filter_by_id_vers 2
+#endif /* !defined(H5Pget_filter_by_id_vers) */
+
+#if !defined(H5Pinsert_vers)
+  #define H5Pinsert_vers 2
+#endif /* !defined(H5Pinsert_vers) */
+
+#if !defined(H5Pregister_vers)
+  #define H5Pregister_vers 2
+#endif /* !defined(H5Pregister_vers) */
+
+#if !defined(H5Rdereference_vers)
+  #define H5Rdereference_vers 2
+#endif /* !defined(H5Rdereference_vers) */
+
+#if !defined(H5Rget_obj_type_vers)
+  #define H5Rget_obj_type_vers 2
+#endif /* !defined(H5Rget_obj_type_vers) */
+
+#if !defined(H5Sencode_vers)
+  #define H5Sencode_vers 2
+#endif /* !defined(H5Sencode_vers) */
+
+#if !defined(H5Tarray_create_vers)
+  #define H5Tarray_create_vers 2
+#endif /* !defined(H5Tarray_create_vers) */
+
+#if !defined(H5Tcommit_vers)
+  #define H5Tcommit_vers 2
+#endif /* !defined(H5Tcommit_vers) */
+
+#if !defined(H5Tget_array_dims_vers)
+  #define H5Tget_array_dims_vers 2
+#endif /* !defined(H5Tget_array_dims_vers) */
+
+#if !defined(H5Topen_vers)
+  #define H5Topen_vers 2
+#endif /* !defined(H5Topen_vers) */
+
+/************/
+/* Typedefs */
+/************/
+
+#if !defined(H5E_auto_t_vers)
+  #define H5E_auto_t_vers 2
+#endif /* !defined(H5E_auto_t_vers) */
+
+#if !defined(H5O_info_t_vers)
+  #define H5O_info_t_vers 2
+#endif /* !defined(H5O_info_t_vers) */
+
+#if !defined(H5O_iterate_t_vers)
+  #define H5O_iterate_t_vers 2
+#endif /* !defined(H5O_iterate_t_vers) */
+
+#if !defined(H5Z_class_t_vers)
+  #define H5Z_class_t_vers 2
+#endif /* !defined(H5Z_class_t_vers) */
+
+#endif /* H5_USE_200_API */
+
 
 /* Choose the correct version of each API symbol, defaulting to the latest
  *      version of each.  The "best" name for API parameters/data structures
@@ -1012,6 +1208,17 @@
   #error "H5Gopen_vers set to invalid value"
 #endif /* H5Gopen_vers */
 
+#if !defined(H5Iregister_type_vers) || H5Iregister_type_vers == 2
+  #ifndef H5Iregister_type_vers
+    #define H5Iregister_type_vers 2
+  #endif /* H5Iregister_type_vers */
+  #define H5Iregister_type H5Iregister_type2
+#elif H5Iregister_type_vers == 1
+  #define H5Iregister_type H5Iregister_type1
+#else /* H5Iregister_type_vers */
+  #error "H5Iregister_type_vers set to invalid value"
+#endif /* H5Iregister_type_vers */
+
 #if !defined(H5Lget_info_vers) || H5Lget_info_vers == 2
   #ifndef H5Lget_info_vers
     #define H5Lget_info_vers 2
diff --git a/src/Makefile.am b/src/Makefile.am
index fa5d4536764..75cc60591b8 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -67,7 +67,7 @@ libhdf5_la_SOURCES= H5.c H5build_settings.c H5checksum.c H5dbg.c H5system.c \
         H5HFspace.c H5HFstat.c H5HFtest.c H5HFtiny.c \
         H5HG.c H5HGcache.c H5HGdbg.c H5HGquery.c \
         H5HL.c H5HLcache.c H5HLdbg.c H5HLint.c H5HLprfx.c H5HLdblk.c \
-        H5I.c H5Idbg.c H5Iint.c H5Itest.c \
+        H5I.c H5Idbg.c H5Ideprec.c H5Iint.c H5Itest.c \
         H5L.c H5Ldeprec.c H5Lexternal.c H5Lint.c \
         H5M.c \
         H5MF.c H5MFaggr.c H5MFdbg.c H5MFsection.c \
diff --git a/src/libhdf5.settings.autotools.in b/src/libhdf5.settings.autotools.in
index e7900a19dfc..77c1b986eb2 100644
--- a/src/libhdf5.settings.autotools.in
+++ b/src/libhdf5.settings.autotools.in
@@ -77,6 +77,7 @@ Dimension scales w/ new references: @DIMENSION_SCALES_WITH_NEW_REF@
                   Build HDF5 Tools: @HDF5_TOOLS@
                            Threads: @THREADS@
                       Threadsafety: @THREADSAFE@
+                       Concurrency: @CONCURRENCY@
                Default API mapping: @DEFAULT_API_VERSION@
     With deprecated public symbols: @DEPRECATED_SYMBOLS@
             I/O filters (external): @EXTERNAL_FILTERS@
diff --git a/src/libhdf5.settings.cmake.in b/src/libhdf5.settings.cmake.in
index 236e1ebb7f7..e6c8d3a7903 100644
--- a/src/libhdf5.settings.cmake.in
+++ b/src/libhdf5.settings.cmake.in
@@ -75,6 +75,7 @@ Dimension scales w/ new references: @DIMENSION_SCALES_WITH_NEW_REF@
                   Build HDF5 Tools: @HDF5_BUILD_TOOLS@
                            Threads: @HDF5_ENABLE_THREADS@
                       Threadsafety: @HDF5_ENABLE_THREADSAFE@
+                       Concurrency: @HDF5_ENABLE_CONCURRENCY@
                Default API mapping: @HDF5_DEFAULT_API_VERSION@
     With deprecated public symbols: @HDF5_ENABLE_DEPRECATED_SYMBOLS@
             I/O filters (external): @EXTERNAL_FILTERS@
diff --git a/test/API/H5_api_async_test.c b/test/API/H5_api_async_test.c
index 04b604f0761..35a97baf3a2 100644
--- a/test/API/H5_api_async_test.c
+++ b/test/API/H5_api_async_test.c
@@ -12,10 +12,10 @@
 
 #include "H5_api_async_test.h"
 
-static void print_async_test_header(const void *params);
+static void print_async_test_header(void *params);
 
 static void
-print_async_test_header(const void H5_ATTR_UNUSED *params)
+print_async_test_header(void H5_ATTR_UNUSED *params)
 {
     printf("\n");
     printf("**********************************************\n");
@@ -31,20 +31,20 @@ print_async_test_header(const void H5_ATTR_UNUSED *params)
 
 #ifdef H5_API_TEST_HAVE_ASYNC
 
-static void test_one_dataset_io(const void *params);
-static void test_multi_dataset_io(const void *params);
-static void test_multi_file_dataset_io(const void *params);
-static void test_multi_file_grp_dset_io(const void *params);
-static void test_set_extent(const void *params);
-static void test_attribute_exists(const void *params);
-static void test_attribute_io(const void *params);
-static void test_attribute_io_tconv(const void *params);
-static void test_attribute_io_compound(const void *params);
-static void test_group(const void *params);
-static void test_link(const void *params);
-static void test_ocopy_orefresh(const void *params);
-static void test_file_reopen(const void *params);
-static void test_file_cleanup(const void *params);
+static void test_one_dataset_io(void *params);
+static void test_multi_dataset_io(void *params);
+static void test_multi_file_dataset_io(void *params);
+static void test_multi_file_grp_dset_io(void *params);
+static void test_set_extent(void *params);
+static void test_attribute_exists(void *params);
+static void test_attribute_io(void *params);
+static void test_attribute_io_tconv(void *params);
+static void test_attribute_io_compound(void *params);
+static void test_group(void *params);
+static void test_link(void *params);
+static void test_ocopy_orefresh(void *params);
+static void test_file_reopen(void *params);
+static void test_file_cleanup(void *params);
 
 /* Highest "printf" file created (starting at 0) */
 int max_printf_file = -1;
@@ -53,7 +53,7 @@ int max_printf_file = -1;
  * Create file and dataset, write to dataset
  */
 static void
-test_one_dataset_io(const void H5_ATTR_UNUSED *params)
+test_one_dataset_io(void H5_ATTR_UNUSED *params)
 {
     hid_t   file_id  = H5I_INVALID_HID;
     hid_t   dset_id  = H5I_INVALID_HID;
@@ -320,7 +320,7 @@ test_one_dataset_io(const void H5_ATTR_UNUSED *params)
  * Create file and multiple datasets, write to them and read from them
  */
 static void
-test_multi_dataset_io(const void H5_ATTR_UNUSED *params)
+test_multi_dataset_io(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id    = H5I_INVALID_HID;
     hid_t dset_id[5] = {H5I_INVALID_HID, H5I_INVALID_HID, H5I_INVALID_HID, H5I_INVALID_HID, H5I_INVALID_HID};
@@ -575,7 +575,7 @@ test_multi_dataset_io(const void H5_ATTR_UNUSED *params)
  * from them
  */
 static void
-test_multi_file_dataset_io(const void H5_ATTR_UNUSED *params)
+test_multi_file_dataset_io(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id[5] = {H5I_INVALID_HID, H5I_INVALID_HID, H5I_INVALID_HID, H5I_INVALID_HID, H5I_INVALID_HID};
     hid_t dset_id[5] = {H5I_INVALID_HID, H5I_INVALID_HID, H5I_INVALID_HID, H5I_INVALID_HID, H5I_INVALID_HID};
@@ -885,7 +885,7 @@ test_multi_file_dataset_io(const void H5_ATTR_UNUSED *params)
  * and read from them
  */
 static void
-test_multi_file_grp_dset_io(const void H5_ATTR_UNUSED *params)
+test_multi_file_grp_dset_io(void H5_ATTR_UNUSED *params)
 {
     hid_t   file_id  = H5I_INVALID_HID;
     hid_t   grp_id   = H5I_INVALID_HID;
@@ -1193,7 +1193,7 @@ test_multi_file_grp_dset_io(const void H5_ATTR_UNUSED *params)
  * Create file and dataset, write to dataset
  */
 static void
-test_set_extent(const void H5_ATTR_UNUSED *params)
+test_set_extent(void H5_ATTR_UNUSED *params)
 {
     hid_t   file_id       = H5I_INVALID_HID;
     hid_t   dset_id       = H5I_INVALID_HID;
@@ -1408,7 +1408,7 @@ test_set_extent(const void H5_ATTR_UNUSED *params)
  * Test H5Aexists()
  */
 static void
-test_attribute_exists(const void H5_ATTR_UNUSED *params)
+test_attribute_exists(void H5_ATTR_UNUSED *params)
 {
     hid_t   file_id  = H5I_INVALID_HID;
     hid_t   dset_id  = H5I_INVALID_HID;
@@ -1529,7 +1529,7 @@ test_attribute_exists(const void H5_ATTR_UNUSED *params)
  * Create file, dataset, and attribute, write to attribute
  */
 static void
-test_attribute_io(const void H5_ATTR_UNUSED *params)
+test_attribute_io(void H5_ATTR_UNUSED *params)
 {
     hid_t   file_id  = H5I_INVALID_HID;
     hid_t   dset_id  = H5I_INVALID_HID;
@@ -1673,7 +1673,7 @@ test_attribute_io(const void H5_ATTR_UNUSED *params)
  * Create file, dataset, and attribute, write to attribute with type conversion
  */
 static void
-test_attribute_io_tconv(const void H5_ATTR_UNUSED *params)
+test_attribute_io_tconv(void H5_ATTR_UNUSED *params)
 {
     hid_t   file_id  = H5I_INVALID_HID;
     hid_t   attr_id  = H5I_INVALID_HID;
@@ -1814,7 +1814,7 @@ typedef struct tattr_cmpd_t {
 } tattr_cmpd_t;
 
 static void
-test_attribute_io_compound(const void H5_ATTR_UNUSED *params)
+test_attribute_io_compound(void H5_ATTR_UNUSED *params)
 {
     hid_t        file_id   = H5I_INVALID_HID;
     hid_t        attr_id   = H5I_INVALID_HID;
@@ -2124,7 +2124,7 @@ test_attribute_io_compound(const void H5_ATTR_UNUSED *params)
  * Test group interfaces
  */
 static void
-test_group(const void H5_ATTR_UNUSED *params)
+test_group(void H5_ATTR_UNUSED *params)
 {
     hid_t      file_id         = H5I_INVALID_HID;
     hid_t      parent_group_id = H5I_INVALID_HID;
@@ -2292,7 +2292,7 @@ test_group(const void H5_ATTR_UNUSED *params)
  * Test link interfaces
  */
 static void
-test_link(const void H5_ATTR_UNUSED *params)
+test_link(void H5_ATTR_UNUSED *params)
 {
     hid_t  file_id         = H5I_INVALID_HID;
     hid_t  parent_group_id = H5I_INVALID_HID;
@@ -2489,7 +2489,7 @@ test_link(const void H5_ATTR_UNUSED *params)
  * Test H5Ocopy() and H5Orefresh()
  */
 static void
-test_ocopy_orefresh(const void H5_ATTR_UNUSED *params)
+test_ocopy_orefresh(void H5_ATTR_UNUSED *params)
 {
     hid_t   file_id         = H5I_INVALID_HID;
     hid_t   parent_group_id = H5I_INVALID_HID;
@@ -2604,7 +2604,7 @@ test_ocopy_orefresh(const void H5_ATTR_UNUSED *params)
  * Test H5Freopen()
  */
 static void
-test_file_reopen(const void H5_ATTR_UNUSED *params)
+test_file_reopen(void H5_ATTR_UNUSED *params)
 {
     hid_t  file_id          = H5I_INVALID_HID;
     hid_t  reopened_file_id = H5I_INVALID_HID;
@@ -2676,7 +2676,7 @@ test_file_reopen(const void H5_ATTR_UNUSED *params)
  * Cleanup temporary test files
  */
 static void
-test_file_cleanup(const void H5_ATTR_UNUSED *params)
+test_file_cleanup(void H5_ATTR_UNUSED *params)
 {
     char file_name[64];
     int  i;
diff --git a/test/API/H5_api_attribute_test.c b/test/API/H5_api_attribute_test.c
index d508889ae31..0a11bdb6fb2 100644
--- a/test/API/H5_api_attribute_test.c
+++ b/test/API/H5_api_attribute_test.c
@@ -20,48 +20,48 @@
  *         order value gets reset when all attributes are removed.
  */
 
-static void print_attribute_test_header(const void *params);
-static void test_create_attribute_on_root(const void *params);
-static void test_create_attribute_on_dataset(const void *params);
-static void test_create_attribute_on_datatype(const void *params);
-static void test_create_attribute_with_null_space(const void *params);
-static void test_create_attribute_with_scalar_space(const void *params);
-static void test_create_attribute_with_space_in_name(const void *params);
-static void test_create_attribute_invalid_params(const void *params);
-static void test_open_attribute(const void *params);
-static void test_open_attribute_invalid_params(const void *params);
-static void test_write_attribute(const void *params);
-static void test_write_attribute_invalid_params(const void *params);
-static void test_read_attribute(const void *params);
-static void test_read_attribute_invalid_params(const void *params);
-static void test_read_empty_attribute(const void *params);
-static void test_close_attribute_invalid_id(const void *params);
-static void test_get_attribute_space_and_type(const void *params);
-static void test_get_attribute_space_and_type_invalid_params(const void *params);
-static void test_attribute_property_lists(const void *params);
-static void test_get_attribute_name(const void *params);
-static void test_get_attribute_name_invalid_params(const void *params);
-static void test_get_attribute_storage_size(const void *params);
-static void test_get_attribute_info(const void *params);
-static void test_get_attribute_info_invalid_params(const void *params);
-static void test_rename_attribute(const void *params);
-static void test_rename_attribute_invalid_params(const void *params);
-static void test_attribute_iterate_group(const void *params);
-static void test_attribute_iterate_dataset(const void *params);
-static void test_attribute_iterate_datatype(const void *params);
-static void test_attribute_iterate_index_saving(const void *params);
-static void test_attribute_iterate_invalid_params(const void *params);
-static void test_attribute_iterate_0_attributes(const void *params);
-static void test_attribute_compound_subset(const void *params);
-static void test_attribute_string_encodings(const void *params);
-static void test_delete_attribute(const void *params);
-static void test_delete_attribute_invalid_params(const void *params);
-static void test_attribute_exists(const void *params);
-static void test_attribute_exists_invalid_params(const void *params);
-static void test_attribute_many(const void *params);
-static void test_attribute_duplicate_id(const void *params);
-static void test_get_number_attributes(const void *params);
-static void test_attr_shared_dtype(const void *params);
+static void print_attribute_test_header(void *params);
+static void test_create_attribute_on_root(void *params);
+static void test_create_attribute_on_dataset(void *params);
+static void test_create_attribute_on_datatype(void *params);
+static void test_create_attribute_with_null_space(void *params);
+static void test_create_attribute_with_scalar_space(void *params);
+static void test_create_attribute_with_space_in_name(void *params);
+static void test_create_attribute_invalid_params(void *params);
+static void test_open_attribute(void *params);
+static void test_open_attribute_invalid_params(void *params);
+static void test_write_attribute(void *params);
+static void test_write_attribute_invalid_params(void *params);
+static void test_read_attribute(void *params);
+static void test_read_attribute_invalid_params(void *params);
+static void test_read_empty_attribute(void *params);
+static void test_close_attribute_invalid_id(void *params);
+static void test_get_attribute_space_and_type(void *params);
+static void test_get_attribute_space_and_type_invalid_params(void *params);
+static void test_attribute_property_lists(void *params);
+static void test_get_attribute_name(void *params);
+static void test_get_attribute_name_invalid_params(void *params);
+static void test_get_attribute_storage_size(void *params);
+static void test_get_attribute_info(void *params);
+static void test_get_attribute_info_invalid_params(void *params);
+static void test_rename_attribute(void *params);
+static void test_rename_attribute_invalid_params(void *params);
+static void test_attribute_iterate_group(void *params);
+static void test_attribute_iterate_dataset(void *params);
+static void test_attribute_iterate_datatype(void *params);
+static void test_attribute_iterate_index_saving(void *params);
+static void test_attribute_iterate_invalid_params(void *params);
+static void test_attribute_iterate_0_attributes(void *params);
+static void test_attribute_compound_subset(void *params);
+static void test_attribute_string_encodings(void *params);
+static void test_delete_attribute(void *params);
+static void test_delete_attribute_invalid_params(void *params);
+static void test_attribute_exists(void *params);
+static void test_attribute_exists_invalid_params(void *params);
+static void test_attribute_many(void *params);
+static void test_attribute_duplicate_id(void *params);
+static void test_get_number_attributes(void *params);
+static void test_attr_shared_dtype(void *params);
 
 static herr_t attr_iter_callback1(hid_t location_id, const char *attr_name, const H5A_info_t *ainfo,
                                   void *op_data);
@@ -69,7 +69,7 @@ static herr_t attr_iter_callback2(hid_t location_id, const char *attr_name, cons
                                   void *op_data);
 
 static void
-print_attribute_test_header(const void H5_ATTR_UNUSED *params)
+print_attribute_test_header(void H5_ATTR_UNUSED *params)
 {
     printf("\n");
     printf("**********************************************\n");
@@ -84,7 +84,7 @@ print_attribute_test_header(const void H5_ATTR_UNUSED *params)
  * the root group.
  */
 static void
-test_create_attribute_on_root(const void H5_ATTR_UNUSED *params)
+test_create_attribute_on_root(void H5_ATTR_UNUSED *params)
 {
     htri_t attr_exists;
     hid_t  file_id = H5I_INVALID_HID;
@@ -221,7 +221,7 @@ test_create_attribute_on_root(const void H5_ATTR_UNUSED *params)
  * a dataset.
  */
 static void
-test_create_attribute_on_dataset(const void H5_ATTR_UNUSED *params)
+test_create_attribute_on_dataset(void H5_ATTR_UNUSED *params)
 {
     htri_t attr_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -406,7 +406,7 @@ test_create_attribute_on_dataset(const void H5_ATTR_UNUSED *params)
  * a committed datatype.
  */
 static void
-test_create_attribute_on_datatype(const void H5_ATTR_UNUSED *params)
+test_create_attribute_on_datatype(void H5_ATTR_UNUSED *params)
 {
     htri_t attr_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -581,7 +581,7 @@ test_create_attribute_on_datatype(const void H5_ATTR_UNUSED *params)
  * NULL dataspace is not problematic.
  */
 static void
-test_create_attribute_with_null_space(const void H5_ATTR_UNUSED *params)
+test_create_attribute_with_null_space(void H5_ATTR_UNUSED *params)
 {
     htri_t attr_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -692,7 +692,7 @@ test_create_attribute_with_null_space(const void H5_ATTR_UNUSED *params)
  * scalar dataspace is not problematic.
  */
 static void
-test_create_attribute_with_scalar_space(const void H5_ATTR_UNUSED *params)
+test_create_attribute_with_scalar_space(void H5_ATTR_UNUSED *params)
 {
     htri_t attr_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -803,7 +803,7 @@ test_create_attribute_with_scalar_space(const void H5_ATTR_UNUSED *params)
  * is not problematic.
  */
 static void
-test_create_attribute_with_space_in_name(const void H5_ATTR_UNUSED *params)
+test_create_attribute_with_space_in_name(void H5_ATTR_UNUSED *params)
 {
     htri_t attr_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -909,7 +909,7 @@ test_create_attribute_with_space_in_name(const void H5_ATTR_UNUSED *params)
  * H5Acreate is passed invalid parameters.
  */
 static void
-test_create_attribute_invalid_params(const void H5_ATTR_UNUSED *params)
+test_create_attribute_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -1352,7 +1352,7 @@ test_create_attribute_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Aopen(_by_idx).
  */
 static void
-test_open_attribute(const void H5_ATTR_UNUSED *params)
+test_open_attribute(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -1756,7 +1756,7 @@ test_open_attribute(const void H5_ATTR_UNUSED *params)
  * H5Aopen(_by_name/_by_idx) is passed invalid parameters.
  */
 static void
-test_open_attribute_invalid_params(const void H5_ATTR_UNUSED *params)
+test_open_attribute_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -2271,7 +2271,7 @@ test_open_attribute_invalid_params(const void H5_ATTR_UNUSED *params)
  * can be made.
  */
 static void
-test_write_attribute(const void H5_ATTR_UNUSED *params)
+test_write_attribute(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[ATTRIBUTE_WRITE_TEST_SPACE_RANK];
     size_t  i, data_size;
@@ -2400,7 +2400,7 @@ test_write_attribute(const void H5_ATTR_UNUSED *params)
  * H5Awrite is passed invalid parameters.
  */
 static void
-test_write_attribute_invalid_params(const void H5_ATTR_UNUSED *params)
+test_write_attribute_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[ATTRIBUTE_WRITE_INVALID_PARAMS_TEST_SPACE_RANK];
     size_t  i, data_size;
@@ -2590,7 +2590,7 @@ test_write_attribute_invalid_params(const void H5_ATTR_UNUSED *params)
  * attribute.
  */
 static void
-test_read_attribute(const void H5_ATTR_UNUSED *params)
+test_read_attribute(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[ATTRIBUTE_READ_TEST_SPACE_RANK];
     size_t  i, data_size;
@@ -2745,7 +2745,7 @@ test_read_attribute(const void H5_ATTR_UNUSED *params)
  * H5Aread is passed invalid parameters.
  */
 static void
-test_read_attribute_invalid_params(const void H5_ATTR_UNUSED *params)
+test_read_attribute_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[ATTRIBUTE_READ_INVALID_PARAMS_TEST_SPACE_RANK];
     size_t  i, data_size;
@@ -2958,7 +2958,7 @@ test_read_attribute_invalid_params(const void H5_ATTR_UNUSED *params)
  * Test reading an empty attribute is ok
  */
 static void
-test_read_empty_attribute(const void H5_ATTR_UNUSED *params)
+test_read_empty_attribute(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[ATTRIBUTE_READ_EMPTY_SPACE_RANK];
     size_t  i, data_size;
@@ -3085,7 +3085,7 @@ test_read_empty_attribute(const void H5_ATTR_UNUSED *params)
  * an invalid attribute ID.
  */
 static void
-test_close_attribute_invalid_id(const void H5_ATTR_UNUSED *params)
+test_close_attribute_invalid_id(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
     hid_t  file_id = H5I_INVALID_HID;
@@ -3140,7 +3140,7 @@ test_close_attribute_invalid_id(const void H5_ATTR_UNUSED *params)
  * H5Aget_space and H5Aget_type, respectively.
  */
 static void
-test_get_attribute_space_and_type(const void H5_ATTR_UNUSED *params)
+test_get_attribute_space_and_type(void H5_ATTR_UNUSED *params)
 {
     hsize_t attr_dims[ATTRIBUTE_GET_SPACE_TYPE_TEST_SPACE_RANK];
     size_t  i;
@@ -3445,7 +3445,7 @@ test_get_attribute_space_and_type(const void H5_ATTR_UNUSED *params)
  * invalid parameters, respectively.
  */
 static void
-test_get_attribute_space_and_type_invalid_params(const void H5_ATTR_UNUSED *params)
+test_get_attribute_space_and_type_invalid_params(void H5_ATTR_UNUSED *params)
 {
     htri_t attr_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -3606,7 +3606,7 @@ test_get_attribute_space_and_type_invalid_params(const void H5_ATTR_UNUSED *para
  * be retrieved later with a call to H5Aget_create_plist.
  */
 static void
-test_attribute_property_lists(const void H5_ATTR_UNUSED *params)
+test_attribute_property_lists(void H5_ATTR_UNUSED *params)
 {
     H5T_cset_t encoding = H5T_CSET_UTF8;
     htri_t     attr_exists;
@@ -3891,7 +3891,7 @@ test_attribute_property_lists(const void H5_ATTR_UNUSED *params)
  * H5Aget_name_by_idx.
  */
 static void
-test_get_attribute_name(const void H5_ATTR_UNUSED *params)
+test_get_attribute_name(void H5_ATTR_UNUSED *params)
 {
     ssize_t name_buf_size;
     htri_t  attr_exists;
@@ -4388,7 +4388,7 @@ test_get_attribute_name(const void H5_ATTR_UNUSED *params)
  * parameters.
  */
 static void
-test_get_attribute_name_invalid_params(const void H5_ATTR_UNUSED *params)
+test_get_attribute_name_invalid_params(void H5_ATTR_UNUSED *params)
 {
     ssize_t name_buf_size;
     htri_t  attr_exists;
@@ -4744,7 +4744,7 @@ test_get_attribute_name_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Aget_storage_size.
  */
 static void
-test_get_attribute_storage_size(const void H5_ATTR_UNUSED *params)
+test_get_attribute_storage_size(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Aget_storage_size");
 
@@ -4757,7 +4757,7 @@ test_get_attribute_storage_size(const void H5_ATTR_UNUSED *params)
  * A test to check the functionality of H5Aget_info(_by_idx).
  */
 static void
-test_get_attribute_info(const void H5_ATTR_UNUSED *params)
+test_get_attribute_info(void H5_ATTR_UNUSED *params)
 {
     H5A_info_t attr_info;
     htri_t     attr_exists;
@@ -5405,7 +5405,7 @@ test_get_attribute_info(const void H5_ATTR_UNUSED *params)
  * doesn't succeed when passed invalid parameters.
  */
 static void
-test_get_attribute_info_invalid_params(const void H5_ATTR_UNUSED *params)
+test_get_attribute_info_invalid_params(void H5_ATTR_UNUSED *params)
 {
     H5A_info_t attr_info;
     htri_t     attr_exists;
@@ -5867,7 +5867,7 @@ test_get_attribute_info_invalid_params(const void H5_ATTR_UNUSED *params)
  * with H5Arename and H5Arename_by_name.
  */
 static void
-test_rename_attribute(const void H5_ATTR_UNUSED *params)
+test_rename_attribute(void H5_ATTR_UNUSED *params)
 {
     htri_t attr_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -6084,7 +6084,7 @@ test_rename_attribute(const void H5_ATTR_UNUSED *params)
  * when H5Arename(_by_name) is passed invalid parameters.
  */
 static void
-test_rename_attribute_invalid_params(const void H5_ATTR_UNUSED *params)
+test_rename_attribute_invalid_params(void H5_ATTR_UNUSED *params)
 {
     htri_t attr_exists;
     herr_t err_ret         = -1;
@@ -6456,7 +6456,7 @@ test_rename_attribute_invalid_params(const void H5_ATTR_UNUSED *params)
  * creation order.
  */
 static void
-test_attribute_iterate_group(const void H5_ATTR_UNUSED *params)
+test_attribute_iterate_group(void H5_ATTR_UNUSED *params)
 {
     size_t link_counter;
     size_t i;
@@ -6848,7 +6848,7 @@ test_attribute_iterate_group(const void H5_ATTR_UNUSED *params)
  * creation order.
  */
 static void
-test_attribute_iterate_dataset(const void H5_ATTR_UNUSED *params)
+test_attribute_iterate_dataset(void H5_ATTR_UNUSED *params)
 {
     size_t link_counter;
     size_t i;
@@ -7272,7 +7272,7 @@ test_attribute_iterate_dataset(const void H5_ATTR_UNUSED *params)
  * creation order.
  */
 static void
-test_attribute_iterate_datatype(const void H5_ATTR_UNUSED *params)
+test_attribute_iterate_datatype(void H5_ATTR_UNUSED *params)
 {
     size_t link_counter;
     size_t i;
@@ -7687,7 +7687,7 @@ test_attribute_iterate_datatype(const void H5_ATTR_UNUSED *params)
  * creation order.
  */
 static void
-test_attribute_iterate_index_saving(const void H5_ATTR_UNUSED *params)
+test_attribute_iterate_index_saving(void H5_ATTR_UNUSED *params)
 {
     TESTING("attribute iteration index saving capability");
 
@@ -7702,7 +7702,7 @@ test_attribute_iterate_index_saving(const void H5_ATTR_UNUSED *params)
  * passed invalid parameters.
  */
 static void
-test_attribute_iterate_invalid_params(const void H5_ATTR_UNUSED *params)
+test_attribute_iterate_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
     htri_t attr_exists;
@@ -8123,7 +8123,7 @@ test_attribute_iterate_invalid_params(const void H5_ATTR_UNUSED *params)
  * not problematic.
  */
 static void
-test_attribute_iterate_0_attributes(const void H5_ATTR_UNUSED *params)
+test_attribute_iterate_0_attributes(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -8315,7 +8315,7 @@ typedef struct attribute_compound_io_t {
  * been written, using subsets of compound datatypes
  */
 static void
-test_attribute_compound_subset(const void H5_ATTR_UNUSED *params)
+test_attribute_compound_subset(void H5_ATTR_UNUSED *params)
 {
     hsize_t                 dims[1] = {ATTRIBUTE_COMPOUND_IO_ATTR_DIMS};
     size_t                  i;
@@ -8572,7 +8572,7 @@ test_attribute_compound_subset(const void H5_ATTR_UNUSED *params)
  * correctness for strings with ASCII or UTF-8 char sets
  */
 static void
-test_attribute_string_encodings(const void H5_ATTR_UNUSED *params)
+test_attribute_string_encodings(void H5_ATTR_UNUSED *params)
 {
     hid_t   file_id                               = H5I_INVALID_HID;
     hid_t   container_group                       = H5I_INVALID_HID;
@@ -8641,15 +8641,15 @@ test_attribute_string_encodings(const void H5_ATTR_UNUSED *params)
         goto error;
     }
 
-    if ((dset_id1 = H5Dcreate(container_group, ATTRIBUTE_STRING_ENCODINGS_DSET_NAME1, type_id1, space_id,
-                              H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) {
+    if ((dset_id1 = H5Dcreate2(container_group, ATTRIBUTE_STRING_ENCODINGS_DSET_NAME1, type_id1, space_id,
+                               H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) {
         H5_FAILED();
         printf("    couldn't create dataset with ascii string\n");
         goto error;
     }
 
-    if ((attr_id1 = H5Acreate(dset_id1, ATTRIBUTE_STRING_ENCODINGS_ATTR_NAME1, type_id1, space_id,
-                              H5P_DEFAULT, H5P_DEFAULT)) < 0) {
+    if ((attr_id1 = H5Acreate2(dset_id1, ATTRIBUTE_STRING_ENCODINGS_ATTR_NAME1, type_id1, space_id,
+                               H5P_DEFAULT, H5P_DEFAULT)) < 0) {
         H5_FAILED();
         printf("    couldn't create attribute with ascii string\n");
         goto error;
@@ -8673,15 +8673,15 @@ test_attribute_string_encodings(const void H5_ATTR_UNUSED *params)
         goto error;
     }
 
-    if ((dset_id2 = H5Dcreate(container_group, ATTRIBUTE_STRING_ENCODINGS_DSET_NAME2, type_id2, space_id,
-                              H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) {
+    if ((dset_id2 = H5Dcreate2(container_group, ATTRIBUTE_STRING_ENCODINGS_DSET_NAME2, type_id2, space_id,
+                               H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) {
         H5_FAILED();
         printf("    couldn't create dataset with UTF-8 string\n");
         goto error;
     }
 
-    if ((attr_id2 = H5Acreate(dset_id2, ATTRIBUTE_STRING_ENCODINGS_ATTR_NAME2, type_id2, space_id,
-                              H5P_DEFAULT, H5P_DEFAULT)) < 0) {
+    if ((attr_id2 = H5Acreate2(dset_id2, ATTRIBUTE_STRING_ENCODINGS_ATTR_NAME2, type_id2, space_id,
+                               H5P_DEFAULT, H5P_DEFAULT)) < 0) {
         H5_FAILED();
         printf("    couldn't create attribute with ascii string\n");
         goto error;
@@ -8836,7 +8836,7 @@ test_attribute_string_encodings(const void H5_ATTR_UNUSED *params)
  * using H5Adelete(_by_idx).
  */
 static void
-test_delete_attribute(const void H5_ATTR_UNUSED *params)
+test_delete_attribute(void H5_ATTR_UNUSED *params)
 {
     htri_t attr_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -10069,7 +10069,7 @@ test_delete_attribute(const void H5_ATTR_UNUSED *params)
  * parameters.
  */
 static void
-test_delete_attribute_invalid_params(const void H5_ATTR_UNUSED *params)
+test_delete_attribute_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
     htri_t attr_exists;
@@ -10487,7 +10487,7 @@ test_delete_attribute_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Aexists and H5Aexists_by_name.
  */
 static void
-test_attribute_exists(const void H5_ATTR_UNUSED *params)
+test_attribute_exists(void H5_ATTR_UNUSED *params)
 {
     htri_t attr_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -10628,7 +10628,7 @@ test_attribute_exists(const void H5_ATTR_UNUSED *params)
  * given invalid parameters.
  */
 static void
-test_attribute_exists_invalid_params(const void H5_ATTR_UNUSED *params)
+test_attribute_exists_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
     htri_t attr_exists;
@@ -10905,7 +10905,7 @@ test_attribute_exists_invalid_params(const void H5_ATTR_UNUSED *params)
  * to the file
  */
 static void
-test_attribute_many(const void H5_ATTR_UNUSED *params)
+test_attribute_many(void H5_ATTR_UNUSED *params)
 {
     unsigned u;
     htri_t   attr_exists;
@@ -11015,7 +11015,7 @@ test_attribute_many(const void H5_ATTR_UNUSED *params)
  * a second time
  */
 static void
-test_attribute_duplicate_id(const void H5_ATTR_UNUSED *params)
+test_attribute_duplicate_id(void H5_ATTR_UNUSED *params)
 {
     htri_t attr_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -11130,7 +11130,7 @@ test_attribute_duplicate_id(const void H5_ATTR_UNUSED *params)
  * XXX: Cover all of the cases and move to H5O tests.
  */
 static void
-test_get_number_attributes(const void H5_ATTR_UNUSED *params)
+test_get_number_attributes(void H5_ATTR_UNUSED *params)
 {
     H5O_info2_t obj_info;
     htri_t      attr_exists;
@@ -11309,7 +11309,7 @@ test_get_number_attributes(const void H5_ATTR_UNUSED *params)
  * XXX: May move to H5O tests.
  */
 static void
-test_attr_shared_dtype(const void H5_ATTR_UNUSED *params)
+test_attr_shared_dtype(void H5_ATTR_UNUSED *params)
 {
     H5O_info2_t obj_info;
     htri_t      attr_exists;
diff --git a/test/API/H5_api_dataset_test.c b/test/API/H5_api_dataset_test.c
index c88288bfd01..633b759ef2b 100644
--- a/test/API/H5_api_dataset_test.c
+++ b/test/API/H5_api_dataset_test.c
@@ -16,91 +16,91 @@
  * XXX: H5Dread_chunk/H5Dwrite_chunk, H5Dfill/scatter/gather
  */
 
-static void print_dataset_test_header(const void *params);
-static void test_create_dataset_under_root(const void *params);
-static void test_create_dataset_under_existing_group(const void *params);
-static void test_create_dataset_invalid_params(const void *params);
-static void test_create_anonymous_dataset(const void *params);
-static void test_create_anonymous_dataset_invalid_params(const void *params);
-static void test_create_dataset_null_space(const void *params);
-static void test_create_dataset_scalar_space(const void *params);
-static void test_create_zero_dim_dset(const void *params);
-static void test_create_dataset_random_shapes(const void *params);
-static void test_create_dataset_predefined_types(const void *params);
-static void test_create_dataset_string_types(const void *params);
-static void test_create_dataset_compound_types(const void *params);
-static void test_create_dataset_enum_types(const void *params);
-static void test_create_dataset_array_types(const void *params);
-static void test_create_dataset_creation_properties(const void *params);
-static void test_create_many_dataset(const void *params);
-static void test_open_dataset(const void *params);
-static void test_open_dataset_invalid_params(const void *params);
-static void test_close_dataset_invalid_params(const void *params);
-static void test_get_dataset_space_and_type(const void *params);
-static void test_get_dataset_space_and_type_invalid_params(const void *params);
-static void test_get_dataset_space_status(const void *params);
-static void test_get_dataset_space_status_invalid_params(const void *params);
-static void test_dataset_property_lists(const void *params);
-static void test_get_dataset_storage_size(const void *params);
-static void test_get_dataset_storage_size_invalid_params(const void *params);
-static void test_get_dataset_chunk_storage_size(const void *params);
-static void test_get_dataset_chunk_storage_size_invalid_params(const void *params);
-static void test_get_dataset_offset(const void *params);
-static void test_get_dataset_offset_invalid_params(const void *params);
-static void test_read_dataset_small_all(const void *params);
-static void test_read_dataset_small_hyperslab(const void *params);
-static void test_read_dataset_small_point_selection(const void *params);
-static void test_read_multi_dataset_small_all(const void *params);
-static void test_read_multi_dataset_small_hyperslab(const void *params);
-static void test_read_multi_dataset_small_point_selection(const void *params);
-static void test_dataset_io_point_selections(const void *params);
-static void test_read_dataset_invalid_params(const void *params);
-static void test_write_dataset_small_all(const void *params);
-static void test_write_dataset_small_hyperslab(const void *params);
-static void test_write_dataset_small_point_selection(const void *params);
-static void test_write_dataset_data_verification(const void *params);
-static void test_write_multi_dataset_small_all(const void *params);
-static void test_write_multi_dataset_small_hyperslab(const void *params);
-static void test_write_multi_dataset_small_point_selection(const void *params);
-static void test_write_multi_dataset_data_verification(const void *params);
-static void test_write_dataset_invalid_params(const void *params);
-static void test_dataset_string_encodings(const void *params);
-static void test_dataset_builtin_type_conversion(const void *params);
-static void test_dataset_real_to_int_conversion(const void *params);
-static void test_dataset_compound_partial_io(const void *params);
-static void test_dataset_vlen_io(const void *params);
-static void test_dataset_set_extent_chunked_unlimited(const void *params);
-static void test_dataset_set_extent_chunked_fixed(const void *params);
-static void test_dataset_set_extent_data(const void *params);
-static void test_dataset_set_extent_double_handles(const void *params);
-static void test_dataset_set_extent_invalid_params(const void *params);
-static void test_flush_dataset(const void *params);
-static void test_flush_dataset_invalid_params(const void *params);
-static void test_refresh_dataset(const void *params);
-static void test_refresh_dataset_invalid_params(const void *params);
+static void print_dataset_test_header(void *params);
+static void test_create_dataset_under_root(void *params);
+static void test_create_dataset_under_existing_group(void *params);
+static void test_create_dataset_invalid_params(void *params);
+static void test_create_anonymous_dataset(void *params);
+static void test_create_anonymous_dataset_invalid_params(void *params);
+static void test_create_dataset_null_space(void *params);
+static void test_create_dataset_scalar_space(void *params);
+static void test_create_zero_dim_dset(void *params);
+static void test_create_dataset_random_shapes(void *params);
+static void test_create_dataset_predefined_types(void *params);
+static void test_create_dataset_string_types(void *params);
+static void test_create_dataset_compound_types(void *params);
+static void test_create_dataset_enum_types(void *params);
+static void test_create_dataset_array_types(void *params);
+static void test_create_dataset_creation_properties(void *params);
+static void test_create_many_dataset(void *params);
+static void test_open_dataset(void *params);
+static void test_open_dataset_invalid_params(void *params);
+static void test_close_dataset_invalid_params(void *params);
+static void test_get_dataset_space_and_type(void *params);
+static void test_get_dataset_space_and_type_invalid_params(void *params);
+static void test_get_dataset_space_status(void *params);
+static void test_get_dataset_space_status_invalid_params(void *params);
+static void test_dataset_property_lists(void *params);
+static void test_get_dataset_storage_size(void *params);
+static void test_get_dataset_storage_size_invalid_params(void *params);
+static void test_get_dataset_chunk_storage_size(void *params);
+static void test_get_dataset_chunk_storage_size_invalid_params(void *params);
+static void test_get_dataset_offset(void *params);
+static void test_get_dataset_offset_invalid_params(void *params);
+static void test_read_dataset_small_all(void *params);
+static void test_read_dataset_small_hyperslab(void *params);
+static void test_read_dataset_small_point_selection(void *params);
+static void test_read_multi_dataset_small_all(void *params);
+static void test_read_multi_dataset_small_hyperslab(void *params);
+static void test_read_multi_dataset_small_point_selection(void *params);
+static void test_dataset_io_point_selections(void *params);
+static void test_read_dataset_invalid_params(void *params);
+static void test_write_dataset_small_all(void *params);
+static void test_write_dataset_small_hyperslab(void *params);
+static void test_write_dataset_small_point_selection(void *params);
+static void test_write_dataset_data_verification(void *params);
+static void test_write_multi_dataset_small_all(void *params);
+static void test_write_multi_dataset_small_hyperslab(void *params);
+static void test_write_multi_dataset_small_point_selection(void *params);
+static void test_write_multi_dataset_data_verification(void *params);
+static void test_write_dataset_invalid_params(void *params);
+static void test_dataset_string_encodings(void *params);
+static void test_dataset_builtin_type_conversion(void *params);
+static void test_dataset_real_to_int_conversion(void *params);
+static void test_dataset_compound_partial_io(void *params);
+static void test_dataset_vlen_io(void *params);
+static void test_dataset_set_extent_chunked_unlimited(void *params);
+static void test_dataset_set_extent_chunked_fixed(void *params);
+static void test_dataset_set_extent_data(void *params);
+static void test_dataset_set_extent_double_handles(void *params);
+static void test_dataset_set_extent_invalid_params(void *params);
+static void test_flush_dataset(void *params);
+static void test_flush_dataset_invalid_params(void *params);
+static void test_refresh_dataset(void *params);
+static void test_refresh_dataset_invalid_params(void *params);
 
 /*
  * Chunking tests
  */
-static void test_create_single_chunk_dataset(const void *params);
-static void test_write_single_chunk_dataset(const void *params);
-static void test_create_multi_chunk_dataset(const void *params);
-static void test_write_multi_chunk_dataset_same_shape_read(const void *params);
-static void test_write_multi_chunk_dataset_diff_shape_read(const void *params);
-static void test_overwrite_multi_chunk_dataset_same_shape_read(const void *params);
-static void test_overwrite_multi_chunk_dataset_diff_shape_read(const void *params);
-static void test_read_partial_chunk_all_selection(const void *params);
-static void test_read_partial_chunk_hyperslab_selection(const void *params);
-static void test_read_partial_chunk_point_selection(const void *params);
-
-static void test_get_vlen_buf_size(const void *params);
+static void test_create_single_chunk_dataset(void *params);
+static void test_write_single_chunk_dataset(void *params);
+static void test_create_multi_chunk_dataset(void *params);
+static void test_write_multi_chunk_dataset_same_shape_read(void *params);
+static void test_write_multi_chunk_dataset_diff_shape_read(void *params);
+static void test_overwrite_multi_chunk_dataset_same_shape_read(void *params);
+static void test_overwrite_multi_chunk_dataset_diff_shape_read(void *params);
+static void test_read_partial_chunk_all_selection(void *params);
+static void test_read_partial_chunk_hyperslab_selection(void *params);
+static void test_read_partial_chunk_point_selection(void *params);
+
+static void test_get_vlen_buf_size(void *params);
 
 static size_t filter(unsigned int flags, size_t H5_ATTR_UNUSED cd_nelmts,
                      const unsigned int H5_ATTR_UNUSED cd_values[], size_t nbytes,
                      size_t H5_ATTR_UNUSED *buf_size, void H5_ATTR_UNUSED **buf);
 
 static void
-print_dataset_test_header(const void H5_ATTR_UNUSED *params)
+print_dataset_test_header(void H5_ATTR_UNUSED *params)
 {
     printf("\n");
     printf("**********************************************\n");
@@ -115,7 +115,7 @@ print_dataset_test_header(const void H5_ATTR_UNUSED *params)
  * created under the root group.
  */
 static void
-test_create_dataset_under_root(const void H5_ATTR_UNUSED *params)
+test_create_dataset_under_root(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id    = H5I_INVALID_HID;
     hid_t dset_id    = H5I_INVALID_HID;
@@ -182,7 +182,7 @@ test_create_dataset_under_root(const void H5_ATTR_UNUSED *params)
  * under a group that is not the root group.
  */
 static void
-test_create_dataset_under_existing_group(const void H5_ATTR_UNUSED *params)
+test_create_dataset_under_existing_group(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -270,7 +270,7 @@ test_create_dataset_under_existing_group(const void H5_ATTR_UNUSED *params)
  * when H5Dcreate is passed invalid parameters.
  */
 static void
-test_create_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
+test_create_dataset_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -526,7 +526,7 @@ test_create_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test to check that an anonymous dataset can be created.
  */
 static void
-test_create_anonymous_dataset(const void H5_ATTR_UNUSED *params)
+test_create_anonymous_dataset(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -613,7 +613,7 @@ test_create_anonymous_dataset(const void H5_ATTR_UNUSED *params)
  * parameters.
  */
 static void
-test_create_anonymous_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
+test_create_anonymous_dataset_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -808,7 +808,7 @@ test_create_anonymous_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
  * dataspace is not problematic.
  */
 static void
-test_create_dataset_null_space(const void H5_ATTR_UNUSED *params)
+test_create_dataset_null_space(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -905,7 +905,7 @@ test_create_dataset_null_space(const void H5_ATTR_UNUSED *params)
  * dataspace is not problematic.
  */
 static void
-test_create_dataset_scalar_space(const void H5_ATTR_UNUSED *params)
+test_create_dataset_scalar_space(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -1002,7 +1002,7 @@ test_create_dataset_scalar_space(const void H5_ATTR_UNUSED *params)
  * which contains a 0-sized dimension is not problematic.
  */
 static void
-test_create_zero_dim_dset(const void H5_ATTR_UNUSED *params)
+test_create_zero_dim_dset(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[ZERO_DIM_DSET_TEST_SPACE_RANK] = {0};
     hid_t   file_id                             = H5I_INVALID_HID;
@@ -1105,7 +1105,7 @@ test_create_zero_dim_dset(const void H5_ATTR_UNUSED *params)
  * a variety of different dataspace shapes.
  */
 static void
-test_create_dataset_random_shapes(const void H5_ATTR_UNUSED *params)
+test_create_dataset_random_shapes(void H5_ATTR_UNUSED *params)
 {
     size_t i;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -1204,7 +1204,7 @@ test_create_dataset_random_shapes(const void H5_ATTR_UNUSED *params)
  * datatypes.
  */
 static void
-test_create_dataset_predefined_types(const void H5_ATTR_UNUSED *params)
+test_create_dataset_predefined_types(void H5_ATTR_UNUSED *params)
 {
     size_t i;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -1310,7 +1310,7 @@ test_create_dataset_predefined_types(const void H5_ATTR_UNUSED *params)
  * string datatypes.
  */
 static void
-test_create_dataset_string_types(const void H5_ATTR_UNUSED *params)
+test_create_dataset_string_types(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -1479,7 +1479,7 @@ test_create_dataset_string_types(const void H5_ATTR_UNUSED *params)
  * a variety of compound datatypes.
  */
 static void
-test_create_dataset_compound_types(const void H5_ATTR_UNUSED *params)
+test_create_dataset_compound_types(void H5_ATTR_UNUSED *params)
 {
     size_t i, j;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -1649,7 +1649,7 @@ test_create_dataset_compound_types(const void H5_ATTR_UNUSED *params)
  * enum datatypes.
  */
 static void
-test_create_dataset_enum_types(const void H5_ATTR_UNUSED *params)
+test_create_dataset_enum_types(void H5_ATTR_UNUSED *params)
 {
     size_t      i;
     hid_t       file_id         = H5I_INVALID_HID;
@@ -1791,7 +1791,7 @@ test_create_dataset_enum_types(const void H5_ATTR_UNUSED *params)
  * array datatypes.
  */
 static void
-test_create_dataset_array_types(const void H5_ATTR_UNUSED *params)
+test_create_dataset_array_types(void H5_ATTR_UNUSED *params)
 {
     hsize_t array_dims1[DATASET_ARRAY_TYPE_TEST_RANK1];
     hsize_t array_dims2[DATASET_ARRAY_TYPE_TEST_RANK2];
@@ -2002,7 +2002,7 @@ filter(unsigned int H5_ATTR_UNUSED flags, size_t H5_ATTR_UNUSED cd_nelmts,
  * dataset creation properties.
  */
 static void
-test_create_dataset_creation_properties(const void H5_ATTR_UNUSED *params)
+test_create_dataset_creation_properties(void H5_ATTR_UNUSED *params)
 {
     hsize_t      dims[DATASET_CREATION_PROPERTIES_TEST_SHAPE_RANK];
     hsize_t      chunk_dims[DATASET_CREATION_PROPERTIES_TEST_SHAPE_RANK];
@@ -2385,9 +2385,9 @@ test_create_dataset_creation_properties(const void H5_ATTR_UNUSED *params)
                 PART_ERROR(DCPL_fill_value_test);
             }
 
-            if ((dset_id =
-                     H5Dcreate(group_id, DATASET_FILL_VALUE_TEST_DSET_NAME1, DATASET_FILL_VALUE_TEST_INT_TYPE,
-                               fspace_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT)) < 0) {
+            if ((dset_id = H5Dcreate2(group_id, DATASET_FILL_VALUE_TEST_DSET_NAME1,
+                                      DATASET_FILL_VALUE_TEST_INT_TYPE, fspace_id, H5P_DEFAULT, dcpl_id,
+                                      H5P_DEFAULT)) < 0) {
                 H5_FAILED();
                 printf("    couldn't create dataset with integer fill value");
                 PART_ERROR(DCPL_fill_value_test);
@@ -3088,7 +3088,7 @@ test_create_dataset_creation_properties(const void H5_ATTR_UNUSED *params)
  * A test to create many small datasets (100,000)
  */
 static void
-test_create_many_dataset(const void H5_ATTR_UNUSED *params)
+test_create_many_dataset(void H5_ATTR_UNUSED *params)
 {
     hid_t         file_id         = H5I_INVALID_HID;
     hid_t         container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -3191,7 +3191,7 @@ test_create_many_dataset(const void H5_ATTR_UNUSED *params)
  * H5Dopen succeeds.
  */
 static void
-test_open_dataset(const void H5_ATTR_UNUSED *params)
+test_open_dataset(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Dopen");
 
@@ -3205,7 +3205,7 @@ test_open_dataset(const void H5_ATTR_UNUSED *params)
  * passed invalid parameters.
  */
 static void
-test_open_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
+test_open_dataset_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -3380,7 +3380,7 @@ test_open_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
  * passed an invalid dataset ID.
  */
 static void
-test_close_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
+test_close_dataset_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
     hid_t  file_id = H5I_INVALID_HID;
@@ -3435,7 +3435,7 @@ test_close_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
  * H5Dget_type, respectively.
  */
 static void
-test_get_dataset_space_and_type(const void H5_ATTR_UNUSED *params)
+test_get_dataset_space_and_type(void H5_ATTR_UNUSED *params)
 {
     hsize_t dset_dims[DATASET_GET_SPACE_TYPE_TEST_SPACE_RANK];
     size_t  i;
@@ -3725,7 +3725,7 @@ test_get_dataset_space_and_type(const void H5_ATTR_UNUSED *params)
  * invalid parameters, respectively.
  */
 static void
-test_get_dataset_space_and_type_invalid_params(const void H5_ATTR_UNUSED *params)
+test_get_dataset_space_and_type_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID;
@@ -3868,7 +3868,7 @@ test_get_dataset_space_and_type_invalid_params(const void H5_ATTR_UNUSED *params
  * A test for H5Dget_space_status.
  */
 static void
-test_get_dataset_space_status(const void H5_ATTR_UNUSED *params)
+test_get_dataset_space_status(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Dget_space_status");
 
@@ -3883,7 +3883,7 @@ test_get_dataset_space_status(const void H5_ATTR_UNUSED *params)
  * it is passed invalid parameters.
  */
 static void
-test_get_dataset_space_status_invalid_params(const void H5_ATTR_UNUSED *params)
+test_get_dataset_space_status_invalid_params(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Dget_space_status with invalid parameters");
 
@@ -3900,7 +3900,7 @@ test_get_dataset_space_status_invalid_params(const void H5_ATTR_UNUSED *params)
  * access can be retrieved with a call to H5Dget_access_plist.
  */
 static void
-test_dataset_property_lists(const void H5_ATTR_UNUSED *params)
+test_dataset_property_lists(void H5_ATTR_UNUSED *params)
 {
     const char *path_prefix = "/test_prefix";
     hsize_t     dims[DATASET_PROPERTY_LIST_TEST_SPACE_RANK];
@@ -4304,7 +4304,7 @@ test_dataset_property_lists(const void H5_ATTR_UNUSED *params)
  * A test for H5Dget_storage_size.
  */
 static void
-test_get_dataset_storage_size(const void H5_ATTR_UNUSED *params)
+test_get_dataset_storage_size(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Dget_storage_size");
 
@@ -4319,7 +4319,7 @@ test_get_dataset_storage_size(const void H5_ATTR_UNUSED *params)
  * invalid parameters.
  */
 static void
-test_get_dataset_storage_size_invalid_params(const void H5_ATTR_UNUSED *params)
+test_get_dataset_storage_size_invalid_params(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Dget_storage_size with invalid parameters");
 
@@ -4332,7 +4332,7 @@ test_get_dataset_storage_size_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Dget_chunk_storage_size.
  */
 static void
-test_get_dataset_chunk_storage_size(const void H5_ATTR_UNUSED *params)
+test_get_dataset_chunk_storage_size(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Dget_chunk_storage_size");
 
@@ -4347,7 +4347,7 @@ test_get_dataset_chunk_storage_size(const void H5_ATTR_UNUSED *params)
  * is passed invalid parameters.
  */
 static void
-test_get_dataset_chunk_storage_size_invalid_params(const void H5_ATTR_UNUSED *params)
+test_get_dataset_chunk_storage_size_invalid_params(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Dget_chunk_storage_size with invalid parameters");
 
@@ -4360,7 +4360,7 @@ test_get_dataset_chunk_storage_size_invalid_params(const void H5_ATTR_UNUSED *pa
  * A test for H5Dget_offset.
  */
 static void
-test_get_dataset_offset(const void H5_ATTR_UNUSED *params)
+test_get_dataset_offset(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Dget_offset");
 
@@ -4375,7 +4375,7 @@ test_get_dataset_offset(const void H5_ATTR_UNUSED *params)
  * parameters.
  */
 static void
-test_get_dataset_offset_invalid_params(const void H5_ATTR_UNUSED *params)
+test_get_dataset_offset_invalid_params(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Dget_offset with invalid parameters");
 
@@ -4389,7 +4389,7 @@ test_get_dataset_offset_invalid_params(const void H5_ATTR_UNUSED *params)
  * read back from a dataset using an H5S_ALL selection.
  */
 static void
-test_read_dataset_small_all(const void H5_ATTR_UNUSED *params)
+test_read_dataset_small_all(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[DATASET_SMALL_READ_TEST_ALL_DSET_SPACE_RANK] = {10, 5, 3};
     size_t  i, data_size;
@@ -4494,7 +4494,7 @@ test_read_dataset_small_all(const void H5_ATTR_UNUSED *params)
  * read back from a dataset using a hyperslab selection.
  */
 static void
-test_read_dataset_small_hyperslab(const void H5_ATTR_UNUSED *params)
+test_read_dataset_small_hyperslab(void H5_ATTR_UNUSED *params)
 {
     hsize_t start[DATASET_SMALL_READ_TEST_HYPERSLAB_DSET_SPACE_RANK];
     hsize_t stride[DATASET_SMALL_READ_TEST_HYPERSLAB_DSET_SPACE_RANK];
@@ -4621,7 +4621,7 @@ test_read_dataset_small_hyperslab(const void H5_ATTR_UNUSED *params)
  * read back from a dataset using a point selection.
  */
 static void
-test_read_dataset_small_point_selection(const void H5_ATTR_UNUSED *params)
+test_read_dataset_small_point_selection(void H5_ATTR_UNUSED *params)
 {
     hsize_t points[DATASET_SMALL_READ_TEST_POINT_SELECTION_NUM_POINTS *
                    DATASET_SMALL_READ_TEST_POINT_SELECTION_DSET_SPACE_RANK];
@@ -4750,7 +4750,7 @@ test_read_dataset_small_point_selection(const void H5_ATTR_UNUSED *params)
  * read back from multiple datasets using H5S_ALL selections.
  */
 static void
-test_read_multi_dataset_small_all(const void H5_ATTR_UNUSED *params)
+test_read_multi_dataset_small_all(void H5_ATTR_UNUSED *params)
 {
 
     hsize_t dims[DATASET_SMALL_READ_TEST_ALL_DSET_SPACE_RANK] = {10, 5, 3};
@@ -4876,7 +4876,7 @@ test_read_multi_dataset_small_all(const void H5_ATTR_UNUSED *params)
  * read back from datasets using hyperslab selections.
  */
 static void
-test_read_multi_dataset_small_hyperslab(const void H5_ATTR_UNUSED *params)
+test_read_multi_dataset_small_hyperslab(void H5_ATTR_UNUSED *params)
 {
     hsize_t start[DATASET_SMALL_READ_TEST_HYPERSLAB_DSET_SPACE_RANK];
     hsize_t stride[DATASET_SMALL_READ_TEST_HYPERSLAB_DSET_SPACE_RANK];
@@ -5029,7 +5029,7 @@ test_read_multi_dataset_small_hyperslab(const void H5_ATTR_UNUSED *params)
  * read back from datasets using point selections.
  */
 static void
-test_read_multi_dataset_small_point_selection(const void H5_ATTR_UNUSED *params)
+test_read_multi_dataset_small_point_selection(void H5_ATTR_UNUSED *params)
 {
     hsize_t points[DATASET_SMALL_READ_TEST_POINT_SELECTION_NUM_POINTS *
                    DATASET_SMALL_READ_TEST_POINT_SELECTION_DSET_SPACE_RANK];
@@ -5199,7 +5199,7 @@ test_read_multi_dataset_small_point_selection(const void H5_ATTR_UNUSED *params)
             } while ((J) < (I));                                                                             \
     }
 static void
-test_dataset_io_point_selections(const void H5_ATTR_UNUSED *params)
+test_dataset_io_point_selections(void H5_ATTR_UNUSED *params)
 {
     hid_t   file_id         = H5I_INVALID_HID;
     hid_t   dset_id         = H5I_INVALID_HID;
@@ -5680,7 +5680,7 @@ test_dataset_io_point_selections(const void H5_ATTR_UNUSED *params)
  * dataset when H5Dread is passed invalid parameters.
  */
 static void
-test_read_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
+test_read_dataset_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[DATASET_READ_INVALID_PARAMS_TEST_DSET_SPACE_RANK] = {10, 5, 3};
     herr_t  err_ret                                                = -1;
@@ -5914,7 +5914,7 @@ test_read_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
  * made to a dataset using an H5S_ALL selection.
  */
 static void
-test_write_dataset_small_all(const void H5_ATTR_UNUSED *params)
+test_write_dataset_small_all(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t  dims[DATASET_SMALL_WRITE_TEST_ALL_DSET_SPACE_RANK] = {10, 5, 3};
@@ -6042,7 +6042,7 @@ test_write_dataset_small_all(const void H5_ATTR_UNUSED *params)
  * to a dataset using a hyperslab selection.
  */
 static void
-test_write_dataset_small_hyperslab(const void H5_ATTR_UNUSED *params)
+test_write_dataset_small_hyperslab(void H5_ATTR_UNUSED *params)
 {
     hsize_t start[DATASET_SMALL_WRITE_TEST_HYPERSLAB_DSET_SPACE_RANK];
     hsize_t stride[DATASET_SMALL_WRITE_TEST_HYPERSLAB_DSET_SPACE_RANK];
@@ -6173,7 +6173,7 @@ test_write_dataset_small_hyperslab(const void H5_ATTR_UNUSED *params)
  * to a dataset using a point selection.
  */
 static void
-test_write_dataset_small_point_selection(const void H5_ATTR_UNUSED *params)
+test_write_dataset_small_point_selection(void H5_ATTR_UNUSED *params)
 {
     hsize_t points[DATASET_SMALL_WRITE_TEST_POINT_SELECTION_NUM_POINTS *
                    DATASET_SMALL_WRITE_TEST_POINT_SELECTION_DSET_SPACE_RANK];
@@ -6305,7 +6305,7 @@ test_write_dataset_small_point_selection(const void H5_ATTR_UNUSED *params)
  * a dataset after it has been written.
  */
 static void
-test_write_dataset_data_verification(const void H5_ATTR_UNUSED *params)
+test_write_dataset_data_verification(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t  dims[DATASET_DATA_VERIFY_WRITE_TEST_DSET_SPACE_RANK] = {10, 10, 10};
@@ -6820,7 +6820,7 @@ test_write_dataset_data_verification(const void H5_ATTR_UNUSED *params)
  * made to a dataset using an H5S_ALL selection.
  */
 static void
-test_write_multi_dataset_small_all(const void H5_ATTR_UNUSED *params)
+test_write_multi_dataset_small_all(void H5_ATTR_UNUSED *params)
 {
     hssize_t    space_npoints;
     hsize_t     dims[DATASET_SMALL_WRITE_TEST_ALL_DSET_SPACE_RANK] = {10, 5, 3};
@@ -6983,7 +6983,7 @@ test_write_multi_dataset_small_all(const void H5_ATTR_UNUSED *params)
  * to a dataset using a hyperslab selection.
  */
 static void
-test_write_multi_dataset_small_hyperslab(const void H5_ATTR_UNUSED *params)
+test_write_multi_dataset_small_hyperslab(void H5_ATTR_UNUSED *params)
 {
     hsize_t     start[DATASET_SMALL_WRITE_TEST_HYPERSLAB_DSET_SPACE_RANK];
     hsize_t     stride[DATASET_SMALL_WRITE_TEST_HYPERSLAB_DSET_SPACE_RANK];
@@ -7144,7 +7144,7 @@ test_write_multi_dataset_small_hyperslab(const void H5_ATTR_UNUSED *params)
  * to a dataset using a point selection.
  */
 static void
-test_write_multi_dataset_small_point_selection(const void H5_ATTR_UNUSED *params)
+test_write_multi_dataset_small_point_selection(void H5_ATTR_UNUSED *params)
 {
     hsize_t     points[DATASET_SMALL_WRITE_TEST_POINT_SELECTION_NUM_POINTS *
                    DATASET_SMALL_WRITE_TEST_POINT_SELECTION_DSET_SPACE_RANK];
@@ -7307,7 +7307,7 @@ test_write_multi_dataset_small_point_selection(const void H5_ATTR_UNUSED *params
  * multiple datasets after it has been written.
  */
 static void
-test_write_multi_dataset_data_verification(const void H5_ATTR_UNUSED *params)
+test_write_multi_dataset_data_verification(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints[DATASET_MULTI_COUNT];
     hsize_t  dims[DATASET_DATA_VERIFY_WRITE_TEST_DSET_SPACE_RANK] = {10, 10, 10};
@@ -7910,7 +7910,7 @@ test_write_multi_dataset_data_verification(const void H5_ATTR_UNUSED *params)
  * when H5Dwrite is passed invalid parameters.
  */
 static void
-test_write_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
+test_write_dataset_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t  dims[DATASET_WRITE_INVALID_PARAMS_TEST_DSET_SPACE_RANK] = {10, 5, 3};
@@ -8151,7 +8151,7 @@ test_write_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
  * can be written to and read from a dataset
  */
 static void
-test_dataset_string_encodings(const void H5_ATTR_UNUSED *params)
+test_dataset_string_encodings(void H5_ATTR_UNUSED *params)
 {
     hid_t   file_id                             = H5I_INVALID_HID;
     hid_t   container_group                     = H5I_INVALID_HID;
@@ -8218,8 +8218,8 @@ test_dataset_string_encodings(const void H5_ATTR_UNUSED *params)
         goto error;
     }
 
-    if ((dset_id1 = H5Dcreate(container_group, DATASET_STRING_ENCODINGS_DSET_NAME1, type_id1, space_id,
-                              H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) {
+    if ((dset_id1 = H5Dcreate2(container_group, DATASET_STRING_ENCODINGS_DSET_NAME1, type_id1, space_id,
+                               H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) {
         H5_FAILED();
         printf("    couldn't create dataset with ascii string\n");
         goto error;
@@ -8243,8 +8243,8 @@ test_dataset_string_encodings(const void H5_ATTR_UNUSED *params)
         goto error;
     }
 
-    if ((dset_id2 = H5Dcreate(container_group, DATASET_STRING_ENCODINGS_DSET_NAME2, type_id2, space_id,
-                              H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) {
+    if ((dset_id2 = H5Dcreate2(container_group, DATASET_STRING_ENCODINGS_DSET_NAME2, type_id2, space_id,
+                               H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) {
         H5_FAILED();
         printf("    couldn't create dataset with UTF-8 string\n");
         goto error;
@@ -8396,7 +8396,7 @@ test_dataset_string_encodings(const void H5_ATTR_UNUSED *params)
  * been written, using type conversion with builtin types.
  */
 static void
-test_dataset_builtin_type_conversion(const void H5_ATTR_UNUSED *params)
+test_dataset_builtin_type_conversion(void H5_ATTR_UNUSED *params)
 {
     hssize_t    space_npoints;
     hsize_t     dims[DATASET_DATA_BUILTIN_CONVERSION_TEST_DSET_SPACE_RANK] = {10, 10, 10};
@@ -8930,7 +8930,7 @@ test_dataset_builtin_type_conversion(const void H5_ATTR_UNUSED *params)
 }
 
 static void
-test_dataset_real_to_int_conversion(const void H5_ATTR_UNUSED *params)
+test_dataset_real_to_int_conversion(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t  dims[DATASET_DATA_REAL_CONVERSION_TEST_DSET_SPACE_RANK] = {10, 10, 10};
@@ -9481,7 +9481,7 @@ typedef struct dataset_compount_partial_io_t {
 } dataset_compount_partial_io_t;
 
 static void
-test_dataset_compound_partial_io(const void H5_ATTR_UNUSED *params)
+test_dataset_compound_partial_io(void H5_ATTR_UNUSED *params)
 {
     hsize_t                       dims[1] = {DATASET_COMPOUND_PARTIAL_IO_DSET_DIMS};
     size_t                        i;
@@ -9736,7 +9736,7 @@ test_dataset_compound_partial_io(const void H5_ATTR_UNUSED *params)
 /* A test to check that vlen sequences can be written and read back
  * with basic parent types and selections */
 static void
-test_dataset_vlen_io(const void H5_ATTR_UNUSED *params)
+test_dataset_vlen_io(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID;
@@ -10223,6 +10223,9 @@ test_dataset_vlen_io(const void H5_ATTR_UNUSED *params)
             }
 
             /* Reset buffers */
+            if (H5Sselect_all(space_id) < 0)
+                PART_TEST_ERROR(rw_point_selection);
+
             if (H5Treclaim(vlen_int, space_id, H5P_DEFAULT, rbuf) < 0)
                 PART_TEST_ERROR(rw_point_selection);
 
@@ -10325,6 +10328,9 @@ test_dataset_vlen_io(const void H5_ATTR_UNUSED *params)
             }
 
             /* Reset buffers */
+            if (H5Sselect_all(space_id) < 0)
+                PART_TEST_ERROR(rw_hyperslab_selection);
+
             if (H5Treclaim(vlen_int, space_id, H5P_DEFAULT, rbuf) < 0)
                 PART_TEST_ERROR(rw_hyperslab_selection);
 
@@ -10399,7 +10405,7 @@ test_dataset_vlen_io(const void H5_ATTR_UNUSED *params)
  * dataset may both shrink and grow.
  */
 static void
-test_dataset_set_extent_chunked_unlimited(const void H5_ATTR_UNUSED *params)
+test_dataset_set_extent_chunked_unlimited(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[DATASET_SET_EXTENT_CHUNKED_UNLIMITED_TEST_SPACE_RANK];
     hsize_t max_dims[DATASET_SET_EXTENT_CHUNKED_UNLIMITED_TEST_SPACE_RANK];
@@ -10632,7 +10638,7 @@ test_dataset_set_extent_chunked_unlimited(const void H5_ATTR_UNUSED *params)
  * dataset may only shrink.
  */
 static void
-test_dataset_set_extent_chunked_fixed(const void H5_ATTR_UNUSED *params)
+test_dataset_set_extent_chunked_fixed(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[DATASET_SET_EXTENT_CHUNKED_FIXED_TEST_SPACE_RANK];
     hsize_t dims2[DATASET_SET_EXTENT_CHUNKED_FIXED_TEST_SPACE_RANK];
@@ -10910,7 +10916,7 @@ test_dataset_set_extent_chunked_fixed(const void H5_ATTR_UNUSED *params)
  * and shrinking the dataset with H5Dset_extent
  */
 static void
-test_dataset_set_extent_data(const void H5_ATTR_UNUSED *params)
+test_dataset_set_extent_data(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims_origin[DATASET_SET_EXTENT_DATA_TEST_SPACE_RANK] = {DATASET_SET_EXTENT_DATA_TEST_SPACE_DIM,
                                                                     DATASET_SET_EXTENT_DATA_TEST_SPACE_DIM};
@@ -11250,7 +11256,7 @@ test_dataset_set_extent_data(const void H5_ATTR_UNUSED *params)
  * return the new size when queried.
  */
 static void
-test_dataset_set_extent_double_handles(const void H5_ATTR_UNUSED *params)
+test_dataset_set_extent_double_handles(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims_origin[DATASET_SET_EXTENT_DOUBLE_HANDLES_TEST_SPACE_RANK] = {
         DATASET_SET_EXTENT_DOUBLE_HANDLES_TEST_SPACE_DIM, DATASET_SET_EXTENT_DOUBLE_HANDLES_TEST_SPACE_DIM};
@@ -11391,7 +11397,7 @@ test_dataset_set_extent_double_handles(const void H5_ATTR_UNUSED *params)
  * changed when H5Dset_extent is passed invalid parameters.
  */
 static void
-test_dataset_set_extent_invalid_params(const void H5_ATTR_UNUSED *params)
+test_dataset_set_extent_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[DATASET_SET_EXTENT_INVALID_PARAMS_TEST_SPACE_RANK];
     hsize_t chunk_dims[DATASET_SET_EXTENT_INVALID_PARAMS_TEST_SPACE_RANK];
@@ -11653,7 +11659,7 @@ test_dataset_set_extent_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Dflush.
  */
 static void
-test_flush_dataset(const void H5_ATTR_UNUSED *params)
+test_flush_dataset(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Dflush");
 
@@ -11667,7 +11673,7 @@ test_flush_dataset(const void H5_ATTR_UNUSED *params)
  * passed invalid parameters.
  */
 static void
-test_flush_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
+test_flush_dataset_invalid_params(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Dflush with invalid parameters");
 
@@ -11680,7 +11686,7 @@ test_flush_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Drefresh.
  */
 static void
-test_refresh_dataset(const void H5_ATTR_UNUSED *params)
+test_refresh_dataset(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Drefresh");
 
@@ -11694,7 +11700,7 @@ test_refresh_dataset(const void H5_ATTR_UNUSED *params)
  * passed invalid parameters.
  */
 static void
-test_refresh_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
+test_refresh_dataset_invalid_params(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Drefresh with invalid parameters");
 
@@ -11707,7 +11713,7 @@ test_refresh_dataset_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test to create a dataset composed of a single chunk.
  */
 static void
-test_create_single_chunk_dataset(const void H5_ATTR_UNUSED *params)
+test_create_single_chunk_dataset(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[DATASET_SINGLE_CHUNK_TEST_SPACE_RANK];
     hsize_t retrieved_chunk_dims[DATASET_SINGLE_CHUNK_TEST_SPACE_RANK];
@@ -11885,7 +11891,7 @@ test_create_single_chunk_dataset(const void H5_ATTR_UNUSED *params)
  * and read correctly.
  */
 static void
-test_write_single_chunk_dataset(const void H5_ATTR_UNUSED *params)
+test_write_single_chunk_dataset(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t  dims[DATASET_SINGLE_CHUNK_WRITE_TEST_DSET_SPACE_RANK];
@@ -12105,7 +12111,7 @@ test_write_single_chunk_dataset(const void H5_ATTR_UNUSED *params)
  * A test to create a dataset composed of multiple chunks.
  */
 static void
-test_create_multi_chunk_dataset(const void H5_ATTR_UNUSED *params)
+test_create_multi_chunk_dataset(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[DATASET_MULTI_CHUNK_TEST_SPACE_RANK]       = {100, 100};
     hsize_t chunk_dims[DATASET_MULTI_CHUNK_TEST_SPACE_RANK] = {10, 10};
@@ -12286,7 +12292,7 @@ test_create_multi_chunk_dataset(const void H5_ATTR_UNUSED *params)
  * used are the same shape.
  */
 static void
-test_write_multi_chunk_dataset_same_shape_read(const void H5_ATTR_UNUSED *params)
+test_write_multi_chunk_dataset_same_shape_read(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[DATASET_MULTI_CHUNK_WRITE_SAME_SPACE_READ_TEST_DSET_SPACE_RANK]       = {100, 100};
     hsize_t chunk_dims[DATASET_MULTI_CHUNK_WRITE_SAME_SPACE_READ_TEST_DSET_SPACE_RANK] = {10, 10};
@@ -12613,7 +12619,7 @@ test_write_multi_chunk_dataset_same_shape_read(const void H5_ATTR_UNUSED *params
  * used are differently shaped.
  */
 static void
-test_write_multi_chunk_dataset_diff_shape_read(const void H5_ATTR_UNUSED *params)
+test_write_multi_chunk_dataset_diff_shape_read(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[DATASET_MULTI_CHUNK_WRITE_DIFF_SPACE_READ_TEST_DSET_SPACE_RANK]       = {100, 100};
     hsize_t chunk_dims[DATASET_MULTI_CHUNK_WRITE_DIFF_SPACE_READ_TEST_DSET_SPACE_RANK] = {10, 10};
@@ -12950,7 +12956,7 @@ test_write_multi_chunk_dataset_diff_shape_read(const void H5_ATTR_UNUSED *params
  * dataspace and memory dataspace used are the same shape.
  */
 static void
-test_overwrite_multi_chunk_dataset_same_shape_read(const void H5_ATTR_UNUSED *params)
+test_overwrite_multi_chunk_dataset_same_shape_read(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[DATASET_MULTI_CHUNK_OVERWRITE_SAME_SPACE_READ_TEST_DSET_SPACE_RANK]       = {100, 100};
     hsize_t chunk_dims[DATASET_MULTI_CHUNK_OVERWRITE_SAME_SPACE_READ_TEST_DSET_SPACE_RANK] = {10, 10};
@@ -13285,7 +13291,7 @@ test_overwrite_multi_chunk_dataset_same_shape_read(const void H5_ATTR_UNUSED *pa
  * dataspace and memory dataspace used are differently shaped.
  */
 static void
-test_overwrite_multi_chunk_dataset_diff_shape_read(const void H5_ATTR_UNUSED *params)
+test_overwrite_multi_chunk_dataset_diff_shape_read(void H5_ATTR_UNUSED *params)
 {
     hsize_t dims[DATASET_MULTI_CHUNK_OVERWRITE_DIFF_SPACE_READ_TEST_DSET_SPACE_RANK]       = {100, 100};
     hsize_t chunk_dims[DATASET_MULTI_CHUNK_OVERWRITE_DIFF_SPACE_READ_TEST_DSET_SPACE_RANK] = {10, 10};
@@ -13633,7 +13639,7 @@ test_overwrite_multi_chunk_dataset_diff_shape_read(const void H5_ATTR_UNUSED *pa
 #define FIXED_DIMSIZE       25
 #define FIXED_CHUNK_DIMSIZE 10
 static void
-test_read_partial_chunk_all_selection(const void H5_ATTR_UNUSED *params)
+test_read_partial_chunk_all_selection(void H5_ATTR_UNUSED *params)
 {
     DATASET_PARTIAL_CHUNK_READ_ALL_SEL_TEST_DSET_CTYPE write_buf[FIXED_DIMSIZE][FIXED_DIMSIZE];
     DATASET_PARTIAL_CHUNK_READ_ALL_SEL_TEST_DSET_CTYPE read_buf[FIXED_DIMSIZE][FIXED_DIMSIZE];
@@ -13819,7 +13825,7 @@ test_read_partial_chunk_all_selection(const void H5_ATTR_UNUSED *params)
 #define FIXED_CHUNK_DIMSIZE 10
 #define FIXED_NCHUNKS       9 /* For convenience - make sure to adjust this as necessary */
 static void
-test_read_partial_chunk_hyperslab_selection(const void H5_ATTR_UNUSED *params)
+test_read_partial_chunk_hyperslab_selection(void H5_ATTR_UNUSED *params)
 {
     DATASET_PARTIAL_CHUNK_READ_HYPER_SEL_TEST_DSET_CTYPE write_buf[FIXED_CHUNK_DIMSIZE][FIXED_CHUNK_DIMSIZE];
     DATASET_PARTIAL_CHUNK_READ_HYPER_SEL_TEST_DSET_CTYPE read_buf[FIXED_CHUNK_DIMSIZE][FIXED_CHUNK_DIMSIZE];
@@ -14112,7 +14118,7 @@ test_read_partial_chunk_hyperslab_selection(const void H5_ATTR_UNUSED *params)
 /* #define FIXED_DIMSIZE       25 */
 /* #define FIXED_CHUNK_DIMSIZE 10 */
 static void
-test_read_partial_chunk_point_selection(const void H5_ATTR_UNUSED *params)
+test_read_partial_chunk_point_selection(void H5_ATTR_UNUSED *params)
 {
     TESTING("reading a partial chunk using a point selection in file dataspace");
     SKIPPED();
@@ -14127,7 +14133,7 @@ test_read_partial_chunk_point_selection(const void H5_ATTR_UNUSED *params)
  * correct size
  */
 static void
-test_get_vlen_buf_size(const void H5_ATTR_UNUSED *params)
+test_get_vlen_buf_size(void H5_ATTR_UNUSED *params)
 {
     hvl_t    wdata[DATASET_GET_VLEN_BUF_SIZE_DSET_SPACE_DIM]; /* Information to write */
     hid_t    file_id         = H5I_INVALID_HID;
diff --git a/test/API/H5_api_datatype_test.c b/test/API/H5_api_datatype_test.c
index 968729df33e..6eca4abe089 100644
--- a/test/API/H5_api_datatype_test.c
+++ b/test/API/H5_api_datatype_test.c
@@ -17,35 +17,35 @@
  */
 #define PROBLEMATIC_TESTS
 
-static void print_datatype_test_header(const void *params);
-static void test_create_committed_datatype(const void *params);
-static void test_create_committed_datatype_invalid_params(const void *params);
-static void test_create_anonymous_committed_datatype(const void *params);
-static void test_create_anonymous_committed_datatype_invalid_params(const void *params);
+static void print_datatype_test_header(void *params);
+static void test_create_committed_datatype(void *params);
+static void test_create_committed_datatype_invalid_params(void *params);
+static void test_create_anonymous_committed_datatype(void *params);
+static void test_create_anonymous_committed_datatype_invalid_params(void *params);
 #ifndef PROBLEMATIC_TESTS
-static void test_create_committed_datatype_empty_types(const void *params);
+static void test_create_committed_datatype_empty_types(void *params);
 #endif
-static void test_recommit_committed_type(const void *params);
-static void test_open_committed_datatype(const void *params);
-static void test_open_committed_datatype_invalid_params(const void *params);
-static void test_reopen_committed_datatype_indirect(const void *params);
-static void test_close_committed_datatype_invalid_id(const void *params);
-static void test_datatype_property_lists(const void *params);
-static void test_create_dataset_with_committed_type(const void *params);
-static void test_create_attribute_with_committed_type(const void *params);
-static void test_delete_committed_type(const void *params);
-static void test_resurrect_datatype(const void *params);
-static void test_flush_committed_datatype(const void *params);
-static void test_flush_committed_datatype_invalid_params(const void *params);
-static void test_refresh_committed_datatype(const void *params);
-static void test_refresh_committed_datatype_invalid_params(const void *params);
+static void test_recommit_committed_type(void *params);
+static void test_open_committed_datatype(void *params);
+static void test_open_committed_datatype_invalid_params(void *params);
+static void test_reopen_committed_datatype_indirect(void *params);
+static void test_close_committed_datatype_invalid_id(void *params);
+static void test_datatype_property_lists(void *params);
+static void test_create_dataset_with_committed_type(void *params);
+static void test_create_attribute_with_committed_type(void *params);
+static void test_delete_committed_type(void *params);
+static void test_resurrect_datatype(void *params);
+static void test_flush_committed_datatype(void *params);
+static void test_flush_committed_datatype_invalid_params(void *params);
+static void test_refresh_committed_datatype(void *params);
+static void test_refresh_committed_datatype_invalid_params(void *params);
 #ifndef PROBLEMATIC_TESTS
-static void test_cant_commit_predefined(const void *params);
+static void test_cant_commit_predefined(void *params);
 #endif
-static void test_cant_modify_committed_type(const void *params);
+static void test_cant_modify_committed_type(void *params);
 
 static void
-print_datatype_test_header(const void H5_ATTR_UNUSED *params)
+print_datatype_test_header(void H5_ATTR_UNUSED *params)
 {
     printf("\n");
     printf("**********************************************\n");
@@ -59,7 +59,7 @@ print_datatype_test_header(const void H5_ATTR_UNUSED *params)
  * A test to check that a committed datatype can be created.
  */
 static void
-test_create_committed_datatype(const void H5_ATTR_UNUSED *params)
+test_create_committed_datatype(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -139,7 +139,7 @@ test_create_committed_datatype(const void H5_ATTR_UNUSED *params)
  * created when H5Tcommit2 is passed invalid parameters.
  */
 static void
-test_create_committed_datatype_invalid_params(const void H5_ATTR_UNUSED *params)
+test_create_committed_datatype_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret         = -1;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -361,7 +361,7 @@ test_create_committed_datatype_invalid_params(const void H5_ATTR_UNUSED *params)
  * can be created with H5Tcommit_anon.
  */
 static void
-test_create_anonymous_committed_datatype(const void H5_ATTR_UNUSED *params)
+test_create_anonymous_committed_datatype(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -440,7 +440,7 @@ test_create_anonymous_committed_datatype(const void H5_ATTR_UNUSED *params)
  * created when H5Tcommit_anon is passed invalid parameters.
  */
 static void
-test_create_anonymous_committed_datatype_invalid_params(const void H5_ATTR_UNUSED *params)
+test_create_anonymous_committed_datatype_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret         = -1;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -606,7 +606,7 @@ test_create_anonymous_committed_datatype_invalid_params(const void H5_ATTR_UNUSE
  */
 #ifndef PROBLEMATIC_TESTS
 static void
-test_create_committed_datatype_empty_types(const void H5_ATTR_UNUSED *params)
+test_create_committed_datatype_empty_types(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret         = FAIL;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -780,7 +780,7 @@ test_create_committed_datatype_empty_types(const void H5_ATTR_UNUSED *params)
  * A test to check that a committed datatype can't be re-committed.
  */
 static void
-test_recommit_committed_type(const void H5_ATTR_UNUSED *params)
+test_recommit_committed_type(void H5_ATTR_UNUSED *params)
 {
     htri_t is_committed = false;
     herr_t err_ret;
@@ -887,7 +887,7 @@ test_recommit_committed_type(const void H5_ATTR_UNUSED *params)
  * can be opened using H5Topen2.
  */
 static void
-test_open_committed_datatype(const void H5_ATTR_UNUSED *params)
+test_open_committed_datatype(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -976,7 +976,7 @@ test_open_committed_datatype(const void H5_ATTR_UNUSED *params)
  * be opened when H5Topen2 is passed invalid parameters.
  */
 static void
-test_open_committed_datatype_invalid_params(const void H5_ATTR_UNUSED *params)
+test_open_committed_datatype_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -1144,7 +1144,7 @@ test_open_committed_datatype_invalid_params(const void H5_ATTR_UNUSED *params)
  * through H5Dget_type without causing problems.
  */
 static void
-test_reopen_committed_datatype_indirect(const void H5_ATTR_UNUSED *params)
+test_reopen_committed_datatype_indirect(void H5_ATTR_UNUSED *params)
 {
     size_t dt_size         = 0;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -1616,7 +1616,7 @@ test_reopen_committed_datatype_indirect(const void H5_ATTR_UNUSED *params)
  * it is passed an invalid datatype ID.
  */
 static void
-test_close_committed_datatype_invalid_id(const void H5_ATTR_UNUSED *params)
+test_close_committed_datatype_invalid_id(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
     hid_t  file_id = H5I_INVALID_HID;
@@ -1672,7 +1672,7 @@ test_close_committed_datatype_invalid_id(const void H5_ATTR_UNUSED *params)
  * be retrieved later with a call to H5Tget_create_plist.
  */
 static void
-test_datatype_property_lists(const void H5_ATTR_UNUSED *params)
+test_datatype_property_lists(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -1887,7 +1887,7 @@ test_datatype_property_lists(const void H5_ATTR_UNUSED *params)
  * a committed datatype.
  */
 static void
-test_create_dataset_with_committed_type(const void H5_ATTR_UNUSED *params)
+test_create_dataset_with_committed_type(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -2030,7 +2030,7 @@ test_create_dataset_with_committed_type(const void H5_ATTR_UNUSED *params)
  * using a committed datatype.
  */
 static void
-test_create_attribute_with_committed_type(const void H5_ATTR_UNUSED *params)
+test_create_attribute_with_committed_type(void H5_ATTR_UNUSED *params)
 {
     htri_t attr_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -2163,7 +2163,7 @@ test_create_attribute_with_committed_type(const void H5_ATTR_UNUSED *params)
  * be deleted.
  */
 static void
-test_delete_committed_type(const void H5_ATTR_UNUSED *params)
+test_delete_committed_type(void H5_ATTR_UNUSED *params)
 {
     htri_t type_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -2275,7 +2275,7 @@ test_delete_committed_type(const void H5_ATTR_UNUSED *params)
  * the link to the datatype is deleted and then a new one is created.
  */
 static void
-test_resurrect_datatype(const void H5_ATTR_UNUSED *params)
+test_resurrect_datatype(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID;
@@ -2412,7 +2412,7 @@ test_resurrect_datatype(const void H5_ATTR_UNUSED *params)
 }
 
 static void
-test_flush_committed_datatype(const void H5_ATTR_UNUSED *params)
+test_flush_committed_datatype(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Tflush");
 
@@ -2422,7 +2422,7 @@ test_flush_committed_datatype(const void H5_ATTR_UNUSED *params)
 }
 
 static void
-test_flush_committed_datatype_invalid_params(const void H5_ATTR_UNUSED *params)
+test_flush_committed_datatype_invalid_params(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Tflush with invalid parameters");
 
@@ -2432,7 +2432,7 @@ test_flush_committed_datatype_invalid_params(const void H5_ATTR_UNUSED *params)
 }
 
 static void
-test_refresh_committed_datatype(const void H5_ATTR_UNUSED *params)
+test_refresh_committed_datatype(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Trefresh");
 
@@ -2442,7 +2442,7 @@ test_refresh_committed_datatype(const void H5_ATTR_UNUSED *params)
 }
 
 static void
-test_refresh_committed_datatype_invalid_params(const void H5_ATTR_UNUSED *params)
+test_refresh_committed_datatype_invalid_params(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Trefresh with invalid parameters");
 
@@ -2458,7 +2458,7 @@ test_refresh_committed_datatype_invalid_params(const void H5_ATTR_UNUSED *params
  */
 #ifndef PROBLEMATIC_TESTS
 static void
-test_cant_commit_predefined(const void H5_ATTR_UNUSED *params)
+test_cant_commit_predefined(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -2536,7 +2536,7 @@ test_cant_commit_predefined(const void H5_ATTR_UNUSED *params)
  * A test to check that a datatype cannot be modified once it has been committed.
  */
 static void
-test_cant_modify_committed_type(const void H5_ATTR_UNUSED *params)
+test_cant_modify_committed_type(void H5_ATTR_UNUSED *params)
 {
     htri_t is_committed = false;
     herr_t err_ret;
diff --git a/test/API/H5_api_file_test.c b/test/API/H5_api_file_test.c
index bdcb9e8f9a7..a57a7dd231c 100644
--- a/test/API/H5_api_file_test.c
+++ b/test/API/H5_api_file_test.c
@@ -12,27 +12,27 @@
 
 #include "H5_api_file_test.h"
 
-static void print_file_test_header(const void *params);
-static void test_create_file(const void *params);
-static void test_create_file_invalid_params(const void *params);
-static void test_create_file_excl(const void *params);
-static void test_open_file(const void *params);
-static void test_open_file_invalid_params(const void *params);
-static void test_open_nonexistent_file(const void *params);
-static void test_file_open_overlap(const void *params);
-static void test_file_permission(const void *params);
-static void test_reopen_file(const void *params);
-static void test_close_file_invalid_id(const void *params);
-static void test_flush_file(const void *params);
-static void test_file_is_accessible(const void *params);
-static void test_file_property_lists(const void *params);
-static void test_get_file_intent(const void *params);
-static void test_get_file_obj_count(const void *params);
-static void test_file_mounts(const void *params);
-static void test_get_file_name(const void *params);
+static void print_file_test_header(void *params);
+static void test_create_file(void *params);
+static void test_create_file_invalid_params(void *params);
+static void test_create_file_excl(void *params);
+static void test_open_file(void *params);
+static void test_open_file_invalid_params(void *params);
+static void test_open_nonexistent_file(void *params);
+static void test_file_open_overlap(void *params);
+static void test_file_permission(void *params);
+static void test_reopen_file(void *params);
+static void test_close_file_invalid_id(void *params);
+static void test_flush_file(void *params);
+static void test_file_is_accessible(void *params);
+static void test_file_property_lists(void *params);
+static void test_get_file_intent(void *params);
+static void test_get_file_obj_count(void *params);
+static void test_file_mounts(void *params);
+static void test_get_file_name(void *params);
 
 static void
-print_file_test_header(const void H5_ATTR_UNUSED *params)
+print_file_test_header(void H5_ATTR_UNUSED *params)
 {
     printf("\n");
     printf("**********************************************\n");
@@ -46,7 +46,7 @@ print_file_test_header(const void H5_ATTR_UNUSED *params)
  * Tests that a file can be created.
  */
 static void
-test_create_file(const void H5_ATTR_UNUSED *params)
+test_create_file(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id           = H5I_INVALID_HID;
     char *prefixed_filename = NULL;
@@ -103,7 +103,7 @@ test_create_file(const void H5_ATTR_UNUSED *params)
  * invalid parameters.
  */
 static void
-test_create_file_invalid_params(const void H5_ATTR_UNUSED *params)
+test_create_file_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id           = H5I_INVALID_HID;
     char *prefixed_filename = NULL;
@@ -254,7 +254,7 @@ test_create_file_invalid_params(const void H5_ATTR_UNUSED *params)
  * using the H5F_ACC_EXCL flag while the file already exists.
  */
 static void
-test_create_file_excl(const void H5_ATTR_UNUSED *params)
+test_create_file_excl(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id           = H5I_INVALID_HID;
     hid_t file_id2          = H5I_INVALID_HID;
@@ -338,7 +338,7 @@ test_create_file_excl(const void H5_ATTR_UNUSED *params)
  * Tests that a file can be opened.
  */
 static void
-test_open_file(const void H5_ATTR_UNUSED *params)
+test_open_file(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id = H5I_INVALID_HID;
 
@@ -422,7 +422,7 @@ test_open_file(const void H5_ATTR_UNUSED *params)
  * invalid parameters.
  */
 static void
-test_open_file_invalid_params(const void H5_ATTR_UNUSED *params)
+test_open_file_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id = H5I_INVALID_HID;
 
@@ -523,7 +523,7 @@ test_open_file_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test to ensure that opening a file which doesn't exist will fail.
  */
 static void
-test_open_nonexistent_file(const void H5_ATTR_UNUSED *params)
+test_open_nonexistent_file(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id           = H5I_INVALID_HID;
     char *prefixed_filename = NULL;
@@ -586,7 +586,7 @@ test_open_nonexistent_file(const void H5_ATTR_UNUSED *params)
  * and things are handled appropriately.
  */
 static void
-test_file_permission(const void H5_ATTR_UNUSED *params)
+test_file_permission(void H5_ATTR_UNUSED *params)
 {
     hid_t  file_id           = H5I_INVALID_HID;
     hid_t  dset_id           = H5I_INVALID_HID;
@@ -825,7 +825,7 @@ test_file_permission(const void H5_ATTR_UNUSED *params)
  * A test to check that a file can be re-opened with H5Freopen.
  */
 static void
-test_reopen_file(const void H5_ATTR_UNUSED *params)
+test_reopen_file(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id  = H5I_INVALID_HID;
     hid_t file_id2 = H5I_INVALID_HID;
@@ -875,7 +875,7 @@ test_reopen_file(const void H5_ATTR_UNUSED *params)
  * A test to check that H5Fclose doesn't succeed for an
  * invalid file ID */
 static void
-test_close_file_invalid_id(const void H5_ATTR_UNUSED *params)
+test_close_file_invalid_id(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
 
@@ -912,7 +912,7 @@ test_close_file_invalid_id(const void H5_ATTR_UNUSED *params)
  * A test to check that a file can be flushed using H5Fflush.
  */
 static void
-test_flush_file(const void H5_ATTR_UNUSED *params)
+test_flush_file(void H5_ATTR_UNUSED *params)
 {
     hid_t    file_id           = H5I_INVALID_HID;
     hid_t    dspace_id         = H5I_INVALID_HID;
@@ -1037,7 +1037,7 @@ test_flush_file(const void H5_ATTR_UNUSED *params)
  * A test for H5Fis_accessible.
  */
 static void
-test_file_is_accessible(const void H5_ATTR_UNUSED *params)
+test_file_is_accessible(void H5_ATTR_UNUSED *params)
 {
     const char *const fake_filename     = "nonexistent_file.h5";
     char             *prefixed_filename = NULL;
@@ -1125,7 +1125,7 @@ test_file_is_accessible(const void H5_ATTR_UNUSED *params)
  * can be retrieved with a call to H5Fget_access_plist.
  */
 static void
-test_file_property_lists(const void H5_ATTR_UNUSED *params)
+test_file_property_lists(void H5_ATTR_UNUSED *params)
 {
     hsize_t prop_val           = 0;
     hid_t   file_id1           = H5I_INVALID_HID;
@@ -1435,7 +1435,7 @@ test_file_property_lists(const void H5_ATTR_UNUSED *params)
  * A test to check that the file intent flags can be retrieved.
  */
 static void
-test_get_file_intent(const void H5_ATTR_UNUSED *params)
+test_get_file_intent(void H5_ATTR_UNUSED *params)
 {
     unsigned file_intent;
     hid_t    file_id           = H5I_INVALID_HID;
@@ -1598,7 +1598,7 @@ test_get_file_intent(const void H5_ATTR_UNUSED *params)
  * can be retrieved.
  */
 static void
-test_get_file_obj_count(const void H5_ATTR_UNUSED *params)
+test_get_file_obj_count(void H5_ATTR_UNUSED *params)
 {
     ssize_t obj_count;
     hid_t   file_id            = H5I_INVALID_HID;
@@ -1951,7 +1951,7 @@ test_get_file_obj_count(const void H5_ATTR_UNUSED *params)
  * works correctly.
  */
 static void
-test_file_open_overlap(const void H5_ATTR_UNUSED *params)
+test_file_open_overlap(void H5_ATTR_UNUSED *params)
 {
     ssize_t obj_count;
     hid_t   file_id           = H5I_INVALID_HID;
@@ -2093,7 +2093,7 @@ test_file_open_overlap(const void H5_ATTR_UNUSED *params)
  * correctly.
  */
 static void
-test_file_mounts(const void H5_ATTR_UNUSED *params)
+test_file_mounts(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id           = H5I_INVALID_HID;
     hid_t child_fid         = H5I_INVALID_HID;
@@ -2185,7 +2185,7 @@ test_file_mounts(const void H5_ATTR_UNUSED *params)
  * A test to ensure that a file's name can be retrieved.
  */
 static void
-test_get_file_name(const void H5_ATTR_UNUSED *params)
+test_get_file_name(void H5_ATTR_UNUSED *params)
 {
     ssize_t file_name_buf_len = 0;
     hid_t   file_id           = H5I_INVALID_HID;
diff --git a/test/API/H5_api_group_test.c b/test/API/H5_api_group_test.c
index ce784e0c04f..6bc490dbb53 100644
--- a/test/API/H5_api_group_test.c
+++ b/test/API/H5_api_group_test.c
@@ -12,29 +12,29 @@
 
 #include "H5_api_group_test.h"
 
-static void print_group_test_header(const void *params);
-static void test_create_group_under_root(const void *params);
-static void test_create_group_under_existing_group(const void *params);
-static void test_create_many_groups(const void *params);
-static void test_create_deep_groups(const void *params);
-static void test_create_intermediate_group(const void *params);
-static void test_create_group_invalid_params(const void *params);
-static void test_create_anonymous_group(const void *params);
-static void test_create_anonymous_group_invalid_params(const void *params);
-static void test_open_nonexistent_group(const void *params);
-static void test_open_group_invalid_params(const void *params);
-static void test_close_group_invalid_id(const void *params);
-static void test_group_property_lists(const void *params);
-static void test_get_group_info(const void *params);
-static void test_get_group_info_invalid_params(const void *params);
-static void test_flush_group(const void *params);
-static void test_flush_group_invalid_params(const void *params);
-static void test_refresh_group(const void *params);
-static void test_refresh_group_invalid_params(const void *params);
+static void print_group_test_header(void *params);
+static void test_create_group_under_root(void *params);
+static void test_create_group_under_existing_group(void *params);
+static void test_create_many_groups(void *params);
+static void test_create_deep_groups(void *params);
+static void test_create_intermediate_group(void *params);
+static void test_create_group_invalid_params(void *params);
+static void test_create_anonymous_group(void *params);
+static void test_create_anonymous_group_invalid_params(void *params);
+static void test_open_nonexistent_group(void *params);
+static void test_open_group_invalid_params(void *params);
+static void test_close_group_invalid_id(void *params);
+static void test_group_property_lists(void *params);
+static void test_get_group_info(void *params);
+static void test_get_group_info_invalid_params(void *params);
+static void test_flush_group(void *params);
+static void test_flush_group_invalid_params(void *params);
+static void test_refresh_group(void *params);
+static void test_refresh_group_invalid_params(void *params);
 static int  create_group_recursive(hid_t parent_gid, unsigned counter);
 
 static void
-print_group_test_header(const void H5_ATTR_UNUSED *params)
+print_group_test_header(void H5_ATTR_UNUSED *params)
 {
     printf("\n");
     printf("**********************************************\n");
@@ -48,7 +48,7 @@ print_group_test_header(const void H5_ATTR_UNUSED *params)
  * A test to check that a group can be created under the root group.
  */
 static void
-test_create_group_under_root(const void H5_ATTR_UNUSED *params)
+test_create_group_under_root(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id    = H5I_INVALID_HID;
     hid_t parent_gid = H5I_INVALID_HID;
@@ -101,7 +101,7 @@ test_create_group_under_root(const void H5_ATTR_UNUSED *params)
  * group which is not the root group.
  */
 static void
-test_create_group_under_existing_group(const void H5_ATTR_UNUSED *params)
+test_create_group_under_existing_group(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t parent_group_id = H5I_INVALID_HID, child_group_id = H5I_INVALID_HID,
@@ -177,7 +177,7 @@ test_create_group_under_existing_group(const void H5_ATTR_UNUSED *params)
  * A test to create many (one million) groups
  */
 static void
-test_create_many_groups(const void H5_ATTR_UNUSED *params)
+test_create_many_groups(void H5_ATTR_UNUSED *params)
 {
     hid_t    file_id         = H5I_INVALID_HID;
     hid_t    container_group = H5I_INVALID_HID;
@@ -257,7 +257,7 @@ test_create_many_groups(const void H5_ATTR_UNUSED *params)
  * A test to create groups of the depth GROUP_DEPTH.
  */
 static void
-test_create_deep_groups(const void H5_ATTR_UNUSED *params)
+test_create_deep_groups(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID;
@@ -365,7 +365,7 @@ create_group_recursive(hid_t parent_gid, unsigned counter)
  * A test to create groups automatically using H5Pset_create_intermediate_group
  */
 static void
-test_create_intermediate_group(const void H5_ATTR_UNUSED *params)
+test_create_intermediate_group(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id           = H5I_INVALID_HID;
     hid_t container_group   = H5I_INVALID_HID;
@@ -514,7 +514,7 @@ test_create_intermediate_group(const void H5_ATTR_UNUSED *params)
  * is passed invalid parameters.
  */
 static void
-test_create_group_invalid_params(const void H5_ATTR_UNUSED *params)
+test_create_group_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id  = H5I_INVALID_HID;
     hid_t group_id = H5I_INVALID_HID;
@@ -689,7 +689,7 @@ test_create_group_invalid_params(const void H5_ATTR_UNUSED *params)
  * H5Gcreate_anon.
  */
 static void
-test_create_anonymous_group(const void H5_ATTR_UNUSED *params)
+test_create_anonymous_group(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, new_group_id = H5I_INVALID_HID;
@@ -749,7 +749,7 @@ test_create_anonymous_group(const void H5_ATTR_UNUSED *params)
  * when H5Gcreate_anon is passed invalid parameters.
  */
 static void
-test_create_anonymous_group_invalid_params(const void H5_ATTR_UNUSED *params)
+test_create_anonymous_group_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, new_group_id = H5I_INVALID_HID;
@@ -874,7 +874,7 @@ test_create_anonymous_group_invalid_params(const void H5_ATTR_UNUSED *params)
  * be opened.
  */
 static void
-test_open_nonexistent_group(const void H5_ATTR_UNUSED *params)
+test_open_nonexistent_group(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id  = H5I_INVALID_HID;
     hid_t group_id = H5I_INVALID_HID;
@@ -929,7 +929,7 @@ test_open_nonexistent_group(const void H5_ATTR_UNUSED *params)
  * is passed invalid parameters.
  */
 static void
-test_open_group_invalid_params(const void H5_ATTR_UNUSED *params)
+test_open_group_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id  = H5I_INVALID_HID;
     hid_t group_id = H5I_INVALID_HID;
@@ -1058,7 +1058,7 @@ test_open_group_invalid_params(const void H5_ATTR_UNUSED *params)
  * invalid group ID.
  */
 static void
-test_close_group_invalid_id(const void H5_ATTR_UNUSED *params)
+test_close_group_invalid_id(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
 
@@ -1097,7 +1097,7 @@ test_close_group_invalid_id(const void H5_ATTR_UNUSED *params)
  * retrieved later with a call to H5Gget_create_plist.
  */
 static void
-test_group_property_lists(const void H5_ATTR_UNUSED *params)
+test_group_property_lists(void H5_ATTR_UNUSED *params)
 {
     unsigned dummy_prop_val  = GROUP_PROPERTY_LIST_TEST_DUMMY_VAL;
     hid_t    file_id         = H5I_INVALID_HID;
@@ -1367,7 +1367,7 @@ test_group_property_lists(const void H5_ATTR_UNUSED *params)
  * A test for the functionality of H5Gget_info(_by_idx).
  */
 static void
-test_get_group_info(const void H5_ATTR_UNUSED *params)
+test_get_group_info(void H5_ATTR_UNUSED *params)
 {
     H5G_info_t group_info;
     unsigned   i;
@@ -1802,7 +1802,7 @@ test_get_group_info(const void H5_ATTR_UNUSED *params)
  * H5Gget_info(_by_name/_by_idx) is passed invalid parameters.
  */
 static void
-test_get_group_info_invalid_params(const void H5_ATTR_UNUSED *params)
+test_get_group_info_invalid_params(void H5_ATTR_UNUSED *params)
 {
     H5G_info_t group_info;
     herr_t     err_ret = -1;
@@ -2158,7 +2158,7 @@ test_get_group_info_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Gflush.
  */
 static void
-test_flush_group(const void H5_ATTR_UNUSED *params)
+test_flush_group(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID;
@@ -2229,7 +2229,7 @@ test_flush_group(const void H5_ATTR_UNUSED *params)
  * is passed invalid parameters.
  */
 static void
-test_flush_group_invalid_params(const void H5_ATTR_UNUSED *params)
+test_flush_group_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t status;
 
@@ -2266,7 +2266,7 @@ test_flush_group_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Grefresh.
  */
 static void
-test_refresh_group(const void H5_ATTR_UNUSED *params)
+test_refresh_group(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID;
@@ -2337,7 +2337,7 @@ test_refresh_group(const void H5_ATTR_UNUSED *params)
  * is passed invalid parameters.
  */
 static void
-test_refresh_group_invalid_params(const void H5_ATTR_UNUSED *params)
+test_refresh_group_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t status;
 
diff --git a/test/API/H5_api_link_test.c b/test/API/H5_api_link_test.c
index 25249976a53..a88a65fec64 100644
--- a/test/API/H5_api_link_test.c
+++ b/test/API/H5_api_link_test.c
@@ -16,63 +16,63 @@
  * TODO: add link tests for short-circuit success in operator callback
  */
 
-static void print_link_test_header(const void *params);
-static void test_create_hard_link(const void *params);
-static void test_create_hard_link_long_name(const void *params);
-static void test_create_hard_link_many(const void *params);
-static void test_create_hard_link_same_loc(const void *params);
-static void test_create_hard_link_invalid_params(const void *params);
-static void test_create_soft_link_existing_relative(const void *params);
-static void test_create_soft_link_existing_absolute(const void *params);
-static void test_create_soft_link_dangling_relative(const void *params);
-static void test_create_soft_link_dangling_absolute(const void *params);
-static void test_create_soft_link_long_name(const void *params);
-static void test_create_soft_link_many(const void *params);
-static void test_create_soft_link_invalid_params(const void *params);
-static void test_create_external_link(const void *params);
-static void test_create_external_link_dangling(const void *params);
-static void test_create_external_link_multi(const void *params);
-static void test_create_external_link_ping_pong(const void *params);
-static void test_create_external_link_invalid_params(const void *params);
-static void test_create_user_defined_link(const void *params);
-static void test_create_user_defined_link_invalid_params(const void *params);
-static void test_delete_link(const void *params);
-static void test_delete_link_reset_grp_max_crt_order(const void *params);
-static void test_delete_link_invalid_params(const void *params);
-static void test_copy_link(const void *params);
-static void test_copy_links_into_group_with_links(const void *params);
-static void test_copy_link_across_files(const void *params);
-static void test_copy_link_invalid_params(const void *params);
-static void test_move_link(const void *params);
-static void test_move_links_into_group_with_links(const void *params);
-static void test_move_link_across_files(const void *params);
-static void test_move_link_reset_grp_max_crt_order(const void *params);
-static void test_move_link_invalid_params(const void *params);
-static void test_get_link_val(const void *params);
-static void test_get_link_val_invalid_params(const void *params);
-static void test_get_link_info(const void *params);
-static void test_get_link_info_invalid_params(const void *params);
-static void test_get_link_name(const void *params);
-static void test_get_link_name_invalid_params(const void *params);
-static void test_link_iterate_hard_links(const void *params);
-static void test_link_iterate_soft_links(const void *params);
-static void test_link_iterate_external_links(const void *params);
-static void test_link_iterate_ud_links(const void *params);
-static void test_link_iterate_mixed_links(const void *params);
-static void test_link_iterate_invalid_params(const void *params);
-static void test_link_iterate_0_links(const void *params);
-static void test_link_visit_hard_links_no_cycles(const void *params);
-static void test_link_visit_soft_links_no_cycles(const void *params);
-static void test_link_visit_external_links_no_cycles(const void *params);
-static void test_link_visit_ud_links_no_cycles(const void *params);
-static void test_link_visit_mixed_links_no_cycles(const void *params);
-static void test_link_visit_hard_links_cycles(const void *params);
-static void test_link_visit_soft_links_cycles(const void *params);
-static void test_link_visit_external_links_cycles(const void *params);
-static void test_link_visit_ud_links_cycles(const void *params);
-static void test_link_visit_mixed_links_cycles(const void *params);
-static void test_link_visit_invalid_params(const void *params);
-static void test_link_visit_0_links(const void *params);
+static void print_link_test_header(void *params);
+static void test_create_hard_link(void *params);
+static void test_create_hard_link_long_name(void *params);
+static void test_create_hard_link_many(void *params);
+static void test_create_hard_link_same_loc(void *params);
+static void test_create_hard_link_invalid_params(void *params);
+static void test_create_soft_link_existing_relative(void *params);
+static void test_create_soft_link_existing_absolute(void *params);
+static void test_create_soft_link_dangling_relative(void *params);
+static void test_create_soft_link_dangling_absolute(void *params);
+static void test_create_soft_link_long_name(void *params);
+static void test_create_soft_link_many(void *params);
+static void test_create_soft_link_invalid_params(void *params);
+static void test_create_external_link(void *params);
+static void test_create_external_link_dangling(void *params);
+static void test_create_external_link_multi(void *params);
+static void test_create_external_link_ping_pong(void *params);
+static void test_create_external_link_invalid_params(void *params);
+static void test_create_user_defined_link(void *params);
+static void test_create_user_defined_link_invalid_params(void *params);
+static void test_delete_link(void *params);
+static void test_delete_link_reset_grp_max_crt_order(void *params);
+static void test_delete_link_invalid_params(void *params);
+static void test_copy_link(void *params);
+static void test_copy_links_into_group_with_links(void *params);
+static void test_copy_link_across_files(void *params);
+static void test_copy_link_invalid_params(void *params);
+static void test_move_link(void *params);
+static void test_move_links_into_group_with_links(void *params);
+static void test_move_link_across_files(void *params);
+static void test_move_link_reset_grp_max_crt_order(void *params);
+static void test_move_link_invalid_params(void *params);
+static void test_get_link_val(void *params);
+static void test_get_link_val_invalid_params(void *params);
+static void test_get_link_info(void *params);
+static void test_get_link_info_invalid_params(void *params);
+static void test_get_link_name(void *params);
+static void test_get_link_name_invalid_params(void *params);
+static void test_link_iterate_hard_links(void *params);
+static void test_link_iterate_soft_links(void *params);
+static void test_link_iterate_external_links(void *params);
+static void test_link_iterate_ud_links(void *params);
+static void test_link_iterate_mixed_links(void *params);
+static void test_link_iterate_invalid_params(void *params);
+static void test_link_iterate_0_links(void *params);
+static void test_link_visit_hard_links_no_cycles(void *params);
+static void test_link_visit_soft_links_no_cycles(void *params);
+static void test_link_visit_external_links_no_cycles(void *params);
+static void test_link_visit_ud_links_no_cycles(void *params);
+static void test_link_visit_mixed_links_no_cycles(void *params);
+static void test_link_visit_hard_links_cycles(void *params);
+static void test_link_visit_soft_links_cycles(void *params);
+static void test_link_visit_external_links_cycles(void *params);
+static void test_link_visit_ud_links_cycles(void *params);
+static void test_link_visit_mixed_links_cycles(void *params);
+static void test_link_visit_invalid_params(void *params);
+static void test_link_visit_0_links(void *params);
 
 static herr_t link_iter_hard_links_cb(hid_t group_id, const char *name, const H5L_info2_t *info,
                                       void *op_data);
@@ -108,7 +108,7 @@ static herr_t link_visit_invalid_params_cb(hid_t group_id, const char *name, con
 static herr_t link_visit_0_links_cb(hid_t group_id, const char *name, const H5L_info2_t *info, void *op_data);
 
 static void
-print_link_test_header(const void H5_ATTR_UNUSED *params)
+print_link_test_header(void H5_ATTR_UNUSED *params)
 {
     printf("\n");
     printf("**********************************************\n");
@@ -123,7 +123,7 @@ print_link_test_header(const void H5_ATTR_UNUSED *params)
  * using H5Lcreate_hard.
  */
 static void
-test_create_hard_link(const void H5_ATTR_UNUSED *params)
+test_create_hard_link(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -206,7 +206,7 @@ test_create_hard_link(const void H5_ATTR_UNUSED *params)
  * using H5Lcreate_hard.
  */
 static void
-test_create_hard_link_long_name(const void H5_ATTR_UNUSED *params)
+test_create_hard_link_long_name(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id = H5I_INVALID_HID;
@@ -314,7 +314,7 @@ test_create_hard_link_long_name(const void H5_ATTR_UNUSED *params)
  * using H5Lcreate_hard.
  */
 static void
-test_create_hard_link_many(const void H5_ATTR_UNUSED *params)
+test_create_hard_link_many(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -511,7 +511,7 @@ test_create_hard_link_many(const void H5_ATTR_UNUSED *params)
  * the H5L_SAME_LOC macro for H5Lcreate_hard().
  */
 static void
-test_create_hard_link_same_loc(const void H5_ATTR_UNUSED *params)
+test_create_hard_link_same_loc(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -642,7 +642,7 @@ test_create_hard_link_same_loc(const void H5_ATTR_UNUSED *params)
  * H5Lcreate_hard is passed invalid parameters.
  */
 static void
-test_create_hard_link_invalid_params(const void H5_ATTR_UNUSED *params)
+test_create_hard_link_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
     htri_t link_exists;
@@ -897,6 +897,12 @@ test_create_hard_link_invalid_params(const void H5_ATTR_UNUSED *params)
                 PART_ERROR(H5Lcreate_hard_across_files);
             }
 
+            if (H5Fclose(ext_file_id) < 0) {
+                H5_FAILED();
+                printf("    couldn't close external file '%s'\n", ext_link_filename);
+                PART_ERROR(H5Lcreate_hard_across_files);
+            }
+
             if (remove_test_file(NULL, ext_link_filename) < 0) {
                 H5_FAILED();
                 printf("    failed to delete external file '%s'\n", ext_link_filename);
@@ -934,8 +940,6 @@ test_create_hard_link_invalid_params(const void H5_ATTR_UNUSED *params)
 
     TESTING_2("test cleanup");
 
-    if (H5Fclose(ext_file_id) < 0)
-        TEST_ERROR;
     if (H5Gclose(group_id) < 0)
         TEST_ERROR;
     if (H5Gclose(container_group) < 0)
@@ -966,7 +970,7 @@ test_create_hard_link_invalid_params(const void H5_ATTR_UNUSED *params)
  * existing object with a relative path, can be created.
  */
 static void
-test_create_soft_link_existing_relative(const void H5_ATTR_UNUSED *params)
+test_create_soft_link_existing_relative(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -1073,7 +1077,7 @@ test_create_soft_link_existing_relative(const void H5_ATTR_UNUSED *params)
  * existing object using an absolute path, can be created.
  */
 static void
-test_create_soft_link_existing_absolute(const void H5_ATTR_UNUSED *params)
+test_create_soft_link_existing_absolute(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -1169,7 +1173,7 @@ test_create_soft_link_existing_absolute(const void H5_ATTR_UNUSED *params)
  * path, can be created.
  */
 static void
-test_create_soft_link_dangling_relative(const void H5_ATTR_UNUSED *params)
+test_create_soft_link_dangling_relative(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -1290,7 +1294,7 @@ test_create_soft_link_dangling_relative(const void H5_ATTR_UNUSED *params)
  * can be created.
  */
 static void
-test_create_soft_link_dangling_absolute(const void H5_ATTR_UNUSED *params)
+test_create_soft_link_dangling_absolute(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -1411,7 +1415,7 @@ test_create_soft_link_dangling_absolute(const void H5_ATTR_UNUSED *params)
  * using H5Lcreate_soft.
  */
 static void
-test_create_soft_link_long_name(const void H5_ATTR_UNUSED *params)
+test_create_soft_link_long_name(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id = H5I_INVALID_HID;
@@ -1519,7 +1523,7 @@ test_create_soft_link_long_name(const void H5_ATTR_UNUSED *params)
  * using H5Lcreate_soft.
  */
 static void
-test_create_soft_link_many(const void H5_ATTR_UNUSED *params)
+test_create_soft_link_many(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists        = FAIL;
     bool   valid_name_matched = false;
@@ -1727,7 +1731,7 @@ test_create_soft_link_many(const void H5_ATTR_UNUSED *params)
  * when H5Lcreate_soft is passed invalid parameters.
  */
 static void
-test_create_soft_link_invalid_params(const void H5_ATTR_UNUSED *params)
+test_create_soft_link_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
     htri_t link_exists;
@@ -1961,7 +1965,7 @@ test_create_soft_link_invalid_params(const void H5_ATTR_UNUSED *params)
  * using H5Lcreate_external.
  */
 static void
-test_create_external_link(const void H5_ATTR_UNUSED *params)
+test_create_external_link(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -2073,7 +2077,7 @@ test_create_external_link(const void H5_ATTR_UNUSED *params)
  * be created.
  */
 static void
-test_create_external_link_dangling(const void H5_ATTR_UNUSED *params)
+test_create_external_link_dangling(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id = H5I_INVALID_HID, ext_file_id = H5I_INVALID_HID;
@@ -2209,7 +2213,7 @@ test_create_external_link_dangling(const void H5_ATTR_UNUSED *params)
  * that crosses several files using H5Lcreate_external.
  */
 static void
-test_create_external_link_multi(const void H5_ATTR_UNUSED *params)
+test_create_external_link_multi(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -2590,7 +2594,7 @@ test_create_external_link_multi(const void H5_ATTR_UNUSED *params)
  *       file2:/link6    -> file1: /final
  */
 static void
-test_create_external_link_ping_pong(const void H5_ATTR_UNUSED *params)
+test_create_external_link_ping_pong(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id                                             = H5I_INVALID_HID;
     hid_t group_id                                            = H5I_INVALID_HID;
@@ -2875,7 +2879,7 @@ test_create_external_link_ping_pong(const void H5_ATTR_UNUSED *params)
  * when H5Lcreate_external is passed invalid parameters.
  */
 static void
-test_create_external_link_invalid_params(const void H5_ATTR_UNUSED *params)
+test_create_external_link_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
     htri_t link_exists;
@@ -3166,7 +3170,7 @@ test_create_external_link_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test to check that a user-defined link can be created.
  */
 static void
-test_create_user_defined_link(const void H5_ATTR_UNUSED *params)
+test_create_user_defined_link(void H5_ATTR_UNUSED *params)
 {
     ssize_t udata_size;
     htri_t  link_exists;
@@ -3256,7 +3260,7 @@ test_create_user_defined_link(const void H5_ATTR_UNUSED *params)
  * it is given invalid parameters.
  */
 static void
-test_create_user_defined_link_invalid_params(const void H5_ATTR_UNUSED *params)
+test_create_user_defined_link_invalid_params(void H5_ATTR_UNUSED *params)
 {
     ssize_t udata_size;
     htri_t  link_exists;
@@ -3504,7 +3508,7 @@ test_create_user_defined_link_invalid_params(const void H5_ATTR_UNUSED *params)
  * using H5Ldelete and H5Ldelete_by_idx.
  */
 static void
-test_delete_link(const void H5_ATTR_UNUSED *params)
+test_delete_link(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id = H5I_INVALID_HID, ext_file_id = H5I_INVALID_HID;
@@ -7072,7 +7076,7 @@ test_delete_link(const void H5_ATTR_UNUSED *params)
  * all the links have been deleted from the group.
  */
 static void
-test_delete_link_reset_grp_max_crt_order(const void H5_ATTR_UNUSED *params)
+test_delete_link_reset_grp_max_crt_order(void H5_ATTR_UNUSED *params)
 {
     H5G_info_t grp_info;
     size_t     i;
@@ -7323,7 +7327,7 @@ test_delete_link_reset_grp_max_crt_order(const void H5_ATTR_UNUSED *params)
 }
 
 static void
-test_delete_link_invalid_params(const void H5_ATTR_UNUSED *params)
+test_delete_link_invalid_params(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     herr_t err_ret         = -1;
@@ -7653,7 +7657,7 @@ test_delete_link_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test to check that a link can be copied using H5Lcopy.
  */
 static void
-test_copy_link(const void H5_ATTR_UNUSED *params)
+test_copy_link(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id = H5I_INVALID_HID, ext_file_id = H5I_INVALID_HID;
@@ -8948,7 +8952,7 @@ test_copy_link(const void H5_ATTR_UNUSED *params)
  * creation order values.
  */
 static void
-test_copy_links_into_group_with_links(const void H5_ATTR_UNUSED *params)
+test_copy_links_into_group_with_links(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Lcopy adjusting creation order values for copied links");
 
@@ -8967,7 +8971,7 @@ test_copy_links_into_group_with_links(const void H5_ATTR_UNUSED *params)
  *       links.
  */
 static void
-test_copy_link_across_files(const void H5_ATTR_UNUSED *params)
+test_copy_link_across_files(void H5_ATTR_UNUSED *params)
 {
     TESTING("link copying across files");
 
@@ -8983,7 +8987,7 @@ test_copy_link_across_files(const void H5_ATTR_UNUSED *params)
  * when H5Lcopy is passed invalid parameters.
  */
 static void
-test_copy_link_invalid_params(const void H5_ATTR_UNUSED *params)
+test_copy_link_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
     htri_t link_exists;
@@ -9328,7 +9332,7 @@ test_copy_link_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test to check that a link can be moved with H5Lmove.
  */
 static void
-test_move_link(const void H5_ATTR_UNUSED *params)
+test_move_link(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -10897,7 +10901,7 @@ test_move_link(const void H5_ATTR_UNUSED *params)
  * creation order values.
  */
 static void
-test_move_links_into_group_with_links(const void H5_ATTR_UNUSED *params)
+test_move_links_into_group_with_links(void H5_ATTR_UNUSED *params)
 {
     H5L_info2_t link_info;
     size_t      i;
@@ -11083,7 +11087,7 @@ test_move_links_into_group_with_links(const void H5_ATTR_UNUSED *params)
  *       moved links retain their original properties.
  */
 static void
-test_move_link_across_files(const void H5_ATTR_UNUSED *params)
+test_move_link_across_files(void H5_ATTR_UNUSED *params)
 {
     TESTING("link moving across files");
 
@@ -11100,7 +11104,7 @@ test_move_link_across_files(const void H5_ATTR_UNUSED *params)
  * all the links have been moved out of the group.
  */
 static void
-test_move_link_reset_grp_max_crt_order(const void H5_ATTR_UNUSED *params)
+test_move_link_reset_grp_max_crt_order(void H5_ATTR_UNUSED *params)
 {
     H5G_info_t grp_info;
     size_t     i;
@@ -11283,7 +11287,7 @@ test_move_link_reset_grp_max_crt_order(const void H5_ATTR_UNUSED *params)
  * invalid parameters.
  */
 static void
-test_move_link_invalid_params(const void H5_ATTR_UNUSED *params)
+test_move_link_invalid_params(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     herr_t err_ret         = -1;
@@ -11691,7 +11695,7 @@ test_move_link_invalid_params(const void H5_ATTR_UNUSED *params)
  * be retrieved by using H5Lget_val and H5Lget_val_by_idx.
  */
 static void
-test_get_link_val(const void H5_ATTR_UNUSED *params)
+test_get_link_val(void H5_ATTR_UNUSED *params)
 {
     H5L_info2_t link_info;
     const char *ext_link_filepath;
@@ -13848,7 +13852,7 @@ test_get_link_val(const void H5_ATTR_UNUSED *params)
  * retrieved when H5Lget_val(_by_idx) is passed invalid parameters.
  */
 static void
-test_get_link_val_invalid_params(const void H5_ATTR_UNUSED *params)
+test_get_link_val_invalid_params(void H5_ATTR_UNUSED *params)
 {
     H5L_info2_t link_info;
     htri_t      link_exists;
@@ -14195,7 +14199,7 @@ test_get_link_val_invalid_params(const void H5_ATTR_UNUSED *params)
  * H5Lget_info_by_idx2.
  */
 static void
-test_get_link_info(const void H5_ATTR_UNUSED *params)
+test_get_link_info(void H5_ATTR_UNUSED *params)
 {
     H5L_info2_t link_info;
     const char *ext_objname = "/";
@@ -16923,7 +16927,7 @@ test_get_link_info(const void H5_ATTR_UNUSED *params)
  * when H5Lget_info(_by_idx)2 is passed invalid parameters.
  */
 static void
-test_get_link_info_invalid_params(const void H5_ATTR_UNUSED *params)
+test_get_link_info_invalid_params(void H5_ATTR_UNUSED *params)
 {
     H5L_info2_t link_info;
     herr_t      err_ret = -1;
@@ -17255,7 +17259,7 @@ test_get_link_info_invalid_params(const void H5_ATTR_UNUSED *params)
  * retrieved by using H5Lget_name_by_idx.
  */
 static void
-test_get_link_name(const void H5_ATTR_UNUSED *params)
+test_get_link_name(void H5_ATTR_UNUSED *params)
 {
     ssize_t link_name_buf_size = 0;
     htri_t  link_exists;
@@ -19516,7 +19520,7 @@ test_get_link_name(const void H5_ATTR_UNUSED *params)
  * when H5Lget_name_by_idx is passed invalid parameters.
  */
 static void
-test_get_link_name_invalid_params(const void H5_ATTR_UNUSED *params)
+test_get_link_name_invalid_params(void H5_ATTR_UNUSED *params)
 {
     ssize_t ret;
     htri_t  link_exists;
@@ -19782,7 +19786,7 @@ test_get_link_name_invalid_params(const void H5_ATTR_UNUSED *params)
  * name and link creation order.
  */
 static void
-test_link_iterate_hard_links(const void H5_ATTR_UNUSED *params)
+test_link_iterate_hard_links(void H5_ATTR_UNUSED *params)
 {
     size_t i;
     htri_t link_exists;
@@ -20163,7 +20167,7 @@ test_link_iterate_hard_links(const void H5_ATTR_UNUSED *params)
  * name and link creation order.
  */
 static void
-test_link_iterate_soft_links(const void H5_ATTR_UNUSED *params)
+test_link_iterate_soft_links(void H5_ATTR_UNUSED *params)
 {
     size_t i;
     htri_t link_exists;
@@ -20516,7 +20520,7 @@ test_link_iterate_soft_links(const void H5_ATTR_UNUSED *params)
  * name and link creation order.
  */
 static void
-test_link_iterate_external_links(const void H5_ATTR_UNUSED *params)
+test_link_iterate_external_links(void H5_ATTR_UNUSED *params)
 {
     size_t i;
     htri_t link_exists;
@@ -20921,7 +20925,7 @@ test_link_iterate_external_links(const void H5_ATTR_UNUSED *params)
  * actually test the order that objects were created in.
  */
 static void
-test_link_iterate_ud_links(const void H5_ATTR_UNUSED *params)
+test_link_iterate_ud_links(void H5_ATTR_UNUSED *params)
 {
     TESTING("link iteration (only user-defined links)");
 
@@ -20945,7 +20949,7 @@ test_link_iterate_ud_links(const void H5_ATTR_UNUSED *params)
  * TODO refactor link saving portion into its own test
  */
 static void
-test_link_iterate_mixed_links(const void H5_ATTR_UNUSED *params)
+test_link_iterate_mixed_links(void H5_ATTR_UNUSED *params)
 {
     hsize_t saved_idx;
     size_t  i;
@@ -21483,7 +21487,7 @@ test_link_iterate_mixed_links(const void H5_ATTR_UNUSED *params)
  * when given invalid parameters.
  */
 static void
-test_link_iterate_invalid_params(const void H5_ATTR_UNUSED *params)
+test_link_iterate_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
     size_t i;
@@ -21896,7 +21900,7 @@ test_link_iterate_invalid_params(const void H5_ATTR_UNUSED *params)
  * group with no links in it is not problematic.
  */
 static void
-test_link_iterate_0_links(const void H5_ATTR_UNUSED *params)
+test_link_iterate_0_links(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -22138,7 +22142,7 @@ test_link_iterate_0_links(const void H5_ATTR_UNUSED *params)
  * creation order.
  */
 static void
-test_link_visit_hard_links_no_cycles(const void H5_ATTR_UNUSED *params)
+test_link_visit_hard_links_no_cycles(void H5_ATTR_UNUSED *params)
 {
     size_t i;
     htri_t link_exists;
@@ -22538,7 +22542,7 @@ test_link_visit_hard_links_no_cycles(const void H5_ATTR_UNUSED *params)
  * creation order.
  */
 static void
-test_link_visit_soft_links_no_cycles(const void H5_ATTR_UNUSED *params)
+test_link_visit_soft_links_no_cycles(void H5_ATTR_UNUSED *params)
 {
     size_t i;
     htri_t link_exists;
@@ -22918,7 +22922,7 @@ test_link_visit_soft_links_no_cycles(const void H5_ATTR_UNUSED *params)
  * creation order.
  */
 static void
-test_link_visit_external_links_no_cycles(const void H5_ATTR_UNUSED *params)
+test_link_visit_external_links_no_cycles(void H5_ATTR_UNUSED *params)
 {
     size_t i;
     htri_t link_exists;
@@ -23319,7 +23323,7 @@ test_link_visit_external_links_no_cycles(const void H5_ATTR_UNUSED *params)
  * actually test the order that objects were created in.
  */
 static void
-test_link_visit_ud_links_no_cycles(const void H5_ATTR_UNUSED *params)
+test_link_visit_ud_links_no_cycles(void H5_ATTR_UNUSED *params)
 {
     TESTING("link visiting without cycles (only user-defined links)");
 
@@ -23344,7 +23348,7 @@ test_link_visit_ud_links_no_cycles(const void H5_ATTR_UNUSED *params)
  * TODO refactor test to create a macroed number of subgroups
  */
 static void
-test_link_visit_mixed_links_no_cycles(const void H5_ATTR_UNUSED *params)
+test_link_visit_mixed_links_no_cycles(void H5_ATTR_UNUSED *params)
 {
     size_t i;
     htri_t link_exists;
@@ -23832,7 +23836,7 @@ test_link_visit_mixed_links_no_cycles(const void H5_ATTR_UNUSED *params)
  * order of both link name and link creation order.
  */
 static void
-test_link_visit_hard_links_cycles(const void H5_ATTR_UNUSED *params)
+test_link_visit_hard_links_cycles(void H5_ATTR_UNUSED *params)
 {
     size_t i;
     htri_t link_exists;
@@ -24205,7 +24209,7 @@ test_link_visit_hard_links_cycles(const void H5_ATTR_UNUSED *params)
  * order of both link name and link creation order.
  */
 static void
-test_link_visit_soft_links_cycles(const void H5_ATTR_UNUSED *params)
+test_link_visit_soft_links_cycles(void H5_ATTR_UNUSED *params)
 {
     size_t i;
     htri_t link_exists;
@@ -24585,7 +24589,7 @@ test_link_visit_soft_links_cycles(const void H5_ATTR_UNUSED *params)
  * creation order.
  */
 static void
-test_link_visit_external_links_cycles(const void H5_ATTR_UNUSED *params)
+test_link_visit_external_links_cycles(void H5_ATTR_UNUSED *params)
 {
     size_t i;
     htri_t link_exists;
@@ -24973,7 +24977,7 @@ test_link_visit_external_links_cycles(const void H5_ATTR_UNUSED *params)
  * actually test the order that objects were created in.
  */
 static void
-test_link_visit_ud_links_cycles(const void H5_ATTR_UNUSED *params)
+test_link_visit_ud_links_cycles(void H5_ATTR_UNUSED *params)
 {
     TESTING("link visiting with cycles (only user-defined links)");
 
@@ -24993,7 +24997,7 @@ test_link_visit_ud_links_cycles(const void H5_ATTR_UNUSED *params)
  * actually test the order that objects were created in.
  */
 static void
-test_link_visit_mixed_links_cycles(const void H5_ATTR_UNUSED *params)
+test_link_visit_mixed_links_cycles(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     size_t i;
@@ -25433,7 +25437,7 @@ test_link_visit_mixed_links_cycles(const void H5_ATTR_UNUSED *params)
  * it is given invalid parameters.
  */
 static void
-test_link_visit_invalid_params(const void H5_ATTR_UNUSED *params)
+test_link_visit_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
     htri_t link_exists;
@@ -25899,7 +25903,7 @@ test_link_visit_invalid_params(const void H5_ATTR_UNUSED *params)
  * not problematic.
  */
 static void
-test_link_visit_0_links(const void H5_ATTR_UNUSED *params)
+test_link_visit_0_links(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
diff --git a/test/API/H5_api_misc_test.c b/test/API/H5_api_misc_test.c
index 14611a001eb..07b5af8d387 100644
--- a/test/API/H5_api_misc_test.c
+++ b/test/API/H5_api_misc_test.c
@@ -12,16 +12,16 @@
 
 #include "H5_api_misc_test.h"
 
-static void print_misc_test_header(const void *params);
-static void test_open_link_without_leading_slash(const void *params);
-static void test_object_creation_by_absolute_path(const void *params);
-static void test_absolute_vs_relative_path(const void *params);
-static void test_dot_for_object_name(const void *params);
-static void test_symbols_in_compound_field_name(const void *params);
-static void test_double_init_term(const void *params);
+static void print_misc_test_header(void *params);
+static void test_open_link_without_leading_slash(void *params);
+static void test_object_creation_by_absolute_path(void *params);
+static void test_absolute_vs_relative_path(void *params);
+static void test_dot_for_object_name(void *params);
+static void test_symbols_in_compound_field_name(void *params);
+static void test_double_init_term(void *params);
 
 static void
-print_misc_test_header(const void H5_ATTR_UNUSED *params)
+print_misc_test_header(void H5_ATTR_UNUSED *params)
 {
     printf("\n");
     printf("**********************************************\n");
@@ -32,7 +32,7 @@ print_misc_test_header(const void H5_ATTR_UNUSED *params)
 }
 
 static void
-test_open_link_without_leading_slash(const void H5_ATTR_UNUSED *params)
+test_open_link_without_leading_slash(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID;
@@ -134,7 +134,7 @@ test_open_link_without_leading_slash(const void H5_ATTR_UNUSED *params)
 }
 
 static void
-test_object_creation_by_absolute_path(const void H5_ATTR_UNUSED *params)
+test_object_creation_by_absolute_path(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -369,7 +369,7 @@ test_object_creation_by_absolute_path(const void H5_ATTR_UNUSED *params)
 
 /* XXX: Add testing for groups */
 static void
-test_absolute_vs_relative_path(const void H5_ATTR_UNUSED *params)
+test_absolute_vs_relative_path(void H5_ATTR_UNUSED *params)
 {
     htri_t link_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -714,7 +714,7 @@ test_absolute_vs_relative_path(const void H5_ATTR_UNUSED *params)
  * A test to check creating/opening objects with the "." as the name
  */
 static void
-test_dot_for_object_name(const void H5_ATTR_UNUSED *params)
+test_dot_for_object_name(void H5_ATTR_UNUSED *params)
 {
     hid_t  file_id         = H5I_INVALID_HID;
     hid_t  container_group = H5I_INVALID_HID, subgroup_id = H5I_INVALID_HID;
@@ -886,7 +886,7 @@ test_dot_for_object_name(const void H5_ATTR_UNUSED *params)
  * at the moment.
  */
 static void
-test_double_init_term(const void H5_ATTR_UNUSED *params)
+test_double_init_term(void H5_ATTR_UNUSED *params)
 {
     TESTING("double init/term correctness");
 
@@ -901,7 +901,7 @@ test_double_init_term(const void H5_ATTR_UNUSED *params)
 }
 
 static void
-test_symbols_in_compound_field_name(const void H5_ATTR_UNUSED *params)
+test_symbols_in_compound_field_name(void H5_ATTR_UNUSED *params)
 {
     size_t i;
     size_t total_type_size;
diff --git a/test/API/H5_api_object_test.c b/test/API/H5_api_object_test.c
index 85714f2ed1f..200879907f6 100644
--- a/test/API/H5_api_object_test.c
+++ b/test/API/H5_api_object_test.c
@@ -12,37 +12,37 @@
 
 #include "H5_api_object_test.h"
 
-static void print_object_test_header(const void *params);
-static void test_open_object(const void *params);
-static void test_open_object_invalid_params(const void *params);
-static void test_object_exists(const void *params);
-static void test_object_exists_invalid_params(const void *params);
-static void test_get_object_info(const void *params);
-static void test_get_object_info_invalid_params(const void *params);
-static void test_link_object(const void *params);
-static void test_link_object_invalid_params(const void *params);
-static void test_incr_decr_object_refcount(const void *params);
-static void test_incr_decr_object_refcount_invalid_params(const void *params);
-static void test_object_copy_basic(const void *params);
-static void test_object_copy_already_existing(const void *params);
-static void test_object_copy_shallow_group_copy(const void *params);
-static void test_object_copy_no_attributes(const void *params);
-static void test_object_copy_by_soft_link(const void *params);
-static void test_object_copy_group_with_soft_links(const void *params);
-static void test_object_copy_between_files(const void *params);
-static void test_object_copy_invalid_params(const void *params);
-static void test_object_comments(const void *params);
-static void test_object_comments_invalid_params(const void *params);
-static void test_object_visit(const void *params);
-static void test_object_visit_soft_link(const void *params);
-static void test_object_visit_invalid_params(const void *params);
-static void test_close_object(const void *params);
-static void test_close_object_invalid_params(const void *params);
-static void test_close_invalid_objects(const void *params);
-static void test_flush_object(const void *params);
-static void test_flush_object_invalid_params(const void *params);
-static void test_refresh_object(const void *params);
-static void test_refresh_object_invalid_params(const void *params);
+static void print_object_test_header(void *params);
+static void test_open_object(void *params);
+static void test_open_object_invalid_params(void *params);
+static void test_object_exists(void *params);
+static void test_object_exists_invalid_params(void *params);
+static void test_get_object_info(void *params);
+static void test_get_object_info_invalid_params(void *params);
+static void test_link_object(void *params);
+static void test_link_object_invalid_params(void *params);
+static void test_incr_decr_object_refcount(void *params);
+static void test_incr_decr_object_refcount_invalid_params(void *params);
+static void test_object_copy_basic(void *params);
+static void test_object_copy_already_existing(void *params);
+static void test_object_copy_shallow_group_copy(void *params);
+static void test_object_copy_no_attributes(void *params);
+static void test_object_copy_by_soft_link(void *params);
+static void test_object_copy_group_with_soft_links(void *params);
+static void test_object_copy_between_files(void *params);
+static void test_object_copy_invalid_params(void *params);
+static void test_object_comments(void *params);
+static void test_object_comments_invalid_params(void *params);
+static void test_object_visit(void *params);
+static void test_object_visit_soft_link(void *params);
+static void test_object_visit_invalid_params(void *params);
+static void test_close_object(void *params);
+static void test_close_object_invalid_params(void *params);
+static void test_close_invalid_objects(void *params);
+static void test_flush_object(void *params);
+static void test_flush_object_invalid_params(void *params);
+static void test_refresh_object(void *params);
+static void test_refresh_object_invalid_params(void *params);
 
 static herr_t object_copy_attribute_iter_callback(hid_t location_id, const char *attr_name,
                                                   const H5A_info_t *ainfo, void *op_data);
@@ -64,7 +64,7 @@ static herr_t object_visit_noop_callback(hid_t o_id, const char *name, const H5O
                                          void *op_data);
 
 static void
-print_object_test_header(const void H5_ATTR_UNUSED *params)
+print_object_test_header(void H5_ATTR_UNUSED *params)
 {
     printf("\n");
     printf("**********************************************\n");
@@ -85,7 +85,7 @@ print_object_test_header(const void H5_ATTR_UNUSED *params)
  * XXX: test opening through dangling and resolving soft links.
  */
 static void
-test_open_object(const void H5_ATTR_UNUSED *params)
+test_open_object(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -399,7 +399,7 @@ test_open_object(const void H5_ATTR_UNUSED *params)
  * are passed invalid parameters.
  */
 static void
-test_open_object_invalid_params(const void H5_ATTR_UNUSED *params)
+test_open_object_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -769,7 +769,7 @@ test_open_object_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Oexists_by_name.
  */
 static void
-test_object_exists(const void H5_ATTR_UNUSED *params)
+test_object_exists(void H5_ATTR_UNUSED *params)
 {
     htri_t object_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -1035,7 +1035,7 @@ test_object_exists(const void H5_ATTR_UNUSED *params)
  * when it is passed invalid parameters.
  */
 static void
-test_object_exists_invalid_params(const void H5_ATTR_UNUSED *params)
+test_object_exists_invalid_params(void H5_ATTR_UNUSED *params)
 {
     htri_t object_exists;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -1194,7 +1194,7 @@ test_object_exists_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Oget_info(_by_name/_by_idx).
  */
 static void
-test_get_object_info(const void H5_ATTR_UNUSED *params)
+test_get_object_info(void H5_ATTR_UNUSED *params)
 {
     TESTING("object info retrieval");
 
@@ -1209,7 +1209,7 @@ test_get_object_info(const void H5_ATTR_UNUSED *params)
  * parameters.
  */
 static void
-test_get_object_info_invalid_params(const void H5_ATTR_UNUSED *params)
+test_get_object_info_invalid_params(void H5_ATTR_UNUSED *params)
 {
     TESTING("object info retrieval with invalid parameters");
 
@@ -1222,7 +1222,7 @@ test_get_object_info_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Olink.
  */
 static void
-test_link_object(const void H5_ATTR_UNUSED *params)
+test_link_object(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -1381,7 +1381,7 @@ test_link_object(const void H5_ATTR_UNUSED *params)
  * parameters.
  */
 static void
-test_link_object_invalid_params(const void H5_ATTR_UNUSED *params)
+test_link_object_invalid_params(void H5_ATTR_UNUSED *params)
 {
     hid_t  file_id         = H5I_INVALID_HID;
     hid_t  container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -1579,7 +1579,7 @@ test_link_object_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Oincr_refcount/H5Odecr_refcount.
  */
 static void
-test_incr_decr_object_refcount(const void H5_ATTR_UNUSED *params)
+test_incr_decr_object_refcount(void H5_ATTR_UNUSED *params)
 {
     H5O_info2_t oinfo; /* Object info struct */
     hid_t       file_id         = H5I_INVALID_HID;
@@ -1878,7 +1878,7 @@ test_incr_decr_object_refcount(const void H5_ATTR_UNUSED *params)
  * fail when passed invalid parameters.
  */
 static void
-test_incr_decr_object_refcount_invalid_params(const void H5_ATTR_UNUSED *params)
+test_incr_decr_object_refcount_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t status;
 
@@ -1945,7 +1945,7 @@ test_incr_decr_object_refcount_invalid_params(const void H5_ATTR_UNUSED *params)
  * Basic tests for H5Ocopy.
  */
 static void
-test_object_copy_basic(const void H5_ATTR_UNUSED *params)
+test_object_copy_basic(void H5_ATTR_UNUSED *params)
 {
     H5O_info2_t object_info;
     H5G_info_t  group_info;
@@ -2522,7 +2522,7 @@ test_object_copy_basic(const void H5_ATTR_UNUSED *params)
  * an object to a destination where the object already exists.
  */
 static void
-test_object_copy_already_existing(const void H5_ATTR_UNUSED *params)
+test_object_copy_already_existing(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -2720,7 +2720,7 @@ test_object_copy_already_existing(const void H5_ATTR_UNUSED *params)
  * for H5Ocopy.
  */
 static void
-test_object_copy_shallow_group_copy(const void H5_ATTR_UNUSED *params)
+test_object_copy_shallow_group_copy(void H5_ATTR_UNUSED *params)
 {
     H5G_info_t group_info;
     htri_t     object_link_exists;
@@ -2947,7 +2947,7 @@ test_object_copy_shallow_group_copy(const void H5_ATTR_UNUSED *params)
  * of H5Ocopy.
  */
 static void
-test_object_copy_no_attributes(const void H5_ATTR_UNUSED *params)
+test_object_copy_no_attributes(void H5_ATTR_UNUSED *params)
 {
     H5O_info2_t object_info;
     htri_t      object_link_exists;
@@ -3460,7 +3460,7 @@ test_object_copy_no_attributes(const void H5_ATTR_UNUSED *params)
  * object specified is a soft link or dangling soft link.
  */
 static void
-test_object_copy_by_soft_link(const void H5_ATTR_UNUSED *params)
+test_object_copy_by_soft_link(void H5_ATTR_UNUSED *params)
 {
     H5O_info2_t object_info;
     H5G_info_t  group_info;
@@ -3793,7 +3793,7 @@ test_object_copy_by_soft_link(const void H5_ATTR_UNUSED *params)
  * flag.
  */
 static void
-test_object_copy_group_with_soft_links(const void H5_ATTR_UNUSED *params)
+test_object_copy_group_with_soft_links(void H5_ATTR_UNUSED *params)
 {
     H5G_info_t group_info;
     htri_t     object_link_exists;
@@ -4151,7 +4151,7 @@ test_object_copy_group_with_soft_links(const void H5_ATTR_UNUSED *params)
  * H5Ocopy.
  */
 static void
-test_object_copy_between_files(const void H5_ATTR_UNUSED *params)
+test_object_copy_between_files(void H5_ATTR_UNUSED *params)
 {
     H5O_info2_t object_info;
     H5G_info_t  group_info;
@@ -4763,7 +4763,7 @@ test_object_copy_between_files(const void H5_ATTR_UNUSED *params)
  * is passed invalid parameters.
  */
 static void
-test_object_copy_invalid_params(const void H5_ATTR_UNUSED *params)
+test_object_copy_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret         = -1;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -4999,7 +4999,7 @@ test_object_copy_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Oset_comment(_by_name)/H5Oget_comment(_by_name).
  */
 static void
-test_object_comments(const void H5_ATTR_UNUSED *params)
+test_object_comments(void H5_ATTR_UNUSED *params)
 {
     TESTING("object comments");
 
@@ -5013,7 +5013,7 @@ test_object_comments(const void H5_ATTR_UNUSED *params)
  * fail when passed invalid parameters.
  */
 static void
-test_object_comments_invalid_params(const void H5_ATTR_UNUSED *params)
+test_object_comments_invalid_params(void H5_ATTR_UNUSED *params)
 {
     TESTING("object comments with invalid parameters");
 
@@ -5028,7 +5028,7 @@ test_object_comments_invalid_params(const void H5_ATTR_UNUSED *params)
  * XXX: Should have test for checking nested object's names/paths.
  */
 static void
-test_object_visit(const void H5_ATTR_UNUSED *params)
+test_object_visit(void H5_ATTR_UNUSED *params)
 {
     size_t   i;
     hid_t    file_id         = H5I_INVALID_HID;
@@ -5638,8 +5638,8 @@ test_object_visit(const void H5_ATTR_UNUSED *params)
 
             i = 0;
 
-            if (H5Ovisit_by_name(attr_id, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, object_visit_simple_callback,
-                                 &i, H5O_INFO_ALL, H5P_DEFAULT) < 0) {
+            if (H5Ovisit_by_name3(attr_id, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, object_visit_simple_callback,
+                                  &i, H5O_INFO_ALL, H5P_DEFAULT) < 0) {
                 H5_FAILED();
                 printf("    H5Ovisit_by_name on an attribute failed!\n");
                 PART_ERROR(H5Ovisit_by_name_attr);
@@ -5729,7 +5729,7 @@ test_object_visit(const void H5_ATTR_UNUSED *params)
  * do not get visited.
  */
 static void
-test_object_visit_soft_link(const void H5_ATTR_UNUSED *params)
+test_object_visit_soft_link(void H5_ATTR_UNUSED *params)
 {
     size_t i;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -6214,7 +6214,7 @@ test_object_visit_soft_link(const void H5_ATTR_UNUSED *params)
  * it is passed invalid parameters.
  */
 static void
-test_object_visit_invalid_params(const void H5_ATTR_UNUSED *params)
+test_object_visit_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret         = -1;
     hid_t  file_id         = H5I_INVALID_HID;
@@ -6531,7 +6531,7 @@ test_object_visit_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Oclose.
  */
 static void
-test_close_object(const void H5_ATTR_UNUSED *params)
+test_close_object(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id         = H5I_INVALID_HID;
     hid_t container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -6731,7 +6731,7 @@ test_close_object(const void H5_ATTR_UNUSED *params)
  * is passed invalid parameters.
  */
 static void
-test_close_object_invalid_params(const void H5_ATTR_UNUSED *params)
+test_close_object_invalid_params(void H5_ATTR_UNUSED *params)
 {
     herr_t err_ret = -1;
     hid_t  file_id = H5I_INVALID_HID;
@@ -6785,7 +6785,7 @@ test_close_object_invalid_params(const void H5_ATTR_UNUSED *params)
  * and attribute) can't be closed with H5Oclose.
  */
 static void
-test_close_invalid_objects(const void H5_ATTR_UNUSED *params)
+test_close_invalid_objects(void H5_ATTR_UNUSED *params)
 {
     hid_t  file_id         = H5I_INVALID_HID;
     hid_t  container_group = H5I_INVALID_HID, group_id = H5I_INVALID_HID;
@@ -6968,7 +6968,7 @@ test_close_invalid_objects(const void H5_ATTR_UNUSED *params)
  * A test for H5Oflush.
  */
 static void
-test_flush_object(const void H5_ATTR_UNUSED *params)
+test_flush_object(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Oflush");
 
@@ -6982,7 +6982,7 @@ test_flush_object(const void H5_ATTR_UNUSED *params)
  * it is passed invalid parameters.
  */
 static void
-test_flush_object_invalid_params(const void H5_ATTR_UNUSED *params)
+test_flush_object_invalid_params(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Oflush with invalid parameters");
 
@@ -6995,7 +6995,7 @@ test_flush_object_invalid_params(const void H5_ATTR_UNUSED *params)
  * A test for H5Orefresh.
  */
 static void
-test_refresh_object(const void H5_ATTR_UNUSED *params)
+test_refresh_object(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Orefresh");
 
@@ -7009,7 +7009,7 @@ test_refresh_object(const void H5_ATTR_UNUSED *params)
  * it is passed invalid parameters.
  */
 static void
-test_refresh_object_invalid_params(const void H5_ATTR_UNUSED *params)
+test_refresh_object_invalid_params(void H5_ATTR_UNUSED *params)
 {
     TESTING("H5Orefresh with invalid parameters");
 
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 52a17922dc1..a0f7519deb9 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -512,7 +512,7 @@ endif ()
 
 #-- Adding test for direct_chunk
 if (H5_ZLIB_HEADER)
-  message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
+  message(VERBOSE "H5_ZLIB_HEADER for direct_chunk=${H5_ZLIB_HEADER}")
 
   set_source_files_properties(${HDF5_TEST_SOURCE_DIR}/direct_chunk.c
                PROPERTIES COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}"
diff --git a/test/cache.c b/test/cache.c
index f3599e8158d..cd2496205eb 100644
--- a/test/cache.c
+++ b/test/cache.c
@@ -222,7 +222,7 @@ static unsigned check_stats(unsigned paged);
 static void check_stats__smoke_check_1(H5F_t *file_ptr);
 #endif /* H5C_COLLECT_CACHE_STATS */
 
-static H5F_t *setup_cache(size_t max_cache_size, size_t min_clean_size, unsigned paged);
+static H5F_t *setup_cache(size_t max_cache_size, size_t min_clean_size, unsigned paged, H5CX_node_t *api_ctx);
 
 static void takedown_cache(H5F_t *file_ptr, bool dump_stats, bool dump_detailed_stats);
 
@@ -298,14 +298,11 @@ smoke_check_1(int express_test, unsigned paged)
     if (show_progress) /* 2 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    file_ptr = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged);
+    file_ptr = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged, &api_ctx);
 
     if (show_progress) /* 3 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     row_major_scan_forward(/* file_ptr               */ file_ptr,
                            /* max_index              */ max_index,
                            /* lag                    */ lag,
@@ -426,9 +423,6 @@ smoke_check_1(int express_test, unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* smoke_check_1() */
 
@@ -499,14 +493,11 @@ smoke_check_2(int express_test, unsigned paged)
     if (show_progress) /* 2 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    file_ptr = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged);
+    file_ptr = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged, &api_ctx);
 
     if (show_progress) /* 3 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     row_major_scan_forward(/* file_ptr               */ file_ptr,
                            /* max_index              */ max_index,
                            /* lag                    */ lag,
@@ -627,9 +618,6 @@ smoke_check_2(int express_test, unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* smoke_check_2() */
 
@@ -699,14 +687,11 @@ smoke_check_3(int express_test, unsigned paged)
     if (show_progress) /* 2 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+    file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
 
     if (show_progress) /* 3 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     row_major_scan_forward(/* file_ptr               */ file_ptr,
                            /* max_index              */ max_index,
                            /* lag                    */ lag,
@@ -827,9 +812,6 @@ smoke_check_3(int express_test, unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* smoke_check_3() */
 
@@ -900,14 +882,11 @@ smoke_check_4(int express_test, unsigned paged)
     if (show_progress) /* 2 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+    file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
 
     if (show_progress) /* 3 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     row_major_scan_forward(/* file_ptr               */ file_ptr,
                            /* max_index              */ max_index,
                            /* lag                    */ lag,
@@ -1028,9 +1007,6 @@ smoke_check_4(int express_test, unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* smoke_check_4() */
 
@@ -1141,12 +1117,9 @@ smoke_check_5(int express_test, unsigned paged)
     if (show_progress) /* 2 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+    file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     cache_ptr = file_ptr->shared->cache;
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
 
         result = H5C_set_cache_auto_resize_config(cache_ptr, &auto_size_ctl);
@@ -1258,9 +1231,6 @@ smoke_check_5(int express_test, unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* smoke_check_5() */
 
@@ -1371,12 +1341,9 @@ smoke_check_6(int express_test, unsigned paged)
     if (show_progress) /* 2 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+    file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     cache_ptr = file_ptr->shared->cache;
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
 
         result = H5C_set_cache_auto_resize_config(cache_ptr, &auto_size_ctl);
@@ -1488,9 +1455,6 @@ smoke_check_6(int express_test, unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* smoke_check_6() */
 
@@ -1602,12 +1566,9 @@ smoke_check_7(int express_test, unsigned paged)
     if (show_progress) /* 2 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+    file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     cache_ptr = file_ptr->shared->cache;
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
 
         result = H5C_set_cache_auto_resize_config(cache_ptr, &auto_size_ctl);
@@ -1719,9 +1680,6 @@ smoke_check_7(int express_test, unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* smoke_check_7() */
 
@@ -1833,12 +1791,9 @@ smoke_check_8(int express_test, unsigned paged)
     if (show_progress) /* 2 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+    file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     cache_ptr = file_ptr->shared->cache;
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
 
         result = H5C_set_cache_auto_resize_config(cache_ptr, &auto_size_ctl);
@@ -1950,9 +1905,6 @@ smoke_check_8(int express_test, unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* smoke_check_8() */
 
@@ -2029,15 +1981,12 @@ smoke_check_9(int express_test, unsigned paged)
     if (show_progress) /* 2 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    file_ptr  = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged);
+    file_ptr  = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged, &api_ctx);
     cache_ptr = file_ptr->shared->cache;
 
     if (show_progress) /* 3 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     /* disable evictions */
     if (pass) {
 
@@ -2260,9 +2209,6 @@ smoke_check_9(int express_test, unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* smoke_check_9() */
 
@@ -2339,12 +2285,9 @@ smoke_check_10(int express_test, unsigned paged)
     if (show_progress) /* 2 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    file_ptr  = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged);
+    file_ptr  = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged, &api_ctx);
     cache_ptr = file_ptr->shared->cache;
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (show_progress) /* 3 */
         fprintf(stdout, "%s() - %0d -- pass = %d -- evictions enabled\n", __func__, mile_stone++, (int)pass);
 
@@ -2566,9 +2509,6 @@ smoke_check_10(int express_test, unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* smoke_check_10() */
 
@@ -2643,14 +2583,11 @@ write_permitted_check(int
     if (show_progress) /* 2 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    file_ptr = setup_cache((size_t)(1 * 1024 * 1024), (size_t)0, paged);
+    file_ptr = setup_cache((size_t)(1 * 1024 * 1024), (size_t)0, paged, &api_ctx);
 
     if (show_progress) /* 3 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     row_major_scan_forward(/* file_ptr              */ file_ptr,
                            /* max_index              */ max_index,
                            /* lag                    */ lag,
@@ -2769,9 +2706,6 @@ write_permitted_check(int
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
 #else /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
 
     SKIPPED();
@@ -2838,13 +2772,10 @@ check_insert_entry(unsigned paged)
 
         reset_entries();
 
-        file_ptr  = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged);
+        file_ptr  = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged, &api_ctx);
         cache_ptr = file_ptr->shared->cache;
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
 
         insert_entry(file_ptr, entry_type, 0, H5C__NO_FLAGS_SET);
@@ -3037,9 +2968,6 @@ check_insert_entry(unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_insert_entry() */
 
@@ -3076,12 +3004,9 @@ check_flush_cache(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     /* first test behaviour on an empty cache.  Can't do much sanity
      * checking in this case, so simply check the return values.
      */
@@ -3127,9 +3052,6 @@ check_flush_cache(unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_flush_cache() */
 
@@ -9590,7 +9512,7 @@ check_get_entry_status(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged, &api_ctx);
 
         if (file_ptr == NULL) {
 
@@ -9604,9 +9526,6 @@ check_get_entry_status(unsigned paged)
         }
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
 
         /* entry not in cache -- only in_cache should be touched by
@@ -9769,9 +9688,6 @@ check_get_entry_status(unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_get_entry_status() */
 
@@ -9811,15 +9727,12 @@ check_expunge_entry(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged, &api_ctx);
 
         base_addr = entries[0];
         entry_ptr = &(base_addr[0]);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
 
         /* entry not in cache -- only in_cache should be touched by
@@ -10050,9 +9963,6 @@ check_expunge_entry(unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_expunge_entry() */
 
@@ -10108,7 +10018,7 @@ check_multiple_read_protect(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
 #if H5C_COLLECT_CACHE_STATS
         cache_ptr = file_ptr->shared->cache;
 #endif /* H5C_COLLECT_CACHE_STATS */
@@ -10123,9 +10033,6 @@ check_multiple_read_protect(unsigned paged)
         }
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
 #if H5C_COLLECT_CACHE_STATS
     if ((cache_ptr->write_protects[0] != 0) || (cache_ptr->read_protects[0] != 0) ||
         (cache_ptr->max_read_protects[0] != 0)) {
@@ -10440,9 +10347,6 @@ check_multiple_read_protect(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_multiple_read_protect() */
 
@@ -10519,12 +10423,9 @@ check_move_entry(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     u = 0;
     while (pass && (u < NELMTS(test_specs))) {
         check_move_entry__run_test(file_ptr, u, &(test_specs[u]));
@@ -10542,9 +10443,6 @@ check_move_entry(unsigned paged)
     if (!pass)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_move_entry() */
 
@@ -10733,7 +10631,7 @@ check_pin_protected_entry(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged, &api_ctx);
 
         if (file_ptr == NULL) {
 
@@ -10742,9 +10640,6 @@ check_pin_protected_entry(unsigned paged)
         }
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
 
         protect_entry(file_ptr, 0, 0);
@@ -10794,9 +10689,6 @@ check_pin_protected_entry(unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_pin_protected_entry() */
 
@@ -10870,7 +10762,7 @@ check_resize_entry(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged, &api_ctx);
 
         if (file_ptr == NULL) {
 
@@ -10886,9 +10778,6 @@ check_resize_entry(unsigned paged)
         }
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
 
         if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0) || (cache_ptr->slist_len != 0) ||
@@ -11659,9 +11548,6 @@ check_resize_entry(unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_resize_entry() */
 
@@ -11742,7 +11628,7 @@ check_evictions_enabled(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(1 * 1024 * 1024), (size_t)(512 * 1024), paged);
+        file_ptr = setup_cache((size_t)(1 * 1024 * 1024), (size_t)(512 * 1024), paged, &api_ctx);
 
         if (file_ptr == NULL) {
 
@@ -11756,9 +11642,6 @@ check_evictions_enabled(unsigned paged)
         }
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (show_progress) /* 2 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
 
@@ -12306,9 +12189,6 @@ check_evictions_enabled(unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_evictions_enabled() */
 
@@ -12343,13 +12223,10 @@ check_flush_protected_err(unsigned paged)
      */
 
     reset_entries();
-    file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+    file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     if (pass)
         cache_ptr = file_ptr->shared->cache;
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass)
         protect_entry(file_ptr, 0, 0);
 
@@ -12395,9 +12272,6 @@ check_flush_protected_err(unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_flush_protected_err() */
 
@@ -12433,12 +12307,9 @@ check_destroy_pinned_err(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         protect_entry(file_ptr, 0, 0);
         unprotect_entry(file_ptr, 0, 0, H5C__PIN_ENTRY_FLAG);
@@ -12485,9 +12356,6 @@ check_destroy_pinned_err(unsigned paged)
     if (!pass)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_destroy_pinned_err() */
 
@@ -12524,12 +12392,9 @@ check_destroy_protected_err(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         /* Note: normally this call would go just before the series of
          * flushes prior to file close -- in particular, all entries
@@ -12584,9 +12449,6 @@ check_destroy_protected_err(unsigned paged)
     if (!pass)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_destroy_protected_err() */
 
@@ -12626,12 +12488,9 @@ check_duplicate_insert_err(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         protect_entry(file_ptr, 0, 0);
 
@@ -12669,9 +12528,6 @@ check_duplicate_insert_err(unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_duplicate_insert_err() */
 
@@ -12711,12 +12567,9 @@ check_double_pin_err(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         protect_entry(file_ptr, 0, 0);
 
@@ -12759,9 +12612,6 @@ check_double_pin_err(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_double_pin_err() */
 
@@ -12803,12 +12653,9 @@ check_double_unpin_err(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         protect_entry(file_ptr, 0, 0);
 
@@ -12858,9 +12705,6 @@ check_double_unpin_err(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_double_unpin_err() */
 
@@ -12906,12 +12750,9 @@ check_pin_entry_errs(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         protect_entry(file_ptr, 0, 0);
 
@@ -12971,9 +12812,6 @@ check_pin_entry_errs(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_pin_entry_errs() */
 
@@ -13012,12 +12850,9 @@ check_double_protect_err(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         protect_entry(file_ptr, 0, 0);
 
@@ -13058,9 +12893,6 @@ check_double_protect_err(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_double_protect_err() */
 
@@ -13099,12 +12931,9 @@ check_double_unprotect_err(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         protect_entry(file_ptr, 0, 0);
 
@@ -13141,9 +12970,6 @@ check_double_unprotect_err(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_double_unprotect_err() */
 
@@ -13186,12 +13012,9 @@ check_mark_entry_dirty_errs(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         protect_entry(file_ptr, 0, 0);
 
@@ -13228,9 +13051,6 @@ check_mark_entry_dirty_errs(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_mark_entry_dirty_errs() */
 
@@ -13276,12 +13096,9 @@ check_expunge_entry_errs(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         entry_ptr = &((entries[0])[0]);
 
@@ -13346,9 +13163,6 @@ check_expunge_entry_errs(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_expunge_entry_errs() */
 
@@ -13389,13 +13203,10 @@ check_move_entry_errs(unsigned paged)
     if (pass) {
         reset_entries();
 
-        file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
         cache_ptr = file_ptr->shared->cache;
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         insert_entry(file_ptr, 0, 0, H5C__NO_FLAGS_SET);
         insert_entry(file_ptr, 0, 1, H5C__NO_FLAGS_SET);
@@ -13427,9 +13238,6 @@ check_move_entry_errs(unsigned paged)
     if (pass)
         takedown_cache(file_ptr, false, false);
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     /* Allocate a cache, protect an entry R/O, and then call
      * H5C_move_entry() to move it -- this should fail.
      *
@@ -13440,13 +13248,10 @@ check_move_entry_errs(unsigned paged)
     if (pass) {
         reset_entries();
 
-        file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
         cache_ptr = file_ptr->shared->cache;
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         insert_entry(file_ptr, 0, 0, H5C__NO_FLAGS_SET);
         protect_entry_ro(file_ptr, 0, 0);
@@ -13476,9 +13281,6 @@ check_move_entry_errs(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     } /* end else */
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_move_entry_errs() */
 
@@ -13523,12 +13325,9 @@ check_resize_entry_errs(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         entry_ptr = &((entries[0])[0]);
 
@@ -13582,9 +13381,6 @@ check_resize_entry_errs(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_resize_entry_errs() */
 
@@ -13623,12 +13419,9 @@ check_unprotect_ro_dirty_err(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         protect_entry_ro(file_ptr, 0, 0);
 
@@ -13656,9 +13449,6 @@ check_unprotect_ro_dirty_err(unsigned paged)
         takedown_cache(file_ptr, false, false);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     /* allocate a another cache, protect an entry read only twice, and
      * then unprotect it with the dirtied flag set.  This should fail.
      * Unprotect it with no flags set twice and then destroy the cache.
@@ -13669,12 +13459,9 @@ check_unprotect_ro_dirty_err(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         protect_entry_ro(file_ptr, 0, 0);
         protect_entry_ro(file_ptr, 0, 0);
@@ -13716,9 +13503,6 @@ check_unprotect_ro_dirty_err(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_unprotect_ro_dirty_err() */
 
@@ -13758,12 +13542,9 @@ check_protect_ro_rw_err(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         protect_entry_ro(file_ptr, 0, 0);
 
@@ -13804,9 +13585,6 @@ check_protect_ro_rw_err(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_protect_ro_rw_err() */
 
@@ -13844,16 +13622,13 @@ check_protect_retries(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
 
         /* Set up read attempts for verifying checksum */
         file_ptr->shared->read_attempts = 10;
         file_ptr->shared->retries_nbins = 1;
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     /* Test only for this type which has a speculative load */
     type = VARIABLE_ENTRY_TYPE;
     idx  = 0;
@@ -13937,9 +13712,6 @@ check_protect_retries(unsigned paged)
         fprintf(stdout, "%s: failure_msg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_protect_retries() */
 
@@ -13987,13 +13759,10 @@ check_check_evictions_enabled_err(unsigned paged)
 
         reset_entries();
 
-        file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
         cache_ptr = file_ptr->shared->cache;
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
 
         result = H5C_get_evictions_enabled(NULL, &evictions_enabled);
@@ -14076,9 +13845,6 @@ check_check_evictions_enabled_err(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_evictions_enabled_err() */
 
@@ -14178,13 +13944,10 @@ check_auto_cache_resize(bool cork_ageout, unsigned paged)
     if (pass) {
         reset_entries();
 
-        file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
         cache_ptr = file_ptr->shared->cache;
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
         result = H5C_set_cache_auto_resize_config(cache_ptr, &auto_size_ctl);
         if (result != SUCCEED) {
@@ -17944,9 +17707,6 @@ check_auto_cache_resize(bool cork_ageout, unsigned paged)
     if (!pass)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_auto_cache_resize() */
 
@@ -18032,7 +17792,7 @@ check_auto_cache_resize_disable(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
 
         if (file_ptr == NULL) {
 
@@ -18045,9 +17805,6 @@ check_auto_cache_resize_disable(unsigned paged)
         }
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
 
         result = H5C_set_cache_auto_resize_config(cache_ptr, &auto_size_ctl);
@@ -20543,9 +20300,6 @@ check_auto_cache_resize_disable(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_auto_cache_resize_disable() */
 
@@ -20627,13 +20381,10 @@ check_auto_cache_resize_epoch_markers(unsigned paged)
 
         reset_entries();
 
-        file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
         cache_ptr = file_ptr->shared->cache;
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
 
         result = H5C_set_cache_auto_resize_config(cache_ptr, &auto_size_ctl);
@@ -21231,9 +20982,6 @@ check_auto_cache_resize_epoch_markers(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_auto_cache_resize_epoch_markers() */
 
@@ -21317,13 +21065,10 @@ check_auto_cache_resize_input_errs(unsigned paged)
 
         reset_entries();
 
-        file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
         cache_ptr = file_ptr->shared->cache;
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
 
         result = H5C_set_cache_auto_resize_config(cache_ptr, &ref_auto_size_ctl);
@@ -23448,9 +23193,6 @@ check_auto_cache_resize_input_errs(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_auto_cache_resize_input_errs() */
 
@@ -23538,13 +23280,10 @@ check_auto_cache_resize_aux_fcns(unsigned paged)
 
         reset_entries();
 
-        file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
         cache_ptr = file_ptr->shared->cache;
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (pass) {
 
         result = H5C_set_cache_auto_resize_config(cache_ptr, &auto_size_ctl);
@@ -23940,9 +23679,6 @@ check_auto_cache_resize_aux_fcns(unsigned paged)
         fprintf(stdout, "%s: failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_auto_cache_resize_aux_fcns() */
 
@@ -24049,7 +23785,7 @@ check_metadata_blizzard_absence(bool fill_via_insertion, unsigned paged)
          * The max_cache_size should have room for 50 entries.
          * The min_clean_size is half of that, or 25 entries.
          */
-        file_ptr = setup_cache((size_t)(50 * entry_size), (size_t)(25 * entry_size), paged);
+        file_ptr = setup_cache((size_t)(50 * entry_size), (size_t)(25 * entry_size), paged, &api_ctx);
 
         if (file_ptr == NULL) {
 
@@ -24060,9 +23796,6 @@ check_metadata_blizzard_absence(bool fill_via_insertion, unsigned paged)
             cache_ptr = file_ptr->shared->cache;
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     if (show_progress) /* 1 */
         fprintf(stdout, "%s: check point %d -- pass %d\n", __func__, checkpoint++, pass);
 
@@ -24675,9 +24408,6 @@ check_metadata_blizzard_absence(bool fill_via_insertion, unsigned paged)
         H5_FAILED();
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_metadata_blizzard_absence() */
 
@@ -24726,16 +24456,13 @@ check_flush_deps(unsigned paged)
      */
 
     reset_entries();
-    file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+    file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     cache_ptr = file_ptr->shared->cache;
     base_addr = entries[entry_type];
 
     if (!pass)
         CACHE_ERROR("setup_cache failed")
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     /* Insert entries to work with into the cache */
     for (u = 0; u < 5; u++) {
         insert_entry(file_ptr, entry_type, (int32_t)u, H5C__NO_FLAGS_SET);
@@ -26602,9 +26329,6 @@ check_flush_deps(unsigned paged)
         fprintf(stdout, "%s.\n", failure_mssg);
     } /* end else */
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_flush_deps() */
 
@@ -26642,13 +26366,10 @@ check_flush_deps_err(unsigned paged)
 
         /* Allocate a cache */
         reset_entries();
-        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
         if (!pass)
             CACHE_ERROR("setup_cache failed")
 
-        /* Push API context */
-        H5CX_push(&api_ctx);
-
         /* Insert entries to work with into the cache */
         for (u = 0; u < 10; u++) {
             insert_entry(file_ptr, entry_type, (int32_t)u, H5C__NO_FLAGS_SET);
@@ -26856,20 +26577,16 @@ check_flush_deps_err(unsigned paged)
         } /* end switch */
 
         takedown_cache(file_ptr, false, false);
-        if (!pass)
+        if (!pass) {
+            file_ptr = NULL;
             CACHE_ERROR("takedown_cache failed")
+        }
         file_ptr = NULL;
-
-        /* Pop API context */
-        H5CX_pop(false);
     } /* end for */
 
 done:
-    if (file_ptr) {
+    if (file_ptr)
         takedown_cache(file_ptr, false, false);
-        /* Pop API context */
-        H5CX_pop(false);
-    }
 
     if (pass)
         PASSED();
@@ -26926,15 +26643,12 @@ check_flush_deps_order(unsigned paged)
      */
 
     reset_entries();
-    file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+    file_ptr  = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     cache_ptr = file_ptr->shared->cache;
 
     if (!pass)
         CACHE_ERROR("setup_cache failed")
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     /* Insert entries to work with into the cache */
     for (u = 0; u < 5; u++) {
         insert_entry(file_ptr, entry_type, (int32_t)u, H5C__NO_FLAGS_SET);
@@ -29609,9 +29323,6 @@ check_flush_deps_order(unsigned paged)
         fprintf(stdout, "%s.\n", failure_mssg);
     } /* end else */
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_flush_deps_order() */
 
@@ -29660,7 +29371,7 @@ check_notify_cb(unsigned paged)
      */
 
     reset_entries();
-    file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged);
+    file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged, &api_ctx);
     if (!file_ptr)
         CACHE_ERROR("setup_cache returned NULL")
     cache_ptr = file_ptr->shared->cache;
@@ -29669,9 +29380,6 @@ check_notify_cb(unsigned paged)
     if (!pass)
         CACHE_ERROR("setup_cache failed")
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     /* Insert entries to work with into the cache */
     for (u = 0; u < 5; u++) {
         insert_entry(file_ptr, entry_type, (int32_t)u, H5C__NO_FLAGS_SET);
@@ -29829,9 +29537,6 @@ check_notify_cb(unsigned paged)
         fprintf(stdout, "%s.\n", failure_mssg);
     } /* end else */
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_notify_cb() */
 
@@ -29921,7 +29626,7 @@ check_metadata_cork(bool fill_via_insertion, unsigned paged)
          * The max_cache_size should have room for 50 entries.
          * The min_clean_size is half of that, or 25 entries.
          */
-        file_ptr = setup_cache((size_t)(50 * entry_size), (size_t)(25 * entry_size), paged);
+        file_ptr = setup_cache((size_t)(50 * entry_size), (size_t)(25 * entry_size), paged, &api_ctx);
 
         if (file_ptr == NULL) {
 
@@ -29932,9 +29637,6 @@ check_metadata_cork(bool fill_via_insertion, unsigned paged)
             cache_ptr = file_ptr->shared->cache;
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     /* Cork the cache entry type */
     cork_entry_type(file_ptr, entry_type);
 
@@ -30423,9 +30125,6 @@ check_metadata_cork(bool fill_via_insertion, unsigned paged)
         H5_FAILED();
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_metadata_cork() */
 
@@ -30474,12 +30173,9 @@ check_entry_deletions_during_scans(unsigned paged)
 
         reset_entries();
 
-        file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged);
+        file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged, &api_ctx);
     }
 
-    /* Push API context */
-    H5CX_push(&api_ctx);
-
     /* run the tests.  This set of tests is somewhat eclectic, as
      * we are trying to test all locations where the deletion of
      * an entry from the cache as a side effect of the fluch of
@@ -30520,9 +30216,6 @@ check_entry_deletions_during_scans(unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
     return (unsigned)!pass;
 } /* check_entry_deletions_during_scans() */
 
@@ -31895,10 +31588,7 @@ check_stats(unsigned paged)
 
     reset_entries();
 
-    file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged);
-
-    /* Push API context */
-    H5CX_push(&api_ctx);
+    file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged, &api_ctx);
 
     if (pass) {
 
@@ -31922,9 +31612,6 @@ check_stats(unsigned paged)
         fprintf(stdout, "%s(): failure_mssg = \"%s\".\n", __func__, failure_mssg);
     }
 
-    /* Pop API context */
-    H5CX_pop(false);
-
 #else /* H5C_COLLECT_CACHE_STATS */
 
     SKIPPED();
@@ -32367,20 +32054,19 @@ check_write_permitted(const H5F_t H5_ATTR_UNUSED *f, bool *write_permitted_ptr)
  *****************************************************************************/
 
 H5F_t *
-setup_cache(size_t max_cache_size, size_t min_clean_size, unsigned paged)
+setup_cache(size_t max_cache_size, size_t min_clean_size, unsigned paged, H5CX_node_t *api_ctx)
 {
-    char        filename[512];
-    bool        show_progress = false;
-    bool        verbose       = true;
-    int         mile_stone    = 1;
-    hid_t       fid           = H5I_INVALID_HID;
-    H5F_t      *file_ptr      = NULL;
-    H5C_t      *cache_ptr     = NULL;
-    H5F_t      *ret_val       = NULL;
-    haddr_t     actual_base_addr;
-    hid_t       fapl_id = H5P_DEFAULT;
-    hid_t       fcpl_id = H5P_DEFAULT;
-    H5CX_node_t api_ctx = {{0}, NULL}; /* API context node to push */
+    char    filename[512];
+    bool    show_progress = false;
+    bool    verbose       = true;
+    int     mile_stone    = 1;
+    hid_t   fid           = H5I_INVALID_HID;
+    H5F_t  *file_ptr      = NULL;
+    H5C_t  *cache_ptr     = NULL;
+    H5F_t  *ret_val       = NULL;
+    haddr_t actual_base_addr;
+    hid_t   fapl_id = H5P_DEFAULT;
+    hid_t   fcpl_id = H5P_DEFAULT;
 
     if (show_progress) /* 1 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
@@ -32471,7 +32157,7 @@ setup_cache(size_t max_cache_size, size_t min_clean_size, unsigned paged)
     }     /* end if */
 
     /* Push API context */
-    H5CX_push(&api_ctx);
+    H5CX_push(api_ctx);
 
     if (show_progress) /* 4 */
         fprintf(stdout, "%s() - %0d -- pass = %d\n", __func__, mile_stone++, (int)pass);
diff --git a/test/dsets.c b/test/dsets.c
index e64ed4441b7..88eee7297d3 100644
--- a/test/dsets.c
+++ b/test/dsets.c
@@ -2845,13 +2845,14 @@ test_missing_filter(hid_t file)
     size_t        i, j;                                   /* Local index variables */
     herr_t        ret;                                    /* Generic return value */
     const char   *testfile       = H5_get_srcdir_filename(FILE_DEFLATE_NAME); /* Corrected test file name */
-    H5CX_node_t   api_ctx        = {{0}, NULL};                               /* API context node to push */
     bool          api_ctx_pushed = false;                                     /* Whether API context pushed */
 
     TESTING("dataset access with missing filter");
 
     /* Unregister the deflate filter */
 #ifdef H5_HAVE_FILTER_DEFLATE
+    H5CX_node_t api_ctx = {{0}, NULL}; /* API context node to push */
+
     /* Verify deflate filter is registered currently */
     if (H5Zfilter_avail(H5Z_FILTER_DEFLATE) != true) {
         H5_FAILED();
diff --git a/test/h5test.c b/test/h5test.c
index e3bd8997b05..15e405b213c 100644
--- a/test/h5test.c
+++ b/test/h5test.c
@@ -959,7 +959,7 @@ h5_show_hostname(void)
 #ifdef H5_HAVE_PARALLEL
     int mpi_rank, mpi_initialized, mpi_finalized;
 #endif
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
     uint64_t thread_id = 0; /* ID of thread */
 
     if (H5TS_thread_id(&thread_id) < 0)
@@ -977,12 +977,12 @@ h5_show_hostname(void)
         MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
         printf("MPI-process %d.", mpi_rank);
     }
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
     else
         printf("thread %" PRIu64 ".", thread_id);
 #endif
 #else
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
     printf("thread %" PRIu64 ".", thread_id);
 #endif
 #endif
diff --git a/test/hdfs.c b/test/hdfs.c
index c09d75bd33e..6f5bfe4dd29 100644
--- a/test/hdfs.c
+++ b/test/hdfs.c
@@ -1062,9 +1062,10 @@ test_H5FDread_without_eoa_set_fails(void)
      * TEST *
      ********/
 
-    H5E_BEGIN_TRY{/* mute stack trace on expected failure */
-                  JSVERIFY(FAIL, H5FDread(file_shakespeare, H5FD_MEM_DRAW, H5P_DEFAULT, 1200699, 102, buffer),
-                           "cannot read before eoa is set")} H5E_END_TRY
+    H5E_BEGIN_TRY
+    {/* mute stack trace on expected failure */
+     JSVERIFY(FAIL, H5FDread(file_shakespeare, H5FD_MEM_DRAW, H5P_DEFAULT, 1200699, 102, buffer),
+              "cannot read before eoa is set")} H5E_END_TRY
     for (i = 0; i < HDFS_TEST_MAX_BUF_SIZE; i++) {
         JSVERIFY(0, (unsigned)buffer[i], "buffer was modified by write!")
     }
@@ -1376,13 +1377,15 @@ test_noops_and_autofails(void)
 
     /* auto-fail calls to write and truncate
      */
-    H5E_BEGIN_TRY{JSVERIFY(FAIL, H5FDwrite(file, H5FD_MEM_DRAW, H5P_DEFAULT, 1000, 35, data),
-                           "write must fail")} H5E_END_TRY
+    H5E_BEGIN_TRY
+    {JSVERIFY(FAIL, H5FDwrite(file, H5FD_MEM_DRAW, H5P_DEFAULT, 1000, 35, data),
+              "write must fail")} H5E_END_TRY
 
-    H5E_BEGIN_TRY{JSVERIFY(FAIL, H5FDtruncate(file, H5P_DEFAULT, false), "truncate must fail")} H5E_END_TRY
+    H5E_BEGIN_TRY
+    {JSVERIFY(FAIL, H5FDtruncate(file, H5P_DEFAULT, false), "truncate must fail")} H5E_END_TRY
 
-    H5E_BEGIN_TRY{
-        JSVERIFY(FAIL, H5FDtruncate(file, H5P_DEFAULT, true), "truncate must fail (closing)")} H5E_END_TRY
+    H5E_BEGIN_TRY
+    {JSVERIFY(FAIL, H5FDtruncate(file, H5P_DEFAULT, true), "truncate must fail (closing)")} H5E_END_TRY
 
     /************
      * TEARDOWN *
@@ -1504,11 +1507,13 @@ test_H5F_integration(void)
 
     /* Read-Write Open access is not allowed with this file driver.
      */
-    H5E_BEGIN_TRY{FAIL_IF(0 <= H5Fopen(filename_example_h5, H5F_ACC_RDWR, fapl_id))} H5E_END_TRY
+    H5E_BEGIN_TRY
+    {FAIL_IF(0 <= H5Fopen(filename_example_h5, H5F_ACC_RDWR, fapl_id))} H5E_END_TRY
 
     /* H5Fcreate() is not allowed with this file driver.
      */
-    H5E_BEGIN_TRY{FAIL_IF(0 <= H5Fcreate(filename_missing, H5F_ACC_RDONLY, H5P_DEFAULT, fapl_id))} H5E_END_TRY
+    H5E_BEGIN_TRY
+    {FAIL_IF(0 <= H5Fcreate(filename_missing, H5F_ACC_RDONLY, H5P_DEFAULT, fapl_id))} H5E_END_TRY
 
     /* Successful open.
      */
diff --git a/test/hyperslab.c b/test/hyperslab.c
index c407facf18a..3a84f9758bc 100644
--- a/test/hyperslab.c
+++ b/test/hyperslab.c
@@ -1150,9 +1150,9 @@ main(int argc, char *argv[])
      * Open the library explicitly for thread-safe builds, so per-thread
      * things are initialized correctly.
      */
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
     H5open();
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
     /*
      *------------------------------
@@ -1353,9 +1353,9 @@ main(int argc, char *argv[])
 
     printf("All hyperslab tests passed.\n");
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
     H5close();
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
     exit(EXIT_SUCCESS);
 }
diff --git a/test/ros3.c b/test/ros3.c
index b5307a7305f..9b21828255e 100644
--- a/test/ros3.c
+++ b/test/ros3.c
@@ -25,6 +25,7 @@
 
 #include "H5FDprivate.h" /* Virtual File Driver utilities */
 #include "H5FDros3.h"    /* this file driver's utilities */
+#define H5FD_S3COMMS_TESTING
 #include "H5FDs3comms.h" /* for loading of credentials */
 
 #ifdef H5_HAVE_ROS3_VFD
@@ -484,7 +485,7 @@ test_eof_eoa(void)
     H5FD_t       *fd           = NULL;
     hid_t         fapl_id      = H5I_INVALID_HID;
 
-    TESTING("ROS3 eof/eoa gets and sets");
+    TESTING("ros3 eof/eoa gets and sets");
 
     if (s3_test_credentials_loaded == 0) {
         SKIPPED();
@@ -1051,7 +1052,8 @@ main(void)
     }
     else {
         strncpy(s3_test_bucket_url, bucket_url_env, S3_TEST_MAX_URL_SIZE);
-        s3_test_bucket_defined = true;
+        s3_test_bucket_url[S3_TEST_MAX_URL_SIZE - 1] = '\0';
+        s3_test_bucket_defined                       = true;
     }
 
     if (S3_TEST_MAX_URL_SIZE < snprintf(url_text_restricted, (size_t)S3_TEST_MAX_URL_SIZE, "%s/%s",
@@ -1089,8 +1091,8 @@ main(void)
     s3_test_aws_region[0]            = '\0';
 
     /* Attempt to load test credentials - if unable, certain tests will be skipped */
-    if (SUCCEED == H5FD_s3comms_load_aws_profile(S3_TEST_PROFILE_NAME, s3_test_aws_access_key_id,
-                                                 s3_test_aws_secret_access_key, s3_test_aws_region)) {
+    if (SUCCEED == H5FD__s3comms_load_aws_profile(S3_TEST_PROFILE_NAME, s3_test_aws_access_key_id,
+                                                  s3_test_aws_secret_access_key, s3_test_aws_region)) {
         s3_test_credentials_loaded = 1;
         strncpy(restricted_access_fa.aws_region, (const char *)s3_test_aws_region, H5FD_ROS3_MAX_REGION_LEN);
         strncpy(restricted_access_fa.secret_id, (const char *)s3_test_aws_access_key_id,
diff --git a/test/s3comms.c b/test/s3comms.c
index 61d73f37f35..5192bf9bf7a 100644
--- a/test/s3comms.c
+++ b/test/s3comms.c
@@ -17,8 +17,9 @@
  */
 
 #include "h5test.h"
+
+#define H5FD_S3COMMS_TESTING
 #include "H5FDs3comms.h"
-#include "H5MMprivate.h" /* memory management */
 
 #ifdef H5_HAVE_ROS3_VFD
 
@@ -84,7 +85,7 @@ test_macro_format_credential(void)
 } /* end test_macro_format_credential() */
 
 /*---------------------------------------------------------------------------
- * Function:    test_aws_canonical_request
+ * Function:    test_make_aws_canonical_request
  *
  * Purpose:     Demonstrate the construction of a Canonical Request (and
  *              Signed Headers)
@@ -98,7 +99,7 @@ test_macro_format_credential(void)
  *---------------------------------------------------------------------------
  */
 static int
-test_aws_canonical_request(void)
+test_make_aws_canonical_request(void)
 {
     struct header {
         const char *name;
@@ -162,7 +163,7 @@ test_aws_canonical_request(void)
     char             sh_dest[64]; /* Signed headers */
     herr_t           ret;
 
-    TESTING("test_aws_canonical_request");
+    TESTING("make AWS canonical request");
 
     for (int i = 0; i < NCASES; i++) {
         C = &cases[i];
@@ -176,19 +177,19 @@ test_aws_canonical_request(void)
         }
 
         /* Create HTTP request object with given verb, resource/path */
-        hrb = H5FD_s3comms_hrb_init_request(C->verb, C->resource, "HTTP/1.1");
+        hrb = H5FD__s3comms_hrb_init_request(C->verb, C->resource, "HTTP/1.1");
         assert(hrb->body == NULL);
 
         /* Create headers list from test case input */
         for (int j = 0; j < C->listsize; j++) {
-            if (H5FD_s3comms_hrb_node_set(&node, C->list[j].name, C->list[j].value) < 0)
+            if (H5FD__s3comms_hrb_node_set(&node, C->list[j].name, C->list[j].value) < 0)
                 TEST_ERROR;
         }
 
         hrb->first_header = node;
 
         /* Test */
-        if (H5FD_s3comms_aws_canonical_request(cr_dest, 512, sh_dest, 64, hrb) < 0)
+        if (H5FD__s3comms_make_aws_canonical_request(cr_dest, 512, sh_dest, 64, hrb) < 0)
             TEST_ERROR;
         if (strncmp(C->exp_headers, sh_dest, 512))
             FAIL_PUTS_ERROR("header string mismatch");
@@ -197,26 +198,26 @@ test_aws_canonical_request(void)
 
         /* Tear-down */
         while (node != NULL) {
-            if (H5FD_s3comms_hrb_node_set(&node, node->name, NULL) < 0)
+            if (H5FD__s3comms_hrb_node_set(&node, node->name, NULL) < 0)
                 TEST_ERROR;
         }
-        if (H5FD_s3comms_hrb_destroy(&hrb) < 0)
+        if (H5FD__s3comms_hrb_destroy(hrb) < 0)
             TEST_ERROR;
     }
 
     /* ERROR CASES - Malformed hrb and/or node-list */
     H5E_BEGIN_TRY
     {
-        ret = H5FD_s3comms_aws_canonical_request(cr_dest, 20, sh_dest, 20, NULL);
+        ret = H5FD__s3comms_make_aws_canonical_request(cr_dest, 20, sh_dest, 20, NULL);
     }
     H5E_END_TRY
     if (ret == SUCCEED)
         FAIL_PUTS_ERROR("http request object cannot be null");
 
-    hrb = H5FD_s3comms_hrb_init_request("GET", "/", "HTTP/1.1");
+    hrb = H5FD__s3comms_hrb_init_request("GET", "/", "HTTP/1.1");
     H5E_BEGIN_TRY
     {
-        ret = H5FD_s3comms_aws_canonical_request(NULL, 20, sh_dest, 20, hrb);
+        ret = H5FD__s3comms_make_aws_canonical_request(NULL, 20, sh_dest, 20, hrb);
     }
     H5E_END_TRY
     if (ret == SUCCEED)
@@ -224,13 +225,13 @@ test_aws_canonical_request(void)
 
     H5E_BEGIN_TRY
     {
-        ret = H5FD_s3comms_aws_canonical_request(cr_dest, 20, NULL, 20, hrb);
+        ret = H5FD__s3comms_make_aws_canonical_request(cr_dest, 20, NULL, 20, hrb);
     }
     H5E_END_TRY
     if (ret == SUCCEED)
         FAIL_PUTS_ERROR("signed headers destination cannot be null");
 
-    if (H5FD_s3comms_hrb_destroy(&hrb) < 0)
+    if (H5FD__s3comms_hrb_destroy(hrb) < 0)
         TEST_ERROR;
 
     PASSED();
@@ -240,90 +241,18 @@ test_aws_canonical_request(void)
 
     if (node != NULL) {
         while (node != NULL)
-            H5FD_s3comms_hrb_node_set(&node, node->name, NULL);
-    }
-    if (hrb != NULL) {
-        H5FD_s3comms_hrb_destroy(&hrb);
+            H5FD__s3comms_hrb_node_set(&node, node->name, NULL);
     }
+    H5FD__s3comms_hrb_destroy(hrb);
 
     return 1;
 
-} /* end test_aws_canonical_request() */
-
-/*---------------------------------------------------------------------------
- * Function:    test_bytes_to_hex
- *
- * Purpose:     Define and verify behavior of `H5FD_s3comms_bytes_to_hex()`.
- *
- * Return:      PASS : 0
- *              FAIL : 1
- *---------------------------------------------------------------------------
- */
-static int
-test_bytes_to_hex(void)
-{
-    struct testcase {
-        const char          exp[17]; /* in size * 2 + 1 for null terminator */
-        const unsigned char in[8];
-        size_t              size;
-        bool                lower;
-    };
-
-    struct testcase cases[] = {
-        {
-            "52F3000C9A",
-            {82, 243, 0, 12, 154},
-            5,
-            false,
-        },
-        {
-            "009a0cf3005200", /* lowercase alphas */
-            {0, 154, 12, 243, 0, 82, 0},
-            7,
-            true,
-        },
-        {
-            "", {17, 63, 26, 56}, 0, false, /* irrelevant */
-        },
-    };
-    const int NCASES = 3;
-    char      out[17];
-    herr_t    ret;
-
-    TESTING("bytes-to-hex");
-
-    for (int i = 0; i < NCASES; i++) {
-        for (int out_off = 0; out_off < 17; out_off++) {
-            out[out_off] = 0;
-        }
-
-        if (H5FD_s3comms_bytes_to_hex(out, cases[i].in, cases[i].size, cases[i].lower) < 0)
-            TEST_ERROR;
-
-        if (strncmp(cases[i].exp, out, 17))
-            FAIL_PUTS_ERROR("incorrect bytes to hex conversion");
-    }
-
-    /* dest cannot be null */
-    H5E_BEGIN_TRY
-    {
-        ret = H5FD_s3comms_bytes_to_hex(NULL, (const unsigned char *)"nada", 5, false);
-    }
-    H5E_END_TRY
-    if (ret == SUCCEED)
-        FAIL_PUTS_ERROR("dest parameter cannot be null");
-
-    PASSED();
-    return 0;
-
-error:
-    return 1;
-} /* end test_bytes_to_hex() */
+} /* end test_make_aws_canonical_request() */
 
 /*---------------------------------------------------------------------------
  * Function:    test_hrb_init_request
  *
- * Purpose:     Define and verify behavior of `H5FD_s3comms_hrb_init_request()`
+ * Purpose:     Define and verify behavior of `H5FD__s3comms_hrb_init_request()`
  *
  * Return:      PASS : 0
  *              FAIL : 1
@@ -391,7 +320,7 @@ test_hrb_init_request(void)
     for (int i = 0; i < NCASES; i++) {
         struct testcase *C = &cases[i];
 
-        req = H5FD_s3comms_hrb_init_request(C->verb, C->resource, C->version);
+        req = H5FD__s3comms_hrb_init_request(C->verb, C->resource, C->version);
 
         if (cases[i].ret_null == true) {
             if (req != NULL)
@@ -418,11 +347,8 @@ test_hrb_init_request(void)
                 TEST_ERROR;
             if (0 != req->body_len)
                 TEST_ERROR;
-            if (H5FD_s3comms_hrb_destroy(&req) < 0)
+            if (H5FD__s3comms_hrb_destroy(req) < 0)
                 FAIL_PUTS_ERROR("unable to destroy hrb_t");
-            /* Should annull pointer as well as free */
-            if (NULL != req)
-                TEST_ERROR;
         }
     }
 
@@ -430,7 +356,7 @@ test_hrb_init_request(void)
     return 0;
 
 error:
-    H5FD_s3comms_hrb_destroy(&req);
+    H5FD__s3comms_hrb_destroy(req);
     return 1;
 } /* end test_hrb_init_request() */
 
@@ -759,14 +685,14 @@ test_hrb_node_set(void)
             const char *name  = test->given[mock_i];
             const char *value = test->given[mock_i + 1];
 
-            if (H5FD_s3comms_hrb_node_set(&list, name, value) < 0)
+            if (H5FD__s3comms_hrb_node_set(&list, name, value) < 0)
                 TEST_ERROR;
         }
 
         /* TEST */
 
         /* Modify list */
-        if (test->returned != H5FD_s3comms_hrb_node_set(&list, test->delta.name, test->delta.value))
+        if (test->returned != H5FD__s3comms_hrb_node_set(&list, test->delta.name, test->delta.value))
             FAIL_PUTS_ERROR(test->message);
 
         /* Verify resulting list */
@@ -792,7 +718,7 @@ test_hrb_node_set(void)
         /* TEARDOWN */
 
         while (list != NULL) {
-            if (H5FD_s3comms_hrb_node_set(&list, list->name, NULL) < 0)
+            if (H5FD__s3comms_hrb_node_set(&list, list->name, NULL) < 0)
                 TEST_ERROR;
         }
     }
@@ -802,403 +728,25 @@ test_hrb_node_set(void)
 
 error:
     while (list != NULL) {
-        H5FD_s3comms_hrb_node_set(&list, list->name, NULL);
+        H5FD__s3comms_hrb_node_set(&list, list->name, NULL);
     }
 
     return 1;
 
 } /* end test_hrb_node_set() */
 
+/* This is difficult to test since we took away the time parameter */
 /*---------------------------------------------------------------------------
- * Function:    test_HMAC_SHA256
+ * Function:    test_make_aws_signing_key
  *
- * Purpose:     Define and verify behavior of `H5FD_s3comms_HMAC_SHA256()`
+ * Purpose:     Verify behavior of `H5FD__s3comms_make_aws_signing_key()`
  *
  * Return:      PASS : 0
  *              FAIL : 1
  *---------------------------------------------------------------------------
  */
 static int
-test_HMAC_SHA256(void)
-{
-    struct testcase {
-        herr_t              ret; /* SUCCEED/FAIL expected from call */
-        const unsigned char key[SHA256_DIGEST_LENGTH];
-        size_t              key_len;
-        const char         *msg;
-        size_t              msg_len;
-        const char         *exp;       /* not used if ret == FAIL */
-        size_t              dest_size; /* if 0, `dest` is not malloc'd */
-    };
-
-    struct testcase cases[] = {
-        {
-            SUCCEED,
-            {
-                0xdb, 0xb8, 0x93, 0xac, 0xc0, 0x10, 0x96, 0x49, 0x18, 0xf1, 0xfd,
-                0x43, 0x3a, 0xdd, 0x87, 0xc7, 0x0e, 0x8b, 0x0d, 0xb6, 0xbe, 0x30,
-                0xc1, 0xfb, 0xea, 0xfe, 0xfa, 0x5e, 0xc6, 0xba, 0x83, 0x78,
-            },
-            SHA256_DIGEST_LENGTH,
-            "AWS4-HMAC-SHA256\n20130524T000000Z\n20130524/us-east-1/s3/"
-            "aws4_request\n7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972",
-            strlen("AWS4-HMAC-SHA256\n20130524T000000Z\n20130524/us-east-1/s3/"
-                   "aws4_request\n7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972"),
-            "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41",
-            SHA256_DIGEST_LENGTH * 2 + 1, /* +1 for null terminator */
-        },
-        {
-            SUCCEED,
-            {'J', 'e', 'f', 'e'},
-            4,
-            "what do ya want for nothing?",
-            28,
-            "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843",
-            SHA256_DIGEST_LENGTH * 2 + 1,
-        },
-        {
-            FAIL, "DOESN'T MATTER", 14, "ALSO IRRELEVANT", 15, NULL,
-            0, /* dest -> null, resulting in immediate error */
-        },
-    };
-    char     *dest   = NULL;
-    const int NCASES = 3;
-
-    TESTING("HMAC_SHA256");
-
-    for (int i = 0; i < NCASES; i++) {
-
-        if (cases[i].dest_size == 0) {
-            dest = NULL;
-        }
-        else {
-            if (NULL == (dest = (char *)malloc(sizeof(char) * cases[i].dest_size)))
-                TEST_ERROR;
-        }
-
-        if (cases[i].ret !=
-            H5FD_s3comms_HMAC_SHA256(cases[i].key, cases[i].key_len, cases[i].msg, cases[i].msg_len, dest))
-            TEST_ERROR;
-
-        if (cases[i].ret == SUCCEED) {
-            if (strncmp(cases[i].exp, dest, strlen(cases[i].exp)))
-                TEST_ERROR;
-        }
-        free(dest);
-    }
-
-    PASSED();
-    return 0;
-
-error:
-    free(dest);
-    return 1;
-
-} /* end test_HMAC_SHA256() */
-
-/*---------------------------------------------------------------------------
- * Function: test_parse_url
- *
- *
- * Return:      PASS : 0
- *              FAIL : 1
- *---------------------------------------------------------------------------
- */
-static int
-test_parse_url(void)
-{
-    typedef struct {
-        const char *scheme;
-        const char *host;
-        const char *port;
-        const char *path;
-        const char *query;
-    } const_purl_t;
-
-    struct testcase {
-        const char  *url;
-        herr_t       exp_ret;  /* expected return */
-        const_purl_t expected; /* unused if exp_ret is FAIL */
-        const char  *msg;
-    };
-
-    parsed_url_t   *purl    = NULL;
-    const int       NCASES  = 15;
-    struct testcase cases[] = {
-        {
-            NULL,
-            FAIL,
-            {NULL, NULL, NULL, NULL, NULL},
-            "null url",
-        },
-        {
-            "",
-            FAIL,
-            {NULL, NULL, NULL, NULL, NULL},
-            "empty url",
-        },
-        {
-            "ftp://[1000:4000:0002:2010]",
-            SUCCEED,
-            {
-                "ftp",
-                "[1000:4000:0002:2010]",
-                NULL,
-                NULL,
-                NULL,
-            },
-            "IPv6 ftp and empty path (root)",
-        },
-        {
-            "ftp://[1000:4000:0002:2010]:2040",
-            SUCCEED,
-            {
-                "ftp",
-                "[1000:4000:0002:2010]",
-                "2040",
-                NULL,
-                NULL,
-            },
-            "root IPv6 ftp with port",
-        },
-        {
-            "http://some.domain.org:9000/path/to/resource.txt",
-            SUCCEED,
-            {
-                "http",
-                "some.domain.org",
-                "9000",
-                "path/to/resource.txt",
-                NULL,
-            },
-            "without query",
-        },
-        {
-            "https://domain.me:00/file.txt?some_params unchecked",
-            SUCCEED,
-            {
-                "https",
-                "domain.me",
-                "00",
-                "file.txt",
-                "some_params unchecked",
-            },
-            "with query",
-        },
-        {
-            "ftp://domain.com/",
-            SUCCEED,
-            {
-                "ftp",
-                "domain.com",
-                NULL,
-                NULL,
-                NULL,
-            },
-            "explicit root w/out port",
-        },
-        {
-            "ftp://domain.com:1234/",
-            SUCCEED,
-            {
-                "ftp",
-                "domain.com",
-                "1234",
-                NULL,
-                NULL,
-            },
-            "explicit root with port",
-        },
-        {
-            "ftp://domain.com:1234/file?",
-            FAIL,
-            {
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-            },
-            "empty query is invalid",
-        },
-        {
-            "ftp://:1234/file",
-            FAIL,
-            {
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-            },
-            "no host",
-        },
-        {
-            "h&r block",
-            FAIL,
-            {
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-            },
-            "no scheme (bad URL)",
-        },
-        {
-            "http://domain.com?a=b&d=b",
-            SUCCEED,
-            {
-                "http",
-                "domain.com",
-                NULL,
-                NULL,
-                "a=b&d=b",
-            },
-            "QUERY with implicit PATH",
-        },
-        {
-            "http://[5]/path?a=b&d=b",
-            SUCCEED,
-            {
-                "http",
-                "[5]",
-                NULL,
-                "path",
-                "a=b&d=b",
-            },
-            "IPv6 extraction is really dumb",
-        },
-        {
-            "http://[1234:5678:0910:1112]:port/path",
-            FAIL,
-            {
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-            },
-            "non-decimal PORT (port)",
-        },
-        {
-            "http://mydomain.com:01a3/path",
-            FAIL,
-            {
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-            },
-            "non-decimal PORT (01a3)",
-        },
-    };
-
-    TESTING("url-parsing functionality");
-
-    /*********
-     * TESTS *
-     *********/
-
-    for (int i = 0; i < NCASES; i++) {
-
-        if (cases[i].exp_ret != H5FD_s3comms_parse_url(cases[i].url, &purl))
-            TEST_ERROR;
-
-        if (cases[i].exp_ret == FAIL) {
-            /* On FAIL, `purl` should be untouched--remains NULL */
-            if (purl != NULL)
-                TEST_ERROR;
-        }
-        else {
-            /* On SUCCEED, `purl` should be set */
-            if (purl == NULL)
-                TEST_ERROR;
-
-            if (cases[i].expected.scheme != NULL) {
-                if (NULL == purl->scheme)
-                    TEST_ERROR;
-                if (strcmp(cases[i].expected.scheme, purl->scheme))
-                    TEST_ERROR;
-            }
-            else {
-                if (NULL != purl->scheme)
-                    TEST_ERROR;
-            }
-
-            if (cases[i].expected.host != NULL) {
-                if (NULL == purl->host)
-                    TEST_ERROR;
-                if (strcmp(cases[i].expected.host, purl->host))
-                    TEST_ERROR;
-            }
-            else {
-                if (NULL != purl->host)
-                    TEST_ERROR;
-            }
-
-            if (cases[i].expected.port != NULL) {
-                if (NULL == purl->port)
-                    TEST_ERROR;
-                if (strcmp(cases[i].expected.port, purl->port))
-                    TEST_ERROR;
-            }
-            else {
-                if (NULL != purl->port)
-                    TEST_ERROR;
-            }
-
-            if (cases[i].expected.path != NULL) {
-                if (NULL == purl->path)
-                    TEST_ERROR;
-                if (strcmp(cases[i].expected.path, purl->path))
-                    TEST_ERROR;
-            }
-            else {
-                if (NULL != purl->path)
-                    TEST_ERROR;
-            }
-
-            if (cases[i].expected.query != NULL) {
-                if (NULL == purl->query)
-                    TEST_ERROR;
-                if (strcmp(cases[i].expected.query, purl->query))
-                    TEST_ERROR;
-            }
-            else {
-                if (NULL != purl->query)
-                    TEST_ERROR;
-            }
-        }
-
-        if (H5FD_s3comms_free_purl(purl) < 0)
-            TEST_ERROR;
-
-        purl = NULL;
-    }
-
-    PASSED();
-    return 0;
-
-error:
-    H5FD_s3comms_free_purl(purl);
-
-    return 1;
-
-} /* end test_parse_url() */
-
-/*---------------------------------------------------------------------------
- * Function:    test_signing_key
- *
- * Purpose:     Verify behavior of `H5FD_s3comms_signing_key()`
- *
- * Return:      PASS : 0
- *              FAIL : 1
- *---------------------------------------------------------------------------
- */
-static int
-test_signing_key(void)
+test_make_aws_signing_key(void)
 {
     struct testcase {
         const char   *region;
@@ -1222,15 +770,14 @@ test_signing_key(void)
 
     unsigned char *key    = NULL;
     const int      NCASES = 1;
-    herr_t         ret;
 
-    TESTING("signing_key");
+    TESTING("make AWS signing key");
 
     for (int i = 0; i < NCASES; i++) {
         if (NULL == (key = (unsigned char *)malloc(sizeof(unsigned char) * SHA256_DIGEST_LENGTH)))
             TEST_ERROR;
 
-        if (H5FD_s3comms_signing_key(key, cases[i].secret_key, cases[i].region, cases[i].when) < 0)
+        if (H5FD__s3comms_make_aws_signing_key(key, cases[i].secret_key, cases[i].region, cases[i].when) < 0)
             TEST_ERROR;
 
         if (strncmp((const char *)cases[i].exp, (const char *)key, SHA256_DIGEST_LENGTH))
@@ -1240,55 +787,16 @@ test_signing_key(void)
         key = NULL;
     }
 
-    /* ERROR CASES */
-
-    if (NULL == (key = (unsigned char *)malloc(sizeof(unsigned char) * SHA256_DIGEST_LENGTH)))
-        TEST_ERROR;
-
-    H5E_BEGIN_TRY
-    {
-        ret = H5FD_s3comms_signing_key(NULL, cases[0].secret_key, cases[0].region, cases[0].when);
-    }
-    H5E_END_TRY
-    if (ret == SUCCEED)
-        FAIL_PUTS_ERROR("destination cannot be NULL");
-
-    H5E_BEGIN_TRY
-    {
-        ret = H5FD_s3comms_signing_key(key, NULL, cases[0].region, cases[0].when);
-    }
-    H5E_END_TRY
-    if (ret == SUCCEED)
-        FAIL_PUTS_ERROR("secret key cannot be NULL");
-
-    H5E_BEGIN_TRY
-    {
-        ret = H5FD_s3comms_signing_key(key, cases[0].secret_key, NULL, cases[0].when);
-    }
-    H5E_END_TRY
-    if (ret == SUCCEED)
-        FAIL_PUTS_ERROR("aws region cannot be NULL");
-
-    H5E_BEGIN_TRY
-    {
-        ret = H5FD_s3comms_signing_key(key, cases[0].secret_key, cases[0].region, NULL);
-    }
-    H5E_END_TRY
-    if (ret == SUCCEED)
-        FAIL_PUTS_ERROR("time string cannot be NULL");
-
-    free(key);
-
     PASSED();
     return 0;
 
 error:
     free(key);
     return 1;
-} /* end test_signing_key() */
+} /* end test_make_aws_signing_key() */
 
 /*---------------------------------------------------------------------------
- * Function:    test_tostringtosign()
+ * Function:    test_make_aws_stringtosign()
  *
  * Purpose:     Verify that we can get the "string to sign" from a Canonical
  *              Request and related information.
@@ -1298,7 +806,7 @@ test_signing_key(void)
  *---------------------------------------------------------------------------
  */
 static int
-test_tostringtosign(void)
+test_make_aws_stringtosign(void)
 {
     const char canonreq[]   = "GET\n/"
                               "test.txt\n\nhost:examplebucket.s3.amazonaws.com\nrange:bytes=0-9\nx-amz-content-"
@@ -1310,9 +818,9 @@ test_tostringtosign(void)
     char       s2s[512];
     herr_t     ret;
 
-    TESTING("s3comms tostringtosign");
+    TESTING("make AWS stringtosign");
 
-    if (H5FD_s3comms_tostringtosign(s2s, canonreq, iso8601now, region) < 0)
+    if (H5FD__s3comms_make_aws_stringtosign(s2s, canonreq, iso8601now, region) < 0)
         FAIL_PUTS_ERROR("unable to create string to sign");
 
     if (strncmp("AWS4-HMAC-SHA256\n20130524T000000Z\n20130524/us-east-1/s3/"
@@ -1324,7 +832,7 @@ test_tostringtosign(void)
 
     H5E_BEGIN_TRY
     {
-        ret = H5FD_s3comms_tostringtosign(s2s, NULL, iso8601now, region);
+        ret = H5FD__s3comms_make_aws_stringtosign(s2s, NULL, iso8601now, region);
     }
     H5E_END_TRY
     if (ret == SUCCEED)
@@ -1332,7 +840,7 @@ test_tostringtosign(void)
 
     H5E_BEGIN_TRY
     {
-        ret = H5FD_s3comms_tostringtosign(s2s, canonreq, NULL, region);
+        ret = H5FD__s3comms_make_aws_stringtosign(s2s, canonreq, NULL, region);
     }
     H5E_END_TRY
     if (ret == SUCCEED)
@@ -1340,7 +848,7 @@ test_tostringtosign(void)
 
     H5E_BEGIN_TRY
     {
-        ret = H5FD_s3comms_tostringtosign(s2s, canonreq, iso8601now, NULL);
+        ret = H5FD__s3comms_make_aws_stringtosign(s2s, canonreq, iso8601now, NULL);
     }
     H5E_END_TRY
     if (ret == SUCCEED)
@@ -1352,12 +860,12 @@ test_tostringtosign(void)
 error:
     return 1;
 
-} /* end test_tostringtosign() */
+} /* end test_make_aws_stringtosign() */
 
 /*---------------------------------------------------------------------------
  * Function:    test_s3r_get_filesize
  *
- * Purpose:     Test H5FD_s3comms_s3r_get_filesize()
+ * Purpose:     Test H5FD__s3comms_s3r_get_filesize()
  *
  * Return:      PASS : 0
  *              FAIL : 1
@@ -1383,16 +891,16 @@ test_s3r_get_filesize(void)
         snprintf(url_raven, S3_TEST_MAX_URL_SIZE, "%s/%s", s3_test_bucket_url, S3_TEST_RESOURCE_TEXT_PUBLIC))
         TEST_ERROR;
 
-    if (0 != H5FD_s3comms_s3r_get_filesize(NULL))
+    if (0 != H5FD__s3comms_s3r_get_filesize(NULL))
         FAIL_PUTS_ERROR("filesize of the null handle should be 0");
 
-    if (NULL == (handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL)))
+    if (NULL == (handle = H5FD__s3comms_s3r_open(url_raven, NULL, NULL)))
         TEST_ERROR;
 
-    if (6464 != H5FD_s3comms_s3r_get_filesize(handle))
+    if (6464 != H5FD__s3comms_s3r_get_filesize(handle))
         FAIL_PUTS_ERROR("incorrect file size - fragile, make sure the file size didn't change");
 
-    if (H5FD_s3comms_s3r_close(handle) < 0)
+    if (H5FD__s3comms_s3r_close(handle) < 0)
         TEST_ERROR;
 
     PASSED();
@@ -1400,14 +908,14 @@ test_s3r_get_filesize(void)
 
 error:
     if (handle != NULL)
-        H5FD_s3comms_s3r_close(handle);
+        H5FD__s3comms_s3r_close(handle);
     return 1;
 } /* end test_s3r_get_filesize() */
 
 /*---------------------------------------------------------------------------
  * Function:    test_s3r_open
  *
- * Purpose:     Test H5FD_s3comms_s3r_open()
+ * Purpose:     Test H5FD__s3comms_s3r_open()
  *
  * Return:      PASS : 0
  *              FAIL : 1
@@ -1416,14 +924,11 @@ test_s3r_get_filesize(void)
 static int
 test_s3r_open(void)
 {
-    char          url_missing[S3_TEST_MAX_URL_SIZE];
-    char          url_raven[S3_TEST_MAX_URL_SIZE];
-    char          url_shakespeare[S3_TEST_MAX_URL_SIZE];
-    unsigned char signing_key[SHA256_DIGEST_LENGTH];
-    struct tm    *now = NULL;
-    char          iso8601now[ISO8601_SIZE];
-    s3r_t        *handle = NULL;
-    parsed_url_t *purl   = NULL;
+    char              url_missing[S3_TEST_MAX_URL_SIZE];
+    char              url_raven[S3_TEST_MAX_URL_SIZE];
+    char              url_shakespeare[S3_TEST_MAX_URL_SIZE];
+    H5FD_ros3_fapl_t *fa     = NULL;
+    s3r_t            *handle = NULL;
 
     TESTING("s3r_open");
 
@@ -1444,6 +949,18 @@ test_s3r_open(void)
      * PRE-TEST SETUP *
      ******************/
 
+    /* Create and fill a common fapl
+     *
+     * Specific fields will be set (and reset) as needed by tests below
+     */
+    if (NULL == (fa = (H5FD_ros3_fapl_t *)calloc(1, sizeof(H5FD_ros3_fapl_t))))
+        TEST_ERROR;
+    fa->version      = H5FD_CURR_ROS3_FAPL_T_VERSION;
+    fa->authenticate = true;
+    strcpy(fa->aws_region, s3_test_aws_region);
+    strcpy(fa->secret_id, s3_test_aws_access_key_id);
+    strcpy(fa->secret_key, s3_test_aws_secret_access_key);
+
     if (S3_TEST_MAX_URL_SIZE < snprintf(url_shakespeare, S3_TEST_MAX_URL_SIZE, "%s/%s", s3_test_bucket_url,
                                         S3_TEST_RESOURCE_TEXT_RESTRICTED))
         TEST_ERROR;
@@ -1456,39 +973,6 @@ test_s3r_open(void)
         snprintf(url_raven, S3_TEST_MAX_URL_SIZE, "%s/%s", s3_test_bucket_url, S3_TEST_RESOURCE_TEXT_PUBLIC))
         TEST_ERROR;
 
-    /* Set given bucket url with invalid/inactive port number for badport.
-     * Note, this sort of micro-management of parsed_url_t is not advised
-     */
-    if (H5FD_s3comms_parse_url(s3_test_bucket_url, &purl) < 0)
-        TEST_ERROR;
-
-    if (purl->port == NULL) {
-        if (NULL == (purl->port = (char *)H5MM_malloc(sizeof(char) * 5)))
-            TEST_ERROR;
-        if (5 < snprintf(purl->port, 5, "9000"))
-            TEST_ERROR;
-    }
-    else if (strcmp(purl->port, "9000") != 0) {
-        if (5 < snprintf(purl->port, 5, "9000"))
-            TEST_ERROR;
-    }
-    else {
-        if (5 < snprintf(purl->port, 5, "1234"))
-            TEST_ERROR;
-    }
-
-    if (NULL == (now = gmnow()))
-        TEST_ERROR;
-    if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1))
-        TEST_ERROR;
-
-    /* It is desired to have means available to verify that signing_key
-     * was set successfully and to an expected value.
-     */
-    if (H5FD_s3comms_signing_key(signing_key, (const char *)s3_test_aws_secret_access_key,
-                                 (const char *)s3_test_aws_region, (const char *)iso8601now) < 0)
-        TEST_ERROR;
-
     /*************************
      * OPEN NONEXISTENT FILE *
      *************************/
@@ -1496,7 +980,7 @@ test_s3r_open(void)
     /* Attempt anonymously */
     H5E_BEGIN_TRY
     {
-        handle = H5FD_s3comms_s3r_open(url_missing, NULL, NULL, NULL, NULL);
+        handle = H5FD__s3comms_s3r_open(url_missing, NULL, NULL);
     }
     H5E_END_TRY
     if (handle != NULL)
@@ -1505,9 +989,7 @@ test_s3r_open(void)
     /* Attempt with authentication */
     H5E_BEGIN_TRY
     {
-        handle = H5FD_s3comms_s3r_open(
-            url_missing, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id,
-            (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token);
+        handle = H5FD__s3comms_s3r_open(url_missing, fa, (const char *)s3_test_aws_security_token);
     }
     H5E_END_TRY
     if (handle != NULL)
@@ -1520,82 +1002,76 @@ test_s3r_open(void)
     /* Anonymous access on restricted file */
     H5E_BEGIN_TRY
     {
-        handle = H5FD_s3comms_s3r_open(url_shakespeare, NULL, NULL, NULL, NULL);
+        handle = H5FD__s3comms_s3r_open(url_shakespeare, NULL, NULL);
     }
     H5E_END_TRY
     if (handle != NULL)
         TEST_ERROR;
 
     /* Pass in a bad ID */
+    strcpy(fa->secret_id, "I_MADE_UP_MY_ID");
     H5E_BEGIN_TRY
     {
-        handle = H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, "I_MADE_UP_MY_ID",
-                                       (const unsigned char *)signing_key,
-                                       (const char *)s3_test_aws_security_token);
+        handle = H5FD__s3comms_s3r_open(url_shakespeare, fa, (const char *)s3_test_aws_security_token);
     }
     H5E_END_TRY
     if (handle != NULL)
         TEST_ERROR;
+    strcpy(fa->secret_id, s3_test_aws_access_key_id);
 
     /* Using an invalid signing key */
+    strcpy(fa->secret_key, "I_AM_A_FAKE_KEY");
     H5E_BEGIN_TRY
     {
-        handle = H5FD_s3comms_s3r_open(
-            url_shakespeare, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id,
-            (const unsigned char *)EMPTY_SHA256, (const char *)s3_test_aws_security_token);
+        handle = H5FD__s3comms_s3r_open(url_shakespeare, fa, (const char *)s3_test_aws_security_token);
     }
     H5E_END_TRY
     if (handle != NULL)
         TEST_ERROR;
+    strcpy(fa->secret_key, s3_test_aws_secret_access_key);
 
     /*******************************
      * SUCCESSFUL OPEN (AND CLOSE) *
      *******************************/
 
     /* Anonymous */
-    handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL);
+    handle = H5FD__s3comms_s3r_open(url_raven, NULL, NULL);
     if (handle == NULL)
         TEST_ERROR;
-    if (6464 != H5FD_s3comms_s3r_get_filesize(handle))
+    if (6464 != H5FD__s3comms_s3r_get_filesize(handle))
         FAIL_PUTS_ERROR("did not get expected filesize");
-    if (H5FD_s3comms_s3r_close(handle) < 0)
+    if (H5FD__s3comms_s3r_close(handle) < 0)
         TEST_ERROR;
     handle = NULL;
 
     /* Using authentication on anonymously-accessible file? */
-    handle = H5FD_s3comms_s3r_open(
-        url_raven, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id,
-        (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token);
+    handle = H5FD__s3comms_s3r_open(url_raven, fa, (const char *)s3_test_aws_security_token);
     if (handle == NULL)
         TEST_ERROR;
-    if (6464 != H5FD_s3comms_s3r_get_filesize(handle))
+    if (6464 != H5FD__s3comms_s3r_get_filesize(handle))
         FAIL_PUTS_ERROR("did not get expected filesize");
-    if (H5FD_s3comms_s3r_close(handle))
+    if (H5FD__s3comms_s3r_close(handle))
         TEST_ERROR;
     handle = NULL;
 
     /* Authenticating */
-    handle = H5FD_s3comms_s3r_open(
-        url_shakespeare, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id,
-        (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token);
+    handle = H5FD__s3comms_s3r_open(url_shakespeare, fa, (const char *)s3_test_aws_security_token);
     if (handle == NULL)
         TEST_ERROR;
-    if (5458199 != H5FD_s3comms_s3r_get_filesize(handle))
+    if (5458199 != H5FD__s3comms_s3r_get_filesize(handle))
         FAIL_PUTS_ERROR("did not get expected filesize");
-    if (H5FD_s3comms_s3r_close(handle) < 0)
+    if (H5FD__s3comms_s3r_close(handle) < 0)
         TEST_ERROR;
     handle = NULL;
 
-    if (H5FD_s3comms_free_purl(purl) < 0)
-        TEST_ERROR;
+    free(fa);
 
     PASSED();
     return 0;
 error:
     if (handle != NULL)
-        H5FD_s3comms_s3r_close(handle);
-    if (purl != NULL)
-        H5FD_s3comms_free_purl(purl);
+        H5FD__s3comms_s3r_close(handle);
+    free(fa);
 
     return 1;
 } /* end test_s3r_open() */
@@ -1606,10 +1082,10 @@ test_s3r_open(void)
  * Purpose:     Specify and demonstrate the use and life cycle of an S3
  *              request handle `s3r_t`, through its related functions.
  *
- *     H5FD_s3comms_s3r_open
+ *     H5FD__s3comms_s3r_open
  *     H5FD_s3comms_s3r_getsize << called by open() _only_
- *     H5FD_s3comms_s3r_read    << called by getsize(), multiple times working
- *     H5FD_s3comms_s3r_close
+ *     H5FD__s3comms_s3r_read    << called by getsize(), multiple times working
+ *     H5FD__s3comms_s3r_close
  *
  *     Shows most basic curl iteration
  *
@@ -1625,7 +1101,7 @@ test_s3r_read(void)
     s3r_t *handle = NULL;
     herr_t ret;
 
-    TESTING("test_s3r_read");
+    TESTING("s3r_read");
 
     /* Initial setup */
     if (false == s3_test_bucket_defined) {
@@ -1640,10 +1116,10 @@ test_s3r_read(void)
         TEST_ERROR;
 
     /* Open file */
-    handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL);
+    handle = H5FD__s3comms_s3r_open(url_raven, NULL, NULL);
     if (handle == NULL)
         TEST_ERROR;
-    if (6464 != H5FD_s3comms_s3r_get_filesize(handle))
+    if (6464 != H5FD__s3comms_s3r_get_filesize(handle))
         TEST_ERROR;
 
     /*****************************
@@ -1652,7 +1128,7 @@ test_s3r_read(void)
 
     /* Read from start of file */
     memset(buffer, 0, S3COMMS_READ_BUFFER_SIZE);
-    if (H5FD_s3comms_s3r_read(handle, (haddr_t)0, (size_t)118, buffer) < 0)
+    if (H5FD__s3comms_s3r_read(handle, (haddr_t)0, (size_t)118, buffer) < 0)
         TEST_ERROR;
     if (strcmp("Once upon a midnight dreary, while I pondered, weak and weary,\n"
                "Over many a quaint and curious volume of forgotten lore",
@@ -1661,21 +1137,21 @@ test_s3r_read(void)
 
     /* Read arbitrary range */
     memset(buffer, 0, S3COMMS_READ_BUFFER_SIZE);
-    if (H5FD_s3comms_s3r_read(handle, (haddr_t)2540, (size_t)54, buffer) < 0)
+    if (H5FD__s3comms_s3r_read(handle, (haddr_t)2540, (size_t)54, buffer) < 0)
         TEST_ERROR;
     if (strcmp("the grave and stern decorum of the countenance it wore", buffer))
         TEST_ERROR;
 
     /* Read one character */
     memset(buffer, 0, S3COMMS_READ_BUFFER_SIZE);
-    if (H5FD_s3comms_s3r_read(handle, (haddr_t)2540, (size_t)1, buffer) < 0)
+    if (H5FD__s3comms_s3r_read(handle, (haddr_t)2540, (size_t)1, buffer) < 0)
         TEST_ERROR;
     if (strcmp("t", buffer))
         TEST_ERROR;
 
     /* Read to EOF */
     memset(buffer, 0, S3COMMS_READ_BUFFER_SIZE);
-    if (H5FD_s3comms_s3r_read(handle, (haddr_t)6370, (size_t)0, buffer) < 0)
+    if (H5FD__s3comms_s3r_read(handle, (haddr_t)6370, (size_t)0, buffer) < 0)
         TEST_ERROR;
     if (strncmp(
             buffer,
@@ -1691,7 +1167,7 @@ test_s3r_read(void)
     memset(buffer, 0, S3COMMS_READ_BUFFER_SIZE);
     H5E_BEGIN_TRY
     {
-        ret = H5FD_s3comms_s3r_read(handle, (haddr_t)6400, (size_t)100, /* 6400+100 > 6464 */ buffer);
+        ret = H5FD__s3comms_s3r_read(handle, (haddr_t)6400, (size_t)100, /* 6400+100 > 6464 */ buffer);
     }
     H5E_END_TRY
     if (ret == SUCCEED)
@@ -1703,7 +1179,7 @@ test_s3r_read(void)
     memset(buffer, 0, S3COMMS_READ_BUFFER_SIZE);
     H5E_BEGIN_TRY
     {
-        ret = H5FD_s3comms_s3r_read(handle, (haddr_t)1200699, /* 1200699 > 6464 */ (size_t)100, buffer);
+        ret = H5FD__s3comms_s3r_read(handle, (haddr_t)1200699, /* 1200699 > 6464 */ (size_t)100, buffer);
     }
     H5E_END_TRY
     if (ret == SUCCEED)
@@ -1715,7 +1191,7 @@ test_s3r_read(void)
     memset(buffer, 0, S3COMMS_READ_BUFFER_SIZE);
     H5E_BEGIN_TRY
     {
-        ret = H5FD_s3comms_s3r_read(handle, (haddr_t)6464, (size_t)0, buffer);
+        ret = H5FD__s3comms_s3r_read(handle, (haddr_t)6464, (size_t)0, buffer);
     }
     H5E_END_TRY
     if (ret == SUCCEED)
@@ -1727,7 +1203,7 @@ test_s3r_read(void)
      * TEAR DOWN *
      *************/
 
-    if (H5FD_s3comms_s3r_close(handle) < 0)
+    if (H5FD__s3comms_s3r_close(handle) < 0)
         TEST_ERROR;
 
     PASSED();
@@ -1735,7 +1211,7 @@ test_s3r_read(void)
 
 error:
     if (handle != NULL)
-        H5FD_s3comms_s3r_close(handle);
+        H5FD__s3comms_s3r_close(handle);
     return 1;
 } /* end test_s3r_read() */
 
@@ -1770,15 +1246,15 @@ main(void)
     s3_test_aws_region[0]            = '\0';
     s3_test_bucket_url[0]            = '\0';
 
-    /* TODO: unit/regression test for H5FD_s3comms_load_aws_profile()
+    /* TODO: unit/regression test for H5FD__s3comms_load_aws_profile()
      * requires a few test files and/or manipulation of default path
      */
 
     /* attempt to load test credentials
      * if unable, certain tests will be skipped
      */
-    if (SUCCEED == H5FD_s3comms_load_aws_profile(S3_TEST_PROFILE_NAME, s3_test_aws_access_key_id,
-                                                 s3_test_aws_secret_access_key, s3_test_aws_region)) {
+    if (SUCCEED == H5FD__s3comms_load_aws_profile(S3_TEST_PROFILE_NAME, s3_test_aws_access_key_id,
+                                                  s3_test_aws_secret_access_key, s3_test_aws_region)) {
         s3_test_credentials_loaded = 1;
     }
 
@@ -1789,20 +1265,18 @@ main(void)
     }
     else {
         strncpy(s3_test_bucket_url, bucket_url_env, S3_TEST_MAX_URL_SIZE);
-        s3_test_bucket_defined = true;
+        s3_test_bucket_url[S3_TEST_MAX_URL_SIZE - 1] = '\0';
+        s3_test_bucket_defined                       = true;
     }
 
     curl_global_init(CURL_GLOBAL_DEFAULT);
 
     nerrors += test_macro_format_credential();
-    nerrors += test_aws_canonical_request();
-    nerrors += test_bytes_to_hex();
+    nerrors += test_make_aws_canonical_request();
     nerrors += test_hrb_init_request();
     nerrors += test_hrb_node_set();
-    nerrors += test_HMAC_SHA256();
-    nerrors += test_parse_url();
-    nerrors += test_signing_key();
-    nerrors += test_tostringtosign();
+    nerrors += test_make_aws_signing_key();
+    nerrors += test_make_aws_stringtosign();
 
     nerrors += test_s3r_get_filesize();
     nerrors += test_s3r_open();
diff --git a/test/tarray.c b/test/tarray.c
index 6a1e3d68ecc..ace3929a222 100644
--- a/test/tarray.c
+++ b/test/tarray.c
@@ -2375,7 +2375,7 @@ test_compat(void)
  *-------------------------------------------------------------------------
  */
 void
-test_array(const void H5_ATTR_UNUSED *params)
+test_array(void H5_ATTR_UNUSED *params)
 {
     /* Output message about test being performed */
     MESSAGE(5, ("Testing Array Datatypes\n"));
diff --git a/test/tattr.c b/test/tattr.c
index 2d1aeed77c2..e06694622cd 100644
--- a/test/tattr.c
+++ b/test/tattr.c
@@ -11937,7 +11937,7 @@ test_attr_delete_last_dense(hid_t fcpl, hid_t fapl)
 **
 ****************************************************************/
 void
-test_attr(const void H5_ATTR_UNUSED *params)
+test_attr(void H5_ATTR_UNUSED *params)
 {
     hid_t    fapl = (H5I_INVALID_HID), fapl2 = (H5I_INVALID_HID); /* File access property lists */
     hid_t    fcpl = (H5I_INVALID_HID), fcpl2 = (H5I_INVALID_HID); /* File creation property lists */
diff --git a/test/tchecksum.c b/test/tchecksum.c
index 9b55f5ec902..d0cb7de25b0 100644
--- a/test/tchecksum.c
+++ b/test/tchecksum.c
@@ -216,7 +216,7 @@ test_chksum_large(void)
 **
 ****************************************************************/
 void
-test_checksum(const void H5_ATTR_UNUSED *params)
+test_checksum(void H5_ATTR_UNUSED *params)
 {
     /* Output message about test being performed */
     MESSAGE(5, ("Testing checksum algorithms\n"));
diff --git a/test/tconfig.c b/test/tconfig.c
index 97a0b6385bc..c36620e6dec 100644
--- a/test/tconfig.c
+++ b/test/tconfig.c
@@ -59,7 +59,7 @@ void test_exit_definitions(void);
  *-------------------------------------------------------------------------
  */
 void
-test_configure(const void H5_ATTR_UNUSED *params)
+test_configure(void H5_ATTR_UNUSED *params)
 {
     /* Output message about test being performed */
     MESSAGE(5, ("Testing configure definitions\n"));
diff --git a/test/tcoords.c b/test/tcoords.c
index 52561317253..d41081404ae 100644
--- a/test/tcoords.c
+++ b/test/tcoords.c
@@ -678,7 +678,7 @@ test_multiple_ends(hid_t file, bool is_chunked)
 **
 ****************************************************************/
 void
-test_coords(const void H5_ATTR_UNUSED *params)
+test_coords(void H5_ATTR_UNUSED *params)
 {
     hid_t  fid;
     bool   is_chunk[2] = {true, false};
diff --git a/test/testframe.c b/test/testframe.c
index 05cb57967ea..9eb76e2f282 100644
--- a/test/testframe.c
+++ b/test/testframe.c
@@ -24,7 +24,7 @@
 typedef struct TestStruct {
     char Name[MAXTESTNAME];
     char Description[MAXTESTDESC];
-    void (*TestFunc)(const void *);
+    void (*TestFunc)(void *);
     void (*TestSetupFunc)(void *);
     void (*TestCleanupFunc)(void *);
     void *TestParameters;
@@ -60,7 +60,7 @@ int TestVerbosity_g          = VERBO_DEF; /* Default Verbosity is Low */
  * Add a new test to the list of tests to be executed
  */
 herr_t
-AddTest(const char *TestName, void (*TestFunc)(const void *), void (*TestSetupFunc)(void *),
+AddTest(const char *TestName, void (*TestFunc)(void *), void (*TestSetupFunc)(void *),
         void (*TestCleanupFunc)(void *), const void *TestData, size_t TestDataSize, const char *TestDescr)
 {
     void *new_test_data = NULL;
@@ -537,6 +537,15 @@ TestShutdown(void)
     return SUCCEED;
 }
 
+/*
+ * Retrieve the MPI rank for this process.
+ */
+H5_ATTR_PURE int
+GetTestFrameworkProcessID(void)
+{
+    return TestFrameworkProcessID_g;
+}
+
 /*
  * Retrieve the verbosity level for the testing framework
  */
diff --git a/test/testframe.h b/test/testframe.h
index 2b24f03c07d..9047d20b6ee 100644
--- a/test/testframe.h
+++ b/test/testframe.h
@@ -91,16 +91,16 @@
  * Verbose queries
  * Only None needs an exact match.  The rest are at least as much.
  */
-#define VERBOSE_NONE (TestVerbosity_g == VERBO_NONE)
-#define VERBOSE_DEF  (TestVerbosity_g >= VERBO_DEF)
-#define VERBOSE_LO   (TestVerbosity_g >= VERBO_LO)
-#define VERBOSE_MED  (TestVerbosity_g >= VERBO_MED)
-#define VERBOSE_HI   (TestVerbosity_g >= VERBO_HI)
+#define VERBOSE_NONE (GetTestVerbosity() == VERBO_NONE)
+#define VERBOSE_DEF  (GetTestVerbosity() >= VERBO_DEF)
+#define VERBOSE_LO   (GetTestVerbosity() >= VERBO_LO)
+#define VERBOSE_MED  (GetTestVerbosity() >= VERBO_MED)
+#define VERBOSE_HI   (GetTestVerbosity() >= VERBO_HI)
 
 /* Used to document process through a test */
 #define MESSAGE(V, A)                                                                                        \
     do {                                                                                                     \
-        if (TestFrameworkProcessID_g == 0 && TestVerbosity_g > (V))                                          \
+        if (GetTestFrameworkProcessID() == 0 && GetTestVerbosity() > (V))                                    \
             printf A;                                                                                        \
     } while (0)
 
@@ -112,9 +112,6 @@
 /* Variables */
 /*************/
 
-H5TEST_DLLVAR int TestFrameworkProcessID_g;
-H5TEST_DLLVAR int TestVerbosity_g;
-
 /**************/
 /* Prototypes */
 /**************/
@@ -343,7 +340,7 @@ H5TEST_DLL void TestInfo(FILE *stream);
  * \see PerformTests()
  *
  */
-H5TEST_DLL herr_t AddTest(const char *TestName, void (*TestFunc)(const void *), void (*TestSetupFunc)(void *),
+H5TEST_DLL herr_t AddTest(const char *TestName, void (*TestFunc)(void *), void (*TestSetupFunc)(void *),
                           void (*TestCleanupFunc)(void *), const void *TestData, size_t TestDataSize,
                           const char *TestDescr);
 
@@ -422,6 +419,20 @@ H5TEST_DLL herr_t PerformTests(void);
  */
 H5TEST_DLL void TestSummary(FILE *stream);
 
+/**
+ * --------------------------------------------------------------------------
+ * \ingroup H5TEST
+ *
+ * \brief Returns the MPI rank for this process
+ *
+ * \return The MPI rank of this process
+ *
+ * \details GetTestFrameworkProcessID() returns the MPI rank for this process.
+ *          Always returns rank 0 in serial HDF5.
+ *
+ */
+H5TEST_DLL int GetTestFrameworkProcessID(void);
+
 /**
  * --------------------------------------------------------------------------
  * \ingroup H5TEST
diff --git a/test/testhdf5.h b/test/testhdf5.h
index 16ec9b4b055..9af0517241c 100644
--- a/test/testhdf5.h
+++ b/test/testhdf5.h
@@ -179,32 +179,32 @@ extern "C" {
 #endif
 
 /* Prototypes for the test routines */
-void test_metadata(const void *params);
-void test_checksum(const void *params);
-void test_refstr(const void *params);
-void test_file(const void *params);
-void test_h5o(const void *params);
-void test_h5t(const void *params);
-void test_h5s(const void *params);
-void test_coords(const void *params);
-void test_h5d(const void *params);
-void test_attr(const void *params);
-void test_select(const void *params);
-void test_time(const void *params);
-void test_reference(const void *params);
-void test_reference_deprec(const void *params);
-void test_vltypes(const void *params);
-void test_vlstrings(const void *params);
-void test_iterate(const void *params);
-void test_array(const void *params);
-void test_genprop(const void *params);
-void test_configure(const void *params);
-void test_h5_system(const void *params);
-void test_misc(const void *params);
-void test_ids(const void *params);
-void test_skiplist(const void *params);
-void test_sohm(const void *params);
-void test_unicode(const void *params);
+void test_metadata(void *params);
+void test_checksum(void *params);
+void test_refstr(void *params);
+void test_file(void *params);
+void test_h5o(void *params);
+void test_h5t(void *params);
+void test_h5s(void *params);
+void test_coords(void *params);
+void test_h5d(void *params);
+void test_attr(void *params);
+void test_select(void *params);
+void test_time(void *params);
+void test_reference(void *params);
+void test_reference_deprec(void *params);
+void test_vltypes(void *params);
+void test_vlstrings(void *params);
+void test_iterate(void *params);
+void test_array(void *params);
+void test_genprop(void *params);
+void test_configure(void *params);
+void test_h5_system(void *params);
+void test_misc(void *params);
+void test_ids(void *params);
+void test_skiplist(void *params);
+void test_sohm(void *params);
+void test_unicode(void *params);
 
 /* Prototypes for the cleanup routines */
 void cleanup_metadata(void *params);
diff --git a/test/tfile.c b/test/tfile.c
index 2d5e71b5135..493e5de2f6d 100644
--- a/test/tfile.c
+++ b/test/tfile.c
@@ -8346,7 +8346,7 @@ test_deprec(const char *driver_name)
 **
 ****************************************************************/
 void
-test_file(const void H5_ATTR_UNUSED *params)
+test_file(void H5_ATTR_UNUSED *params)
 {
     const char *driver_name;               /* File Driver value from environment */
     hid_t       fapl_id = H5I_INVALID_HID; /* VFD-dependent fapl ID */
diff --git a/test/tgenprop.c b/test/tgenprop.c
index 8d8e39a5b79..22495f54382 100644
--- a/test/tgenprop.c
+++ b/test/tgenprop.c
@@ -1926,6 +1926,80 @@ test_genprop_refcount(void)
 
 } /* ent test_genprop_refcount() */
 
+/****************************************************************
+**
+** test_set_default_plist_fail(): Test that the default property lists are unmodifiable
+**
+****************************************************************/
+static void
+test_set_default_plist_fail(void)
+{
+    hid_t  vol_id = H5I_INVALID_HID;
+    herr_t ret    = FAIL;
+
+    /* Output message about test being performed */
+    MESSAGE(5, ("Testing that default property lists are unmodifiable\n"));
+
+    /* Attempt to modify the default generic property list */
+    H5E_BEGIN_TRY
+    {
+        ret = H5Pset_vol(H5P_DEFAULT, H5VL_NATIVE, NULL);
+    }
+    H5E_END_TRY
+
+    VERIFY(ret, FAIL, "H5Pset_vol");
+
+    H5E_BEGIN_TRY
+    {
+        ret = H5Pset_userblock(H5P_FILE_CREATE_DEFAULT, 1024);
+    }
+    H5E_END_TRY
+
+    VERIFY(ret, FAIL, "H5Pset_userblock");
+
+    H5E_BEGIN_TRY
+    {
+        ret = H5Pset_layout(H5P_DATASET_CREATE_DEFAULT, H5D_CONTIGUOUS);
+    }
+    H5E_END_TRY
+
+    VERIFY(ret, FAIL, "H5Pset_layout");
+
+    H5E_BEGIN_TRY
+    {
+        ret = H5Pset_efile_prefix(H5P_DATASET_ACCESS_DEFAULT, "prefix");
+    }
+    H5E_END_TRY
+
+    VERIFY(ret, FAIL, "H5Pset_efile_prefix");
+
+    H5E_BEGIN_TRY
+    {
+        ret = H5Pset_vol(H5P_FILE_ACCESS_DEFAULT, vol_id, NULL);
+    }
+    H5E_END_TRY
+
+    VERIFY(ret, FAIL, "H5Pset_vol");
+
+    H5E_BEGIN_TRY
+    {
+        ret = H5Pset_preserve(H5P_DATASET_XFER_DEFAULT, true);
+    }
+    H5E_END_TRY
+
+    VERIFY(ret, FAIL, "H5Pset_preserve");
+
+    H5E_BEGIN_TRY
+    {
+        ret = H5Pset_local_heap_size_hint(H5P_GROUP_CREATE_DEFAULT, 0);
+    }
+    H5E_END_TRY
+
+    VERIFY(ret, FAIL, "H5Pset_local_heap_size_hint");
+
+    return;
+}
+
 #ifndef H5_NO_DEPRECATED_SYMBOLS
 /****************************************************************
 **
@@ -2145,7 +2219,7 @@ test_genprop_deprec_list(void)
 **
 ****************************************************************/
 void
-test_genprop(const void H5_ATTR_UNUSED *params)
+test_genprop(void H5_ATTR_UNUSED *params)
 {
     /* Output message about test being performed */
     MESSAGE(5, ("Testing Generic Properties\n"));
@@ -2167,6 +2241,8 @@ test_genprop(const void H5_ATTR_UNUSED *params)
     test_genprop_list_add_remove_prop(); /* Test adding and removing the same property several times to HDF5
                                             property list */
 
+    test_set_default_plist_fail(); /* Test that default property lists cannot be modified */
+
     test_genprop_equal();    /* Tests for more H5Pequal verification */
     test_genprop_path();     /* Tests for class path verification */
     test_genprop_refcount(); /* Tests for class reference counting */
diff --git a/test/th5_system.c b/test/th5_system.c
index 9980e6a788f..bbbe0e47753 100644
--- a/test/th5_system.c
+++ b/test/th5_system.c
@@ -551,7 +551,7 @@ test_h5_strndup(void)
 }
 
 void
-test_h5_system(const void H5_ATTR_UNUSED *params)
+test_h5_system(void H5_ATTR_UNUSED *params)
 {
     MESSAGE(5, ("Testing H5system routines\n"));
 
diff --git a/test/th5o.c b/test/th5o.c
index e605d4e6f84..beda55c0d59 100644
--- a/test/th5o.c
+++ b/test/th5o.c
@@ -2045,7 +2045,7 @@ test_h5o_visit2(void)
 **
 ****************************************************************/
 void
-test_h5o(const void H5_ATTR_UNUSED *params)
+test_h5o(void H5_ATTR_UNUSED *params)
 {
     /* Output message about test being performed */
     MESSAGE(5, ("Testing Objects\n"));
diff --git a/test/th5s.c b/test/th5s.c
index 4835e1f0e03..061af06a53f 100644
--- a/test/th5s.c
+++ b/test/th5s.c
@@ -3480,7 +3480,7 @@ test_versionbounds(void)
 **
 ****************************************************************/
 void
-test_h5s(const void H5_ATTR_UNUSED *params)
+test_h5s(void H5_ATTR_UNUSED *params)
 {
     H5F_libver_t low, high; /* Low and high bounds */
 
diff --git a/test/tid.c b/test/tid.c
index c4ec0d8ea40..f0edd810d3e 100644
--- a/test/tid.c
+++ b/test/tid.c
@@ -79,9 +79,9 @@ basic_id_test(void)
         goto out;
 
     /* Register a type */
-    myType = H5Iregister_type((size_t)64, 0, free_wrapper);
+    myType = H5Iregister_type2(0, free_wrapper);
 
-    CHECK(myType, H5I_BADID, "H5Iregister_type");
+    CHECK(myType, H5I_BADID, "H5Iregister_type2");
     if (myType == H5I_BADID)
         goto out;
 
@@ -173,9 +173,9 @@ basic_id_test(void)
     H5E_END_TRY
 
     /* Register another type and another object in that type */
-    myType = H5Iregister_type((size_t)64, 0, free_wrapper);
+    myType = H5Iregister_type2(0, free_wrapper);
 
-    CHECK(myType, H5I_BADID, "H5Iregister_type");
+    CHECK(myType, H5I_BADID, "H5Iregister_type2");
     if (myType == H5I_BADID)
         goto out;
 
@@ -246,6 +246,225 @@ basic_id_test(void)
     return -1;
 }
 
+#ifndef H5_NO_DEPRECATED_SYMBOLS
+/* Test that H5Iregister_type1() works correctly (just a copy of the basic test) */
+static int
+H5Iregister_type1_test(void)
+{
+    H5I_type_t myType  = H5I_BADID;
+    hid_t      arrayID = H5I_INVALID_HID;
+    void      *testObj = NULL;
+    void      *testPtr = NULL;
+    char       nameString[10];
+    hid_t      testID;
+    ssize_t    testSize = -1;
+    herr_t     err;
+    int        num_ref;
+    hsize_t    num_members;
+
+    /* Try to register an ID with fictitious types */
+    H5E_BEGIN_TRY
+    arrayID = H5Iregister((H5I_type_t)420, testObj);
+    H5E_END_TRY
+
+    VERIFY(arrayID, H5I_INVALID_HID, "H5Iregister");
+    if (arrayID != H5I_INVALID_HID)
+        goto out;
+
+    H5E_BEGIN_TRY
+    arrayID = H5Iregister((H5I_type_t)-1, testObj);
+    H5E_END_TRY
+
+    VERIFY(arrayID, H5I_INVALID_HID, "H5Iregister");
+    if (arrayID != H5I_INVALID_HID)
+        goto out;
+
+    /* Try to access IDs with fictitious types */
+    H5E_BEGIN_TRY
+    testPtr = H5Iobject_verify((hid_t)100, (H5I_type_t)0);
+    H5E_END_TRY
+
+    CHECK_PTR_NULL(testPtr, "H5Iobject_verify");
+    if (testPtr != NULL)
+        goto out;
+
+    H5E_BEGIN_TRY
+    testPtr = H5Iobject_verify((hid_t)700, (H5I_type_t)700);
+    H5E_END_TRY
+
+    CHECK_PTR_NULL(testPtr, "H5Iobject_verify");
+    if (testPtr != NULL)
+        goto out;
+
+    /* Register a type */
+    myType = H5Iregister_type1(64, 0, free_wrapper);
+
+    CHECK(myType, H5I_BADID, "H5Iregister_type1");
+    if (myType == H5I_BADID)
+        goto out;
+
+    /* Register an ID and retrieve the object it points to.
+     * Once the ID has been registered, testObj will be freed when
+     * its ID type is destroyed.
+     */
+    testObj = malloc(7 * sizeof(int));
+    arrayID = H5Iregister(myType, testObj);
+
+    CHECK(arrayID, H5I_INVALID_HID, "H5Iregister");
+    if (arrayID == H5I_INVALID_HID) {
+        free(testObj);
+        goto out;
+    }
+
+    testPtr = (int *)H5Iobject_verify(arrayID, myType);
+
+    CHECK_PTR_EQ(testPtr, testObj, "H5Iobject_verify");
+    if (testPtr != testObj)
+        goto out;
+
+    /* Ensure that H5Iget_file_id and H5Iget_name() fail, since this
+     * is an hid_t for the wrong kind of object
+     */
+    H5E_BEGIN_TRY
+    testID = H5Iget_file_id(arrayID);
+    H5E_END_TRY
+
+    VERIFY(testID, H5I_INVALID_HID, "H5Iget_file_id");
+    if (testID != H5I_INVALID_HID)
+        goto out;
+
+    H5E_BEGIN_TRY
+    testSize = H5Iget_name(arrayID, nameString, (size_t)9);
+    H5E_END_TRY
+
+    VERIFY(testSize, -1, "H5Iget_name");
+    if (testSize != -1)
+        goto out;
+
+    /* Make sure H5Iremove_verify catches objects of the wrong type */
+    H5E_BEGIN_TRY
+    testPtr = (int *)H5Iremove_verify(arrayID, (H5I_type_t)0);
+    H5E_END_TRY
+
+    CHECK_PTR_NULL(testPtr, "H5Iremove_verify");
+    if (testPtr != NULL)
+        goto out;
+
+    H5E_BEGIN_TRY
+    testPtr = (int *)H5Iremove_verify(arrayID, (H5I_type_t)((int)myType - 1));
+    H5E_END_TRY
+
+    CHECK_PTR_NULL(testPtr, "H5Iremove_verify");
+    if (testPtr != NULL)
+        goto out;
+
+    /* Remove an ID and make sure we can't access it */
+    testPtr = (int *)H5Iremove_verify(arrayID, myType);
+
+    CHECK_PTR(testPtr, "H5Iremove_verify");
+    if (testPtr == NULL)
+        goto out;
+
+    H5E_BEGIN_TRY
+    testPtr = (int *)H5Iobject_verify(arrayID, myType);
+    H5E_END_TRY
+
+    CHECK_PTR_NULL(testPtr, "H5Iobject_verify");
+    if (testPtr != NULL)
+        goto out;
+
+    /* Delete the type and make sure we can't access objects within it */
+    arrayID = H5Iregister(myType, testObj);
+
+    err = H5Idestroy_type(myType);
+    VERIFY(err, 0, "H5Idestroy_type");
+    if (err != 0)
+        goto out;
+    VERIFY(H5Itype_exists(myType), 0, "H5Itype_exists");
+    if (H5Itype_exists(myType) != 0)
+        goto out;
+
+    H5E_BEGIN_TRY
+    VERIFY(H5Inmembers(myType, NULL), -1, "H5Inmembers");
+    if (H5Inmembers(myType, NULL) != -1)
+        goto out;
+    H5E_END_TRY
+
+    /* Register another type and another object in that type */
+    myType = H5Iregister_type1(64, 0, free_wrapper);
+
+    CHECK(myType, H5I_BADID, "H5Iregister_type1");
+    if (myType == H5I_BADID)
+        goto out;
+
+    /* The memory that testObj pointed to should already have been
+     * freed when the previous type was destroyed.  Allocate new
+     * memory for it.
+     */
+    testObj = malloc(7 * sizeof(int));
+    arrayID = H5Iregister(myType, testObj);
+
+    CHECK(arrayID, H5I_INVALID_HID, "H5Iregister");
+    if (arrayID == H5I_INVALID_HID) {
+        free(testObj);
+        goto out;
+    }
+
+    err = H5Inmembers(myType, &num_members);
+    CHECK(err, -1, "H5Inmembers");
+    if (err < 0)
+        goto out;
+    VERIFY(num_members, 1, "H5Inmembers");
+    if (num_members != 1)
+        goto out;
+
+    /* Increment references to type and ensure that dec_type_ref
+     * doesn't destroy the type
+     */
+    num_ref = H5Iinc_type_ref(myType);
+    VERIFY(num_ref, 2, "H5Iinc_type_ref");
+    if (num_ref != 2)
+        goto out;
+    num_ref = H5Idec_type_ref(myType);
+    VERIFY(num_ref, 1, "H5Idec_type_ref");
+    if (num_ref != 1)
+        goto out;
+    err = H5Inmembers(myType, &num_members);
+    CHECK(err, -1, "H5Inmembers");
+    if (err < 0)
+        goto out;
+    VERIFY(num_members, 1, "H5Inmembers");
+    if (num_members != 1)
+        goto out;
+
+    /* This call to dec_type_ref should destroy the type */
+    num_ref = H5Idec_type_ref(myType);
+    VERIFY(num_ref, 0, "H5Idec_type_ref");
+    if (num_ref != 0)
+        goto out;
+    VERIFY(H5Itype_exists(myType), 0, "H5Itype_exists");
+    if (H5Itype_exists(myType) != 0)
+        goto out;
+
+    H5E_BEGIN_TRY
+    err = H5Inmembers(myType, &num_members);
+    if (err >= 0)
+        goto out;
+    H5E_END_TRY
+
+    return 0;
+
+out:
+    /* Clean up type if it has been allocated and free memory used
+     * by testObj
+     */
+    if (myType >= 0)
+        H5Idestroy_type(myType);
+
+    return -1;
+}
+#endif /* H5_NO_DEPRECATED_SYMBOLS */
+
 /* A dummy search function for the next test */
 static int
 test_search_func(void H5_ATTR_UNUSED *ptr1, hid_t H5_ATTR_UNUSED id, void H5_ATTR_UNUSED *ptr2)
@@ -508,47 +727,47 @@ test_id_type_list(void)
     H5I_type_t testType;
     int        i; /* Just a counter variable */
 
-    startType = H5Iregister_type((size_t)8, 0, free_wrapper);
-    CHECK(startType, H5I_BADID, "H5Iregister_type");
+    startType = H5Iregister_type2(0, free_wrapper);
+    CHECK(startType, H5I_BADID, "H5Iregister_type2");
     if (startType == H5I_BADID)
         goto out;
 
     /* Sanity check */
     if ((int)startType >= H5I_MAX_NUM_TYPES || startType < H5I_NTYPES) {
         /* Error condition, throw an error */
-        ERROR("H5Iregister_type");
+        ERROR("H5Iregister_type2");
         goto out;
     }
     /* Create types up to H5I_MAX_NUM_TYPES */
     for (i = startType + 1; i < H5I_MAX_NUM_TYPES; i++) {
-        currentType = H5Iregister_type((size_t)8, 0, free_wrapper);
-        CHECK(currentType, H5I_BADID, "H5Iregister_type");
+        currentType = H5Iregister_type2(0, free_wrapper);
+        CHECK(currentType, H5I_BADID, "H5Iregister_type2");
         if (currentType == H5I_BADID)
             goto out;
     }
 
     /* Wrap around to low type ID numbers */
     for (i = H5I_NTYPES; i < startType; i++) {
-        currentType = H5Iregister_type((size_t)8, 0, free_wrapper);
-        CHECK(currentType, H5I_BADID, "H5Iregister_type");
+        currentType = H5Iregister_type2(0, free_wrapper);
+        CHECK(currentType, H5I_BADID, "H5Iregister_type2");
         if (currentType == H5I_BADID)
             goto out;
     }
 
     /* There should be no room at the inn for a new ID type*/
     H5E_BEGIN_TRY
-    testType = H5Iregister_type((size_t)8, 0, free_wrapper);
+    testType = H5Iregister_type2(0, free_wrapper);
     H5E_END_TRY
 
-    VERIFY(testType, H5I_BADID, "H5Iregister_type");
+    VERIFY(testType, H5I_BADID, "H5Iregister_type2");
     if (testType != H5I_BADID)
         goto out;
 
     /* Now delete a type and try to insert again */
     H5Idestroy_type(H5I_NTYPES);
-    testType = H5Iregister_type((size_t)8, 0, free_wrapper);
+    testType = H5Iregister_type2(0, free_wrapper);
 
-    VERIFY(testType, H5I_NTYPES, "H5Iregister_type");
+    VERIFY(testType, H5I_NTYPES, "H5Iregister_type2");
     if (testType != H5I_NTYPES)
         goto out;
 
@@ -700,8 +919,8 @@ test_remove_clear_type(void)
     herr_t         ret; /* return value */
 
     /* Register a user-defined type with our custom ID-deleting callback */
-    obj_type = H5Iregister_type((size_t)8, 0, rct_free_cb);
-    CHECK(obj_type, H5I_BADID, "H5Iregister_type");
+    obj_type = H5Iregister_type2(0, rct_free_cb);
+    CHECK(obj_type, H5I_BADID, "H5Iregister_type2");
     if (obj_type == H5I_BADID)
         goto error;
 
@@ -994,8 +1213,8 @@ test_future_ids(void)
     herr_t        ret;             /* Return value */
 
     /* Register a user-defined type with our custom ID-deleting callback */
-    obj_type = H5Iregister_type((size_t)15, 0, free_actual_object);
-    CHECK(obj_type, H5I_BADID, "H5Iregister_type");
+    obj_type = H5Iregister_type2(0, free_actual_object);
+    CHECK(obj_type, H5I_BADID, "H5Iregister_type2");
     if (H5I_BADID == obj_type)
         goto error;
 
@@ -1054,8 +1273,8 @@ test_future_ids(void)
         goto error;
 
     /* Re-register a user-defined type with our custom ID-deleting callback */
-    obj_type = H5Iregister_type((size_t)15, 0, free_actual_object);
-    CHECK(obj_type, H5I_BADID, "H5Iregister_type");
+    obj_type = H5Iregister_type2(0, free_actual_object);
+    CHECK(obj_type, H5I_BADID, "H5Iregister_type2");
     if (H5I_BADID == obj_type)
         goto error;
 
@@ -1094,8 +1313,8 @@ test_future_ids(void)
         goto error;
 
     /* Re-register a user-defined type with our custom ID-deleting callback */
-    obj_type = H5Iregister_type((size_t)15, 0, free_actual_object);
-    CHECK(obj_type, H5I_BADID, "H5Iregister_type");
+    obj_type = H5Iregister_type2(0, free_actual_object);
+    CHECK(obj_type, H5I_BADID, "H5Iregister_type2");
     if (H5I_BADID == obj_type)
         goto error;
 
@@ -1495,13 +1714,17 @@ test_appropriate_ids(void)
 }
 
 void
-test_ids(const void H5_ATTR_UNUSED *params)
+test_ids(void H5_ATTR_UNUSED *params)
 {
     /* Set the random # seed */
     srand((unsigned)time(NULL));
 
     if (basic_id_test() < 0)
         TestErrPrintf("Basic ID test failed\n");
+#ifndef H5_NO_DEPRECATED_SYMBOLS
+    if (H5Iregister_type1_test() < 0)
+        TestErrPrintf("H5Iregister_type1() test failed\n");
+#endif /* H5_NO_DEPRECATED_SYMBOLS */
     if (id_predefined_test() < 0)
         TestErrPrintf("Predefined ID type test failed\n");
     if (test_is_valid() < 0)
diff --git a/test/titerate.c b/test/titerate.c
index 0450500bd38..7efae347247 100644
--- a/test/titerate.c
+++ b/test/titerate.c
@@ -1243,7 +1243,7 @@ test_links_deprec(hid_t fapl)
 **
 ****************************************************************/
 void
-test_iterate(const void H5_ATTR_UNUSED *params)
+test_iterate(void H5_ATTR_UNUSED *params)
 {
     hid_t    fapl, fapl2; /* File access property lists */
     unsigned new_format;  /* Whether to use the new format or not */
diff --git a/test/tmeta.c b/test/tmeta.c
index 5a48836bf9d..f9c7fa79088 100644
--- a/test/tmeta.c
+++ b/test/tmeta.c
@@ -53,7 +53,7 @@ static uint8_t encode_buffer[sizeof(compar_buffer)];
 **
 ****************************************************************/
 void
-test_metadata(const void H5_ATTR_UNUSED *params)
+test_metadata(void H5_ATTR_UNUSED *params)
 {
     int16_t  ei16 = TEST_INT16_VALUE; /* variables to hold the values to encode */
     uint16_t eu16 = TEST_UINT16_VALUE;
diff --git a/test/tmisc.c b/test/tmisc.c
index 6a0674d41bc..c614696dffc 100644
--- a/test/tmisc.c
+++ b/test/tmisc.c
@@ -7125,7 +7125,7 @@ test_misc41(void)
 **
 ****************************************************************/
 void
-test_misc(const void H5_ATTR_UNUSED *params)
+test_misc(void H5_ATTR_UNUSED *params)
 {
     bool default_driver = h5_using_default_driver(NULL);
 
diff --git a/test/trefer.c b/test/trefer.c
index bc443b7b4be..a7cbcaa183f 100644
--- a/test/trefer.c
+++ b/test/trefer.c
@@ -3834,7 +3834,7 @@ test_reference_perf(void)
 **
 ****************************************************************/
 void
-test_reference(const void H5_ATTR_UNUSED *params)
+test_reference(void H5_ATTR_UNUSED *params)
 {
     H5F_libver_t low, high;   /* Low and high bounds */
     const char  *driver_name; /* File Driver value from environment */
diff --git a/test/trefer_deprec.c b/test/trefer_deprec.c
index d06f5e39ef1..2407b87b68e 100644
--- a/test/trefer_deprec.c
+++ b/test/trefer_deprec.c
@@ -1917,7 +1917,7 @@ test_reference_compat(void)
 **
 ****************************************************************/
 void
-test_reference_deprec(const void H5_ATTR_UNUSED *params)
+test_reference_deprec(void H5_ATTR_UNUSED *params)
 {
     H5F_libver_t low, high; /* Low and high bounds */
     bool         vol_is_native;
diff --git a/test/trefstr.c b/test/trefstr.c
index 93c0a14b3f6..849c4fcf4f0 100644
--- a/test/trefstr.c
+++ b/test/trefstr.c
@@ -512,7 +512,7 @@ test_refstr_finalize(void)
 **
 ****************************************************************/
 void
-test_refstr(const void H5_ATTR_UNUSED *params)
+test_refstr(void H5_ATTR_UNUSED *params)
 {
     /* Output message about test being performed */
     MESSAGE(5, ("Testing Reference Counted Strings\n"));
diff --git a/test/tselect.c b/test/tselect.c
index 21c3648d587..03477d16919 100644
--- a/test/tselect.c
+++ b/test/tselect.c
@@ -16060,7 +16060,7 @@ test_h5s_set_extent_none(void)
 **
 ****************************************************************/
 void
-test_select(const void H5_ATTR_UNUSED *params)
+test_select(void H5_ATTR_UNUSED *params)
 {
     hid_t       plist_id;                     /* Property list for reading random hyperslabs */
     hid_t       fapl;                         /* Property list accessing the file */
diff --git a/test/tskiplist.c b/test/tskiplist.c
index d9a765aafb1..ba3a2d971b8 100644
--- a/test/tskiplist.c
+++ b/test/tskiplist.c
@@ -1555,7 +1555,7 @@ test_skiplist_term(void)
 **
 ****************************************************************/
 void
-test_skiplist(const void H5_ATTR_UNUSED *params)
+test_skiplist(void H5_ATTR_UNUSED *params)
 {
     /* Output message about test being performed */
     MESSAGE(5, ("Testing Skip Lists\n"));
diff --git a/test/tsohm.c b/test/tsohm.c
index 71d3578bcbb..4ce224bfc00 100644
--- a/test/tsohm.c
+++ b/test/tsohm.c
@@ -3708,7 +3708,7 @@ test_sohm_external_dtype(void)
 **
 ****************************************************************/
 void
-test_sohm(const void H5_ATTR_UNUSED *params)
+test_sohm(void H5_ATTR_UNUSED *params)
 {
     const char *driver_name;
     bool        vol_is_native;
diff --git a/test/ttime.c b/test/ttime.c
index 1218e262694..43e4e08c946 100644
--- a/test/ttime.c
+++ b/test/ttime.c
@@ -198,7 +198,7 @@ test_time_io(void)
 **
 ****************************************************************/
 void
-test_time(const void H5_ATTR_UNUSED *params)
+test_time(void H5_ATTR_UNUSED *params)
 {
     /* Output message about test being performed */
     MESSAGE(5, ("Testing Time Datatypes\n"));
diff --git a/test/ttsafe.c b/test/ttsafe.c
index 04f30c15138..fde42220eb5 100644
--- a/test/ttsafe.c
+++ b/test/ttsafe.c
@@ -51,18 +51,18 @@ num_digits(int num)
 
 /* Test the H5is_library_threadsafe() function */
 void
-tts_is_threadsafe(const void H5_ATTR_UNUSED *params)
+tts_is_threadsafe(void H5_ATTR_UNUSED *params)
 {
     bool is_ts;
     bool should_be;
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
     is_ts     = false;
     should_be = true;
-#else  /* H5_HAVE_THREADSAFE */
+#else  /* H5_HAVE_THREADSAFE_API */
     is_ts     = true;
     should_be = false;
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
     if (H5is_library_threadsafe(&is_ts) != SUCCEED)
         TestErrPrintf("H5_is_library_threadsafe() call failed - test failed\n");
@@ -117,7 +117,7 @@ main(int argc, char *argv[])
 #ifdef H5_HAVE_STDATOMIC_H
     MESSAGE(2, ("\tC11 atomics enabled\n"));
 #endif
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
     MESSAGE(2, ("\tThreadsafe API enabled\n"));
 #endif
 #endif
@@ -144,11 +144,9 @@ main(int argc, char *argv[])
 #endif /* !H5_HAVE_WIN_THREADS */
     AddTest("semaphore", tts_semaphore, NULL, NULL, NULL, 0, "lightweight system semaphores");
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
     AddTest("thread_id", tts_thread_id, NULL, NULL, NULL, 0, "thread IDs");
 
-    /* Error stack test must be done after thread_id test to not mess up expected IDs */
-    AddTest("error_stacks", tts_error_stacks, NULL, NULL, NULL, 0, "error stack tests");
     AddTest("dcreate", tts_dcreate, NULL, cleanup_dcreate, NULL, 0, "multi-dataset creation");
     AddTest("error", tts_error, NULL, cleanup_error, NULL, 0, "per-thread error stacks");
 #ifdef H5_HAVE_PTHREAD_H
@@ -158,14 +156,17 @@ main(int argc, char *argv[])
     AddTest("acreate", tts_acreate, NULL, cleanup_acreate, NULL, 0, "multi-attribute creation");
     AddTest("attr_vlen", tts_attr_vlen, NULL, cleanup_attr_vlen, NULL, 0, "multi-file-attribute-vlen read");
 
+    /* Error stack test must be done after thread_id test to not mess up expected IDs */
+    AddTest("error_stacks", tts_error_stacks, NULL, NULL, NULL, 0, "error stack tests");
+
     /* Developer API routine tests */
     AddTest("developer", tts_develop_api, NULL, NULL, NULL, 0, "developer API routines");
 
-#else /* H5_HAVE_THREADSAFE */
+#else /* H5_HAVE_THREADSAFE_API */
 
     printf("Most thread-safety tests skipped because THREADSAFE not enabled\n");
 
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 
 #else /* H5_HAVE_THREADS */
 
diff --git a/test/ttsafe.h b/test/ttsafe.h
index 87b41d83bc6..6739f89bfcf 100644
--- a/test/ttsafe.h
+++ b/test/ttsafe.h
@@ -34,27 +34,30 @@
 extern char *gen_name(int);
 
 /* Prototypes for the test routines */
-void tts_is_threadsafe(const void *);
+void tts_is_threadsafe(void *);
 #ifdef H5_HAVE_THREADS
-void tts_thread_pool(const void *);
-void tts_atomics(const void *);
-void tts_rwlock(const void *);
-void tts_semaphore(const void *);
+void tts_thread_pool(void *);
+#ifndef H5_HAVE_STDATOMIC_H
+/* C11 atomics only tested when emulated */
+void tts_atomics(void *);
+#endif /* H5_HAVE_STDATOMIC_H */
+void tts_rwlock(void *);
+void tts_semaphore(void *);
 #ifndef H5_HAVE_WIN_THREADS
-void tts_rec_rwlock_smoke_check_1(const void *);
-void tts_rec_rwlock_smoke_check_2(const void *);
-void tts_rec_rwlock_smoke_check_3(const void *);
-void tts_rec_rwlock_smoke_check_4(const void *);
+void tts_rec_rwlock_smoke_check_1(void *);
+void tts_rec_rwlock_smoke_check_2(void *);
+void tts_rec_rwlock_smoke_check_3(void *);
+void tts_rec_rwlock_smoke_check_4(void *);
 #endif /* !H5_HAVE_WIN_THREADS */
-#ifdef H5_HAVE_THREADSAFE
-void tts_dcreate(const void *);
-void tts_error(const void *);
-void tts_cancel(const void *);
-void tts_acreate(const void *);
-void tts_attr_vlen(const void *);
-void tts_thread_id(const void *);
-void tts_develop_api(const void *);
-void tts_error_stacks(const void *);
+#ifdef H5_HAVE_THREADSAFE_API
+void tts_dcreate(void *);
+void tts_error(void *);
+void tts_cancel(void *);
+void tts_acreate(void *);
+void tts_attr_vlen(void *);
+void tts_thread_id(void *);
+void tts_develop_api(void *);
+void tts_error_stacks(void *);
 
 /* Prototypes for the cleanup routines */
 void cleanup_dcreate(void *);
@@ -63,6 +66,6 @@ void cleanup_cancel(void *);
 void cleanup_acreate(void *);
 void cleanup_attr_vlen(void *);
 
-#endif /* H5_HAVE_THREADSAFE */
+#endif /* H5_HAVE_THREADSAFE_API */
 #endif /* H5_HAVE_THREADS */
 #endif /* TTSAFE_H */
diff --git a/test/ttsafe_acreate.c b/test/ttsafe_acreate.c
index 98533203ac5..51074d5419d 100644
--- a/test/ttsafe_acreate.c
+++ b/test/ttsafe_acreate.c
@@ -29,7 +29,7 @@
 
 #include "ttsafe.h"
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 
 #define FILENAME    "ttsafe_acreate.h5"
 #define DATASETNAME "IntData"
@@ -45,7 +45,7 @@ typedef struct acreate_data_struct {
 } ttsafe_name_data_t;
 
 void
-tts_acreate(const void H5_ATTR_UNUSED *params)
+tts_acreate(void H5_ATTR_UNUSED *params)
 {
     /* Thread declarations */
     H5TS_thread_t threads[NUM_THREADS];
@@ -189,4 +189,4 @@ cleanup_acreate(void H5_ATTR_UNUSED *params)
     }
 }
 
-#endif /*H5_HAVE_THREADSAFE*/
+#endif /* H5_HAVE_THREADSAFE_API */
diff --git a/test/ttsafe_atomic.c b/test/ttsafe_atomic.c
index 6dc294ea8ca..00c5fdf4f34 100644
--- a/test/ttsafe_atomic.c
+++ b/test/ttsafe_atomic.c
@@ -18,7 +18,7 @@
 
 #include "ttsafe.h"
 
-#ifdef H5_HAVE_THREADS
+#if defined(H5_HAVE_THREADS) && !defined(H5_HAVE_STDATOMIC_H)
 
 #define NUM_THREADS 16
 
@@ -64,7 +64,7 @@ decr_task(void *_counter)
  **********************************************************************
  */
 void
-tts_atomics(const void H5_ATTR_UNUSED *params)
+tts_atomics(void H5_ATTR_UNUSED *params)
 {
     H5TS_pool_t *pool = NULL;
     herr_t       result;
@@ -176,4 +176,4 @@ tts_atomics(const void H5_ATTR_UNUSED *params)
 
 } /* end tts_atomics() */
 
-#endif /* H5_HAVE_THREADS */
+#endif /* defined(H5_HAVE_THREADS) && !defined(H5_HAVE_STDATOMIC_H) */
diff --git a/test/ttsafe_attr_vlen.c b/test/ttsafe_attr_vlen.c
index 13240e26ee9..0f1b3aefeb6 100644
--- a/test/ttsafe_attr_vlen.c
+++ b/test/ttsafe_attr_vlen.c
@@ -42,7 +42,7 @@
 
 #include "ttsafe.h"
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 
 #define FILENAME    "ttsafe_attr_vlen.h5"
 #define ATTR_NAME   "root_attr"
@@ -51,7 +51,7 @@
 H5TS_THREAD_RETURN_TYPE tts_attr_vlen_thread(void *);
 
 void
-tts_attr_vlen(const void H5_ATTR_UNUSED *params)
+tts_attr_vlen(void H5_ATTR_UNUSED *params)
 {
     H5TS_thread_t threads[NUM_THREADS] = {0};             /* Thread declaration */
     hid_t         fid                  = H5I_INVALID_HID; /* File ID */
@@ -187,4 +187,4 @@ cleanup_attr_vlen(void H5_ATTR_UNUSED *params)
     }
 }
 
-#endif /*H5_HAVE_THREADSAFE*/
+#endif /* H5_HAVE_THREADSAFE_API */
diff --git a/test/ttsafe_cancel.c b/test/ttsafe_cancel.c
index 0a5dbabcc98..680080c8984 100644
--- a/test/ttsafe_cancel.c
+++ b/test/ttsafe_cancel.c
@@ -29,14 +29,13 @@
  ********************************************************************/
 #include "ttsafe.h"
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 #ifdef H5_HAVE_PTHREAD_H
 
 #define FILENAME    "ttsafe_cancel.h5"
 #define DATASETNAME "commonname"
 
 void  *tts_cancel_thread(void *);
-void   tts_cancel_barrier(void);
 herr_t tts_cancel_callback(void *, hid_t, unsigned, const hsize_t *, void *);
 void   cancellation_cleanup(void *);
 
@@ -55,7 +54,7 @@ pthread_t             childthread;
 static H5TS_barrier_t barrier;
 
 void
-tts_cancel(const void H5_ATTR_UNUSED *params)
+tts_cancel(void H5_ATTR_UNUSED *params)
 {
     hid_t dataset;
     int   buffer;
@@ -210,5 +209,5 @@ cleanup_cancel(void H5_ATTR_UNUSED *params)
     }
 }
 
-#endif /*H5_HAVE_PTHREAD_H*/
-#endif /*H5_HAVE_THREADSAFE*/
+#endif /* H5_HAVE_PTHREAD_H */
+#endif /* H5_HAVE_THREADSAFE_API */
diff --git a/test/ttsafe_dcreate.c b/test/ttsafe_dcreate.c
index 14f751055f8..e2b693bc96d 100644
--- a/test/ttsafe_dcreate.c
+++ b/test/ttsafe_dcreate.c
@@ -25,7 +25,7 @@
  ********************************************************************/
 #include "ttsafe.h"
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 
 #define FILENAME   "ttsafe_dcreate.h5"
 #define NUM_THREAD 16
@@ -54,7 +54,7 @@ thr_info thread_out[NUM_THREAD];
  **********************************************************************
  */
 void
-tts_dcreate(const void H5_ATTR_UNUSED *params)
+tts_dcreate(void H5_ATTR_UNUSED *params)
 {
     /* thread definitions */
     H5TS_thread_t threads[NUM_THREAD];
@@ -158,4 +158,4 @@ cleanup_dcreate(void H5_ATTR_UNUSED *params)
         HDunlink(FILENAME);
     }
 }
-#endif /*H5_HAVE_THREADSAFE*/
+#endif /* H5_HAVE_THREADSAFE_API */
diff --git a/test/ttsafe_develop.c b/test/ttsafe_develop.c
index fbe91949486..aec6f291922 100644
--- a/test/ttsafe_develop.c
+++ b/test/ttsafe_develop.c
@@ -16,9 +16,13 @@
  *
  ********************************************************************/
 
+#define H5VL_FRIEND /* Suppress error about including H5VLpkg    */
+#define H5VL_TESTING
+
 #include "ttsafe.h"
+#include "H5VLpkg.h" /* Virtual Object Layer                 */
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 
 typedef struct {
     H5TS_barrier_t *barrier;
@@ -97,28 +101,44 @@ tts_develop_api_thr_2(void *_udata)
  **********************************************************************
  */
 void
-tts_develop_api(const void H5_ATTR_UNUSED *params)
+tts_develop_api(void H5_ATTR_UNUSED *params)
 {
+    hid_t                   def_fapl = H5I_INVALID_HID;
+    hid_t                   vol_id   = H5I_INVALID_HID;
     H5TS_thread_t           thread_1, thread_2;
     H5TS_barrier_t          barrier;
     unsigned                lock_count = UINT_MAX;
     bool                    acquired   = false;
     tts_develop_api_udata_t udata;
     unsigned                api_count_1 = 0, api_count_2 = 0;
+    int                     is_native;
     herr_t                  result;
 
-    /* Check that API count increases with each API call */
-    result = H5TSmutex_get_attempt_count(&api_count_1);
-    CHECK_I(result, "H5TSmutex_get_attempt_count");
+    def_fapl = H5Pcreate(H5P_FILE_ACCESS);
+    CHECK(def_fapl, H5I_INVALID_HID, "H5Pcreate");
+
+    result = H5Pget_vol_id(def_fapl, &vol_id);
+    CHECK(result, FAIL, "H5Pget_vol_id");
+
+    is_native = H5VL__is_native_connector_test(vol_id);
+    CHECK(is_native, FAIL, "H5VL__is_native_connector_test");
+
+    if (is_native) {
+        /* Check that API count increases with each API call */
+        result = H5TSmutex_get_attempt_count(&api_count_1);
+        CHECK_I(result, "H5TSmutex_get_attempt_count");
 
-    /* No-op API call, to increment the API counter */
-    result = H5garbage_collect();
-    CHECK_I(result, "H5garbage_collect");
+        /* No-op API call, to increment the API counter */
+        result = H5garbage_collect();
+        CHECK_I(result, "H5garbage_collect");
 
-    result = H5TSmutex_get_attempt_count(&api_count_2);
-    CHECK_I(result, "H5TSmutex_get_attempt_count");
+        result = H5TSmutex_get_attempt_count(&api_count_2);
+        CHECK_I(result, "H5TSmutex_get_attempt_count");
 
-    VERIFY(api_count_2, (api_count_1 + 1), "H5TSmutex_get_attempt_count");
+        VERIFY(api_count_2, (api_count_1 + 1), "H5TSmutex_get_attempt_count");
+    } /* end if */
+    else
+        printf("Non-native VOL connector used, skipping mutex attempt count test\n");
 
     /* Check H5TSmutex_acquire & H5TSmutex_release in thread callbacks */
 
@@ -162,4 +182,4 @@ tts_develop_api(const void H5_ATTR_UNUSED *params)
 
 } /* end tts_develop_api() */
 
-#endif /*H5_HAVE_THREADSAFE*/
+#endif /* H5_HAVE_THREADSAFE_API */
diff --git a/test/ttsafe_error.c b/test/ttsafe_error.c
index 9322bf460ca..fc0920fde6b 100644
--- a/test/ttsafe_error.c
+++ b/test/ttsafe_error.c
@@ -30,7 +30,7 @@
 #define H5VL_TESTING
 #include "H5VLpkg.h" /* Virtual Object Layer                 */
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 
 #define NUM_THREAD 16
 #define FILENAME   "ttsafe_error.h5"
@@ -59,7 +59,7 @@ static herr_t                  walk_error_callback(unsigned, const H5E_error2_t
 static H5TS_THREAD_RETURN_TYPE tts_error_thread(void *);
 
 void
-tts_error(const void H5_ATTR_UNUSED *params)
+tts_error(void H5_ATTR_UNUSED *params)
 {
     hid_t         def_fapl = H5I_INVALID_HID;
     hid_t         vol_id   = H5I_INVALID_HID;
@@ -265,4 +265,4 @@ cleanup_error(void H5_ATTR_UNUSED *params)
     }
 }
 
-#endif /*H5_HAVE_THREADSAFE*/
+#endif /* H5_HAVE_THREADSAFE_API */
diff --git a/test/ttsafe_error_stacks.c b/test/ttsafe_error_stacks.c
index 6edca5b5fe2..a062628ef0b 100644
--- a/test/ttsafe_error_stacks.c
+++ b/test/ttsafe_error_stacks.c
@@ -11,7 +11,7 @@
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 #include "ttsafe.h"
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 
 #define ERR_CLS_NAME        "Custom error class"
 #define ERR_CLS_LIB_NAME    "example_lib"
@@ -77,7 +77,7 @@ generate_user_error(void H5_ATTR_UNUSED *arg)
 **********************************************************************
 */
 void
-tts_error_stacks(const void H5_ATTR_UNUSED *params)
+tts_error_stacks(void H5_ATTR_UNUSED *params)
 {
     H5TS_thread_t threads[2];
     herr_t        status = FAIL;
diff --git a/test/ttsafe_rec_rwlock.c b/test/ttsafe_rec_rwlock.c
index 37bf094fb80..ba9c8aeb0ce 100644
--- a/test/ttsafe_rec_rwlock.c
+++ b/test/ttsafe_rec_rwlock.c
@@ -263,7 +263,7 @@ tts_rec_rwlock_smoke_check_test_thread(void *_udata)
  **********************************************************************
  */
 void
-tts_rec_rwlock_smoke_check_1(const void H5_ATTR_UNUSED *params)
+tts_rec_rwlock_smoke_check_1(void H5_ATTR_UNUSED *params)
 {
     herr_t result;
 #if H5TS_ENABLE_REC_RWLOCK_STATS
@@ -546,7 +546,7 @@ tts_rec_rwlock_smoke_check_1(const void H5_ATTR_UNUSED *params)
  **********************************************************************
  */
 void
-tts_rec_rwlock_smoke_check_2(const void H5_ATTR_UNUSED *params)
+tts_rec_rwlock_smoke_check_2(void H5_ATTR_UNUSED *params)
 {
     herr_t                   result;
     int                      express_test;
@@ -752,7 +752,7 @@ tts_rec_rwlock_smoke_check_2(const void H5_ATTR_UNUSED *params)
  **********************************************************************
  */
 void
-tts_rec_rwlock_smoke_check_3(const void H5_ATTR_UNUSED *params)
+tts_rec_rwlock_smoke_check_3(void H5_ATTR_UNUSED *params)
 {
     herr_t                   result;
     int                      i;
@@ -958,7 +958,7 @@ tts_rec_rwlock_smoke_check_3(const void H5_ATTR_UNUSED *params)
  **********************************************************************
  */
 void
-tts_rec_rwlock_smoke_check_4(const void H5_ATTR_UNUSED *params)
+tts_rec_rwlock_smoke_check_4(void H5_ATTR_UNUSED *params)
 {
     herr_t                   result;
     int                      i;
diff --git a/test/ttsafe_rwlock.c b/test/ttsafe_rwlock.c
index b8f6b24b546..de2ab79bad0 100644
--- a/test/ttsafe_rwlock.c
+++ b/test/ttsafe_rwlock.c
@@ -182,7 +182,7 @@ verify_counting(void *_counter)
  **********************************************************************
  */
 void
-tts_rwlock(const void H5_ATTR_UNUSED *params)
+tts_rwlock(void H5_ATTR_UNUSED *params)
 {
     H5TS_thread_t    threads[NUM_THREADS];
     H5TS_pool_t     *pool = NULL;
diff --git a/test/ttsafe_semaphore.c b/test/ttsafe_semaphore.c
index 26802ad96a2..6da69b4a387 100644
--- a/test/ttsafe_semaphore.c
+++ b/test/ttsafe_semaphore.c
@@ -229,7 +229,7 @@ tts_semaphore_clientserver(void)
  **********************************************************************
  */
 void
-tts_semaphore(const void H5_ATTR_UNUSED *params)
+tts_semaphore(void H5_ATTR_UNUSED *params)
 {
     H5TS_semaphore_t sem;
     herr_t           result;
diff --git a/test/ttsafe_thread_id.c b/test/ttsafe_thread_id.c
index 540857cae71..9509a391f40 100644
--- a/test/ttsafe_thread_id.c
+++ b/test/ttsafe_thread_id.c
@@ -18,7 +18,7 @@
 
 #include "ttsafe.h"
 
-#ifdef H5_HAVE_THREADSAFE
+#ifdef H5_HAVE_THREADSAFE_API
 
 #define CYCLE_COUNT 2
 #define NTHREADS    5
@@ -93,7 +93,7 @@ thread_main(void H5_ATTR_UNUSED *arg)
  **********************************************************************
  */
 void
-tts_thread_id(const void H5_ATTR_UNUSED *params)
+tts_thread_id(void H5_ATTR_UNUSED *params)
 {
     H5TS_thread_t threads[NTHREADS];
     uint64_t      tid;
@@ -135,4 +135,4 @@ tts_thread_id(const void H5_ATTR_UNUSED *params)
 
 } /* end tts_thread_id() */
 
-#endif /*H5_HAVE_THREADSAFE*/
+#endif /* H5_HAVE_THREADSAFE_API */
diff --git a/test/ttsafe_thread_pool.c b/test/ttsafe_thread_pool.c
index 2229fa9b40e..7a6f0abe16c 100644
--- a/test/ttsafe_thread_pool.c
+++ b/test/ttsafe_thread_pool.c
@@ -92,7 +92,7 @@ decr_task(void *_counter)
  **********************************************************************
  */
 void
-tts_thread_pool(const void H5_ATTR_UNUSED *params)
+tts_thread_pool(void H5_ATTR_UNUSED *params)
 {
     H5TS_pool_t *pool = NULL;
     herr_t       result;
diff --git a/test/tunicode.c b/test/tunicode.c
index 78a3381dcfc..2349f99ae9c 100644
--- a/test/tunicode.c
+++ b/test/tunicode.c
@@ -802,7 +802,7 @@ dump_string(const char *string)
  * that string.
  */
 void
-test_unicode(const void H5_ATTR_UNUSED *params)
+test_unicode(void H5_ATTR_UNUSED *params)
 {
     char         test_string[MAX_STRING_LENGTH];
     unsigned int cur_pos = 0;   /* Current position in test_string */
diff --git a/test/tvlstr.c b/test/tvlstr.c
index e6f33e7f94c..68e29ee731d 100644
--- a/test/tvlstr.c
+++ b/test/tvlstr.c
@@ -968,7 +968,7 @@ test_write_same_element(void)
 **
 ****************************************************************/
 void
-test_vlstrings(const void H5_ATTR_UNUSED *params)
+test_vlstrings(void H5_ATTR_UNUSED *params)
 {
     /* Output message about test being performed */
     MESSAGE(5, ("Testing Variable-Length Strings\n"));
diff --git a/test/tvltypes.c b/test/tvltypes.c
index fd367ebde66..0f559e77f1a 100644
--- a/test/tvltypes.c
+++ b/test/tvltypes.c
@@ -3233,7 +3233,7 @@ test_vltypes_fill_value(void)
 **
 ****************************************************************/
 void
-test_vltypes(const void H5_ATTR_UNUSED *params)
+test_vltypes(void H5_ATTR_UNUSED *params)
 {
     /* Output message about test being performed */
     MESSAGE(5, ("Testing Variable-Length Datatypes\n"));
diff --git a/testpar/API/H5_api_async_test_parallel.c b/testpar/API/H5_api_async_test_parallel.c
index 450b69f1464..0a390e076d4 100644
--- a/testpar/API/H5_api_async_test_parallel.c
+++ b/testpar/API/H5_api_async_test_parallel.c
@@ -12,10 +12,10 @@
 
 #include "H5_api_async_test_parallel.h"
 
-static void print_async_test_header(const void *params);
+static void print_async_test_header(void *params);
 
 static void
-print_async_test_header(const void *params)
+print_async_test_header(void *params)
 {
     bool coll_md_read = true;
 
@@ -50,19 +50,19 @@ print_async_test_header(const void *params)
 
 #ifdef H5_API_TEST_HAVE_ASYNC
 
-static void test_async_one_dataset_io(const void *params);
-static void test_async_multi_dataset_io(const void *params);
-static void test_async_multi_file_dataset_io(const void *params);
-static void test_async_multi_file_grp_dset_io(const void *params);
-static void test_async_set_extent(const void *params);
-static void test_async_attribute_exists(const void *params);
-static void test_async_attribute_io(const void *params);
-static void test_async_attribute_io_tconv(const void *params);
-static void test_async_attribute_io_compound(const void *params);
-static void test_async_group(const void *params);
-static void test_async_link(const void *params);
-static void test_async_ocopy_orefresh(const void *params);
-static void test_async_file_reopen(const void *params);
+static void test_async_one_dataset_io(void *params);
+static void test_async_multi_dataset_io(void *params);
+static void test_async_multi_file_dataset_io(void *params);
+static void test_async_multi_file_grp_dset_io(void *params);
+static void test_async_set_extent(void *params);
+static void test_async_attribute_exists(void *params);
+static void test_async_attribute_io(void *params);
+static void test_async_attribute_io_tconv(void *params);
+static void test_async_attribute_io_compound(void *params);
+static void test_async_group(void *params);
+static void test_async_link(void *params);
+static void test_async_ocopy_orefresh(void *params);
+static void test_async_file_reopen(void *params);
 
 /* Highest "printf" file created (starting at 0) */
 int max_printf_file = -1;
@@ -73,7 +73,7 @@ int max_printf_file = -1;
  */
 #define ONE_DATASET_IO_TEST_SPACE_RANK 2
 static void
-test_async_one_dataset_io(const void *params)
+test_async_one_dataset_io(void *params)
 {
     hsize_t *dims = NULL;
     hsize_t  start[ONE_DATASET_IO_TEST_SPACE_RANK];
@@ -438,7 +438,7 @@ test_async_one_dataset_io(const void *params)
 #define MULTI_DATASET_IO_TEST_SPACE_RANK 2
 #define MULTI_DATASET_IO_TEST_NDSETS     5
 static void
-test_async_multi_dataset_io(const void *params)
+test_async_multi_dataset_io(void *params)
 {
     hsize_t *dims = NULL;
     hsize_t  start[MULTI_DATASET_IO_TEST_SPACE_RANK];
@@ -763,7 +763,7 @@ test_async_multi_dataset_io(const void *params)
 #define MULTI_FILE_DATASET_IO_TEST_SPACE_RANK 2
 #define MULTI_FILE_DATASET_IO_TEST_NFILES     5
 static void
-test_async_multi_file_dataset_io(const void *params)
+test_async_multi_file_dataset_io(void *params)
 {
     hsize_t *dims = NULL;
     hsize_t  start[MULTI_FILE_DATASET_IO_TEST_SPACE_RANK];
@@ -1189,7 +1189,7 @@ test_async_multi_file_dataset_io(const void *params)
 #define MULTI_FILE_GRP_DSET_IO_TEST_SPACE_RANK 2
 #define MULTI_FILE_GRP_DSET_IO_TEST_NFILES     5
 static void
-test_async_multi_file_grp_dset_io(const void *params)
+test_async_multi_file_grp_dset_io(void *params)
 {
     hsize_t *dims = NULL;
     hsize_t  start[MULTI_FILE_GRP_DSET_IO_TEST_SPACE_RANK];
@@ -1606,7 +1606,7 @@ test_async_multi_file_grp_dset_io(const void *params)
 #define SET_EXTENT_TEST_SPACE_RANK  2
 #define SET_EXTENT_TEST_NUM_EXTENDS 6
 static void
-test_async_set_extent(const void *params)
+test_async_set_extent(void *params)
 {
     hsize_t *dims    = NULL;
     hsize_t *maxdims = NULL;
@@ -1949,7 +1949,7 @@ test_async_set_extent(const void *params)
  */
 #define ATTRIBUTE_EXISTS_TEST_SPACE_RANK 2
 static void
-test_async_attribute_exists(const void *params)
+test_async_attribute_exists(void *params)
 {
     hsize_t *dims          = NULL;
     bool     op_failed     = false;
@@ -2102,7 +2102,7 @@ test_async_attribute_exists(const void *params)
  */
 #define ATTRIBUTE_IO_TEST_SPACE_RANK 2
 static void
-test_async_attribute_io(const void *params)
+test_async_attribute_io(void *params)
 {
     hsize_t *dims          = NULL;
     bool     op_failed     = false;
@@ -2311,7 +2311,7 @@ test_async_attribute_io(const void *params)
  */
 #define ATTRIBUTE_IO_TCONV_TEST_SPACE_RANK 2
 static void
-test_async_attribute_io_tconv(const void *params)
+test_async_attribute_io_tconv(void *params)
 {
     hsize_t *dims = NULL;
     bool     op_failed;
@@ -2511,7 +2511,7 @@ typedef struct tattr_cmpd_t {
 
 #define ATTRIBUTE_IO_COMPOUND_TEST_SPACE_RANK 2
 static void
-test_async_attribute_io_compound(const void *params)
+test_async_attribute_io_compound(void *params)
 {
     hsize_t      *dims = NULL;
     bool          op_failed;
@@ -2915,7 +2915,7 @@ test_async_attribute_io_compound(const void *params)
  * Tests async group interfaces in parallel
  */
 static void
-test_async_group(const void *params)
+test_async_group(void *params)
 {
     hid_t      file_id         = H5I_INVALID_HID;
     hid_t      fapl_id         = H5I_INVALID_HID;
@@ -3093,7 +3093,7 @@ test_async_group(const void *params)
  * Tests async link interfaces in parallel
  */
 static void
-test_async_link(const void *params)
+test_async_link(void *params)
 {
     hid_t  file_id         = H5I_INVALID_HID;
     hid_t  fapl_id         = H5I_INVALID_HID;
@@ -3354,7 +3354,7 @@ test_async_link(const void *params)
  */
 #define OCOPY_REFRESH_TEST_SPACE_RANK 2
 static void
-test_async_ocopy_orefresh(const void *params)
+test_async_ocopy_orefresh(void *params)
 {
     hsize_t *dims            = NULL;
     hid_t    file_id         = H5I_INVALID_HID;
@@ -3518,7 +3518,7 @@ test_async_ocopy_orefresh(const void *params)
  * Tests H5Freopen_async in parallel
  */
 static void
-test_async_file_reopen(const void *params)
+test_async_file_reopen(void *params)
 {
     hid_t  file_id          = H5I_INVALID_HID;
     hid_t  fapl_id          = H5I_INVALID_HID;
@@ -3600,7 +3600,7 @@ test_async_file_reopen(const void *params)
  * Cleanup temporary test files
  */
 static void
-test_async_file_cleanup(const void H5_ATTR_UNUSED *params)
+test_async_file_cleanup(void H5_ATTR_UNUSED *params)
 {
     char file_name[64];
     int  i;
diff --git a/testpar/API/H5_api_attribute_test_parallel.c b/testpar/API/H5_api_attribute_test_parallel.c
index c06ac918147..7bc94d0d212 100644
--- a/testpar/API/H5_api_attribute_test_parallel.c
+++ b/testpar/API/H5_api_attribute_test_parallel.c
@@ -12,10 +12,10 @@
 
 #include "H5_api_attribute_test_parallel.h"
 
-static void print_attribute_test_header(const void *params);
+static void print_attribute_test_header(void *params);
 
 static void
-print_attribute_test_header(const void H5_ATTR_UNUSED *params)
+print_attribute_test_header(void H5_ATTR_UNUSED *params)
 {
     if (MAINPROCESS) {
         printf("\n");
diff --git a/testpar/API/H5_api_dataset_test_parallel.c b/testpar/API/H5_api_dataset_test_parallel.c
index d4e208aaee9..edc57771ed1 100644
--- a/testpar/API/H5_api_dataset_test_parallel.c
+++ b/testpar/API/H5_api_dataset_test_parallel.c
@@ -16,38 +16,38 @@
  */
 #include "H5_api_dataset_test_parallel.h"
 
-static void print_dataset_test_header(const void *params);
-static void test_write_dataset_data_verification(const void *params);
-static void test_write_dataset_independent(const void *params);
-static void test_write_dataset_one_proc_0_selection(const void *params);
-static void test_write_dataset_one_proc_none_selection(const void *params);
-static void test_write_dataset_one_proc_all_selection(const void *params);
-static void test_write_dataset_hyper_file_all_mem(const void *params);
-static void test_write_dataset_all_file_hyper_mem(const void *params);
-static void test_write_dataset_point_file_all_mem(const void *params);
-static void test_write_dataset_all_file_point_mem(const void *params);
-static void test_write_dataset_hyper_file_point_mem(const void *params);
-static void test_write_dataset_point_file_hyper_mem(const void *params);
-static void test_read_dataset_one_proc_0_selection(const void *params);
-static void test_read_dataset_one_proc_none_selection(const void *params);
-static void test_read_dataset_one_proc_all_selection(const void *params);
-static void test_read_dataset_hyper_file_all_mem(const void *params);
-static void test_read_dataset_all_file_hyper_mem(const void *params);
-static void test_read_dataset_point_file_all_mem(const void *params);
-static void test_read_dataset_all_file_point_mem(const void *params);
-static void test_read_dataset_hyper_file_point_mem(const void *params);
-static void test_read_dataset_point_file_hyper_mem(const void *params);
+static void print_dataset_test_header(void *params);
+static void test_write_dataset_data_verification(void *params);
+static void test_write_dataset_independent(void *params);
+static void test_write_dataset_one_proc_0_selection(void *params);
+static void test_write_dataset_one_proc_none_selection(void *params);
+static void test_write_dataset_one_proc_all_selection(void *params);
+static void test_write_dataset_hyper_file_all_mem(void *params);
+static void test_write_dataset_all_file_hyper_mem(void *params);
+static void test_write_dataset_point_file_all_mem(void *params);
+static void test_write_dataset_all_file_point_mem(void *params);
+static void test_write_dataset_hyper_file_point_mem(void *params);
+static void test_write_dataset_point_file_hyper_mem(void *params);
+static void test_read_dataset_one_proc_0_selection(void *params);
+static void test_read_dataset_one_proc_none_selection(void *params);
+static void test_read_dataset_one_proc_all_selection(void *params);
+static void test_read_dataset_hyper_file_all_mem(void *params);
+static void test_read_dataset_all_file_hyper_mem(void *params);
+static void test_read_dataset_point_file_all_mem(void *params);
+static void test_read_dataset_all_file_point_mem(void *params);
+static void test_read_dataset_hyper_file_point_mem(void *params);
+static void test_read_dataset_point_file_hyper_mem(void *params);
 
 /*
  * Chunking tests
  */
-static void test_write_multi_chunk_dataset_same_shape_read(const void *params);
-static void test_write_multi_chunk_dataset_diff_shape_read(const void *params);
-static void test_overwrite_multi_chunk_dataset_same_shape_read(const void *params);
-static void test_overwrite_multi_chunk_dataset_diff_shape_read(const void *params);
+static void test_write_multi_chunk_dataset_same_shape_read(void *params);
+static void test_write_multi_chunk_dataset_diff_shape_read(void *params);
+static void test_overwrite_multi_chunk_dataset_same_shape_read(void *params);
+static void test_overwrite_multi_chunk_dataset_diff_shape_read(void *params);
 
 static void
-print_dataset_test_header(const void H5_ATTR_UNUSED *params)
+print_dataset_test_header(void H5_ATTR_UNUSED *params)
 {
     if (MAINPROCESS) {
         printf("\n");
@@ -73,7 +73,7 @@ print_dataset_test_header(const void H5_ATTR_UNUSED *params)
 #define DATASET_WRITE_DATA_VERIFY_TEST_DSET_NAME2 "dataset_write_data_verification_hyperslab"
 #define DATASET_WRITE_DATA_VERIFY_TEST_DSET_NAME3 "dataset_write_data_verification_points"
 static void
-test_write_dataset_data_verification(const void H5_ATTR_UNUSED *params)
+test_write_dataset_data_verification(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *dims = NULL;
@@ -891,7 +891,7 @@ test_write_dataset_data_verification(const void H5_ATTR_UNUSED *params)
 #define DATASET_INDEPENDENT_WRITE_TEST_DSET_NAME1 "dset1"
 #define DATASET_INDEPENDENT_WRITE_TEST_DSET_NAME2 "dset2"
 static void
-test_write_dataset_independent(const void H5_ATTR_UNUSED *params)
+test_write_dataset_independent(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *dims = NULL;
@@ -1230,7 +1230,7 @@ test_write_dataset_independent(const void H5_ATTR_UNUSED *params)
 #define DATASET_WRITE_ONE_PROC_0_SEL_TEST_GROUP_NAME "one_rank_0_sel_write_test"
 #define DATASET_WRITE_ONE_PROC_0_SEL_TEST_DSET_NAME  "one_rank_0_sel_dset"
 static void
-test_write_dataset_one_proc_0_selection(const void H5_ATTR_UNUSED *params)
+test_write_dataset_one_proc_0_selection(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *dims = NULL;
@@ -1527,7 +1527,7 @@ test_write_dataset_one_proc_0_selection(const void H5_ATTR_UNUSED *params)
 #define DATASET_WRITE_ONE_PROC_NONE_SEL_TEST_GROUP_NAME "one_rank_none_sel_write_test"
 #define DATASET_WRITE_ONE_PROC_NONE_SEL_TEST_DSET_NAME  "one_rank_none_sel_dset"
 static void
-test_write_dataset_one_proc_none_selection(const void H5_ATTR_UNUSED *params)
+test_write_dataset_one_proc_none_selection(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *dims = NULL;
@@ -1841,7 +1841,7 @@ test_write_dataset_one_proc_none_selection(const void H5_ATTR_UNUSED *params)
 #define DATASET_WRITE_ONE_PROC_ALL_SEL_TEST_GROUP_NAME "one_rank_all_sel_write_test"
 #define DATASET_WRITE_ONE_PROC_ALL_SEL_TEST_DSET_NAME  "one_rank_all_sel_dset"
 static void
-test_write_dataset_one_proc_all_selection(const void H5_ATTR_UNUSED *params)
+test_write_dataset_one_proc_all_selection(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *dims = NULL;
@@ -2133,7 +2133,7 @@ test_write_dataset_one_proc_all_selection(const void H5_ATTR_UNUSED *params)
 #define DATASET_WRITE_HYPER_FILE_ALL_MEM_TEST_DSET_NAME  "hyper_sel_file_all_sel_mem_dset"
 #endif
 static void
-test_write_dataset_hyper_file_all_mem(const void H5_ATTR_UNUSED *params)
+test_write_dataset_hyper_file_all_mem(void H5_ATTR_UNUSED *params)
 {
 #ifdef BROKEN
     hssize_t space_npoints;
@@ -2408,7 +2408,7 @@ test_write_dataset_hyper_file_all_mem(const void H5_ATTR_UNUSED *params)
 #define DATASET_WRITE_ALL_FILE_HYPER_MEM_TEST_GROUP_NAME "all_sel_file_hyper_sel_mem_write_test"
 #define DATASET_WRITE_ALL_FILE_HYPER_MEM_TEST_DSET_NAME  "all_sel_file_hyper_sel_mem_dset"
 static void
-test_write_dataset_all_file_hyper_mem(const void H5_ATTR_UNUSED *params)
+test_write_dataset_all_file_hyper_mem(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *dims = NULL;
@@ -2695,7 +2695,7 @@ test_write_dataset_all_file_hyper_mem(const void H5_ATTR_UNUSED *params)
  * in the memory dataspace.
  */
 static void
-test_write_dataset_point_file_all_mem(const void H5_ATTR_UNUSED *params)
+test_write_dataset_point_file_all_mem(void H5_ATTR_UNUSED *params)
 {
     TESTING("write to dataset with point sel. for file space; all sel. for memory");
 
@@ -2715,7 +2715,7 @@ test_write_dataset_point_file_all_mem(const void H5_ATTR_UNUSED *params)
 #define DATASET_WRITE_ALL_FILE_POINT_MEM_TEST_GROUP_NAME "all_sel_file_point_sel_mem_write_test"
 #define DATASET_WRITE_ALL_FILE_POINT_MEM_TEST_DSET_NAME  "all_sel_file_point_sel_mem_dset"
 static void
-test_write_dataset_all_file_point_mem(const void H5_ATTR_UNUSED *params)
+test_write_dataset_all_file_point_mem(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *points = NULL;
@@ -3026,7 +3026,7 @@ test_write_dataset_all_file_point_mem(const void H5_ATTR_UNUSED *params)
 #define DATASET_WRITE_HYPER_FILE_POINT_MEM_TEST_GROUP_NAME "hyper_sel_file_point_sel_mem_write_test"
 #define DATASET_WRITE_HYPER_FILE_POINT_MEM_TEST_DSET_NAME  "hyper_sel_file_point_sel_mem_dset"
 static void
-test_write_dataset_hyper_file_point_mem(const void H5_ATTR_UNUSED *params)
+test_write_dataset_hyper_file_point_mem(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *dims   = NULL;
@@ -3350,7 +3350,7 @@ test_write_dataset_hyper_file_point_mem(const void H5_ATTR_UNUSED *params)
 #define DATASET_WRITE_POINT_FILE_HYPER_MEM_TEST_GROUP_NAME "point_sel_file_hyper_sel_mem_write_test"
 #define DATASET_WRITE_POINT_FILE_HYPER_MEM_TEST_DSET_NAME  "point_sel_file_hyper_sel_mem_dset"
 static void
-test_write_dataset_point_file_hyper_mem(const void H5_ATTR_UNUSED *params)
+test_write_dataset_point_file_hyper_mem(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *dims   = NULL;
@@ -3667,7 +3667,7 @@ test_write_dataset_point_file_hyper_mem(const void H5_ATTR_UNUSED *params)
 #define DATASET_READ_ONE_PROC_0_SEL_TEST_GROUP_NAME "one_rank_0_sel_read_test"
 #define DATASET_READ_ONE_PROC_0_SEL_TEST_DSET_NAME  "one_rank_0_sel_dset"
 static void
-test_read_dataset_one_proc_0_selection(const void H5_ATTR_UNUSED *params)
+test_read_dataset_one_proc_0_selection(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *dims = NULL;
@@ -4001,7 +4001,7 @@ test_read_dataset_one_proc_0_selection(const void H5_ATTR_UNUSED *params)
 #define DATASET_READ_ONE_PROC_NONE_SEL_TEST_GROUP_NAME "one_rank_none_sel_read_test"
 #define DATASET_READ_ONE_PROC_NONE_SEL_TEST_DSET_NAME  "one_rank_none_sel_dset"
 static void
-test_read_dataset_one_proc_none_selection(const void H5_ATTR_UNUSED *params)
+test_read_dataset_one_proc_none_selection(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *dims = NULL;
@@ -4351,7 +4351,7 @@ test_read_dataset_one_proc_none_selection(const void H5_ATTR_UNUSED *params)
 #define DATASET_READ_ONE_PROC_ALL_SEL_TEST_GROUP_NAME "one_rank_all_sel_read_test"
 #define DATASET_READ_ONE_PROC_ALL_SEL_TEST_DSET_NAME  "one_rank_all_sel_dset"
 static void
-test_read_dataset_one_proc_all_selection(const void H5_ATTR_UNUSED *params)
+test_read_dataset_one_proc_all_selection(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *dims = NULL;
@@ -4683,7 +4683,7 @@ test_read_dataset_one_proc_all_selection(const void H5_ATTR_UNUSED *params)
  * selection in the memory dataspace.
  */
 static void
-test_read_dataset_hyper_file_all_mem(const void H5_ATTR_UNUSED *params)
+test_read_dataset_hyper_file_all_mem(void H5_ATTR_UNUSED *params)
 {
     TESTING("read from dataset with hyperslab sel. for file space; all sel. for memory");
 
@@ -4703,7 +4703,7 @@ test_read_dataset_hyper_file_all_mem(const void H5_ATTR_UNUSED *params)
 #define DATASET_READ_ALL_FILE_HYPER_MEM_TEST_GROUP_NAME "all_sel_file_hyper_sel_mem_read_test"
 #define DATASET_READ_ALL_FILE_HYPER_MEM_TEST_DSET_NAME  "all_sel_file_hyper_sel_mem_dset"
 static void
-test_read_dataset_all_file_hyper_mem(const void H5_ATTR_UNUSED *params)
+test_read_dataset_all_file_hyper_mem(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *dims = NULL;
@@ -5030,7 +5030,7 @@ test_read_dataset_all_file_hyper_mem(const void H5_ATTR_UNUSED *params)
  * in the memory dataspace.
  */
 static void
-test_read_dataset_point_file_all_mem(const void H5_ATTR_UNUSED *params)
+test_read_dataset_point_file_all_mem(void H5_ATTR_UNUSED *params)
 {
     TESTING("read from dataset with point sel. for file space; all sel. for memory");
 
@@ -5050,7 +5050,7 @@ test_read_dataset_point_file_all_mem(const void H5_ATTR_UNUSED *params)
 #define DATASET_READ_ALL_FILE_POINT_MEM_TEST_GROUP_NAME "all_sel_file_point_sel_mem_read_test"
 #define DATASET_READ_ALL_FILE_POINT_MEM_TEST_DSET_NAME  "all_sel_file_point_sel_mem_dset"
 static void
-test_read_dataset_all_file_point_mem(const void H5_ATTR_UNUSED *params)
+test_read_dataset_all_file_point_mem(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *points = NULL;
@@ -5398,7 +5398,7 @@ test_read_dataset_all_file_point_mem(const void H5_ATTR_UNUSED *params)
 #define DATASET_READ_HYPER_FILE_POINT_MEM_TEST_GROUP_NAME "hyper_sel_file_point_sel_mem_read_test"
 #define DATASET_READ_HYPER_FILE_POINT_MEM_TEST_DSET_NAME  "hyper_sel_file_point_sel_mem_dset"
 static void
-test_read_dataset_hyper_file_point_mem(const void H5_ATTR_UNUSED *params)
+test_read_dataset_hyper_file_point_mem(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *dims   = NULL;
@@ -5756,7 +5756,7 @@ test_read_dataset_hyper_file_point_mem(const void H5_ATTR_UNUSED *params)
 #define DATASET_READ_POINT_FILE_HYPER_MEM_TEST_GROUP_NAME "point_sel_file_hyper_sel_mem_read_test"
 #define DATASET_READ_POINT_FILE_HYPER_MEM_TEST_DSET_NAME  "point_sel_file_hyper_sel_mem_dset"
 static void
-test_read_dataset_point_file_hyper_mem(const void H5_ATTR_UNUSED *params)
+test_read_dataset_point_file_hyper_mem(void H5_ATTR_UNUSED *params)
 {
     hssize_t space_npoints;
     hsize_t *dims   = NULL;
@@ -6119,7 +6119,7 @@ test_read_dataset_point_file_hyper_mem(const void H5_ATTR_UNUSED *params)
     "multi_chunk_dataset_write_same_space_read_test"
 #define DATASET_MULTI_CHUNK_WRITE_SAME_SPACE_READ_TEST_DSET_NAME "multi_chunk_dataset"
 static void
-test_write_multi_chunk_dataset_same_shape_read(const void H5_ATTR_UNUSED *params)
+test_write_multi_chunk_dataset_same_shape_read(void H5_ATTR_UNUSED *params)
 {
     hsize_t *dims       = NULL;
     hsize_t *chunk_dims = NULL;
@@ -6581,7 +6581,7 @@ test_write_multi_chunk_dataset_same_shape_read(const void H5_ATTR_UNUSED *params
     "multi_chunk_dataset_write_diff_space_read_test"
 #define DATASET_MULTI_CHUNK_WRITE_DIFF_SPACE_READ_TEST_DSET_NAME "multi_chunk_dataset"
 static void
-test_write_multi_chunk_dataset_diff_shape_read(const void H5_ATTR_UNUSED *params)
+test_write_multi_chunk_dataset_diff_shape_read(void H5_ATTR_UNUSED *params)
 {
     hsize_t *dims       = NULL;
     hsize_t *chunk_dims = NULL;
@@ -7046,7 +7046,7 @@ test_write_multi_chunk_dataset_diff_shape_read(const void H5_ATTR_UNUSED *params
 #define DATASET_MULTI_CHUNK_OVERWRITE_SAME_SPACE_READ_TEST_DSET_NAME "multi_chunk_dataset"
 #define DATASET_MULTI_CHUNK_OVERWRITE_SAME_SPACE_READ_TEST_NITERS    10
 static void
-test_overwrite_multi_chunk_dataset_same_shape_read(const void H5_ATTR_UNUSED *params)
+test_overwrite_multi_chunk_dataset_same_shape_read(void H5_ATTR_UNUSED *params)
 {
     hsize_t *dims       = NULL;
     hsize_t *chunk_dims = NULL;
@@ -7571,7 +7571,7 @@ test_overwrite_multi_chunk_dataset_same_shape_read(const void H5_ATTR_UNUSED *pa
 #define DATASET_MULTI_CHUNK_OVERWRITE_DIFF_SPACE_READ_TEST_DSET_NAME "multi_chunk_dataset"
 #define DATASET_MULTI_CHUNK_OVERWRITE_DIFF_SPACE_READ_TEST_NITERS    10
 static void
-test_overwrite_multi_chunk_dataset_diff_shape_read(const void H5_ATTR_UNUSED *params)
+test_overwrite_multi_chunk_dataset_diff_shape_read(void H5_ATTR_UNUSED *params)
 {
     hsize_t *dims       = NULL;
     hsize_t *chunk_dims = NULL;
diff --git a/testpar/API/H5_api_datatype_test_parallel.c b/testpar/API/H5_api_datatype_test_parallel.c
index 6b5faf2d04e..ea927b47286 100644
--- a/testpar/API/H5_api_datatype_test_parallel.c
+++ b/testpar/API/H5_api_datatype_test_parallel.c
@@ -12,10 +12,10 @@
 
 #include "H5_api_datatype_test_parallel.h"
 
-static void print_datatype_test_header(const void *params);
+static void print_datatype_test_header(void *params);
 
 static void
-print_datatype_test_header(const void H5_ATTR_UNUSED *params)
+print_datatype_test_header(void H5_ATTR_UNUSED *params)
 {
     if (MAINPROCESS) {
         printf("\n");
diff --git a/testpar/API/H5_api_file_test_parallel.c b/testpar/API/H5_api_file_test_parallel.c
index c86517c7347..74b72157cd0 100644
--- a/testpar/API/H5_api_file_test_parallel.c
+++ b/testpar/API/H5_api_file_test_parallel.c
@@ -12,13 +12,13 @@
 
 #include "H5_api_file_test_parallel.h"
 
-static void print_file_test_header(const void *params);
-static void test_create_file(const void *params);
-static void test_open_file(const void *params);
-static void test_split_comm_file_access(const void *params);
+static void print_file_test_header(void *params);
+static void test_create_file(void *params);
+static void test_open_file(void *params);
+static void test_split_comm_file_access(void *params);
 
 static void
-print_file_test_header(const void H5_ATTR_UNUSED *params)
+print_file_test_header(void H5_ATTR_UNUSED *params)
 {
     if (MAINPROCESS) {
         printf("\n");
@@ -35,7 +35,7 @@ print_file_test_header(const void H5_ATTR_UNUSED *params)
  */
 #define FILE_CREATE_TEST_FILENAME "test_file_parallel.h5"
 static void
-test_create_file(const void H5_ATTR_UNUSED *params)
+test_create_file(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id = H5I_INVALID_HID;
     hid_t fapl_id = H5I_INVALID_HID;
@@ -86,7 +86,7 @@ test_create_file(const void H5_ATTR_UNUSED *params)
  * A test to ensure that a file can be opened in parallel.
  */
 static void
-test_open_file(const void H5_ATTR_UNUSED *params)
+test_open_file(void H5_ATTR_UNUSED *params)
 {
     hid_t file_id = H5I_INVALID_HID;
     hid_t fapl_id = H5I_INVALID_HID;
@@ -195,7 +195,7 @@ test_open_file(const void H5_ATTR_UNUSED *params)
  */
 #define SPLIT_FILE_COMM_TEST_FILE_NAME "split_comm_file.h5"
 static void
-test_split_comm_file_access(const void H5_ATTR_UNUSED *params)
+test_split_comm_file_access(void H5_ATTR_UNUSED *params)
 {
     MPI_Comm comm;
     MPI_Info info    = MPI_INFO_NULL;
diff --git a/testpar/API/H5_api_group_test_parallel.c b/testpar/API/H5_api_group_test_parallel.c
index ffacc870ac0..9826a157cda 100644
--- a/testpar/API/H5_api_group_test_parallel.c
+++ b/testpar/API/H5_api_group_test_parallel.c
@@ -12,10 +12,10 @@
 
 #include "H5_api_group_test_parallel.h"
 
-static void print_group_test_header(const void *params);
+static void print_group_test_header(void *params);
 
 static void
-print_group_test_header(const void H5_ATTR_UNUSED *params)
+print_group_test_header(void H5_ATTR_UNUSED *params)
 {
     if (MAINPROCESS) {
         printf("\n");
diff --git a/testpar/API/H5_api_link_test_parallel.c b/testpar/API/H5_api_link_test_parallel.c
index 8df1d74ba24..cb751cc4128 100644
--- a/testpar/API/H5_api_link_test_parallel.c
+++ b/testpar/API/H5_api_link_test_parallel.c
@@ -12,10 +12,10 @@
 
 #include "H5_api_link_test_parallel.h"
 
-static void print_link_test_header(const void *params);
+static void print_link_test_header(void *params);
 
 static void
-print_link_test_header(const void H5_ATTR_UNUSED *params)
+print_link_test_header(void H5_ATTR_UNUSED *params)
 {
     if (MAINPROCESS) {
         printf("\n");
diff --git a/testpar/API/H5_api_misc_test_parallel.c b/testpar/API/H5_api_misc_test_parallel.c
index 37ce0374bba..6413c59e46f 100644
--- a/testpar/API/H5_api_misc_test_parallel.c
+++ b/testpar/API/H5_api_misc_test_parallel.c
@@ -12,10 +12,10 @@
 
 #include "H5_api_misc_test_parallel.h"
 
-static void print_misc_test_header(const void *params);
+static void print_misc_test_header(void *params);
 
 static void
-print_misc_test_header(const void H5_ATTR_UNUSED *params)
+print_misc_test_header(void H5_ATTR_UNUSED *params)
 {
     if (MAINPROCESS) {
         printf("\n");
diff --git a/testpar/API/H5_api_object_test_parallel.c b/testpar/API/H5_api_object_test_parallel.c
index e7b3804281a..8f1df90726a 100644
--- a/testpar/API/H5_api_object_test_parallel.c
+++ b/testpar/API/H5_api_object_test_parallel.c
@@ -12,10 +12,10 @@
 
 #include "H5_api_object_test_parallel.h"
 
-static void print_object_test_header(const void *params);
+static void print_object_test_header(void *params);
 
 static void
-print_object_test_header(const void H5_ATTR_UNUSED *params)
+print_object_test_header(void H5_ATTR_UNUSED *params)
 {
     if (MAINPROCESS) {
         printf("\n");
diff --git a/testpar/API/H5_api_test_parallel.c b/testpar/API/H5_api_test_parallel.c
index 7a80c9b1bdd..e1f217ee2d6 100644
--- a/testpar/API/H5_api_test_parallel.c
+++ b/testpar/API/H5_api_test_parallel.c
@@ -29,11 +29,6 @@ char H5_api_test_parallel_filename[H5_API_TEST_FILENAME_MAX_LENGTH];
 
 const char *test_path_prefix;
 
-size_t n_tests_run_g;
-size_t n_tests_passed_g;
-size_t n_tests_failed_g;
-size_t n_tests_skipped_g;
-
 int mpi_size;
 int mpi_rank;
 
diff --git a/testpar/t_2Gio.c b/testpar/t_2Gio.c
index 258601d1776..440d52e2bc3 100644
--- a/testpar/t_2Gio.c
+++ b/testpar/t_2Gio.c
@@ -619,7 +619,7 @@ MpioTest2G(MPI_Comm comm)
  */
 
 static void
-dataset_writeInd(const void *params)
+dataset_writeInd(void *params)
 {
     hid_t   fid;                /* HDF5 file ID */
     hid_t   acc_tpl;            /* File access templates */
@@ -764,7 +764,7 @@ dataset_writeInd(const void *params)
 
 /* Example of using the parallel HDF5 library to read a dataset */
 static void
-dataset_readInd(const void *params)
+dataset_readInd(void *params)
 {
     hid_t       fid;                 /* HDF5 file ID */
     hid_t       acc_tpl;             /* File access templates */
@@ -886,7 +886,7 @@ dataset_readInd(const void *params)
  */
 
 static void
-dataset_writeAll(const void *params)
+dataset_writeAll(void *params)
 {
     hid_t   fid;                                    /* HDF5 file ID */
     hid_t   acc_tpl;                                /* File access templates */
@@ -1472,7 +1472,7 @@ dataset_writeAll(const void *params)
  */
 
 static void
-dataset_readAll(const void *params)
+dataset_readAll(void *params)
 {
     hid_t       fid;                                              /* HDF5 file ID */
     hid_t       acc_tpl;                                          /* File access templates */
@@ -1928,7 +1928,7 @@ dataset_readAll(const void *params)
  */
 
 static void
-extend_writeInd2(const void *params)
+extend_writeInd2(void *params)
 {
     const char *filename;
     hid_t       fid;             /* HDF5 file ID */
@@ -2091,7 +2091,7 @@ extend_writeInd2(const void *params)
  */
 #ifdef H5_HAVE_FILTER_DEFLATE
 static void
-compress_readAll(const void *params)
+compress_readAll(void *params)
 {
     hid_t       fid;                           /* HDF5 file ID */
     hid_t       acc_tpl;                       /* File access templates */
@@ -2276,7 +2276,7 @@ compress_readAll(const void *params)
  */
 
 static void
-none_selection_chunk(const void *params)
+none_selection_chunk(void *params)
 {
     hid_t       fid;                /* HDF5 file ID */
     hid_t       acc_tpl;            /* File access templates */
diff --git a/testpar/t_chunk_alloc.c b/testpar/t_chunk_alloc.c
index c9896a48508..9ffcab09336 100644
--- a/testpar/t_chunk_alloc.c
+++ b/testpar/t_chunk_alloc.c
@@ -456,7 +456,7 @@ verify_data(const char *filename, int chunk_factor, write_type write_pattern, in
  * it, read to verify all data are as written.
  */
 void
-test_chunk_alloc(const void *params)
+test_chunk_alloc(void *params)
 {
     const char *filename;
     hid_t       file_id, dataset;
@@ -542,7 +542,7 @@ test_chunk_alloc(const void *params)
  *     fashion.
  */
 void
-test_chunk_alloc_incr_ser_to_par(const void *params)
+test_chunk_alloc_incr_ser_to_par(void *params)
 {
     H5D_space_status_t space_status;
     const char        *filename;
diff --git a/testpar/t_coll_chunk.c b/testpar/t_coll_chunk.c
index 2569cdd9e8e..9c4c36b2c18 100644
--- a/testpar/t_coll_chunk.c
+++ b/testpar/t_coll_chunk.c
@@ -64,7 +64,7 @@ static void coll_chunktest(const char *filename, int chunk_factor, int select_fa
  */
 
 void
-coll_chunk1(const void *params)
+coll_chunk1(void *params)
 {
     const char *filename = ((const H5Ptest_param_t *)params)->name;
     int         mpi_rank;
@@ -126,7 +126,7 @@ coll_chunk1(const void *params)
  * ------------------------------------------------------------------------
  */
 void
-coll_chunk2(const void *params)
+coll_chunk2(void *params)
 {
     const char *filename = ((const H5Ptest_param_t *)params)->name;
     int         mpi_rank;
@@ -189,7 +189,7 @@ coll_chunk2(const void *params)
  */
 
 void
-coll_chunk3(const void *params)
+coll_chunk3(void *params)
 {
     const char *filename = ((const H5Ptest_param_t *)params)->name;
     int         mpi_size;
@@ -254,7 +254,7 @@ coll_chunk3(const void *params)
  */
 
 void
-coll_chunk4(const void *params)
+coll_chunk4(void *params)
 {
     const char *filename = ((const H5Ptest_param_t *)params)->name;
     int         mpi_rank;
@@ -317,7 +317,7 @@ coll_chunk4(const void *params)
  */
 
 void
-coll_chunk5(const void *params)
+coll_chunk5(void *params)
 {
     const char *filename = ((const H5Ptest_param_t *)params)->name;
     int         mpi_rank;
@@ -382,7 +382,7 @@ coll_chunk5(const void *params)
  */
 
 void
-coll_chunk6(const void *params)
+coll_chunk6(void *params)
 {
     const char *filename = ((const H5Ptest_param_t *)params)->name;
     int         mpi_rank;
@@ -445,7 +445,7 @@ coll_chunk6(const void *params)
  */
 
 void
-coll_chunk7(const void *params)
+coll_chunk7(void *params)
 {
     const char *filename = ((const H5Ptest_param_t *)params)->name;
     int         mpi_rank;
@@ -508,7 +508,7 @@ coll_chunk7(const void *params)
  */
 
 void
-coll_chunk8(const void *params)
+coll_chunk8(void *params)
 {
     const char *filename = ((const H5Ptest_param_t *)params)->name;
     int         mpi_rank;
@@ -571,7 +571,7 @@ coll_chunk8(const void *params)
  */
 
 void
-coll_chunk9(const void *params)
+coll_chunk9(void *params)
 {
     const char *filename = ((const H5Ptest_param_t *)params)->name;
     int         mpi_rank;
@@ -634,7 +634,7 @@ coll_chunk9(const void *params)
  */
 
 void
-coll_chunk10(const void *params)
+coll_chunk10(void *params)
 {
     const char *filename = ((const H5Ptest_param_t *)params)->name;
     int         mpi_rank;
diff --git a/testpar/t_coll_md.c b/testpar/t_coll_md.c
index 461c2a8febe..e1113ca9f82 100644
--- a/testpar/t_coll_md.c
+++ b/testpar/t_coll_md.c
@@ -63,7 +63,7 @@
  * arbitrary number (0 was chosen).
  */
 void
-test_partial_no_selection_coll_md_read(const void *params)
+test_partial_no_selection_coll_md_read(void *params)
 {
     const char *filename;
     hsize_t    *dataset_dims = NULL;
@@ -262,7 +262,7 @@ test_partial_no_selection_coll_md_read(const void *params)
  *
  */
 void
-test_multi_chunk_io_addrmap_issue(const void *params)
+test_multi_chunk_io_addrmap_issue(void *params)
 {
     const char *filename;
     hsize_t     start[MULTI_CHUNK_IO_ADDRMAP_ISSUE_DIMS];
@@ -390,7 +390,7 @@ test_multi_chunk_io_addrmap_issue(const void *params)
  *2096 but expected 320000 major: Internal error (too specific to document in detail) minor: MPI Error String
  */
 void
-test_link_chunk_io_sort_chunk_issue(const void *params)
+test_link_chunk_io_sort_chunk_issue(void *params)
 {
     const char *filename;
     hsize_t     dataset_dims[LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS];
@@ -554,7 +554,7 @@ test_link_chunk_io_sort_chunk_issue(const void *params)
  * heap data is not correctly mapped as raw data.
  */
 void
-test_collective_global_heap_write(const void *params)
+test_collective_global_heap_write(void *params)
 {
     const char *filename;
     hsize_t     attr_dims[COLL_GHEAP_WRITE_ATTR_DIMS];
@@ -634,7 +634,7 @@ test_collective_global_heap_write(const void *params)
  * collective metadata writes are NOT requested.
  */
 void
-test_coll_io_ind_md_write(const void *params)
+test_coll_io_ind_md_write(void *params)
 {
     const char *filename;
     long long  *data = NULL;
diff --git a/testpar/t_dset.c b/testpar/t_dset.c
index 92884c5b270..fa9d97d490c 100644
--- a/testpar/t_dset.c
+++ b/testpar/t_dset.c
@@ -224,7 +224,7 @@ dataset_vrfy(hsize_t start[], hsize_t count[], hsize_t stride[], hsize_t block[]
  */
 
 void
-dataset_writeInd(const void *params)
+dataset_writeInd(void *params)
 {
     hid_t       fid;                /* HDF5 file ID */
     hid_t       acc_tpl;            /* File access templates */
@@ -376,7 +376,7 @@ dataset_writeInd(const void *params)
 
 /* Example of using the parallel HDF5 library to read a dataset */
 void
-dataset_readInd(const void *params)
+dataset_readInd(void *params)
 {
     hid_t       fid;                 /* HDF5 file ID */
     hid_t       acc_tpl;             /* File access templates */
@@ -511,7 +511,7 @@ dataset_readInd(const void *params)
  */
 
 void
-dataset_writeAll(const void *params)
+dataset_writeAll(void *params)
 {
     hid_t       fid;                                    /* HDF5 file ID */
     hid_t       acc_tpl;                                /* File access templates */
@@ -1107,7 +1107,7 @@ dataset_writeAll(const void *params)
  */
 
 void
-dataset_readAll(const void *params)
+dataset_readAll(void *params)
 {
     hid_t       fid;                                              /* HDF5 file ID */
     hid_t       acc_tpl;                                          /* File access templates */
@@ -1582,7 +1582,7 @@ dataset_readAll(const void *params)
  */
 
 void
-extend_writeInd(const void *params)
+extend_writeInd(void *params)
 {
     hid_t       fid;                /* HDF5 file ID */
     hid_t       acc_tpl;            /* File access templates */
@@ -1816,7 +1816,7 @@ extend_writeInd(const void *params)
  */
 
 void
-extend_writeInd2(const void *params)
+extend_writeInd2(void *params)
 {
     const char *filename;
     hid_t       fid;             /* HDF5 file ID */
@@ -1989,7 +1989,7 @@ extend_writeInd2(const void *params)
 
 /* Example of using the parallel HDF5 library to read an extendible dataset */
 void
-extend_readInd(const void *params)
+extend_readInd(void *params)
 {
     hid_t       fid;                 /* HDF5 file ID */
     hid_t       acc_tpl;             /* File access templates */
@@ -2179,7 +2179,7 @@ extend_readInd(const void *params)
  */
 
 void
-extend_writeAll(const void *params)
+extend_writeAll(void *params)
 {
     hid_t       fid;                /* HDF5 file ID */
     hid_t       acc_tpl;            /* File access templates */
@@ -2432,7 +2432,7 @@ extend_writeAll(const void *params)
 
 /* Example of using the parallel HDF5 library to read an extendible dataset */
 void
-extend_readAll(const void *params)
+extend_readAll(void *params)
 {
     hid_t       fid;                /* HDF5 file ID */
     hid_t       acc_tpl;            /* File access templates */
@@ -2638,7 +2638,7 @@ extend_readAll(const void *params)
  */
 #ifdef H5_HAVE_FILTER_DEFLATE
 void
-compress_readAll(const void *params)
+compress_readAll(void *params)
 {
     hid_t       fid;                           /* HDF5 file ID */
     hid_t       acc_tpl;                       /* File access templates */
@@ -2834,7 +2834,7 @@ compress_readAll(const void *params)
  */
 
 void
-none_selection_chunk(const void *params)
+none_selection_chunk(void *params)
 {
     hid_t       fid;                /* HDF5 file ID */
     hid_t       acc_tpl;            /* File access templates */
@@ -3549,7 +3549,7 @@ test_actual_io_mode(const void *params, int selection_mode)
  *
  */
 void
-actual_io_mode_tests(const void *params)
+actual_io_mode_tests(void *params)
 {
     H5D_selection_io_mode_t selection_io_mode;
     hid_t                   dxpl_id = H5I_INVALID_HID;
@@ -3999,7 +3999,7 @@ test_no_collective_cause_mode(const void *params, int selection_mode)
  *
  */
 void
-no_collective_cause_tests(const void *params)
+no_collective_cause_tests(void *params)
 {
     /*
      * Test individual cause
@@ -4036,7 +4036,7 @@ no_collective_cause_tests(const void *params)
  */
 
 void
-dataset_atomicity(const void *params)
+dataset_atomicity(void *params)
 {
     hid_t       fid;              /* HDF5 file ID */
     hid_t       acc_tpl;          /* File access templates */
@@ -4384,7 +4384,7 @@ dataset_atomicity(const void *params)
  *
  */
 void
-test_dense_attr(const void *params)
+test_dense_attr(void *params)
 {
     int         mpi_size, mpi_rank;
     hid_t       fpid, fid;
diff --git a/testpar/t_file.c b/testpar/t_file.c
index 51b244e99d8..b4d9f2c74c7 100644
--- a/testpar/t_file.c
+++ b/testpar/t_file.c
@@ -57,7 +57,7 @@ static int open_file(const char *filename, hid_t fapl, int metadata_write_strate
  * sooner or later due to barrier mixed up.
  */
 void
-test_split_comm_access(const void *params)
+test_split_comm_access(void *params)
 {
     MPI_Comm    comm;
     MPI_Info    info = MPI_INFO_NULL;
@@ -134,7 +134,7 @@ test_split_comm_access(const void *params)
 }
 
 void
-test_page_buffer_access(const void *params)
+test_page_buffer_access(void *params)
 {
     const char *filename;
     hid_t       file_id = H5I_INVALID_HID; /* File ID */
@@ -793,7 +793,7 @@ open_file(const char *filename, hid_t fapl, int metadata_write_strategy, hsize_t
  *        multiple opens of the same file.
  */
 void
-test_file_properties(const void *params)
+test_file_properties(void *params)
 {
     hid_t       fid          = H5I_INVALID_HID; /* HDF5 file ID */
     hid_t       fapl_id      = H5I_INVALID_HID; /* File access plist */
@@ -998,7 +998,7 @@ test_file_properties(const void *params)
 } /* end test_file_properties() */
 
 void
-test_delete(const void *params)
+test_delete(void *params)
 {
     hid_t       fid           = H5I_INVALID_HID; /* HDF5 file ID */
     hid_t       fapl_id       = H5I_INVALID_HID; /* File access plist */
@@ -1078,7 +1078,7 @@ test_delete(const void *params)
  * due to an invalid library version bounds setting
  */
 void
-test_invalid_libver_bounds_file_close_assert(const void *params)
+test_invalid_libver_bounds_file_close_assert(void *params)
 {
     const char *filename = NULL;
     MPI_Comm    comm     = MPI_COMM_WORLD;
@@ -1128,7 +1128,7 @@ test_invalid_libver_bounds_file_close_assert(const void *params)
  * called by multiple ranks.
  */
 void
-test_evict_on_close_parallel_unsupp(const void *params)
+test_evict_on_close_parallel_unsupp(void *params)
 {
     const char *filename = NULL;
     MPI_Comm    comm     = MPI_COMM_WORLD;
@@ -1188,7 +1188,7 @@ test_evict_on_close_parallel_unsupp(const void *params)
  * This is a test program from the user.
  */
 void
-test_fapl_preserve_hints(const void *params)
+test_fapl_preserve_hints(void *params)
 {
     const char *filename;
     const char *key       = "hdf_info_fapl";
diff --git a/testpar/t_file_image.c b/testpar/t_file_image.c
index e8f064cae33..f60ef9c1ea6 100644
--- a/testpar/t_file_image.c
+++ b/testpar/t_file_image.c
@@ -58,7 +58,7 @@
  *                                               JRM -- 11/28/11
  */
 void
-file_image_daisy_chain_test(const void H5_ATTR_UNUSED *params)
+file_image_daisy_chain_test(void H5_ATTR_UNUSED *params)
 {
     char       file_name[1024] = "\0";
     int        mpi_size, mpi_rank;
diff --git a/testpar/t_filter_read.c b/testpar/t_filter_read.c
index fc64680c732..6949ba4eb69 100644
--- a/testpar/t_filter_read.c
+++ b/testpar/t_filter_read.c
@@ -192,7 +192,7 @@ filter_read_internal(const char *filename, hid_t dcpl, hsize_t *dset_size)
  */
 
 void
-test_filter_read(const void *params)
+test_filter_read(void *params)
 {
     hid_t         dc;                                       /* HDF5 IDs */
     const hsize_t chunk_size[2] = {CHUNK_DIM1, CHUNK_DIM2}; /* Chunk dimensions */
diff --git a/testpar/t_mdset.c b/testpar/t_mdset.c
index d966bfb0f44..063c5ae1724 100644
--- a/testpar/t_mdset.c
+++ b/testpar/t_mdset.c
@@ -75,7 +75,7 @@ get_size(void)
  *
  */
 void
-zero_dim_dset(const void *params)
+zero_dim_dset(void *params)
 {
     int         mpi_size, mpi_rank;
     const char *filename;
@@ -144,7 +144,7 @@ zero_dim_dset(const void *params)
  * a slab of array to the file.
  */
 void
-multiple_dset_write(const void *params)
+multiple_dset_write(void *params)
 {
     int                    i, j, n, mpi_size, mpi_rank, size;
     hid_t                  iof, plist, dataset, memspace, filespace;
@@ -238,7 +238,7 @@ multiple_dset_write(const void *params)
 /* Example of using PHDF5 to create, write, and read compact dataset.
  */
 void
-compact_dataset(const void *params)
+compact_dataset(void *params)
 {
     int         i, j, mpi_size, mpi_rank, size, err_num = 0;
     hid_t       iof, plist, dcpl, dxpl, dataset, filespace;
@@ -378,7 +378,7 @@ compact_dataset(const void *params)
  * of Null dataspace.
  */
 void
-null_dataset(const void *params)
+null_dataset(void *params)
 {
     int         mpi_size, mpi_rank;
     hid_t       iof, plist, dxpl, dataset, attr, sid;
@@ -495,7 +495,7 @@ null_dataset(const void *params)
  * the boundary of interest.
  */
 void
-big_dataset(const void *params)
+big_dataset(void *params)
 {
     int   mpi_size, mpi_rank;        /* MPI info */
     hid_t iof,                       /* File ID */
@@ -637,7 +637,7 @@ big_dataset(const void *params)
  * default fill value of zeros to work correctly.
  */
 void
-dataset_fillvalue(const void *params)
+dataset_fillvalue(void *params)
 {
     int   mpi_size, mpi_rank;             /* MPI info */
     int   err_num;                        /* Number of errors */
@@ -891,7 +891,7 @@ dataset_fillvalue(const void *params)
 
 /* combined cngrpw and ingrpr tests because ingrpr reads file created by cngrpw. */
 void
-collective_group_write_independent_group_read(const void *params)
+collective_group_write_independent_group_read(void *params)
 {
     collective_group_write(params);
     independent_group_read(params);
@@ -901,7 +901,7 @@ collective_group_write_independent_group_read(const void *params)
  * These groups and datasets are for testing independent read later.
  */
 void
-collective_group_write(const void *params)
+collective_group_write(void *params)
 {
     int                    mpi_rank, mpi_size, size;
     int                    i, j, m;
@@ -1014,7 +1014,7 @@ collective_group_write(const void *params)
  * datasets independently.
  */
 void
-independent_group_read(const void *params)
+independent_group_read(void *params)
 {
     int                    mpi_rank, m;
     hid_t                  plist, fid;
@@ -1142,7 +1142,7 @@ group_dataset_read(hid_t fid, int mpi_rank, int m)
  *
  */
 void
-multiple_group_write(const void *params)
+multiple_group_write(void *params)
 {
     int                    mpi_rank, mpi_size, size;
     int                    m;
@@ -1311,7 +1311,7 @@ create_group_recursive(hid_t memspace, hid_t filespace, hid_t gid, int counter)
  * every dataset in every group and check their correctness.
  */
 void
-multiple_group_read(const void *params)
+multiple_group_read(void *params)
 {
     int                    mpi_rank, mpi_size, error_num, size;
     int                    m;
@@ -1620,7 +1620,7 @@ get_slab(hsize_t chunk_origin[], hsize_t chunk_dims[], hsize_t count[], hsize_t
 #define N 4
 
 void
-io_mode_confusion(const void *params)
+io_mode_confusion(void *params)
 {
     /*
      * HDF5 APIs definitions
@@ -1900,7 +1900,7 @@ const char *lg_att_name[NUM_DATA_SETS]  = {"large_attribute_0", "large_attribute
                                            "large_attribute_3"};
 
 void
-rr_obj_hdr_flush_confusion(const void *params)
+rr_obj_hdr_flush_confusion(void *params)
 {
     /* MPI variables */
     /* private communicator size and rank */
@@ -2705,7 +2705,7 @@ rr_obj_hdr_flush_confusion_reader(const void *params, MPI_Comm comm)
 #define EXTRA_ALIGN 100
 
 void
-chunk_align_bug_1(const void *params)
+chunk_align_bug_1(void *params)
 {
     int            mpi_rank;
     hid_t          file_id, dset_id, fapl_id, dcpl_id, space_id;
diff --git a/testpar/t_oflush.c b/testpar/t_oflush.c
index 4bca1fe1d43..7f00f2a5c78 100644
--- a/testpar/t_oflush.c
+++ b/testpar/t_oflush.c
@@ -25,7 +25,7 @@
 #define RANK        2
 
 void
-test_oflush(const void *params)
+test_oflush(void *params)
 {
     int         mpi_size, mpi_rank;
     hid_t       file, dataset;
diff --git a/testpar/t_ph5basic.c b/testpar/t_ph5basic.c
index 8cecffb4f72..47a23196859 100644
--- a/testpar/t_ph5basic.c
+++ b/testpar/t_ph5basic.c
@@ -28,7 +28,7 @@
  *-------------------------------------------------------------------------
  */
 void
-test_fapl_mpio_dup(const void H5_ATTR_UNUSED *params)
+test_fapl_mpio_dup(void H5_ATTR_UNUSED *params)
 {
     int      mpi_size, mpi_rank;
     MPI_Comm comm, comm_tmp;
@@ -190,7 +190,7 @@ test_fapl_mpio_dup(const void H5_ATTR_UNUSED *params)
  *-------------------------------------------------------------------------
  */
 void
-test_get_dxpl_mpio(const void *params)
+test_get_dxpl_mpio(void *params)
 {
     hid_t            fid  = H5I_INVALID_HID;
     hid_t            sid  = H5I_INVALID_HID;
diff --git a/testpar/t_prop.c b/testpar/t_prop.c
index 257c4330629..ee9b007b39e 100644
--- a/testpar/t_prop.c
+++ b/testpar/t_prop.c
@@ -88,7 +88,7 @@ test_encode_decode(hid_t orig_pl, int mpi_rank, int recv_proc)
 }
 
 void
-test_plist_ed(const void H5_ATTR_UNUSED *params)
+test_plist_ed(void H5_ATTR_UNUSED *params)
 {
     hid_t dcpl;   /* dataset create prop. list */
     hid_t dapl;   /* dataset access prop. list */
@@ -451,7 +451,7 @@ test_plist_ed(const void H5_ATTR_UNUSED *params)
 }
 
 void
-external_links(const void H5_ATTR_UNUSED *params)
+external_links(void H5_ATTR_UNUSED *params)
 {
     hid_t lcpl  = H5I_INVALID_HID; /* link create prop. list */
     hid_t lapl  = H5I_INVALID_HID; /* link access prop. list */
diff --git a/testpar/t_shapesame.c b/testpar/t_shapesame.c
index 56b460d7895..193f6e76bd9 100644
--- a/testpar/t_shapesame.c
+++ b/testpar/t_shapesame.c
@@ -4179,56 +4179,56 @@ parse_options(int argc, char **argv)
 
 /* Shape Same test using contiguous hyperslab using independent IO on contiguous datasets */
 static void
-sscontig1(const void *params)
+sscontig1(void *params)
 {
     contig_hs_dr_pio_test(params, IND_CONTIG);
 }
 
 /* Shape Same test using contiguous hyperslab using collective IO on contiguous datasets */
 static void
-sscontig2(const void *params)
+sscontig2(void *params)
 {
     contig_hs_dr_pio_test(params, COL_CONTIG);
 }
 
 /* Shape Same test using contiguous hyperslab using independent IO on chunked datasets */
 static void
-sscontig3(const void *params)
+sscontig3(void *params)
 {
     contig_hs_dr_pio_test(params, IND_CHUNKED);
 }
 
 /* Shape Same test using contiguous hyperslab using collective IO on chunked datasets */
 static void
-sscontig4(const void *params)
+sscontig4(void *params)
 {
     contig_hs_dr_pio_test(params, COL_CHUNKED);
 }
 
 /* Shape Same test using checker hyperslab using independent IO on contiguous datasets */
 static void
-sschecker1(const void *params)
+sschecker1(void *params)
 {
     ckrbrd_hs_dr_pio_test(params, IND_CONTIG);
 }
 
 /* Shape Same test using checker hyperslab using collective IO on contiguous datasets */
 static void
-sschecker2(const void *params)
+sschecker2(void *params)
 {
     ckrbrd_hs_dr_pio_test(params, COL_CONTIG);
 }
 
 /* Shape Same test using checker hyperslab using independent IO on chunked datasets */
 static void
-sschecker3(const void *params)
+sschecker3(void *params)
 {
     ckrbrd_hs_dr_pio_test(params, IND_CHUNKED);
 }
 
 /* Shape Same test using checker hyperslab using collective IO on chunked datasets */
 static void
-sschecker4(const void *params)
+sschecker4(void *params)
 {
     ckrbrd_hs_dr_pio_test(params, COL_CHUNKED);
 }
diff --git a/testpar/t_span_tree.c b/testpar/t_span_tree.c
index 7619404e866..9aac650d729 100644
--- a/testpar/t_span_tree.c
+++ b/testpar/t_span_tree.c
@@ -52,7 +52,7 @@ static void coll_read_test(const void *params);
  *-------------------------------------------------------------------------
  */
 void
-coll_irregular_cont_write(const void *params)
+coll_irregular_cont_write(void *params)
 {
     int mpi_rank;
 
@@ -87,7 +87,7 @@ coll_irregular_cont_write(const void *params)
  *-------------------------------------------------------------------------
  */
 void
-coll_irregular_cont_read(const void *params)
+coll_irregular_cont_read(void *params)
 {
     int mpi_rank;
 
@@ -122,7 +122,7 @@ coll_irregular_cont_read(const void *params)
  *-------------------------------------------------------------------------
  */
 void
-coll_irregular_simple_chunk_write(const void *params)
+coll_irregular_simple_chunk_write(void *params)
 {
     int mpi_rank;
 
@@ -157,7 +157,7 @@ coll_irregular_simple_chunk_write(const void *params)
  *-------------------------------------------------------------------------
  */
 void
-coll_irregular_simple_chunk_read(const void *params)
+coll_irregular_simple_chunk_read(void *params)
 {
     int mpi_rank;
 
@@ -192,7 +192,7 @@ coll_irregular_simple_chunk_read(const void *params)
  *-------------------------------------------------------------------------
  */
 void
-coll_irregular_complex_chunk_write(const void *params)
+coll_irregular_complex_chunk_write(void *params)
 {
     int mpi_rank;
 
@@ -227,7 +227,7 @@ coll_irregular_complex_chunk_write(const void *params)
  *-------------------------------------------------------------------------
  */
 void
-coll_irregular_complex_chunk_read(const void *params)
+coll_irregular_complex_chunk_read(void *params)
 {
     int mpi_rank;
 
@@ -2349,7 +2349,7 @@ lower_dim_size_comp_test__run_test(const void *params, const int chunk_edge_size
  */
 
 void
-lower_dim_size_comp_test(const void *params)
+lower_dim_size_comp_test(void *params)
 {
     /* const char *fcnName = "lower_dim_size_comp_test()"; */
     int chunk_edge_size = 0;
@@ -2411,7 +2411,7 @@ lower_dim_size_comp_test(const void *params)
 #define LINK_CHUNK_COLLECTIVE_IO_TEST_CHUNK_SIZE 16
 
 void
-link_chunk_collective_io_test(const void *params)
+link_chunk_collective_io_test(void *params)
 {
     /* const char *fcnName = "link_chunk_collective_io_test()"; */
     const char *filename;
diff --git a/testpar/testphdf5.h b/testpar/testphdf5.h
index 2de49c39501..2d256b375ab 100644
--- a/testpar/testphdf5.h
+++ b/testpar/testphdf5.h
@@ -195,78 +195,76 @@ extern int facc_type;            /*Test file access type */
 extern int dxfer_coll_type;
 
 /* Test program prototypes */
-void test_plist_ed(const void *params);
-void external_links(const void *params);
-void zero_dim_dset(const void *params);
-void test_file_properties(const void *params);
-void test_delete(const void *params);
-void test_invalid_libver_bounds_file_close_assert(const void *params);
-void test_evict_on_close_parallel_unsupp(const void *params);
-void test_fapl_preserve_hints(const void *params);
-void multiple_dset_write(const void *params);
-void multiple_group_write(const void *params);
-void multiple_group_read(const void *params);
-void collective_group_write_independent_group_read(const void *params);
-void collective_group_write(const void *params);
-void independent_group_read(const void *params);
-void test_fapl_mpio_dup(const void *params);
-void test_get_dxpl_mpio(const void *params);
-void test_split_comm_access(const void *params);
-void test_page_buffer_access(const void *params);
-void dataset_atomicity(const void *params);
-void dataset_writeInd(const void *params);
-void dataset_writeAll(const void *params);
-void extend_writeInd(const void *params);
-void extend_writeInd2(const void *params);
-void extend_writeAll(const void *params);
-void dataset_readInd(const void *params);
-void dataset_readAll(const void *params);
-void extend_readInd(const void *params);
-void extend_readAll(const void *params);
-void none_selection_chunk(const void *params);
-void actual_io_mode_tests(const void *params);
-void no_collective_cause_tests(const void *params);
-void test_chunk_alloc(const void *params);
-void test_chunk_alloc_incr_ser_to_par(const void *params);
-void test_filter_read(const void *params);
-void compact_dataset(const void *params);
-void null_dataset(const void *params);
-void big_dataset(const void *params);
-void dataset_fillvalue(const void *params);
-void coll_chunk1(const void *params);
-void coll_chunk2(const void *params);
-void coll_chunk3(const void *params);
-void coll_chunk4(const void *params);
-void coll_chunk5(const void *params);
-void coll_chunk6(const void *params);
-void coll_chunk7(const void *params);
-void coll_chunk8(const void *params);
-void coll_chunk9(const void *params);
-void coll_chunk10(const void *params);
-void coll_irregular_cont_read(const void *params);
-void coll_irregular_cont_write(const void *params);
-void coll_irregular_simple_chunk_read(const void *params);
-void coll_irregular_simple_chunk_write(const void *params);
-void coll_irregular_complex_chunk_read(const void *params);
-void coll_irregular_complex_chunk_write(const void *params);
-void io_mode_confusion(const void *params);
-void rr_obj_hdr_flush_confusion(const void *params);
-void chunk_align_bug_1(const void *params);
-void lower_dim_size_comp_test(const void *params);
-void link_chunk_collective_io_test(const void *params);
-void contig_hyperslab_dr_pio_test(ShapeSameTestMethods sstest_type);
-void checker_board_hyperslab_dr_pio_test(ShapeSameTestMethods sstest_type);
-void file_image_daisy_chain_test(const void *params);
+void test_plist_ed(void *params);
+void external_links(void *params);
+void zero_dim_dset(void *params);
+void test_file_properties(void *params);
+void test_delete(void *params);
+void test_invalid_libver_bounds_file_close_assert(void *params);
+void test_evict_on_close_parallel_unsupp(void *params);
+void test_fapl_preserve_hints(void *params);
+void multiple_dset_write(void *params);
+void multiple_group_write(void *params);
+void multiple_group_read(void *params);
+void collective_group_write_independent_group_read(void *params);
+void collective_group_write(void *params);
+void independent_group_read(void *params);
+void test_fapl_mpio_dup(void *params);
+void test_get_dxpl_mpio(void *params);
+void test_split_comm_access(void *params);
+void test_page_buffer_access(void *params);
+void dataset_atomicity(void *params);
+void dataset_writeInd(void *params);
+void dataset_writeAll(void *params);
+void extend_writeInd(void *params);
+void extend_writeInd2(void *params);
+void extend_writeAll(void *params);
+void dataset_readInd(void *params);
+void dataset_readAll(void *params);
+void extend_readInd(void *params);
+void extend_readAll(void *params);
+void none_selection_chunk(void *params);
+void actual_io_mode_tests(void *params);
+void no_collective_cause_tests(void *params);
+void test_chunk_alloc(void *params);
+void test_chunk_alloc_incr_ser_to_par(void *params);
+void test_filter_read(void *params);
+void compact_dataset(void *params);
+void null_dataset(void *params);
+void big_dataset(void *params);
+void dataset_fillvalue(void *params);
+void coll_chunk1(void *params);
+void coll_chunk2(void *params);
+void coll_chunk3(void *params);
+void coll_chunk4(void *params);
+void coll_chunk5(void *params);
+void coll_chunk6(void *params);
+void coll_chunk7(void *params);
+void coll_chunk8(void *params);
+void coll_chunk9(void *params);
+void coll_chunk10(void *params);
+void coll_irregular_cont_read(void *params);
+void coll_irregular_cont_write(void *params);
+void coll_irregular_simple_chunk_read(void *params);
+void coll_irregular_simple_chunk_write(void *params);
+void coll_irregular_complex_chunk_read(void *params);
+void coll_irregular_complex_chunk_write(void *params);
+void io_mode_confusion(void *params);
+void rr_obj_hdr_flush_confusion(void *params);
+void chunk_align_bug_1(void *params);
+void lower_dim_size_comp_test(void *params);
+void link_chunk_collective_io_test(void *params);
+void file_image_daisy_chain_test(void *params);
 #ifdef H5_HAVE_FILTER_DEFLATE
-void compress_readAll(const void *params);
+void compress_readAll(void *params);
 #endif /* H5_HAVE_FILTER_DEFLATE */
-void test_dense_attr(const void *params);
-void test_partial_no_selection_coll_md_read(const void *params);
-void test_multi_chunk_io_addrmap_issue(const void *params);
-void test_link_chunk_io_sort_chunk_issue(const void *params);
-void test_collective_global_heap_write(const void *params);
-void test_coll_io_ind_md_write(const void *params);
-void test_oflush(const void *params);
+void test_dense_attr(void *params);
+void test_partial_no_selection_coll_md_read(void *params);
+void test_multi_chunk_io_addrmap_issue(void *params);
+void test_link_chunk_io_sort_chunk_issue(void *params);
+void test_collective_global_heap_write(void *params);
+void test_coll_io_ind_md_write(void *params);
+void test_oflush(void *params);
 
 /* commonly used prototypes */
 MPI_Offset h5_mpi_get_file_size(const char *filename, MPI_Comm comm, MPI_Info info);
diff --git a/tools/lib/h5tools_error.h b/tools/lib/h5tools_error.h
index 2496d585419..c7361345af6 100644
--- a/tools/lib/h5tools_error.h
+++ b/tools/lib/h5tools_error.h
@@ -16,7 +16,7 @@
 #ifndef H5TOOLS_ERROR_H
 #define H5TOOLS_ERROR_H
 
-#include "H5Epublic.h"
+#include "H5private.h"
 #include "H5Eprivate.h" /* Error handling       */
 
 /* tools-HDF5 Error variables */
diff --git a/tools/src/h5repack/h5repack_refs.c b/tools/src/h5repack/h5repack_refs.c
index 0c281b006e8..95f396663af 100644
--- a/tools/src/h5repack/h5repack_refs.c
+++ b/tools/src/h5repack/h5repack_refs.c
@@ -631,7 +631,7 @@ copy_refs_attr(hid_t loc_in, hid_t loc_out, trav_table_t *travt, hid_t fidout) /
                  * create output
                  *-------------------------------------------------------------------------
                  */
-                refbuf = (hdset_reg_ref_t *)calloc(sizeof(hdset_reg_ref_t), (size_t)nelmts); /*init to zero */
+                refbuf = (hdset_reg_ref_t *)calloc((size_t)nelmts, sizeof(hdset_reg_ref_t)); /*init to zero */
                 if (refbuf == NULL) {
                     printf("cannot allocate memory\n");
                     H5TOOLS_GOTO_ERROR((-1), "calloc failed");
diff --git a/tools/test/h5diff/CMakeTests.cmake b/tools/test/h5diff/CMakeTests.cmake
index 5d36e0cd764..112d5066de1 100644
--- a/tools/test/h5diff/CMakeTests.cmake
+++ b/tools/test/h5diff/CMakeTests.cmake
@@ -808,7 +808,7 @@ ADD_H5_TEST (h5diff_63 1 -v ${STRINGS1} ${STRINGS2} string4 string4)
 ADD_H5_TEST (h5diff_600 1 ${FILE1})
 
 # 6.1: Check if non-exist object name is specified
-ADD_H5_CMP_TEST (h5diff_601 2 "Object could not be found" ${FILE1} ${FILE1} nono_obj)
+ADD_H5_CMP_TEST (h5diff_601 2 "could not be found" ${FILE1} ${FILE1} nono_obj)
 
 # ##############################################################################
 # # -d
diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c
index 2a06a2323a3..c81d114164f 100644
--- a/tools/test/h5dump/h5dumpgentest.c
+++ b/tools/test/h5dump/h5dumpgentest.c
@@ -2406,10 +2406,10 @@ gent_attrreg(void)
     int              i;                                   /* counting variables */
 
     /* Allocate write & read buffers */
-    wbuf  = (hdset_reg_ref_t *)calloc(sizeof(hdset_reg_ref_t), SPACE1_DIM1);
-    rbuf  = (hdset_reg_ref_t *)malloc(sizeof(hdset_reg_ref_t) * SPACE1_DIM1);
-    dwbuf = (uint8_t *)malloc(sizeof(uint8_t) * SPACE2_DIM1 * SPACE2_DIM2);
-    drbuf = (uint8_t *)calloc(sizeof(uint8_t), SPACE2_DIM1 * SPACE2_DIM2);
+    wbuf  = (hdset_reg_ref_t *)calloc(SPACE1_DIM1, sizeof(hdset_reg_ref_t));
+    rbuf  = (hdset_reg_ref_t *)calloc(SPACE1_DIM1, sizeof(hdset_reg_ref_t));
+    dwbuf = (uint8_t *)calloc(SPACE2_DIM1 * SPACE2_DIM2, sizeof(uint8_t));
+    drbuf = (uint8_t *)calloc(SPACE2_DIM1 * SPACE2_DIM2, sizeof(uint8_t));
 
     /* Create file */
     fid1 = H5Fcreate(FILE64, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
@@ -3121,7 +3121,7 @@ gent_array1_big(void)
     block[0]  = 1;
 
     /* Allocate write & read buffers */
-    wbuf  = (hdset_reg_ref_t *)calloc(sizeof(hdset_reg_ref_t), SPACE1_DIM1);
+    wbuf  = (hdset_reg_ref_t *)calloc(SPACE1_DIM1, sizeof(hdset_reg_ref_t));
     wdata = (int *)malloc(sizeof(int) * (size_t)(SPACE_ARRAY1BIG_DIM * ARRAY1BIG_DIM));
 
     /* Allocate and initialize array data to write */
diff --git a/tools/test/perform/CMakeLists.txt b/tools/test/perform/CMakeLists.txt
index 646bd1a6de2..46170f9237d 100644
--- a/tools/test/perform/CMakeLists.txt
+++ b/tools/test/perform/CMakeLists.txt
@@ -113,7 +113,7 @@ set (zip_perf_SOURCES
     ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/zip_perf.c
 )
 if (H5_ZLIB_HEADER)
-  message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}")
+  message (VERBOSE "H5_ZLIB_HEADER for zip_perf=${H5_ZLIB_HEADER}")
   if (HDF5_USE_ZLIB_NG)
     add_compile_definitions(H5_HAVE_ZLIBNG_H=1 H5_ZLIB_HEADER="${H5_ZLIB_HEADER}")
   endif ()