Use Preferences for using custom hdf5 library (#1061)

* use system library using Preferences.jl * allow specific library filenames * fix typo * Apply suggestions from code review Co-authored-by: Michael Schlottke-Lakemper <[email protected]> * import dlsym * remove deps folder from Format check * set julia compat to 1.6 * extend docs * format * bump version * bump HDF5 compat versions of filters to 0.17 * update HISTORY.md * add note on migration instructions * add example of LocalPreferences.toml in docs * CI test for system libraries with Preferences * format * fix pwd * add Project.toml * rm Project.toml * add julia-actions to system library CI * debug CI * add Project.toml * remove additional julia-actions * debug CI * debug CI * explicitly state possible library names in docs * set prefs to each library * update docs (HDF5_jll comes with MPI) * workaround * fix workaround * disable virtual dataset tests for libhdf5 version < 1.12 * fix typo in docs * raise warning if JULIA_HDF5_PATH is detected * format * Apply suggestions from code review Co-authored-by: Simon Byrne <[email protected]> Co-authored-by: Mark Kittisopikul <[email protected]> * set both preferences in one call * set both preferences in one call * Apply suggestions from code review Co-authored-by: Mark Kittisopikul <[email protected]> --------- Co-authored-by: Michael Schlottke-Lakemper <[email protected]> Co-authored-by: Mark Kittisopikul <[email protected]> Co-authored-by: Simon Byrne <[email protected]>
JuliaIO · Jun 2, 2023 · 4d446fc · 4d446fc
1 parent cfba706
commit 4d446fc
Show file tree

Hide file tree

Showing 18 changed files with 179 additions and 127 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -16,7 +16,8 @@ jobs:
     strategy:
       matrix:
         version:
-          - '1.3'
+          - '1.6'
+          - '1'
         os:
           - ubuntu-20.04 # required for libhdf5 v1.10.4 support
         arch:
@@ -27,15 +28,20 @@ jobs:
           sudo apt-get update
           sudo apt-get install mpich libhdf5-mpich-dev
           echo "JULIA_HDF5_PATH=/usr/lib/x86_64-linux-gnu/hdf5/mpich/" >> $GITHUB_ENV
-          echo "JULIA_MPI_BINARY=system" >> $GITHUB_ENV
       - uses: actions/checkout@v3
       - uses: julia-actions/setup-julia@latest
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
       - uses: julia-actions/julia-buildpkg@latest
+      - name: Configure MPI.jl
+        shell: julia --color=yes {0}
+        run: |
+          @show pwd()
+          include(joinpath(pwd(), "test", "configure_packages.jl"))
       - uses: julia-actions/julia-runtest@latest
 
+
   HDF5:
     name: julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
@@ -44,7 +50,7 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.3'
+          - '1.6'
           - '1'
           - 'nightly'
         os:
@@ -59,13 +65,13 @@ jobs:
             arch: x86
           - os: ubuntu-latest # excluded because HDF5_jll v1.12 does not support i686
             arch: x86
-          - version: '1.3'
+          - version: '1.6'
             arch: x86
           - version: 'nightly'
             arch: x86
-          - version: '1.3'
+          - version: '1.6'
             os: macOS-latest
-          - version: '1.3'
+          - version: '1.6'
             os: windows-latest
     steps:
       - uses: actions/checkout@v3

diff --git a/.github/workflows/Format-check.yml b/.github/workflows/Format-check.yml
@@ -25,7 +25,7 @@ jobs:
       - name: Install JuliaFormatter and format
         run: |
           julia -e 'using Pkg; Pkg.add(PackageSpec(name="JuliaFormatter"))'
-          julia -e 'using JuliaFormatter; format(["src", "test", "deps", "filters", "gen"], verbose=true)'
+          julia -e 'using JuliaFormatter; format(["src", "test", "filters", "gen"], verbose=true)'
       - name: Format check
         run: |
           julia -e '

diff --git a/.gitignore b/.gitignore
@@ -4,3 +4,4 @@
 *.jl.mem
 /docs/build/
 Manifest.toml
+LocalPreferences.toml
diff --git a/HISTORY.md b/HISTORY.md
@@ -2,6 +2,9 @@
 
 Please also see the [release notes](https://github.com/JuliaIO/HDF5.jl/releases) for additional details.
 
+## v0.17.0
+* Replace build step by using Preferences.jl to use system HDF5 library
+
 ## v0.16.14
 * Allow `begin` to work in a dataset
 * Simplify MPIO tests and internals

diff --git a/Project.toml b/Project.toml
@@ -1,22 +1,26 @@
 name = "HDF5"
 uuid = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
-version = "0.16.15"
+version = "0.17.0"
 
 [deps]
 Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
 HDF5_jll = "0234f1f7-429e-5d53-9886-15a909be8d59"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
+Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267"
 
 [compat]
 Compat = "3.1.0, 4"
 HDF5_jll = "~1.10.5, ~1.12.0, ~1.14.0"
+MPIPreferences = "0.1.7"
+Preferences = "1.3"
 Requires = "1.0"
-julia = "1.3"
+julia = "1.6"
 
 [extras]
 CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc"

diff --git a/deps/.gitignore b/deps/.gitignore
diff --git a/deps/build.jl b/deps/build.jl
diff --git a/docs/src/index.md b/docs/src/index.md
@@ -27,24 +27,58 @@ Starting from Julia 1.3, the HDF5 binaries are by default downloaded using the `
 
 ### Using custom or system provided HDF5 binaries
 
-To use system-provided HDF5 binaries instead, set the environment variable `JULIA_HDF5_PATH` to the top-level installation directory HDF5, i.e. the library should be located in `${JULIA_HDF5_PATH}/lib` or `${JULIA_HDF5_PATH}/lib64`, or alternatively simply in `${JULIA_HDF5_PATH}`. Then run `import Pkg; Pkg.build("HDF5")`. In particular, this is required if you need parallel HDF5 support, which is not provided by the `HDF5_jll` binaries.
+!!! note "Migration from HDF5.jl v0.16 and earlier"
+    How to use a system-provided HDF5 library has been changed in HDF5.jl v0.17. Previously,
+    the library path was set by the environment variable `JULIA_HDF5_PATH`, which required to
+    rebuild HDF5.jl afterwards. The environment variable has been removed and no longer has an
+    effect (for backward compatibility it is still recommended to **also** set the environment
+    variable). Instead, proceed as described below.
 
-If the library is in your library search path, then `JULIA_HDF5_PATH` can be set to an empty string.
+To use system-provided HDF5 binaries instead, set the preferences `libhdf5` and `libhdf5_hl`, see also [Preferences.jl](https://github.com/JuliaPackaging/Preferences.jl). These need to point to the local paths of the libraries `libhdf5` and `libhdf5_hl`.
 
-For example, to use HDF5 (`libhdf5-mpich-dev`) with MPI using system libraries on Ubuntu 20.04, you would run:
+For example, to use HDF5 (`libhdf5-mpich-dev`) with MPI using system libraries on Ubuntu 20.04, you would run
 
 ```sh
 $ sudo apt install mpich libhdf5-mpich-dev
-$ JULIA_HDF5_PATH=/usr/lib/x86_64-linux-gnu/hdf5/mpich/
-$ JULIA_MPI_BINARY=system
 ```
 
-Then in Julia, run:
+If your system HDF5 library is compiled with MPI, you need to tell MPI.jl to use the same locally installed MPI implementation. This can be done in Julia by running:
 
 ```julia
-pkg> build
+using MPIPreferences
+MPIPreferences.use_system_binary()
 ```
 
+to set the MPI preferences, see the [documentation of MPI.jl](https://juliaparallel.org/MPI.jl/stable/configuration/). You can set the path to the system library using [Preferences.jl](https://github.com/JuliaPackaging/Preferences.jl) by:
+
+```julia
+using Preferences, UUIDs
+
+set_preferences!(
+    UUID("f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"), # UUID of HDF5.jl
+    "libhdf5" => "/usr/lib/x86_64-linux-gnu/hdf5/mpich/libhdf5.so",
+    "libhdf5_hl" => "/usr/lib/x86_64-linux-gnu/hdf5/mpich/libhdf5_hl.so", force = true)
+```
+
+Also see the file `test/configure_packages.jl` for an example.
+
+Both, the MPI preferences and the preferences for HDF5.jl write to a file called LocalPreferences.toml in the project directory. After performing the described steps this file could look like the following:
+
+```toml
+[MPIPreferences]
+_format = "1.0"
+abi = "MPICH"
+binary = "system"
+libmpi = "/software/mpi/lib/libmpi.so"
+mpiexec = "/software/mpi/bin/mpiexec"
+
+[HDF5]
+libhdf5 = "/usr/lib/x86_64-linux-gnu/hdf5/mpich/libhdf5.so"
+libhdf5_hl = "/usr/lib/x86_64-linux-gnu/hdf5/mpich/libhdf5_hl.so"
+```
+
+If you want to switch to another HDF5 library or the library moved, you can call the `set_preferences!` commands again (or manually edit LocalPreferences.toml) to set the new paths. Using the default implementation provided by HDF5_jll can be done by simply manually deleting the LocalPreferences.toml file.
+
 ## Opening and closing files
 
 "Plain" (i.e., with no extra formatting conventions) HDF5 files are created and/or opened with the `h5open` command:

diff --git a/docs/src/mpi.md b/docs/src/mpi.md
@@ -20,7 +20,7 @@ In practice, this means that in Julia code, `MPI` must be imported _before_
 The following step-by-step guide assumes one already has access to
 parallel-enabled HDF5 libraries linked to an existent MPI installation.
 
-### 1. Using system-provided MPI libraries
+### [1. Using system-provided MPI libraries](@id using_system_MPI)
 
 Using a system-provided MPI library can be done with MPIPreferences.jl.
 After installing MPIPreferences.jl and running
@@ -31,12 +31,28 @@ See the [MPI.jl
 docs](https://juliaparallel.org/MPI.jl/stable/configuration/#Using-a-system-provided-MPI-backend)
 for details.
 
-### 2. Using parallel HDF5 libraries
+### [2. Using parallel HDF5 libraries](@id using_parallel_HDF5)
+
+!!! note "Migration from HDF5.jl v0.16 and earlier"
+    How to use a system-provided HDF5 library has been changed in HDF5.jl v0.17. Previously,
+    the library path was set by the environment variable `JULIA_HDF5_PATH`, which required to
+    rebuild HDF5.jl afterwards. The environment variable has been removed and no longer has an
+    effect (for backward compatibility it is still recommended to **also** set the environment
+    variable). Instead, proceed as described below.
 
 As detailed in [Using custom or system provided HDF5 binaries](@ref), set the
-`JULIA_HDF5_PATH` environment variable to the path where the parallel HDF5
-binaries are located.
-Then run `]build HDF5` from Julia.
+preferences `libhdf5` and `libhdf5_hl` to the full path, where the parallel HDF5 binaries are located.
+This can be done by:
+
+```julia
+julia> using Preferences, UUIDs
+
+julia> set_preferences!(
+           UUID("f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"), # UUID of HDF5.jl
+           "libhdf5" => "/path/to/your/libhdf5.so",
+           "libhdf5_hl" => "/path/to/your/libhdf5_hl.so",
+           force = true)
+```
 
 ### 3. Loading MPI-enabled HDF5
 
@@ -50,6 +66,25 @@ using HDF5
 @assert HDF5.has_parallel()
 ```
 
+### Notes to HPC cluster administrators
+
+More information for a setup at an HPC cluster can be found in the [docs of MPI.jl](https://juliaparallel.org/MPI.jl/stable/configuration/#Notes-to-HPC-cluster-administrators).
+After performing the steps [1.](@ref using_system_MPI) and [2.](@ref using_parallel_HDF5) the
+LocalPreferences.toml file could look something like the following:
+
+```toml
+[MPIPreferences]
+_format = "1.0"
+abi = "OpenMPI"
+binary = "system"
+libmpi = "/software/mpi/lib/libmpi.so"
+mpiexec = "/software/mpi/bin/mpiexec"
+
+[HDF5]
+libhdf5 = "/path/to/your/libhdf5.so"
+libhdf5_hl = "/path/to/your/libhdf5_hl.so"
+```
+
 ### Reading and writing data in parallel
 
 A parallel HDF5 file may be opened by passing a `MPI.Comm` (and optionally a

diff --git a/filters/H5Zbitshuffle/Project.toml b/filters/H5Zbitshuffle/Project.toml
@@ -8,6 +8,6 @@ HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 bitshuffle_jll = "228fe19c-1b83-5282-a626-13744502a320"
 
 [compat]
-HDF5 = "0.16"
+HDF5 = "0.17"
 bitshuffle_jll = "0.4.2"
 julia = "1.6"
diff --git a/filters/H5Zblosc/Project.toml b/filters/H5Zblosc/Project.toml
@@ -7,6 +7,6 @@ Blosc = "a74b3585-a348-5f62-a45c-50e91977d574"
 HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 
 [compat]
-HDF5 = "0.16"
+HDF5 = "0.17"
 Blosc = "0.7.3"
 julia = "1.3"
diff --git a/filters/H5Zbzip2/Project.toml b/filters/H5Zbzip2/Project.toml
@@ -7,6 +7,6 @@ CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd"
 HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 
 [compat]
-HDF5 = "0.16"
+HDF5 = "0.17"
 CodecBzip2 = "0.7"
 julia = "1.3"
diff --git a/filters/H5Zlz4/Project.toml b/filters/H5Zlz4/Project.toml
@@ -7,6 +7,6 @@ CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561"
 HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 
 [compat]
-HDF5 = "0.16"
+HDF5 = "0.17"
 CodecLz4 = "0.4"
 julia = "1.3"
diff --git a/filters/H5Zzstd/Project.toml b/filters/H5Zzstd/Project.toml
@@ -7,6 +7,6 @@ CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2"
 HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 
 [compat]
-HDF5 = "0.16"
+HDF5 = "0.17"
 CodecZstd = "0.7"
 julia = "1.3"