From c6050e63dc126c7e04d87090d8cbde900131df1b Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <jpblaschke@lbl.gov>
Date: Sat, 25 Feb 2023 21:24:45 -0800
Subject: [PATCH 01/25] add libgtl

---
 lib/MPIPreferences/src/MPIPreferences.jl | 24 +++++++++++++++++++++---
 lib/MPIPreferences/src/system.jl         | 14 ++++++++++++++
 src/MPI.jl                               |  5 +++++
 src/api/api.jl                           | 13 +++++++++++--
 4 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl
index 364ed14e6..a025bb9be 100644
--- a/lib/MPIPreferences/src/MPIPreferences.jl
+++ b/lib/MPIPreferences/src/MPIPreferences.jl
@@ -81,7 +81,8 @@ function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_j
         "binary" => binary,
         "libmpi" => nothing,
         "abi" => nothing,
-        "mpiexec" => nothing;
+        "mpiexec" => nothing,
+        "libgtl" => nothing;
         export_prefs=export_prefs,
         force=force
     )
@@ -114,6 +115,7 @@ end
         mpiexec = "mpiexec",
         abi = nothing,
         export_prefs = false,
+        gtl_names=nothing,
         force = true)
 
 Switches the underlying MPI implementation to a system provided one. A restart
@@ -136,6 +138,9 @@ Options:
   using [`identify_abi`](@ref). See [`abi`](@ref) for currently supported
   values.
 
+- `gtl_names`: can be either `nothing` or a list of GPU Transport Layer (GTL)
+  names. On Cray systems, this is usually `["libmpi_gtl_cuda", "libmpi_gtl_hsa"]`.
+
 - `export_prefs`: if `true`, the preferences into the `Project.toml` instead of
   `LocalPreferences.toml`.
 
@@ -145,8 +150,9 @@ function use_system_binary(;
         library_names=["libmpi", "libmpi_ibm", "msmpi", "libmpich", "libmpi_cray", "libmpitrampoline"],
         mpiexec="mpiexec",
         abi=nothing,
+        gtl_names=nothing,
         export_prefs=false,
-        force=true,
+        force=true
     )
     binary = "system"
     # Set `ZES_ENABLE_SYSMAN` to work around https://github.com/open-mpi/ompi/issues/10142
@@ -166,12 +172,24 @@ function use_system_binary(;
     if mpiexec isa Cmd
         mpiexec = collect(mpiexec)
     end
+    libgtl = isnothing(gtl_names) ? nothing : find_library(gtl_names)
+    if libgtl == ""
+        error("""
+            GTL library could not be found with the following name(s):
+                $(gtl_names)
+            If you want to use differnt name(s) for the GTL library, use
+                MPIPreferences.use_system_binary(; gtl_names=[...])
+            If you don't want to enable GTL, use
+                MPIPreferences.use_system_binary(; gtl_names=nothing)
+            """)
+    end
     set_preferences!(MPIPreferences,
         "_format" => "1.0",
         "binary" => binary,
         "libmpi" => libmpi,
         "abi" => abi,
         "mpiexec" => mpiexec,
+        "libgtl" => libgtl;
         export_prefs=export_prefs,
         force=force
     )
@@ -186,7 +204,7 @@ function use_system_binary(;
     end
 
     if binary == MPIPreferences.binary && abi == MPIPreferences.abi && libmpi == System.libmpi && mpiexec == System.mpiexec_path
-        @info "MPIPreferences unchanged" binary libmpi abi mpiexec
+        @info "MPIPreferences unchanged" binary libmpi abi mpiexec libgtl
     else
         PREFS_CHANGED[] = true
         @info "MPIPreferences changed" binary libmpi abi mpiexec
diff --git a/lib/MPIPreferences/src/system.jl b/lib/MPIPreferences/src/system.jl
index b5fd8bd49..b7e72428d 100644
--- a/lib/MPIPreferences/src/system.jl
+++ b/lib/MPIPreferences/src/system.jl
@@ -2,12 +2,26 @@ module System
     export libmpi, mpiexec
     using Preferences, Libdl
     const libmpi = @load_preference("libmpi")
+    const libgtl = @load_preference("libgtl")
     const mpiexec_path = @load_preference("mpiexec")
     mpiexec(;adjust_PATH=true, adjust_LIBPATH=true) = `$mpiexec_path`
     mpiexec(f;adjust_PATH=true, adjust_LIBPATH=true) = f(`$mpiexec_path`)
 
+    libgtl_handle = C_NULL
     libmpi_handle = C_NULL
     function __init__()
+        # dlopen GTL library (if needed) before dlopen'ing the MPI library:
+        #
+        if ! isnothing(libgtl)
+            global libgtl_handle = try
+                Libdl.dlopen(libgtl, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
+            catch error
+                @error """
+                       $(libgtl) could not be loaded, see error message below.
+                       Use `MPIPreferences.use_system_binary` or `MPIPreferences.use_jll_binary` to reconfigure the package and then restart Julia.
+                       """ error
+            end
+        end
         global libmpi_handle = try
             Libdl.dlopen(libmpi, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
         catch error
diff --git a/src/MPI.jl b/src/MPI.jl
index 0e8ff7b5b..71e2fc736 100644
--- a/src/MPI.jl
+++ b/src/MPI.jl
@@ -103,6 +103,11 @@ function __init__()
     end
 
     @static if Sys.isunix()
+        # dlopen GTL library (if needed) before dlopen'ing the MPI library:
+        #
+        if ! isnothing(libgtl)
+            Libdl.dlopen(libgtl, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
+        end
         # dlopen the MPI library before any ccall:
         # - RTLD_GLOBAL is required for Open MPI
         #   https://www.open-mpi.org/community/lists/users/2010/04/12803.php
diff --git a/src/api/api.jl b/src/api/api.jl
index db8448034..2ee0f62cf 100644
--- a/src/api/api.jl
+++ b/src/api/api.jl
@@ -3,7 +3,8 @@ module API
 export MPI_Aint, MPI_Count, MPI_Offset, MPI_Status,
     MPI_Comm, MPI_Datatype, MPI_Errhandler, MPI_File, MPI_Group,
     MPI_Info, MPI_Message, MPI_Op, MPI_Request, MPI_Win,
-    libmpi, mpiexec, @mpichk, @mpicall, MPIPtr, SentinelPtr, FeatureLevelError
+    libmpi, mpiexec, @mpichk, @mpicall, MPIPtr, SentinelPtr, FeatureLevelError,
+    libgtl
 
 import MPIPreferences
 using Libdl
@@ -11,17 +12,25 @@ using Libdl
 if MPIPreferences.binary == "MPICH_jll"
     import MPICH_jll: libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
+    const libgtl = nothing
 elseif MPIPreferences.binary == "OpenMPI_jll"
     import OpenMPI_jll: libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
+    const libgtl = nothing
 elseif MPIPreferences.binary == "MicrosoftMPI_jll"
     import MicrosoftMPI_jll: libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
+    const libgtl = nothing
 elseif MPIPreferences.binary == "MPItrampoline_jll"
     import MPItrampoline_jll: MPItrampoline_jll, libmpi, libmpi_handle, mpiexec
     const libmpiconstants = MPItrampoline_jll.libload_time_mpi_constants_path
+    # TODO: We'll probably need GTL with libmpitrampoline, and until 
+    # MPItrampoline_jll "understands" libgtl, this should be an acceptable
+    # workaround
+    const libgtl = MPIPreferences.Preferences.@load_preference("libgtl")
 elseif MPIPreferences.binary == "system"
-    import MPIPreferences.System: libmpi, libmpi_handle, mpiexec
+    import MPIPreferences.System: libmpi, libmpi_handle, mpiexec,
+        libgtl, libgtl_handle
     const libmpiconstants = nothing
 else
     error("Unknown MPI binary: $(MPIPreferences.binary)")

From c24bf46d5f8080e7bffd2a5f170843cb7ee53ccd Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <jpblaschke@lbl.gov>
Date: Sat, 25 Feb 2023 21:46:20 -0800
Subject: [PATCH 02/25] add reference to PR in comment

---
 lib/MPIPreferences/src/system.jl | 2 +-
 src/MPI.jl                       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/MPIPreferences/src/system.jl b/lib/MPIPreferences/src/system.jl
index b7e72428d..ff6eb34bb 100644
--- a/lib/MPIPreferences/src/system.jl
+++ b/lib/MPIPreferences/src/system.jl
@@ -11,7 +11,7 @@ module System
     libmpi_handle = C_NULL
     function __init__()
         # dlopen GTL library (if needed) before dlopen'ing the MPI library:
-        #
+        # https://github.com/JuliaParallel/MPI.jl/pull/716
         if ! isnothing(libgtl)
             global libgtl_handle = try
                 Libdl.dlopen(libgtl, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
diff --git a/src/MPI.jl b/src/MPI.jl
index 71e2fc736..7aafd562f 100644
--- a/src/MPI.jl
+++ b/src/MPI.jl
@@ -104,7 +104,7 @@ function __init__()
 
     @static if Sys.isunix()
         # dlopen GTL library (if needed) before dlopen'ing the MPI library:
-        #
+        # https://github.com/JuliaParallel/MPI.jl/pull/716
         if ! isnothing(libgtl)
             Libdl.dlopen(libgtl, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
         end

From 9b55e095c0b6ce897ec0fee47dac971ed53409c0 Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <jpblaschke@lbl.gov>
Date: Sun, 23 Apr 2023 21:23:58 -0700
Subject: [PATCH 03/25] work on cray cc parser

---
 lib/MPIPreferences/src/parse_cray_cc.jl | 86 +++++++++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 lib/MPIPreferences/src/parse_cray_cc.jl

diff --git a/lib/MPIPreferences/src/parse_cray_cc.jl b/lib/MPIPreferences/src/parse_cray_cc.jl
new file mode 100644
index 000000000..e11bb3827
--- /dev/null
+++ b/lib/MPIPreferences/src/parse_cray_cc.jl
@@ -0,0 +1,86 @@
+module CrayParser
+
+import Base: filter, map, reduce
+filter(f::Function)::Function = x -> filter(f, x)
+map(f::Function)::Function    = x -> map(f, x)
+reduce(f::Function)::Function = x -> reduce(f, x)
+
+function communicate(cmd::Cmd, input="")
+    inp = Pipe()
+    out = Pipe()
+    err = Pipe()
+
+    process = run(pipeline(cmd, stdin=inp, stdout=out, stderr=err), wait=false)
+
+    close(out.in)
+    close(err.in)
+
+    stdout = @async String(read(out))
+    stderr = @async String(read(err))
+
+    write(process, input)
+    close(inp)
+
+    wait(process)
+    return (
+        stdout = fetch(stdout),
+        stderr = fetch(stderr),
+        code = process.exitcode
+    )
+end
+
+const libmpi_prefix = "mpi_"
+const libgtl_prefix = "mpi_gtl_"
+
+function cray_mpi(libs)
+    x = libs |>
+        filter(x-> startswith(x, libmpi_prefix)) |> 
+        filter(x->!startswith(x, libgtl_prefix))
+    @assert length(x) == 1
+    return x[1]
+end
+
+function cray_gtl(libs)
+    x = libs |>
+        filter(x->startswith(x, libmpi_prefix)) |> 
+        filter(x->startswith(x, libgtl_prefix))
+    @assert length(x) == 1
+    return x[1]
+end
+
+function other_libs(libs)
+    x = libs |>
+        filter(x->!startswith(x, libmpi_prefix)) |> 
+        filter(x->!startswith(x, libgtl_prefix))
+    return x
+end
+
+function analyze_cray_cc()
+    cray_opts = communicate(Cmd(["cc", "--cray-print-opts=all"]))
+    @assert cray_opts.code == 0
+
+    ld_paths = SubString{String}[]
+    libs = SubString{String}[]
+    for opt in split(cray_opts.stdout, " ") |>
+               map(x->split(x, ","))        |>
+               reduce(vcat)                 |>
+               map(x->replace(x, "\n"=>""))
+        if startswith(opt, "-L")
+            push!(ld_paths, @view opt[3:end])
+        end
+
+        if startswith(opt, "-l")
+            push!(libs, @view opt[3:end])
+        end
+    end
+
+
+    println(ld_paths)
+    println(other_libs(libs))
+    println(cray_mpi(libs))
+    println(cray_gtl(libs))
+end
+
+analyze_cray_cc()
+
+end

From d7e0af6ad1182e8c8def1fc3fca7cd9d92157495 Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <jpblaschke@lbl.gov>
Date: Mon, 24 Apr 2023 13:33:14 -0700
Subject: [PATCH 04/25] wip

---
 lib/MPIPreferences/src/parse_cray_cc.jl | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/lib/MPIPreferences/src/parse_cray_cc.jl b/lib/MPIPreferences/src/parse_cray_cc.jl
index e11bb3827..770e0b781 100644
--- a/lib/MPIPreferences/src/parse_cray_cc.jl
+++ b/lib/MPIPreferences/src/parse_cray_cc.jl
@@ -5,6 +5,20 @@ filter(f::Function)::Function = x -> filter(f, x)
 map(f::Function)::Function    = x -> map(f, x)
 reduce(f::Function)::Function = x -> reduce(f, x)
 
+struct CrayPE
+    libmpi::String
+    libgtl::String
+    cclibs::Vector{String}
+    gtl_env_switch::String
+    
+    CrayPE(mpi_dl::T, gtl_dl::T, cclibs::Vector{T}) where T <:AbstractString = new(
+        "lib" * mpi_dl * ".so", # Assuming Linux -- CrayPE is only avaialbe for linux anyway
+        "lib" * gtl_dl * ".so",
+        cclibs,
+        "MPICH_GPU_SUPPORT_ENABLED"
+    )
+end
+
 function communicate(cmd::Cmd, input="")
     inp = Pipe()
     out = Pipe()
@@ -74,13 +88,9 @@ function analyze_cray_cc()
         end
     end
 
-
-    println(ld_paths)
-    println(other_libs(libs))
-    println(cray_mpi(libs))
-    println(cray_gtl(libs))
+    CrayPE(cray_mpi(libs), cray_gtl(libs), other_libs(libs))
 end
 
-analyze_cray_cc()
+println(analyze_cray_cc())
 
 end

From ebe0c277a02b3dd46c2286b98814e0786c74cc6f Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <jpblaschke@lbl.gov>
Date: Mon, 24 Apr 2023 22:11:09 -0700
Subject: [PATCH 05/25] wip

---
 lib/MPIPreferences/src/MPIPreferences.jl | 46 ++++++++++++++++--------
 lib/MPIPreferences/src/parse_cray_cc.jl  |  2 +-
 lib/MPIPreferences/src/system.jl         | 46 +++++++++++++++++-------
 src/MPI.jl                               | 16 +++++++--
 src/api/api.jl                           | 22 ++++++++----
 5 files changed, 96 insertions(+), 36 deletions(-)

diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl
index a025bb9be..0bae3fbd4 100644
--- a/lib/MPIPreferences/src/MPIPreferences.jl
+++ b/lib/MPIPreferences/src/MPIPreferences.jl
@@ -52,6 +52,7 @@ end
 
 @static if binary == "system"
     include("system.jl")
+    include("parase_cray_cc.jl")
 end
 
 """
@@ -82,7 +83,9 @@ function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_j
         "libmpi" => nothing,
         "abi" => nothing,
         "mpiexec" => nothing,
-        "libgtl" => nothing;
+        "preloads" => nothing,
+        "preloads_env_switch" => nothing,
+        "cclibs" => nothing;
         export_prefs=export_prefs,
         force=force
     )
@@ -138,8 +141,7 @@ Options:
   using [`identify_abi`](@ref). See [`abi`](@ref) for currently supported
   values.
 
-- `gtl_names`: can be either `nothing` or a list of GPU Transport Layer (GTL)
-  names. On Cray systems, this is usually `["libmpi_gtl_cuda", "libmpi_gtl_hsa"]`.
+- `vendor`: can be either `nothing` or a vendor name (such a `"cray"`).
 
 - `export_prefs`: if `true`, the preferences into the `Project.toml` instead of
   `LocalPreferences.toml`.
@@ -150,11 +152,23 @@ function use_system_binary(;
         library_names=["libmpi", "libmpi_ibm", "msmpi", "libmpich", "libmpi_cray", "libmpitrampoline"],
         mpiexec="mpiexec",
         abi=nothing,
-        gtl_names=nothing,
+        vendor=nothing,
         export_prefs=false,
         force=true
     )
     binary = "system"
+    # vendor workarounds
+    preloads = []
+    preloads_env_switch = nothing
+    cclibs = []
+    if vendor == "cray"
+        cray_pe = CrayParser.analyze_cray_cc()
+        library_names = [cray_pe.libmpi]
+        preloads = [cray_pe.libgtl]
+        preloads_env_switch = cray_pe.gtl_env_switch
+        cclibs = cray_pe.cclibs
+    end
+
     # Set `ZES_ENABLE_SYSMAN` to work around https://github.com/open-mpi/ompi/issues/10142
     libmpi = withenv("ZES_ENABLE_SYSMAN" => "1") do
         find_library(library_names)
@@ -172,16 +186,16 @@ function use_system_binary(;
     if mpiexec isa Cmd
         mpiexec = collect(mpiexec)
     end
-    libgtl = isnothing(gtl_names) ? nothing : find_library(gtl_names)
-    if libgtl == ""
-        error("""
-            GTL library could not be found with the following name(s):
-                $(gtl_names)
-            If you want to use differnt name(s) for the GTL library, use
-                MPIPreferences.use_system_binary(; gtl_names=[...])
-            If you don't want to enable GTL, use
-                MPIPreferences.use_system_binary(; gtl_names=nothing)
-            """)
+    # libgtl = isnothing(gtl_names) ? nothing : find_library(gtl_names)
+    # if libgtl == ""
+    #     error("""
+    #         GTL library could not be found with the following name(s):
+    #             $(gtl_names)
+    #         If you want to use differnt name(s) for the GTL library, use
+    #             MPIPreferences.use_system_binary(; gtl_names=[...])
+    #         If you don't want to enable GTL, use
+    #             MPIPreferences.use_system_binary(; gtl_names=nothing)
+    #         """)
     end
     set_preferences!(MPIPreferences,
         "_format" => "1.0",
@@ -189,7 +203,9 @@ function use_system_binary(;
         "libmpi" => libmpi,
         "abi" => abi,
         "mpiexec" => mpiexec,
-        "libgtl" => libgtl;
+        "preloads" => preloads,
+        "preloads_env_switch" => preloads_env_switch,
+        "cclibs" => cclibs;
         export_prefs=export_prefs,
         force=force
     )
diff --git a/lib/MPIPreferences/src/parse_cray_cc.jl b/lib/MPIPreferences/src/parse_cray_cc.jl
index 770e0b781..defaff6c8 100644
--- a/lib/MPIPreferences/src/parse_cray_cc.jl
+++ b/lib/MPIPreferences/src/parse_cray_cc.jl
@@ -91,6 +91,6 @@ function analyze_cray_cc()
     CrayPE(cray_mpi(libs), cray_gtl(libs), other_libs(libs))
 end
 
-println(analyze_cray_cc())
+# println(analyze_cray_cc())
 
 end
diff --git a/lib/MPIPreferences/src/system.jl b/lib/MPIPreferences/src/system.jl
index ff6eb34bb..b64b26a12 100644
--- a/lib/MPIPreferences/src/system.jl
+++ b/lib/MPIPreferences/src/system.jl
@@ -2,26 +2,48 @@ module System
     export libmpi, mpiexec
     using Preferences, Libdl
     const libmpi = @load_preference("libmpi")
-    const libgtl = @load_preference("libgtl")
+    # const libgtl = @load_preference("libgtl")
+    const preloads = @load_preference("preloads")
+    const preloads_env_switch = @load_preference("preloads_env_switch")
     const mpiexec_path = @load_preference("mpiexec")
     mpiexec(;adjust_PATH=true, adjust_LIBPATH=true) = `$mpiexec_path`
     mpiexec(f;adjust_PATH=true, adjust_LIBPATH=true) = f(`$mpiexec_path`)
 
-    libgtl_handle = C_NULL
+    # libgtl_handle = C_NULL
     libmpi_handle = C_NULL
     function __init__()
-        # dlopen GTL library (if needed) before dlopen'ing the MPI library:
-        # https://github.com/JuliaParallel/MPI.jl/pull/716
-        if ! isnothing(libgtl)
-            global libgtl_handle = try
-                Libdl.dlopen(libgtl, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
-            catch error
-                @error """
-                       $(libgtl) could not be loaded, see error message below.
-                       Use `MPIPreferences.use_system_binary` or `MPIPreferences.use_jll_binary` to reconfigure the package and then restart Julia.
-                       """ error
+        # # dlopen GTL library (if needed) before dlopen'ing the MPI library:
+        # # https://github.com/JuliaParallel/MPI.jl/pull/716
+        # if ! isnothing(libgtl)
+        #     global libgtl_handle = try
+        #         Libdl.dlopen(libgtl, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
+        #     catch error
+        #         @error """
+        #                $(libgtl) could not be loaded, see error message below.
+        #                Use `MPIPreferences.use_system_binary` or `MPIPreferences.use_jll_binary` to reconfigure the package and then restart Julia.
+        #                """ error
+        #     end
+        # end
+        preload_enabled = false
+        if isnothing(preloads_env_switch)
+            preload_enabled = true
+        elseif get(ENV, preloads_env_switch, "0") == "1"
+            preload_enabled = true
+        end
+
+        if preload_enabled
+            for preload in preloads
+                try
+                    Libdl.dlopen(preload, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
+                catch error
+                    @error """
+                        $(preload) could not be loaded, see error message below.
+                        Use `MPIPreferences.use_system_binary` or `MPIPreferences.use_jll_binary` to reconfigure the package and then restart Julia.
+                        """ error
+                end
             end
         end
+
         global libmpi_handle = try
             Libdl.dlopen(libmpi, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
         catch error
diff --git a/src/MPI.jl b/src/MPI.jl
index 7aafd562f..5f71f22f1 100644
--- a/src/MPI.jl
+++ b/src/MPI.jl
@@ -105,9 +105,21 @@ function __init__()
     @static if Sys.isunix()
         # dlopen GTL library (if needed) before dlopen'ing the MPI library:
         # https://github.com/JuliaParallel/MPI.jl/pull/716
-        if ! isnothing(libgtl)
-            Libdl.dlopen(libgtl, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
+        # if ! isnothing(libgtl)
+        #     Libdl.dlopen(libgtl, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
+        # end
+        preload_enabled = false
+        if isnothing(preloads_env_switch)
+            preload_enabled = true
+        elseif get(ENV, preloads_env_switch, "0") == "1"
+            preload_enabled = true
         end
+        if preload_enabled
+            for preload in preloads
+                Libdl.dlopen(preload, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
+            end
+        end
+
         # dlopen the MPI library before any ccall:
         # - RTLD_GLOBAL is required for Open MPI
         #   https://www.open-mpi.org/community/lists/users/2010/04/12803.php
diff --git a/src/api/api.jl b/src/api/api.jl
index 2ee0f62cf..ae9ae81eb 100644
--- a/src/api/api.jl
+++ b/src/api/api.jl
@@ -4,7 +4,8 @@ export MPI_Aint, MPI_Count, MPI_Offset, MPI_Status,
     MPI_Comm, MPI_Datatype, MPI_Errhandler, MPI_File, MPI_Group,
     MPI_Info, MPI_Message, MPI_Op, MPI_Request, MPI_Win,
     libmpi, mpiexec, @mpichk, @mpicall, MPIPtr, SentinelPtr, FeatureLevelError,
-    libgtl
+    preloads, preloads_env_switch
+    # libgtl
 
 import MPIPreferences
 using Libdl
@@ -12,25 +13,34 @@ using Libdl
 if MPIPreferences.binary == "MPICH_jll"
     import MPICH_jll: libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
-    const libgtl = nothing
+    # const libgtl = nothing
+    const preloads = nothing
+    const preloads_env_switch = nothing
 elseif MPIPreferences.binary == "OpenMPI_jll"
     import OpenMPI_jll: libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
-    const libgtl = nothing
+    # const libgtl = nothing
+    const preloads = nothing
+    const preloads_env_switch = nothing
 elseif MPIPreferences.binary == "MicrosoftMPI_jll"
     import MicrosoftMPI_jll: libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
-    const libgtl = nothing
+    # const libgtl = nothing
+    const preloads = nothing
+    const preloads_env_switch = nothing
 elseif MPIPreferences.binary == "MPItrampoline_jll"
     import MPItrampoline_jll: MPItrampoline_jll, libmpi, libmpi_handle, mpiexec
     const libmpiconstants = MPItrampoline_jll.libload_time_mpi_constants_path
     # TODO: We'll probably need GTL with libmpitrampoline, and until 
     # MPItrampoline_jll "understands" libgtl, this should be an acceptable
     # workaround
-    const libgtl = MPIPreferences.Preferences.@load_preference("libgtl")
+    # const libgtl = MPIPreferences.Preferences.@load_preference("libgtl")
+    const preloads = MPIPreferences.Preferences.@load_preference("preloads")
+    const preloads_env_switch = MPIPreferences.Preferences.@load_preference("preloads_env_switch")
 elseif MPIPreferences.binary == "system"
     import MPIPreferences.System: libmpi, libmpi_handle, mpiexec,
-        libgtl, libgtl_handle
+        # libgtl, libgtl_handle
+        preloads, preloads_env_switch
     const libmpiconstants = nothing
 else
     error("Unknown MPI binary: $(MPIPreferences.binary)")

From dae1a196e7eae1882f088731606550c827a76959 Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <jpblaschke@lbl.gov>
Date: Mon, 24 Apr 2023 22:53:04 -0700
Subject: [PATCH 06/25] cleanup

---
 lib/MPIPreferences/src/MPIPreferences.jl | 23 +++++++----------------
 lib/MPIPreferences/src/parse_cray_cc.jl  |  2 --
 lib/MPIPreferences/src/system.jl         | 17 +++--------------
 src/MPI.jl                               |  7 ++-----
 src/api/api.jl                           | 11 +++--------
 5 files changed, 15 insertions(+), 45 deletions(-)

diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl
index 0bae3fbd4..d9189be5b 100644
--- a/lib/MPIPreferences/src/MPIPreferences.jl
+++ b/lib/MPIPreferences/src/MPIPreferences.jl
@@ -50,9 +50,9 @@ else
     error("Unknown binary: $binary")
 end
 
+include("parse_cray_cc.jl")
 @static if binary == "system"
     include("system.jl")
-    include("parase_cray_cc.jl")
 end
 
 """
@@ -83,7 +83,7 @@ function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_j
         "libmpi" => nothing,
         "abi" => nothing,
         "mpiexec" => nothing,
-        "preloads" => nothing,
+        "preloads" => [],
         "preloads_env_switch" => nothing,
         "cclibs" => nothing;
         export_prefs=export_prefs,
@@ -180,23 +180,15 @@ function use_system_binary(;
             If you want to try different name(s) for the MPI library, use
                 MPIPreferences.use_system_binary(; library_names=[...])""")
     end
+
     if isnothing(abi)
         abi = identify_abi(libmpi)
     end
+
     if mpiexec isa Cmd
         mpiexec = collect(mpiexec)
     end
-    # libgtl = isnothing(gtl_names) ? nothing : find_library(gtl_names)
-    # if libgtl == ""
-    #     error("""
-    #         GTL library could not be found with the following name(s):
-    #             $(gtl_names)
-    #         If you want to use differnt name(s) for the GTL library, use
-    #             MPIPreferences.use_system_binary(; gtl_names=[...])
-    #         If you don't want to enable GTL, use
-    #             MPIPreferences.use_system_binary(; gtl_names=nothing)
-    #         """)
-    end
+
     set_preferences!(MPIPreferences,
         "_format" => "1.0",
         "binary" => binary,
@@ -210,7 +202,6 @@ function use_system_binary(;
         force=force
     )
 
-
     if VERSION <= v"1.6.5" || VERSION == v"1.7.0"
         @warn """
         Due to a bug in Julia (until 1.6.5 and 1.7.1), setting preferences in transitive dependencies
@@ -220,10 +211,10 @@ function use_system_binary(;
     end
 
     if binary == MPIPreferences.binary && abi == MPIPreferences.abi && libmpi == System.libmpi && mpiexec == System.mpiexec_path
-        @info "MPIPreferences unchanged" binary libmpi abi mpiexec libgtl
+        @info "MPIPreferences unchanged" binary libmpi abi mpiexec preloads preloads_env_switch
     else
         PREFS_CHANGED[] = true
-        @info "MPIPreferences changed" binary libmpi abi mpiexec
+        @info "MPIPreferences changed" binary libmpi abi mpiexec preloads preloads_env_switch
 
         if DEPS_LOADED[]
             error("You will need to restart Julia for the changes to take effect")
diff --git a/lib/MPIPreferences/src/parse_cray_cc.jl b/lib/MPIPreferences/src/parse_cray_cc.jl
index defaff6c8..d3ca16c2e 100644
--- a/lib/MPIPreferences/src/parse_cray_cc.jl
+++ b/lib/MPIPreferences/src/parse_cray_cc.jl
@@ -91,6 +91,4 @@ function analyze_cray_cc()
     CrayPE(cray_mpi(libs), cray_gtl(libs), other_libs(libs))
 end
 
-# println(analyze_cray_cc())
-
 end
diff --git a/lib/MPIPreferences/src/system.jl b/lib/MPIPreferences/src/system.jl
index b64b26a12..5fda0b570 100644
--- a/lib/MPIPreferences/src/system.jl
+++ b/lib/MPIPreferences/src/system.jl
@@ -2,28 +2,16 @@ module System
     export libmpi, mpiexec
     using Preferences, Libdl
     const libmpi = @load_preference("libmpi")
-    # const libgtl = @load_preference("libgtl")
     const preloads = @load_preference("preloads")
     const preloads_env_switch = @load_preference("preloads_env_switch")
     const mpiexec_path = @load_preference("mpiexec")
     mpiexec(;adjust_PATH=true, adjust_LIBPATH=true) = `$mpiexec_path`
     mpiexec(f;adjust_PATH=true, adjust_LIBPATH=true) = f(`$mpiexec_path`)
 
-    # libgtl_handle = C_NULL
     libmpi_handle = C_NULL
     function __init__()
-        # # dlopen GTL library (if needed) before dlopen'ing the MPI library:
-        # # https://github.com/JuliaParallel/MPI.jl/pull/716
-        # if ! isnothing(libgtl)
-        #     global libgtl_handle = try
-        #         Libdl.dlopen(libgtl, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
-        #     catch error
-        #         @error """
-        #                $(libgtl) could not be loaded, see error message below.
-        #                Use `MPIPreferences.use_system_binary` or `MPIPreferences.use_jll_binary` to reconfigure the package and then restart Julia.
-        #                """ error
-        #     end
-        # end
+        # preload any dependencies of libmpi (if needed, eg. GTL on cray) before
+        # dlopen'ing the MPI library: https://github.com/JuliaParallel/MPI.jl/pull/716
         preload_enabled = false
         if isnothing(preloads_env_switch)
             preload_enabled = true
@@ -41,6 +29,7 @@ module System
                         Use `MPIPreferences.use_system_binary` or `MPIPreferences.use_jll_binary` to reconfigure the package and then restart Julia.
                         """ error
                 end
+                # TODO: do we want to expose the preload handles?
             end
         end
 
diff --git a/src/MPI.jl b/src/MPI.jl
index 5f71f22f1..10022095c 100644
--- a/src/MPI.jl
+++ b/src/MPI.jl
@@ -103,11 +103,8 @@ function __init__()
     end
 
     @static if Sys.isunix()
-        # dlopen GTL library (if needed) before dlopen'ing the MPI library:
-        # https://github.com/JuliaParallel/MPI.jl/pull/716
-        # if ! isnothing(libgtl)
-        #     Libdl.dlopen(libgtl, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
-        # end
+        # preload any dependencies of libmpi (if needed, eg. GTL on cray) before
+        # dlopen'ing the MPI library: https://github.com/JuliaParallel/MPI.jl/pull/716
         preload_enabled = false
         if isnothing(preloads_env_switch)
             preload_enabled = true
diff --git a/src/api/api.jl b/src/api/api.jl
index ae9ae81eb..1f01c1c67 100644
--- a/src/api/api.jl
+++ b/src/api/api.jl
@@ -13,33 +13,28 @@ using Libdl
 if MPIPreferences.binary == "MPICH_jll"
     import MPICH_jll: libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
-    # const libgtl = nothing
     const preloads = nothing
     const preloads_env_switch = nothing
 elseif MPIPreferences.binary == "OpenMPI_jll"
     import OpenMPI_jll: libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
-    # const libgtl = nothing
     const preloads = nothing
     const preloads_env_switch = nothing
 elseif MPIPreferences.binary == "MicrosoftMPI_jll"
     import MicrosoftMPI_jll: libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
-    # const libgtl = nothing
     const preloads = nothing
     const preloads_env_switch = nothing
 elseif MPIPreferences.binary == "MPItrampoline_jll"
     import MPItrampoline_jll: MPItrampoline_jll, libmpi, libmpi_handle, mpiexec
     const libmpiconstants = MPItrampoline_jll.libload_time_mpi_constants_path
-    # TODO: We'll probably need GTL with libmpitrampoline, and until 
-    # MPItrampoline_jll "understands" libgtl, this should be an acceptable
-    # workaround
-    # const libgtl = MPIPreferences.Preferences.@load_preference("libgtl")
+    # TODO: We'll probably need the preloads -- like cray's GTL -- with
+    # libmpitrampoline, and until MPItrampoline_jll "understands" preloads, this
+    # should be an acceptable workaround
     const preloads = MPIPreferences.Preferences.@load_preference("preloads")
     const preloads_env_switch = MPIPreferences.Preferences.@load_preference("preloads_env_switch")
 elseif MPIPreferences.binary == "system"
     import MPIPreferences.System: libmpi, libmpi_handle, mpiexec,
-        # libgtl, libgtl_handle
         preloads, preloads_env_switch
     const libmpiconstants = nothing
 else

From 564fb43ccc785894bf9d70fe3e892207f89404bc Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <johannes.blaschke@gmail.com>
Date: Mon, 24 Apr 2023 23:55:23 -0700
Subject: [PATCH 07/25] Fix nothing => []

---
 src/api/api.jl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/api/api.jl b/src/api/api.jl
index 1f01c1c67..5130cd58c 100644
--- a/src/api/api.jl
+++ b/src/api/api.jl
@@ -13,17 +13,17 @@ using Libdl
 if MPIPreferences.binary == "MPICH_jll"
     import MPICH_jll: libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
-    const preloads = nothing
+    const preloads = []
     const preloads_env_switch = nothing
 elseif MPIPreferences.binary == "OpenMPI_jll"
     import OpenMPI_jll: libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
-    const preloads = nothing
+    const preloads = []
     const preloads_env_switch = nothing
 elseif MPIPreferences.binary == "MicrosoftMPI_jll"
     import MicrosoftMPI_jll: libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
-    const preloads = nothing
+    const preloads = []
     const preloads_env_switch = nothing
 elseif MPIPreferences.binary == "MPItrampoline_jll"
     import MPItrampoline_jll: MPItrampoline_jll, libmpi, libmpi_handle, mpiexec

From d238adc46b3f9ebd464a8005c558d87a687fe04b Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <jpblaschke@lbl.gov>
Date: Wed, 5 Jul 2023 21:49:34 -0700
Subject: [PATCH 08/25] use readchomp

---
 lib/MPIPreferences/src/parse_cray_cc.jl | 29 ++-----------------------
 1 file changed, 2 insertions(+), 27 deletions(-)

diff --git a/lib/MPIPreferences/src/parse_cray_cc.jl b/lib/MPIPreferences/src/parse_cray_cc.jl
index d3ca16c2e..077ab5c9c 100644
--- a/lib/MPIPreferences/src/parse_cray_cc.jl
+++ b/lib/MPIPreferences/src/parse_cray_cc.jl
@@ -19,30 +19,6 @@ struct CrayPE
     )
 end
 
-function communicate(cmd::Cmd, input="")
-    inp = Pipe()
-    out = Pipe()
-    err = Pipe()
-
-    process = run(pipeline(cmd, stdin=inp, stdout=out, stderr=err), wait=false)
-
-    close(out.in)
-    close(err.in)
-
-    stdout = @async String(read(out))
-    stderr = @async String(read(err))
-
-    write(process, input)
-    close(inp)
-
-    wait(process)
-    return (
-        stdout = fetch(stdout),
-        stderr = fetch(stderr),
-        code = process.exitcode
-    )
-end
-
 const libmpi_prefix = "mpi_"
 const libgtl_prefix = "mpi_gtl_"
 
@@ -70,12 +46,11 @@ function other_libs(libs)
 end
 
 function analyze_cray_cc()
-    cray_opts = communicate(Cmd(["cc", "--cray-print-opts=all"]))
-    @assert cray_opts.code == 0
+    cray_opts = readchomp(Cmd(["cc", "--cray-print-opts=all"]))
 
     ld_paths = SubString{String}[]
     libs = SubString{String}[]
-    for opt in split(cray_opts.stdout, " ") |>
+    for opt in split(cray_opts, " ")        |>
                map(x->split(x, ","))        |>
                reduce(vcat)                 |>
                map(x->replace(x, "\n"=>""))

From 6f7ea3c2bbedd1a82a9ddbb724f70c2e6cd00d59 Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <jpblaschke@lbl.gov>
Date: Wed, 5 Jul 2023 21:57:43 -0700
Subject: [PATCH 09/25] use fix1

---
 lib/MPIPreferences/src/parse_cray_cc.jl | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/MPIPreferences/src/parse_cray_cc.jl b/lib/MPIPreferences/src/parse_cray_cc.jl
index 077ab5c9c..b3d8c7924 100644
--- a/lib/MPIPreferences/src/parse_cray_cc.jl
+++ b/lib/MPIPreferences/src/parse_cray_cc.jl
@@ -1,9 +1,8 @@
 module CrayParser
 
-import Base: filter, map, reduce
-filter(f::Function)::Function = x -> filter(f, x)
-map(f::Function)::Function    = x -> map(f, x)
-reduce(f::Function)::Function = x -> reduce(f, x)
+filter(f::Function)::Function = Base.Fix1(Base.filter, f)
+map(f::Function)::Function    = Base.Fix1(Base.map, f)
+reduce(f::Function)::Function = Base.Fix1(Base.reduce, f)
 
 struct CrayPE
     libmpi::String

From a1fbd21cc59e36f26e4be3cfafeafea243662375 Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <jpblaschke@lbl.gov>
Date: Wed, 5 Jul 2023 22:22:37 -0700
Subject: [PATCH 10/25] update docs

---
 docs/src/configuration.md                | 44 +++++++++++++++++++++++-
 lib/MPIPreferences/src/MPIPreferences.jl |  7 +++-
 2 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/docs/src/configuration.md b/docs/src/configuration.md
index aa1ec89cf..277652977 100644
--- a/docs/src/configuration.md
+++ b/docs/src/configuration.md
@@ -60,7 +60,6 @@ If the implementation is changed, you will need to call this function again. See
     from transitive dependencies is broken ([Preferences.jl#24](https://github.com/JuliaPackaging/Preferences.jl/issues/24)).
     To fix this update your version of Julia, or add `MPIPreferences` as a direct dependency to your project.
 
-
 ### Notes to HPC cluster administrators
 
 Preferences are merged across the Julia load path, such that it is feasible to provide a module file that appends a path to
@@ -107,6 +106,49 @@ Preferences are merged across the Julia load path, such that it is feasible to p
    that will take precedent by modifying the local `Project.toml` or by providing a
    `LocalPreferences.toml` file.
 
+### Notes about vendor-provided MPI backends
+
+`MPIPreferences` can load vendor-specific libraries and settings using the
+`vendor` parameter, eg `MPIPreferences.use_system_binary(mpiexec="srun", vendor="cray")`
+configures `MPIPreferences` for use on Cray systems with `srun`.
+
+!!! note
+    Currently `vendor` only supports Cray systems.
+
+This populates the `library_names`, `preloads`, `preloads_env_switch` and
+`cclibs` preferences. These are defermined by parsing `cc --cray-print-opts=all`
+emitted from the Cray Compiler Wrappers. Therefore `use_system_binary` needs
+to be run on the target system, with the corresponding `PrgEnv` loaded.
+
+The function of these settings are as follows:
+ * `preloads` specifies a list of libraries that are to be loaded (in order)
+   before `libmpi`.
+ * `preloads_env_switch` specifies the name of an environment variable that, if
+   set to `0`, can disable the `preloads`
+ * `cclibs` is a list of libraries also linked by the compiler wrappers. This is
+   recorded mainly for debugging purposes, and the libraries listed here are not
+   explicitly loaded by `MPI.jl`.
+
+An example of running `MPIPreferences.use_system_library(vendor="cray")` in
+`PrgEnv-gnu` is:
+
+   ```toml
+   [MPIPreferences]
+   _format = "1.0"
+   abi = "MPICH"
+   binary = "system"
+   cclibs = ["cupti", "cudart", "cuda", "sci_gnu_82_mpi", "sci_gnu_82", "dl", "dsmml", "xpmem"]
+   libmpi = "libmpi_gnu_91.so"
+   mpiexec = "mpiexec"
+   preloads = ["libmpi_gtl_cuda.so"]
+   preloads_env_switch = "MPICH_GPU_SUPPORT_ENABLED"
+   ```
+
+This is an example of CrayMPICH requiring `libmpi_gtl_cuda.so` to be preloaded,
+unless `MPICH_GPU_SUPPORT_ENABLED=0` (the latter allowing MPI-enabled code to
+run on a non-GPU enabled node without needing a seperate `LocalPreferences.toml`).
+
+
 ## [Using an alternative JLL-provided MPI library](@id configure_jll_binary)
 
 The following MPI implementations are provided as JLL packages and automatically obtained when installing MPI.jl:
diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl
index 86da1db35..63691b765 100644
--- a/lib/MPIPreferences/src/MPIPreferences.jl
+++ b/lib/MPIPreferences/src/MPIPreferences.jl
@@ -141,7 +141,12 @@ Options:
   using [`identify_abi`](@ref). See [`abi`](@ref) for currently supported
   values.
 
-- `vendor`: can be either `nothing` or a vendor name (such a `"cray"`).
+- `vendor`: can be either `nothing` or a vendor name (such a `"cray"`). If
+  `vendor` has the value "cray", then the output from `cc --cray-print-opts=all`
+  is parsed for which libraries are linked by the Cray Compiler Wrappers. Note
+  that if `mpi_gtl_*` is present, then this .so will be added to the preloads.
+  Also note that the inputs to `library_names` will be overwritten by the
+  library name used by the compiler wrapper.
 
 - `export_prefs`: if `true`, the preferences into the `Project.toml` instead of
   `LocalPreferences.toml`.

From 8d8af3cdaee0cf78a35c85d3e69f38932c4d5fff Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <jpblaschke@lbl.gov>
Date: Wed, 5 Jul 2023 22:25:01 -0700
Subject: [PATCH 11/25] this is neater

---
 lib/MPIPreferences/src/parse_cray_cc.jl | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/lib/MPIPreferences/src/parse_cray_cc.jl b/lib/MPIPreferences/src/parse_cray_cc.jl
index b3d8c7924..f167e6fcc 100644
--- a/lib/MPIPreferences/src/parse_cray_cc.jl
+++ b/lib/MPIPreferences/src/parse_cray_cc.jl
@@ -25,16 +25,14 @@ function cray_mpi(libs)
     x = libs |>
         filter(x-> startswith(x, libmpi_prefix)) |> 
         filter(x->!startswith(x, libgtl_prefix))
-    @assert length(x) == 1
-    return x[1]
+    return only(x)
 end
 
 function cray_gtl(libs)
     x = libs |>
         filter(x->startswith(x, libmpi_prefix)) |> 
         filter(x->startswith(x, libgtl_prefix))
-    @assert length(x) == 1
-    return x[1]
+    return only(x)
 end
 
 function other_libs(libs)

From 8d2d7819ec93cd4ed35bb3f7d9a020c50aaf88c5 Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <jpblaschke@lbl.gov>
Date: Wed, 5 Jul 2023 22:27:17 -0700
Subject: [PATCH 12/25] bump format version number

---
 lib/MPIPreferences/src/MPIPreferences.jl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl
index 63691b765..4894e7be0 100644
--- a/lib/MPIPreferences/src/MPIPreferences.jl
+++ b/lib/MPIPreferences/src/MPIPreferences.jl
@@ -4,7 +4,7 @@ export use_jll_binary, use_system_binary
 
 using Preferences, Libdl
 
-if !(VersionNumber(@load_preference("_format", "1.0")) <= v"1.0")
+if !(VersionNumber(@load_preference("_format", "1.1")) <= v"1.1")
     error("The preferences attached to MPIPreferences are incompatible with this version of the package.")
 end
 
@@ -78,7 +78,7 @@ function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_j
               Accepted options are:
                   $(join(known_binaries, ", "))""")
     set_preferences!(MPIPreferences,
-        "_format" => "1.0",
+        "_format" => "1.1",
         "binary" => binary,
         "libmpi" => nothing,
         "abi" => nothing,
@@ -195,7 +195,7 @@ function use_system_binary(;
     end
 
     set_preferences!(MPIPreferences,
-        "_format" => "1.0",
+        "_format" => "1.1",
         "binary" => binary,
         "libmpi" => libmpi,
         "abi" => abi,

From 07f6350fad99737c5e4cbb01c9aa45afe3bc4731 Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <jpblaschke@lbl.gov>
Date: Wed, 5 Jul 2023 22:37:34 -0700
Subject: [PATCH 13/25] formatting

---
 docs/src/configuration.md | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/docs/src/configuration.md b/docs/src/configuration.md
index 277652977..fc6540f31 100644
--- a/docs/src/configuration.md
+++ b/docs/src/configuration.md
@@ -132,23 +132,22 @@ The function of these settings are as follows:
 An example of running `MPIPreferences.use_system_library(vendor="cray")` in
 `PrgEnv-gnu` is:
 
-   ```toml
-   [MPIPreferences]
-   _format = "1.0"
-   abi = "MPICH"
-   binary = "system"
-   cclibs = ["cupti", "cudart", "cuda", "sci_gnu_82_mpi", "sci_gnu_82", "dl", "dsmml", "xpmem"]
-   libmpi = "libmpi_gnu_91.so"
-   mpiexec = "mpiexec"
-   preloads = ["libmpi_gtl_cuda.so"]
-   preloads_env_switch = "MPICH_GPU_SUPPORT_ENABLED"
-   ```
+```toml
+[MPIPreferences]
+_format = "1.0"
+abi = "MPICH"
+binary = "system"
+cclibs = ["cupti", "cudart", "cuda", "sci_gnu_82_mpi", "sci_gnu_82", "dl", "dsmml", "xpmem"]
+libmpi = "libmpi_gnu_91.so"
+mpiexec = "mpiexec"
+preloads = ["libmpi_gtl_cuda.so"]
+preloads_env_switch = "MPICH_GPU_SUPPORT_ENABLED"
+```
 
 This is an example of CrayMPICH requiring `libmpi_gtl_cuda.so` to be preloaded,
 unless `MPICH_GPU_SUPPORT_ENABLED=0` (the latter allowing MPI-enabled code to
 run on a non-GPU enabled node without needing a seperate `LocalPreferences.toml`).
 
-
 ## [Using an alternative JLL-provided MPI library](@id configure_jll_binary)
 
 The following MPI implementations are provided as JLL packages and automatically obtained when installing MPI.jl:

From 7a81fc478411187be8c5c3335076cdca3b5319b4 Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <johannes.blaschke@gmail.com>
Date: Thu, 13 Jul 2023 10:14:28 -0700
Subject: [PATCH 14/25] Update lib/MPIPreferences/src/MPIPreferences.jl

Only bump format for where the new version is needed

Co-authored-by: Simon Byrne <simonbyrne@gmail.com>
---
 lib/MPIPreferences/src/MPIPreferences.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl
index 4894e7be0..4b0f1bc11 100644
--- a/lib/MPIPreferences/src/MPIPreferences.jl
+++ b/lib/MPIPreferences/src/MPIPreferences.jl
@@ -78,7 +78,7 @@ function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_j
               Accepted options are:
                   $(join(known_binaries, ", "))""")
     set_preferences!(MPIPreferences,
-        "_format" => "1.1",
+        "_format" => "1.0",
         "binary" => binary,
         "libmpi" => nothing,
         "abi" => nothing,

From 15e064aea920046292c371067f8d127f332cc48b Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <johannes.blaschke@gmail.com>
Date: Thu, 13 Jul 2023 10:15:46 -0700
Subject: [PATCH 15/25] Update lib/MPIPreferences/src/MPIPreferences.jl

only require v1.1 if vendor is input

Co-authored-by: Simon Byrne <simonbyrne@gmail.com>
---
 lib/MPIPreferences/src/MPIPreferences.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl
index 4b0f1bc11..3be2e7394 100644
--- a/lib/MPIPreferences/src/MPIPreferences.jl
+++ b/lib/MPIPreferences/src/MPIPreferences.jl
@@ -195,7 +195,7 @@ function use_system_binary(;
     end
 
     set_preferences!(MPIPreferences,
-        "_format" => "1.1",
+        "_format" => isnothing(vendor) ? "1.0" : "1.1",
         "binary" => binary,
         "libmpi" => libmpi,
         "abi" => abi,

From d06178a5e97a273a46c9eb971aff719e2c6d7412 Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <johannes.blaschke@gmail.com>
Date: Thu, 13 Jul 2023 10:16:41 -0700
Subject: [PATCH 16/25] Update docs/src/configuration.md

Co-authored-by: Simon Byrne <simonbyrne@gmail.com>
---
 docs/src/configuration.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/configuration.md b/docs/src/configuration.md
index fc6540f31..c1fc21270 100644
--- a/docs/src/configuration.md
+++ b/docs/src/configuration.md
@@ -134,7 +134,7 @@ An example of running `MPIPreferences.use_system_library(vendor="cray")` in
 
 ```toml
 [MPIPreferences]
-_format = "1.0"
+_format = "1.1"
 abi = "MPICH"
 binary = "system"
 cclibs = ["cupti", "cudart", "cuda", "sci_gnu_82_mpi", "sci_gnu_82", "dl", "dsmml", "xpmem"]

From cef0d0fe397dd44b0cd08d694554d5d63113fef1 Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <johannes.blaschke@gmail.com>
Date: Thu, 13 Jul 2023 10:16:51 -0700
Subject: [PATCH 17/25] Update docs/src/configuration.md

Co-authored-by: Simon Byrne <simonbyrne@gmail.com>
---
 docs/src/configuration.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/src/configuration.md b/docs/src/configuration.md
index c1fc21270..4479db88a 100644
--- a/docs/src/configuration.md
+++ b/docs/src/configuration.md
@@ -129,6 +129,8 @@ The function of these settings are as follows:
    recorded mainly for debugging purposes, and the libraries listed here are not
    explicitly loaded by `MPI.jl`.
 
+If these are set, the `_format` key will be set to `"1.1"`.
+
 An example of running `MPIPreferences.use_system_library(vendor="cray")` in
 `PrgEnv-gnu` is:
 

From a73e3992ad7c8897f7f33d33642ad37b66b71966 Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <johannes.blaschke@gmail.com>
Date: Wed, 19 Jul 2023 17:21:29 -0700
Subject: [PATCH 18/25] Update src/MPI.jl

Co-authored-by: Simon Byrne <simonbyrne@gmail.com>
---
 src/MPI.jl | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/MPI.jl b/src/MPI.jl
index 575586a0b..2d174252d 100644
--- a/src/MPI.jl
+++ b/src/MPI.jl
@@ -115,15 +115,11 @@ function __init__()
     @static if Sys.isunix()
         # preload any dependencies of libmpi (if needed, eg. GTL on cray) before
         # dlopen'ing the MPI library: https://github.com/JuliaParallel/MPI.jl/pull/716
-        preload_enabled = false
-        if isnothing(preloads_env_switch)
-            preload_enabled = true
-        elseif get(ENV, preloads_env_switch, "0") == "1"
-            preload_enabled = true
-        end
-        if preload_enabled
-            for preload in preloads
-                Libdl.dlopen(preload, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
+        if !isnothing(preloads)
+            if isnothing(preloads_env_switch) || get(ENV, preloads_env_switch, "0") == "1"
+                for preload in preloads
+                    Libdl.dlopen(preload, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
+                end
             end
         end
 

From 7c4b3938de816d7993f31e304323f386d3eba98a Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <johannes.blaschke@gmail.com>
Date: Thu, 20 Jul 2023 10:53:44 -0700
Subject: [PATCH 19/25] Update lib/MPIPreferences/src/MPIPreferences.jl

Co-authored-by: Valentin Churavy <vchuravy@users.noreply.github.com>
---
 lib/MPIPreferences/src/MPIPreferences.jl | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl
index 3be2e7394..7222a6201 100644
--- a/lib/MPIPreferences/src/MPIPreferences.jl
+++ b/lib/MPIPreferences/src/MPIPreferences.jl
@@ -166,12 +166,15 @@ function use_system_binary(;
     preloads = []
     preloads_env_switch = nothing
     cclibs = []
-    if vendor == "cray"
+    if vendor === nothing
+    elseif vendor == "cray"
         cray_pe = CrayParser.analyze_cray_cc()
         library_names = [cray_pe.libmpi]
         preloads = [cray_pe.libgtl]
         preloads_env_switch = cray_pe.gtl_env_switch
         cclibs = cray_pe.cclibs
+    else
+        error("Unknown vendor $vendor")
     end
 
     # Set `ZES_ENABLE_SYSMAN` to work around https://github.com/open-mpi/ompi/issues/10142

From a1b225147b9da529f76dc1a9fb14d8268b65f21d Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <jpblaschke@lbl.gov>
Date: Thu, 20 Jul 2023 15:16:28 -0700
Subject: [PATCH 20/25] clean up, and sanitize preloading

---
 lib/MPIPreferences/src/MPIPreferences.jl |  8 +++--
 lib/MPIPreferences/src/preloads.jl       | 44 ++++++++++++++++++++++++
 lib/MPIPreferences/src/system.jl         | 23 ++-----------
 src/MPI.jl                               | 14 +++-----
 src/api/api.jl                           | 18 ++--------
 5 files changed, 59 insertions(+), 48 deletions(-)
 create mode 100644 lib/MPIPreferences/src/preloads.jl

diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl
index 7222a6201..0a4b9cf68 100644
--- a/lib/MPIPreferences/src/MPIPreferences.jl
+++ b/lib/MPIPreferences/src/MPIPreferences.jl
@@ -4,7 +4,7 @@ export use_jll_binary, use_system_binary
 
 using Preferences, Libdl
 
-if !(VersionNumber(@load_preference("_format", "1.1")) <= v"1.1")
+if !(VersionNumber(@load_preference("_format", "1.0")) <= v"1.1")
     error("The preferences attached to MPIPreferences are incompatible with this version of the package.")
 end
 
@@ -50,7 +50,11 @@ else
     error("Unknown binary: $binary")
 end
 
+include("preloads.jl")
+using .Preloads: dlopen_preloads, preloads, preloads_env_switch
+
 include("parse_cray_cc.jl")
+
 @static if binary == "system"
     include("system.jl")
 end
@@ -117,8 +121,8 @@ end
         library_names = ["libmpi", "libmpi_ibm", "msmpi", "libmpich", "libmpi_cray", "libmpitrampoline"],
         mpiexec = "mpiexec",
         abi = nothing,
+        vendor = nothing,
         export_prefs = false,
-        gtl_names=nothing,
         force = true)
 
 Switches the underlying MPI implementation to a system provided one. A restart
diff --git a/lib/MPIPreferences/src/preloads.jl b/lib/MPIPreferences/src/preloads.jl
new file mode 100644
index 000000000..2380c8339
--- /dev/null
+++ b/lib/MPIPreferences/src/preloads.jl
@@ -0,0 +1,44 @@
+module Preloads
+
+any(f::Function)::Function = Base.Fix1(Base.any, f)
+
+using Preferences, Libdl
+
+const preloads = @load_preference("preloads")
+const preloads_env_switch = @load_preference("preloads_env_switch")
+
+
+function is_loaded(name)
+    if dllist() |> any(x->endswith(x, name))
+        return true
+    end
+    return false
+end
+
+"""
+    dlopen_preloads()
+
+dlopen's all preloads specified in the preloads section of MPIPreferences
+"""
+function dlopen_preloads()
+    if !isnothing(preloads)
+        if isnothing(preloads_env_switch) || get(ENV, preloads_env_switch, "0") == "1"
+            for preload in preloads
+                if is_loaded(preload)
+                    continue
+                end
+
+                try
+                    dlopen(preload, RTLD_LAZY | RTLD_GLOBAL)
+                catch error
+                    @error """
+                        $(preload) could not be loaded, see error message below.
+                        Use `MPIPreferences` to reconfigure the package and then restart Julia.
+                        """ error
+                end
+            end
+        end
+    end
+end
+
+end
\ No newline at end of file
diff --git a/lib/MPIPreferences/src/system.jl b/lib/MPIPreferences/src/system.jl
index 5fda0b570..3d16043d0 100644
--- a/lib/MPIPreferences/src/system.jl
+++ b/lib/MPIPreferences/src/system.jl
@@ -1,4 +1,6 @@
 module System
+    include("preloads.jl")
+
     export libmpi, mpiexec
     using Preferences, Libdl
     const libmpi = @load_preference("libmpi")
@@ -12,26 +14,7 @@ module System
     function __init__()
         # preload any dependencies of libmpi (if needed, eg. GTL on cray) before
         # dlopen'ing the MPI library: https://github.com/JuliaParallel/MPI.jl/pull/716
-        preload_enabled = false
-        if isnothing(preloads_env_switch)
-            preload_enabled = true
-        elseif get(ENV, preloads_env_switch, "0") == "1"
-            preload_enabled = true
-        end
-
-        if preload_enabled
-            for preload in preloads
-                try
-                    Libdl.dlopen(preload, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
-                catch error
-                    @error """
-                        $(preload) could not be loaded, see error message below.
-                        Use `MPIPreferences.use_system_binary` or `MPIPreferences.use_jll_binary` to reconfigure the package and then restart Julia.
-                        """ error
-                end
-                # TODO: do we want to expose the preload handles?
-            end
-        end
+        Preloads.dlopen_preloads()
 
         global libmpi_handle = try
             Libdl.dlopen(libmpi, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
diff --git a/src/MPI.jl b/src/MPI.jl
index 2d174252d..7d74e49e0 100644
--- a/src/MPI.jl
+++ b/src/MPI.jl
@@ -112,17 +112,11 @@ function __init__()
         """ ENV["JULIA_MPI_BINARY"]=mpi_env_binary MPIPreferences.binary
     end
 
-    @static if Sys.isunix()
-        # preload any dependencies of libmpi (if needed, eg. GTL on cray) before
-        # dlopen'ing the MPI library: https://github.com/JuliaParallel/MPI.jl/pull/716
-        if !isnothing(preloads)
-            if isnothing(preloads_env_switch) || get(ENV, preloads_env_switch, "0") == "1"
-                for preload in preloads
-                    Libdl.dlopen(preload, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
-                end
-            end
-        end
+    # preload any dependencies of libmpi (if needed, eg. GTL on cray) before
+    # dlopen'ing the MPI library: https://github.com/JuliaParallel/MPI.jl/pull/716
+    MPIPreferences.dlopen_preloads()
 
+    @static if Sys.isunix()
         # dlopen the MPI library before any ccall:
         # - RTLD_GLOBAL is required for Open MPI
         #   https://www.open-mpi.org/community/lists/users/2010/04/12803.php
diff --git a/src/api/api.jl b/src/api/api.jl
index 411a5a5fb..af5822857 100644
--- a/src/api/api.jl
+++ b/src/api/api.jl
@@ -3,9 +3,7 @@ module API
 export MPI_Aint, MPI_Count, MPI_Offset, MPI_Status,
     MPI_Comm, MPI_Datatype, MPI_Errhandler, MPI_File, MPI_Group,
     MPI_Info, MPI_Message, MPI_Op, MPI_Request, MPI_Win,
-    libmpi, mpiexec, @mpichk, @mpicall, MPIPtr, SentinelPtr, FeatureLevelError,
-    preloads, preloads_env_switch
-    # libgtl
+    libmpi, mpiexec, @mpichk, @mpicall, MPIPtr, SentinelPtr, FeatureLevelError
 
 import MPIPreferences
 using Libdl
@@ -13,29 +11,17 @@ using Libdl
 if MPIPreferences.binary == "MPICH_jll"
     import MPICH_jll: MPICH_jll, libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
-    const preloads = []
-    const preloads_env_switch = nothing
 elseif MPIPreferences.binary == "OpenMPI_jll"
     import OpenMPI_jll: OpenMPI_jll, libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
-    const preloads = []
-    const preloads_env_switch = nothing
 elseif MPIPreferences.binary == "MicrosoftMPI_jll"
     import MicrosoftMPI_jll: MicrosoftMPI_jll, libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
-    const preloads = []
-    const preloads_env_switch = nothing
 elseif MPIPreferences.binary == "MPItrampoline_jll"
     import MPItrampoline_jll: MPItrampoline_jll, libmpi, libmpi_handle, mpiexec
     const libmpiconstants = MPItrampoline_jll.libload_time_mpi_constants_path
-    # TODO: We'll probably need the preloads -- like cray's GTL -- with
-    # libmpitrampoline, and until MPItrampoline_jll "understands" preloads, this
-    # should be an acceptable workaround
-    const preloads = MPIPreferences.Preferences.@load_preference("preloads")
-    const preloads_env_switch = MPIPreferences.Preferences.@load_preference("preloads_env_switch")
 elseif MPIPreferences.binary == "system"
-    import MPIPreferences.System: libmpi, libmpi_handle, mpiexec,
-        preloads, preloads_env_switch
+    import MPIPreferences.System: libmpi, libmpi_handle, mpiexec
     const libmpiconstants = nothing
 else
     error("Unknown MPI binary: $(MPIPreferences.binary)")

From 82b2750d45249921fc7eaedbfe796e70d8b1ab17 Mon Sep 17 00:00:00 2001
From: Johannes Blaschke <jpblaschke@lbl.gov>
Date: Thu, 20 Jul 2023 16:54:56 -0700
Subject: [PATCH 21/25] add docstring to docs

---
 docs/src/reference/mpipreferences.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/src/reference/mpipreferences.md b/docs/src/reference/mpipreferences.md
index d17849e71..1b2649ae0 100644
--- a/docs/src/reference/mpipreferences.md
+++ b/docs/src/reference/mpipreferences.md
@@ -22,6 +22,7 @@ MPIPreferences.use_jll_binary
 ```@docs
 MPIPreferences.check_unchanged
 MPIPreferences.identify_abi
+MPIPreferences.dlopen_preloads
 ```
 
 ## Preferences schema

From c38a6dc1fc56e9c0c07d1d7511e063afc4aaaf0f Mon Sep 17 00:00:00 2001
From: Simon Byrne <simonbyrne@gmail.com>
Date: Fri, 21 Jul 2023 12:16:14 -0700
Subject: [PATCH 22/25] Update src/MPI.jl

---
 src/MPI.jl | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/MPI.jl b/src/MPI.jl
index 7d74e49e0..49fb40e55 100644
--- a/src/MPI.jl
+++ b/src/MPI.jl
@@ -112,9 +112,11 @@ function __init__()
         """ ENV["JULIA_MPI_BINARY"]=mpi_env_binary MPIPreferences.binary
     end
 
-    # preload any dependencies of libmpi (if needed, eg. GTL on cray) before
-    # dlopen'ing the MPI library: https://github.com/JuliaParallel/MPI.jl/pull/716
-    MPIPreferences.dlopen_preloads()
+    if isdefined(MPIPreferences, :dlopen_preloads)
+        # preload any dependencies of libmpi (if needed, eg. GTL on cray) before
+        # dlopen'ing the MPI library: https://github.com/JuliaParallel/MPI.jl/pull/716
+        MPIPreferences.dlopen_preloads()
+    end
 
     @static if Sys.isunix()
         # dlopen the MPI library before any ccall:

From 26cfb69702132e9700cec14a4bb8282fc937770d Mon Sep 17 00:00:00 2001
From: Simon Byrne <simonbyrne@gmail.com>
Date: Fri, 21 Jul 2023 12:17:15 -0700
Subject: [PATCH 23/25] Bump patch

---
 lib/MPIPreferences/Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/MPIPreferences/Project.toml b/lib/MPIPreferences/Project.toml
index 5099bc642..4ad8d8c0e 100644
--- a/lib/MPIPreferences/Project.toml
+++ b/lib/MPIPreferences/Project.toml
@@ -1,7 +1,7 @@
 name = "MPIPreferences"
 uuid = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267"
 authors = []
-version = "0.1.8"
+version = "0.1.9"
 
 [deps]
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

From aa1689702bd741dd0686e620084dd987e94c2cc8 Mon Sep 17 00:00:00 2001
From: Simon Byrne <simonbyrne@gmail.com>
Date: Mon, 24 Jul 2023 10:05:38 -0400
Subject: [PATCH 24/25] Bump version

---
 Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Project.toml b/Project.toml
index 4539f0b8a..e447bdc19 100644
--- a/Project.toml
+++ b/Project.toml
@@ -1,7 +1,7 @@
 name = "MPI"
 uuid = "da04e1cc-30fd-572f-bb4f-1f8673147195"
 authors = []
-version = "0.20.11"
+version = "0.20.12"
 
 [deps]
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"

From 41475006d944312faa35d9bdd6cd3847fd6b2436 Mon Sep 17 00:00:00 2001
From: Valentin Churavy <vchuravy@users.noreply.github.com>
Date: Mon, 24 Jul 2023 10:17:00 -0400
Subject: [PATCH 25/25] lowerbound MPIPreferences

---
 Project.toml | 2 +-
 src/MPI.jl   | 8 +++-----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/Project.toml b/Project.toml
index e447bdc19..030dee921 100644
--- a/Project.toml
+++ b/Project.toml
@@ -22,7 +22,7 @@ Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 AMDGPU = "0.3, 0.4"
 CUDA = "3, 4"
 DocStringExtensions = "0.8, 0.9"
-MPIPreferences = "0.1.6"
+MPIPreferences = "0.1.8"
 PkgVersion = "0.3"
 PrecompileTools = "1.0.1"
 Requires = "~0.5, 1.0"
diff --git a/src/MPI.jl b/src/MPI.jl
index 49fb40e55..7d74e49e0 100644
--- a/src/MPI.jl
+++ b/src/MPI.jl
@@ -112,11 +112,9 @@ function __init__()
         """ ENV["JULIA_MPI_BINARY"]=mpi_env_binary MPIPreferences.binary
     end
 
-    if isdefined(MPIPreferences, :dlopen_preloads)
-        # preload any dependencies of libmpi (if needed, eg. GTL on cray) before
-        # dlopen'ing the MPI library: https://github.com/JuliaParallel/MPI.jl/pull/716
-        MPIPreferences.dlopen_preloads()
-    end
+    # preload any dependencies of libmpi (if needed, eg. GTL on cray) before
+    # dlopen'ing the MPI library: https://github.com/JuliaParallel/MPI.jl/pull/716
+    MPIPreferences.dlopen_preloads()
 
     @static if Sys.isunix()
         # dlopen the MPI library before any ccall: