Add tests to MPIPreferences starting with GTL parsing

lib/MPIPreferences/src/parse_cray_cc.jl

@@ -64,7 +64,14 @@ function other_libs(libs)
 end
 
 function analyze_cray_cc()
-    cray_opts = readchomp(Cmd(["cc", "--cray-print-opts=all"]))
+    opts_file = get(ENV, "JULIA_MPI_CC_OPTS_FILE", "")
+    cray_opts = ""
+    if ! isempty(opts_file)
+        @warn "Loading Cray parameters from file at: JULIA_MPI_CC_OPTS_FILE=$(opts_file)"
+        cray_opts = readchomp(opts_file)
+    else
+        cray_opts = readchomp(Cmd(["cc", "--cray-print-opts=all"]))
+    end
 
     ld_paths = SubString{String}[]
     libs = SubString{String}[]

lib/MPIPreferences/test/Project.toml

@@ -0,0 +1,5 @@
+[deps]
+MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

lib/MPIPreferences/test/runtests.jl

@@ -0,0 +1,56 @@
+using Pkg, Test, MPIPreferences
+
+project_path = dirname(Pkg.project().path)
+test_dir = @__DIR__
+test_data = joinpath(@__DIR__, "test_data")
+
+cmd(f; a="") = `$(Base.julia_cmd()) --startup-file=no --project=$(project_path) $(joinpath(test_dir, f)) $(a)`
+
+function test_generated_prefs(suffix)
+    MPIPreferences.use_system_binary(vendor="cray"; mpiexec="srun")
+
+    local_pref_1 = joinpath(project_path, "JuliaLocalPreferences.toml")
+    local_pref_2 = joinpath(project_path, "LocalPreferences.toml")
+    pref_file = isfile(local_pref_1) ? local_pref_1 : local_pref_2
+
+    genereated_settings = readchomp(pref_file)
+    expected_settings = readchomp(
+        joinpath(test_data, "LocalPreferences_$(suffix).toml")
+    )
+
+    @test genereated_settings == expected_settings
+    r = run(cmd("test_libmpi.jl"))
+    @test success(r)
+end
+
+function test_existing_prefs(suffix)
+    local_pref_1 = joinpath(project_path, "JuliaLocalPreferences.toml")
+    local_pref_2 = joinpath(project_path, "LocalPreferences.toml")
+
+    # ensure any previous tests have been cleaned up
+    if isfile(local_pref_1) rm(local_pref_1) end
+    if isfile(local_pref_2) rm(local_pref_2) end
+
+    cp(
+        joinpath(test_data, "LocalPreferences_$(suffix).toml"),
+        joinpath(project_path, "LocalPreferences.toml")
+    )
+
+    r = run(cmd("test_libmpi.jl"))
+    @test success(r)
+end
+
+withenv("JULIA_MPI_CC_OPTS_FILE" => joinpath(test_data, "cc_opts_cuda.txt")) do
+    test_generated_prefs("cuda")
+end
+
+withenv("JULIA_MPI_CC_OPTS_FILE" => joinpath(test_data, "cc_opts_cpu.txt")) do
+    test_generated_prefs("cpu")
+end
+
+test_existing_prefs("cuda")
+r = run(cmd("test_gtl_preload.jl"; a="cuda"))
+@test success(r)
+test_existing_prefs("cpu")
+r = run(cmd("test_gtl_preload.jl"; a="cpu"))
+@test success(r)

lib/MPIPreferences/test/test_data/LocalPreferences_cpu.toml

@@ -0,0 +1,9 @@
+[MPIPreferences]
+__clear__ = ["preloads"]
+_format = "1.1"
+abi = "MPICH"
+binary = "system"
+cclibs = ["sci_gnu_82_mpi", "sci_gnu_82", "dl", "dsmml", "xpmem"]
+libmpi = "libmpi_gnu_91.so"
+mpiexec = "srun"
+preloads_env_switch = "MPICH_GPU_SUPPORT_ENABLED"

lib/MPIPreferences/test/test_data/LocalPreferences_cuda.toml

@@ -0,0 +1,9 @@
+[MPIPreferences]
+_format = "1.1"
+abi = "MPICH"
+binary = "system"
+cclibs = ["cupti", "cudart", "cuda", "sci_gnu_82_mpi", "sci_gnu_82", "dl", "dsmml", "xpmem"]
+libmpi = "libmpi_gnu_91.so"
+mpiexec = "srun"
+preloads = ["libmpi_gtl_cuda.so"]
+preloads_env_switch = "MPICH_GPU_SUPPORT_ENABLED"

lib/MPIPreferences/test/test_data/cc_opts_cpu.txt

@@ -0,0 +1 @@
+-I/opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/include -I/opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64/include -I/opt/cray/pe/dsmml/0.2.2/dsmml//include -I/opt/cray/xpmem/2.6.2-2.5_2.27__gd067c3f.shasta/include -L/opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/lib -L/opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64/lib -L/opt/cray/pe/dsmml/0.2.2/dsmml//lib -L/opt/cray/xpmem/2.6.2-2.5_2.27__gd067c3f.shasta/lib64 -Wl,--as-needed,-lmpi_gnu_91,--no-as-needed -Wl,--as-needed,-lsci_gnu_82_mpi,--no-as-needed -Wl,--as-needed,-lsci_gnu_82,--no-as-needed -ldl -Wl,--as-needed,-ldsmml,--no-as-needed -lxpmem

lib/MPIPreferences/test/test_data/cc_opts_cuda.txt

@@ -0,0 +1 @@
+-I/opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/include -I/opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64/include -I/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/nvvm/include -I/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/extras/CUPTI/include -I/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/extras/Debugger/include -I/opt/cray/pe/dsmml/0.2.2/dsmml//include -I/opt/cray/xpmem/2.6.2-2.5_2.27__gd067c3f.shasta/include -L/opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/lib -L/opt/cray/pe/mpich/8.1.25/gtl/lib -L/opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64/lib -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/lib64/stubs -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/lib64 -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/nvvm/lib64 -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/extras/CUPTI/lib64 -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/extras/Debugger/lib64 -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/math_libs/11.7/lib64 -L/opt/cray/pe/dsmml/0.2.2/dsmml//lib -L/opt/cray/xpmem/2.6.2-2.5_2.27__gd067c3f.shasta/lib64 -Wl,--as-needed,-lcupti,-lcudart,--no-as-needed -lcuda -Wl,--as-needed,-lmpi_gnu_91,--no-as-needed -lmpi_gtl_cuda -Wl,--as-needed,-lsci_gnu_82_mpi,--no-as-needed -Wl,--as-needed,-lsci_gnu_82,--no-as-needed -ldl -Wl,--as-needed,-ldsmml,--no-as-needed -lxpmem

lib/MPIPreferences/test/test_gtl_preload.jl

@@ -0,0 +1,7 @@
+using Test, MPIPreferences
+
+if ARGS[1] == "cuda"
+    @test only(MPIPreferences.System.preloads) ==  "libmpi_gtl_cuda.so"
+elseif ARGS[1] == "cpu"
+    @test isnothing(MPIPreferences.System.preloads)
+end

lib/MPIPreferences/test/test_libmpi.jl

@@ -0,0 +1,3 @@
+using Test, MPI
+
+@test MPI.libmpi == "libmpi_gnu_91.so"