From 74fb87c8ba9afeaf95dcbe3118fdd755abb8b39b Mon Sep 17 00:00:00 2001 From: Gabriele Bozzola Date: Mon, 18 Mar 2024 09:55:27 -0700 Subject: [PATCH] Add ClimaArtifacts --- .buildkite/pipeline.yml | 27 +++ Project.toml | 12 +- docs/Manifest.toml | 241 +++++++++++--------- docs/Project.toml | 1 + docs/src/index.md | 6 + src/ClimaArtifacts.jl | 186 +++++++++++++++ src/ClimaComms.jl | 5 + test/test_artifacts/Artifacts.toml | 14 ++ test/test_artifacts/Project.toml | 10 + test/test_artifacts/test_clima_artifacts.jl | 45 ++++ 10 files changed, 431 insertions(+), 116 deletions(-) create mode 100644 src/ClimaArtifacts.jl create mode 100644 test/test_artifacts/Artifacts.toml create mode 100644 test/test_artifacts/Project.toml create mode 100644 test/test_artifacts/test_clima_artifacts.jl diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b424399d..cd814c09 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -19,6 +19,8 @@ steps: - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" - "julia --project -e 'using Pkg; Pkg.status()'" + - "julia --project=test/test_artifacts -e 'using Pkg; Pkg.develop(;path=\".\"); Pkg.instantiate()'" + agents: slurm_cpus_per_task: 8 env: @@ -55,3 +57,28 @@ steps: slurm_nodes: 1 slurm_ntasks_per_node: 2 slurm_gpus_per_task: 1 + + # The artifacts jobs cannot be concurrent + - label: ":amphora: artifacts" + key: "artifacts" + command: + - julia --project=test/test_artifacts test/test_artifacts/test_clima_artifacts.jl + env: + CLIMACOMMS_TEST_DEVICE: CPU + agents: + slurm_nodes: 1 + slurm_ntasks_per_node: 1 + concurrency: 1 + concurrency_group: 'artifacts' + + - label: ":amphora::amphora: artifacts (MPI)" + key: "mpi_artifacts" + command: + - julia --project=test/test_artifacts test/test_artifacts/test_clima_artifacts.jl + env: + CLIMACOMMS_TEST_DEVICE: CPU + agents: + slurm_nodes: 1 + slurm_ntasks_per_node: 2 + concurrency: 1 + concurrency_group: 'artifacts' diff --git a/Project.toml b/Project.toml index 30a31424..fafb191c 100644 --- a/Project.toml +++ b/Project.toml @@ -1,16 +1,10 @@ name = "ClimaComms" uuid = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d" -authors = [ - "Kiran Pamnany ", - "Simon Byrne ", - "Charles Kawczynski ", - "Sriharsha Kandala ", - "Jake Bolewski ", - "Gabriele Bozzola ", -] -version = "0.5.7" +authors = ["Kiran Pamnany ", "Simon Byrne ", "Charles Kawczynski ", "Sriharsha Kandala ", "Jake Bolewski ", "Gabriele Bozzola "] +version = "0.5.8" [deps] +Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 1f6a1426..5aa3c4c6 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.9.3" +julia_version = "1.10.2" manifest_format = "2.0" -project_hash = "2a3f6f2093cc9e00b435b32de577d1b8ccb4f7ac" +project_hash = "e4c6632eef3aaf209e39b70b0251ebbdbb42c2c1" [[deps.ANSIColoredPrinters]] git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" @@ -24,15 +24,15 @@ version = "1.5.0" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [[deps.AbstractTrees]] -git-tree-sha1 = "faa260e4cb5aba097a73fab382dd4b5819d8ec8c" +git-tree-sha1 = "2d9c9a55f9c93e8887ad391fbae72f8ef55e1177" uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" -version = "0.4.4" +version = "0.4.5" [[deps.Adapt]] deps = ["LinearAlgebra", "Requires"] -git-tree-sha1 = "76289dc51920fdc6e0013c872ba9551d54961c24" +git-tree-sha1 = "cea4ac3f5b4bc4b3000aa55afb6e5626518948fa" uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" -version = "3.6.2" +version = "4.0.3" weakdeps = ["StaticArrays"] [deps.Adapt.extensions] @@ -61,45 +61,47 @@ version = "0.4.2" uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" [[deps.CEnum]] -git-tree-sha1 = "eb4cb44a499229b3b8426dcfb5dd85333951ff90" +git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc" uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" -version = "0.4.2" +version = "0.5.0" [[deps.CUDA]] -deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "Crayons", "DataFrames", "ExprTools", "GPUArrays", "GPUCompiler", "KernelAbstractions", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "NVTX", "Preferences", "PrettyTables", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "Statistics", "UnsafeAtomicsLLVM"] -git-tree-sha1 = "f062a48c26ae027f70c44f48f244862aec47bf99" +deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "Crayons", "DataFrames", "ExprTools", "GPUArrays", "GPUCompiler", "KernelAbstractions", "LLVM", "LLVMLoopInfo", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "NVTX", "Preferences", "PrettyTables", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "StaticArrays", "Statistics"] +git-tree-sha1 = "baa8ea7a1ea63316fa3feb454635215773c9c845" uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" -version = "5.0.0" +version = "5.2.0" [deps.CUDA.extensions] + ChainRulesCoreExt = "ChainRulesCore" SpecialFunctionsExt = "SpecialFunctions" [deps.CUDA.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" [[deps.CUDA_Driver_jll]] deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"] -git-tree-sha1 = "35a37bb72b35964f2895c12c687ae263b4ac170c" +git-tree-sha1 = "d01bfc999768f0a31ed36f5d22a76161fc63079c" uuid = "4ee394cb-3365-5eb0-8335-949819d2adfc" -version = "0.6.0+3" +version = "0.7.0+1" [[deps.CUDA_Runtime_Discovery]] deps = ["Libdl"] -git-tree-sha1 = "bcc4a23cbbd99c8535a5318455dcf0f2546ec536" +git-tree-sha1 = "2cb12f6b2209f40a4b8967697689a47c50485490" uuid = "1af6417a-86b4-443c-805f-a4643ffb695f" -version = "0.2.2" +version = "0.2.3" [[deps.CUDA_Runtime_jll]] deps = ["Artifacts", "CUDA_Driver_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] -git-tree-sha1 = "bfe5a693a11522d58392f742243f2b50dc27afd6" +git-tree-sha1 = "8e25c009d2bf16c2c31a70a6e9e8939f7325cc84" uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" -version = "0.9.2+0" +version = "0.11.1+0" [[deps.ClimaComms]] -deps = ["CUDA", "MPI"] +deps = ["Artifacts", "CUDA", "MPI"] path = ".." uuid = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d" -version = "0.5.4" +version = "0.5.8" [[deps.ColorTypes]] deps = ["FixedPointNumbers", "Random"] @@ -114,10 +116,10 @@ uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" version = "0.12.10" [[deps.Compat]] -deps = ["UUIDs"] -git-tree-sha1 = "8a62af3e248a8c4bad6b32cbbe663ae02275e32c" +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "c955881e3c981181362ae4088b35995446298b80" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.10.0" +version = "4.14.0" weakdeps = ["Dates", "LinearAlgebra"] [deps.Compat.extensions] @@ -126,7 +128,7 @@ weakdeps = ["Dates", "LinearAlgebra"] [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.0.5+0" +version = "1.1.0+0" [[deps.Crayons]] git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" @@ -134,9 +136,9 @@ uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" version = "4.1.1" [[deps.DataAPI]] -git-tree-sha1 = "8da84edb865b0b5b0100c0666a9bc9a0b71c553c" +git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" -version = "1.15.0" +version = "1.16.0" [[deps.DataFrames]] deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "REPL", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] @@ -146,9 +148,9 @@ version = "1.6.1" [[deps.DataStructures]] deps = ["Compat", "InteractiveUtils", "OrderedCollections"] -git-tree-sha1 = "3dbd312d370723b6bb43ba9d02fc36abade4518d" +git-tree-sha1 = "0f4b5d62a88d8f59003e43c25a8a90de9eb76317" uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.18.15" +version = "0.18.18" [[deps.DataValueInterfaces]] git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" @@ -200,27 +202,33 @@ uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" [[deps.GPUArrays]] deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] -git-tree-sha1 = "8ad8f375ae365aa1eb2f42e2565a40b55a4b69a8" +git-tree-sha1 = "47e4686ec18a9620850bad110b79966132f14283" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "9.0.0" +version = "10.0.2" [[deps.GPUArraysCore]] deps = ["Adapt"] -git-tree-sha1 = "2d6ca471a6c7b536127afccfa7564b5b39227fe0" +git-tree-sha1 = "ec632f177c0d990e64d955ccc1b8c04c485a0950" uuid = "46192b85-c4d5-4398-a991-12ede77f4527" -version = "0.1.5" +version = "0.1.6" [[deps.GPUCompiler]] deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Scratch", "TimerOutputs", "UUIDs"] -git-tree-sha1 = "5e4487558477f191c043166f8301dd0b4be4e2b2" +git-tree-sha1 = "a846f297ce9d09ccba02ead0cae70690e072a119" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" -version = "0.24.5" +version = "0.25.0" + +[[deps.Hwloc_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "ca0f6bf568b4bfc807e7537f081c81e35ceca114" +uuid = "e33a78d0-f292-5ffc-b300-72abe9b543c8" +version = "2.10.0+0" [[deps.IOCapture]] deps = ["Logging", "Random"] -git-tree-sha1 = "d75853a0bdbfb1ac815478bacd89cd27b550ace6" +git-tree-sha1 = "8b72179abc660bfab5e28472e019392b97d0985c" uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" -version = "0.2.3" +version = "0.2.4" [[deps.InlineStrings]] deps = ["Parsers"] @@ -262,9 +270,9 @@ version = "0.2.1+0" [[deps.KernelAbstractions]] deps = ["Adapt", "Atomix", "InteractiveUtils", "LinearAlgebra", "MacroTools", "PrecompileTools", "Requires", "SparseArrays", "StaticArrays", "UUIDs", "UnsafeAtomics", "UnsafeAtomicsLLVM"] -git-tree-sha1 = "4c5875e4c228247e1c2b087669846941fb6e0118" +git-tree-sha1 = "ed7167240f40e62d97c1f5f7735dea6de3cc5c49" uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" -version = "0.9.8" +version = "0.9.18" [deps.KernelAbstractions.extensions] EnzymeExt = "EnzymeCore" @@ -273,26 +281,35 @@ version = "0.9.8" EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" [[deps.LLVM]] -deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Printf", "Unicode"] -git-tree-sha1 = "a9d2ce1d5007b1e8f6c5b89c5a31ff8bd146db5c" +deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Preferences", "Printf", "Requires", "Unicode"] +git-tree-sha1 = "7c6650580b4c3169d9905858160db895bff6d2e2" uuid = "929cbde3-209d-540e-8aea-75f648917ca0" -version = "6.2.1" +version = "6.6.1" +weakdeps = ["BFloat16s"] + + [deps.LLVM.extensions] + BFloat16sExt = "BFloat16s" [[deps.LLVMExtra_jll]] deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] -git-tree-sha1 = "7ca6850ae880cc99b59b88517545f91a52020afa" +git-tree-sha1 = "88b916503aac4fb7f701bb625cd84ca5dd1677bc" uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" -version = "0.0.25+0" +version = "0.0.29+0" + +[[deps.LLVMLoopInfo]] +git-tree-sha1 = "2e5c102cfc41f48ae4740c7eca7743cc7e7b75ea" +uuid = "8b046642-f1f6-4319-8d3c-209ddc03c586" +version = "1.0.0" [[deps.LaTeXStrings]] -git-tree-sha1 = "f2355693d6778a178ade15952b7ac47a4ff97996" +git-tree-sha1 = "50901ebc375ed41dbf8058da26f9de442febbbec" uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" -version = "1.3.0" +version = "1.3.1" [[deps.LazilyInitializedFields]] -git-tree-sha1 = "410fe4739a4b092f2ffe36fcb0dcc3ab12648ce1" +git-tree-sha1 = "8f7f3cabab0fd1800699663533b6d5cb3fc0e612" uuid = "0e77f7df-68c5-4e49-93ce-4cd80f5598bf" -version = "1.2.1" +version = "1.2.2" [[deps.LazyArtifacts]] deps = ["Artifacts", "Pkg"] @@ -301,21 +318,26 @@ uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -version = "0.6.3" +version = "0.6.4" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "7.84.0+0" +version = "8.4.0+0" [[deps.LibGit2]] -deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.6.4+0" + [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.10.2+0" +version = "1.11.0+1" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -329,9 +351,9 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" [[deps.MPI]] deps = ["Distributed", "DocStringExtensions", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "PkgVersion", "PrecompileTools", "Requires", "Serialization", "Sockets"] -git-tree-sha1 = "b4d8707e42b693720b54f0b3434abee6dd4d947a" +git-tree-sha1 = "4e3136db3735924f96632a5b40a5979f1f53fa07" uuid = "da04e1cc-30fd-572f-bb4f-1f8673147195" -version = "0.20.16" +version = "0.20.19" [deps.MPI.extensions] AMDGPUExt = "AMDGPU" @@ -342,28 +364,28 @@ version = "0.20.16" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" [[deps.MPICH_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] -git-tree-sha1 = "8a5b4d2220377d1ece13f49438d71ad20cf1ba83" +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] +git-tree-sha1 = "656036b9ed6f942d35e536e249600bc31d0f9df8" uuid = "7cb0a576-ebde-5e09-9194-50597f1243b4" -version = "4.1.2+0" +version = "4.2.0+0" [[deps.MPIPreferences]] deps = ["Libdl", "Preferences"] -git-tree-sha1 = "781916a2ebf2841467cda03b6f1af43e23839d85" +git-tree-sha1 = "8f6af051b9e8ec597fa09d8885ed79fd582f33c9" uuid = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" -version = "0.1.9" +version = "0.1.10" [[deps.MPItrampoline_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] -git-tree-sha1 = "6979eccb6a9edbbb62681e158443e79ecc0d056a" +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] +git-tree-sha1 = "77c3bd69fdb024d75af38713e883d0f249ce19c2" uuid = "f1f71cc9-e9ae-5b93-9b94-4fe0e1ad3748" -version = "5.3.1+0" +version = "5.3.2+0" [[deps.MacroTools]] deps = ["Markdown", "Random"] -git-tree-sha1 = "9ee1618cbf5240e6d4e0371d6f24065083f60c48" +git-tree-sha1 = "2fa9ee3e63fd3a4f7a9a4f4744a52f4856de82df" uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -version = "0.5.11" +version = "0.5.13" [[deps.Markdown]] deps = ["Base64"] @@ -371,20 +393,20 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[deps.MarkdownAST]] deps = ["AbstractTrees", "Markdown"] -git-tree-sha1 = "e8513266815200c0c8f522d6d44ffb5e9b366ae4" +git-tree-sha1 = "465a70f0fc7d443a00dcdc3267a497397b8a3899" uuid = "d0879d2d-cac2-40c8-9cee-1863dc0c7391" -version = "0.1.1" +version = "0.1.2" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.2+0" +version = "2.28.2+1" [[deps.MicrosoftMPI_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "a8027af3d1743b3bfae34e54872359fdebb31422" +git-tree-sha1 = "f12a29c4400ba812841c6ace3f4efbb6dbb3ba01" uuid = "9237b28f-5490-5468-be7b-bb81f5f5e6cf" -version = "10.1.3+4" +version = "10.1.4+2" [[deps.Missings]] deps = ["DataAPI"] @@ -397,13 +419,13 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2022.10.11" +version = "2023.1.10" [[deps.NVTX]] deps = ["Colors", "JuliaNVTXCallbacks_jll", "Libdl", "NVTX_jll"] -git-tree-sha1 = "8bc9ce4233be3c63f8dcd78ccaf1b63a9c0baa34" +git-tree-sha1 = "53046f0483375e3ed78e49190f1154fa0a4083a1" uuid = "5da4648a-3479-48b8-97b9-01cb529c0a1f" -version = "0.3.3" +version = "0.3.4" [[deps.NVTX_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -418,29 +440,29 @@ version = "1.2.0" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.21+4" +version = "0.3.23+4" [[deps.OpenMPI_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] -git-tree-sha1 = "f3080f4212a8ba2ceb10a34b938601b862094314" +git-tree-sha1 = "e25c1778a98e34219a00455d6e4384e017ea9762" uuid = "fe0851c0-eecd-5654-98d4-656369965a5c" -version = "4.1.5+0" +version = "4.1.6+0" [[deps.OrderedCollections]] -git-tree-sha1 = "2e73fe17cac3c62ad1aebe70d44c963c3cfdc3e3" +git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.6.2" +version = "1.6.3" [[deps.Parsers]] deps = ["Dates", "PrecompileTools", "UUIDs"] -git-tree-sha1 = "716e24b21538abc91f6205fd1d8363f39b442851" +git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.7.2" +version = "2.8.1" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.9.2" +version = "1.10.0" [[deps.PkgVersion]] deps = ["Pkg"] @@ -456,21 +478,21 @@ version = "1.4.3" [[deps.PrecompileTools]] deps = ["Preferences"] -git-tree-sha1 = "03b4c25b43cb84cee5c90aa9b5ea0a78fd848d2f" +git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.2.0" +version = "1.2.1" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "00805cd429dcb4870060ff49ef443486c262e38e" +git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.1" +version = "1.4.3" [[deps.PrettyTables]] -deps = ["Crayons", "LaTeXStrings", "Markdown", "Printf", "Reexport", "StringManipulation", "Tables"] -git-tree-sha1 = "ee094908d720185ddbdc58dbe0c1cbe35453ec7a" +deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "88b895d13d53b5577fd53379d913b9ab9ac82660" uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" -version = "2.2.7" +version = "2.3.1" [[deps.Printf]] deps = ["Unicode"] @@ -481,14 +503,14 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[deps.Random]] -deps = ["SHA", "Serialization"] +deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[deps.Random123]] deps = ["Random", "RandomNumbers"] -git-tree-sha1 = "552f30e847641591ba3f39fd1bed559b9deb0ef3" +git-tree-sha1 = "4743b43e5a9c4a2ede372de7061eed81795b12e7" uuid = "74087812-796a-5b5d-8853-05524746bad3" -version = "1.6.1" +version = "1.7.0" [[deps.RandomNumbers]] deps = ["Random", "Requires"] @@ -519,15 +541,15 @@ version = "0.7.0" [[deps.Scratch]] deps = ["Dates"] -git-tree-sha1 = "30449ee12237627992a99d5e30ae63e4d78cd24a" +git-tree-sha1 = "3bac05bc7e74a75fd9cba4295cde4045d9fe2386" uuid = "6c6a2e73-6563-6170-7368-637461726353" -version = "1.2.0" +version = "1.2.1" [[deps.SentinelArrays]] deps = ["Dates", "Random"] -git-tree-sha1 = "04bdff0b09c65ff3e06a05e3eb7b120223da3d39" +git-tree-sha1 = "0e7508ff27ba32f26cd459474ca2ede1bc10991f" uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" -version = "1.4.0" +version = "1.4.1" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" @@ -537,24 +559,29 @@ uuid = "6462fe0b-24de-5631-8697-dd941f90decc" [[deps.SortingAlgorithms]] deps = ["DataStructures"] -git-tree-sha1 = "c60ec5c62180f27efea3ba2908480f8055e17cee" +git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "1.1.1" +version = "1.2.1" [[deps.SparseArrays]] deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.10.0" [[deps.StaticArrays]] -deps = ["LinearAlgebra", "Random", "StaticArraysCore"] -git-tree-sha1 = "0adf069a2a490c47273727e029371b31d44b72b2" +deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] +git-tree-sha1 = "bf074c045d3d5ffd956fa0a461da38a44685d6b2" uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "1.6.5" -weakdeps = ["Statistics"] +version = "1.9.3" [deps.StaticArrays.extensions] + StaticArraysChainRulesCoreExt = "ChainRulesCore" StaticArraysStatisticsExt = "Statistics" + [deps.StaticArrays.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + [[deps.StaticArraysCore]] git-tree-sha1 = "36b3d696ce6366023a0ea192b4cd442268995a0d" uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" @@ -563,7 +590,7 @@ version = "1.4.2" [[deps.Statistics]] deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -version = "1.9.0" +version = "1.10.0" [[deps.StringManipulation]] deps = ["PrecompileTools"] @@ -572,9 +599,9 @@ uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" version = "0.3.4" [[deps.SuiteSparse_jll]] -deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" -version = "5.10.1+6" +version = "7.2.1+1" [[deps.TOML]] deps = ["Dates"] @@ -589,9 +616,9 @@ version = "1.0.1" [[deps.Tables]] deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits"] -git-tree-sha1 = "a1f34829d5ac0ef499f6d84428bd6b4c71f02ead" +git-tree-sha1 = "cb76cf677714c095e535e3501ac7954732aeea2d" uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -version = "1.11.0" +version = "1.11.1" [[deps.Tar]] deps = ["ArgTools", "SHA"] @@ -629,19 +656,19 @@ version = "0.1.3" [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.2.13+0" +version = "1.2.13+1" [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.8.0+0" +version = "5.8.0+1" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.48.0+0" +version = "1.52.0+1" [[deps.p7zip_jll]] deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" -version = "17.4.0+0" +version = "17.4.0+2" diff --git a/docs/Project.toml b/docs/Project.toml index f85258f0..fa02d22f 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,3 +1,4 @@ [deps] +Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" ClimaComms = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" diff --git a/docs/src/index.md b/docs/src/index.md index e2d0f376..ea36bc48 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -63,3 +63,9 @@ ClimaComms.start ClimaComms.progress ClimaComms.finish ``` + +## Artifacts + +```@docs +ClimaComms.@clima_artifact +``` diff --git a/src/ClimaArtifacts.jl b/src/ClimaArtifacts.jl new file mode 100644 index 00000000..772514c2 --- /dev/null +++ b/src/ClimaArtifacts.jl @@ -0,0 +1,186 @@ +module ClimaArtifacts + +import Base.BinaryPlatforms: HostPlatform +import Artifacts as JuliaArtifacts + +import ..SingletonCommsContext, + ..MPICommsContext, ..AbstractCommsContext, ..iamroot, ..barrier + +const ACCESSED_ARTIFACTS::Set{String} = Set(String[]) + +root_or_singleton(::Nothing) = true +root_or_singleton(::SingletonCommsContext) = true +root_or_singleton(ctx::MPICommsContext) = iamroot(ctx) + +maybe_wait(::Nothing) = nothing +maybe_wait(::SingletonCommsContext) = nothing +maybe_wait(ctx::MPICommsContext) = barrier(ctx) + +# This code is largely a re-implementation of Artifacts.artifact_str extended to add +# instrumentation and control MPI +""" + @clima_artifact(artifact_name, context = nothing) + +Return the path of the given artifact name. + +For MPI contexts and lazily downloaded artifacts. + +The context is required only for lazy artifacts. +""" +macro clima_artifact(name, context = nothing) + # Find Artifacts.toml file we're going to load from + srcfile = string(__source__.file) + if ( + (isinteractive() && startswith(srcfile, "REPL[")) || + (!isinteractive() && srcfile == "none") + ) && !isfile(srcfile) + srcfile = pwd() + end + local artifacts_toml = JuliaArtifacts.find_artifacts_toml(srcfile) + if isnothing(artifacts_toml) + error( + string( + "Cannot locate '(Julia)Artifacts.toml' file when attempting to use artifact '", + name, + "' in '", + __module__, + "'", + ), + ) + end + + # Load Artifacts.toml at compile time, so that we don't have to use `__source__.file` + # at runtime, which gets stale if the `.ji` file is relocated. + local artifact_dict = JuliaArtifacts.load_artifacts_toml(artifacts_toml) + + # Invalidate calling .ji file if Artifacts.toml file changes + Base.include_dependency(artifacts_toml) + + # Check if the user has provided `LazyArtifacts`, and thus supports lazy artifacts + # If not, check to see if `Pkg` or `Pkg.Artifacts` has been imported. + lazyartifacts = nothing + for module_name in (:LazyArtifacts, :Pkg, :Artifacts) + if isdefined(__module__, module_name) + lazyartifacts = GlobalRef(__module__, module_name) + break + end + end + + # Artifacts.artifact_str deals with platforms, but we do not need to support that + # feature + platform = HostPlatform() + + # If `name` is a constant, we can actually load and parse the `Artifacts.toml` file now, + # saving the work from runtime. + if isa(name, AbstractString) + # To support slash-indexing, we need to split the artifact name from the path tail: + artifact_name, artifact_path_tail, hash = + JuliaArtifacts.artifact_slash_lookup( + name, + artifact_dict, + artifacts_toml, + platform, + ) + meta = JuliaArtifacts.artifact_meta( + artifact_name, + artifact_dict, + artifacts_toml; + platform, + ) + if !isnothing(meta) && get(meta, "lazy", false) + # This is a lazy artifact, we can only process it with a context + isnothing(context) && + error("Lazy artifacts required @clima_artifact(name, context)") + end + return quote + # We call JuliaArtifacts._artifact_str twice, the first time only with the root + # process (to avoid race conditions), the second time to ensure that all the + # processes have the artifact string + if Base.invokelatest(root_or_singleton, $(esc(context))) + artifact_path = Base.invokelatest( + JuliaArtifacts._artifact_str, + $(__module__), + $(artifacts_toml), + $(artifact_name), + $(artifact_path_tail), + $(artifact_dict), + $(hash), + $(platform), + $(lazyartifacts), + )::String + push!(ACCESSED_ARTIFACTS, artifact_path) + end + Base.invokelatest(maybe_wait, $(esc(context))) + # When we call _artifact_str again, we can now assume that the artifact is + # available + return Base.invokelatest( + JuliaArtifacts._artifact_str, + $(__module__), + $(artifacts_toml), + $(artifact_name), + $(artifact_path_tail), + $(artifact_dict), + $(hash), + $(platform), + $(lazyartifacts), + )::String + end + else + # If artifact_name is not a string (e.g., it is a variable), we have to do all the + # work at runtime + return quote + local platform = $(esc(platform)) + local artifact_name, artifact_path_tail, hash = + JuliaArtifacts.artifact_slash_lookup( + $(esc(name)), + $(artifact_dict), + $(artifacts_toml), + platform, + ) + meta = JuliaArtifacts.artifact_meta( + artifact_name, + $artifact_dict, + $artifacts_toml; + platform, + ) + if !isnothing(meta) && get(meta, "lazy", false) + # This is a lazy artifact, we can only process it with a context + isnothing($context) && error( + "Lazy artifacts required @clima_artifact(name, context)", + ) + end + + # We call JuliaArtifacts._artifact_str twice, the first time only with the root + # process (to avoid race conditions), the second time to ensure that all the + # processes have the artifact string + if Base.invokelatest(root_or_singleton, $(esc(context))) + artifact_path = Base.invokelatest( + JuliaArtifacts._artifact_str, + $(__module__), + $(artifacts_toml), + artifact_name, + artifact_path_tail, + $(artifact_dict), + hash, + platform, + $(lazyartifacts), + )::String + push!(ACCESSED_ARTIFACTS, artifact_path) + end + Base.invokelatest(maybe_wait, $(esc(context))) + return Base.invokelatest( + JuliaArtifacts._artifact_str, + $(__module__), + $(artifacts_toml), + artifact_name, + artifact_path_tail, + $(artifact_dict), + hash, + platform, + $(lazyartifacts), + )::String + end + end +end + +end diff --git a/src/ClimaComms.jl b/src/ClimaComms.jl index 3d4084c3..00113592 100644 --- a/src/ClimaComms.jl +++ b/src/ClimaComms.jl @@ -8,6 +8,7 @@ in order to: etc.), and - transparently support single or double buffering for GPUs, depending on whether the transport has the ability to access GPU memory. +- lazily download artifacts in a process-safe way. """ module ClimaComms @@ -16,4 +17,8 @@ include("context.jl") include("singleton.jl") include("mpi.jl") +include("ClimaArtifacts.jl") +import .ClimaArtifacts: @clima_artifact +export @clima_artifact + end # module diff --git a/test/test_artifacts/Artifacts.toml b/test/test_artifacts/Artifacts.toml new file mode 100644 index 00000000..5587d6b5 --- /dev/null +++ b/test/test_artifacts/Artifacts.toml @@ -0,0 +1,14 @@ +[socrates] +git-tree-sha1 = "43563e7631a7eafae1f9f8d9d332e3de44ad7239" +lazy = true + + [[socrates.download]] + url = "https://github.com/staticfloat/small_bin/raw/master/socrates.tar.gz" + sha256 = "e65d2f13f2085f2c279830e863292312a72930fee5ba3c792b14c33ce5c5cc58" + +[laskar2004] +git-tree-sha1 = "e4044f4093e731021a3e62428f594ee7b5ea56e2" + + [[laskar2004.download]] + sha256 = "8b3a9510ade307e1834c51968421ab803d0d570f1c27529cb417d7a185fbcc3e" + url = "https://data.caltech.edu/records/v9dp1-jb227/files/laskar2004.tar.gz" diff --git a/test/test_artifacts/Project.toml b/test/test_artifacts/Project.toml new file mode 100644 index 00000000..29fe1091 --- /dev/null +++ b/test/test_artifacts/Project.toml @@ -0,0 +1,10 @@ +name = "ClimaTestArtifact" + +[deps] +Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +ClimaComms = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d" +LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +ClimaComms = "0.5.8" diff --git a/test/test_artifacts/test_clima_artifacts.jl b/test/test_artifacts/test_clima_artifacts.jl new file mode 100644 index 00000000..1bc32a5e --- /dev/null +++ b/test/test_artifacts/test_clima_artifacts.jl @@ -0,0 +1,45 @@ +using Artifacts +using LazyArtifacts +using Test + +import ClimaComms + +const context = ClimaComms.context() + +@show expected_path = artifact"socrates" + +# Remove the artifact, so that we test that we are downloading it +Base.Filesystem.rm(dirname(expected_path), recursive = true) +@info "Removed artifact" + +@testset "Artifact, context: $context" begin + @test_throws ErrorException @macroexpand ClimaComms.@clima_artifact( + "socrates" + ) + @test ClimaComms.@clima_artifact("socrates", context) == expected_path + + # Test with name as a variable + Base.Filesystem.rm(dirname(expected_path), recursive = true) + artifact_name = "socrates" + + @test ClimaComms.@clima_artifact(artifact_name, context) == expected_path + + @test_throws ErrorException ClimaComms.@clima_artifact(artifact_name) +end + +# Test with a non lazy-artifact + +@show expected_path2 = artifact"laskar2004" + +# Remove the artifact, so that we test that we are downloading it +Base.Filesystem.rm(dirname(expected_path2), recursive = true) + +@testset "Non-lazy artifact, context: $context" begin + @test ClimaComms.@clima_artifact("laskar2004") == expected_path2 + + # Test with name as a variable + Base.Filesystem.rm(dirname(expected_path2), recursive = true) + artifact_name = "laskar2004" + + @test ClimaComms.@clima_artifact(artifact_name) == expected_path2 +end