From a315d0e3e8e74aa9be4db60f088077b7dab01df6 Mon Sep 17 00:00:00 2001 From: Gabriele Bozzola Date: Mon, 27 Jan 2025 08:33:47 -0800 Subject: [PATCH] Consolidate Manifests Manifests.toml are a burden to maintain and add a lot of git diff noise. The one benefit of Manifests.toml is that they help with tracking version dependencies. However: - For `docs`, the precise version of dependencies is not important, there is pretty much no version-dependent code being executed there - `perf` and `examples` can be consolidated into a single `.buildkite` environment. This environment only has to have one manifest checked, the one for which buildkite is run (and not one for each version of Julia). - The `perf` environment can be removed all together. It is a more niche environment: if reproducing a specific result is required, developers can use the `.buildkite` Manifest. If not, developers can also consider keeping the relevant tools (mainly JET) in the base environment. This is the same pattern we have in other repos such as ClimaCore. --- {perf => .buildkite}/Manifest-v1.11.toml | 0 .buildkite/gpu_pipeline/pipeline.yml | 50 +- .buildkite/longruns_gpu/pipeline.yml | 46 +- .buildkite/pipeline.yml | 263 +- .buildkite/scaling/pipeline.sh | 156 +- docs/Manifest-v1.11.toml | 2606 -------------------- docs/Manifest.toml | 2571 ------------------- docs/src/contributor_guide.md | 2 +- examples/Manifest-v1.11.toml | 2782 --------------------- examples/Manifest.toml | 2745 --------------------- perf/Manifest.toml | 2872 ---------------------- perf/Project.toml | 96 - perf/jet.jl | 2 +- 13 files changed, 252 insertions(+), 13939 deletions(-) rename {perf => .buildkite}/Manifest-v1.11.toml (100%) delete mode 100644 docs/Manifest-v1.11.toml delete mode 100644 docs/Manifest.toml delete mode 100644 examples/Manifest-v1.11.toml delete mode 100644 examples/Manifest.toml delete mode 100644 perf/Manifest.toml delete mode 100644 perf/Project.toml diff --git a/perf/Manifest-v1.11.toml b/.buildkite/Manifest-v1.11.toml similarity index 100% rename from perf/Manifest-v1.11.toml rename to .buildkite/Manifest-v1.11.toml diff --git a/.buildkite/gpu_pipeline/pipeline.yml b/.buildkite/gpu_pipeline/pipeline.yml index e7858cfa3df..7ba6cc4d514 100644 --- a/.buildkite/gpu_pipeline/pipeline.yml +++ b/.buildkite/gpu_pipeline/pipeline.yml @@ -20,19 +20,13 @@ steps: key: "init_gpu_env" command: - echo "--- Instantiate examples" - - julia --project=examples -e 'using Pkg; Pkg.instantiate(;verbose=true)' - - julia --project=examples -e 'using Pkg; Pkg.precompile()' - - julia --project=examples -e 'using CUDA; CUDA.precompile_runtime()' - - julia --project=examples -e 'using Pkg; Pkg.status()' - - - echo "--- Instantiate perf" - - julia --project=perf -e 'using Pkg; Pkg.instantiate(;verbose=true)' - - julia --project=perf -e 'using Pkg; Pkg.precompile()' - - julia --project=perf -e 'using CUDA; CUDA.precompile_runtime()' - - julia --project=perf -e 'using Pkg; Pkg.status()' + - julia --project=.buildkite -e 'using Pkg; Pkg.instantiate(;verbose=true)' + - julia --project=.buildkite -e 'using Pkg; Pkg.precompile()' + - julia --project=.buildkite -e 'using CUDA; CUDA.precompile_runtime()' + - julia --project=.buildkite -e 'using Pkg; Pkg.status()' - echo "--- Download artifacts" - - julia --project=examples artifacts/download_artifacts.jl + - julia --project=.buildkite artifacts/download_artifacts.jl agents: slurm_gpus: 1 @@ -52,7 +46,7 @@ steps: - mkdir -p target_gpu_implicit_baroclinic_wave - > nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=target_gpu_implicit_baroclinic_wave/output_active/report - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${GPU_CONFIG_PATH}target_gpu_implicit_baroclinic_wave.yml --job_id target_gpu_implicit_baroclinic_wave artifact_paths: "target_gpu_implicit_baroclinic_wave/output_active/*" @@ -69,7 +63,7 @@ steps: - mkdir -p gpu_hs_rhoe_equil_55km_nz63_0M - > nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_hs_rhoe_equil_55km_nz63_0M/output_active/report - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${GPU_CONFIG_PATH}gpu_hs_rhoe_equil_0M.yml --job_id gpu_hs_rhoe_equil_55km_nz63_0M artifact_paths: "gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*" @@ -87,7 +81,7 @@ steps: - > srun --cpu-bind=threads --cpus-per-task=4 nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_hs_rhoe_equil_55km_nz63_0M_4process/output_active/report-%q{PMI_RANK} - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${GPU_CONFIG_PATH}gpu_hs_rhoe_equil_0M.yml --job_id gpu_hs_rhoe_equil_55km_nz63_0M_4process artifact_paths: "gpu_hs_rhoe_equil_55km_nz63_0M_4process/output_active/*" @@ -107,7 +101,7 @@ steps: - > srun --cpu-bind=threads --cpus-per-task=4 nsys profile --delay 100 --trace=osrt,nvtx,cuda,mpi,ucx --output=target_gpu_implicit_baroclinic_wave_4process/output_active/report-%q{PMI_RANK} - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${GPU_CONFIG_PATH}target_gpu_implicit_baroclinic_wave.yml --job_id target_gpu_implicit_baroclinic_wave_4process artifact_paths: "target_gpu_implicit_baroclinic_wave_4process/output_active/*" @@ -128,7 +122,7 @@ steps: - mkdir -p gpu_aquaplanet_dyamond_diag_1process - > srun --cpu-bind=threads --cpus-per-task=4 - nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_dyamond_diag_1process/output_active/report julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_dyamond_diag_1process/output_active/report julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_diag_1process.yml --job_id gpu_aquaplanet_dyamond_diag_1process artifact_paths: "gpu_aquaplanet_dyamond_diag_1process/output_active/*" @@ -149,7 +143,7 @@ steps: - > srun --cpu-bind=threads --cpus-per-task=4 nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_dyamond_ss_1process/output_active/report - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ss.yml --job_id gpu_aquaplanet_dyamond_ss_1process artifact_paths: "gpu_aquaplanet_dyamond_ss_1process/output_active/*" @@ -169,7 +163,7 @@ steps: - mkdir -p gpu_aquaplanet_dyamond_ss_2process - > srun --cpu-bind=threads --cpus-per-task=4 - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ss.yml --job_id gpu_aquaplanet_dyamond_ss_2process artifact_paths: "gpu_aquaplanet_dyamond_ss_2process/output_active/*" @@ -189,7 +183,7 @@ steps: - mkdir -p gpu_aquaplanet_dyamond_ss_4process - > srun --cpu-bind=threads --cpus-per-task=4 - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ss.yml --job_id gpu_aquaplanet_dyamond_ss_4process artifact_paths: "gpu_aquaplanet_dyamond_ss_4process/output_active/*" @@ -210,7 +204,7 @@ steps: command: - mkdir -p gpu_aquaplanet_dyamond_ss - > - julia --color=yes --project=examples post_processing/plot_gpu_strong_scaling.jl gpu_aquaplanet_dyamond_ss + julia --color=yes --project=.buildkite post_processing/plot_gpu_strong_scaling.jl gpu_aquaplanet_dyamond_ss artifact_paths: "gpu_aquaplanet_dyamond_ss/*" env: CLIMACOMMS_CONTEXT: "MPI" @@ -227,7 +221,7 @@ steps: - mkdir -p gpu_aquaplanet_dyamond_ws_1process - > srun --cpu-bind=threads --cpus-per-task=4 - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ws_1process.yml --job_id gpu_aquaplanet_dyamond_ws_1process artifact_paths: "gpu_aquaplanet_dyamond_ws_1process/output_active/*" @@ -247,7 +241,7 @@ steps: - mkdir -p gpu_aquaplanet_dyamond_ws_2process - > srun --cpu-bind=threads --cpus-per-task=4 - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ws_2process.yml --job_id gpu_aquaplanet_dyamond_ws_2process artifact_paths: "gpu_aquaplanet_dyamond_ws_2process/output_active/*" @@ -267,7 +261,7 @@ steps: - mkdir -p gpu_aquaplanet_dyamond_ws_4process - > srun --cpu-bind=threads --cpus-per-task=4 - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ws_4process.yml --job_id gpu_aquaplanet_dyamond_ws_4process artifact_paths: "gpu_aquaplanet_dyamond_ws_4process/output_active/*" @@ -288,7 +282,7 @@ steps: command: - mkdir -p gpu_aquaplanet_dyamond_ws - > - julia --color=yes --project=examples post_processing/plot_gpu_weak_scaling.jl gpu_aquaplanet_dyamond_ws + julia --color=yes --project=.buildkite post_processing/plot_gpu_weak_scaling.jl gpu_aquaplanet_dyamond_ws artifact_paths: "gpu_aquaplanet_dyamond_ws/*" env: CLIMACOMMS_DEVICE: "CUDA" @@ -308,7 +302,7 @@ steps: - mkdir -p gpu_aquaplanet_diagedmf - > nsys profile --delay 200 --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_diagedmf/output_active/report - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${MODEL_CONFIG_PATH}aquaplanet_diagedmf.yml --job_id gpu_aquaplanet_diagedmf artifact_paths: "gpu_aquaplanet_diagedmf/output_active/*" @@ -324,7 +318,7 @@ steps: - label: "gpu_aquaplanet_diagedmf_benchmark" command: > - julia --color=yes --project=perf perf/benchmark.jl + julia --color=yes --project=.buildkite perf/benchmark.jl --config_file ${MODEL_CONFIG_PATH}aquaplanet_diagedmf.yml --job_id gpu_aquaplanet_diagedmf_benchmark artifact_paths: "gpu_aquaplanet_diagedmf_benchmark/output_active/*" @@ -342,7 +336,7 @@ steps: - mkdir -p gpu_aquaplanet_progedmf - > nsys profile --delay 100 --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_progedmf/output_active/report - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${MODEL_CONFIG_PATH}aquaplanet_progedmf.yml --job_id gpu_aquaplanet_progedmf artifact_paths: "gpu_aquaplanet_progedmf/output_active/*" @@ -358,7 +352,7 @@ steps: - label: "gpu_aquaplanet_progedmf_benchmark" command: > - julia --color=yes --project=perf perf/benchmark.jl + julia --color=yes --project=.buildkite perf/benchmark.jl --config_file ${MODEL_CONFIG_PATH}aquaplanet_progedmf.yml --job_id gpu_aquaplanet_progedmf_benchmark artifact_paths: "gpu_aquaplanet_progedmf_benchmark/output_active/*" diff --git a/.buildkite/longruns_gpu/pipeline.yml b/.buildkite/longruns_gpu/pipeline.yml index 206ce0885ff..598abf8ef07 100644 --- a/.buildkite/longruns_gpu/pipeline.yml +++ b/.buildkite/longruns_gpu/pipeline.yml @@ -21,13 +21,13 @@ steps: key: "init_gpu_env" command: - echo "--- Instantiate examples" - - julia --project=examples -e 'using Pkg; Pkg.instantiate(;verbose=true)' - - julia --project=examples -e 'using Pkg; Pkg.precompile()' - - julia --project=examples -e 'using CUDA; CUDA.precompile_runtime()' - - julia --project=examples -e 'using Pkg; Pkg.status()' + - julia --project=.buildkite -e 'using Pkg; Pkg.instantiate(;verbose=true)' + - julia --project=.buildkite -e 'using Pkg; Pkg.precompile()' + - julia --project=.buildkite -e 'using CUDA; CUDA.precompile_runtime()' + - julia --project=.buildkite -e 'using Pkg; Pkg.status()' - echo "--- Download artifacts" - - julia --project=examples artifacts/download_artifacts.jl + - julia --project=.buildkite artifacts/download_artifacts.jl agents: slurm_gpus: 1 @@ -43,7 +43,7 @@ steps: - label: ":computer: hydrostatic balance (ρe_tot)" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -54,7 +54,7 @@ steps: - label: ":computer: dry baroclinic wave" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -65,7 +65,7 @@ steps: - label: ":computer: dry baroclinic wave high res" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -76,7 +76,7 @@ steps: - label: ":computer: baroclinic wave equilmoist" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -87,7 +87,7 @@ steps: - label: ":computer: baroclinic wave equilmoist high res" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -98,7 +98,7 @@ steps: - label: ":computer: dry held-suarez" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -109,7 +109,7 @@ steps: - label: ":computer: held-suarez, equilmoist" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -120,7 +120,7 @@ steps: - label: ":computer: held-suarez equilmoist + deep-atmosphere eqns" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" env: CLIMACOMMS_DEVICE: "CUDA" @@ -134,7 +134,7 @@ steps: - label: ":computer: aquaplanet equilmoist allsky radiation + 0M microphysics" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -145,7 +145,7 @@ steps: - label: ":computer: aquaplanet equilmoist allsky radiation + diagnostic edmf + 0M microphysics" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -156,7 +156,7 @@ steps: - label: ":computer: aquaplanet equilmoist allsky radiation + prognostic edmf diffusion only + 0M microphysics" command: - - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -167,7 +167,7 @@ steps: - label: ":computer: aquaplanet equilmoist allsky radiation + 0M microphysics + earth topography" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -178,7 +178,7 @@ steps: - label: ":umbrella: aquaplanet equilmoist allsky radiation + 1M microphysics" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -189,7 +189,7 @@ steps: - label: ":umbrella: aquaplanet non-equilmoist allsky radiation + 1M microphysics" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -200,7 +200,7 @@ steps: - label: ":computer: aquaplanet equilmoist allsky radiation + time-varying insolation + 0M microphysics + slab ocean" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -215,7 +215,7 @@ steps: - label: ":computer: aquaplanet dyamond" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -230,7 +230,7 @@ steps: - label: ":computer: amip target diagnostic edmf" command: - - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + - srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME artifact_paths: "$$JOB_NAME/output_active/*" agents: slurm_gpus: 1 @@ -246,7 +246,7 @@ steps: # TODO: uncomment when zalesak works # - label: ":computer: lim ARS zalesak baroclinic wave (ρe_tot) equilmoist high resolution" # command: - # - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME + # - srun julia --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME # artifact_paths: "$$JOB_NAME/output_active/*" # env: # CLIMACOMMS_CONTEXT: "MPI" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2f4b200b4b1..4c8e155b262 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -28,16 +28,11 @@ steps: - "julia --project -e 'using Pkg; Pkg.precompile()'" - "julia --project -e 'using Pkg; Pkg.status()'" - - echo "--- Instantiate examples" - - "julia --project=examples -e 'using Pkg; Pkg.instantiate(;verbose=true)'" - - "julia --project=examples -e 'using Pkg; Pkg.precompile()'" - - "julia --project=examples -e 'using CUDA; CUDA.precompile_runtime()'" - - "julia --project=examples -e 'using Pkg; Pkg.status()'" - - - echo "--- Instantiate perf" - - "julia --project=perf -e 'using Pkg; Pkg.instantiate(;verbose=true)'" - - "julia --project=perf -e 'using Pkg; Pkg.precompile()'" - - "julia --project=perf -e 'using Pkg; Pkg.status()'" + - echo "--- Instantiate .buildkite" + - "julia --project=.buildkite -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + - "julia --project=.buildkite -e 'using Pkg; Pkg.precompile()'" + - "julia --project=.buildkite -e 'using CUDA; CUDA.precompile_runtime()'" + - "julia --project=.buildkite -e 'using Pkg; Pkg.status()'" - echo "--- Instantiate calibration/test" - "julia --project=calibration/test -e 'using Pkg; Pkg.develop(;path=\".\"); Pkg.instantiate(;verbose=true)'" @@ -45,7 +40,7 @@ steps: - "julia --project=calibration/test -e 'using Pkg; Pkg.status()'" - echo "--- Download artifacts" - - "julia --project=examples artifacts/download_artifacts.jl" + - "julia --project=.buildkite artifacts/download_artifacts.jl" agents: slurm_cpus_per_task: 8 @@ -60,35 +55,35 @@ steps: steps: - label: ":computer: Test reproducibility infrastructure" - command: "julia --color=yes --project=examples test/unit_reproducibility_infra.jl" + command: "julia --color=yes --project=.buildkite test/unit_reproducibility_infra.jl" - group: "Radiation" steps: - label: ":computer: single column radiative equilibrium gray" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/single_column_radiative_equilibrium_gray.yml --job_id single_column_radiative_equilibrium_gray artifact_paths: "single_column_radiative_equilibrium_gray/output_active/*" - label: ":computer: single column radiative equilibrium clearsky" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/single_column_radiative_equilibrium_clearsky.yml --job_id single_column_radiative_equilibrium_clearsky artifact_paths: "single_column_radiative_equilibrium_clearsky/output_active/*" - label: ":computer: single column radiative equilibrium clearsky prognostic surface temperature" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/single_column_radiative_equilibrium_clearsky_prognostic_surface_temp.yml --job_id single_column_radiative_equilibrium_clearsky_prognostic_surface_temp artifact_paths: "single_column_radiative_equilibrium_clearsky_prognostic_surface_temp/output_active/*" - label: ":computer: single column radiative equilibrium allsky idealized clouds varying insolation" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/single_column_radiative_equilibrium_allsky_idealized_clouds.yml --job_id single_column_radiative_equilibrium_allsky_idealized_clouds artifact_paths: "single_column_radiative_equilibrium_allsky_idealized_clouds/output_active/*" @@ -97,7 +92,7 @@ steps: steps: - label: ":umbrella: 1-moment precipitation sanity test single column" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/single_column_precipitation_test.yml --job_id single_column_precipitation_test artifact_paths: "single_column_precipitation_test/output_active/*" @@ -106,32 +101,32 @@ steps: steps: - label: ":computer: non-orographic gravity wave parameterization unit test 3d" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_3d.jl" + command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_3d.jl" artifact_paths: "nonorographic_gravity_wave_test_3d/*" agents: slurm_mem: 20GB - label: ":computer: non-orographic gravity wave parameterization test with MiMA output" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_mima.jl" + command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_mima.jl" artifact_paths: "nonorographic_gravity_wave_test_mima/*" agents: slurm_mem: 20GB - label: ":computer: non-orographic gravity wave parameterization unit test single column" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_single_column.jl" + command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_single_column.jl" artifact_paths: "nonorographic_gravity_wave_test_single_column/*" - label: ":computer: orographic gravity wave parameterization unit test for base flux calculation" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_baseflux.jl" + command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_baseflux.jl" artifact_paths: "orographic_gravity_wave_test_baseflux/*" - label: ":computer: orographic gravity wave parameterization unit test for 3d calculation" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_3d.jl" + command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_3d.jl" artifact_paths: "orographic_gravity_wave_test_3d/*" - label: ":computer: single column non-orographic gravity wave parameterization" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/single_column_nonorographic_gravity_wave.yml --job_id single_column_nonorographic_gravity_wave artifact_paths: "single_column_nonorographic_gravity_wave/*" @@ -141,11 +136,11 @@ steps: - label: ":computer: single column hydrostatic balance float64" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/single_column_hydrostatic_balance_ft64.yml --job_id single_column_hydrostatic_balance_ft64 - julia --color=yes --project=examples reproducibility_tests/test_mse.jl + julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl --job_id single_column_hydrostatic_balance_ft64 --out_dir single_column_hydrostatic_balance_ft64/output_active artifact_paths: "single_column_hydrostatic_balance_ft64/output_active/*" @@ -155,21 +150,21 @@ steps: - label: ":computer: Box hydrostatic balance (ρe_tot)" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/box_hydrostatic_balance_rhoe.yml --job_id box_hydrostatic_balance_rhoe artifact_paths: "box_hydrostatic_balance_rhoe/output_active/*" - label: ":computer: 3D density current" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/box_density_current_test.yml --job_id box_density_current_test artifact_paths: "box_density_current_test/output_active/*" - label: ":computer: Box (ρe_tot) rcemipii with diagnostic edmf" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/rcemipii_box_diagnostic_edmfx.yml --job_id rcemipii_box_diagnostic_edmfx artifact_paths: "rcemipii_box_diagnostic_edmfx/output_active/*" @@ -178,7 +173,7 @@ steps: - label: ":genie: LES ISDAC in a box" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/les_isdac_box.yml --job_id les_isdac_box artifact_paths: "les_isdac_box/*" @@ -193,21 +188,21 @@ steps: steps: - label: ":computer: Agnesi linear hydrostatic mountain experiment (uniform)" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/plane_agnesi_mountain_test_uniform.yml --job_id plane_agnesi_mountain_test_uniform artifact_paths: "plane_agnesi_mountain_test_uniform/output_active/*" - label: ":computer: Agnesi linear hydrostatic mountain experiment (stretched)" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/plane_agnesi_mountain_test_stretched.yml --job_id plane_agnesi_mountain_test_stretched artifact_paths: "plane_agnesi_mountain_test_stretched/output_active/*" - label: ":computer: Density current experiment" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/plane_density_current_test.yml --job_id plane_density_current_test artifact_paths: "plane_density_current_test/output_active/*" @@ -218,25 +213,25 @@ steps: - label: ":computer: baroclinic wave check conservation" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_conservation.yml + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_conservation.yml --job_id sphere_baroclinic_wave_conservation artifact_paths: "sphere_baroclinic_wave_conservation/output_active/*" - label: ":computer: baroclinic wave moist check conservation" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_equilmoist_conservation.yml + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_equilmoist_conservation.yml --job_id sphere_baroclinic_wave_equilmoist_conservation artifact_paths: "sphere_baroclinic_wave_equilmoist_conservation/output_active/*" - label: ":computer: baroclinic wave moist check conservation float64" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_equilmoist_conservation_ft64.yml + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_equilmoist_conservation_ft64.yml --job_id sphere_baroclinic_wave_equilmoist_conservation_ft64 artifact_paths: "sphere_baroclinic_wave_equilmoist_conservation_ft64/output_active/*" - label: ":computer: baroclinic wave moist check conservation with sources" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_equilmoist_conservation_source.yml + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_equilmoist_conservation_source.yml --job_id sphere_baroclinic_wave_equilmoist_conservation_source artifact_paths: "sphere_baroclinic_wave_equilmoist_conservation_source/output_active/*" @@ -245,7 +240,7 @@ steps: - label: ":computer: hydrostatic balance (ρe) float64" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_hydrostatic_balance_rhoe_ft64.yml --job_id sphere_hydrostatic_balance_rhoe_ft64 artifact_paths: "sphere_hydrostatic_balance_rhoe_ft64/output_active/*" @@ -255,18 +250,18 @@ steps: - label: ":computer: baroclinic wave (ρe)" key: sphere_baroclinic_wave_rhoe command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe.yml --job_id sphere_baroclinic_wave_rhoe artifact_paths: "sphere_baroclinic_wave_rhoe/output_active/*" - label: ":computer: no lim baroclinic wave (ρe) equilmoist" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_equilmoist.yml --job_id sphere_baroclinic_wave_rhoe_equilmoist - julia --color=yes --project=examples reproducibility_tests/test_mse.jl + julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl --job_id sphere_baroclinic_wave_rhoe_equilmoist --out_dir sphere_baroclinic_wave_rhoe_equilmoist/output_active artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist/output_active/*" @@ -275,11 +270,11 @@ steps: - label: ":computer: no lim baroclinic wave (ρe) equilmoist (deep sphere)" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/deep_sphere_baroclinic_wave_rhoe_equilmoist.yml --job_id deep_sphere_baroclinic_wave_rhoe_equilmoist - julia --color=yes --project=examples reproducibility_tests/test_mse.jl + julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl --job_id deep_sphere_baroclinic_wave_rhoe_equilmoist --out_dir deep_sphere_baroclinic_wave_rhoe_equilmoist/output_active artifact_paths: "deep_sphere_baroclinic_wave_rhoe_equilmoist/output_active/*" @@ -289,7 +284,7 @@ steps: # Add this back when we figure out what to do with SSP and zalesak # - label: ":computer: SSP zalesak tracer & energy upwind baroclinic wave (ρe_tot) equilmoist" # command: > - # julia --color=yes --project=examples examples/hybrid/driver.jl + # julia --color=yes --project=.buildkite examples/hybrid/driver.jl # --config_file $CONFIG_PATH/$$JOB_NAME.yml # --job_id JOB_NAME # artifact_paths: "$$JOB_NAME/output_active/*" @@ -301,18 +296,18 @@ steps: - label: ":computer: held suarez (ρe) hightop" key: sphere_held_suarez_rhoe_hightop command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_hightop.yml --job_id sphere_held_suarez_rhoe_hightop artifact_paths: "sphere_held_suarez_rhoe_hightop/output_active/*" - label: ":computer: no lim held suarez (ρe) equilmoist hightop sponge" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_equilmoist_hightop_sponge.yml --job_id sphere_held_suarez_rhoe_equilmoist_hightop_sponge - julia --color=yes --project=examples reproducibility_tests/test_mse.jl + julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl --job_id sphere_held_suarez_rhoe_equilmoist_hightop_sponge --out_dir sphere_held_suarez_rhoe_equilmoist_hightop_sponge/output_active artifact_paths: "sphere_held_suarez_rhoe_equilmoist_hightop_sponge/output_active/*" @@ -324,11 +319,11 @@ steps: - label: ":umbrella: aquaplanet (ρe_tot) equil allsky monin_obukhov varying insol gravity wave (gfdl_restart) high top 1-moment" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res.yml --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res - julia --color=yes --project=examples reproducibility_tests/test_mse.jl + julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res --out_dir sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res/output_active artifact_paths: "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res/output_active/*" @@ -337,11 +332,11 @@ steps: - label: ":computer: aquaplanet (ρe_tot) equil allsky monin_obukhov varying insol gravity wave (raw_topo) high top zonally asymmetric" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric.yml --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric - julia --color=yes --project=examples reproducibility_tests/test_mse.jl + julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric --out_dir sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric/output_active artifact_paths: "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric/output_active/*" @@ -351,7 +346,7 @@ steps: - label: ":computer: aquaplanet (ρe_tot) nonequilmoist allsky radiation monin_obukhov varying insolation high top 1-moment" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_nonequilmoist_allsky.yml --job_id sphere_aquaplanet_rhoe_nonequilmoist_allsky artifact_paths: "sphere_aquaplanet_rhoe_nonequilmoist_allsky/output_active/*" @@ -360,7 +355,7 @@ steps: - label: ":computer: aquaplanet (ρe_tot) slabocean equilmoist allsky radiation monin_obukhov varying insolation" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean.yml --job_id aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean artifact_paths: "aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean/output_active/*" @@ -369,7 +364,7 @@ steps: - label: ":computer: aquaplanet (ρe_tot) slabocean equilmoist allsky radiation monin_obukhov varying insolation float64" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean_ft64.yml --job_id aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean_ft64 artifact_paths: "aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean_ft64/output_active/*" @@ -378,7 +373,7 @@ steps: - label: ":computer: aquaplanet (ρe_tot) rcemipii with diagnostic edmf" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/rcemipii_sphere_diagnostic_edmfx.yml --job_id rcemipii_sphere_diagnostic_edmfx artifact_paths: "rcemipii_sphere_diagnostic_edmfx/output_active/*" @@ -390,21 +385,21 @@ steps: - label: ":computer: baroclinic wave (ρe) topography (dcmip)" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_topography_dcmip_rs.yml --job_id sphere_baroclinic_wave_rhoe_topography_dcmip_rs artifact_paths: "sphere_baroclinic_wave_rhoe_topography_dcmip_rs/output_active/*" - label: ":computer: Diagnostic Earth surface elevation spectra" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth.yml --job_id sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth artifact_paths: "sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth/output_active/*" - label: ":computer: Baroclinic wave (ρe) equilmoist (Hughes2023 double mountain config)" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_hughes2023.yml --job_id sphere_baroclinic_wave_rhoe_hughes2023 artifact_paths: "sphere_baroclinic_wave_rhoe_hughes2023/output_active/*" @@ -414,13 +409,13 @@ steps: - label: ":computer: test restart" command: > - julia --color=yes --project=examples test/restart.jl + julia --color=yes --project=.buildkite test/restart.jl agents: slurm_mem: 16GB - label: ":computer: test restart GPU" command: > - julia --color=yes --project=examples test/restart.jl + julia --color=yes --project=.buildkite test/restart.jl env: CLIMACOMMS_DEVICE: "CUDA" agents: @@ -429,13 +424,13 @@ steps: - label: ":computer: test restart" command: > - julia --color=yes --project=examples test/restart.jl --manytests true + julia --color=yes --project=.buildkite test/restart.jl --manytests true agents: slurm_mem: 20GB - label: ":computer: test restart GPU" command: > - julia --color=yes --project=examples test/restart.jl --manytests true + julia --color=yes --project=.buildkite test/restart.jl --manytests true env: CLIMACOMMS_DEVICE: "CUDA" agents: @@ -446,7 +441,7 @@ steps: # lost because of timeout - label: ":computer: test restart MPI" command: > - srun julia --color=yes --project=examples test/restart.jl + srun julia --color=yes --project=.buildkite test/restart.jl env: CLIMACOMMS_CONTEXT: "MPI" timeout_in_minutes: 20 @@ -461,7 +456,7 @@ steps: # lost because of timeout - label: ":computer: test restart GPU MPI" command: > - srun julia --color=yes --project=examples test/restart.jl + srun julia --color=yes --project=.buildkite test/restart.jl env: CLIMACOMMS_CONTEXT: "MPI" CLIMACOMMS_DEVICE: "CUDA" @@ -480,7 +475,7 @@ steps: - label: ":computer: Prep restart for MPI" key: "mpi_baro_wave_make_restart" command: > - srun julia --color=yes --project=examples examples/hybrid/driver.jl + srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $MPI_CONFIG_PATH/mpi_make_restart.yml --job_id mpi_make_restart artifact_paths: "mpi_make_restart/output_active/*" @@ -496,7 +491,7 @@ steps: command: > tar xvf mpi_make_restart/output_active/hdf5_files.tar -C mpi_make_restart/output_active - srun julia --color=yes --project=examples examples/hybrid/driver.jl + srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $MPI_CONFIG_PATH/restart_mpi_baroclinic_wave_rhoe.yml --job_id restart_mpi_baroclinic_wave_rhoe artifact_paths: "restart_mpi_baroclinic_wave_rhoe/output_active/*" @@ -512,7 +507,7 @@ steps: - label: ":computer: MPI no lim aquaplanet (ρe) equilmoist clearsky radiation" command: > - srun julia --color=yes --project=examples examples/hybrid/driver.jl + srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $MPI_CONFIG_PATH/mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky.yml --job_id mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky artifact_paths: "mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky/output_active/*" @@ -525,7 +520,7 @@ steps: - label: ":computer: Prep for calling remap pipeline" key: "prep_remap" command: > - srun julia --color=yes --project=examples examples/hybrid/driver.jl + srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $MPI_CONFIG_PATH/prep_remap.yml --job_id prep_remap artifact_paths: "prep_remap/output_active/*" @@ -540,7 +535,7 @@ steps: command: > tar xvf prep_remap/output_active/hdf5_files.tar -C prep_remap/output_active - julia --color=yes --project=examples post_processing/remap/remap_pipeline.jl + julia --color=yes --project=.buildkite post_processing/remap/remap_pipeline.jl --data_dir prep_remap/output_active --out_dir remap_pipeline_output artifact_paths: "remap_pipeline_output/*" @@ -550,7 +545,7 @@ steps: # TODO: we should somehow decouple this unit test from the perf env / scripts - label: ":computer: checkbounds" command: > - julia --color=yes --check-bounds=yes --project=perf perf/benchmark.jl + julia --color=yes --check-bounds=yes --project=.buildkite perf/benchmark.jl --config_file $PERF_CONFIG_PATH/checkbounds.yml --job_id checkbounds artifact_paths: "checkbounds/output_active/*" @@ -575,7 +570,7 @@ steps: - label: ":genie: Diagnostic EDMFX test in a box" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/diagnostic_edmfx_test_box.yml --job_id diagnostic_edmfx_test_box artifact_paths: "diagnostic_edmfx_test_box/output_active/*" @@ -584,7 +579,7 @@ steps: - label: ":genie: Diagnostic EDMFX GABLS in a box" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/diagnostic_edmfx_gabls_box.yml --job_id diagnostic_edmfx_gabls_box artifact_paths: "diagnostic_edmfx_gabls_box/output_active/*" @@ -593,7 +588,7 @@ steps: - label: ":genie: Diagnostic EDMFX Bomex in a box" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/diagnostic_edmfx_bomex_box.yml --job_id diagnostic_edmfx_bomex_box artifact_paths: "diagnostic_edmfx_bomex_box/output_active/*" @@ -602,7 +597,7 @@ steps: - label: ":genie: Diagnostic EDMFX Bomex stretched grid in a box" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/diagnostic_edmfx_bomex_stretched_box.yml --job_id diagnostic_edmfx_bomex_stretched_box artifact_paths: "diagnostic_edmfx_bomex_stretched_box/output_active/*" @@ -611,7 +606,7 @@ steps: - label: ":genie: Diagnostic EDMFX DYCOMS_RF01 in a box (explicit)" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf01_explicit_box.yml --job_id diagnostic_edmfx_dycoms_rf01_explicit_box artifact_paths: "diagnostic_edmfx_dycoms_rf01_explicit_box/output_active/*" @@ -620,7 +615,7 @@ steps: - label: ":genie: Diagnostic EDMFX DYCOMS_RF01 in a box" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf01_box.yml --job_id diagnostic_edmfx_dycoms_rf01_box artifact_paths: "diagnostic_edmfx_dycoms_rf01_box/output_active/*" @@ -629,7 +624,7 @@ steps: - label: ":umbrella: Diagnostic EDMFX DYCOMS_RF02 in a box" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf02_box.yml --job_id diagnostic_edmfx_dycoms_rf02_box artifact_paths: "diagnostic_edmfx_dycoms_rf02_box/output_active/*" @@ -638,7 +633,7 @@ steps: - label: ":umbrella: Diagnostic EDMFX Rico in a box" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/diagnostic_edmfx_rico_box.yml --job_id diagnostic_edmfx_rico_box artifact_paths: "diagnostic_edmfx_rico_box/output_active/*" @@ -647,7 +642,7 @@ steps: - label: ":umbrella: Diagnostic EDMFX TRMM in a box" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_box.yml --job_id diagnostic_edmfx_trmm_box artifact_paths: "diagnostic_edmfx_trmm_box/output_active/*" @@ -656,7 +651,7 @@ steps: - label: ":umbrella: Diagnostic EDMFX TRMM stretched grid in a box" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_stretched_box.yml --job_id diagnostic_edmfx_trmm_stretched_box artifact_paths: "diagnostic_edmfx_trmm_stretched_box/output_active/*" @@ -665,7 +660,7 @@ steps: - label: ":genie: Diagnostic EDMFX TRMM 0M in a box" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_box_0M.yml --job_id diagnostic_edmfx_trmm_box_0M artifact_paths: "diagnostic_edmfx_trmm_box_0M/output_active/*" @@ -674,11 +669,11 @@ steps: - label: ":genie: Diagnostic EDMFX aquaplanet with TKE" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet.yml --job_id diagnostic_edmfx_aquaplanet - julia --color=yes --project=examples reproducibility_tests/test_mse.jl + julia --color=yes --project=.buildkite reproducibility_tests/test_mse.jl --job_id diagnostic_edmfx_aquaplanet --out_dir diagnostic_edmfx_aquaplanet/output_active artifact_paths: "diagnostic_edmfx_aquaplanet/output_active/*" @@ -691,7 +686,7 @@ steps: - label: ":genie: Prognostic EDMFX advection test in a column" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_adv_test_column.yml --job_id prognostic_edmfx_adv_test_column artifact_paths: "prognostic_edmfx_adv_test_column/output_active/*" @@ -701,7 +696,7 @@ steps: - label: ":genie: Prognostic EDMFX simple plume test in a column" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_simpleplume_column.yml --job_id prognostic_edmfx_simpleplume_column artifact_paths: "prognostic_edmfx_simpleplume_column/output_active/*" @@ -710,7 +705,7 @@ steps: - label: ":genie: Prognostic EDMFX GABLS in a column" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_gabls_column.yml --job_id prognostic_edmfx_gabls_column artifact_paths: "prognostic_edmfx_gabls_column/output_active/*" @@ -719,7 +714,7 @@ steps: - label: ":genie: Prognostic EDMFX Bomex in a column (pi group)" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_bomex_pigroup_column.yml --job_id prognostic_edmfx_bomex_pigroup_column artifact_paths: "prognostic_edmfx_bomex_pigroup_column/output_active/*" @@ -728,7 +723,7 @@ steps: - label: ":genie: Prognostic EDMFX Bomex with precribed TKE in a column" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_bomex_fixtke_column.yml --job_id prognostic_edmfx_bomex_fixtke_column artifact_paths: "prognostic_edmfx_bomex_fixtke_column/output_active/*" @@ -737,7 +732,7 @@ steps: - label: ":genie: Prognostic EDMFX Bomex stretched grid in a column" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_bomex_stretched_column.yml --job_id prognostic_edmfx_bomex_stretched_column artifact_paths: "prognostic_edmfx_bomex_stretched_column/output_active/*" @@ -746,7 +741,7 @@ steps: - label: ":genie: Prognostic EDMFX Bomex in a column" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_bomex_column.yml --job_id prognostic_edmfx_bomex_column artifact_paths: "prognostic_edmfx_bomex_column/output_active/*" @@ -755,7 +750,7 @@ steps: - label: ":genie: Prognostic EDMFX Bomex in a column (implicit)" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_bomex_implicit_column.yml --job_id prognostic_edmfx_bomex_column_implicit artifact_paths: "prognostic_edmfx_bomex_column_implicit/output_active/*" @@ -764,7 +759,7 @@ steps: - label: ":genie: Prognostic EDMFX Dycoms RF01 in a column" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_dycoms_rf01_column.yml --job_id prognostic_edmfx_dycoms_rf01_column artifact_paths: "prognostic_edmfx_dycoms_rf01_column/output_active/*" @@ -773,7 +768,7 @@ steps: - label: ":umbrella: Prognostic EDMFX Rico in a column" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_rico_column.yml --job_id prognostic_edmfx_rico_column artifact_paths: "prognostic_edmfx_rico_column/output_active/*" @@ -782,7 +777,7 @@ steps: - label: ":umbrella: Prognostic EDMFX TRMM in a column" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_trmm_column.yml --job_id prognostic_edmfx_trmm_column artifact_paths: "prognostic_edmfx_trmm_column/output_active/*" @@ -791,7 +786,7 @@ steps: - label: ":genie: Prognostic EDMFX TRMM with 0-moment in a column" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_trmm_column_0M.yml --job_id prognostic_edmfx_trmm_column_0M artifact_paths: "prognostic_edmfx_trmm_column_0M/output_active/*" @@ -800,7 +795,7 @@ steps: - label: ":genie: Prognostic EDMFX GCM driven in a column" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_gcmdriven_column.yml --job_id prognostic_edmfx_gcmdriven_column artifact_paths: "prognostic_edmfx_gcmdriven_column/output_active/*" @@ -810,7 +805,7 @@ steps: - label: ":genie: Prognostic EDMFX Bomex in a box" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_bomex_box.yml --job_id prognostic_edmfx_bomex_box artifact_paths: "prognostic_edmfx_bomex_box/*" @@ -819,7 +814,7 @@ steps: - label: ":genie: Prognostic EDMFX aquaplanet" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_aquaplanet.yml --job_id prognostic_edmfx_aquaplanet artifact_paths: "prognostic_edmfx_aquaplanet/output_active/*" @@ -828,7 +823,7 @@ steps: - label: ":genie: Prognostic EDMFX Soares in a column" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_soares_column.yml --job_id prognostic_edmfx_soares_column artifact_paths: "prognostic_edmfx_soares_column/output_active/*" @@ -840,7 +835,7 @@ steps: - label: "GPU:Gravity waves" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_nonorographic_gravity_wave.yml --job_id sphere_nonorographic_gravity_wave artifact_paths: "sphere_nonorographic_gravity_wave/*" @@ -853,7 +848,7 @@ steps: - label: "GPU: baroclinic wave" key: "sphere_baroclinic_wave_rhoe_gpu" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe.yml --job_id sphere_baroclinic_wave_rhoe_gpu artifact_paths: "sphere_baroclinic_wave_rhoe_gpu/output_active/*" @@ -869,7 +864,7 @@ steps: tar xvf sphere_baroclinic_wave_rhoe_gpu/output_active/hdf5_files.tar -C sphere_baroclinic_wave_rhoe_gpu - julia --color=yes --project=examples post_processing/compare_outputs.jl + julia --color=yes --project=.buildkite post_processing/compare_outputs.jl --output_folder_1 sphere_baroclinic_wave_rhoe --output_folder_2 sphere_baroclinic_wave_rhoe_gpu --t_end 10days --compare_state false @@ -883,7 +878,7 @@ steps: - mkdir -p target_gpu_implicit_baroclinic_wave - > nsys profile --delay 100 --trace=nvtx,cuda --output=target_gpu_implicit_baroclinic_wave/output_active/report - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${GPU_CONFIG_PATH}/target_gpu_implicit_baroclinic_wave.yml --job_id target_gpu_implicit_baroclinic_wave artifact_paths: "target_gpu_implicit_baroclinic_wave/output_active/*" @@ -900,12 +895,12 @@ steps: # - > # srun --cpu-bind=threads --cpus-per-task=4 # nsys profile --delay 100 --trace=nvtx,cuda,mpi --output=target_gpu_implicit_baroclinic_wave_4process/output_active/report-%q{PMI_RANK} - # julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + # julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl # --config_file ${GPU_CONFIG_PATH}/target_gpu_implicit_baroclinic_wave.yml # --job_id target_gpu_implicit_baroclinic_wave_4process - > srun --cpu-bind=threads --cpus-per-task=4 - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + julia --threads=3 --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${GPU_CONFIG_PATH}/target_gpu_implicit_baroclinic_wave.yml --job_id target_gpu_implicit_baroclinic_wave_4process artifact_paths: "target_gpu_implicit_baroclinic_wave_4process/output_active/*" @@ -922,7 +917,7 @@ steps: command: - > nsys profile --delay 100 --trace=nvtx,cuda --output=central_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/report - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/central_gpu_hs_rhoe_equil_0M.yml --job_id central_gpu_hs_rhoe_equil_55km_nz63_0M artifact_paths: "central_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*" @@ -936,7 +931,7 @@ steps: command: - > nsys profile --delay 100 --trace=nvtx,cuda --output=central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/report - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/central_cloud_diag_gpu_hs_rhoe_equil_0M.yml --job_id central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M artifact_paths: "central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*" @@ -950,7 +945,7 @@ steps: command: - mkdir -p gpu_aquaplanet_dyamond - > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file ${CONFIG_PATH}/gpu_aquaplanet_dyamond.yml --job_id gpu_aquaplanet_dyamond artifact_paths: "gpu_aquaplanet_dyamond/output_active/*" @@ -962,7 +957,7 @@ steps: - label: "GPU: test DYAMOND interpolated initial conditions" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $GPU_CONFIG_PATH/gpu_aquaplanet_dyamond_summer.yml --job_id gpu_aquaplanet_dyamond_summer artifact_paths: "gpu_aquaplanet_dyamond_summer/output_active/*" @@ -975,7 +970,7 @@ steps: - label: "GPU: Diagnostic EDMFX aquaplanet" key: "diagnostic_edmfx_aquaplanet_gpu" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet_gpu.yml --job_id diagnostic_edmfx_aquaplanet_gpu artifact_paths: "diagnostic_edmfx_aquaplanet_gpu/output_active/*" @@ -987,7 +982,7 @@ steps: - label: "GPU: Prognostic EDMFX aquaplanet" command: > - julia --color=yes --project=examples examples/hybrid/driver.jl + julia --color=yes --project=.buildkite examples/hybrid/driver.jl --config_file $CONFIG_PATH/prognostic_edmfx_aquaplanet.yml --job_id prognostic_edmfx_aquaplanet_gpu artifact_paths: "prognostic_edmfx_aquaplanet_gpu/output_active/*" @@ -1003,7 +998,7 @@ steps: - label: ":computer: Benchmark: GPU implicit baro wave" command: > - julia --color=yes --project=examples perf/benchmark_step.jl + julia --color=yes --project=.buildkite perf/benchmark_step.jl --config_file $PERF_CONFIG_PATH/gpu_implicit_barowave.yml --job_id gpu_implicit_barowave artifact_paths: "gpu_implicit_barowave/output_active/*" @@ -1015,7 +1010,7 @@ steps: - label: ":computer: Benchmark: GPU implicit baro wave moist" command: > - julia --color=yes --project=examples perf/benchmark_step.jl + julia --color=yes --project=.buildkite perf/benchmark_step.jl --config_file $PERF_CONFIG_PATH/gpu_implicit_barowave_moist.yml --job_id gpu_implicit_barowave_moist artifact_paths: "gpu_implicit_barowave_moist/output_active/*" @@ -1027,7 +1022,7 @@ steps: - label: ":computer: Benchmark: CPU implicit baro wave" command: > - julia --color=yes --project=examples perf/benchmark_step.jl + julia --color=yes --project=.buildkite perf/benchmark_step.jl --config_file $PERF_CONFIG_PATH/cpu_implicit_barowave.yml --job_id cpu_implicit_barowave artifact_paths: "cpu_implicit_barowave/output_active/*" @@ -1035,7 +1030,7 @@ steps: # Benchmarks - label: ":computer: Benchmark: CPU perf target (default)" command: > - julia --color=yes --project=perf perf/benchmark.jl + julia --color=yes --project=.buildkite perf/benchmark.jl --config_file $PERF_CONFIG_PATH/bm_perf_target.yml --job_id bm_perf_target agents: @@ -1043,7 +1038,7 @@ steps: - label: ":computer: Benchmark: GPU perf target" command: > - julia --color=yes --project=perf perf/benchmark.jl + julia --color=yes --project=.buildkite perf/benchmark.jl --config_file $PERF_CONFIG_PATH/bm_perf_target.yml --job_id bm_perf_target_gpu env: @@ -1054,7 +1049,7 @@ steps: - label: ":computer: Benchmark: CPU perf target (Threaded)" command: > - julia --color=yes --threads 8 --project=perf perf/benchmark.jl + julia --color=yes --threads 8 --project=.buildkite perf/benchmark.jl --config_file $PERF_CONFIG_PATH/bm_perf_target_threaded.yml --job_id bm_perf_target_threaded agents: @@ -1063,7 +1058,7 @@ steps: - label: ":computer: Benchmark: GPU diag edmf" command: > - julia --color=yes --project=perf perf/benchmark.jl + julia --color=yes --project=.buildkite perf/benchmark.jl --config_file ${CONFIG_PATH}/aquaplanet_diagedmf.yml --job_id bm_diag_edmf_gpu env: @@ -1074,7 +1069,7 @@ steps: - label: ":computer: Benchmark: GPU prog edmf" command: > - julia --color=yes --project=perf perf/benchmark.jl + julia --color=yes --project=.buildkite perf/benchmark.jl --config_file ${CONFIG_PATH}/aquaplanet_progedmf.yml --job_id bm_prog_edmf_gpu env: @@ -1088,7 +1083,7 @@ steps: - label: ":fire: Flame graph: perf target (default)" command: > - julia --color=yes --project=perf perf/flame.jl + julia --color=yes --project=.buildkite perf/flame.jl --config_file $PERF_CONFIG_PATH/flame_perf_target.yml --job_id flame_perf_target artifact_paths: "flame_perf_target/*" @@ -1097,7 +1092,7 @@ steps: - label: ":fire: Flame graph: perf target (with tracers)" command: > - julia --color=yes --project=perf perf/flame.jl + julia --color=yes --project=.buildkite perf/flame.jl --config_file $PERF_CONFIG_PATH/flame_perf_target_tracers.yml --job_id flame_perf_target_tracers artifact_paths: "flame_perf_target_tracers/*" @@ -1106,7 +1101,7 @@ steps: - label: ":fire: Flame graph: perf target (diagnostic edmfx)" command: > - julia --color=yes --project=perf perf/flame.jl + julia --color=yes --project=.buildkite perf/flame.jl --config_file $PERF_CONFIG_PATH/flame_perf_target_diagnostic_edmfx.yml --job_id flame_perf_target_diagnostic_edmfx artifact_paths: "flame_perf_target_diagnostic_edmfx/*" @@ -1115,7 +1110,7 @@ steps: - label: ":fire: Flame graph: perf target (prognostic edmfx aquaplanet)" command: > - julia --color=yes --project=perf perf/flame.jl + julia --color=yes --project=.buildkite perf/flame.jl --config_file $PERF_CONFIG_PATH/flame_perf_target_prognostic_edmfx_aquaplanet.yml --job_id flame_perf_target_prognostic_edmfx_aquaplanet artifact_paths: "flame_perf_target_prognostic_edmfx_aquaplanet/*" @@ -1124,7 +1119,7 @@ steps: - label: ":fire: Flame graph: perf target (frierson diffusion)" command: > - julia --color=yes --project=perf perf/flame.jl + julia --color=yes --project=.buildkite perf/flame.jl --config_file $PERF_CONFIG_PATH/flame_perf_target_frierson.yml --job_id flame_perf_target_frierson artifact_paths: "flame_perf_target_frierson/*" @@ -1133,7 +1128,7 @@ steps: - label: ":fire: Flame graph: perf target (Threaded)" command: > - julia --threads 8 --color=yes --project=perf perf/flame.jl + julia --threads 8 --color=yes --project=.buildkite perf/flame.jl --config_file $PERF_CONFIG_PATH/flame_perf_target_threaded.yml --job_id flame_perf_target_threaded artifact_paths: "flame_perf_target_threaded/*" @@ -1143,7 +1138,7 @@ steps: - label: ":fire: Flame graph: perf target (Callbacks)" command: > - julia --color=yes --project=perf perf/flame.jl + julia --color=yes --project=.buildkite perf/flame.jl --config_file $PERF_CONFIG_PATH/flame_perf_target_callbacks.yml --job_id flame_perf_target_callbacks artifact_paths: "flame_perf_target_callbacks/*" @@ -1152,7 +1147,7 @@ steps: - label: ":fire: Flame graph: gravity wave" command: > - julia --color=yes --project=perf perf/flame.jl + julia --color=yes --project=.buildkite perf/flame.jl --config_file $PERF_CONFIG_PATH/flame_perf_gw.yml --job_id flame_perf_gw artifact_paths: "flame_perf_gw/*" @@ -1161,7 +1156,7 @@ steps: - label: ":fire: Flame graph: perf target (diagnostics)" command: > - julia --color=yes --project=perf perf/flame.jl + julia --color=yes --project=.buildkite perf/flame.jl --config_file $PERF_CONFIG_PATH/flame_perf_diagnostics.yml --job_id flame_perf_diagnostics artifact_paths: "flame_perf_diagnostics/*" @@ -1170,7 +1165,7 @@ steps: - label: ":fire: Flame graph: gpu job" command: > - julia --color=yes --project=perf perf/flame.jl + julia --color=yes --project=.buildkite perf/flame.jl --config_file $PERF_CONFIG_PATH/flame_gpu_implicit_barowave_moist.yml --job_id flame_gpu_implicit_barowave_moist artifact_paths: "flame_gpu_implicit_barowave_moist/*" @@ -1187,7 +1182,7 @@ steps: # Inference - label: ":rocket: JET n-failures (inference)" command: > - julia --color=yes --project=perf perf/jet_test_nfailures.jl + julia --color=yes --project=.buildkite perf/jet_test_nfailures.jl --config_file $PERF_CONFIG_PATH/jet_n_failures.yml --job_id jet_n_failures agents: @@ -1196,7 +1191,7 @@ steps: # Latency - label: ":mag::rocket: Invalidations" command: > - julia --color=yes --project=perf perf/invalidations.jl + julia --color=yes --project=.buildkite perf/invalidations.jl artifact_paths: "invalidations/*" agents: slurm_mem: 24GB @@ -1205,13 +1200,13 @@ steps: continue_on_failure: true - label: ":robot_face: Print new mse tables" - command: "julia --color=yes --project=examples reproducibility_tests/mse_summary.jl" + command: "julia --color=yes --project=.buildkite reproducibility_tests/mse_summary.jl" - label: ":robot_face: Print new reference counter" - command: "julia --color=yes --project=examples reproducibility_tests/print_new_ref_counter.jl" + command: "julia --color=yes --project=.buildkite reproducibility_tests/print_new_ref_counter.jl" - label: ":bar_chart: Tabulate performance summary" - command: "julia --color=yes --project=perf perf/tabulate_perf_summary.jl" + command: "julia --color=yes --project=.buildkite perf/tabulate_perf_summary.jl" # - label: ":chart_with_downwards_trend: build history" # command: @@ -1222,4 +1217,4 @@ steps: - wait - label: ":robot_face: Move main results" - command: "julia --color=yes --project=examples reproducibility_tests/move_output.jl" + command: "julia --color=yes --project=.buildkite reproducibility_tests/move_output.jl" diff --git a/.buildkite/scaling/pipeline.sh b/.buildkite/scaling/pipeline.sh index 0d12baf2698..aa7734ab536 100755 --- a/.buildkite/scaling/pipeline.sh +++ b/.buildkite/scaling/pipeline.sh @@ -13,57 +13,54 @@ mpi_impl="openmpi" parent_folder=scaling_configs mkdir -p "$parent_folder" -low_resolution_lines=\ -"t_end: 10days +low_resolution_lines="t_end: 10days dt: 400secs z_elem: 10 h_elem: 6" -medium_resolution_lines=\ -"t_end: 4days +medium_resolution_lines="t_end: 4days dt: 150secs z_elem: 45 dz_bottom: 30 h_elem: 16" -high_resolution_lines=\ -"t_end: 1days +high_resolution_lines="t_end: 1days dt: 50secs z_elem: 45 h_elem: 30" # Create configuration files for each resolution for i in "${!resolutions[@]}"; do - resolution="${resolutions[$i]}" - proc_counts="${process_counts[$i]}" - - for nprocs in $proc_counts; do - filename="${resolution}_res_${FT}_${nprocs}.yml" - folder_name="${resolution}_res_Float32" - mkdir -p "$parent_folder/$folder_name" - filepath="$parent_folder/$folder_name/$filename" - - echo "job_id: sphere_held_suarez_${resolution}_res_rhoe_${nprocs}" > "$filepath" - echo "forcing: held_suarez" >> "$filepath" - echo "FLOAT_TYPE: $FT" >> "$filepath" - echo "tracer_upwinding: none" >> "$filepath" - - case "$resolution" in - "low") - echo -e "$low_resolution_lines" >> "$filepath" - ;; - "mid") - echo -e "$medium_resolution_lines" >> "$filepath" - ;; - "high") - echo -e "$high_resolution_lines" >> "$filepath" - ;; - esac - done + resolution="${resolutions[$i]}" + proc_counts="${process_counts[$i]}" + + for nprocs in $proc_counts; do + filename="${resolution}_res_${FT}_${nprocs}.yml" + folder_name="${resolution}_res_Float32" + mkdir -p "$parent_folder/$folder_name" + filepath="$parent_folder/$folder_name/$filename" + + echo "job_id: sphere_held_suarez_${resolution}_res_rhoe_${nprocs}" >"$filepath" + echo "forcing: held_suarez" >>"$filepath" + echo "FLOAT_TYPE: $FT" >>"$filepath" + echo "tracer_upwinding: none" >>"$filepath" + + case "$resolution" in + "low") + echo -e "$low_resolution_lines" >>"$filepath" + ;; + "mid") + echo -e "$medium_resolution_lines" >>"$filepath" + ;; + "high") + echo -e "$high_resolution_lines" >>"$filepath" + ;; + esac + done done # set up environment and agents -cat << 'EOM' +cat <<'EOM' agents: queue: new-central modules: climacommon/2024_10_08 @@ -83,9 +80,9 @@ steps: key: "init_cpu_env" command: - echo "--- Instantiate" - - "julia --project=examples -e 'using Pkg; Pkg.instantiate(;verbose=true)'" - - "julia --project=examples -e 'using Pkg; Pkg.precompile()'" - - "julia --project=examples -e 'using Pkg; Pkg.status()'" + - "julia --project=.buildkite -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + - "julia --project=.buildkite -e 'using Pkg; Pkg.precompile()'" + - "julia --project=.buildkite -e 'using Pkg; Pkg.status()'" agents: slurm_cpus_per_task: 8 env: @@ -97,57 +94,56 @@ EOM for i in "${!resolutions[@]}"; do -res="${resolutions[$i]}" -proc_counts="${process_counts[$i]}" + res="${resolutions[$i]}" + proc_counts="${process_counts[$i]}" -cat << EOM + cat <