From bde16cde751f4c64a592035fc23f3910ae08176c Mon Sep 17 00:00:00 2001
From: Kevin Phan <98072684+ph-kev@users.noreply.github.com>
Date: Wed, 29 Jan 2025 17:11:08 -0800
Subject: [PATCH 1/4] Investigating benchmarks

---
 experiments/benchmarks/bucket.jl     | 1 +
 experiments/benchmarks/land.jl       | 1 +
 experiments/benchmarks/richards.jl   | 1 +
 experiments/benchmarks/snowy_land.jl | 1 +
 4 files changed, 4 insertions(+)

diff --git a/experiments/benchmarks/bucket.jl b/experiments/benchmarks/bucket.jl
index 27cdc7b928..5bd909a4b9 100644
--- a/experiments/benchmarks/bucket.jl
+++ b/experiments/benchmarks/bucket.jl
@@ -197,6 +197,7 @@ std_timing_s = round(
 @info "Max time: $(max_timing_s) s"
 @info "Min time: $(min_timing_s) s"
 @info "Standard deviation time: $(std_timing_s) s"
+@info "Timings: $timings_s"
 @info "Done profiling"
 
 prob, ode_algo, Δt, cb = setup_simulation()
diff --git a/experiments/benchmarks/land.jl b/experiments/benchmarks/land.jl
index b8e08254ba..667ab54053 100644
--- a/experiments/benchmarks/land.jl
+++ b/experiments/benchmarks/land.jl
@@ -418,6 +418,7 @@ std_timing_s = round(
 @info "Max time: $(max_timing_s) s"
 @info "Min time: $(min_timing_s) s"
 @info "Standard deviation time: $(std_timing_s) s"
+@info "Timings: $timings_s"
 @info "Done profiling"
 
 prob, ode_algo, Δt, cb = setup_simulation()
diff --git a/experiments/benchmarks/richards.jl b/experiments/benchmarks/richards.jl
index 5731546629..ea7ab69b28 100644
--- a/experiments/benchmarks/richards.jl
+++ b/experiments/benchmarks/richards.jl
@@ -248,6 +248,7 @@ std_timing_s = round(
 @info "Max time: $(max_timing_s) s"
 @info "Min time: $(min_timing_s) s"
 @info "Standard deviation time: $(std_timing_s) s"
+@info "Timings: $timings_s"
 @info "Done profiling"
 
 prob, ode_algo, Δt, cb = setup_simulation()
diff --git a/experiments/benchmarks/snowy_land.jl b/experiments/benchmarks/snowy_land.jl
index 16fc55eee2..a0e1d88430 100644
--- a/experiments/benchmarks/snowy_land.jl
+++ b/experiments/benchmarks/snowy_land.jl
@@ -429,6 +429,7 @@ std_timing_s = round(
 @info "Max time: $(max_timing_s) s"
 @info "Min time: $(min_timing_s) s"
 @info "Standard deviation time: $(std_timing_s) s"
+@info "Timings: $timings_s"
 @info "Done profiling"
 
 prob, ode_algo, Δt, cb = setup_simulation()

From b8f2c12ac6231b3bf0628f1253b6f003ec18eedc Mon Sep 17 00:00:00 2001
From: Kevin Phan <98072684+ph-kev@users.noreply.github.com>
Date: Thu, 30 Jan 2025 11:33:51 -0800
Subject: [PATCH 2/4] Run: 2

---
 experiments/benchmarks/bucket.jl     | 1 +
 experiments/benchmarks/land.jl       | 1 +
 experiments/benchmarks/richards.jl   | 1 +
 experiments/benchmarks/snowy_land.jl | 1 +
 4 files changed, 4 insertions(+)

diff --git a/experiments/benchmarks/bucket.jl b/experiments/benchmarks/bucket.jl
index 5bd909a4b9..c570204d6e 100644
--- a/experiments/benchmarks/bucket.jl
+++ b/experiments/benchmarks/bucket.jl
@@ -192,6 +192,7 @@ std_timing_s = round(
     sqrt(sum(((timings_s .- average_timing_s) .^ 2) / num_samples)),
     sigdigits = 3,
 )
+# Runs: 2
 @info "Num samples: $num_samples"
 @info "Average time: $(average_timing_s) s"
 @info "Max time: $(max_timing_s) s"
diff --git a/experiments/benchmarks/land.jl b/experiments/benchmarks/land.jl
index 667ab54053..6e308ee775 100644
--- a/experiments/benchmarks/land.jl
+++ b/experiments/benchmarks/land.jl
@@ -413,6 +413,7 @@ std_timing_s = round(
     sqrt(sum(((timings_s .- average_timing_s) .^ 2) / num_samples)),
     sigdigits = 3,
 )
+# Runs: 2
 @info "Num samples: $num_samples"
 @info "Average time: $(average_timing_s) s"
 @info "Max time: $(max_timing_s) s"
diff --git a/experiments/benchmarks/richards.jl b/experiments/benchmarks/richards.jl
index ea7ab69b28..f9c9406733 100644
--- a/experiments/benchmarks/richards.jl
+++ b/experiments/benchmarks/richards.jl
@@ -243,6 +243,7 @@ std_timing_s = round(
     sqrt(sum(((timings_s .- average_timing_s) .^ 2) / num_samples)),
     sigdigits = 3,
 )
+# Runs: 2
 @info "Num samples: $num_samples"
 @info "Average time: $(average_timing_s) s"
 @info "Max time: $(max_timing_s) s"
diff --git a/experiments/benchmarks/snowy_land.jl b/experiments/benchmarks/snowy_land.jl
index a0e1d88430..2be42ca5a0 100644
--- a/experiments/benchmarks/snowy_land.jl
+++ b/experiments/benchmarks/snowy_land.jl
@@ -424,6 +424,7 @@ std_timing_s = round(
     sqrt(sum(((timings_s .- average_timing_s) .^ 2) / num_samples)),
     sigdigits = 3,
 )
+# Runs: 2
 @info "Num samples: $num_samples"
 @info "Average time: $(average_timing_s) s"
 @info "Max time: $(max_timing_s) s"

From 356ca4907d153f896483bc9233f61f2482a336a0 Mon Sep 17 00:00:00 2001
From: Kevin Phan <98072684+ph-kev@users.noreply.github.com>
Date: Fri, 31 Jan 2025 10:24:28 -0800
Subject: [PATCH 3/4] Run 3

---
 experiments/benchmarks/bucket.jl     | 4 +++-
 experiments/benchmarks/land.jl       | 4 +++-
 experiments/benchmarks/richards.jl   | 4 +++-
 experiments/benchmarks/snowy_land.jl | 4 +++-
 4 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/experiments/benchmarks/bucket.jl b/experiments/benchmarks/bucket.jl
index c570204d6e..f83d45e4f8 100644
--- a/experiments/benchmarks/bucket.jl
+++ b/experiments/benchmarks/bucket.jl
@@ -171,9 +171,11 @@ MAX_PROFILING_TIME_SECONDS = 500
 MAX_PROFILING_SAMPLES = 100
 time_now = time()
 timings_s = Float64[]
+ClimaComms.device() isa ClimaComms.CUDADevice && import CUDA
 while (time() - time_now) < MAX_PROFILING_TIME_SECONDS &&
     length(timings_s) < MAX_PROFILING_SAMPLES
     lprob, lode_algo, lΔt, lcb = setup_simulation()
+    ClimaComms.device() isa ClimaComms.CUDADevice && CUDA.device_synchronize()
     push!(
         timings_s,
         ClimaComms.@elapsed device SciMLBase.solve(
@@ -192,7 +194,7 @@ std_timing_s = round(
     sqrt(sum(((timings_s .- average_timing_s) .^ 2) / num_samples)),
     sigdigits = 3,
 )
-# Runs: 2
+# Runs: 3
 @info "Num samples: $num_samples"
 @info "Average time: $(average_timing_s) s"
 @info "Max time: $(max_timing_s) s"
diff --git a/experiments/benchmarks/land.jl b/experiments/benchmarks/land.jl
index 6e308ee775..5e3eeaab01 100644
--- a/experiments/benchmarks/land.jl
+++ b/experiments/benchmarks/land.jl
@@ -392,9 +392,11 @@ MAX_PROFILING_TIME_SECONDS = 500
 MAX_PROFILING_SAMPLES = 100
 time_now = time()
 timings_s = Float64[]
+ClimaComms.device() isa ClimaComms.CUDADevice && import CUDA
 while (time() - time_now) < MAX_PROFILING_TIME_SECONDS &&
     length(timings_s) < MAX_PROFILING_SAMPLES
     lprob, lode_algo, lΔt, lcb = setup_simulation()
+    ClimaComms.device() isa ClimaComms.CUDADevice && CUDA.device_synchronize()
     push!(
         timings_s,
         ClimaComms.@elapsed device SciMLBase.solve(
@@ -413,7 +415,7 @@ std_timing_s = round(
     sqrt(sum(((timings_s .- average_timing_s) .^ 2) / num_samples)),
     sigdigits = 3,
 )
-# Runs: 2
+# Runs: 3
 @info "Num samples: $num_samples"
 @info "Average time: $(average_timing_s) s"
 @info "Max time: $(max_timing_s) s"
diff --git a/experiments/benchmarks/richards.jl b/experiments/benchmarks/richards.jl
index f9c9406733..23651cd17a 100644
--- a/experiments/benchmarks/richards.jl
+++ b/experiments/benchmarks/richards.jl
@@ -222,9 +222,11 @@ MAX_PROFILING_TIME_SECONDS = 500
 MAX_PROFILING_SAMPLES = 100
 time_now = time()
 timings_s = Float64[]
+ClimaComms.device() isa ClimaComms.CUDADevice && import CUDA
 while (time() - time_now) < MAX_PROFILING_TIME_SECONDS &&
     length(timings_s) < MAX_PROFILING_SAMPLES
     lprob, lode_algo, lΔt, lcb = setup_simulation()
+    ClimaComms.device() isa ClimaComms.CUDADevice && CUDA.device_synchronize()
     push!(
         timings_s,
         ClimaComms.@elapsed device SciMLBase.solve(
@@ -243,7 +245,7 @@ std_timing_s = round(
     sqrt(sum(((timings_s .- average_timing_s) .^ 2) / num_samples)),
     sigdigits = 3,
 )
-# Runs: 2
+# Runs: 3
 @info "Num samples: $num_samples"
 @info "Average time: $(average_timing_s) s"
 @info "Max time: $(max_timing_s) s"
diff --git a/experiments/benchmarks/snowy_land.jl b/experiments/benchmarks/snowy_land.jl
index 2be42ca5a0..1c8e11b204 100644
--- a/experiments/benchmarks/snowy_land.jl
+++ b/experiments/benchmarks/snowy_land.jl
@@ -403,9 +403,11 @@ MAX_PROFILING_TIME_SECONDS = 500
 MAX_PROFILING_SAMPLES = 100
 time_now = time()
 timings_s = Float64[]
+ClimaComms.device() isa ClimaComms.CUDADevice && import CUDA
 while (time() - time_now) < MAX_PROFILING_TIME_SECONDS &&
     length(timings_s) < MAX_PROFILING_SAMPLES
     lprob, lode_algo, lΔt, lcb = setup_simulation()
+    ClimaComms.device() isa ClimaComms.CUDADevice && CUDA.device_synchronize()
     push!(
         timings_s,
         ClimaComms.@elapsed device SciMLBase.solve(
@@ -424,7 +426,7 @@ std_timing_s = round(
     sqrt(sum(((timings_s .- average_timing_s) .^ 2) / num_samples)),
     sigdigits = 3,
 )
-# Runs: 2
+# Runs: 3
 @info "Num samples: $num_samples"
 @info "Average time: $(average_timing_s) s"
 @info "Max time: $(max_timing_s) s"

From adce17b05a3150032f53de1a4de323596456c571 Mon Sep 17 00:00:00 2001
From: Kevin Phan <98072684+ph-kev@users.noreply.github.com>
Date: Fri, 31 Jan 2025 10:51:09 -0800
Subject: [PATCH 4/4] Run: 4

---
 experiments/benchmarks/bucket.jl     | 4 ++--
 experiments/benchmarks/land.jl       | 5 +++--
 experiments/benchmarks/richards.jl   | 2 +-
 experiments/benchmarks/snowy_land.jl | 4 ++--
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/experiments/benchmarks/bucket.jl b/experiments/benchmarks/bucket.jl
index f83d45e4f8..6e64ecaa3e 100644
--- a/experiments/benchmarks/bucket.jl
+++ b/experiments/benchmarks/bucket.jl
@@ -194,7 +194,7 @@ std_timing_s = round(
     sqrt(sum(((timings_s .- average_timing_s) .^ 2) / num_samples)),
     sigdigits = 3,
 )
-# Runs: 3
+# Runs: 4
 @info "Num samples: $num_samples"
 @info "Average time: $(average_timing_s) s"
 @info "Max time: $(max_timing_s) s"
@@ -248,7 +248,7 @@ if ClimaComms.device() isa ClimaComms.CUDADevice
 end
 
 if get(ENV, "BUILDKITE_PIPELINE_SLUG", nothing) == "climaland-benchmark"
-    PREVIOUS_BEST_TIME = 0.477
+    PREVIOUS_BEST_TIME = 0.486
     if average_timing_s > PREVIOUS_BEST_TIME + std_timing_s
         @info "Possible performance regression, previous average time was $(PREVIOUS_BEST_TIME)"
     elseif average_timing_s < PREVIOUS_BEST_TIME - std_timing_s
diff --git a/experiments/benchmarks/land.jl b/experiments/benchmarks/land.jl
index 5e3eeaab01..27824de51c 100644
--- a/experiments/benchmarks/land.jl
+++ b/experiments/benchmarks/land.jl
@@ -397,6 +397,7 @@ while (time() - time_now) < MAX_PROFILING_TIME_SECONDS &&
     length(timings_s) < MAX_PROFILING_SAMPLES
     lprob, lode_algo, lΔt, lcb = setup_simulation()
     ClimaComms.device() isa ClimaComms.CUDADevice && CUDA.device_synchronize()
+    ClimaComms.device() isa ClimaComms.CUDADevice && println("This should work!")
     push!(
         timings_s,
         ClimaComms.@elapsed device SciMLBase.solve(
@@ -415,7 +416,7 @@ std_timing_s = round(
     sqrt(sum(((timings_s .- average_timing_s) .^ 2) / num_samples)),
     sigdigits = 3,
 )
-# Runs: 3
+# Runs: 4
 @info "Num samples: $num_samples"
 @info "Average time: $(average_timing_s) s"
 @info "Max time: $(max_timing_s) s"
@@ -469,7 +470,7 @@ if ClimaComms.device() isa ClimaComms.CUDADevice
 end
 
 if get(ENV, "BUILDKITE_PIPELINE_SLUG", nothing) == "climaland-benchmark"
-    PREVIOUS_BEST_TIME = 6.1
+    PREVIOUS_BEST_TIME = 6.35
     if average_timing_s > PREVIOUS_BEST_TIME + std_timing_s
         @info "Possible performance regression, previous average time was $(PREVIOUS_BEST_TIME)"
     elseif average_timing_s < PREVIOUS_BEST_TIME - std_timing_s
diff --git a/experiments/benchmarks/richards.jl b/experiments/benchmarks/richards.jl
index 23651cd17a..10b0da2aea 100644
--- a/experiments/benchmarks/richards.jl
+++ b/experiments/benchmarks/richards.jl
@@ -245,7 +245,7 @@ std_timing_s = round(
     sqrt(sum(((timings_s .- average_timing_s) .^ 2) / num_samples)),
     sigdigits = 3,
 )
-# Runs: 3
+# Runs: 4
 @info "Num samples: $num_samples"
 @info "Average time: $(average_timing_s) s"
 @info "Max time: $(max_timing_s) s"
diff --git a/experiments/benchmarks/snowy_land.jl b/experiments/benchmarks/snowy_land.jl
index 1c8e11b204..7001a55129 100644
--- a/experiments/benchmarks/snowy_land.jl
+++ b/experiments/benchmarks/snowy_land.jl
@@ -426,7 +426,7 @@ std_timing_s = round(
     sqrt(sum(((timings_s .- average_timing_s) .^ 2) / num_samples)),
     sigdigits = 3,
 )
-# Runs: 3
+# Runs: 4
 @info "Num samples: $num_samples"
 @info "Average time: $(average_timing_s) s"
 @info "Max time: $(max_timing_s) s"
@@ -480,7 +480,7 @@ if ClimaComms.device() isa ClimaComms.CUDADevice
 end
 
 if get(ENV, "BUILDKITE_PIPELINE_SLUG", nothing) == "climaland-benchmark"
-    PREVIOUS_BEST_TIME = 6.5
+    PREVIOUS_BEST_TIME = 0.545
     if average_timing_s > PREVIOUS_BEST_TIME + std_timing_s
         @info "Possible performance regression, previous average time was $(PREVIOUS_BEST_TIME)"
     elseif average_timing_s < PREVIOUS_BEST_TIME - std_timing_s