Add tests and plots for steady-state velocity

.buildkite/ci_driver.jl

@@ -28,7 +28,7 @@ sol_res = CA.solve_atmos!(simulation)
 (; p) = integrator
 
 import ClimaCore
-import ClimaCore: Topologies, Quadratures, Spaces
+import ClimaCore: Topologies, Quadratures, Spaces, Fields
 import ClimaComms
 using SciMLBase
 using PrettyTables
@@ -38,6 +38,8 @@ using ClimaTimeSteppers
 using Test
 import Tar
 import Base.Filesystem: rm
+import Statistics: mean
+import LinearAlgebra: norm_sqr
 include(joinpath(pkgdir(CA), "post_processing", "ci_plots.jl"))
 
 ref_job_id = config.parsed_args["reference_job_id"]
@@ -108,6 +110,42 @@ end
 # Write diagnostics that are in DictWriter to text files
 CA.write_diagnostics_as_txt(simulation)
 
+if config.parsed_args["check_steady_state"]
+    @info "Comparing final state to predicted steady-state solution"
+
+    Y_end = integrator.sol.u[end]
+    FT = eltype(Y_end)
+
+    (; steady_state_velocity, params) = integrator.p
+    (; zd_rayleigh) = params
+    @assert !isnothing(steady_state_velocity)
+
+    ᶜuₕ_error_squared =
+        norm_sqr.(Y_end.c.uₕ .- CA.C12.(steady_state_velocity.ᶜu))
+    ᶠu₃_error_squared =
+        norm_sqr.(Y_end.f.u₃ .- CA.C3.(steady_state_velocity.ᶠu))
+
+    # Ignore all errors in the sponge layer.
+    ᶜsponge_mask = FT.(Fields.coordinate_field(Y_end.c).z .< zd_rayleigh)
+    ᶠsponge_mask = FT.(Fields.coordinate_field(Y_end.f).z .< zd_rayleigh)
+    uₕ_rmse = sqrt(sum(ᶜuₕ_error_squared .* ᶜsponge_mask) / sum(ᶜsponge_mask))
+    u₃_rmse = sqrt(sum(ᶠu₃_error_squared .* ᶠsponge_mask) / sum(ᶠsponge_mask))
+
+    uₕ_rmse_by_level = map(1:5) do level
+        sqrt(mean(Fields.level(ᶜuₕ_error_squared, level)))
+    end
+    u₃_rmse_by_level = map(1:5) do level
+        sqrt(mean(Fields.level(ᶠu₃_error_squared, level - Fields.half)))
+    end
+
+    @info "    RMSE of uₕ below sponge layer: $uₕ_rmse"
+    @info "    RMSE of u₃ below sponge layer: $u₃_rmse"
+    @info "    RMSE of uₕ on first 5 levels: $uₕ_rmse_by_level"
+    @info "    RMSE of u₃ on first 5 levels: $u₃_rmse_by_level"
+
+    # TODO: Figure out an appropriate @test for the steady state solution.
+end
+
 # Conservation checks
 if config.parsed_args["check_conservation"]
     FT = Spaces.undertype(axes(sol.u[end].c.ρ))

.buildkite/pipeline.yml

@@ -30,6 +30,7 @@ steps:
 
       - echo "--- Instantiate .buildkite"
       - "julia --project=.buildkite -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia --project=.buildkite -e 'using Pkg; Pkg.add(Pkg.PackageSpec(;name=\"ClimaCore\", rev=\"dy/extruded_1d_cuda\"))'"
       - "julia --project=.buildkite -e 'using Pkg; Pkg.precompile()'"
       - "julia --project=.buildkite -e 'using CUDA; CUDA.precompile_runtime()'"
       - "julia --project=.buildkite -e 'using Pkg; Pkg.status()'"
@@ -89,6 +90,92 @@ steps:
           --job_id single_column_precipitation_test
         artifact_paths: "single_column_precipitation_test/output_active/*"
 
+  - group: "Analytic Tests"
+    steps:
+      - label: ":computer: Cosine Hills Test (2D, Float64)"
+        command: >
+          julia --color=yes --project=.buildkite examples/hybrid/driver.jl
+          --config_file $CONFIG_PATH/plane_cosine_hills_float64_test.yml
+          --job_id plane_cosine_hills_float64_test
+        artifact_paths: "plane_cosine_hills_float64_test/output_active/*"
+
+      - label: "GPU: No Topography Test (2D, Float64, Long Duration, Discrete Balance)"
+        command: >
+          julia --color=yes --project=.buildkite examples/hybrid/driver.jl
+          --config_file $CONFIG_PATH/plane_no_topography_long_float64_test.yml
+          --job_id gpu_plane_no_topography_long_float64_test
+        artifact_paths: "gpu_plane_no_topography_long_float64_test/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+
+      - label: "GPU: Cosine Hills Test (2D, Float64, High Resolution)"
+        command: >
+          julia --color=yes --project=.buildkite examples/hybrid/driver.jl
+          --config_file $CONFIG_PATH/plane_cosine_hills_high_res_float64_test.yml
+          --job_id gpu_plane_cosine_hills_high_res_float64_test
+        artifact_paths: "gpu_plane_cosine_hills_high_res_float64_test/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+
+      - label: "GPU: Cosine Hills Test (Extruded 2D, Float32)"
+        command: >
+          julia --color=yes --project=.buildkite examples/hybrid/driver.jl
+          --config_file $CONFIG_PATH/extruded_plane_cosine_hills_float32_test.yml
+          --job_id gpu_extruded_plane_cosine_hills_float32_test
+        artifact_paths: "gpu_extruded_plane_cosine_hills_float32_test/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+
+      - label: "GPU: Cosine Hills Test (3D, Float32)"
+        command: >
+          julia --color=yes --project=.buildkite examples/hybrid/driver.jl
+          --config_file $CONFIG_PATH/box_cosine_hills_float32_test.yml
+          --job_id gpu_box_cosine_hills_float32_test
+        artifact_paths: "gpu_box_cosine_hills_float32_test/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+
+      - label: "GPU: Agnesi Mountain Test (2D, Float64)"
+        command: >
+          julia --color=yes --project=.buildkite examples/hybrid/driver.jl
+          --config_file $CONFIG_PATH/plane_agnesi_mountain_float64_test.yml
+          --job_id gpu_plane_agnesi_mountain_float64_test
+        artifact_paths: "gpu_plane_agnesi_mountain_float64_test/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+
+      - label: "GPU: Schar Mountain Test (2D, Float64)"
+        command: >
+          julia --color=yes --project=.buildkite examples/hybrid/driver.jl
+          --config_file $CONFIG_PATH/plane_schar_mountain_float64_test.yml
+          --job_id gpu_plane_schar_mountain_float64_test
+        artifact_paths: "gpu_plane_schar_mountain_float64_test/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+
+      - label: "GPU: Schar Mountain Test (2D, Float32)"
+        command: >
+          julia --color=yes --project=.buildkite examples/hybrid/driver.jl
+          --config_file $CONFIG_PATH/plane_schar_mountain_float32_test.yml
+          --job_id gpu_plane_schar_mountain_float32_test
+        artifact_paths: "gpu_plane_schar_mountain_float32_test/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+
   - group: "Gravity wave"
     steps:
 

config/default_configs/default_config.yml

@@ -35,7 +35,7 @@ scalar_hyperdiffusion_coefficient:
   value: 1.5
 # Topography
 topography:
-  help: "Define the surface elevation profile [`NoWarp` (default),`Earth`,`DCMIP200`,`Agnesi`, `Schar`, `Hughes2023`]"
+  help: "Define the surface elevation profile [`NoWarp` (default), `Earth`, `DCMIP200`, `Hughes2023`, `Agnesi`, `Schar`, `Cosine2D`, `Cosine3D`]"
   value: "NoWarp"
 mesh_warp_type:
   help: "Sets the interior mesh warp method [`Linear`, `SLEVE`]"
@@ -226,6 +226,9 @@ reproducibility_test:
 check_conservation:
   help: "Check conservation of mass and energy [`false` (default), `true`]"
   value: false
+check_steady_state:
+  help: "Compare steady-state velocity to analytic solution; only available for certain choices of `topography` [`false` (default), `true`]"
+  value: false
 ls_adv:
   help: "Large-scale advection [`nothing` (default), `Bomex`, `LifeCycleTan2018`, `Rico`, `ARM_SGP`, `GATE_III`]"
   value: ~

config/model_configs/box_cosine_hills_float32_test.yml

@@ -0,0 +1,20 @@
+config: "box"
+FLOAT_TYPE: "Float32"
+initial_condition: "ConstantBuoyancyFrequencyProfile"
+topography: "Cosine3D"
+x_max: 300e3
+y_max: 300e3
+z_max: 21e3
+x_elem: 15
+y_elem: 15
+z_elem: 50
+z_stretch: false
+dt: "12secs"
+t_end: "10days"
+rayleigh_sponge: true
+toml: [toml/analytic_topography_test.toml]
+check_steady_state: true
+output_default_diagnostics: false
+diagnostics:
+  - short_name: [orog, ua, wa, uapredicted, wapredicted, uaerror, waerror]
+    period: 1hours

config/model_configs/extruded_plane_cosine_hills_float32_test.yml

@@ -0,0 +1,20 @@
+config: "box"
+FLOAT_TYPE: "Float32"
+initial_condition: "ConstantBuoyancyFrequencyProfile"
+topography: "Cosine2D"
+x_max: 300e3
+y_max: 40e3
+z_max: 21e3
+x_elem: 15
+y_elem: 2
+z_elem: 50
+z_stretch: false
+dt: "12secs"
+t_end: "10days"
+rayleigh_sponge: true
+toml: [toml/analytic_topography_test.toml]
+check_steady_state: true
+output_default_diagnostics: false
+diagnostics:
+  - short_name: [orog, ua, wa, uapredicted, wapredicted, uaerror, waerror]
+    period: 1hours

config/model_configs/plane_agnesi_mountain_float64_test.yml

@@ -0,0 +1,19 @@
+config: "plane"
+FLOAT_TYPE: "Float64"
+initial_condition: "ConstantBuoyancyFrequencyProfile"
+topography: "Agnesi"
+x_max: 100e3
+z_max: 21e3
+x_elem: 100
+z_elem: 100
+dz_bottom: 10
+dz_top: 1000
+dt: "0.7secs"
+t_end: "4days"
+rayleigh_sponge: true
+toml: [toml/analytic_topography_test.toml]
+check_steady_state: true
+output_default_diagnostics: false
+diagnostics:
+  - short_name: [orog, ua, wa, uapredicted, wapredicted, uaerror, waerror]
+    period: 1hours

config/model_configs/plane_cosine_hills_float64_test.yml

@@ -0,0 +1,19 @@
+config: "plane"
+FLOAT_TYPE: "Float64"
+initial_condition: "ConstantBuoyancyFrequencyProfile"
+topography: "Cosine2D"
+x_max: 300e3
+z_max: 21e3
+x_elem: 15
+z_elem: 50
+z_stretch: false
+nh_poly: 2
+dt: "12secs"
+t_end: "50days"
+rayleigh_sponge: true
+toml: [toml/analytic_topography_test.toml]
+check_steady_state: true
+output_default_diagnostics: false
+diagnostics:
+  - short_name: [orog, ua, wa, uapredicted, wapredicted, uaerror, waerror]
+    period: 1hours

config/model_configs/plane_cosine_hills_high_res_float64_test.yml

@@ -0,0 +1,19 @@
+config: "plane"
+FLOAT_TYPE: "Float64"
+initial_condition: "ConstantBuoyancyFrequencyProfile"
+topography: "Cosine2D"
+x_max: 300e3
+z_max: 21e3
+x_elem: 50
+z_elem: 50
+z_stretch: false
+nh_poly: 2
+dt: "2secs"
+t_end: "1days"
+rayleigh_sponge: true
+toml: [toml/analytic_topography_test.toml]
+check_steady_state: true
+output_default_diagnostics: false
+diagnostics:
+  - short_name: [orog, ua, wa, uapredicted, wapredicted, uaerror, waerror]
+    period: 1hours

config/model_configs/plane_no_topography_long_float64_test.yml

@@ -0,0 +1,20 @@
+config: "plane"
+FLOAT_TYPE: "Float64"
+initial_condition: "ConstantBuoyancyFrequencyProfile"
+topography: "NoWarp"
+discrete_hydrostatic_balance: true
+x_max: 300e3
+z_max: 21e3
+x_elem: 3
+z_elem: 50
+z_stretch: false
+nh_poly: 2
+dt: "5secs"
+t_end: "40days"
+rayleigh_sponge: true
+toml: [toml/analytic_topography_test.toml]
+check_steady_state: true
+output_default_diagnostics: false
+diagnostics:
+  - short_name: [orog, ua, wa, uapredicted, wapredicted, uaerror, waerror]
+    period: 1hours

config/model_configs/plane_schar_mountain_float32_test.yml

@@ -0,0 +1,19 @@
+config: "plane"
+FLOAT_TYPE: "Float32"
+initial_condition: "ConstantBuoyancyFrequencyProfile"
+topography: "Schar"
+x_max: 100e3
+z_max: 21e3
+x_elem: 100
+z_elem: 100
+dz_bottom: 10
+dz_top: 1000
+dt: "0.7secs"
+t_end: "4days"
+rayleigh_sponge: true
+toml: [toml/analytic_topography_test.toml]
+check_steady_state: true
+output_default_diagnostics: false
+diagnostics:
+  - short_name: [orog, ua, wa, uapredicted, wapredicted, uaerror, waerror]
+    period: 1hours

config/model_configs/plane_schar_mountain_float64_test.yml

@@ -0,0 +1,19 @@
+config: "plane"
+FLOAT_TYPE: "Float64"
+initial_condition: "ConstantBuoyancyFrequencyProfile"
+topography: "Schar"
+x_max: 100e3
+z_max: 21e3
+x_elem: 100
+z_elem: 100
+dz_bottom: 10
+dz_top: 1000
+dt: "0.7secs"
+t_end: "4days"
+rayleigh_sponge: true
+toml: [toml/analytic_topography_test.toml]
+check_steady_state: true
+output_default_diagnostics: false
+diagnostics:
+  - short_name: [orog, ua, wa, uapredicted, wapredicted, uaerror, waerror]
+    period: 1hours