More noise in vorticity

[kenflat2]

The following model (an even more simplified version of the model in a previous issue) exhibits strange small grid-scale oscillations in vorticity.

using Oceananigans
using Oceananigans.Units
using CairoMakie

##

Lx = 2000.0kilometers
Ly = 1000.0kilometers
Lz = 4.0kilometers

Δx = 25.0kilometers
n = 30

filename = "testbed_just_water"

grid = RectilinearGrid(size=(80, 40, n), 
                            x=(0, Lx), y=(0, Ly), z=(-Lz, 0),
                            halo=(4,4,4),
                            topology=(Periodic, Bounded, Bounded))


coriolis = FPlane(f=-1e-4)

model = HydrostaticFreeSurfaceModel(;grid, 
                                    coriolis,
                                    momentum_advection=WENO(order=5))

function uᵢ(x, y, z)
    return 0.1 * exp(z/500.0)
end

# Plot uᵢ as a function of z
#region
z_vals = range(-Lz, stop=0, length=100)
u_vals = [uᵢ(0, 0, z) for z in z_vals]

fig = Figure()
ax = Axis(fig[1, 1], xlabel="uᵢ [m/s]", ylabel="z [km]", title="Initial Velocity Profile")
lines!(ax, u_vals, z_vals / 1e3)

fig
#endregion

set!(model, u=uᵢ)

simulation = Simulation(model, Δt=20minutes, stop_time=3 * 30days)

##

#region Callbacks and Checkpointing
using Printf

wall_clock = Ref(time_ns())

function progress(sim)
    elapsed = 1e-9 * (time_ns() - wall_clock[])

    msg = @sprintf("iteration: %d, time: %s, wall time: %s, max|u|: %6.3e, m s⁻¹, max|v|: %6.3e, m s⁻¹, max|w|: %6.3e, m s⁻¹\n",
                   iteration(sim), prettytime(sim), prettytime(elapsed),
                   maximum(abs, sim.model.velocities.u),
                   maximum(abs, sim.model.velocities.v),
                   maximum(abs, sim.model.velocities.w))

    wall_clock[] = time_ns()

    @info msg

    return nothing
end

add_callback!(simulation, progress, name=:progress, IterationInterval(200))

# Write checkpoint files every year
checkpoint_time = 10days
simulation.output_writers[:checkpointer] = Checkpointer(model;
                                                        schedule = TimeInterval(checkpoint_time),
                                                        prefix = filename * "_checkpoint",
                                                        overwrite_existing = true)
#endregion

##

run!(simulation)

Here is a plot of vorticity after 3 months at the surface.

[glwagner]

Are we sure this is noise? It looks rather smooth, perhaps an adjustment from the initial condition? A movie could help, if it is growing in time or something.

[navidcy]

Or also if you increase the resolution and the structure doesn't change that would also suggest that it's not noise.

[kenflat2]

I can run another version of the above simulation at a higher resolution to see if that resolves problems. What I am trying to understand though is noise in vorticity in one of my more complex simulations. Below is a plot of the temperature and vorticity fields of my simulation of the Southern Ocean. It is a channel model with a Gaussian Ridge at the bottom, westerly wind forcing and a strong temperature relaxation at the surface. The plot is taken from a snapshot 40 years after the initial condition. The temperature fields looks quite smooth, but vorticity is a bit of a mess.

using Oceananigans
using Oceananigans.Units
using Oceananigans.Grids
using Oceananigans.TurbulenceClosures
using CairoMakie
using CUDA
using Adapt
using Statistics
using JLD2
using Oceananigans.Operators: ℑxyᶠᶜᵃ, ℑxyᶜᶠᵃ, ℑxzᶠᵃᶜ, ℑxzᶜᵃᶠ, ℑyzᵃᶠᶜ, ℑyzᵃᶜᶠ

#region Model Parameters
##

Lx = 30 # degrees
Ly = 15 # degrees
Lz = 4kilometers # Depth of the domain
start_latitude = -70.0 # degrees

Δx = 1/16 # Degree
nx = Int(Lx / Δx)
ny = Int(Ly / Δx)

# Make the grid resolution in the vertical direction get more fine as depth decreases.
nz = 100

coriolis = HydrostaticSphericalCoriolis()

buoyancy = SeawaterBuoyancy(equation_of_state=LinearEquationOfState())
leos = LinearEquationOfState()

function Tᵢ(x, y, z)
    function Tₐ(y) 
        Ly = 15 # degrees
        start_latitude = -70.0 # degrees
        return 0.0 + 4.0 * (y - start_latitude) / Ly
    end

    function H(x)
        h₀ = 1kilometers # [m] height of the ridge
        σ = 2 # degree 100kilometers # [m] width of the ridge
    
        Lx = 30 # degrees kilometers
        Lz = 4.0kilometers

        return Lz - h₀ * exp(-((x - Lx/2) / (2σ))^2)
    end

    # Lz = 4.0kilometers
    h = 1.0kilometers
    return Tₐ(y) * (exp(z/h) - exp(-H(x)/h)) / (1 - exp(-H(x)/h))
end

function Sᵢ(x, y, z)
    return 35.0
end

function uᵢ(x, y, z)
    return 0.0
end

function vᵢ(x, y, z)
    return 0.0
end

function wᵢ(x, y, z)
    return 0.0
end

function surface_temp_flux(i, j, grid, clock, fields)
    # make this the same as initial condition, no seasonal variation
    function Tₐ(y) 
        Ly = 15 # degrees
        start_latitude = -70.0 # degrees
        return 0.0 + 4.0 * (y - start_latitude) / Ly
    end

    # We have to redefine inside the function to convince the compiler that there
    # are no dependencies on the global variables
    y = φnode(j, grid, Center())

    Δz = 10.0meters
    λ = 1e6days # 100 days is in line with Abernathy, 200 is conservative
    ρ₀ = 1025.0 #kilograms / meters^3
    cₚ = 4000.0 #joules / (kilograms * kelvin)

    T = @inbounds fields.T[i, j, grid.Nz]

    return Δz * ρ₀ * cₚ * (T - Tₐ(y)) / λ
end

T_top_flux = FluxBoundaryCondition(surface_temp_flux, discrete_form=true)
T_bcs = FieldBoundaryConditions(top = T_top_flux) 

# surface momentum flux
function surface_momentum_flux(i, j, grid, clock, fields)
    y = ynode(j, grid, Center())

    u₁₀ = 10.0    # m s⁻¹, average wind velocity 10 meters above the ocean
    cᴰ = 2.5e-3 # dimensionless drag coefficient
    ρₐ = 1.225  # kg m⁻³, average density of air at sea-level
    ρₒ = 1025.0   # kg m⁻³, average density of seawater

    Qᵘ = ρₐ / ρₒ * cᴰ * u₁₀ * abs(u₁₀) # m² s⁻²

    Ly = 15 # degrees 1000.0kilometers

    return -Qᵘ * sin(π * y / Ly) # * 1e2
end

no_slip_bc = ValueBoundaryCondition(0.0)

immersed_no_slip_bottom_bc = ImmersedBoundaryCondition(bottom = no_slip_bc)

u_bcs = FieldBoundaryConditions(immersed=immersed_no_slip_bottom_bc, top=FluxBoundaryCondition(surface_momentum_flux, discrete_form=true))
v_bcs = FieldBoundaryConditions(immersed=immersed_no_slip_bottom_bc)

# Exponential spaced depth grid
function Φ(z)
    Δzₘ = 300meters # chosen so that first grid cell gap is ~10 meters
    Lz = 4kilometers # depth of the domain
    nz = 100 # number of vertical grid cells

    α = (1 / (nz - 1)) * log(Lz / Δzₘ + 1)# log(Lz / Δzₘ)

    return - Δzₘ * (exp(α * (nz-z)) - 1)
end

function ν_v(x, y, z, t)
    νₐ = 1e-4 # abyssal viscosity
    νₘ = 3e-2 # surface viscosity
    
    h₀ = 4kilometers - 1kilometers # [m] height of the ridge

    ml_depth = 200.0meters
    ml_l = 100.0meters

    return νₐ + (νₘ - νₐ) * (tanh((z + ml_depth)/ml_l) + 1.0) / 2.0
end

function κ_v(x, y, z, t)
    κₐ = 3e-5 # abyssal diffusivity
    κₘ = 3e-3 # surface diffusivity
    
    h₀ = 4kilometers - 1kilometers # [m] height of the ridge

    ml_depth = 200.0meters
    ml_l = 100.0meters

    return κₐ + (κₘ - κₐ) * (tanh((z + ml_depth)/ml_l) + 1.0) / 2.0
end

vertical_closure = VerticalScalarDiffusivity(ν=ν_v, κ=κ_v)
convective_adjustment = RiBasedVerticalDiffusivity()

##

#region Model Setup
grid = LatitudeLongitudeGrid( GPU(); size=(nx, ny, nz),
                              longitude=(0, Lx),
                              latitude=(start_latitude, start_latitude + Ly),
                              z=Φ,
                              topology=(Periodic, Bounded, Bounded),
                              halo=(7,7,7));

function bottom(x, y)
    function ridge(x, y)
        h₀ = 1kilometers # [m] height of the ridge
        σ = 2 # degree 100kilometers # [m] width of the ridge
    
        Lx = 30 # degrees kilometers
    
        return h₀ * exp(-((x - Lx/2) / (2σ))^2)
    end

    Lz = 4kilometers # [m] depth of the domain

    return -Lz + ridge(x, y)
end

ridge_grid = ImmersedBoundaryGrid(grid, GridFittedBottom(bottom));

free_surface = SplitExplicitFreeSurface(grid; cfl = 0.2)

momentum_advection = VectorInvariant(vorticity_scheme=WENO(order=9),
                                     vertical_scheme=WENO(grid))

model = HydrostaticFreeSurfaceModel(; grid=ridge_grid,
                                      coriolis = coriolis, 
                                      buoyancy = buoyancy,
                                      free_surface = free_surface,
                                      tracers = (:T, :S),
                                      closure = (vertical_closure, convective_adjustment),
                                      momentum_advection = momentum_advection,
                                      tracer_advection = WENO(grid, order=7), # make sure to use 7th order for tracers, 7th order tracer advection corresponds to 9th order advection for momentum
                                      # closure = (horizontal_closure, vertical_closure), # , convective_adjustment), # , hyperviscous_closure),
                                      boundary_conditions = (T=T_bcs, u=u_bcs, v=v_bcs))

set!(model, T=Tᵢ, S=Sᵢ, u=uᵢ, v=vᵢ, w=wᵢ)

simulation = Simulation(model, Δt=1seconds, stop_time=30days)

#endregion

#region Callbacks and Output Writers
##
using Printf

wall_clock = Ref(time_ns())

function print_progress(sim)
    u, v, w = model.velocities
    # T, S = model.tracers
    progress = 100 * (time(sim) / sim.stop_time)
    elapsed = (time_ns() - wall_clock[]) / 1e9

    @printf("[%05.2f%%] i: %d, t: %s, wall time: %s, max(u): (%6.3e, %6.3e, %6.3e) m/s, next Δt: %s\n",
            progress, iteration(sim), prettytime(sim), prettytime(elapsed),
            maximum(abs, u), maximum(abs, v), maximum(abs, w), prettytime(sim.Δt))

    wall_clock[] = time_ns()

    return nothing
end

add_callback!(simulation, print_progress, IterationInterval(100))

checkpoint_time = 24hours

simulation.output_writers[:checkpointer] = Checkpointer(model;
                                                        schedule = TimeInterval(checkpoint_time),
                                                        prefix = sim_folder * "/" * filename * "_checkpoint",
                                                        overwrite_existing = true)

simulation.output_writers

@info "Running simulation..."

run!(simulation)

@info "Simulation completed in " * prettytime(simulation.run_wall_time)

I'll see if I can get a more simple example that exhibits this behavior, just wanted to raise the current issue as soon as I could.

[glwagner]

There's something wrong. Your code says you are using a grid which is (480, 240, 100) --- quite a high resolution. But the visualization shows a fairly low resolution:

The plot looks more like ~60 points in the y-direction, not 240.

A few more things:

• it's a good idea to initalize a simulation with noise
• use lower order WENO to make a simulation more diffusive
• w is a diagnostic variable in the hydrostatic model, so you should not set! it

[kenflat2]

I have a new version of the simulation that incorporates some of the changes mentioned by Greg and other changes that still exhibits some noise in vorticity.

After 1 year, here are the temperature and vorticity fields at the surface. The grid resolution is 1/4 of a degree and the vorticity field is quite noisy. This could be due to the coarse resolution relative to the true lengthscale of dynamics - we expect a detailed submesoscale field to develop at high resolutions here. We are also using 9th order WENOVectorInvariant so the potential for a very rich velocity field is present but the resolution may be insufficient. In simulations with a 5th order advection scheme, the vorticity is much less noisy. Of most concern is the high vorticity values at the Southern Boundary.

Below I plot the raw u and v values, also at the surface and after 1 year and can see strange behavior near the Southern boundary. The wall at the South is an idealization and hence is not of interest in our study. However, the high velocities there significantly increase the maximum velocity and increase the computational expense under the CFL condition. A key step therefore is to change the simulation so these detailed dynamics at the Southern boundary don't occur.

I have a parallel simulation running at 1/16 of a degree and, though it is early in its integration, we see more detailed dynamics appear immediately in the South. We have strong negative temperature forcing in the South, could these dynamics be due to convection?

Here is the code:

using Oceananigans
using Oceananigans.Units
using Oceananigans.Grids
using Oceananigans.TurbulenceClosures
using CairoMakie
using CUDA
using Adapt
using Statistics
using JLD2
using Oceananigans.Operators: ℑxyᶠᶜᵃ, ℑxyᶜᶠᵃ, ℑxzᶠᵃᶜ, ℑxzᶜᵃᶠ, ℑyzᵃᶠᶜ, ℑyzᵃᶜᶠ

CUDA.device!(1)

filename = "submesoscale_southern_ocean_1-16"
data_folder = "/home/kengee/oceanography/data/simulations"
sim_folder = data_folder * "/" * filename

# Create the data folder if it doesn't exist
if !isdir(sim_folder)
    mkpath(sim_folder)
end

cfl = 0.1

Lx = 30 # degrees
Ly = 20 # degrees
Lz = 3kilometers # Depth of the domain
start_latitude = -70 # degrees

Δx = 1/4 # Degree
nx = Int(Lx / Δx)
ny = Int(Ly / Δx)
nz = 100

# This implies a beta plane
coriolis = HydrostaticSphericalCoriolis()

# Buoyancy
buoyancy = SeawaterBuoyancy(equation_of_state=LinearEquationOfState())
leos = LinearEquationOfState()

function Tᵢ(x, y, z)
    function Tₐ(y) 
        Ly = 20 # degrees
        start_latitude = -70 # degrees
        return 0 + 8 * (y - start_latitude) / Ly
    end

    function H(x)
        h₀ = 1kilometers # [m] height of the ridge
        σ = 2 # degree 100kilometers # [m] width of the ridge
    
        Lx = 30 # degrees kilometers
        Lz = 3kilometers

        return Lz - h₀ * exp(-((x - Lx/2) / (2σ))^2)
    end

    function random_noise(x, y, z)
        φ = Base.MathConstants.golden
        return mod(φ * x ^ 2 * y ^ 2 * z ^ 2, 1)
    end

    # Lz = 4.0kilometers
    h = 1kilometers
    return (Tₐ(y) + 1e-1 * random_noise(x, y, z)) * (exp(z/h) - exp(-H(x)/h)) / (1 - exp(-H(x)/h))
end

function Sᵢ(x, y, z)
    return 35
end

#region Plot Initial Temperature
fig = Figure();
ax = Axis(fig[1,1], xlabel="y", ylabel="Depth [m]", title="Initial Temperature Heatmap")
y = collect(start_latitude:Δx:start_latitude+Ly)
z = collect(-Lz:Lz/nz:0)
T_init = [Tᵢ(0, yᵢ, zᵢ) for yᵢ in y, zᵢ in z]
ax_t = heatmap!(ax, y, z, T_init, colormap=:balance, colorrange=(0, 4))
Colorbar(fig[1,2], ax_t)

fig
#endregion

function uᵢ(x, y, z)
    # Pseudorandom initial condition. Can replace with rand()
    return 1e-1 * mod(Base.MathConstants.golden * (x ^ 2 +  y ^ 2 + z ^ 2), 1)
end

function vᵢ(x, y, z)
    # Rough attempt to add some noise to the velocity field
    return 1e-1 * mod(Base.MathConstants.golden * (x ^ 2 +  y ^ 2 + z ^ 2), 1)
end

function surface_temp_flux(i, j, grid, clock, fields)
    # make this the same as initial condition, no seasonal variation
    function Tₐ(y) 
        Ly = 20 # degrees
        start_latitude = -70 # degrees
        return 0 + 8 * (y - start_latitude) / Ly
    end

    # We have to redefine inside the function to convince the compiler that there
    # are no dependencies on the global variables
    y = φnode(j, grid, Center())

    Δz = 10meters
    λ = 7days

    T = @inbounds fields.T[i, j, grid.Nz]

    return Δz * (T - Tₐ(y)) / λ
end

T_top_flux = FluxBoundaryCondition(surface_temp_flux, discrete_form=true)
T_bcs = FieldBoundaryConditions(top = T_top_flux)

# surface momentum flux
function surface_momentum_flux(i, j, grid, clock, fields)
    y = φnode(j, grid, Center())

    u₁₀ = 10.0    # m s⁻¹, average wind velocity 10 meters above the ocean
    cᴰ = 2.5e-3 # dimensionless drag coefficient
    ρₐ = 1.225  # kg m⁻³, average density of air at sea-level
    ρₒ = 1025.0   # kg m⁻³, average density of seawater

    Qᵘ = ρₐ / ρₒ * cᴰ * u₁₀ * abs(u₁₀) # m² s⁻²

    Ly = 20 # degrees 1000.0kilometers

    return -Qᵘ * (1 - cos(2 * π * y / Ly)) / 2 # * 1e2
end

function bottom_drag_u(i, j, k, grid, clock, fields)
    drag_coeff = 1e-3 # dimensionless
    u = @inbounds fields.u[i, j, k]
    v = ℑxyᶠᶜᵃ(i, j, k, grid, fields.v)
    return - drag_coeff * u * sqrt(u^2 + v^2)
end

function bottom_drag_v(i, j, k, grid, clock, fields)
    drag_coeff = 1e-3 # dimensionless
    u = ℑxyᶜᶠᵃ(i, j, k, grid, fields.u)
    v = @inbounds fields.v[i, j, k]
    return - drag_coeff * v * sqrt(u^2 + v^2)
end

u_bottom_bc = FluxBoundaryCondition(bottom_drag_u, discrete_form=true)
v_bottom_bc = FluxBoundaryCondition(bottom_drag_v, discrete_form=true)

u_bcs = FieldBoundaryConditions(immersed=u_bottom_bc, top=FluxBoundaryCondition(surface_momentum_flux, discrete_form=true)) # , north=u_wall_bc, south=u_wall_bc)
v_bcs = FieldBoundaryConditions(immersed=v_bottom_bc)

# Exponential spaced depth grid
function Φ(z)
    Δzₘ = 300meters # chosen so that first grid cell gap is ~10 meters
    Lz = 3kilometers # depth of the domain
    nz = 100 # number of vertical grid cells

    α = (1 / (nz - 1)) * log(Lz / Δzₘ + 1)# log(Lz / Δzₘ)

    return - Δzₘ * (exp(α * (nz-z)) - 1)
end
#region Plot the depth grid
fig = Figure();
ax = Axis(fig[1, 1], title="Exponential Spaced Depth Grid", xlabel="j", ylabel="Depth [m]")
plot!(ax, 1:nz, Φ.(1:nz))

Φ.(1:nz)

fig
#endregion

function ν_v(x, y, z, t)
    νₐ = 1e-4 # abyssal viscosity
    νₘ = 3e-2 # surface viscosity

    ml_depth = 150meters
    ml_l = 50meters

    return νₐ + (νₘ - νₐ) * (tanh((z + ml_depth)/ml_l) + 1) / 2
end

function κ_v(x, y, z, t)
    κₐ = 3e-5 # abyssal diffusivity
    κₘ = 3e-3 # surface diffusivity
    
    h₀ = 4kilometers - 1kilometers # [m] height of the ridge

    ml_depth = 150meters
    ml_l = 50meters

    return κₐ + (κₘ - κₐ) * (tanh((z + ml_depth)/ml_l) + 1) / 2
end

#region Plot vertical diffusivity profile
fig = Figure();
ax = Axis(fig[1,1], xlabel="log10(κ) [m² s⁻¹]", ylabel="Depth [m]", title="Horizontal Viscosity Profile")

depths = collect(Φ.(1:nz))
zs = zeros(length(depths))
lines!(ax, log10.(ν_v.(zs, zs, depths, zs)), depths, label="κ(z)")

fig
#endregion

vertical_closure = VerticalScalarDiffusivity(ν=ν_v, κ=κ_v)
convective_adjustment = RiBasedVerticalDiffusivity()

grid = RectilinearGrid( GPU(), size=(nx, ny, nz),
                        x=(0, Lx),
grid = LatitudeLongitudeGrid( GPU(); size=(nx, ny, nz),
                              longitude=(0, Lx),
                              latitude=(start_latitude, start_latitude + Ly),
                              z=Φ,
                              topology=(Periodic, Bounded, Bounded),
                              halo=(7,7,7));

function bottom(x, y)
    function ridge(x, y)
        h₀ = 1kilometers # [m] height of the ridge
        σ = 2 # degree 100kilometers # [m] width of the ridge
    
        Lx = 30 # degrees kilometers
    
        return h₀ * exp(-((x - Lx/2) / (2σ))^2)
    end

    Lz = 3kilometers # [m] depth of the domain

    return -Lz + ridge(x, y)
end

ridge_grid = ImmersedBoundaryGrid(grid, GridFittedBottom(bottom));

#region Plot Ridge
collect(0:Δx:Lx)

fig = Figure();
ax = Axis(fig[1,1], xlabel="x [km]", ylabel="z [km]", title="Ridge")
lines!(ax, 0:Δx:Lx, bottom.(0:Δx:Lx, zeros(nx+1)), label="Ridge")
ylims!(ax, -Lz, 0)

fig
#endregion

free_surface = SplitExplicitFreeSurface(grid; cfl = cfl)

momentum_advection = WENOVectorInvariant(order=9)

model = HydrostaticFreeSurfaceModel(; grid=ridge_grid,
                                      coriolis = coriolis, # BetaPlane(latitude=45),
                                      buoyancy = buoyancy,
                                      free_surface = free_surface,
                                      timestepper = :SplitRungeKutta3, # Use RK3
                                      tracers = (:T, :S),
                                      closure = (vertical_closure, convective_adjustment),
                                      momentum_advection = momentum_advection,
                                      tracer_advection = WENO(order=7), # make sure to use 7th order for tracers, 7th order tracer advection corresponds to 9th order advection for momentum
                                      boundary_conditions = (T=T_bcs, u=u_bcs, v=v_bcs))



set!(model, T=Tᵢ, S=Sᵢ, u=uᵢ, v=vᵢ)

simulation = Simulation(model, Δt=1seconds, stop_time=12 * 30days)

# A CFL of 0.1 is conservative for this time stepper
conjure_time_step_wizard!(simulation, IterationInterval(20), cfl=cfl, max_Δt=5minutes)

using Printf

wall_clock = Ref(time_ns())

function print_progress(sim)
    u, v, w = model.velocities
    # T, S = model.tracers
    progress = 100 * (time(sim) / sim.stop_time)
    elapsed = (time_ns() - wall_clock[]) / 1e9

    @printf("[%05.2f%%] i: %d, t: %s, wall time: %s, max(u): (%6.3e, %6.3e, %6.3e) m/s, next Δt: %s\n",
            progress, iteration(sim), prettytime(sim), prettytime(elapsed),
            maximum(abs, u), maximum(abs, v), maximum(abs, w), prettytime(sim.Δt))

    wall_clock[] = time_ns()

    return nothing
end

add_callback!(simulation, print_progress, IterationInterval(100))

checkpoint_time = 7days

simulation.output_writers[:checkpointer] = Checkpointer(model;
                                                        schedule = TimeInterval(checkpoint_time),
                                                        prefix = sim_folder * "/" * filename * "_checkpoint",
                                                        overwrite_existing = true)
                                                    
# checkpoint_outputs!(simulation, filename)

@info "Running simulation..."

run!(simulation)

@info "Simulation completed in " * prettytime(simulation.run_wall_time)

#endregion

[glwagner]

For sure, this is coarse, you can tell just by looking at it. You can make out individual pixels by eye.