From bf404f57a8a5db1c57f8855b249da9c3b4848425 Mon Sep 17 00:00:00 2001 From: Albert de Montserrat Date: Thu, 18 Jul 2024 10:47:06 +0200 Subject: [PATCH 1/5] fix & update 3D shear band miniapps --- .../stokes3D/shear_band/ShearBand3D.jl | 111 ++++++++---------- .../stokes3D/shear_band/ShearBand3D_MPI.jl | 53 +++++++-- 2 files changed, 92 insertions(+), 72 deletions(-) diff --git a/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D.jl b/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D.jl index 48256186..1f09b8cb 100644 --- a/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D.jl +++ b/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D.jl @@ -1,11 +1,37 @@ +const isCUDA = false + +@static if isCUDA + using CUDA +end + using JustRelax, JustRelax.JustRelax3D, JustRelax.DataIO import JustRelax.@cell -const backend_JR = CPUBackend -using Printf, GeoParams, GLMakie, CellArrays, CSV, DataFrames +const backend_JR = @static if isCUDA + CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +else + JustRelax.CPUBackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +end + +using ParallelStencil, ParallelStencil.FiniteDifferences3D + +@static if isCUDA + @init_parallel_stencil(CUDA, Float64, 3) +else + @init_parallel_stencil(Threads, Float64, 3) +end + +using JustPIC, JustPIC._3D +# Threads is the default backend, +# to run on a CUDA GPU load CUDA.jl (i.e. "using CUDA") at the beginning of the script, +# and to run on an AMD GPU load AMDGPU.jl (i.e. "using AMDGPU") at the beginning of the script. +const backend = @static if isCUDA + CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +else + JustPIC.CPUBackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +end -using ParallelStencil -@init_parallel_stencil(Threads, Float64, 3) +using GeoParams, GLMakie, CellArrays # HELPER FUNCTIONS --------------------------------------------------------------- solution(ε, t, G, η) = 2 * ε * η * (1 - exp(-G * t / η)) @@ -17,7 +43,7 @@ function init_phases!(phase_ratios, xci, radius) @parallel_indices (i, j, k) function init_phases!(phases, xc, yc, zc, o_x, o_y, o_z) x, y, z = xc[i], yc[j], zc[k] - if ((x-o_x)^2 + (y-o_y)^2 + (z-o_z)^2) > radius + if ((x-o_x)^2 + (y-o_y)^2 + (z-o_z)^2) > radius^2 JustRelax.@cell phases[1, i, j, k] = 1.0 JustRelax.@cell phases[2, i, j, k] = 0.0 @@ -52,7 +78,7 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") G0 = 1.0 # elastic shear modulus Gi = G0/(6.0-4.0) # elastic shear modulus perturbation εbg = 1.0 # background strain-rate - η_reg = 1.25e-2 # regularisation "viscosity" + η_reg = 8e-3 # regularisation "viscosity" dt = η0/G0/4.0 # assumes Maxwell time of 4 el_bg = ConstantElasticity(; G=G0, ν=0.5) el_inc = ConstantElasticity(; G=Gi, ν=0.5) @@ -81,20 +107,21 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") Elasticity = el_inc, ), ) - + # Initialize phase ratios ------------------------------- - radius = 0.01 - phase_ratios = PhaseRatio(ni, length(rheology)) + radius = 0.1 + phase_ratios = PhaseRatio(backend_JR, ni, length(rheology)) init_phases!(phase_ratios, xci, radius) # STOKES --------------------------------------------- # Allocate arrays needed for every Stokes problem stokes = StokesArrays(backend_JR, ni) - pt_stokes = PTStokesCoeffs(li, di; ϵ = 1e-4, CFL = 0.05 / √3.1) + pt_stokes = PTStokesCoeffs(li, di; ϵ = 1e-6, CFL = 0.05 / √3.1) # Buoyancy forces ρg = @zeros(ni...), @zeros(ni...), @zeros(ni...) args = (; T = @zeros(ni...), P = stokes.P, dt = Inf) # Rheology + cutoff_visc = -Inf, Inf compute_viscosity!(stokes, phase_ratios, args, rheology, cutoff_visc) # Boundary conditions @@ -102,9 +129,9 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") free_slip = (left = true , right = true , top = true , bot = true , back = true , front = true), no_slip = (left = false, right = false, top = false, bot = false, back = false, front = false), ) - stokes.V.Vx .= PTArray([ x*εbg/2 for x in xvi[1], _ in 1:ny+2, _ in 1:nz+2]) - stokes.V.Vy .= PTArray([ y*εbg/2 for _ in 1:nx+2, y in xvi[2], _ in 1:nz+2]) - stokes.V.Vz .= PTArray([-z*εbg for _ in 1:nx+2, _ in 1:nx+2, z in xvi[3]]) + stokes.V.Vx .= PTArray(backend_JR)([ x*εbg for x in xvi[1], _ in 1:ny+2, _ in 1:nz+2]) + stokes.V.Vy .= PTArray(backend_JR)([ 0 for _ in 1:nx+2, y in xvi[2], _ in 1:nz+2]) + stokes.V.Vz .= PTArray(backend_JR)([-z*εbg for _ in 1:nx+2, _ in 1:nx+2, z in xvi[3]]) flow_bcs!(stokes, flow_bcs) # apply boundary conditions update_halo!(@velocity(stokes)...) @@ -151,72 +178,38 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") println("it = $it; t = $t \n") - velocity2vertex!(Vx_v, Vy_v, Vz_v, @velocity(stokes)...) - data_v = (; - T = Array(T_buffer), - τII = Array(stokes.τ.II), - εII = Array(stokes.ε.II), - Vx = Array(Vx_v), - Vy = Array(Vy_v), - Vz = Array(Vz_v), - ) - data_c = (; - P = Array(stokes.P), - η = Array(stokes.viscosity.η_vep), - ) - velocity_v = ( - Array(Vx_v), - Array(Vy_v), - Array(Vz_v), - ) - save_vtk( - joinpath(vtk_dir, "vtk_" * lpad("$it", 6, "0")), - xvi, - xci, - data_v, - data_c, - velocity_v - ) - # visualisation + slice_j = ny >>> 1 fig = Figure(size = (1600, 1600), title = "t = $t") - ax1 = Axis3(fig[1,1], aspect = (1, 1, 1), title = "τII") - ax2 = Axis3(fig[2,1], aspect = (1, 1, 1), title = "η_vep") - ax3 = Axis3(fig[1,2], aspect = (1, 1, 1), title = "log10(εxy)") + ax1 = Axis(fig[1,1], aspect = 1, title = "τII") + ax2 = Axis(fig[2,1], aspect = 1, title = "η_vep") + ax3 = Axis(fig[1,2], aspect = 1, title = "log10(εII)") ax4 = Axis(fig[2,2], aspect = 1) - volume!(ax1, xci..., Array(stokes.τ.II) , colormap=:batlow) - volume!(ax2, xci..., Array(η_vep) , colormap=:batlow) - volume!(ax3, xci..., Array(log10.(stokes.ε.xy)) , colormap=:batlow) + heatmap!(ax1, xci[1], xci[3], Array(stokes.τ.II[:, slice_j, :]) , colormap=:batlow) + heatmap!(ax2, xci[1], xci[3], Array(log10.(stokes.viscosity.η_vep)[:, slice_j, :]) , colormap=:batlow) + heatmap!(ax3, xci[1], xci[3], Array(log10.(stokes.ε.II)[:, slice_j, :]) , colormap=:batlow) lines!(ax4, ttot, τII, color = :black) lines!(ax4, ttot, sol, color = :red) hidexdecorations!(ax1) hidexdecorations!(ax3) + for ax in (ax1, ax2, ax3) xlims!(ax, (0, 1)) ylims!(ax, (0, 1)) - zlims!(ax, (0, 1)) end save(joinpath(figdir, "$(it).png"), fig) - end - df = DataFrame( - t = ttot, - τII = τII, - sol = sol, - ) - - CSV.write(joinpath(figdir, "data_$(nx).csv"), df) - return nothing end -n = 64 + 2 -nx = ny = nz = n - 2 -figdir = "ShearBand3D" +n = 32 +nx = ny = nz = n +figdir = "ShearBand3D_noMPI" igg = if !(JustRelax.MPI.Initialized()) IGG(init_global_grid(nx, ny, nz; init_MPI = true)...) else igg end + main(igg; figdir = figdir, nx = nx, ny = ny, nz = nz); diff --git a/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl b/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl index 45bae0a2..1d9ee742 100644 --- a/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl +++ b/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl @@ -1,11 +1,37 @@ +const isCUDA = false + +@static if isCUDA + using CUDA +end + using JustRelax, JustRelax.JustRelax3D, JustRelax.DataIO import JustRelax.@cell -const backend_JR = CPUBackend -using GeoParams, GLMakie +const backend_JR = @static if isCUDA + CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +else + JustRelax.CPUBackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +end + +using ParallelStencil, ParallelStencil.FiniteDifferences3D + +@static if isCUDA + @init_parallel_stencil(CUDA, Float64, 3) +else + @init_parallel_stencil(Threads, Float64, 3) +end -using ParallelStencil -@init_parallel_stencil(Threads, Float64, 3) +using JustPIC, JustPIC._3D +# Threads is the default backend, +# to run on a CUDA GPU load CUDA.jl (i.e. "using CUDA") at the beginning of the script, +# and to run on an AMD GPU load AMDGPU.jl (i.e. "using AMDGPU") at the beginning of the script. +const backend = @static if isCUDA + CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +else + JustPIC.CPUBackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +end + +using GeoParams, GLMakie, CellArrays # HELPER FUNCTIONS --------------------------------------------------------------- solution(ε, t, G, η) = 2 * ε * η * (1 - exp(-G * t / η)) @@ -99,16 +125,16 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") compute_viscosity!(stokes, phase_ratios, args, rheology, cutoff_visc) # Boundary conditions - flow_bcs = VelocityBoundaryConditions(; - free_slip = (left = true , right = true , top = true , bot = true , back = true , front = true), - no_slip = (left = false, right = false, top = false, bot = false, back = false, front = false), + flow_bcs = VelocityBoundaryConditions(; + free_slip = (left = true , right = true , top = true , bot = true , back = true , front = true), + no_slip = (left = false, right = false, top = false, bot = false, back = false, front = false), ) - stokes.V.Vx .= PTArray(backend_JR)([ x*εbg/2 for x in xvi[1], _ in 1:ny+2, _ in 1:nz+2]) - stokes.V.Vy .= PTArray(backend_JR)([ y*εbg/2 for _ in 1:nx+2, y in xvi[2], _ in 1:nz+2]) - stokes.V.Vz .= PTArray(backend_JR)([-z*εbg for _ in 1:nx+2, _ in 1:nx+2, z in xvi[3]]) + stokes.V.Vx .= PTArray(backend_JR)([ x*εbg for x in xvi[1], _ in 1:ny+2, _ in 1:nz+2]) + stokes.V.Vy .= PTArray(backend_JR)([ 0 for _ in 1:nx+2, y in xvi[2], _ in 1:nz+2]) + stokes.V.Vz .= PTArray(backend_JR)([-z*εbg for _ in 1:nx+2, _ in 1:nx+2, z in xvi[3]]) flow_bcs!(stokes, flow_bcs) # apply boundary conditions update_halo!(@velocity(stokes)...) - + # IO ------------------------------------------------ # if it does not exist, make folder where figures are stored !isdir(figdir) && mkpath(figdir) @@ -200,14 +226,15 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") return nothing end -n = 32 +n = 18 nx = n ny = n nz = n # if only 2 CPU/GPU are used nx = 17 - 2 with N =32 -figdir = "ShearBand3D" +figdir = "ShearBand3D_MPI" igg = if !(JustRelax.MPI.Initialized()) IGG(init_global_grid(nx, ny, nz; init_MPI = true, select_device=false)...) else igg end + main(igg; figdir = figdir, nx = nx, ny = ny, nz = nz); From e188ecec43a93eefc6d2de831295716a00cb51d5 Mon Sep 17 00:00:00 2001 From: Albert de Montserrat Date: Thu, 18 Jul 2024 11:26:07 +0200 Subject: [PATCH 2/5] add missing backend --- miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl b/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl index 1d9ee742..0c830b28 100644 --- a/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl +++ b/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl @@ -110,7 +110,7 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") # Initialize phase ratios ------------------------------- radius = 0.01 - phase_ratios = PhaseRatio(ni, length(rheology)) + phase_ratios = PhaseRatio(backend_JR, ni, length(rheology)) init_phases!(phase_ratios, xci, radius) # STOKES --------------------------------------------- From 13c7ff9725b92916547e6f1fd675944e6041c56e Mon Sep 17 00:00:00 2001 From: Albert de Montserrat Date: Mon, 22 Jul 2024 16:18:18 +0200 Subject: [PATCH 3/5] debugging miniapp --- .../stokes3D/shear_band/ShearBand3D_MPI.jl | 76 ++++++++++++++++--- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl b/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl index 0c830b28..eaf161f0 100644 --- a/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl +++ b/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D_MPI.jl @@ -1,4 +1,5 @@ const isCUDA = false +using JLD2 @static if isCUDA using CUDA @@ -43,7 +44,7 @@ function init_phases!(phase_ratios, xci, radius) @parallel_indices (i, j, k) function init_phases!(phases, xc, yc, zc, o_x, o_y, o_z) x, y, z = xc[i], yc[j], zc[k] - if ((x-o_x)^2 + (y-o_y)^2 + (z-o_z)^2) > radius + if ((x-o_x)^2 + (y-o_y)^2 + (z-o_z)^2) > radius^2 @cell phases[1, i, j, k] = 1.0 @cell phases[2, i, j, k] = 0.0 @@ -109,14 +110,15 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") ) # Initialize phase ratios ------------------------------- - radius = 0.01 + radius = 0.1 phase_ratios = PhaseRatio(backend_JR, ni, length(rheology)) + init_phases!(phase_ratios, xci, radius) # STOKES --------------------------------------------- # Allocate arrays needed for every Stokes problem stokes = StokesArrays(backend_JR, ni) - pt_stokes = PTStokesCoeffs(li, di; ϵ = 1e-4, CFL = 0.05 / √3.1) + pt_stokes = PTStokesCoeffs(li, di; ϵ = 1e-6, CFL = 0.5 / √3.1) # Buoyancy forces ρg = @zeros(ni...), @zeros(ni...), @zeros(ni...) args = (; T = @zeros(ni...), P = stokes.P, dt = Inf) @@ -131,7 +133,7 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") ) stokes.V.Vx .= PTArray(backend_JR)([ x*εbg for x in xvi[1], _ in 1:ny+2, _ in 1:nz+2]) stokes.V.Vy .= PTArray(backend_JR)([ 0 for _ in 1:nx+2, y in xvi[2], _ in 1:nz+2]) - stokes.V.Vz .= PTArray(backend_JR)([-z*εbg for _ in 1:nx+2, _ in 1:nx+2, z in xvi[3]]) + stokes.V.Vz .= PTArray(backend_JR)([-z*εbg for _ in 1:nx+2, _ in 1:ny+2, z in xvi[3]]) flow_bcs!(stokes, flow_bcs) # apply boundary conditions update_halo!(@velocity(stokes)...) @@ -160,6 +162,22 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") sol = Float64[] ttot = Float64[] + Vx_v = @zeros(ni .+ 1...) + Vy_v = @zeros(ni .+ 1...) + Vz_v = @zeros(ni .+ 1...) + + println(" + rank $(igg.me): + ni = $(size(stokes.P)) + dxi = $di + xvi[1] = $(xvi[1]) + xvi[2] = $(xvi[2]) + xvi[3] = $(xvi[3]) + ") + + out = joinpath(figdir, "rank_$(igg.me)") + !isdir(out) && mkpath(out) + while t < tmax # Stokes solver ---------------- solve!( @@ -174,12 +192,17 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") dt, igg; kwargs = (; - iterMax = 150e3, + iterMax = 15e3, nout = 1e3, viscosity_cutoff = (-Inf, Inf) ) ) + jldsave( + joinpath(out, "it_" * lpad("$it", 6, "0") * ".jld2"), + stokes=stokes + ) + tensor_invariant!(stokes.ε) push!(τII, maximum(stokes.τ.xx)) @@ -199,6 +222,37 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") gather!(η_vep_nohalo, η_vep_v) gather!(εII_nohalo, εII_v) + # velocity2vertex!(Vx_v, Vy_v, Vz_v, @velocity(stokes)...) + # data_v = (; + # τxy= stokes.τ.xy, + # εxy= stokes.ε.xy, + # ) + # data_c = (; + # P = stokes.P, + # τxx = stokes.τ.xx, + # τyy = stokes.τ.yy, + # τzz = stokes.τ.zz, + # τII = stokes.τ.II, + # εxx = stokes.ε.xx, + # εyy = stokes.ε.yy, + # εzz = stokes.ε.zz, + # εII = stokes.ε.II, + # η = stokes.viscosity.η, + # ) + # velocity_v = ( + # Vx_v, + # Vy_v, + # Vz_v, + # ) + # save_vtk( + # joinpath(figdir, "vtk_rank_$(igg.me)_" * lpad("$it", 6, "0")), + # xvi, + # xci, + # data_v, + # data_c, + # velocity_v + # ) + # visualisation th = 0:pi/50:3*pi; xunit = @. radius * cos(th) + 0.5; @@ -226,12 +280,14 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") return nothing end -n = 18 -nx = n -ny = n -nz = n # if only 2 CPU/GPU are used nx = 17 - 2 with N =32 -figdir = "ShearBand3D_MPI" +n = 24 +nx = n * 2 +ny = n * 2 +nz = n * 2 # if only 2 CPU/GPU are used nx = 17 - 2 with N =32 +figdir = "ShearBand3D_MPI_1rank" +# figdir = "ShearBand3D_MPI_1rank" igg = if !(JustRelax.MPI.Initialized()) + # IGG(init_global_grid(((nx, ny, nz) .+ 1)...; init_MPI = true, select_device=false)...) IGG(init_global_grid(nx, ny, nz; init_MPI = true, select_device=false)...) else igg From 0c3569260433bb922bebf0a6913f555d7685a599 Mon Sep 17 00:00:00 2001 From: Albert de Montserrat Date: Mon, 22 Jul 2024 16:18:44 +0200 Subject: [PATCH 4/5] debugging upgrades --- src/Interpolations.jl | 11 ++--- src/stokes/Stokes3D.jl | 88 +++++++++++++++++++++------------------- src/topology/Topology.jl | 15 ++++--- 3 files changed, 60 insertions(+), 54 deletions(-) diff --git a/src/Interpolations.jl b/src/Interpolations.jl index 628fb181..2a1fc0fb 100644 --- a/src/Interpolations.jl +++ b/src/Interpolations.jl @@ -80,7 +80,9 @@ end end function center2vertex!(vertex_yz, vertex_xz, vertex_xy, center_yz, center_xz, center_xy) - @parallel center2vertex_kernel!( + ni = size(center_yz) + + @parallel (@idx ni.+1) center2vertex_kernel!( vertex_yz, vertex_xz, vertex_xy, center_yz, center_xz, center_xy ) return nothing @@ -91,8 +93,7 @@ end ) i1, j1, k1 = (i, j, k) .+ 1 nx, ny, nz = size(center_yz) - - if i ≤ nx && 1 < j1 ≤ ny && 1 < k1 ≤ nz + if i ≤ nx && j1 ≤ ny && k1 ≤ nz vertex_yz[i, j1, k1] = 0.25 * ( center_yz[i, j, k] + @@ -101,7 +102,7 @@ end center_yz[i, j1, k1] ) end - if 1 < i1 ≤ nx && j ≤ ny && 1 < k1 ≤ nz + if i1 ≤ nx && j ≤ ny && k1 ≤ nz vertex_xz[i1, j, k1] = 0.25 * ( center_xz[i, j, k] + @@ -110,7 +111,7 @@ end center_xz[i1, j, k1] ) end - if 1 < i1 ≤ nx && 1 < j1 ≤ ny && k ≤ nz + if i1 ≤ nx && j1 ≤ ny && k ≤ nz vertex_xy[i1, j1, k] = 0.25 * ( center_xy[i, j, k] + diff --git a/src/stokes/Stokes3D.jl b/src/stokes/Stokes3D.jl index 4df97ea7..28ee3436 100644 --- a/src/stokes/Stokes3D.jl +++ b/src/stokes/Stokes3D.jl @@ -267,17 +267,30 @@ function _solve!( args, ) - @parallel (@idx ni .+ 1) compute_τ_vertex!( - @shear(stokes.τ)..., - @shear(stokes.τ_o)..., - @shear(stokes.ε)..., - η_vep, - G, - dt, - pt_stokes.θ_dτ, + center2vertex!( + stokes.τ.yz, + stokes.τ.xz, + stokes.τ.xy, + stokes.τ.yz_c, + stokes.τ.xz_c, + stokes.τ.xy_c, ) + # update_halo!(stokes.τ.yz) + # update_halo!(stokes.τ.xz) + # update_halo!(stokes.τ.xy) + update_halo!(stokes.τ.xy, stokes.τ.yz,stokes.τ.xz) - @hide_communication b_width begin # communication/computation overlap + # @parallel (@idx ni .+ 1) compute_τ_vertex!( + # @shear(stokes.τ)..., + # @shear(stokes.τ_o)..., + # @shear(stokes.ε)..., + # η_vep, + # G, + # dt, + # pt_stokes.θ_dτ, + # ) + + # @hide_communication b_width begin # communication/computation overlap @parallel compute_V!( @velocity(stokes)..., stokes.R.Rx, @@ -291,36 +304,25 @@ function _solve!( _di..., ) # apply boundary conditions - velocity2displacement!(stokes, dt) flow_bcs!(stokes, flow_bcs) update_halo!(@velocity(stokes)...) - end + # update_halo!(stokes.V.Vx) + # update_halo!(stokes.V.Vy) + # update_halo!(stokes.V.Vz) + # end end + velocity2displacement!(stokes, dt) iter += 1 if iter % nout == 0 && iter > 1 cont += 1 - # push!(norm_Rx, maximum_mpi(abs.(stokes.R.Rx))) - # push!(norm_Ry, maximum_mpi(abs.(stokes.R.Ry))) - # push!(norm_Rz, maximum_mpi(abs.(stokes.R.Rz))) - # push!(norm_∇V, maximum_mpi(abs.(stokes.R.RP))) - push!( - norm_Rx, - norm_mpi(stokes.R.Rx[2:(end - 1), 2:(end - 1), 2:(end - 1)]) / - length(stokes.R.Rx), - ) - push!( - norm_Ry, - norm_mpi(stokes.R.Ry[2:(end - 1), 2:(end - 1), 2:(end - 1)]) / - length(stokes.R.Ry), - ) - push!( - norm_Rz, - norm_mpi(stokes.R.Rz[2:(end - 1), 2:(end - 1), 2:(end - 1)]) / - length(stokes.R.Rz), - ) - push!(norm_∇V, norm_mpi(stokes.R.RP) / length(stokes.R.RP)) - + for (norm_Ri, Ri) in zip((norm_Rx, norm_Ry, norm_Rz), @residuals(stokes.R)) + push!( + norm_Ri, + norm_mpi(Ri[2:(end - 1), 2:(end - 1), 2:(end - 1)]) / (length(Ri) * igg.nprocs), + ) + end + push!(norm_∇V, norm_mpi(stokes.R.RP) / (length(stokes.R.RP) * igg.nprocs )) err = max(norm_Rx[cont], norm_Ry[cont], norm_Rz[cont], norm_∇V[cont]) push!(err_evo1, err) push!(err_evo2, iter) @@ -486,13 +488,16 @@ function _solve!( stokes.τ.xz_c, stokes.τ.xy_c, ) + # update_halo!(stokes.τ.yz) + # update_halo!(stokes.τ.xz) + # update_halo!(stokes.τ.xy) update_halo!(stokes.τ.yz, stokes.τ.xz, stokes.τ.xy) # @parallel (@idx ni .+ 1) compute_τ_vertex!( # @shear(stokes.τ)..., @shear(stokes.ε)..., η_vep, pt_stokes.θ_dτ # ) - @hide_communication b_width begin # communication/computation overlap + # @hide_communication b_width begin # communication/computation overlap @parallel compute_V!( @velocity(stokes)..., @residuals(stokes.R)..., @@ -504,14 +509,15 @@ function _solve!( _di..., ) # apply boundary conditions - velocity2displacement!(stokes, dt) - free_surface_bcs!(stokes, flow_bcs, η, rheology, phase_ratios, dt, di) + # free_surface_bcs!(stokes, flow_bcs, η, rheology, phase_ratios, dt, di) flow_bcs!(stokes, flow_bcs) - update_halo!(@velocity(stokes)...) - end + update_halo!(stokes.V.Vx) + update_halo!(stokes.V.Vy) + update_halo!(stokes.V.Vz) + # update_halo!(@velocity(stokes)...) + # end end - - stokes.P .= θ + velocity2displacement!(stokes, dt) iter += 1 if iter % nout == 0 && iter > 1 @@ -519,10 +525,10 @@ function _solve!( for (norm_Ri, Ri) in zip((norm_Rx, norm_Ry, norm_Rz), @residuals(stokes.R)) push!( norm_Ri, - norm_mpi(Ri[2:(end - 1), 2:(end - 1), 2:(end - 1)]) / length(Ri), + norm_mpi(Ri[2:(end - 1), 2:(end - 1), 2:(end - 1)]) / (length(Ri) * igg.nprocs), ) end - push!(norm_∇V, norm_mpi(stokes.R.RP) / length(stokes.R.RP)) + push!(norm_∇V, norm_mpi(stokes.R.RP) / (length(stokes.R.RP) * igg.nprocs )) err = max(norm_Rx[cont], norm_Ry[cont], norm_Rz[cont], norm_∇V[cont]) push!(err_evo1, err) push!(err_evo2, iter) diff --git a/src/topology/Topology.jl b/src/topology/Topology.jl index 25c6f51e..c7f410e2 100644 --- a/src/topology/Topology.jl +++ b/src/topology/Topology.jl @@ -52,24 +52,23 @@ function lazy_grid(di::NTuple{N,T1}, ni; origin=ntuple(_ -> zero(T1), Val(N))) w xci = ntuple(Val(N)) do i Base.@_inline_meta rank_origin = f_g[i](1, di[i], ni[i]) - local_origin = rank_origin + origin[i] - rank_end = f_g[i](ni[i], di[i], ni[i]) + + rank_end = f_g[i](ni[i] + 1, di[i], ni[i]) local_end = rank_end + origin[i] - - @inbounds LinRange(local_origin[i] + di[i] / 2, local_end[i] + di[i] / 2, ni[i]) + + @inbounds LinRange(local_origin[i] + di[i] / 2, local_end[i]- di[i] / 2, ni[i]) end # nodes at the vertices of the grid cells xvi = ntuple(Val(N)) do i - # println("potato") Base.@_inline_meta rank_origin = f_g[i](1, di[i], ni[i]) local_origin = rank_origin + origin[i] - - rank_end = f_g[i](ni[i] + 1, di[i], ni[i]) + + rank_end = f_g[i](ni[i] + 1 , di[i], ni[i]) local_end = rank_end + origin[i] - + @inbounds LinRange(local_origin[i], local_end[i], ni[i] + 1) end From 07541b02d522a40dca5a19ec998995703e78351b Mon Sep 17 00:00:00 2001 From: Albert de Montserrat Date: Thu, 8 Aug 2024 13:30:23 +0200 Subject: [PATCH 5/5] adjust CFL --- .../stokes3D/shear_band/ShearBand3D.jl | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D.jl b/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D.jl index 1f09b8cb..44b44dcd 100644 --- a/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D.jl +++ b/miniapps/benchmarks/stokes3D/shear_band/ShearBand3D.jl @@ -116,7 +116,7 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") # STOKES --------------------------------------------- # Allocate arrays needed for every Stokes problem stokes = StokesArrays(backend_JR, ni) - pt_stokes = PTStokesCoeffs(li, di; ϵ = 1e-6, CFL = 0.05 / √3.1) + pt_stokes = PTStokesCoeffs(li, di; ϵ = 1e-6, CFL = 0.5 / √3.1) # Buoyancy forces ρg = @zeros(ni...), @zeros(ni...), @zeros(ni...) args = (; T = @zeros(ni...), P = stokes.P, dt = Inf) @@ -179,32 +179,34 @@ function main(igg; nx=64, ny=64, nz=64, figdir="model_figs") println("it = $it; t = $t \n") # visualisation - slice_j = ny >>> 1 - fig = Figure(size = (1600, 1600), title = "t = $t") - ax1 = Axis(fig[1,1], aspect = 1, title = "τII") - ax2 = Axis(fig[2,1], aspect = 1, title = "η_vep") - ax3 = Axis(fig[1,2], aspect = 1, title = "log10(εII)") - ax4 = Axis(fig[2,2], aspect = 1) - heatmap!(ax1, xci[1], xci[3], Array(stokes.τ.II[:, slice_j, :]) , colormap=:batlow) - heatmap!(ax2, xci[1], xci[3], Array(log10.(stokes.viscosity.η_vep)[:, slice_j, :]) , colormap=:batlow) - heatmap!(ax3, xci[1], xci[3], Array(log10.(stokes.ε.II)[:, slice_j, :]) , colormap=:batlow) - lines!(ax4, ttot, τII, color = :black) - lines!(ax4, ttot, sol, color = :red) - hidexdecorations!(ax1) - hidexdecorations!(ax3) - - for ax in (ax1, ax2, ax3) - xlims!(ax, (0, 1)) - ylims!(ax, (0, 1)) + if igg.me == 0 + slice_j = ny >>> 1 + fig = Figure(size = (1600, 1600), title = "t = $t") + ax1 = Axis(fig[1,1], aspect = 1, title = "τII") + ax2 = Axis(fig[2,1], aspect = 1, title = "η_vep") + ax3 = Axis(fig[1,2], aspect = 1, title = "log10(εII)") + ax4 = Axis(fig[2,2], aspect = 1) + heatmap!(ax1, xci[1], xci[3], Array(stokes.τ.II[:, slice_j, :]) , colormap=:batlow) + heatmap!(ax2, xci[1], xci[3], Array(log10.(stokes.viscosity.η_vep)[:, slice_j, :]) , colormap=:batlow) + heatmap!(ax3, xci[1], xci[3], Array(log10.(stokes.ε.II)[:, slice_j, :]) , colormap=:batlow) + lines!(ax4, ttot, τII, color = :black) + lines!(ax4, ttot, sol, color = :red) + hidexdecorations!(ax1) + hidexdecorations!(ax3) + + for ax in (ax1, ax2, ax3) + xlims!(ax, (0, 1)) + ylims!(ax, (0, 1)) + end + save(joinpath(figdir, "$(it).png"), fig) end - save(joinpath(figdir, "$(it).png"), fig) end return nothing end -n = 32 -nx = ny = nz = n +n = 32 + 4 +nx = ny = nz = n #÷ 2 figdir = "ShearBand3D_noMPI" igg = if !(JustRelax.MPI.Initialized()) IGG(init_global_grid(nx, ny, nz; init_MPI = true)...) @@ -212,4 +214,4 @@ else igg end -main(igg; figdir = figdir, nx = nx, ny = ny, nz = nz); +main(igg; figdir = figdir, nx = nx, ny = ny, nz = nz); \ No newline at end of file