Skip to content

Commit

Permalink
created acc multicore routines
Browse files Browse the repository at this point in the history
  • Loading branch information
@mbeidler3
mbeidler3 committed May 21, 2024
1 parent 6beca22 commit 191323b
Show file tree
Hide file tree
Showing 3 changed files with 784 additions and 17 deletions.
170 changes: 170 additions & 0 deletions src/korc_collisions.f90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2586,6 +2586,176 @@ subroutine include_CoulombCollisions_GC_p(tt,params,Y_R,Y_PHI,Y_Z, &

end subroutine include_CoulombCollisions_GC_p

subroutine include_CoulombCollisions_GC_p_ACC(tt,params,Y_R,Y_PHI,Y_Z, &
Ppll,Pmu,me,flagCon,flagCol,F,P,E_PHI,ne,Te,Zeff,nimp,PSIp)
!$acc routine seq
TYPE(PROFILES), INTENT(IN) :: P
TYPE(FIELDS), INTENT(IN) :: F
TYPE(KORC_PARAMS), INTENT(INOUT) :: params
REAL(rp), INTENT(INOUT) :: Ppll
REAL(rp), INTENT(INOUT) :: Pmu
REAL(rp) :: Bmag
REAL(rp) :: B_R,B_PHI,B_Z
REAL(rp) :: curlb_R,curlb_PHI,curlb_Z
REAL(rp) :: gradB_R,gradB_PHI,gradB_Z
REAL(rp) :: E_R,E_Z
REAL(rp), INTENT(OUT) :: E_PHI,ne,PSIp,Te,Zeff
REAL(rp), INTENT(IN) :: Y_R,Y_PHI,Y_Z
INTEGER(is), INTENT(INOUT) :: flagCol
INTEGER(is), INTENT(INOUT) :: flagCon
REAL(rp),INTENT(IN) :: me
REAL(rp) :: nAr0,nAr1,nAr2,nAr3,nAr4
REAL(rp) :: nD,nD1
REAL(rp),DIMENSION(2) :: dW
REAL(rp),DIMENSION(2) :: rnd1
REAL(rp) :: dt,time
REAL(rp) :: pm
REAL(rp) :: dp
REAL(rp) :: xi
REAL(rp) :: dxi
REAL(rp) :: v,gam
!! speed of particle
REAL(rp) :: CAL
REAL(rp) :: dCAL
REAL(rp) :: CFL
REAL(rp) :: CBL
REAL(rp) :: SC_p,SC_mu,BREM_p
REAL(rp) :: kappa
integer(ip),INTENT(IN) :: tt
REAL(rp), DIMENSION(params%num_impurity_species), INTENT(OUT) :: nimp
REAL(rp) :: E_PHI_tmp

!$acc routine (calculate_GCfieldswE_p_ACC) seq
!$acc routine (interp_Hcollision_p_ACC) seq

pchunk=params%pchunk

if (MODULO(params%it+tt,cparams_ss%subcycling_iterations) .EQ. 0_ip) then
dt = REAL(cparams_ss%subcycling_iterations,rp)*params%dt
time=params%init_time+(params%it-1+tt)*params%dt

#ifdef PSPLINE

call calculate_GCfieldswE_p_ACC(F,Y_R,Y_PHI,Y_Z,B_R,B_PHI,B_Z, &
E_R,E_PHI,E_Z,curlb_R,curlb_PHI,curlb_Z, &
gradB_R,gradB_PHI,gradB_Z,flagCon,PSIp)

#endif PSPLINE

#ifdef PSPLINE

call interp_Hcollision_p_ACC(params,Y_R,Y_PHI,Y_Z,ne,Te,Zeff, &
nAr0,nAr1,nAr2,nAr3,nAr4,nD,nD1,flagCon)

nimp(1)=nAr0
nimp(2)=nAr1
nimp(3)=nAr2
nimp(4)=nAr3
nimp(5)=nAr4
nimp(6)=nD
nimp(7)=nD1


!write(6,*) 'collisions ne',ne(1)*params%cpp%density


#endif PSPLINE


E_PHI_tmp=E_PHI
if (.not.params%FokPlan) E_PHI=0._rp

Bmag=sqrt(B_R*B_R+B_PHI*B_PHI+B_Z*B_Z)
! Transform p_pll,mu to P,eta
pm = SQRT(Ppll*Ppll+2*me*Bmag*Pmu)
xi = Ppll/pm

gam = sqrt(1+pm*pm)

v = pm/gam
! normalized speed (v_K=v_P/c)


#ifdef PARALLEL_RANDOM
rnd1(1) = get_random_U()
rnd1(2) = get_random_U()
! rnd1(:,1) = get_random_mkl()
! rnd1(:,2) = get_random_mkl()
#else
call RANDOM_NUMBER(rnd1)
#endif PARALLEL_RANDOM

dW(1) = SQRT(3*dt)*(-1+2*rnd1(1))
dW(2) = SQRT(3*dt)*(-1+2*rnd1(2))

! write(output_unit_write,'("dW1: ",E17.10)') dW(cc,1)
! write(output_unit_write,'("dW2: ",E17.10)') dW(cc,2)

if (params%profile_model(10:10).eq.'H') then
CAL = CA_SD(v,ne,Te)
dCAL= dCA_SD(v,me,ne,Te)
CFL = CF_SD_FIO(params,v,ne,Te,nimp)
CBL = (CB_ee_SD(v,ne,Te,Zeff)+ &
CB_ei_SD_FIO(params,v,ne,Te,nimp,Zeff))
else
CAL = CA_SD(v,ne,Te)
dCAL= dCA_SD(v,me,ne,Te)
CFL = CF_SD(params,v,ne,Te,P,Y_R,Y_Z)
CBL = (CB_ee_SD(v,ne,Te,Zeff)+ &
CB_ei_SD(params,v,ne,Te,Zeff,P,Y_R,Y_Z))
endif

if (.not.cparams_ss%slowing_down) CFL=0._rp
if (.not.cparams_ss%pitch_diffusion) CBL=0._rp
if (.not.cparams_ss%energy_diffusion) THEN
CAL=0._rp
dCAL=0._rp
ENDIF

dp=REAL(flagCol)*REAL(flagCon)* &
((-CFL+dCAL+E_PHI*xi)*dt+ &
sqrt(2.0_rp*CAL)*dW(1))

dxi=REAL(flagCol)*REAL(flagCon)* &
((-2*xi*CBL/(pm*pm)+ &
E_PHI*(1-xi*xi)/pm)*dt- &
sqrt(2.0_rp*CBL*(1-xi*xi))/pm*dW(2))


pm=pm+dp
xi=xi+dxi




if (xi>1) then

xi=1-mod(xi,1._rp)
else if (xi<-1) then
xi=-1-mod(xi,-1._rp)

endif



! Transform P,xi to p_pll,mu
Ppll=pm*xi
Pmu=(pm*pm-Ppll*Ppll)/(2*me*Bmag)



if ((pm.lt.min(cparams_ss%p_min*cparams_ss%pmin_scale, &
p_therm)).and.flagCol.eq.1_ip) then
flagCol=0_ip
end if


if (.not.params%FokPlan) E_PHI=E_PHI_tmp

end if

end subroutine include_CoulombCollisions_GC_p_ACC

subroutine include_CoulombCollisionsLA_GC_p(spp,achunk,tt,params, &
Y_R,Y_PHI,Y_Z,Ppll,Pmu,me,flagCon,flagCol,F,P,E_PHI,ne,Te,PSIp)

Expand Down
131 changes: 114 additions & 17 deletions src/korc_interp.f90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3367,6 +3367,76 @@ subroutine calculate_GCfieldswE_p(pchunk,F,Y_R,Y_PHI,Y_Z,B_R,B_PHI,B_Z,E_R,E_PHI

end subroutine calculate_GCfieldswE_p

subroutine calculate_GCfieldswE_p_ACC(F,Y_R,Y_PHI,Y_Z,B_R,B_PHI,B_Z,E_R,E_PHI,E_Z, &
curlb_R,curlb_PHI,curlb_Z,gradB_R,gradB_PHI,gradB_Z,flag_cache,PSIp)
!$acc routine seq
REAL(rp), INTENT(IN) :: Y_R,Y_PHI,Y_Z
TYPE(FIELDS), INTENT(IN) :: F
REAL(rp), INTENT(OUT) :: B_R,B_PHI,B_Z
REAL(rp), INTENT(OUT) :: gradB_R,gradB_PHI,gradB_Z
REAL(rp), INTENT(OUT) :: curlb_R,curlb_PHI,curlb_Z
REAL(rp), INTENT(OUT) :: E_R,E_PHI,E_Z
REAL(rp) :: Bmag,EPHI
INTEGER :: cc
REAL(rp),INTENT(OUT) :: PSIp
REAL(rp), DIMENSION(6) :: A
INTEGER(is),INTENT(INOUT) :: flag_cache
REAL(rp) :: psip_conv

!$acc routine (EZspline_interp2_GCvarswE) seq
!$acc routine (EZspline_error) seq

psip_conv=F%psip_conv

call EZspline_interp2_GCvarswE(bfield_2d%A, efield_2d%PHI, Y_R, Y_Z, A, &
EPHI, ezerr)
call EZspline_error(ezerr)

!A(:,1) = PSIp
!A(:,2) = dPSIp/dR
!A(:,3) = dPSIp/dZ
!A(:,4) = d^2PSIp/dR^2
!A(:,5) = d^2PSIp/dZ^2
!A(:,6) = d^2PSIp/dRdZ

PSIp=A(1)

B_R = psip_conv*A(3)/Y_R
! BR = (dA/dZ)/R
B_PHI = -F%Bo*F%Ro/Y_R
! BPHI = Fo*Ro/R
B_Z = -psip_conv*A(2)/Y_R
! BR = -(dA/dR)/R



Bmag=sqrt(B_R*B_R+B_PHI*B_PHI+B_Z*B_Z)

gradB_R=(B_R*psip_conv*A(6)-B_Z*psip_conv*A(4)- &
Bmag*Bmag)/(Y_R*Bmag)
gradB_PHI=0._rp
gradB_Z=(B_R*psip_conv*A(5)-B_Z*psip_conv*A(6))/ &
(Y_R*Bmag)

curlb_R=B_PHI*gradB_Z/(Bmag*Bmag)
curlb_PHI=(Bmag/Y_R*(B_Z+psip_conv*A(4)+ &
psip_conv*A(5))-B_R*gradB_Z+B_Z*gradB_R)/ &
(Bmag*Bmag)
curlb_Z=-B_PHI*gradB_R/(Bmag*Bmag)

if (F%E_2x1t) then
E_R = 0._rp
E_PHI = EPHI
E_Z = 0._rp
else
E_R = 0._rp
E_PHI = 0._rp
E_Z = 0._rp
end if


end subroutine calculate_GCfieldswE_p_ACC

subroutine calculate_GCfields_p(pchunk,F,Y_R,Y_PHI,Y_Z,B_R,B_PHI,B_Z, &
E_R,E_PHI,E_Z, &
curlb_R,curlb_PHI,curlb_Z,gradB_R,gradB_PHI,gradB_Z,flag_cache,PSIp)
Expand Down Expand Up @@ -3778,36 +3848,63 @@ end subroutine interp_fields

#ifdef PSPLINE
subroutine interp_Hcollision_p(params,pchunk,Y_R,Y_PHI,Y_Z,ne,Te,Zeff, &
nAr0,nAr1,nAr2,nAr3,nAr4,nD,nD1,flag_cache)
TYPE(KORC_PARAMS), INTENT(IN) :: params
INTEGER, INTENT(IN) :: pchunk
REAL(rp),DIMENSION(pchunk),INTENT(IN) :: Y_R,Y_PHI,Y_Z
REAL(rp),DIMENSION(pchunk),INTENT(OUT) :: ne,Te,Zeff
REAL(rp),DIMENSION(pchunk),INTENT(OUT) :: nAr0,nAr1,nAr2,nAr3,nAr4,nD,nD1
INTEGER(is),DIMENSION(pchunk),INTENT(INOUT) :: flag_cache
INTEGER :: cc

call check_if_in_profiles_domain_p(pchunk,Y_R,Y_PHI,Y_Z,flag_cache)
! write(output_unit_write,'("YR: ",E17.10)') Y_R(1)
! write(output_unit_write,'("YPHI: ",E17.10)') Y_PHI(1)
! write(output_unit_write,'("YZ: ",E17.10)') Y_Z(1)

! write(output_unit_write,'("Te_interp_R",E17.10)') profiles_2d%Te%x1
! write(output_unit_write,'("Te_interp_Z",E17.10)') profiles_2d%Te%x2

do cc=1_idef,pchunk
call EZspline_interp2_Hcollision(profiles_2d%ne,profiles_2d%Te,profiles_2d%Zeff, &
profiles_2d%nAr0,profiles_2d%nAr1,profiles_2d%nAr2,profiles_2d%nAr3,profiles_2d%nAr4, &
profiles_2d%nD,profiles_2d%nD1,Y_R(cc),Y_Z(cc),ne(cc),Te(cc),Zeff(cc), &
nAr0(cc),nAr1(cc),nAr2(cc),nAr3(cc),nAr4(cc),nD(cc),nD1(cc),ezerr)
call EZspline_error(ezerr)
end do

!write(6,*) 'interp ne%fspl',profiles_2d%ne%fspl(1,12,15)*params%cpp%density

!write(6,*) 'interp ne',ne(1)

end subroutine interp_Hcollision_p

subroutine interp_Hcollision_p_ACC(params,Y_R,Y_PHI,Y_Z,ne,Te,Zeff, &
nAr0,nAr1,nAr2,nAr3,nAr4,nD,nD1,flag_cache)
!$acc routine seq
TYPE(KORC_PARAMS), INTENT(IN) :: params
INTEGER, INTENT(IN) :: pchunk
REAL(rp),DIMENSION(pchunk),INTENT(IN) :: Y_R,Y_PHI,Y_Z
REAL(rp),DIMENSION(pchunk),INTENT(OUT) :: ne,Te,Zeff
REAL(rp),DIMENSION(pchunk),INTENT(OUT) :: nAr0,nAr1,nAr2,nAr3,nAr4,nD,nD1
INTEGER(is),DIMENSION(pchunk),INTENT(INOUT) :: flag_cache
INTEGER :: cc
REAL(rp),INTENT(IN) :: Y_R,Y_PHI,Y_Z
REAL(rp),INTENT(OUT) :: ne,Te,Zeff
REAL(rp),INTENT(OUT) :: nAr0,nAr1,nAr2,nAr3,nAr4,nD,nD1
INTEGER(is),INTENT(INOUT) :: flag_cache

call check_if_in_profiles_domain_p(pchunk,Y_R,Y_PHI,Y_Z,flag_cache)
! write(output_unit_write,'("YR: ",E17.10)') Y_R(1)
! write(output_unit_write,'("YPHI: ",E17.10)') Y_PHI(1)
! write(output_unit_write,'("YZ: ",E17.10)') Y_Z(1)

! write(output_unit_write,'("Te_interp_R",E17.10)') profiles_2d%Te%x1
! write(output_unit_write,'("Te_interp_Z",E17.10)') profiles_2d%Te%x2
!$acc routine (EZspline_interp2_Hcollision) seq
!$acc routine (EZspline_error) seq


do cc=1_idef,pchunk
call EZspline_interp2_Hcollision(profiles_2d%ne,profiles_2d%Te,profiles_2d%Zeff, &
profiles_2d%nAr0,profiles_2d%nAr1,profiles_2d%nAr2,profiles_2d%nAr3,profiles_2d%nAr4, &
profiles_2d%nD,profiles_2d%nD1,Y_R(cc),Y_Z(cc),ne(cc),Te(cc),Zeff(cc), &
nAr0(cc),nAr1(cc),nAr2(cc),nAr3(cc),nAr4(cc),nD(cc),nD1(cc),ezerr)
profiles_2d%nD,profiles_2d%nD1,Y_R,Y_Z,ne,Te,Zeff, &
nAr0,nAr1,nAr2,nAr3,nAr4,nD,nD1,ezerr)
call EZspline_error(ezerr)
end do


!write(6,*) 'interp ne%fspl',profiles_2d%ne%fspl(1,12,15)*params%cpp%density

!write(6,*) 'interp ne',ne(1)

end subroutine interp_Hcollision_p
end subroutine interp_Hcollision_p_ACC

subroutine interp_nRE(params,Y_R,Y_PHI,Y_Z,PSIp,EPHI,ne,Te,nRE, &
nAr0,nAr1,nAr2,nAr3,nAr4,nD,nD1,g_test,fRE_out,rho1D)
Expand Down
Loading

0 comments on commit 191323b

Please sign in to comment.