setting fortran as linking language, fixing bug with AORSA interpolation

.github/workflows/spack.yml

@@ -82,7 +82,7 @@ jobs:
           set -x
 
           cd ./KORC
-          ./build.sh
+          ./build_gnu.sh
 
       - name: Show Cache
         if: failure()

build.sh → build_gnu.sh

@@ -12,7 +12,7 @@ cmake -DCMAKE_BUILD_TYPE:String=$BUILD_TYPE \
       -DUSE_PSPLINE=OFF \
       -DUSE_FIO=OFF \
       -DKORC_TEST=OFF \
-      -DCMAKE_Fortran_FLAGS="-DHDF5_DOUBLE_PRESICION -malign-double -fconvert='big-endian'" \
+      -DCMAKE_Fortran_FLAGS="-DHDF5_DOUBLE_PRESICION -malign-double -fconvert='big-endian' -fPIE" \
       -DCMAKE_C_FLAGS="-malign-double"  \
       -DCMAKE_CXX_FLAGS="-malign-double" \
       -DCMAKE_Fortran_FLAGS_DEBUG="-g3 -ffpe-trap=zero,overflow -fbacktrace" \

build_scripts/build_gnu.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+BUILD_TYPE=Debug
+
+rm -f CMakeCache.txt
+rm -rf CMakeFiles
+
+rm -rf ./build && mkdir $_
+cd build
+
+cmake -DCMAKE_BUILD_TYPE:String=$BUILD_TYPE \
+      -DUSE_PSPLINE=OFF \
+      -DUSE_FIO=OFF \
+      -DKORC_TEST=OFF \
+      -DCMAKE_Fortran_FLAGS="-DHDF5_DOUBLE_PRESICION -malign-double -fconvert='big-endian' -fPIE" \
+      -DCMAKE_C_FLAGS="-malign-double"  \
+      -DCMAKE_CXX_FLAGS="-malign-double" \
+      -DCMAKE_Fortran_FLAGS_DEBUG="-g3 -ffpe-trap=zero,overflow -fbacktrace" \
+      -DCMAKE_C_FLAGS_DEBUG="-g3" \
+      -DCMAKE_CXX_FLAGS_DEBUG="-g3" ../
+
+make -j VERBOSE=1

build_scripts/build_nv_cpu.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+BUILD_TYPE=Debug
+
+rm -f CMakeCache.txt
+rm -rf CMakeFiles
+
+rm -rf ./build && mkdir $_
+cd build
+
+cmake \
+  -DCMAKE_BUILD_TYPE:String=$BUILD_TYPE \
+  -DUSE_OMP=ON \
+  -DUSE_PSPLINE=ON \
+  -DCMAKE_Fortran_FLAGS="-Mfree -Mpreprocess -fPIC -O3 -mp -Mvect=simd:256 -c++libs -Mbyteswapio -DHDF5_DOUBLE_PRESICION" \
+  -DCMAKE_Fortran_FLAGS_DEBUG="-g -Minstrument -traceback -lnvhpcwrapnvtx" \
+  -DCMAKE_C_FLAGS="-O3 -mp -Mvect=simd:256" \
+  -DCMAKE_C_FLAGS_DEBUG="-g -traceback" \
+  -DCMAKE_CXX_FLAGS="-O3 -std=c++11 -mp -Mvect=simd:256" \
+  -DCMAKE_CXX_FLAGS_DEBUG="-g -traceback" \
+  ..
+
+make -j VERBOSE=1

build_scripts/build_nv_gpu.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+BUILD_TYPE=Debug
+
+rm -f CMakeCache.txt
+rm -rf CMakeFiles
+
+rm -rf ./build && mkdir $_
+cd build
+
+  cmake \
+    -DCMAKE_BUILD_TYPE:String=$BUILD_TYPE \
+    -DUSE_OMP=OFF \
+    -DUSE_ACC=ON \
+    -DUSE_PSPLINE=ON \
+    -DCMAKE_Fortran_FLAGS="-acc=gpu -gpu=deepcopy,debug -Mfree -fPIC -c++libs -Mpreprocess -DHDF5_DOUBLE_PRESICION" \
+    -DCMAKE_Fortran_FLAGS_DEBUG="-g -Minstrument -traceback -lnvhpcwrapnvtx" \
+    -DCMAKE_C_FLAGS="-mp" \
+    -DCMAKE_C_FLAGS_DEBUG="-g -G -traceback" \
+    -DCMAKE_CXX_FLAGS="-std=c++11 -mp" \
+    -DCMAKE_CXX_FLAGS_DEBUG="-g -traceback" \
+    ..
+
+make -j VERBOSE=1

setup_cmake

@@ -92,29 +92,31 @@ then
     rm -f CMakeCache.txt
     rm -rf CMakeFiles
 
-    rm -rf ./build && mkdir $_
+    rm -rf ./build && mkdir $_ && cd $_
 
     COMPILER_TYPE=NV
-    HARDWARE_TYPE=GPU
+    HARDWARE_TYPE=CPU
 
     if [ "$COMPILER_TYPE" == GNU ]
     then
-      export CC=/usr/bin/gcc-12
-      export CXX=/usr/bin/g++-12
-      export FC=/usr/bin/gfortran-12
+
+      export FC=/home/21b/KORC/build_gnu/.spack-env/view/bin/gfortran
+      export CC=/home/21b/KORC/build_gnu/.spack-env/view/bin/gcc
+      export CXX=/home/21b/KORC/build_gnu/.spack-env/view/bin/g++
 
       cmake -DCMAKE_BUILD_TYPE:String=$BUILD_TYPE \
-        -DUSE_OMP=OFF \
-        -DUSE_PSPLINE=OFF \
+        -DUSE_OMP=ON \
+        -DUSE_PSPLINE=ON \
         -DUSE_FIO=OFF \
         -DCORI_DIR=OFF \
         -DKORC_TEST=OFF \
         -DCMAKE_Fortran_FLAGS="-O3 -DHDF5_DOUBLE_PRESICION -fopenmp" \
         -DCMAKE_C_FLAGS="-O3 -fopenmp"  \
         -DCMAKE_CXX_FLAGS="-O3 -fopenmp" \
-        -DCMAKE_Fortran_FLAGS_DEBUG="-g -ffpe-trap=invalid,zero,overflow -fbacktrace -Werror" \
-        -DCMAKE_C_FLAGS_DEBUG="-g" \
-        -DCMAKE_CXX_FLAGS_DEBUG="-g" \
+        -DCMAKE_Fortran_FLAGS_DEBUG="-g3 -ffpe-trap=zero,overflow -fbacktrace" \
+        -DCMAKE_C_FLAGS_DEBUG="-g3" \
+        -DCMAKE_CXX_FLAGS_DEBUG="-g3" \
+        ..
 
     elif [ "$COMPILER_TYPE" == NV ]
     then
@@ -124,12 +126,13 @@ then
           -DCMAKE_BUILD_TYPE:String=$BUILD_TYPE \
           -DUSE_OMP=ON \
           -DCMAKE_Fortran_FLAGS="-Mfree -Mpreprocess -fPIC -O3 -mp -Mvect=simd:256 -c++libs -Mbyteswapio -DHDF5_DOUBLE_PRESICION" \
-          -DCMAKE_Fortran_FLAGS_DEBUG="-g -Minfo=all -Minstrument -traceback -lnvhpcwrapnvtx" \
+          -DCMAKE_Fortran_FLAGS_DEBUG="-g -Minstrument -traceback -lnvhpcwrapnvtx" \
           -DCMAKE_C_FLAGS="-O3 -mp -Mvect=simd:256" \
-          -DCMAKE_C_FLAGS_DEBUG="-g -Minfo=all -traceback" \
+          -DCMAKE_C_FLAGS_DEBUG="-g -traceback" \
           -DCMAKE_CXX_FLAGS="-O3 -std=c++11 -mp -Mvect=simd:256" \
-          -DCMAKE_CXX_FLAGS_DEBUG="-g -Minfo=all -traceback" \
-          -DUSE_PSPLINE=ON 
+          -DCMAKE_CXX_FLAGS_DEBUG="-g -traceback" \
+          -DUSE_PSPLINE=ON \
+          ..
 
       elif [ "$HARDWARE_TYPE" == GPU ]
       then
@@ -138,11 +141,11 @@ then
           -DUSE_OMP=OFF \
           -DUSE_ACC=ON \
           -DUSE_PSPLINE=ON \
-          -DCMAKE_Fortran_FLAGS="-fast -acc=gpu -gpu=deepcopy -Mfree -fPIC -O3 -c++libs -Mpreprocess -DHDF5_DOUBLE_PRESICION" \
+          -DCMAKE_Fortran_FLAGS="-acc=gpu -gpu=deepcopy,debug -Mfree -fPIC -c++libs -Mpreprocess -DHDF5_DOUBLE_PRESICION" \
           -DCMAKE_Fortran_FLAGS_DEBUG="-g -Minfo=all -Minstrument -traceback -lnvhpcwrapnvtx" \
-          -DCMAKE_C_FLAGS="-O3 -mp" \
-          -DCMAKE_C_FLAGS_DEBUG="-g -Minfo=all -traceback" \
-          -DCMAKE_CXX_FLAGS="-O3 -std=c++11 -mp" \
+          -DCMAKE_C_FLAGS="-mp" \
+          -DCMAKE_C_FLAGS_DEBUG="-g -G -Minfo=all -traceback" \
+          -DCMAKE_CXX_FLAGS="-std=c++11 -mp" \
           -DCMAKE_CXX_FLAGS_DEBUG="-g -Minfo=all -traceback" 
 
       else

src/CMakeLists.txt

@@ -36,6 +36,7 @@ target_link_libraries (korc
                        korc_depends
 )
 
+
 add_executable (xkorc)
 target_sources (xkorc
 
@@ -49,3 +50,7 @@ target_link_libraries (xkorc
 
                        korc
 )
+
+
+set_property(TARGET xkorc PROPERTY LINKER_LANGUAGE Fortran)
+

src/korc_fields.f90

@@ -882,6 +882,24 @@ subroutine uniform_magnetic_field(F,B)
    B(:,2:3) = 0.0_rp
 end subroutine uniform_magnetic_field
 
+subroutine uniform_fields_p_ACC(Bo,Eo,B_X,B_Y,B_Z,E_X,E_Y,E_Z)
+   !$acc routine seq
+   REAL(rp), INTENT(IN)   :: Bo,Eo
+   REAL(rp), INTENT(OUT)   :: B_X,B_Y,B_Z
+   REAL(rp), INTENT(OUT)   :: E_X,E_Y,E_Z
+   !! Magnetic field components in Cartesian coordinates;
+   !! B(1,:) = \(B_x\), B(2,:) = \(B_y\), B(3,:) = \(B_z\)
+
+   B_X = Bo
+   B_Y = 0._rp
+   B_Z = 0._rp
+
+   E_X = Eo
+   E_Y = 0._rp
+   E_Z = 0._rp
+
+end subroutine uniform_fields_p_ACC
+
 subroutine uniform_fields_p(pchunk,F,B_X,B_Y,B_Z,E_X,E_Y,E_Z)
    INTEGER, INTENT(IN) :: pchunk
    !! @note Subroutine that returns the value of a uniform magnetic
@@ -1075,6 +1093,8 @@ subroutine unitVectors(params,Xo,F,b1,b2,b3,flag,cart,hint,Bo)
    ALLOCATE( vars%X(ppp,3) )
    ALLOCATE( vars%Y(ppp,3) )
    ALLOCATE( vars%B(ppp,3) )
+   ALLOCATE( vars%BR(ppp,3) ) !using for B0 in MARS field evaluation
+   ALLOCATE( vars%BPHI(ppp,3) ) !using for B1 in MARS field evaluation
    ALLOCATE( vars%gradB(ppp,3) )
    ALLOCATE( vars%curlb(ppp,3) )
    ALLOCATE( vars%PSI_P(ppp) )
@@ -1093,6 +1113,8 @@ subroutine unitVectors(params,Xo,F,b1,b2,b3,flag,cart,hint,Bo)
    vars%flagCon = flag
    vars%initLCFS = 0_is
    vars%B=0._rp
+   vars%BR=0._rp
+   vars%BPHI=0._rp
    vars%PSI_P=0._rp
    vars%cart=.false.
 
@@ -1112,11 +1134,14 @@ subroutine unitVectors(params,Xo,F,b1,b2,b3,flag,cart,hint,Bo)
 
       !write(output_unit_write,*) 'before b1,b2,b3 calculation'
 
-    Bo=vars%B
-
    tmpvec=(/1.0_rp,1.0_rp,1.0_rp/)
 
    do ii=1_idef,ppp
+
+      !if ((vars%B(ii,1).ne.vars%BR(ii,1)).or.(vars%B(ii,2).ne.vars%BR(ii,2)).or.(vars%B(ii,3).ne.vars%BR(ii,3))) then
+      !   write(6,*) 'fields',ii,vars%B(ii,1),vars%B(ii,2),vars%B(ii,3),vars%BR(ii,1),vars%BR(ii,2),vars%BR(ii,3),vars%BPHI(ii,1),vars%BPHI(ii,2),vars%BPHI(ii,3)
+      !endif
+
       !write(6,*) 'ii',ii
       if ( vars%flagCon(ii) .EQ. 1_idef ) then
          b1tmp = vars%B(ii,:)/sqrt(vars%B(ii,1)*vars%B(ii,1)+ &
@@ -1150,6 +1175,8 @@ subroutine unitVectors(params,Xo,F,b1,b2,b3,flag,cart,hint,Bo)
    DEALLOCATE( vars%X )
    DEALLOCATE( vars%Y )
    DEALLOCATE( vars%B )
+   DEALLOCATE( vars%BR )
+   DEALLOCATE( vars%BPHI )
    DEALLOCATE( vars%PSI_P )
    DEALLOCATE( vars%gradB )
    DEALLOCATE( vars%curlb )

src/korc_initialize.f90

@@ -536,6 +536,8 @@ subroutine initialize_particles(params,F,P,spp)
        ALLOCATE( spp(ii)%vars%Yborn(spp(ii)%ppp,3) )
        ALLOCATE( spp(ii)%vars%E(spp(ii)%ppp,3) )
        ALLOCATE( spp(ii)%vars%B(spp(ii)%ppp,3) )
+       ALLOCATE( spp(ii)%vars%BR(spp(ii)%ppp,3) ) !dummies for MARS interp debug
+       ALLOCATE( spp(ii)%vars%BPHI(spp(ii)%ppp,3) ) !dummies for MARS interp debug
        ALLOCATE( spp(ii)%vars%PSI_P(spp(ii)%ppp) )
        ALLOCATE( spp(ii)%vars%ne(spp(ii)%ppp) )
        ALLOCATE( spp(ii)%vars%ni(spp(ii)%ppp) )
@@ -564,6 +566,8 @@ subroutine initialize_particles(params,F,P,spp)
        spp(ii)%vars%Yborn = 0.0_rp
        spp(ii)%vars%E = 0.0_rp
        spp(ii)%vars%B = 0.0_rp
+       spp(ii)%vars%BR = 0.0_rp
+       spp(ii)%vars%BPHI = 0.0_rp
        spp(ii)%vars%PSI_P = 0.0_rp
        spp(ii)%vars%ne = 0.0_rp
        spp(ii)%vars%ni = 0.0_rp
@@ -726,6 +730,7 @@ subroutine set_up_particles_ic(params,F,spp,P)
        if (params%mpi_params%rank .EQ. 0) then
           write(output_unit_write,'("* * * * * * * * * * * * * * * * * * * * * * * *",/)')
        end if
+       flush(output_unit_write)
 
        if (params%mpi_params%rank .EQ. 0) then
           write(output_unit_write,'("* * * * INITIALIZING VELOCITY COMPONENTS * * * *")')
@@ -734,6 +739,7 @@ subroutine set_up_particles_ic(params,F,spp,P)
        if (params%mpi_params%rank .EQ. 0) then
           write(output_unit_write,'("* * * * * * * * * * * * * * * * * * * * * * * *",/)')
        end if
+       flush(output_unit_write)
     end if
 
   end subroutine set_up_particles_ic