merging in changes from main and random_test

.github/workflows/cachepoke.yml

@@ -0,0 +1,29 @@
+#
+#  Workflow runs weekly to prevent GH dependency cache eviction
+#
+#  GitHub Actions run in UTC time
+#
+
+name: cachepoke
+on:
+  schedule:
+    - cron: '47 18 * * SAT'  # Trigger workflow every Sat at 18:47 UTC
+
+jobs:
+  job_one:
+    runs-on: ubuntu-latest
+    steps:
+      - name: spack cache
+        id: spack-cache
+        uses: actions/cache@v3
+        with:
+          key: spack-cache-sandlot-6
+          path: |
+            ./spack
+            ~/.spack
+
+
+#
+# https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy
+# https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#schedule
+#

.github/workflows/onyx.yml

@@ -61,10 +61,11 @@ jobs:
     #
 
     steps:
-      - name: build korc
+      - name: run ci_stub/onyx_job_build_cpu.sh
         run: |
           ./KORC/ci_stub/onyx_job_build_cpu.sh
 
+
   job_setup_spack:
     if: true
     runs-on: [ self-hosted ]
@@ -94,29 +95,22 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: 'spack/spack'
-          ref: 'v0.21.1'
+          ref: 'v0.22.2'
           path: './spack'
         #
-        #   git clone --depth=1 --single-branch --branch v0.21.1 https://github.com/spack/spack.git
+        #   git clone --depth=1 --single-branch --branch v0.22.2 https://github.com/spack/spack.git
         #
 
       - if: ${{ steps.check_spack.outputs.exist != 'true' }}
-        name: load nvhpc from environment module
-        run: |
-          module load nvhpc-hpcx/24.5
-          module list
-
-      - if: ${{ steps.check_spack.outputs.exist != 'true' }}
-        name: spack install
+        name: spack env activate
         run: |
           . ./spack/share/spack/setup-env.sh
-          spack config add config:install_tree:padded_length:128
-          spack compiler find
-          spack compiler remove [email protected]
-          spack install hdf5 +fortran -mpi %[email protected]
-          spack load    hdf5 +fortran -mpi %[email protected]
+          spack env activate ./KORC/spackenv/excl-milan_linux-ubuntu22.04-zen3
+          spack concretize -f
+          spack install
 
-          spack gc -y
+        # spack install --overwrite
+        # spack gc -y
 
 
   job_build_gpu:
@@ -134,7 +128,11 @@ jobs:
     steps:
       - name: run ci_stub/onyx_job_build_gpu.sh
         run: |
-           ./KORC/ci_stub/onyx_job_build_gpu.sh
+          . ./spack/share/spack/setup-env.sh
+          spack env activate ./KORC/spackenv/excl-milan_linux-ubuntu22.04-zen3
+          spack install
+          spack build-env hdf5%[email protected]+fortran~mpi -- ./KORC/ci_stub/onyx_job_build_gpu.sh
+
 
   job_test_cpu:
     if: true
@@ -153,9 +151,11 @@ jobs:
         uses: SimenB/github-actions-cpu-cores@v2
         id: cpu-cores
 
-      - name: ctest
+      - name: run ci_stub/onyx_job_test_cpu.sh
+        env:
+          JOB_COUNT: "${{ steps.cpu-cores.outputs.count }}"
         run: |
-          cd ./KORC/build_cpu && ctest --output-on-failure
+          ./KORC/ci_stub/onyx_job_test_cpu.sh
 
 
   job_test_gpu:
@@ -175,13 +175,10 @@ jobs:
         uses: SimenB/github-actions-cpu-cores@v2
         id: cpu-cores
 
-      - name: ctest
+      - name: run ci_stub/onyx_job_test_gpu.sh
+        env:
+          JOB_COUNT: "${{ steps.cpu-cores.outputs.count }}"
         run: |
-          module load nvhpc-hpcx/24.5
-          module list
-          . ./spack/share/spack/setup-env.sh
-          spack load hdf5 +fortran -mpi %[email protected]
-
+          ./KORC/ci_stub/onyx_job_test_gpu.sh
 
-          cd ./KORC/build_gpu && ctest -j ${{ steps.cpu-cores.outputs.count }} --output-on-failure
 

ci_stub/onyx_job_build_gpu.sh

@@ -1,10 +1,9 @@
 #!/bin/bash
 
-# load nvhpc from environment module
-module load nvhpc-hpcx/24.5
+which h5diff
+which nvfortran
+which cmake
 module list
-. ./spack/share/spack/setup-env.sh
-spack load hdf5 +fortran -mpi %[email protected]
 
 cd ./KORC
 rm -f CMakeCache.txt

ci_stub/onyx_job_test_cpu.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+env | grep JOB_COUNT
+
+JOB_COUNT=1
+
+cd ./KORC/build_cpu && ctest -j --output-on-failure
+

ci_stub/onyx_job_test_gpu.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+env | grep JOB_COUNT
+
+JOB_COUNT=1
+
+pushd ./KORC/build_gpu >/dev/null
+
+ctest -j --output-on-failure
+
+pushd ./bin >/dev/null
+
+for j in "mars_test" "egyro_test"; do
+  for i in $(ls -1 ./${j}/ | sort -k1.6n); do
+    for x in {1..20}; do echo "."; done
+    echo "onyx_job_test_gpu >> [Info] << ${j}/${i}/output.korc"
+    for x in {1..5}; do echo "."; done
+    cat ./${j}/${i}/output.korc
+  done
+done
+
+
+popd >/dev/null
+popd >/dev/null

spackenv/excl-milan_linux-ubuntu22.04-zen3/spack.yaml

@@ -0,0 +1,45 @@
+# This is a Spack Environment file.
+#
+# It describes a set of packages to be installed, along with
+# configuration settings.
+spack:
+  # add package specs to the `specs` list
+  specs:
+  - "hdf5%[email protected]+fortran~mpi"
+
+  view: true
+  concretizer:
+    unify: true
+  config:
+    install_tree:
+      padded_length: 128
+
+  compilers:
+  - compiler:
+      spec: nvhpc@=24.5
+      paths:
+        cc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvc
+        cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvc++
+        f77: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvfortran
+        fc: /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvfortran
+      flags: {}
+      operating_system: ubuntu22.04
+      target: x86_64
+      modules:
+        - nvhpc-hpcx/24.5
+      environment: {}
+      extra_rpaths: []
+
+  - compiler:
+      spec: gcc@=11.4.0
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      flags: {}
+      operating_system: ubuntu22.04
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []

src/korc_ppusher.f90

@@ -375,13 +375,13 @@ subroutine FO_init(params,F,spp,output,step)
             eta_tmp=180.0_rp/C_PI* &
               ACOS((B_X(cc)*V_X(cc)+B_Y(cc)*V_Y(cc)+B_Z(cc)*V_Z(cc))/(Bmag(cc)*v(cc)))
 
-            if ((eta_tmp.gt.20._rp).or. &
-              ((spp(ii)%vars%eta(pp-1+cc)-eta_tmp)/spp(ii)%vars%eta(pp-1+cc).gt.0.01)) then 
-                write(6,*) 'abberation',pp
-                write(6,*) pp,spp(ii)%vars%eta(pp-1+cc),eta_tmp,'B', &
-                  spp(ii)%vars%B(pp-1+cc,:),'PSI_P',spp(ii)%vars%PSI_P(pp-1+cc)
+            !if ((eta_tmp.gt.20._rp).or. &
+            !  ((spp(ii)%vars%eta(pp-1+cc)-eta_tmp)/spp(ii)%vars%eta(pp-1+cc).gt.0.01)) then 
+            !    write(6,*) 'abberation',pp
+            !    write(6,*) pp,spp(ii)%vars%eta(pp-1+cc),eta_tmp,'B', &
+            !      spp(ii)%vars%B(pp-1+cc,:),'PSI_P',spp(ii)%vars%PSI_P(pp-1+cc)
 
-            endif  
+            !endif  
 
             spp(ii)%vars%eta(pp-1+cc) = eta_tmp 
 

src/korc_velocity_distribution.f90

@@ -384,12 +384,12 @@ subroutine gyro_distribution(params,random,F,spp)
     ALLOCATE( theta(spp%ppp) )
 
     ! * * * * INITIALIZE VELOCITY * * * *
-    if (.not.params%SameRandSeed) then
-      CALL random%uniform%set(0.0_rp,1.0_rp)
-      do pp=1_idef,spp%ppp
-         theta(pp)=random%uniform%get()
-      enddo
-    endif
+    !if (.not.params%SameRandSeed) then
+    CALL random%uniform%set(0.0_rp,1.0_rp)
+    do pp=1_idef,spp%ppp
+      theta(pp)=random%uniform%get()
+    enddo
+    !endif
     theta = 2.0_rp*C_PI*theta
 
     if (spp%spatial_distribution.eq.'TRACER') theta=2.0*C_PI

test/CMakeLists.txt

@@ -8,6 +8,7 @@ target_sources(xtest
                $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/korc_test.f90>
                $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/test_io.f90>
                $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/test_hpc.f90>
+               $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/test_random.f90>
 )
 target_link_libraries (xtest
 
@@ -22,10 +23,17 @@ set_property(TARGET xtest PROPERTY LINKER_LANGUAGE Fortran)
 configure_file(${CMAKE_SOURCE_DIR}/test/egyro/korc_egyro.sh.in ${CMAKE_BINARY_DIR}/egyro_test/korc_egyro.sh)
 configure_file(${CMAKE_SOURCE_DIR}/test/mars/korc_mars.sh.in ${CMAKE_BINARY_DIR}/mars_test/korc_mars.sh)
 
+#if(USE_PSPLINE)
+#  add_test (NAME mars_test_1
+#            COMMAND ${CMAKE_BINARY_DIR}/mars_test/korc_mars.sh 1
+#            WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+#  set_tests_properties (mars_test_1 PROPERTIES PROCESSORS 1 ENVIRONMENT OMP_NUM_THREADS=1)
+#endif()
+
 file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/bin/egyro_test)
 file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/bin/mars_test)
 
-foreach(RANK IN ITEMS 1 2 4 8 16)
+foreach(RANK IN ITEMS 1 2 4 8) #16)
   if(${RANK} LESS_EQUAL ${MPIEXEC_MAX_NUMPROCS})
     add_test (NAME egyro_test_${RANK}
 	      COMMAND ${CMAKE_BINARY_DIR}/egyro_test/korc_egyro.sh ${RANK}
@@ -34,7 +42,7 @@ foreach(RANK IN ITEMS 1 2 4 8 16)
 
     if(USE_PSPLINE)
       add_test (NAME mars_test_${RANK}
-                COMMAND ${CMAKE_BINARY_DIR}/mars_test/korc_mars.sh ${RANK}
+          COMMAND ${CMAKE_BINARY_DIR}/mars_test/korc_mars.sh ${RANK}
                 WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
       set_tests_properties (mars_test_${RANK} PROPERTIES PROCESSORS ${RANK} ENVIRONMENT OMP_NUM_THREADS=1)
     endif()

test/korc_test.f90

@@ -2,11 +2,17 @@ program korc_test
   use fruit
   use test_io
   use test_hpc
+  use test_random
+  use korc_hpc, only : initialize_mpi,finalize_mpi
   implicit none
   logical ok
 
+  TYPE(KORC_PARAMS)           :: params
   CHARACTER(MX_STRING_LENGTH) :: path_to_outputs
 
+  params%path_to_inputs='TEST'
+  call initialize_mpi(params)
+
   ! create output file for testing
   call set_paths(path_to_outputs)
 
@@ -15,7 +21,8 @@ program korc_test
 
   ! run tests
   write(test_unit_write,*) 'Testing MPI initialization...'
-  call test_mpi_initialization
+  call test_mpi_initialization(params)
+  call test_random_auto
 
   ! compile summary and finalize fruit
   call fruit_summary(test_unit_write)
@@ -29,5 +36,6 @@ program korc_test
 
   close(test_unit_write)
 
-
+  call finalize_mpi(params)
+
 end program korc_test

test/mars/input_file_D3D_191366_1762ms_MARS.korc

@@ -5,10 +5,10 @@
     ! Append simulation results after previous simulation_time reached
   reinit = F
     ! Begin a new simulation, reinitializing from restart file state
-  simulation_time = 1.E-7
+  simulation_time = 1.E-8
     ! Total aimed simulation time in seconds   
     ! Run 10 mu s If transients exist put 5 extra mu s.
-  snapshot_frequency = 1.E-8
+  snapshot_frequency = 1.E-9
     ! Time between snapshots in seconds
   restart_overwrite_frequency = 1.E-1	
     ! Time between overwritting of restart file in seconds

test/test_hpc.f90

@@ -6,17 +6,12 @@ module test_hpc
 
 contains
 
-  subroutine test_mpi_initialization
-    use korc_hpc, only : initialize_mpi,finalize_mpi
-    TYPE(KORC_PARAMS) :: params
+  subroutine test_mpi_initialization(params)
+    TYPE(KORC_PARAMS), INTENT(INOUT) :: params
     integer :: ierror
     integer :: size, rank
     integer :: size_k, rank_k
 
-    params%path_to_inputs='TEST'
-
-    call initialize_mpi(params)
-
     size_k=params%mpi_params%nmpi
     rank_k=params%mpi_params%rank
 
@@ -25,8 +20,6 @@ subroutine test_mpi_initialization
 
     call assert_equals(size_k,size)
     call assert_equals(rank_k,rank)
-
-    call finalize_mpi(params)
 
   end subroutine test_mpi_initialization