Add nearly exact restart. Restart is not binary identical yet. Also s…

…aving restart files is slower on multiple processors.
ORNL-Fusion · Jan 26, 2025 · 5eec8d0 · 5eec8d0
1 parent d580700
commit 5eec8d0
Show file tree

Hide file tree

Showing 7 changed files with 112 additions and 10 deletions.
diff --git a/.github/workflows/ci_doxygen.yaml b/.github/workflows/ci_doxygen.yaml
@@ -28,6 +28,8 @@ jobs:
                 sudo apt-get install libopenmpi-dev
                 sudo apt-get install libscalapack-openmpi-dev
                 sudo apt-get install doxygen
+                sudo apt-get install libblas-dev
+                sudo apt-get install liblapack-dev
             - name: Configure CMake
               shell: bash
               working-directory: ${{runner.workspace}}/Stellarator-Tools/build

diff --git a/.github/workflows/ci_test.yaml b/.github/workflows/ci_test.yaml
@@ -41,6 +41,8 @@ jobs:
                 sudo apt-get install libnetcdff-dev
                 sudo apt-get install libopenmpi-dev
                 sudo apt-get install libscalapack-openmpi-dev
+                sudo apt-get install libblas-dev
+                sudo apt-get install liblapack-dev
             - name: Configure CMake Mac
               if: ${{matrix.os == 'macos-latest'}}
               shell: bash

diff --git a/.github/workflows/ci_test_master.yaml b/.github/workflows/ci_test_master.yaml
@@ -41,6 +41,8 @@ jobs:
                 sudo apt-get install libnetcdff-dev
                 sudo apt-get install libopenmpi-dev
                 sudo apt-get install libscalapack-openmpi-dev
+                sudo apt-get install libblas-dev
+                sudo apt-get install liblapack-dev
             - name: Configure CMake Mac
               if: ${{matrix.os == 'macos-latest'}}
               shell: bash

diff --git a/Sources/Input_Output/CMakeLists.txt b/Sources/Input_Output/CMakeLists.txt
@@ -22,4 +22,5 @@ target_sources(vmec
                $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/vmercier.f>
                $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/write_dcon.f>
                $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/wrout.f>
+               $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/restart.f>
 )
diff --git a/Sources/TimeStep/eqsolve.f b/Sources/TimeStep/eqsolve.f
@@ -1,4 +1,4 @@
-      SUBROUTINE eqsolve(ier_flag, lscreen)
+      SUBROUTINE eqsolve(ier_flag, lscreen, restart_obj)
       USE vmec_main
       USE vmec_params, ONLY: ntmax, ns4, jac75_flag, norm_term_flag,
      &                       bad_jacobian_flag, more_iter_flag,
@@ -11,14 +11,17 @@ SUBROUTINE eqsolve(ier_flag, lscreen)
 ! Add below JDH 2010-08-03
       USE vmec_history
       USE parallel_include_module
-      USE parallel_vmec_module, ONLY: ZeroLastNType
+      USE parallel_vmec_module, ONLY: ZeroLastNType, Parallel2Serial4X
       USE vacmod, ONLY: nuv, nuv3
+      USE restart
+
       IMPLICIT NONE
 !-----------------------------------------------
 !   D u m m y   A r g u m e n t s
 !-----------------------------------------------
-      INTEGER :: ier_flag
-      LOGICAL :: lscreen
+      INTEGER, INTENT(inout)            :: ier_flag
+      LOGICAL, INTENT(in)               :: lscreen
+      CLASS (restart_class), INTENT(in) :: restart_obj
 !-----------------------------------------------
 !   L o c a l   P a r a m e t e r s
 !-----------------------------------------------
@@ -177,6 +180,14 @@ SUBROUTINE eqsolve(ier_flag, lscreen)
      &       iter2            .eq. 1 .or.
      &       .not.liter_flag) THEN
             CALL printout(iter2, delt0r, w0, lscreen)
+
+            IF (PARVMEC) THEN
+               CALL Gather4XArray(pxc)
+            END IF
+            IF (grank .EQ. 0) THEN
+               CALL Parallel2Serial4X(pxc, xc)
+               CALL restart_obj%write()
+            END IF
          END IF
          iter2 = iter2 + 1
          iterc = iterc + 1

diff --git a/Sources/TimeStep/runvmec.f b/Sources/TimeStep/runvmec.f
@@ -1,11 +1,13 @@
       SUBROUTINE runvmec(ictrl_array, input_file0, 
-     &                   lscreen, COMM_WORLD, reset_file_name)
+     &                   lscreen, COMM_WORLD, reset_file_name,
+     &                   restart_file_name)
       USE vmec_main
       USE vmec_params, ONLY: bad_jacobian_flag, more_iter_flag,
      &                       norm_term_flag, successful_term_flag,
      &                       restart_flag, readin_flag,
      &                       timestep_flag, ns_error_flag,
      &                       reset_jacdt_flag, lamscale
+      USE restart
       USE realspace
       USE vmec_params, ONLY: ntmax
       USE vacmod, ONLY: nuv, nuv3
@@ -29,6 +31,7 @@ SUBROUTINE runvmec(ictrl_array, input_file0,
       LOGICAL, INTENT(in)            :: lscreen
       CHARACTER(LEN=*), INTENT(in)   :: input_file0
       CHARACTER(LEN=*), OPTIONAL     :: reset_file_name
+      CHARACTER(LEN=*), OPTIONAL     :: restart_file_name
       INTEGER, INTENT(IN), OPTIONAL  :: COMM_WORLD
 C-----------------------------------------------
 C   L o c a l   V a r i a b l e s
@@ -50,6 +53,7 @@ SUBROUTINE runvmec(ictrl_array, input_file0,
       INTEGER                        :: blklength, grid_id, i, js,
      &                                  nsmin, nsmax
       CHARACTER(LEN=20)              :: fname
+      CLASS (restart_class), POINTER :: restart_obj
 
 C-----------------------------------------------
 !
@@ -110,6 +114,13 @@ SUBROUTINE initialize_radial(nsval, ns_old, delt0,
          LOGICAL, INTENT(in) :: lscreen
          REAL(rprec), INTENT(out) :: delt0
          END SUBROUTINE initialize_radial
+
+         SUBROUTINE eqsolve(ier_flag, lscreen, restart_obj)
+         USE restart
+         INTEGER, INTENT(inout)            :: ier_flag
+         LOGICAL, INTENT(in)               :: lscreen
+         CLASS (restart_class), INTENT(in) :: restart_obj
+         END SUBROUTINE
       END INTERFACE
 
       RUNVMEC_PASS = RUNVMEC_PASS + 1
@@ -319,7 +330,7 @@ END SUBROUTINE initialize_radial
 
          grid_size(grid_id) = nsval
          grid_procs(grid_id) = nranks
-         
+
 !  JDH 2012-06-20. V3FIT fix, inserted with change from VMEC 8.48 -> 8.49
 !  (Not sure just what in initialize_radial messes up convergence - happens slowly)
 !  Logical l_v3fit is declared in vmec_input, available via vmec_main
@@ -339,7 +350,29 @@ END SUBROUTINE initialize_radial
             niter = numsteps + iter2 - 1
          END IF
 
-         CALL eqsolve (ier_flag, lscreen)
+         IF (PRESENT(restart_file_name)         .and.                       &
+     &       LEN_TRIM(restart_file_name) .ne. 0 .and.                       &
+     &       .not. (ns_old .ne. nsval)) THEN
+            IF (grank .EQ. 0) THEN
+               restart_obj => restart_construct_open(restart_file_name)
+               CALL restart_obj%read()
+               CALL Serial2Parallel4X(xc,pxc)
+            END IF
+            IF (PARVMEC) THEN
+               CALL MPI_Bcast(pxc, SIZE(pxc), MPI_REAL8, 0,
+     &                        RUNVMEC_COMM_WORLD, MPI_ERR)
+            END IF
+         ELSE
+            IF (grank .EQ. 0) THEN
+               restart_obj => restart_construct_new('restart.nc')
+            END IF
+         END IF
+
+         CALL eqsolve (ier_flag, lscreen, restart_obj)
+
+         IF (grank .EQ. 0) THEN
+            DEALLOCATE(restart_obj)
+         END IF
 
          IF (numsteps .GT. 0) THEN
             niter = niter_store

diff --git a/Sources/TimeStep/vmec.f b/Sources/TimeStep/vmec.f
@@ -1,3 +1,30 @@
+!-------------------------------------------------------------------------------
+!  The @header2, @begin_table, @item3 and @end_table commands are custom defined
+!  commands in Doxygen.in. They are defined under ALIASES. For the page created
+!  here, the 80 column limit is exceeded. Arguments of aliases are separated by
+!  ','. If you intended ',' to be a string you must use an escaped comma '\,'.
+!
+!>  @page vmec_cl_parsing_sec Command Line Arguments
+!>
+!>  @section vmec_cl_parsing_intro Introduction
+!>  This contains a description of the command line arguments. All arguments
+!>  take the form of
+!>
+!>  @fixed_width{xvmec <namelist_input> [<option>=<value>]}
+!>
+!>  @section vmec_cl_parsing_arg_sec Command Line Arguments
+!>  @header2{Argument, Takes Value, Discription}
+!>  @begin_table
+!>     @item3{@fixed_width{-h},      N, Show help info.}
+!>     @item3{@fixed_width{reset},   Y, Reset from the wout file.}
+!>     @item3{@fixed_width{restart}, Y, Restart from the restart file.}
+!>  @end_table
+!>
+!>  @section vmec_cl_pasring_prog_ref_sec Programmers Reference
+!>  Reference material for the coding to implement commandline parsing is found
+!>  in the @ref vmec.f.
+!-------------------------------------------------------------------------------
+
       PROGRAM vmec
       USE vmec_input
       USE vmec_seq
@@ -13,6 +40,7 @@ PROGRAM vmec
       USE parallel_vmec_module, ONLY: MyEnvVariables, 
      &                                InitializeParallel,
      &                                FinalizeParallel
+      USE profiler
       IMPLICIT NONE
 C-----------------------------------------------
 C   L o c a l   P a r a m e t e r s
@@ -31,6 +59,7 @@ PROGRAM vmec
      &   index_dat, iunit, ncount, nsteps, i
       INTEGER :: ictrl(5)
       CHARACTER(LEN=120) :: input_file, seq_ext, reset_file_name, arg
+      CHARACTER(LEN=120) :: restart_file_name
       CHARACTER(LEN=120) :: log_file
       CHARACTER(LEN=120), DIMENSION(10) :: command_arg
       LOGICAL :: lscreen
@@ -139,14 +168,16 @@ PROGRAM vmec
 !     screen display information, and restart information
 !
       INTERFACE
-         SUBROUTINE runvmec(ictrl_array, input_file0, 
-     &                      lscreen, RVC_COMM, reset_file_name)
+         SUBROUTINE runvmec(ictrl_array, input_file0,                          &
+     &                      lscreen, RVC_COMM, reset_file_name,                &
+     &                      restart_file_name)
          IMPLICIT NONE
          INTEGER, INTENT(inout), TARGET :: ictrl_array(5)
          LOGICAL, INTENT(in) :: lscreen
          CHARACTER(LEN=*), INTENT(in) :: input_file0
          INTEGER, INTENT(in), OPTIONAL :: RVC_COMM
          CHARACTER(LEN=*), OPTIONAL :: reset_file_name
+         CHARACTER(LEN=*), OPTIONAL :: restart_file_name
          END SUBROUTINE runvmec
       END INTERFACE
 
@@ -156,6 +187,8 @@ END SUBROUTINE runvmec
       CALL second0(totalton)
       ton = totalton
 
+      CALL profiler_construct
+
       CALL getcarg(1, command_arg(1), numargs)
       DO iseq = 2, numargs
          CALL getcarg(iseq, command_arg(iseq), numargs)
@@ -189,13 +222,16 @@ END SUBROUTINE runvmec
      &           ' allowed:'
          PRINT *
          PRINT *,'  xvmec <filename> [noscreen] [reset=reset_wout_file]'
+         PRINT *,'                              [restart=restart_file] '
          PRINT *
          PRINT *,' noscreen: supresses all output to screen ',
      &           ' (default, or "screen", displays output)'
          PRINT *,' name of reset wout file (defaults to none)'
 
          STOP
       ELSE
+         reset_file_name = ''
+         restart_file_name = ''
          DO iseq = 2, MIN(numargs,10)
             arg = command_arg(iseq)
             IF (TRIM(arg) .eq. 'noscreen' .or.                                 &
@@ -205,7 +241,16 @@ END SUBROUTINE runvmec
             index_end = INDEX(arg, "reset=")
             index_seq = MAX(INDEX(arg, "RESET="), index_end)
             IF (index_seq .gt. 0) reset_file_name = arg(index_seq+6:)
+            index_end = INDEX(arg, "restart=")
+            index_seq = MAX(INDEX(arg, "RESTART="), index_end)
+            IF (index_seq .gt. 0) restart_file_name = arg(index_seq+8:)
          END DO
+
+         IF (LEN(TRIM(reset_file_name)) .gt. 0 .and.                                 &
+     &       LEN(TRIM(restart_file_name)) .gt. 0 ) THEN
+            WRITE (*,*) LEN(reset_file_name), LEN(restart_file_name)
+            STOP 'Cannot restart from both wout and restart files.'
+         END IF
       END IF
 
 !
@@ -331,7 +376,7 @@ END SUBROUTINE runvmec
 
          RVCCALLNUM = 1
          CALL runvmec(ictrl, extension(index_seq), lscreen, RVC_COMM,
-     &                reset_file_name)
+     &                reset_file_name, restart_file_name)
 
          ierr_vmec = ictrl(2)
 
@@ -440,6 +485,12 @@ END SUBROUTINE runvmec
       total_time = total_time + (totaltoff - totalton)
       toff = totaltoff
       IF (.NOT.LV3FITCALL .AND. lactive) CALL WriteTimes('timings.txt')
+
+      IF (grank .EQ. 0) THEN
+         CALL profiler_write(6)
+      END IF
+      CALL profiler_destruct
+
       CALL FinalizeParallel
 
       END PROGRAM vmec