moving grb2index core code into g2 library (#615)

* moving grb2index core code into g2 library

* moving grb2index code into g2 library

* moving grb2index code into g2 library

* moved code to create index into library

* cleanup

* working on test

* working on test

* working on test

* working on test

* fixed leak in test

* more index testing

* more index testing

* working on index test

* working on index test

* commented out all test code in test_create_index.F90

* uncommented out some test code in test_create_index.F90

* uncommented out some more test code in test_create_index.F90

* adding intents

* more testing

* changed filename in index header record

* initialized variables

* uncommented more test code, again

* uncommented more test code, again

* fixed test

* test working

src/g2index.F90

@@ -2,6 +2,150 @@
 !> @brief Subroutines for dealing with indexes.
 !> @author Edward Hartnett @date Jan 31, 2024
 
+!> Create a version 1 or 2 index file for a GRIB2 file.
+!>
+!> @param[in] lugb Logical unit of opened GRIB2 file.
+!> @param[in] lugi Logical unit file opened to write index to.
+!> @param[in] idxver Index version.
+!> @param[in] filename Name of GRIB2 file. 
+!> @param[out] iret Return code:
+!> - 0 success
+!> - 90 problem opening GRIB2 file.
+!> - 91 problem opening index file.
+!> - 92 no messages found in GRIB2 file.
+!>
+!> @author Ed Hartnett, Mark Iredell @date Feb 15, 2024
+subroutine g2_create_index(lugb, lugi, idxver, filename, iret)
+  implicit none
+
+  integer, intent(in) :: lugb, lugi, idxver
+  character*(*) :: filename
+  integer, intent(out) :: iret
+
+  integer (kind = 8) :: msk1, msk2
+  parameter(msk1 = 32000_8, msk2 = 4000_8)
+  character(len=1), pointer, dimension(:) :: cbuf
+  integer :: numtot, nnum, nlen, mnum, kw
+  integer :: irgi, iw, nmess
+
+  interface
+     subroutine getg2i2r(lugb, msk1, msk2, mnum, idxver, cbuf, &
+          nlen, nnum, nmess, iret)
+       integer, intent(in) :: lugb
+       integer (kind = 8), intent(in) :: msk1, msk2
+       integer, intent(in) :: mnum, idxver
+       character(len = 1), pointer, dimension(:) :: cbuf
+       integer, intent(out) :: nlen, nnum, nmess, iret
+     end subroutine getg2i2r
+  end interface
+
+  ! Assume success.
+  iret = 0
+  numtot = 0
+  nlen = 0
+
+  ! Generate index records for all messages in file, or until memory
+  ! runs out.
+  mnum = 0
+  call getg2i2r(lugb, msk1, msk2, mnum, idxver, cbuf, &
+       nlen, nnum, nmess, irgi)
+  if (irgi .gt. 1 .or. nnum .eq. 0 .or. nlen .eq. 0) then
+     iret = 92
+     return
+  endif
+  numtot = numtot + nnum
+  mnum = mnum + nmess
+
+  ! Write headers.
+  call g2_write_index_headers(lugi, nlen, numtot, filename)
+  iw = 162
+
+  ! Write the index data we have so far.
+  call bawrite(lugi, iw, nlen, kw, cbuf)
+  iw = iw + nlen
+
+  ! Extend index file if index buffer length too large to hold in memory.
+  if (irgi .eq. 1) then
+     do while (irgi .eq. 1 .and. nnum .gt. 0)
+        if (associated(cbuf)) then
+           deallocate(cbuf)
+           nullify(cbuf)
+        endif
+        call getg2i2r(11, msk1, msk2, mnum, idxver, cbuf, &
+             nlen, nnum, nmess, irgi)
+        if (irgi .le. 1 .and. nnum .gt. 0) then
+           numtot = numtot + nnum
+           mnum = mnum + nmess
+           call bawrite(lugi, iw, nlen, kw, cbuf)
+           iw = iw + nlen
+        endif
+     enddo
+     ! Go back and overwrite headers with new info.
+     call g2_write_index_headers(lugi, iw, numtot, filename)
+  endif
+  deallocate(cbuf)  
+
+end subroutine g2_create_index
+
+!> Write index headers.
+!>
+!> @param[in] lugi integer logical unit of output index file
+!> @param[in] nlen integer total length of index records
+!> @param[in] nnum integer number of index records
+!> @param[in] filename character name of GRIB file
+!>
+!> @author Iredell @date 93-11-22
+subroutine g2_write_index_headers(lugi, nlen, nnum, filename)
+  implicit none
+
+  integer, intent(in) :: lugi, nlen, nnum
+  character, intent(in) :: filename*(*)
+
+  character cd8*8, ct10*10, hostname*15
+#ifdef __GFORTRAN__
+  integer istat
+#else
+  character hostnam*15
+  integer hostnm
+#endif
+  character chead(2)*81
+  integer :: kw
+
+  !  fill first 81-byte header
+  call date_and_time(cd8, ct10)
+  chead(1) = '!GFHDR!'
+  chead(1)(9:10) = ' 1'
+  chead(1)(12:14) = '  1'
+  write(chead(1)(16:20),'(i5)') 162
+  chead(1)(22:31) = cd8(1:4) // '-' // cd8(5:6) // '-' // cd8(7:8)
+  chead(1)(33:40) = ct10(1:2) // ':' // ct10(3:4) // ':' // ct10(5:6)
+  chead(1)(42:47) = 'GB2IX1'
+  chead(1)(49:54) = '      '
+#ifdef __GFORTRAN__
+  istat = hostnm(hostname)
+  if (istat .eq. 0) then
+     chead(1)(56:70) = '0000'
+  else
+     chead(1)(56:70) = '0001'
+  endif
+#else
+  chead(1)(56:70) = hostnam(hostname)
+#endif
+  chead(1)(72:80) = 'grb2index'
+  chead(1)(81:81) = char(10)
+
+  !  fill second 81-byte header
+  chead(2) = 'IX1FORM:'
+  write(chead(2)(9:38),'(3i10)') 162, nlen, nnum
+  chead(2)(41:80) = filename
+  chead(2)(81:81) = char(10)
+
+  !  write headers at beginning of index file
+  call bawrite(lugi, 0, 162, kw, chead)
+
+  return
+end subroutine g2_write_index_headers
+
 !> Find, read or generate a version 1 GRIB2 index for a GRIB2 file
 !> (which must be < 2 GB).
 !>
@@ -363,7 +507,6 @@ subroutine getg2i2(lugi, cbuf, idxver, nlen, nnum, iret)
   iret = 4
   call baread(lugi, 0, 162, lhead, chead)
   if (lhead .eq. 162 .and. chead(42:47) .eq. 'GB2IX1') then
-!     read(chead(82:162), '(8x, 3i10, 2x, a40)', iostat = ios) nskp, nlen, nnum
      read(chead(82:162), '(2x, i1, 5x, 3i10, 2x, a40)', iostat = ios) idxver, nskp, nlen, nnum
      if (ios .eq. 0) then
         allocate(cbuf(nlen), stat = istat)    ! Allocate space for cbuf.
@@ -378,7 +521,7 @@ subroutine getg2i2(lugi, cbuf, idxver, nlen, nnum, iret)
   endif
 end subroutine getg2i2
 
-!> Generate an index record for a message in a GRIB2 file.
+!> Generate a version 1 index record for each message in a GRIB2 file.
 !>
 !> The index record contains byte offsets to the message, it's length,
 !> and byte offsets within the message to each section. The index file
@@ -438,7 +581,8 @@ end subroutine getg2i2r
   call getg2i2r(lugb, msk1_8, msk2_8, mnum, 1, cbuf, nlen, nnum, nmess, iret)
 end subroutine getg2ir
 
-!> Generate an index record for a message in a GRIB2 file.
+!> Generate a version 1 or 2 index record for each message in a GRIB2
+!> file.
 !>
 !> The index record contains byte offsets to the message, it's length,
 !> and byte offsets within the message to each section. The index file
@@ -816,7 +960,7 @@ end subroutine gf_unpack5
   endif
 
   ! Search for request.
-  do while(iret.ne.0 .and. k.lt.nnum)
+  do while(iret .ne. 0 .and. k .lt. nnum)
      k = k + 1
      ! Get length of current index record.
      call g2_gbytec(cbuf, inlen, ipos * 8, 4 * 8)    

tests/CMakeLists.txt

@@ -163,6 +163,7 @@ foreach(kind ${kinds})
   create_test(test_mkieee ${kind})
   create_test(test_getlocal ${kind})
   create_test(test_index_gdas ${kind} index_rec.F90)
+  create_test(test_create_index ${kind})
   create_test(test_getgb2p_gdas ${kind})
   set_tests_properties(test_getgb2p_gdas_${kind} PROPERTIES LABELS noMemcheck)  
   create_test(test_realloc ${kind})

tests/test_create_index.F90

@@ -0,0 +1,133 @@
+! This is a test program for NCEPLIBS-g2.
+!
+! This program tests index file functionality with g2_create_index().
+!
+! Ed Hartnett 2/15/24
+program test_create_index
+  use grib_mod
+  implicit none
+
+  ! These are the test files we will use.
+  character(*) :: TEST_FILE_GDAS
+  parameter (TEST_FILE_GDAS = 'gdaswave.t00z.wcoast.0p16.f000.grib2')
+  character(*) :: TEST_FILE_GDAS_INDEX
+  parameter (TEST_FILE_GDAS_INDEX = 'test_create_index_gdaswave.grb2index')
+  character(len=1), pointer, dimension(:) :: cbuf(:)
+  integer :: idxver = 1, nlen, nnum, lugi = 31, lugb = 11
+  integer :: j, jdisc, jpdtn, jgdtn
+  integer :: jids(13), jpdt(100), jgdt(250)
+  integer :: i
+  integer :: k, lpos, iret
+  type(gribfield) :: gfld
+  integer :: expected_idsect(13) = (/ 7, 0, 2, 1, 1, 2021, 11, 30, 0, 0, 0, 0, 1 /)
+  integer :: expected_igdtmpl(19) = (/ 6, 0, 0, 0, 0, 0, 0, 241, 151, 0, 0, 50000000, &
+       210000000, 48, 25000000, 250000000, 166667, 166667, 0 /)
+  integer :: expected_ipdtmpl(15) = (/ 2, 1, 2, 0, 11, 0, 0, 1, 0, 1, 0, 1, 255, 0, 0 /)
+  integer :: expected_idrtmpl(7) = (/ 1092616192, 0, 2, 11, 0, 0, 255 /)
+  integer :: ios
+
+  interface
+     subroutine getg2i2(lugi, cbuf, idxver, nlen, nnum, iret)
+       integer, intent(in) :: lugi
+       character(len=1), pointer, dimension(:) :: cbuf
+       integer, intent(out) :: idxver, nlen, nnum, iret
+     end subroutine getg2i2
+     subroutine g2_create_index(lugb, lugi, idxver, filename, iret)
+       integer, intent(in) :: lugb, lugi, idxver
+       character*(*) :: filename
+       integer, intent(out) :: iret
+     end subroutine g2_create_index
+  end interface
+
+  print *, 'Testing index creation and reading.'
+  print *, 'testing g2_create_index on ', TEST_FILE_GDAS
+
+  ! Open GRIB2 file for reading.
+  call baopenr(lugb, TEST_FILE_GDAS, ios)
+  if (ios .ne. 0) stop 2
+
+  ! Open output file where index will be written.
+  call baopen(lugi, TEST_FILE_GDAS_INDEX, ios)
+  if (ios .ne. 0) stop 3
+
+  call g2_create_index(lugb, lugi, idxver, TEST_FILE_GDAS, iret)
+  if (iret .ne. 0) stop 10
+
+  call baclose(lugb, ios)
+  if (ios .ne. 0) stop 11
+  call baclose(lugi, ios)
+  if (ios .ne. 0) stop 12
+
+  print *, 'OK!'
+  print *, 'testing that index file can be read with getg2i2()...'
+
+  ! Open the index file.
+  call baopen(lugi, TEST_FILE_GDAS_INDEX, iret)
+  if (iret .ne. 0) stop 20
+
+  ! Read the index file.
+  call getg2i2(lugi, cbuf, idxver, nlen, nnum, iret)
+  if (nlen .ne. 3800 .or. nnum .ne. 19 .or. iret .ne. 0) stop 80
+
+  ! Close the index file.
+  call baclose(lugi, iret)
+  if (iret .ne. 0) stop 100
+
+  print *, 'OK!'
+  print *, 'testing that index buffer can be understood by getgb2s2()...'
+
+  ! Parse the index info in cbuf, and fill gfld with the info about
+  ! the first message.
+  j = 0
+  jdisc = -1
+  do i = 1, 13
+     jids(i) = -9999
+  end do
+  jpdtn = -1
+  do i = 1, 100
+     jpdt(i) = -9999
+  end do
+  jgdtn = -1
+  do i = 1, 250
+     jgdt(i) = -9999
+  end do
+  call getgb2s2(cbuf, idxver, nlen, nnum, j, jdisc, jids, jpdtn, jpdt, jgdtn, &
+       jgdt, k, gfld, lpos, iret)
+  if (iret .ne. 0) stop 101
+
+  ! Check that the information is correct for the first record.
+  if (gfld%version .ne. 2 .or. gfld%discipline .ne. 0) stop 102
+  if (gfld%idsectlen .ne. 13) stop 103
+  if (gfld%ifldnum .ne. 1) stop 105
+  if (gfld%griddef .ne. 0) stop 106
+  if (gfld%ngrdpts .ne. 36391) stop 107
+  if (gfld%numoct_opt .ne. 0 .or. gfld%interp_opt .ne. 0 .or. gfld%num_opt .ne. 0) stop 108
+  if (gfld%igdtnum .ne. 0 .or. gfld%igdtlen .ne. 19) stop 109
+  if (gfld%ipdtnum .ne. 0 .or. gfld%ipdtlen .ne. 15 .or. gfld%num_coord .ne. 0) stop 110
+  if (gfld%unpacked .neqv. .FALSE.) stop 112
+  if (gfld%ibmap .ne. 0) stop 113
+  do i = 1, gfld%idsectlen
+     if (gfld%idsect(i) .ne. expected_idsect(i)) stop 200
+  end do
+  do i = 1, gfld%igdtlen
+     if (gfld%igdtmpl(i) .ne. expected_igdtmpl(i)) stop 201
+  end do
+  do i = 1, gfld%ipdtlen
+     if (gfld%ipdtmpl(i) .ne. expected_ipdtmpl(i)) stop 202
+  end do
+  do i = 1, gfld%idrtlen
+     if (gfld%idrtmpl(i) .ne. expected_idrtmpl(i)) stop 203
+  end do
+
+  ! Free memory.
+  call gf_free(gfld)
+
+  print *, 'OK!'
+
+  ! Clean up.
+  deallocate(cbuf)
+  call gf_finalize(iret)
+  if (iret .ne. 0) stop 200
+
+  print *, 'SUCCESS!...'
+end program test_create_index