Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

updated CI, adding tests, dealing with index format 2 #599

Closed
wants to merge 12 commits into from
Closed

updated CI, adding tests, dealing with index format 2 #599

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
715f5f6
updated CI
edwardhartnett Feb 3, 2024
925b994
working on index
edwardhartnett Feb 3, 2024
0e1abb9
more testing
edwardhartnett Feb 4, 2024
716abd6
more testing
edwardhartnett Feb 4, 2024
e97583e
more testing
edwardhartnett Feb 4, 2024
6fe8b94
more testing
edwardhartnett Feb 4, 2024
1c13fd4
more testing
edwardhartnett Feb 4, 2024
56ece6f
working on index version 2
edwardhartnett Feb 4, 2024
54eccff
added test
edwardhartnett Feb 4, 2024
d5ad459
more work on new index format
edwardhartnett Feb 5, 2024
f601c55
further index progress
edwardhartnett Feb 5, 2024
9104bae
fixed documentation
edwardhartnett Feb 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/developer.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ jobs:

- name: upload-test-coverage
if: matrix.config == 'asan/code coverage'
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: g2-test-coverage
path: |
Expand Down
4 changes: 2 additions & 2 deletions src/g2_gbytesc.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
!> has more elements, use g2_sbytesc().
!>
!> @param[in] in Array input.
!> @param[out] iout Unpacked array output.
!> @param[inout] iout Unpacked array output.
!> @param[in] iskip Initial number of bits to skip.
!> @param[in] nbits Number of bits of each integer in IN to take.
!>
Expand All @@ -32,7 +32,7 @@ end subroutine g2_gbytec
!> This should be used when input array IN has only one element. If IN
!> has more elements, use G2_SBYTESC().
!>
!> @param[out] out packed array output
!> @param[inout] out packed array output
!> @param[in] in unpacked array input
!> @param[in] iskip initial number of bits to skip
!> @param[in] nbits Number of bits of each integer in OUT to fill.
Expand Down
233 changes: 188 additions & 45 deletions src/g2index.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,14 +212,15 @@ SUBROUTINE GETG2I(LUGI, CBUF, NLEN, NNUM, IRET)
INTEGER, INTENT(OUT) :: NLEN, NNUM, IRET
CHARACTER CHEAD*162
integer :: ios, istat, lbuf, lhead, nskp
integer :: index_version

NULLIFY(CBUF)
NLEN = 0
NNUM = 0
IRET = 4
CALL BAREAD(LUGI, 0, 162, LHEAD, CHEAD)
IF (LHEAD .EQ. 162 .AND. CHEAD(42:47) .EQ. 'GB2IX1') THEN
READ(CHEAD(82:162), '(8X, 3I10, 2X, A40)', IOSTAT = IOS) NSKP, NLEN, NNUM
READ(CHEAD(82:162), '(2X, I1, 5X, 3I10, 2X, A40)', IOSTAT = IOS) index_version, NSKP, NLEN, NNUM
IF (IOS .EQ. 0) THEN
ALLOCATE(CBUF(NLEN), STAT = ISTAT) ! ALLOCATE SPACE FOR CBUF
IF (ISTAT .NE. 0) THEN
Expand All @@ -239,8 +240,8 @@ END SUBROUTINE GETG2I
!> and byte offsets within the message to each section. The index file
!> record format is documented in subroutine ixgb2().
!>
!> @note Subprogram can be called from a multiprocessing environment.
!> Do not engage the same logical unit from more than one processor.
!> This is the legacy function, which always creates version 1 index
!> files. To create index files of version 1 or 2, use getg2i2r().
!>
!> @param[in] lugb Unit of the unblocked GRIB file. Must
!> be opened by [baopen() or baopenr()]
Expand All @@ -265,45 +266,108 @@ END SUBROUTINE GETG2I
!> - 3 Error deallocating memory.
!>
!> @author Mark Iredell @date 1995-10-31
SUBROUTINE GETG2IR(LUGB, MSK1, MSK2, MNUM, CBUF, NLEN, NNUM, NMESS, IRET)
USE RE_ALLOC ! NEEDED FOR SUBROUTINE REALLOC
subroutine getg2ir(lugb, msk1, msk2, mnum, cbuf, nlen, nnum, nmess, iret)
use re_alloc ! needed for subroutine realloc
implicit none
character(len = 1), pointer, dimension(:) :: cbuf
integer, intent(in) :: lugb, msk1, msk2, mnum
integer, intent(out) :: nlen, nnum, nmess, iret

interface ! required for cbuf pointer
subroutine getg2i2r(lugb, msk1, msk2, mnum, idxver, cbuf, nlen, &
nnum, nmess, iret)
character(len = 1), pointer, dimension(:) :: cbuf
integer, intent(in) :: lugb, msk1, msk2, mnum, idxver
integer, intent(out) :: nlen, nnum, nmess, iret
end subroutine getg2i2r
end interface

CHARACTER(LEN = 1), POINTER, DIMENSION(:) :: CBUF
INTEGER, INTENT(IN) :: LUGB, MSK1, MSK2, MNUM
INTEGER, INTENT(OUT) :: NLEN, NNUM, NMESS, IRET
CHARACTER(LEN = 1), POINTER, DIMENSION(:) :: CBUFTMP
call getg2i2r(lugb, msk1, msk2, mnum, 1, cbuf, nlen, nnum, nmess, iret)
end subroutine getg2ir

!> Generate an index record for a message in a GRIB2 file.
!>
!> The index record contains byte offsets to the message, it's length,
!> and byte offsets within the message to each section. The index file
!> record format is documented in subroutine ixgb2().
!>
!> This subroutine is like getg2ir(), but this subroutine can generate
!> index files in version 1 or 2 format. Use index version 2 for files
!> > 2 GB.
!>
!> @note Subprogram can be called from a multiprocessing environment.
!> Do not engage the same logical unit from more than one processor.
!>
!> @param[in] lugb Unit of the unblocked GRIB file. Must
!> be opened by [baopen() or baopenr()]
!> (https://noaa-emc.github.io/NCEPLIBS-bacio/).
!> @param[in] msk18 Number of bytes to search for first message.
!> @param[in] msk28 Number of bytes to search for other messages.
!> @param[in] mnum Number of GRIB messages to skip (usually 0).
!> @param[in] idxver Index version, 1 for legacy, 2 for files > 2 GB.
!> @param[out] cbuf Pointer to a buffer that will get the index
!> records. If any memory is associated with cbuf when this subroutine
!> is called, cbuf will be nullified in the subroutine. Initially cbuf
!> will get an allocation of 5000 bytes. realloc() will be used to
!> increase the size if necessary. Users must free memory that cbuf
!> points to when cbuf is no longer needed.
!> @param[out] nlen Total length of index record buffer in bytes.
!> @param[out] nnum Number of index records, zero if no GRIB
!> messages are found.
!> @param[out] nmess Last GRIB message in file successfully processed
!> @param[out] iret Return code.
!> - 0 No error.
!> - 1 Not enough memory available to hold full index buffer.
!> - 2 Not enough memory to allocate initial index buffer.
!> - 3 Error deallocating memory.
!>
!> @author Mark Iredell, Edward Hartnett @date Feb 4, 2024
subroutine getg2i2r(lugb, msk18, msk28, mnum, idxver, cbuf, nlen, &
nnum, nmess, iret)
use re_alloc ! needed for subroutine realloc
implicit none

character(len = 1), pointer, dimension(:) :: cbuf
integer, intent(in) :: lugb
integer (kind = 8), intent(in) :: msk18, msk28
integer :: msk1, msk2
integer, intent(in) :: mnum, idxver
integer, intent(out) :: nlen, nnum, nmess, iret
character(len = 1), pointer, dimension(:) :: cbuftmp
integer :: nbytes, newsize, next, numfld, m, mbuf, lskip, lgrib
integer :: istat, iseek, init, iret1
PARAMETER(INIT = 50000, NEXT = 10000)
parameter(init = 50000, next = 10000)

INTERFACE ! REQUIRED FOR CBUF POINTER
SUBROUTINE IXGB2(LUGB, LSKIP, LGRIB, CBUF, NUMFLD, MLEN, IRET)
INTEGER, INTENT(IN) :: LUGB, LSKIP, LGRIB
CHARACTER(LEN = 1), POINTER, DIMENSION(:) :: CBUF
INTEGER, INTENT(OUT) :: NUMFLD, MLEN, IRET
END SUBROUTINE IXGB2
END INTERFACE
interface ! required for cbuf pointer
subroutine ixgb2(lugb, lskip, lgrib, cbuf, numfld, mlen, iret)
integer :: lugb, lskip, lgrib
character(len = 1), pointer, dimension(:) :: cbuf
integer :: numfld, mlen, iret
end subroutine ixgb2
end interface

msk1 = msk18
msk2 = msk28

! Initialize.
IRET = 0
NULLIFY(CBUF)
MBUF = INIT
ALLOCATE(CBUF(MBUF), STAT = ISTAT) ! Allocate initial space for cbuf.
IF (ISTAT .NE. 0) THEN
IRET = 2
RETURN
ENDIF
iret = 0
nullify(cbuf)
mbuf = init
allocate(cbuf(mbuf), stat = istat) ! allocate initial space for cbuf.
if (istat .ne. 0) then
iret = 2
return
endif

! Search for first grib message.
ISEEK = 0
CALL SKGB(LUGB, ISEEK, MSK1, LSKIP, LGRIB)
DO M = 1, MNUM
IF(LGRIB.GT.0) THEN
ISEEK = LSKIP + LGRIB
CALL SKGB(LUGB, ISEEK, MSK2, LSKIP, LGRIB)
ENDIF
ENDDO
iseek = 0
call skgb(lugb, iseek, msk1, lskip, lgrib)
do m = 1, mnum
if(lgrib.gt.0) then
iseek = lskip + lgrib
call skgb(lugb, iseek, msk2, lskip, lgrib)
endif
enddo

! Get index records for every grib message found.
NLEN = 0
Expand Down Expand Up @@ -342,7 +406,7 @@ END SUBROUTINE IXGB2
ISEEK = LSKIP + LGRIB
CALL SKGB(LUGB, ISEEK, MSK2, LSKIP, LGRIB)
ENDDO
END SUBROUTINE GETG2IR
END SUBROUTINE GETG2I2R

!> Find information about a GRIB field from the index and fill a @ref
!> grib_mod::gribfield.
Expand Down Expand Up @@ -646,21 +710,93 @@ END SUBROUTINE GETGB2S
!> - byte kk + 1- ll the data representation section (drs)
!> - byte ll + 1-ll + 6 first 6 bytes of the bit map section (bms)
!>
!> @param[in] lugb Unit of the unblocked GRIB file. Must
!> @param lugb Unit of the unblocked GRIB file. Must
!> be opened by [baopen() or baopenr()]
!> (https://noaa-emc.github.io/NCEPLIBS-bacio/).
!> @param[in] lskip Number of bytes to skip before GRIB message.
!> @param[in] lgrib Number of bytes in GRIB message. When subroutine is
!> @param lskip Number of bytes to skip before GRIB message.
!> @param lgrib Number of bytes in GRIB message. When subroutine is
!> called, this must be set to the size of the cbuf buffer.
!> @param[out] cbuf Pointer to a buffer that will get the index
!> @param cbuf Pointer to a buffer that will get the index
!> records. If any memory is associated with cbuf when this subroutine
!> is called, cbuf will be nullified in the subroutine. Initially cbuf
!> will get an allocation of 5000 bytes. realloc() will be used to
!> increase the size if necessary. Users must free memory that cbuf
!> points to when cbuf is no longer needed.
!> @param numfld Number of index records created.
!> @param mlen Total length of all index records.
!> @param iret Return code
!> - 0 No error
!> - 1 Not enough memory available to hold full index buffer.
!> - 2 I/O error in read.
!> - 3 GRIB message is not edition 2.
!> - 4 Not enough memory to allocate extent to index buffer.
!> - 5 Unidentified GRIB section encountered.
!>
!> @author Mark Iredell @date 1995-10-31
subroutine ixgb2(lugb, lskip, lgrib, cbuf, numfld, mlen, iret)
implicit none
integer :: lugb, lskip, lgrib
character(len = 1), pointer, dimension(:) :: cbuf
integer :: numfld, mlen, iret
integer (kind = 8) :: lskip8

interface
subroutine ix2gb2(lugb, lskip8, idxver, lgrib, cbuf, numfld, mlen, iret)
integer :: lugb
integer (kind = 8) :: lskip8
integer :: idxver, lgrib
character(len = 1), pointer, dimension(:) :: cbuf
integer :: numfld, mlen, iret
end subroutine ix2gb2
end interface

lskip8 = lskip
call ix2gb2(lugb, lskip8, 1, lgrib, cbuf, numfld, mlen, iret)
end SUBROUTINE IXGB2

!> Generate an index record for each field in a GRIB2 message. The index
!> records are written to index buffer pointed to by cbuf. All integers
!> in the index are in big-endian format.
!>
!> This subroutine is called by getg2ir(), which packages the index
!> records into an index file.
!>
!> The index buffer returned contains index records with the
!> format:
!> - byte 001 - 004 length of index record
!> - byte 005 - 008 bytes to skip in data file before GRIB message
!> - byte 009 - 012 bytes to skip in message before lus (local use) set = 0, if no local section.
!> - byte 013 - 016 bytes to skip in message before gds
!> - byte 017 - 020 bytes to skip in message before pds
!> - byte 021 - 024 bytes to skip in message before drs
!> - byte 025 - 028 bytes to skip in message before bms
!> - byte 029 - 032 bytes to skip in message before data section
!> - byte 033 - 040 bytes total in the message
!> - byte 041 - 041 GRIB version number (2)
!> - byte 042 - 042 message discipline
!> - byte 043 - 044 field number within GRIB2 message
!> - byte 045 - ii identification section (ids)
!> - byte ii + 1- jj grid definition section (gds)
!> - byte jj + 1- kk product definition section (pds)
!> - byte kk + 1- ll the data representation section (drs)
!> - byte ll + 1-ll + 6 first 6 bytes of the bit map section (bms)
!>
!> @param lugb Unit of the unblocked GRIB file. Must
!> be opened by [baopen() or baopenr()]
!> (https://noaa-emc.github.io/NCEPLIBS-bacio/).
!> @param lskip8 Number of bytes to skip before GRIB message.
!> @param idxver Index version, 1 for legacy, 2 for GRIB2 files > 2 GB.
!> @param lgrib Number of bytes in GRIB message. When subroutine is
!> called, this must be set to the size of the cbuf buffer.
!> @param cbuf Pointer to a buffer that will get the index
!> records. If any memory is associated with cbuf when this subroutine
!> is called, cbuf will be nullified in the subroutine. Initially cbuf
!> will get an allocation of 5000 bytes. realloc() will be used to
!> increase the size if necessary. Users must free memory that cbuf
!> points to when cbuf is no longer needed.
!> @param[out] numfld Number of index records created.
!> @param[out] mlen Total length of all index records.
!> @param[out] iret Return code
!> @param numfld Number of index records created.
!> @param mlen Total length of all index records.
!> @param iret Return code
!> - 0 No error
!> - 1 Not enough memory available to hold full index buffer.
!> - 2 I/O error in read.
Expand All @@ -669,19 +805,25 @@ END SUBROUTINE GETGB2S
!> - 5 Unidentified GRIB section encountered.
!>
!> @author Mark Iredell @date 1995-10-31
SUBROUTINE IXGB2(LUGB, LSKIP, LGRIB, CBUF, NUMFLD, MLEN, IRET)
USE RE_ALLOC ! NEEDED FOR SUBROUTINE REALLOC
subroutine ix2gb2(lugb, lskip8, idxver, lgrib, cbuf, numfld, mlen, iret)
use re_alloc ! needed for subroutine realloc
implicit none

integer :: lugb
integer (kind = 8) :: lskip8
integer :: idxver, lgrib
CHARACTER(LEN = 1), POINTER, DIMENSION(:) :: CBUF
integer :: numfld, mlen, iret
integer :: lskip

CHARACTER CVER, CDISC
CHARACTER(LEN = 4) :: CTEMP
INTEGER LOCLUS, LOCGDS, LENGDS, LOCBMS
integer :: indbmp, numsec, next, newsize, mova2i, mbuf, lindex
integer :: linmax, ixskp
integer :: mxspd, mxskp, mxsgd, mxsdr, mxsbm, mxlus
integer :: mxlen, mxds, mxfld, mxbms
integer :: init, ixlus, lugb, lskip, lgrib, numfld, mlen, iret
integer :: init, ixlus
integer :: ixsgd, ibread, ibskip, ilndrs, ilnpds, istat, ixds
integer :: ixspd, ixfld, ixids, ixlen, ixsbm, ixsdr
integer :: lbread, lensec, lensec1
Expand All @@ -693,6 +835,7 @@ SUBROUTINE IXGB2(LUGB, LSKIP, LGRIB, CBUF, NUMFLD, MLEN, IRET)
CHARACTER CBREAD(LINMAX), CINDEX(LINMAX)
CHARACTER CIDS(LINMAX), CGDS(LINMAX)

lskip = lskip8
LOCLUS = 0
IRET = 0
MLEN = 0
Expand Down Expand Up @@ -811,7 +954,7 @@ SUBROUTINE IXGB2(LUGB, LSKIP, LGRIB, CBUF, NUMFLD, MLEN, IRET)
ENDIF
IBSKIP = IBSKIP + LENSEC
ENDDO
END SUBROUTINE IXGB2
end subroutine ix2gb2

!> Free all memory associated with the library.
!>
Expand Down
Loading
Loading