Skip to content

Commit

Permalink
Add ability to read ID 31, clean up altimeter attrs
Browse files Browse the repository at this point in the history
  • Loading branch information
@jmcvey3
jmcvey3 committed Feb 9, 2024
1 parent 9affa83 commit e45186e
Show file tree
Hide file tree
Showing 3 changed files with 74 additions and 52 deletions.
8 changes: 6 additions & 2 deletions dolfyn/io/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ def _create_dataset(data):
readers.
Direction 'dir' coordinates are set in `set_coords`
"""
tag = ['_avg', '_b5', '_echo', '_bt', '_gps', '_altraw', '_sl']
tag = ['_avg', '_b5', '_echo', '_bt', '_gps', '_altraw', '_altraw_avg', '_sl']

ds_dict = {}
for key in data['coords']:
Expand Down Expand Up @@ -185,7 +185,9 @@ def _create_dataset(data):
else:
shp = data['data_vars'][key].shape
if len(shp) == 1: # 1D variables
if any(val in key for val in tag):
if '_altraw_avg' in key:
tg = '_altraw_avg'
elif any(val in key for val in tag):
tg = '_' + key.rsplit('_')[-1]
else:
tg = ''
Expand All @@ -196,6 +198,8 @@ def _create_dataset(data):
ds_dict[key] = {"dims": ("range_echo", "time_echo"), "data": data['data_vars'][key]}
elif key == 'samp_altraw':
ds_dict[key] = {"dims": ("n_altraw", "time_altraw"), "data": data['data_vars'][key]}
elif key == 'samp_altraw_avg':
ds_dict[key] = {"dims": ("n_altraw_avg", "time_altraw_avg"), "data": data['data_vars'][key]}

# ADV/ADCP instrument vector data, bottom tracking
elif shp[0] == n_beams and not any(val in key for val in tag[:3]):
Expand Down
112 changes: 65 additions & 47 deletions dolfyn/io/nortek2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,16 +226,20 @@ def init_data(self, ens_start, ens_stop):
outdat = {}
nens = int(ens_stop - ens_start)

# ID 26 usually only recorded in first ensemble
n26 = ((self._index['ID'] == 26) &
(self._index['ens'] >= ens_start) &
(self._index['ens'] < ens_stop)).sum()
if not n26 and 26 in self._burst_readers:
# ID 26 and 31 recorded infrequently
def n_id(id):
return ((self._index['ID'] == id) &
(self._index['ens'] >= ens_start) &
(self._index['ens'] < ens_stop)).sum()
n_altraw = {26: n_id(26), 31: n_id(31)}
if not n_altraw[26] and 26 in self._burst_readers:
self._burst_readers.pop(26)
if not n_altraw[31] and 31 in self._burst_readers:
self._burst_readers.pop(31)

for ky in self._burst_readers:
if ky == 26:
n = n26
if (ky == 26) or (ky == 31):
n = n_altraw[ky]
ens = np.zeros(n, dtype='uint32')
else:
ens = np.arange(ens_start,
Expand Down Expand Up @@ -278,7 +282,7 @@ def readfile(self, ens_start=0, ens_stop=None):
outdat['filehead_config'] = self.filehead_config
print('Reading file %s ...' % self.fname)
c = 0
c26 = 0
c_altraw = {26: 0, 31: 0}
self.f.seek(self._ens_pos[ens_start], 0)
while True:
try:
Expand All @@ -290,9 +294,9 @@ def readfile(self, ens_start=0, ens_stop=None):
# "avg data record" (vel_avg + ast_avg), "bottom track data record" (bt),
# "interleaved burst data record" (vel_b5), "echosounder record" (echo)
self._read_burst(id, outdat[id], c)
elif id in [26]:
# "burst altimeter raw record" (alt_raw) - recorded on nens==0
rdr = self._burst_readers[26]
elif id in [26, 31]:
# "burst altimeter raw record" (_altraw), "avg altimeter raw record" (_altraw_avg)
rdr = self._burst_readers[id]
if not hasattr(rdr, '_nsamp_index'):
first_pass = True
tmp_idx = rdr._nsamp_index = rdr._names.index('nsamp_alt')
Expand All @@ -317,21 +321,21 @@ def readfile(self, ens_start=0, ens_stop=None):
rdr._cs_struct = defs.Struct(
'<' + '{}H'.format(int(rdr.nbyte // 2)))
# Initialize the array
outdat[26]['samp_alt'] = defs._nans(
outdat[id]['samp_alt'] = defs._nans(
[rdr._N[tmp_idx],
len(outdat[26]['samp_alt'])],
len(outdat[id]['samp_alt'])],
dtype=np.uint16)
else:
if sz != rdr._N[tmp_idx]:
raise Exception(
"The number of samples in this 'Altimeter Raw' "
"burst is different from prior bursts.")
self._read_burst(id, outdat[id], c26)
outdat[id]['ensemble'][c26] = c
c26 += 1
self._read_burst(id, outdat[id], c_altraw[id])
outdat[id]['ensemble'][c_altraw[id]] = c
c_altraw[id] += 1

elif id in [27, 29, 30, 31, 35, 36]: # unknown how to handle
# "bottom track record", DVL, "altimeter record", "avg altimeter raw record",
elif id in [27, 29, 30, 35, 36]: # unknown how to handle
# "bottom track record", DVL, "altimeter record",
# "raw echosounder data record", "raw echosounder transmit data record"
if self.debug:
logging.debug(
Expand Down Expand Up @@ -396,11 +400,33 @@ def sci_data(self, dat):
dnow['vel'] = (dnow['vel'] *
10.0 ** dnow['vel_scale']).astype('float32')

def __exit__(self, type, value, trace,):
self.f.close()

def __enter__(self,):
return self
def _altraw_reorg(outdat, tag=''):
"""Submethod for `_reorg` particular to raw altimeter pings (ID 26 and 31)
"""
for ky in list(outdat['data_vars']):
if ky.endswith('raw' + tag) and not ky.endswith('_altraw' + tag):
outdat['data_vars'].pop(ky)
outdat['coords']['time_altraw' + tag] = outdat['coords'].pop('timeraw' + tag)
# convert "signed fractional" to float
outdat['data_vars']['samp_altraw' + tag] = outdat['data_vars']['samp_altraw' + tag].astype('float32') / 2**8

# Read altimeter status
outdat['data_vars'].pop('status_altraw' + tag)
status_alt = lib._alt_status2data(outdat['data_vars']['status_alt' + tag])
for ky in status_alt:
outdat['attrs'][ky + tag] = lib._collapse(
status_alt[ky].astype('uint8'), name=ky)
outdat['data_vars'].pop('status_alt' + tag)

# Power level index
power = {0: 'high', 1: 'med-high', 2: 'med-low', 3: 'low'}
outdat['attrs']['power_level_alt' + tag] = power[outdat['attrs'].pop('power_level_idx_alt' + tag)]

# Other attrs
for ky in list(outdat['attrs']):
if ky.endswith('raw' + tag):
outdat['attrs'][ky.split('raw')[0] + '_alt' + tag] = outdat['attrs'].pop(ky)


def _reorg(dat):
Expand All @@ -418,7 +444,8 @@ def _reorg(dat):
cfg['inst_type'] = 'ADCP'

for id, tag in [(21, ''), (22, '_avg'), (23, '_bt'),
(24, '_b5'), (26, 'raw'), (28, '_echo')]:
(24, '_b5'), (26, 'raw'), (28, '_echo'),
(31, 'raw_avg')]:
if id in [24, 26]:
collapse_exclude = [0]
else:
Expand Down Expand Up @@ -482,24 +509,9 @@ def _reorg(dat):

# Move 'altimeter raw' data to its own down-sampled structure
if 26 in dat:
for ky in list(outdat['data_vars']):
if ky.endswith('raw') and not ky.endswith('_altraw'):
outdat['data_vars'].pop(ky)
outdat['coords']['time_altraw'] = outdat['coords'].pop('timeraw')
outdat['data_vars']['samp_altraw'] = outdat['data_vars']['samp_altraw'].astype('float32') / 2**8 # convert "signed fractional" to float

# Read altimeter status
outdat['data_vars'].pop('status_altraw')
status_alt = lib._alt_status2data(outdat['data_vars']['status_alt'])
for ky in status_alt:
outdat['attrs'][ky] = lib._collapse(
status_alt[ky].astype('uint8'), name=ky)
outdat['data_vars'].pop('status_alt')

# Power level index
power = {0: 'high', 1: 'med-high', 2: 'med-low', 3: 'low'}
outdat['attrs']['power_level_alt'] = power[outdat['attrs'].pop(
'power_level_idx_alt')]
_altraw_reorg(outdat)
if 31 in dat:
_altraw_reorg(outdat, tag='_avg')

# Read status data
status0_vars = [x for x in outdat['data_vars'] if 'status0' in x]
Expand Down Expand Up @@ -537,7 +549,7 @@ def _reorg(dat):
outdat['attrs'][ky] = lib._collapse(
status0_data[ky].astype('uint8'), name=ky)

# Remove status0 variables - keep status variables as they useful for finding missing pings
# Remove status0 variables - keep status variables as they are useful for finding missing pings
[outdat['data_vars'].pop(var) for var in status0_vars]

# Set coordinate system
Expand All @@ -563,9 +575,11 @@ def _reorg(dat):

return outdat


def _clean_dp_skips(data):
"""Removes zeros from interwoven measurements taken in a dual profile
configuration."""
configuration.
"""
for id in data:
if id == 'filehead_config':
continue
Expand All @@ -575,6 +589,7 @@ def _clean_dp_skips(data):
if var not in ['units', 'long_name', 'standard_name']:
data[id][var] = np.squeeze(data[id][var][..., skips])


def _reduce(data):
"""This function takes the output from `reorg`, and further simplifies the
data. Mostly this is combining system, environmental, and orientation data
Expand Down Expand Up @@ -647,19 +662,22 @@ def _reduce(data):
time = val
dc['time'] = dc[time]


def split_dp_datasets(ds):
"""Splits a dataset containing dual profiles into individual profiles
"""
# Figure out which variables belong to which profile based on time variables
# Figure out which variables belong to which profile based on length of time variables
t_dict = {}
for t in ds.coords:
if 'altraw' in t:
continue
elif 'time' in t:
if 'time' in t:
t_dict[t] = ds[t].size

other_coords = []
for key, val in t_dict.items():
if val != t_dict['time']:
if key.endswith('altraw'):
# altraw goes with burst, altraw_avg goes with avg
continue
other_coords.append(key)

# Fetch variables, coordinates, and attrs for second profiling configuration
Expand Down
6 changes: 3 additions & 3 deletions dolfyn/io/nortek2_lib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,7 @@ def _collapse(vec, name=None, exclude=[]):
"Values found: {} (counts: {}).\n"
"Using the most common value: {}".format(
name, list(uniq), list(counts), val))

return val


Expand All @@ -487,11 +487,11 @@ def _calc_config(index):
ids = np.unique(index['ID'])
config = {}
for id in ids:
if id not in [21, 22, 23, 24, 26, 28]:
if id not in [21, 22, 23, 24, 26, 28, 31]:
continue
if id == 23:
type = 'bt'
elif id == 22:
elif (id == 22) or (id == 31):
type = 'avg'
else:
type = 'burst'
Expand Down

0 comments on commit e45186e

Please sign in to comment.