Skip to content

Commit

Permalink
Merge pull request #80 from Unidata/msround
Browse files Browse the repository at this point in the history 
replace small offset with rounding of microseconds (issue #78)
  • Loading branch information
@jswhit
jswhit authored Nov 15, 2018
2 parents 2f022eb + 391be1a commit a93a172
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 71 deletions.
118 changes: 63 additions & 55 deletions cftime/_cftime.pyx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ cdef int[13] _spm_366day = [0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 33
_rop_lookup = {Py_LT: '__gt__', Py_LE: '__ge__', Py_EQ: '__eq__',
Py_GT: '__lt__', Py_GE: '__le__', Py_NE: '__ne__'}

__version__ = '1.0.2.1'
__version__ = '1.0.3'

# Adapted from http://delete.me.uk/2005/03/iso8601.html
# Note: This regex ensures that all ISO8601 timezone formats are accepted - but, due to legacy support for other timestrings, not all incorrect formats can be rejected.
Expand Down Expand Up @@ -151,7 +151,7 @@ def date2num(dates,units,calendar='standard'):
Default is `'standard'`, which is a mixed Julian/Gregorian calendar.
returns a numeric time value, or an array of numeric time values
with approximately 10 microsecond accuracy.
with approximately 100 microsecond accuracy.
"""
calendar = calendar.lower()
basedate = _dateparse(units)
Expand Down Expand Up @@ -238,7 +238,7 @@ def num2date(times,units,calendar='standard',only_use_cftime_datetimes=False):
subclass cftime.datetime are returned for all calendars.
returns a datetime instance, or an array of datetime instances with
approximately 10 microsecond accuracy.
approximately 100 microsecond accuracy.
***Note***: The datetime instances returned are 'real' python datetime
objects if `calendar='proleptic_gregorian'`, or
Expand Down Expand Up @@ -376,7 +376,7 @@ def JulianDayFromDate(date, calendar='standard'):
"""JulianDayFromDate(date, calendar='standard')
creates a Julian Day from a 'datetime-like' object. Returns the fractional
Julian Day (approximately 10 microsecond accuracy).
Julian Day (approximately 100 microsecond accuracy).
if calendar='standard' or 'gregorian' (default), Julian day follows Julian
Calendar on and before 1582-10-5, Gregorian calendar after 1582-10-15.
Expand Down Expand Up @@ -422,14 +422,6 @@ def JulianDayFromDate(date, calendar='standard'):
fracday = hour / 24.0 + minute / 1440.0 + (second + microsecond/1.e6) / 86400.0
jd = jd - 0.5 + fracday

# Add a small offset (proportional to Julian date) for correct re-conversion.
# This is about 45 microseconds in 2000 for Julian date starting -4712.
# (pull request #433).
if calendar not in ['all_leap','no_leap','360_day','365_day','366_day']:
eps = np.array(np.finfo(np.float64).eps,np.longdouble)
eps = np.maximum(eps*jd, eps)
jd += eps

if isscalar:
return jd[0]
else:
Expand All @@ -440,7 +432,7 @@ def DateFromJulianDay(JD, calendar='standard', only_use_cftime_datetimes=False,
"""
returns a 'datetime-like' object given Julian Day. Julian Day is a
fractional day with approximately 10 microsecond accuracy.
fractional day with approximately 100 microsecond accuracy.
if calendar='standard' or 'gregorian' (default), Julian day follows Julian
Calendar on and before 1582-10-5, Gregorian calendar after 1582-10-15.
Expand All @@ -457,48 +449,64 @@ def DateFromJulianDay(JD, calendar='standard', only_use_cftime_datetimes=False,
objects are used, which are actually instances of cftime.datetime.
"""

julian = np.array(JD, dtype=np.longdouble)

# get the day (Z) and the fraction of the day (F)
# use 'round half up' rounding instead of numpy's even rounding
# so that 0.5 is rounded to 1.0, not 0 (cftime issue #49)
Z = np.atleast_1d(np.int32(_round_half_up(julian)))
F = np.atleast_1d(julian + 0.5 - Z).astype(np.longdouble)

cdef Py_ssize_t i_max = len(Z)
year = np.empty(i_max, dtype=np.int32)
month = np.empty(i_max, dtype=np.int32)
day = np.empty(i_max, dtype=np.int32)
dayofyr = np.zeros(i_max,dtype=np.int32)
dayofwk = np.zeros(i_max,dtype=np.int32)
cdef int ijd
cdef Py_ssize_t i
for i in range(i_max):
ijd = Z[i]
year[i],month[i],day[i],dayofwk[i],dayofyr[i] = _IntJulianDayToDate(ijd,calendar)

if calendar in ['standard', 'gregorian']:
ind_before = np.where(julian < 2299160.5)[0]

# Subtract the offset from JulianDayFromDate from the microseconds (pull
# request #433).
hour = np.clip((F * 24.).astype(np.int64), 0, 23)
F -= hour / 24.
minute = np.clip((F * 1440.).astype(np.int64), 0, 59)
# this is an overestimation due to added offset in JulianDayFromDate
second = np.clip((F - minute / 1440.) * 86400., 0, None)
microsecond = (second % 1)*1.e6
# remove the offset from the microsecond calculation.
if calendar not in ['all_leap','no_leap','360_day','365_day','366_day']:
eps = np.array(np.finfo(np.float64).eps,np.longdouble)
eps = np.maximum(eps*julian, eps)
microsecond = np.clip(microsecond - eps*86400.*1e6, 0, 999999)

# convert hour, minute, second to int32
hour = hour.astype(np.int32)
minute = minute.astype(np.int32)
second = second.astype(np.int32)
microsecond = microsecond.astype(np.int32)
julian = np.atleast_1d(np.array(JD, dtype=np.longdouble))

def getdateinfo(julian):
# get the day (Z) and the fraction of the day (F)
# use 'round half up' rounding instead of numpy's even rounding
# so that 0.5 is rounded to 1.0, not 0 (cftime issue #49)
Z = np.int32(_round_half_up(julian))
F = (julian + 0.5 - Z).astype(np.longdouble)

cdef Py_ssize_t i_max = len(Z)
year = np.empty(i_max, dtype=np.int32)
month = np.empty(i_max, dtype=np.int32)
day = np.empty(i_max, dtype=np.int32)
dayofyr = np.zeros(i_max,dtype=np.int32)
dayofwk = np.zeros(i_max,dtype=np.int32)
cdef int ijd
cdef Py_ssize_t i
for i in range(i_max):
ijd = Z[i]
year[i],month[i],day[i],dayofwk[i],dayofyr[i] = _IntJulianDayToDate(ijd,calendar)

if calendar in ['standard', 'gregorian']:
ind_before = np.where(julian < 2299160.5)[0]
else:
ind_before = None

# compute hour, minute, second, microsecond, convert to int32
hour = np.clip((F * 24.).astype(np.int64), 0, 23)
F -= hour / 24.
minute = np.clip((F * 1440.).astype(np.int64), 0, 59)
second = np.clip((F - minute / 1440.) * 86400., 0, None)
microsecond = (second % 1)*1.e6
hour = hour.astype(np.int32)
minute = minute.astype(np.int32)
second = second.astype(np.int32)
microsecond = microsecond.astype(np.int32)

return year,month,day,hour,minute,second,microsecond,dayofyr,dayofwk,ind_before

year,month,day,hour,minute,second,microsecond,dayofyr,dayofwk,ind_before =\
getdateinfo(julian)
# round to nearest second if within ms_eps microseconds
# (to avoid ugly errors in datetime formatting - alternative
# to adding small offset all the time as was done previously)
# see netcdf4-python issue #433 and cftime issue #78
# this is done by rounding microsends up or down, then
# recomputing year,month,day etc
# ms_eps is proportional to julian day,
# about 47 microseconds in 2000 for Julian base date in -4713
ms_eps = np.array(np.finfo(np.float64).eps,np.longdouble)
ms_eps = 86400000000.*np.maximum(ms_eps*julian, ms_eps)
microsecond = np.where(microsecond < ms_eps, 0, microsecond)
indxms = microsecond > 1000000-ms_eps
if indxms.any():
julian[indxms] = julian[indxms] + 2*ms_eps/86400000000.
year,month,day,hour,minute,second,microsecond,dayofyr,dayofwk,ind_before =\
getdateinfo(julian)
microsecond[indxms] = 0

# check if input was scalar and change return accordingly
isscalar = False
Expand Down
30 changes: 14 additions & 16 deletions test/test_cftime.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
# test cftime module for netCDF time <--> python datetime conversions.

dtime = namedtuple('dtime', ('values', 'units', 'calendar'))
dateformat = '%Y-%m-%d %H:%M:%S'


class CFTimeVariable(object):
Expand Down Expand Up @@ -131,11 +132,11 @@ def runTest(self):
self.assertTrue(self.cdftime_pg.calendar == 'proleptic_gregorian')
# check date2num method.
d = datetime(1990, 5, 5, 2, 17)
t1 = np.around(self.cdftime_pg.date2num(d))
self.assertTrue(t1 == 62777470620.0)
t1 = self.cdftime_pg.date2num(d)
self.assertTrue(np.around(t1) == 62777470620.0)
# check num2date method.
d2 = self.cdftime_pg.num2date(t1)
self.assertTrue(str(d) == str(d2))
self.assertTrue(d.strftime(dateformat) == d2.strftime(dateformat))
# check day of year.
ndayr = d.timetuple()[7]
self.assertTrue(ndayr == 125)
Expand Down Expand Up @@ -252,7 +253,6 @@ def runTest(self):
# day goes out of range).
t = 733499.0
d = num2date(t, units='days since 0001-01-01 00:00:00')
dateformat = '%Y-%m-%d %H:%M:%S'
assert_equal(d.strftime(dateformat), '2009-04-01 00:00:00')
# test edge case of issue 75 for numerical problems
for t in (733498.999, 733498.9999, 733498.99999, 733498.999999, 733498.9999999):
Expand Down Expand Up @@ -336,17 +336,11 @@ def runTest(self):
# also tests error found in issue #349
calendars=['standard', 'gregorian', 'proleptic_gregorian', 'noleap', 'julian',\
'all_leap', '365_day', '366_day', '360_day']
dateformat = '%Y-%m-%d %H:%M:%S'
dateref = datetime(2015,2,28,12)
ntimes = 1001
verbose = True # print out max error diagnostics
precis = np.finfo(np.longdouble).precision
if precis < 18:
fact = 10
else:
fact = 1.
for calendar in calendars:
eps = 10.*fact
eps = 100.
units = 'microseconds since 2000-01-30 01:01:01'
microsecs1 = date2num(dateref,units,calendar=calendar)
maxerr = 0
Expand All @@ -363,7 +357,7 @@ def runTest(self):
print('calender = %s max abs err (microsecs) = %s eps = %s' % \
(calendar,maxerr,eps))
units = 'milliseconds since 1800-01-30 01:01:01'
eps = 0.01*fact
eps = 0.1
millisecs1 = date2num(dateref,units,calendar=calendar)
maxerr = 0.
for n in range(ntimes):
Expand All @@ -378,7 +372,7 @@ def runTest(self):
if verbose:
print('calender = %s max abs err (millisecs) = %s eps = %s' % \
(calendar,maxerr,eps))
eps = 1.e-4*fact
eps = 1.e-3
units = 'seconds since 0001-01-30 01:01:01'
secs1 = date2num(dateref,units,calendar=calendar)
maxerr = 0.
Expand All @@ -394,7 +388,7 @@ def runTest(self):
if verbose:
print('calender = %s max abs err (secs) = %s eps = %s' % \
(calendar,maxerr,eps))
eps = 1.e-6*fact
eps = 1.e-5
units = 'minutes since 0001-01-30 01:01:01'
mins1 = date2num(dateref,units,calendar=calendar)
maxerr = 0.
Expand All @@ -410,7 +404,7 @@ def runTest(self):
if verbose:
print('calender = %s max abs err (mins) = %s eps = %s' % \
(calendar,maxerr,eps))
eps = 1.e-7*fact
eps = 1.e-6
units = 'hours since 0001-01-30 01:01:01'
hrs1 = date2num(dateref,units,calendar=calendar)
maxerr = 0.
Expand All @@ -426,7 +420,7 @@ def runTest(self):
if verbose:
print('calender = %s max abs err (hours) = %s eps = %s' % \
(calendar,maxerr,eps))
eps = 1.e-9*fact
eps = 1.e-8
units = 'days since 0001-01-30 01:01:01'
days1 = date2num(dateref,units,calendar=calendar)
maxerr = 0.
Expand Down Expand Up @@ -673,6 +667,10 @@ def runTest(self):
1, 'months since 01-01-01',calendar='standard')
self.assertRaises(ValueError, utime, \
'months since 01-01-01', calendar='standard')
# issue #78 - extra digits due to roundoff
assert(cftime.date2num(cftime.datetime(1, 12, 1, 0, 0, 0, 0, -1, 1), units='days since 01-01-01',calendar='noleap') == 334.0)
assert(cftime.date2num(cftime.num2date(1.0,units='days since 01-01-01',calendar='noleap'),units='days since 01-01-01',calendar='noleap') == 1.0)
assert(cftime.date2num(cftime.DatetimeNoLeap(1980, 1, 1, 0, 0, 0, 0, 6, 1),'days since 1970-01-01','noleap') == 3650.0)


class TestDate2index(unittest.TestCase):
Expand Down

0 comments on commit a93a172

Please sign in to comment.