Skip to content

Commit

Permalink
adjusting timestamps to unix base
Browse files Browse the repository at this point in the history
  • Loading branch information
@BryonLewis
BryonLewis committed Feb 11, 2025
1 parent 16fd490 commit 59b1389
Show file tree
Hide file tree
Showing 5 changed files with 132 additions and 45 deletions.
16 changes: 8 additions & 8 deletions client/src/components/DataSelection/NetCDFDataConfigurator.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,12 +194,12 @@ export default defineComponent({
}
const data = getVariableInformation(newLayerSlice.value);
if (data.startDate) {
sliceLayerRangeStep.value = (data.max / 1e6 - data.min / 1e6) / (data.steps || 1);
const startDate = new Date(data.min / 1e6);
const endDate = new Date(data.max / 1e6);
sliceLayerRangeStep.value = (data.max - data.min) / (data.steps || 1);
const startDate = new Date(data.min);
const endDate = new Date(data.max);
const diffMilli = endDate.getTime() - startDate.getTime();
const differenceInHours = diffMilli / (1000 * 60 * 60);
sliceLayerRange.value = [data.min / 1e6, data.max / 1e6];
sliceLayerRange.value = [data.min, data.max];
sliceLayerRangeStep.value = Math.round(differenceInHours / (data.steps || 1)) * (1000 * 60 * 60);
} else {
sliceLayerRange.value = [data.min, data.max];
Expand Down Expand Up @@ -654,8 +654,8 @@ export default defineComponent({
<v-range-slider
v-model="sliceLayerRange"
:step="sliceLayerRangeStep"
:min="getVariableInformation(newLayerSlice).min / 1e6"
:max="getVariableInformation(newLayerSlice).max / 1e6"
:min="getVariableInformation(newLayerSlice).min"
:max="getVariableInformation(newLayerSlice).max"
class="pt-2"
hide-details
>
Expand All @@ -664,10 +664,10 @@ export default defineComponent({
</template>

<template #prepend>
<span>{{ convertTimestampNSToDatetimeString(getVariableInformation(newLayerSlice)?.min / 1e6) }}</span>
<span>{{ convertTimestampNSToDatetimeString(getVariableInformation(newLayerSlice)?.min) }}</span>
</template>
<template #append>
<span>{{ convertTimestampNSToDatetimeString(getVariableInformation(newLayerSlice)?.max / 1e6) }}</span>
<span>{{ convertTimestampNSToDatetimeString(getVariableInformation(newLayerSlice)?.max) }}</span>
</template>
</v-range-slider>
<v-row align="center" justify="center" class="py-2">
Expand Down
2 changes: 1 addition & 1 deletion client/src/components/NetCDFLayerConfig.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ export default defineComponent({
if (found.sliding.variable === 'time') {
// convert unix timestamp to human readable date YYYY-MM-DD
try {
const date = new Date((mapSlicer[i] as number) / 1e6);
const date = new Date((mapSlicer[i] as number) * 1000);
// eslint-disable-next-line prefer-destructuring
mapSlicer[i] = date.toISOString().split('T')[0];
} catch (e) {
Expand Down
60 changes: 56 additions & 4 deletions client/src/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,11 +174,11 @@ const formatNumPrecision = (num: number, range?: number) => {
return num;
};

function convertTimestampNSToDatetimeString(timestamp: number): string {
function convertTimestampNSToDatetimeString(timestamp: number, format = 'date'): string {
// Convert the nanoseconds to milliseconds
const milliseconds = Math.round(timestamp);
const seconds = Math.round(timestamp);
// Create a Date object
const date = new Date(milliseconds);
const date = new Date(seconds * 1000);
// Extract the parts of the date in UTC
const year = date.getUTCFullYear();
const month = (date.getUTCMonth() + 1).toString().padStart(2, '0'); // Months are 0-indexed
Expand All @@ -188,7 +188,57 @@ function convertTimestampNSToDatetimeString(timestamp: number): string {
const hours = date.getUTCHours().toString().padStart(2, '0');
const minutes = date.getUTCMinutes().toString().padStart(2, '0');
// Format the date as YYYYMMDD HH:MM
return `${year}${month}${day} ${hours}:${minutes}`;
if (format === 'date') {
return `${year}-${month}-${day}`;
} if (format === 'year') {
return `${year}`;
}
return `${year}-${month}-${day} ${hours}:${minutes}`;
}

function formatISOToYYMMDD(dateString: string): string | null {
const isoRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?$/;

if (!isoRegex.test(dateString)) {
return null; // Not a valid ISO 8601 string
}

try {
const date = new Date(dateString);
if (Number.isNaN(date.getTime())) {
return null; // Invalid date
}

const yy = String(date.getFullYear()).slice(-2);
const mm = String(date.getMonth() + 1).padStart(2, '0');
const dd = String(date.getDate()).padStart(2, '0');

return `${yy}-${mm}-${dd}`;
} catch {
return null;
}
}

function formatCompactToISO(compactNumber: number, format: 'date' | 'datetime' = 'date'): string | null {
const compactStr = compactNumber.toString();

if (!/^(\d{12}|\d{8})$/.test(compactStr)) {
return null; // Invalid format
}

const year = compactStr.slice(0, 4);
const month = compactStr.slice(4, 6);
const day = compactStr.slice(6, 8);

if (format === 'date') {
return `${year}-${month}-${day}`;
} if (compactStr.length === 12) {
const hour = compactStr.slice(8, 10);
const minute = compactStr.slice(10, 12);
return `${year}-${month}-${day} ${hour}:${minute}`;
}

return `${year}-${month}-${day}`; // Default fallback
}

function convert360Longitude(longitude: number): number {
Expand All @@ -206,5 +256,7 @@ export {
formatNumPrecision,
createColorNumberPairs,
convertTimestampNSToDatetimeString,
formatCompactToISO,
formatISOToYYMMDD,
convert360Longitude,
};
2 changes: 1 addition & 1 deletion uvdat/core/rest/netcdf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ def images(self, request, *args, **kwargs):
"max": sliding_dim.get('max', step_count),
"variable": sliding_dim.get('variable', 'time'),
}
sliding_data['step'] = (sliding_data['max'] - sliding_data['min']) / step_count
sliding_data['step'] = (sliding_data['max'] - sliding_data['min']) / (step_count - 1)
response_data = {
'netCDFLayer': int(netcdf_layer_id),
'parent_bounds': netcdf_layer.bounds.envelope.coords, # 4-tuple for the parent layer bounds
Expand Down
97 changes: 66 additions & 31 deletions uvdat/core/tasks/netcdf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,49 @@
logger = logging.getLogger(__name__)


def convert_time(obj, output='compact'):
if isinstance(obj, str): # Handle ctime (string)
dt_obj = datetime.strptime(obj, '%a %b %d %H:%M:%S %Y')
elif isinstance(obj, np.datetime64): # Handle datetime64
dt_obj = pd.Timestamp(obj).to_pydatetime()
elif isinstance(obj, datetime): # Handle Python datetime objects
dt_obj = obj
elif isinstance(obj, (cftime.DatetimeNoLeap, cftime.DatetimeAllLeap, cftime.Datetime360Day, cftime.DatetimeJulian)):
dt_obj = datetime(obj.year, obj.month, obj.day, obj.hour, obj.minute, obj.second)
elif isinstance(obj, (int, float)):
if obj > 1e10: # Assume milliseconds timestamp
dt_obj = datetime.fromtimestamp(obj / 1000)
else: # Assume seconds timestamp
dt_obj = datetime.fromtimestamp(obj)
else:
return obj # Return as-is if the type is unrecognized

if output == 'iso':
return dt_obj.isoformat()
elif output == 'datetime':
return dt_obj
elif output == 'compact':
return int(dt_obj.strftime('%Y%m%d%H%M%S'))
elif output == 'unix':
return dt_obj.timestamp()


def convert_to_timestamp(obj):
if isinstance(obj, str): # Handle ctime (string)
dt_obj = datetime.strptime(obj, '%a %b %d %H:%M:%S %Y')
return dt_obj.timestamp()
elif isinstance(obj, np.datetime64): # Handle datetime64
return (obj - np.datetime64('1970-01-01T00:00:00')) / np.timedelta64(1, 's')
elif isinstance(obj, datetime): # Handle Python datetime objects
return obj.timestamp()
elif isinstance(obj, (cftime.DatetimeNoLeap, cftime.DatetimeAllLeap, cftime.Datetime360Day, cftime.DatetimeJulian)):
dt = obj
dt_obj = datetime(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
unix_timestamp = pd.Timestamp(dt_obj).timestamp()
return unix_timestamp
else:
return obj

def create_netcdf_data_layer(file_item, metadata):
with tempfile.TemporaryDirectory() as temp_dir:
raw_data_path = Path(temp_dir, 'netcdf.nc')
Expand Down Expand Up @@ -51,34 +94,38 @@ def create_netcdf_data_layer(file_item, metadata):
# Calculate min and max values if the variable has numeric data
if isinstance(variable.values[0], (cftime.DatetimeNoLeap, cftime.DatetimeAllLeap, cftime.Datetime360Day, cftime.DatetimeJulian)):
vals = []
str_vals = []
for item in variable.values:
print(item)
dt = item
dt_obj = datetime.datetime(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
unix_timestamp = pd.Timestamp(dt_obj).timestamp()
str_vals.append(pd.Timestamp(dt_obj).isoformat())
vals.append(unix_timestamp)
var_min = float(min(vals)) if variable.size > 0 else None
var_max = float(max(vals)) if variable.size > 0 else None
var_info['min'] = var_min
var_info['max'] = var_max
dt_obj = datetime(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
timeobj = convert_time(pd.Timestamp(dt_obj), 'datetime')
vals.append(timeobj)
var_min = (min(vals)) if variable.size > 0 else None
var_max = (max(vals)) if variable.size > 0 else None
var_info['min'] = convert_time(var_min, 'unix')
var_info['max'] = convert_time(var_max, 'unix')
var_info['startDate'] = convert_time(var_min, 'iso')
var_info['endDate'] = convert_time(var_max, 'iso')
var_info['timeType'] = 'unix'
var_info['steps'] = variable.size
# var_info['timeMap'] = str_vals
else:
try:
var_min = float(variable.min().values) if variable.size > 0 else None
var_max = float(variable.max().values) if variable.size > 0 else None
if 'datetime' in str(variable.dtype):
var_info['startDate'] = str(variable.min().values)
var_info['endDate'] = str(variable.max().values)
# str_vals = []
# for item in variable.values:
# str_vals.append(str(item))
# var_info['timeMap'] = str_vals

var_info['min'] = var_min
var_info['max'] = var_max

if 'time' in var_name:
var_info['min'] = convert_time(var_min, 'unix')
var_info['max'] = convert_time(var_max, 'unix')
var_info['startDate'] = convert_time(var_min, 'iso')
var_info['endDate'] = convert_time(var_max, 'iso')

var_info['timeType'] = 'unix'
else:
var_info['min'] = var_min
var_info['max'] = var_max
var_info['steps'] = variable.size

if var_name in description['dimensions'].keys():
Expand Down Expand Up @@ -321,18 +368,6 @@ def preview_netcdf_slice(
return base64_image


def convert_to_timestamp(obj):
if isinstance(obj, str): # Handle ctime (string)
dt_obj = datetime.strptime(obj, '%a %b %d %H:%M:%S %Y')
return dt_obj.timestamp()
elif isinstance(obj, np.datetime64): # Handle datetime64
return (obj - np.datetime64('1970-01-01T00:00:00')) / np.timedelta64(1, 's')
elif isinstance(obj, datetime): # Handle Python datetime objects
return obj.timestamp()
else:
return obj


@shared_task
def create_netcdf_slices(
netcdf_data_id: int,
Expand Down Expand Up @@ -546,8 +581,8 @@ def create_netcdf_slices(
bounds = None
if slicer_range:
slicer_range = [
convert_to_timestamp(slicer_range[0]),
convert_to_timestamp(slicer_range[1]),
convert_time(slicer_range[0], 'unix'),
convert_time(slicer_range[1], 'unix'),
]
if variables:
try:
Expand Down

0 comments on commit 59b1389

Please sign in to comment.