-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathema_features.py
132 lines (109 loc) · 5.28 KB
/
ema_features.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
#%%
# Importing libraries and dependencies
import pandas as pd
import numpy as np
import matplotlib as plot
import matplotlib.pylab as plt
import sklearn as sk
import seaborn as sns
import importlib
from helpers import *
from sklearn import preprocessing
# region [cell] Init variables
#%%
SLIDING_WINDOW = 7
SAMPLE_TIME = '1d'
SELF_INIT_MAX = 5
MAX_ENGAGEMENT_SCORE = SELF_INIT_MAX + 2
# endregion
#%%
# Reading all of the EMA data, and fusing the date and time together.
def get_EMA_features_and_target_for_patient(patient_id):
patient_features, patient_self_init_features = get_patient_features(patient_id)
# todo Include patient_init_features
patient_engagement = get_engagement(patient_features, patient_self_init_features).rename('prior_engagement')
patient_EMA_features = patient_features.join(patient_engagement)
return (patient_EMA_features, patient_engagement)
def read_EMA_code():
full_EMA = pd.read_csv('data/v2/ema_logs/ECD_X970_12345678.csv',
parse_dates=[['xEmaDate', 'xEmaTime']])
full_EMA['xEmaDate'] = full_EMA['xEmaDate_xEmaTime']
full_EMA = full_EMA.drop(['xEmaDate_xEmaTime'], axis=1)
return full_EMA
def normalize_column(column):
norm_column = (column - column.min()) / (column.max() - column.min())
return norm_column
def init_EMA_patient(full_EMA, patient_id):
patient_df = full_EMA[full_EMA['ECD_ID'] == int(patient_id)]
patient_df.index = patient_df['xEmaDate']
return patient_df
def split_self_init_sessions(patient_df):
patient_self_init_df = patient_df[patient_df['xEmaSchedule'] == 4]
patient_df = patient_df[patient_df['xEmaSchedule'] != 4]
return (patient_df, patient_self_init_df)
def get_patient_features(patient_id):
full_EMA = read_EMA_code()
patient_df = init_EMA_patient(full_EMA, patient_id)
patient_df = extract_ema_vals(patient_df)
patient_df, patient_self_init_df = split_self_init_sessions(patient_df)
if len(patient_df) == 0:
return None
patient_df = resample_patient_dataframe(patient_df, SAMPLE_TIME)
patient_self_init_df = resample_patient_dataframe(patient_self_init_df, SAMPLE_TIME)
if len(patient_self_init_df) == 0:
return None
return (patient_df.fillna(0), patient_self_init_df.fillna(0))
def extract_ema_vals(patient_df):
ema_q = one_hot_encode_feature(patient_df, 'xEmaQuestion', prefix='ema_q')
ema_a = ema_q.multiply(patient_df['xEmaRating'], axis='index')
ema_q = ema_q.add_prefix('count_')
ema_a = ema_a.add_prefix('average_')
patient_df = patient_df.join([ema_q, ema_a])
return patient_df
def get_engagement(patient_df, patient_self_init_df):
count_regex = 'count_ema_.*'
patient_asked_count = patient_df.filter(regex=count_regex).sum(axis=1)
patient_self_init_count = patient_df.filter(regex=count_regex).sum(axis=1).apply(lambda row: min(row, SELF_INIT_MAX))
bool_patient_asked = patient_asked_count.apply(lambda row: min(1, row))
bool_patient_init = patient_self_init_count.apply(lambda row: min(1, row))
# TODO We should probably normalize in a better way somehow.
return (patient_self_init_count + bool_patient_asked + bool_patient_init)
def one_hot_encode_feature(patient_df, feature, prefix):
return pd.get_dummies(patient_df[feature], prefix=prefix)
def resample_datetimeindex(dt_index, sample_time):
date_min = dt_index.min()
date_max = dt_index.max() + pd.DateOffset(days=1)
resampled_date = pd.date_range(date_min, date_max, freq=sample_time)[:-1]
fill_data = pd.Series(resampled_date, resampled_date.floor(sample_time))
return fill_data.index
def resample_patient_dataframe(patient_df, sample_time):
if len(patient_df) == 0:
return pd.DataFrame()
ema_q_resampled = count_patient_ema_questions(patient_df, sample_time)
ema_a_resampled = avg_patient_ema_values(patient_df, sample_time)
return ema_q_resampled.join(ema_a_resampled)
def avg_patient_ema_values(patient_df, sample_time):
ema_columns = patient_df.filter(regex="average_ema_*.")
ema_columns = ema_columns.replace('', np.nan, regex=True).fillna(0)
ema_columns = ema_columns.replace(' ', np.nan).fillna(0)
ema_columns = ema_columns.astype(float)
patient_df = pd.DataFrame(index=resample_datetimeindex(patient_df.index, sample_time))
patient_df = patient_df.join(ema_columns.resample(sample_time).mean())
return patient_df
def count_patient_ema_questions(patient_df, sample_time):
ema_columns = patient_df.filter(regex="count_ema_*.")
patient_df = pd.DataFrame(index=resample_datetimeindex(patient_df.index, sample_time))
patient_df = patient_df.join(ema_columns.resample(sample_time).sum())
return patient_df
def convert_features_to_statistics(features, window):
patient_ml = pd.DataFrame(index=features.index)
for col in features.fillna(0):
patient_ml['avg_'+col+'_'+str(window)+'_days'] = features[col].rolling(
str(window)+'d').mean().shift(1)
patient_ml['min_'+col+'_'+str(window)+'_days'] = features[col].rolling(
str(window)+'d').min().shift(1)
patient_ml['max_'+col+'_'+str(window)+'_days'] = features[col].rolling(
str(window)+'d').max().shift(1)
patient_ml['std_'+col+'_'+str(window)+'_days'] = features[col].rolling(
str(window)+'d').std().shift(1)
return patient_ml