TF implementation of Inception Time - optimized for each task using Grid Search

code/configs/tf_inception_time_config.py

@@ -0,0 +1,22 @@
+conf_tf_inception = {'modelname':'tf_inception', 'modeltype':'inception_time_model', 
+    'parameters':dict()}
+
+
+conf_tf_inception_all = {'modelname':'tf_inception_all', 'modeltype':'inception_time_model', 
+    'parameters':dict(epoch=15, batch_size=16, lr_init=0.001, lr_red="no", model_depth=9 , loss="bce" , kernel_size=60)}
+
+conf_tf_inception_diagnostic = {'modelname':'tf_inception_diagnostic', 'modeltype':'inception_time_model', 
+    'parameters':dict(epoch=25, batch_size=32, lr_init=0.001 ,lr_red="no", model_depth=6 , loss="bce" , kernel_size=60)}
+
+conf_tf_inception_form = {'modelname':'tf_inception_form', 'modeltype':'inception_time_model', 
+    'parameters':dict(epoch=25, batch_size=64, lr_init=0.001 ,lr_red="no", model_depth=6 , loss="bce" , kernel_size=20)}
+
+conf_tf_inception_rhythm = {'modelname':'tf_inception_rhythm', 'modeltype':'inception_time_model', 
+    'parameters':dict(epoch=25, batch_size=16, lr_init=0.001, lr_red="no", model_depth=9 , loss="wbce" , kernel_size=40)}
+
+conf_tf_inception_subdiagnostic = {'modelname':'tf_inception_subdiagnostic', 'modeltype':'inception_time_model', 
+    'parameters':dict(epoch=15, batch_size=64, lr_init=0.001 , lr_red="no", model_depth=6 , loss="wbce" , kernel_size=20)}
+
+conf_tf_inception_superdiagnostic = {'modelname':'tf_inception_superdiagnostic', 'modeltype':'inception_time_model', 
+    'parameters':dict(epoch=25, batch_size=64, lr_init=0.001 ,lr_red="yes", model_depth=12 , loss="bce" , kernel_size=40)}
+

code/experiments/scp_experiment.py

@@ -23,6 +23,8 @@ def __init__(self, experiment_name, task, datafolder, outputfolder, models, samp
         self.outputfolder = outputfolder
         self.datafolder = datafolder
         self.sampling_frequency = sampling_frequency
+        self.noise_mean = 0
+        self.noise_std_scale = 0.1
 
         # create folder structure if needed
         if not os.path.exists(self.outputfolder+self.experiment_name):
@@ -34,7 +36,7 @@ def __init__(self, experiment_name, task, datafolder, outputfolder, models, samp
             if not os.path.exists(outputfolder+self.experiment_name+'/data/'):
                 os.makedirs(self.outputfolder+self.experiment_name+'/data/')
 
-    def prepare(self):
+    def prepare(self, add_noise=False):
         # Load PTB-XL data
         self.data, self.raw_labels = utils.load_dataset(self.datafolder, self.sampling_frequency)
 
@@ -59,6 +61,11 @@ def prepare(self):
         self.X_train, self.X_val, self.X_test = utils.preprocess_signals(self.X_train, self.X_val, self.X_test, self.outputfolder+self.experiment_name+'/data/')
         self.n_classes = self.y_train.shape[1]
 
+        # Add noise to test data
+        if add_noise == True:
+            noise = np.random.normal(self.noise_mean,self.X_test.std() * self.noise_std_scale, size = self.X_test.shape)
+            self.X_test = self.X_test + noise
+
         # save train and test labels
         self.y_train.dump(self.outputfolder + self.experiment_name+ '/data/y_train.npy')
         self.y_val.dump(self.outputfolder + self.experiment_name+ '/data/y_val.npy')
@@ -100,6 +107,9 @@ def perform(self):
             elif modeltype == "fastai_model":
                 from models.fastai_model import fastai_model
                 model = fastai_model(modelname, n_classes, self.sampling_frequency, mpath, self.input_shape, **modelparams)
+            elif modeltype == "inception_time_model":
+                from code.models.inception_time import inception_time_model
+                model = inception_time_model(modelname, n_classes, self.sampling_frequency, mpath, self.input_shape, **modelparams)
             elif modeltype == "YOUR_MODEL_TYPE":
                 # YOUR MODEL GOES HERE!
                 from models.your_model import YourModel

code/models/inception_time.py

@@ -0,0 +1,120 @@
+from models.base_model import ClassificationModel
+import tensorflow as tf
+import numpy as np
+import tensorflow_addons as tfa
+
+class inception_time_model(ClassificationModel):
+    def __init__(self, name, n_classes,  sampling_frequency, outputfolder, input_shape, epoch=30, batch_size=32, lr_init = 0.001, lr_red="yes", model_depth=6, loss="bce", kernel_size=40, bottleneck_size=32, nb_filters=32, clf="binary", verbose=1):
+        super(inception_time_model, self).__init__()
+        self.name = name
+        self.n_classes = n_classes
+        self.sampling_frequency = sampling_frequency
+        self.outputfolder = outputfolder
+        self.input_shape = input_shape
+        self.epoch = epoch 
+        self.batch_size = batch_size
+        self.lr_red = lr_red
+        if loss == "bce":
+            self.loss = tf.keras.losses.BinaryCrossentropy()
+        elif loss == "wbce":
+            self.loss = tfa.losses.SigmoidFocalCrossEntropy() #focal instead of weighted bce
+        self.verbose = verbose
+        self.model = build_model((self.sampling_frequency*10,12),self.n_classes,lr_init = lr_init, depth=model_depth, kernel_size=kernel_size, bottleneck_size=bottleneck_size, nb_filters=nb_filters,clf=clf, loss = self.loss)
+
+
+    def fit(self, X_train, y_train, X_val, y_val):
+        if self.lr_red == "no":
+            self.model.fit(X_train, y_train, epochs=self.epoch, batch_size=self.batch_size, 
+            validation_data=(X_val, y_val), verbose=self.verbose)
+        elif self.lr_red == "yes":
+            self.model.fit(X_train, y_train, epochs=self.epoch, batch_size=self.batch_size, 
+            validation_data=(X_val, y_val), verbose=self.verbose,
+            callbacks = [tf.keras.callbacks.LearningRateScheduler(scheduler, verbose=0)])
+        else:
+            print("Error: wrong lr_red argument")
+    def predict(self, X):
+        return self.model.predict(X)
+
+
+def _inception_module(input_tensor, stride=1, activation='linear', use_bottleneck=True, kernel_size=40, bottleneck_size=32, nb_filters=32):
+
+    if use_bottleneck and int(input_tensor.shape[-1]) > 1:
+        input_inception = tf.keras.layers.Conv1D(filters=bottleneck_size, kernel_size=1,
+                                              padding='same', activation=activation, use_bias=False)(input_tensor)
+    else:
+        input_inception = input_tensor
+
+    # kernel_size_s = [3, 5, 8, 11, 17]
+    kernel_size_s = [kernel_size // (2 ** i) for i in range(3)]
+
+    conv_list = []
+
+    for i in range(len(kernel_size_s)):
+        conv_list.append(tf.keras.layers.Conv1D(filters=nb_filters, kernel_size=kernel_size_s[i],
+                                              strides=stride, padding='same', activation=activation, use_bias=False)(
+            input_inception))
+
+    max_pool_1 = tf.keras.layers.MaxPool1D(pool_size=3, strides=stride, padding='same')(input_tensor)
+
+    conv_6 = tf.keras.layers.Conv1D(filters=nb_filters, kernel_size=1,
+                                  padding='same', activation=activation, use_bias=False)(max_pool_1)
+
+    conv_list.append(conv_6)
+
+    x = tf.keras.layers.Concatenate(axis=2)(conv_list)
+    x = tf.keras.layers.BatchNormalization()(x)
+    x = tf.keras.layers.Activation(activation='relu')(x)
+    return x
+
+def _shortcut_layer(input_tensor, out_tensor):
+    shortcut_y = tf.keras.layers.Conv1D(filters=int(out_tensor.shape[-1]), kernel_size=1,
+                                      padding='same', use_bias=False)(input_tensor)
+    shortcut_y = tf.keras.layers.BatchNormalization()(shortcut_y)
+
+    x = tf.keras.layers.Add()([shortcut_y, out_tensor])
+    x = tf.keras.layers.Activation('relu')(x)
+    return x
+
+def build_model(input_shape, nb_classes, depth=6, use_residual=True, lr_init = 0.001, kernel_size=40, bottleneck_size=32, nb_filters=32, clf="binary", loss= tf.keras.losses.BinaryCrossentropy()):
+    input_layer = tf.keras.layers.Input(input_shape)
+
+    x = input_layer
+    input_res = input_layer
+
+    for d in range(depth):
+
+        x = _inception_module(x,kernel_size = kernel_size, bottleneck_size=bottleneck_size, nb_filters=nb_filters)
+
+        if use_residual and d % 3 == 2:
+            x = _shortcut_layer(input_res, x)
+            input_res = x
+
+    gap_layer = tf.keras.layers.GlobalAveragePooling1D()(x)
+
+    output_layer = tf.keras.layers.Dense(units=nb_classes,activation='sigmoid')(gap_layer)  
+    model = tf.keras.models.Model(inputs=input_layer, outputs=output_layer)
+    model.compile(loss=loss, optimizer=tf.keras.optimizers.Adam(learning_rate=lr_init), 
+                  metrics=[tf.keras.metrics.BinaryAccuracy(),
+                           tf.keras.metrics.AUC(
+                        num_thresholds=200,
+                        curve='ROC',
+                        summation_method='interpolation',
+                        name="ROC",
+                        multi_label=True,
+                        ),
+                       tf.keras.metrics.AUC(
+                        num_thresholds=200,
+                        curve='PR',
+                        summation_method='interpolation',
+                        name="PRC",
+                        multi_label=True,
+                        )
+              ])
+    print("Inception model built.")
+    return model
+
+def scheduler(epoch, lr):
+    if epoch % 5 == 0:
+        return lr*0.1
+    else:
+        return lr

code/reproduce_inception_time_results.py

@@ -0,0 +1,69 @@
+from experiments.scp_experiment import SCP_Experiment
+from utils import utils
+# model configs
+from configs.fastai_configs import *
+from configs.wavelet_configs import *
+from code.configs.tf_inception_time_config import *
+import sys
+
+def main(datafolder, datafolder_icbeb, outputfolder):
+
+
+    models = [
+        #conf_fastai_xresnet1d101,
+        #conf_fastai_resnet1d_wang,
+        #conf_fastai_lstm,
+        #conf_fastai_lstm_bidir,
+        #conf_fastai_fcn_wang,
+        #conf_fastai_inception1d,
+        #conf_wavelet_standard_nn,
+        conf_tf_inception,
+        conf_tf_inception_all,
+        conf_tf_inception_diagnostic,
+        conf_tf_inception_form,
+        conf_tf_inception_rhythm,
+        conf_tf_inception_subdiagnostic,
+        conf_tf_inception_superdiagnostic
+        ]
+
+    ##########################################
+    # STANDARD SCP EXPERIMENTS ON PTBXL
+    ##########################################
+
+    experiments = [
+        ('exp0', 'all'),
+        ('exp1', 'diagnostic'),
+        ('exp1.1', 'subdiagnostic'),
+        ('exp1.1.1', 'superdiagnostic'),
+        ('exp2', 'form'),
+        ('exp3', 'rhythm')
+       ]
+
+    for name, task in experiments:
+        print(name)
+        print(task)
+        e = SCP_Experiment(name, task, datafolder, outputfolder, models)
+        e.prepare()
+        e.perform()
+        e.evaluate(n_bootstraping_samples=100, bootstrap_eval=True, dumped_bootstraps=False)
+
+    # generate greate summary table
+    utils.generate_ptbxl_summary_table(folder = outputfolder)
+
+    ##########################################
+    # EXPERIMENT BASED ICBEB DATA
+    ##########################################
+
+    e = SCP_Experiment('exp_ICBEB', 'all', datafolder_icbeb, outputfolder, models)
+    e.prepare()
+    e.perform()
+    e.evaluate()
+
+    # generate greate summary table
+    utils.ICBEBE_table(folder=outputfolder)
+
+if __name__ == "__main__":
+    if not (len(sys.argv) == 4):
+        raise Exception('Include the data and model folders as arguments, e.g., python reproduce_results.py ./path/to/data/ ./path/to/icbeb/ ./path/to/output/')
+    else:
+        main(datafolder = sys.argv[1], datafolder_icbeb = sys.argv[2], outputfolder = sys.argv[3])

code/utils/utils.py

@@ -114,7 +114,7 @@ def apply_thresholds(preds, thresholds):
 # DATA PROCESSING STUFF
 
 def load_dataset(path, sampling_rate, release=False):
-    if path.split('/')[-2] == 'ptbxl':
+    if path.split('/')[-2] == 'ptb-xl-a-large-publicly-available-electrocardiography-dataset-1.0.1':
         # load and convert annotation data
         Y = pd.read_csv(path+'ptbxl_database.csv', index_col='ecg_id')
         Y.scp_codes = Y.scp_codes.apply(lambda x: ast.literal_eval(x))
@@ -335,7 +335,7 @@ def apply_standardizer(X, ss):
 
 # DOCUMENTATION STUFF
 
-def generate_ptbxl_summary_table(selection=None, folder='../output/'):
+def generate_ptbxl_summary_table(selection=None, folder: str="./your/path/to/ptbxl/"):
 
     exps = ['exp0', 'exp1', 'exp1.1', 'exp1.1.1', 'exp2', 'exp3']
     metric1 = 'macro_auc'
@@ -405,7 +405,7 @@ def generate_ptbxl_summary_table(selection=None, folder='../output/'):
             md_source += '| ' + row[0].replace('fastai_', '') + ' | ' + row[1] + ' | [our work]('+our_work+') | [this repo]('+our_repo+')| \n'
     print(md_source)
 
-def ICBEBE_table(selection=None, folder='../output/'):
+def ICBEBE_table(selection=None, folder:str="./your/path/to/icbeb/"):
     cols = ['macro_auc', 'F_beta_macro', 'G_beta_macro']
 
     if selection is None: