Add files via upload

main.py

@@ -0,0 +1,21 @@
+import numpy as np
+from utils.generate_csv import generate_csv 
+from utils.k_fold_splits import k_fold_splits 
+from utils.k_fold_separate import k_fold_separate 
+
+df = generate_csv()
+
+y = np.array([i for i in df["class"]])
+x = np.array([i for i in df["path"]]) 
+
+files_for_train_x = []
+files_for_validation_x = [] 
+files_for_train_y = []
+files_for_validation_y = []
+
+k_fold_splits(x,y, files_for_train_x ,  files_for_validation_x , files_for_train_y , files_for_validation_y ) # n_splits = 5 
+
+n = len(files_for_train_x)
+for i in range(0,n):
+
+     k_fold_separate(files_for_train_x[i] , files_for_train_y[i] , files_for_validation_x[i] ,files_for_validation_y[i] , "InceptionV3" , "MobileNetV2" ,"InceptionResNetV2" ,i+1 , NUM_EPOCHS = 1) 

requirements.txt

@@ -1,9 +1,173 @@
-numpy==1.21.0
-pandas==1.1.5
-matplotlib==3.2.2
-seaborn==2.5.0
-opencv-python==4.2.0.32
-tensorflow==2.5.3
-sklearn==0.24.1
-skimage==0.19.0
-scipy==1.7.1
+# This file may be used to create an environment using:
+# $ conda create --name <env> --file <this file>
+# platform: win-64
+_tflow_select=2.3.0=eigen
+absl-py=0.15.0=pyhd3eb1b0_0
+aiohttp=3.8.1=py38h2bbff1b_0
+aiosignal=1.2.0=pyhd3eb1b0_0
+argon2-cffi=20.1.0=py38h2bbff1b_1
+astor=0.8.1=py38haa95532_0
+astunparse=1.6.3=py_0
+async-timeout=4.0.1=pyhd3eb1b0_0
+async_generator=1.10=pyhd3eb1b0_0
+attrs=21.4.0=pyhd3eb1b0_0
+backcall=0.2.0=pyhd3eb1b0_0
+blas=1.0=mkl
+bleach=4.1.0=pyhd3eb1b0_0
+blinker=1.4=py38haa95532_0
+bottleneck=1.3.2=py38h2a96729_1
+brotli=1.0.9=ha925a31_2
+brotlipy=0.7.0=py38h2bbff1b_1003
+ca-certificates=2020.10.14=0
+cachetools=4.2.2=pyhd3eb1b0_0
+certifi=2020.6.20=py38_0
+cffi=1.15.0=py38h2bbff1b_1
+charset-normalizer=2.0.4=pyhd3eb1b0_0
+click=8.0.3=pyhd3eb1b0_0
+colorama=0.4.4=pyhd3eb1b0_0
+cryptography=3.4.8=py38h71e12ea_0
+cycler=0.11.0=pyhd3eb1b0_0
+dataclasses=0.8=pyh6d0b6a4_7
+debugpy=1.5.1=py38hd77b12b_0
+decorator=5.1.1=pyhd3eb1b0_0
+defusedxml=0.7.1=pyhd3eb1b0_0
+entrypoints=0.3=py38_0
+fonttools=4.25.0=pyhd3eb1b0_0
+freetype=2.10.4=hd328e21_0
+frozenlist=1.2.0=py38h2bbff1b_0
+gast=0.4.0=pyhd3eb1b0_0
+google-auth=1.33.0=pyhd3eb1b0_0
+google-auth-oauthlib=0.4.4=pyhd3eb1b0_0
+google-pasta=0.2.0=pyhd3eb1b0_0
+grpcio=1.42.0=py38hc60d5dd_0
+h5py=2.10.0=py38h5e291fa_0
+hdf5=1.10.4=h7ebc959_0
+icc_rt=2019.0.0=h0cc432a_1
+icu=58.2=ha925a31_3
+idna=3.3=pyhd3eb1b0_0
+importlib-metadata=4.8.2=py38haa95532_0
+importlib_metadata=4.8.2=hd3eb1b0_0
+intel-openmp=2021.4.0=haa95532_3556
+ipykernel=6.4.1=py38haa95532_1
+ipython=7.31.1=py38haa95532_0
+ipython_genutils=0.2.0=pyhd3eb1b0_1
+ipywidgets=7.6.5=pyhd3eb1b0_1
+jedi=0.18.1=py38haa95532_1
+jinja2=3.0.2=pyhd3eb1b0_0
+joblib=1.1.0=pyhd3eb1b0_0
+jpeg=9d=h2bbff1b_0
+jsonschema=3.2.0=pyhd3eb1b0_2
+jupyter=1.0.0=py38_7
+jupyter_client=7.1.2=pyhd3eb1b0_0
+jupyter_console=6.4.0=pyhd3eb1b0_0
+jupyter_core=4.9.1=py38haa95532_0
+jupyterlab_pygments=0.1.2=py_0
+jupyterlab_widgets=1.0.0=pyhd3eb1b0_1
+keras=2.4.3=pyhd8ed1ab_0
+keras-applications=1.0.8=py_1
+keras-preprocessing=1.1.2=pyhd3eb1b0_0
+kiwisolver=1.3.1=py38hd77b12b_0
+libpng=1.6.37=h2a8f88b_0
+libprotobuf=3.19.1=h23ce68f_0
+libtiff=4.2.0=hd0e1b90_0
+libwebp=1.2.0=h2bbff1b_0
+lz4-c=1.9.3=h2bbff1b_1
+markdown=3.3.4=py38haa95532_0
+markupsafe=2.0.1=py38h2bbff1b_0
+matplotlib=3.5.0=py38haa95532_0
+matplotlib-base=3.5.0=py38h6214cd6_0
+matplotlib-inline=0.1.2=pyhd3eb1b0_2
+mistune=0.8.4=py38he774522_1000
+mkl=2021.4.0=haa95532_640
+mkl-service=2.4.0=py38h2bbff1b_0
+mkl_fft=1.3.1=py38h277e83a_0
+mkl_random=1.2.2=py38hf11a4ad_0
+multidict=5.1.0=py38h2bbff1b_2
+munkres=1.1.4=py_0
+nbclient=0.5.3=pyhd3eb1b0_0
+nbconvert=6.1.0=py38haa95532_0
+nbformat=5.1.3=pyhd3eb1b0_0
+nest-asyncio=1.5.1=pyhd3eb1b0_0
+notebook=6.4.6=py38haa95532_0
+numexpr=2.8.1=py38hb80d3ca_0
+numpy=1.21.5=py38ha4e8547_0
+numpy-base=1.21.5=py38hc2deb75_0
+oauthlib=3.1.1=pyhd3eb1b0_0
+olefile=0.46=pyhd3eb1b0_0
+openssl=1.1.1m=h2bbff1b_0
+opt_einsum=3.3.0=pyhd3eb1b0_1
+packaging=21.3=pyhd3eb1b0_0
+pandas=1.3.5=py38h6214cd6_0
+pandocfilters=1.5.0=pyhd3eb1b0_0
+parso=0.8.3=pyhd3eb1b0_0
+pickleshare=0.7.5=pyhd3eb1b0_1003
+pillow=8.4.0=py38hd45dc43_0
+pip=21.2.2=py38haa95532_0
+prometheus_client=0.13.1=pyhd3eb1b0_0
+prompt-toolkit=3.0.20=pyhd3eb1b0_0
+prompt_toolkit=3.0.20=hd3eb1b0_0
+protobuf=3.19.1=py38hd77b12b_0
+pyasn1=0.4.8=pyhd3eb1b0_0
+pyasn1-modules=0.2.8=py_0
+pycparser=2.21=pyhd3eb1b0_0
+pygments=2.11.2=pyhd3eb1b0_0
+pyjwt=2.1.0=py38haa95532_0
+pyopenssl=21.0.0=pyhd3eb1b0_1
+pyparsing=3.0.4=pyhd3eb1b0_0
+pyqt=5.9.2=py38hd77b12b_6
+pyreadline=2.1=py38_1
+pyrsistent=0.18.0=py38h196d8e1_0
+pysocks=1.7.1=py38haa95532_0
+python=3.8.12=h6244533_0
+python-dateutil=2.8.2=pyhd3eb1b0_0
+python_abi=3.8=2_cp38
+pytz=2021.3=pyhd3eb1b0_0
+pywin32=302=py38h827c3e9_1
+pywinpty=2.0.2=py38h5da7b33_0
+pyyaml=6.0=py38h294d835_3
+pyzmq=22.3.0=py38hd77b12b_2
+qt=5.9.7=vc14h73c81de_0
+qtconsole=5.2.2=pyhd3eb1b0_0
+qtpy=1.11.2=pyhd3eb1b0_0
+requests=2.27.1=pyhd3eb1b0_0
+requests-oauthlib=1.3.0=py_0
+rsa=4.7.2=pyhd3eb1b0_1
+scikit-learn=1.0.2=py38hf11a4ad_1
+scipy=1.7.3=py38h0a974cb_0
+send2trash=1.8.0=pyhd3eb1b0_1
+setuptools=58.0.4=py38haa95532_0
+sip=4.19.13=py38hd77b12b_0
+six=1.16.0=pyhd3eb1b0_0
+sqlite=3.37.2=h2bbff1b_0
+tensorboard=2.4.0=pyhc547734_0
+tensorboard-plugin-wit=1.6.0=py_0
+tensorflow=2.3.0=mkl_py38h8c0d9a2_0
+tensorflow-base=2.3.0=eigen_py38h75a453f_0
+tensorflow-estimator=2.6.0=pyh7b7c402_0
+termcolor=1.1.0=py38haa95532_1
+terminado=0.9.4=py38haa95532_0
+testpath=0.5.0=pyhd3eb1b0_0
+threadpoolctl=2.2.0=pyh0d69192_0
+tk=8.6.11=h2bbff1b_0
+tornado=6.1=py38h2bbff1b_0
+traitlets=5.1.1=pyhd3eb1b0_0
+typing-extensions=3.10.0.2=hd3eb1b0_0
+typing_extensions=3.10.0.2=pyh06a4308_0
+urllib3=1.26.8=pyhd3eb1b0_0
+vc=14.2=h21ff451_1
+vs2015_runtime=14.27.29016=h5e58377_2
+wcwidth=0.2.5=pyhd3eb1b0_0
+webencodings=0.5.1=py38_1
+werkzeug=2.0.2=pyhd3eb1b0_0
+wheel=0.37.1=pyhd3eb1b0_0
+widgetsnbextension=3.5.1=py38_0
+win_inet_pton=1.1.0=py38haa95532_0
+wincertstore=0.2=py38haa95532_2
+winpty=0.4.3=4
+wrapt=1.13.3=py38h2bbff1b_2
+xz=5.2.5=h62dcd97_0
+yaml=0.2.5=h8ffe710_2
+yarl=1.6.3=py38h2bbff1b_0
+zipp=3.7.0=pyhd3eb1b0_0
+zlib=1.2.11=h8cc25b3_4
+zstd=1.4.9=h19a0ad4_0

utils/create_model.py

@@ -0,0 +1,36 @@
+import tensorflow as tf
+from tensorflow.keras.models import Model
+
+def create_model(model_name,IMG_SIZE = 256):
+
+
+    IMG_SHAPE = (IMG_SIZE, IMG_SIZE, 3)  # IMG_SIZE = 256
+
+    MobileNetV2_model = tf.keras.applications.MobileNetV2(input_shape=IMG_SHAPE,
+                                                        include_top=False,
+                                                        weights='imagenet')
+
+    InceptionV3_model = tf.keras.applications.inception_v3.InceptionV3(input_shape=IMG_SHAPE,
+                                                                     include_top=False,
+                                                                     weights='imagenet')
+
+    InceptionResNetV2_model = tf.keras.applications.inception_resnet_v2.InceptionResNetV2(input_shape=IMG_SHAPE,
+                                                                                        include_top=False,
+                                                                                        weights='imagenet')
+
+    models = {"MobileNetV2": MobileNetV2_model, "InceptionV3": InceptionV3_model,
+              "InceptionResNetV2": InceptionResNetV2_model}
+
+
+    model = models[model_name]
+
+    x = tf.keras.layers.Conv2D(128, (3, 3), activation='relu')(model.output)
+    x = tf.keras.layers.MaxPooling2D(pool_size=(2, 2))(x)
+    x = tf.keras.layers.Flatten()(x)
+    x = tf.keras.layers.Dense(100, activation='relu')(x)
+    x = tf.keras.layers.Dense(5, activation='softmax')(x)
+
+    model = Model(inputs=model.input, outputs=x)
+
+    my_model = tf.keras.models.clone_model(model)
+    return my_model

utils/fuzzy_dist_ensemble.py

@@ -0,0 +1,18 @@
+import numpy as np
+import scipy
+def fuzzy_dist(classifier1, classifier2, classifier3, verbose=False):
+    out = np.empty(len(classifier1))
+    for i in range(len(classifier1)):
+        if np.argmax(classifier1[i]) == np.argmax(classifier2[i]) == np.argmax(classifier3[i]):
+            out[i] = np.argmax(classifier2[i])
+        else:
+            measure = np.zeros(len(classifier1[i]))
+            for j in range(len(classifier1[i])):
+                scores = np.array(
+                    [classifier1[i, j], classifier2[i, j], classifier3[i, j]])
+                measure[j] = scipy.spatial.distance.cosine(np.ones(3), scores)*scipy.spatial.distance.euclidean(
+                    np.ones(3), scores)*scipy.spatial.distance.cityblock(np.ones(3), scores)
+                if verbose:
+                    print(measure)
+            out[i] = np.argmin(measure)
+    return out

utils/generate_csv.py

@@ -0,0 +1,48 @@
+from msilib.schema import Directory
+import os 
+import pandas as pd 
+from sklearn.utils import shuffle
+
+def generate_csv():
+    print("runing")
+    uniques = ["Dyskeratotic" , "Koilocytotic" , "Metaplastic" , "Parabasal" , "SuperficialIntermediate"]
+    dirs = ["train" , "test"]
+
+    """
+            +-- train
+            |   +-- Dyskeratotic
+            |   +-- Koilocytotic
+            |   +-- Metaplastic
+            |   +-- Parabasal
+            |   +-- SuperficialIntermediate
+
+            +-- test
+            |   +-- Dyskeratotic
+            |   +-- Koilocytotic
+            |   +-- Metaplastic
+            |   +-- Parabasal
+            |   +-- SuperficialIntermediate
+
+    
+    """
+
+    data = []
+    for dir in dirs :
+        for unique in uniques:
+            directory = "data/SiPakMed/" + dir + "/" + unique
+
+            for filename in os.listdir(directory):
+
+                path = directory + "/" + filename 
+                data.append([ filename , path  , unique])
+
+    df = pd.DataFrame(data, columns = ["filename" ,"path", "class"]) 
+    df = shuffle(df)
+    name = "csv_files/" + "Data-full"
+    df.to_csv(name, index = False )
+    print("runing")
+
+    return df    
+
+
+