Merge pull request #26 from enricobu96/feature/add-metrics

Feature/add metrics
enricobu96 · Jul 5, 2021 · 7a156b2 · 7a156b2
2 parents d394aca + adf546c
commit 7a156b2
Show file tree

Hide file tree

Showing 51 changed files with 310 additions and 20 deletions.
diff --git a/extra/test_reports/Final/APE mediapipe/Confusion matrices/conf_matrix_lr.png b/extra/test_reports/Final/APE mediapipe/Confusion matrices/conf_matrix_lr.png
diff --git a/extra/test_reports/Final/APE mediapipe/Confusion matrices/conf_matrix_mlp.png b/extra/test_reports/Final/APE mediapipe/Confusion matrices/conf_matrix_mlp.png
diff --git a/extra/test_reports/Final/APE mediapipe/Confusion matrices/conf_matrix_rc.png b/extra/test_reports/Final/APE mediapipe/Confusion matrices/conf_matrix_rc.png
diff --git a/extra/test_reports/Final/APE mediapipe/Confusion matrices/conf_matrix_rf.png b/extra/test_reports/Final/APE mediapipe/Confusion matrices/conf_matrix_rf.png
diff --git a/extra/test_reports/Final/APE mediapipe/Confusion matrices/conf_matrix_svm.png b/extra/test_reports/Final/APE mediapipe/Confusion matrices/conf_matrix_svm.png
diff --git a/extra/test_reports/Final/APE mediapipe/Learning curves/learning_lr.png b/extra/test_reports/Final/APE mediapipe/Learning curves/learning_lr.png
diff --git a/extra/test_reports/Final/APE mediapipe/Learning curves/learning_mlp.png b/extra/test_reports/Final/APE mediapipe/Learning curves/learning_mlp.png
diff --git a/extra/test_reports/Final/APE mediapipe/Learning curves/learning_rc.png b/extra/test_reports/Final/APE mediapipe/Learning curves/learning_rc.png
diff --git a/extra/test_reports/Final/APE mediapipe/Learning curves/learning_rf.png b/extra/test_reports/Final/APE mediapipe/Learning curves/learning_rf.png
diff --git a/extra/test_reports/Final/APE mediapipe/Learning curves/learning_svm.png b/extra/test_reports/Final/APE mediapipe/Learning curves/learning_svm.png
diff --git a/extra/test_reports/Final/APE mediapipe/test_reports.txt b/extra/test_reports/Final/APE mediapipe/test_reports.txt
@@ -0,0 +1,80 @@
+lr
+
+              precision    recall  f1-score   support
+
+        balc       0.59      0.67      0.63       873
+        bend       0.52      0.65      0.58      1236
+        boxx       0.93      0.79      0.85      1059
+        clap       0.77      0.67      0.72       921
+        marc       0.88      0.64      0.74       799
+        onew       0.75      0.88      0.81       918
+        wave       0.64      0.59      0.62      1160
+
+    accuracy                           0.70      6966
+   macro avg       0.72      0.70      0.71      6966
+weighted avg       0.72      0.70      0.70      6966
+
+rc
+
+              precision    recall  f1-score   support
+
+        balc       0.64      0.64      0.64       873
+        bend       0.51      0.68      0.58      1236
+        boxx       0.89      0.81      0.85      1059
+        clap       0.79      0.66      0.72       921
+        marc       0.85      0.59      0.70       799
+        onew       0.78      0.92      0.85       918
+        wave       0.66      0.60      0.63      1160
+
+    accuracy                           0.70      6966
+   macro avg       0.73      0.70      0.71      6966
+weighted avg       0.72      0.70      0.70      6966
+
+rf
+
+              precision    recall  f1-score   support
+
+        balc       0.76      0.79      0.78       873
+        bend       0.86      0.71      0.78      1236
+        boxx       1.00      0.89      0.94      1059
+        clap       0.76      0.94      0.84       921
+        marc       0.93      0.69      0.79       799
+        onew       0.78      0.99      0.87       918
+        wave       0.77      0.79      0.78      1160
+
+    accuracy                           0.83      6966
+   macro avg       0.84      0.83      0.83      6966
+weighted avg       0.84      0.83      0.82      6966
+
+svm
+
+              precision    recall  f1-score   support
+
+        balc       0.79      0.71      0.75       873
+        bend       0.86      0.77      0.82      1236
+        boxx       0.97      1.00      0.98      1059
+        clap       0.92      0.97      0.95       921
+        marc       0.78      0.81      0.80       799
+        onew       0.98      1.00      0.99       918
+        wave       0.77      0.82      0.79      1160
+
+    accuracy                           0.87      6966
+   macro avg       0.87      0.87      0.87      6966
+weighted avg       0.87      0.87      0.87      6966
+
+mlp
+
+              precision    recall  f1-score   support
+
+        balc       0.50      0.66      0.57       873
+        bend       0.61      0.58      0.59      1236
+        boxx       0.95      0.84      0.89      1059
+        clap       0.79      0.71      0.74       921
+        marc       0.78      0.68      0.73       799
+        onew       0.74      0.90      0.81       918
+        wave       0.68      0.62      0.65      1160
+
+    accuracy                           0.71      6966
+   macro avg       0.72      0.71      0.71      6966
+weighted avg       0.72      0.71      0.71      6966
+
diff --git a/...st_reports/Final/Custom dataset mediapipe/Confusion matrices/conf_matrix_lr.png b/...st_reports/Final/Custom dataset mediapipe/Confusion matrices/conf_matrix_lr.png
diff --git a/...t_reports/Final/Custom dataset mediapipe/Confusion matrices/conf_matrix_mlp.png b/...t_reports/Final/Custom dataset mediapipe/Confusion matrices/conf_matrix_mlp.png
diff --git a/...st_reports/Final/Custom dataset mediapipe/Confusion matrices/conf_matrix_rc.png b/...st_reports/Final/Custom dataset mediapipe/Confusion matrices/conf_matrix_rc.png
diff --git a/...st_reports/Final/Custom dataset mediapipe/Confusion matrices/conf_matrix_rf.png b/...st_reports/Final/Custom dataset mediapipe/Confusion matrices/conf_matrix_rf.png
diff --git a/...t_reports/Final/Custom dataset mediapipe/Confusion matrices/conf_matrix_svm.png b/...t_reports/Final/Custom dataset mediapipe/Confusion matrices/conf_matrix_svm.png
diff --git a/extra/test_reports/Final/Custom dataset mediapipe/Learning curves/learning_lr.png b/extra/test_reports/Final/Custom dataset mediapipe/Learning curves/learning_lr.png
diff --git a/extra/test_reports/Final/Custom dataset mediapipe/Learning curves/learning_mlp.png b/extra/test_reports/Final/Custom dataset mediapipe/Learning curves/learning_mlp.png
diff --git a/extra/test_reports/Final/Custom dataset mediapipe/Learning curves/learning_rc.png b/extra/test_reports/Final/Custom dataset mediapipe/Learning curves/learning_rc.png
diff --git a/extra/test_reports/Final/Custom dataset mediapipe/Learning curves/learning_rf.png b/extra/test_reports/Final/Custom dataset mediapipe/Learning curves/learning_rf.png
diff --git a/extra/test_reports/Final/Custom dataset mediapipe/Learning curves/learning_svm.png b/extra/test_reports/Final/Custom dataset mediapipe/Learning curves/learning_svm.png
diff --git a/extra/test_reports/Final/Custom dataset mediapipe/test_reports.txt b/extra/test_reports/Final/Custom dataset mediapipe/test_reports.txt
@@ -0,0 +1,80 @@
+lr
+
+              precision    recall  f1-score   support
+
+         dab       1.00      0.98      0.99      1351
+   l_arm_mid       0.84      0.90      0.87      1314
+   l_arm_top       0.97      0.93      0.95      1293
+   r_arm_mid       0.97      0.98      0.97      1385
+   r_arm_top       0.97      0.99      0.98      1315
+      steady       0.89      0.88      0.89      1327
+       tpose       0.99      0.95      0.97      1282
+
+    accuracy                           0.95      9267
+   macro avg       0.95      0.95      0.95      9267
+weighted avg       0.95      0.95      0.95      9267
+
+rc
+
+              precision    recall  f1-score   support
+
+         dab       0.99      0.97      0.98      1351
+   l_arm_mid       0.78      0.92      0.84      1314
+   l_arm_top       0.96      0.93      0.95      1293
+   r_arm_mid       0.96      0.99      0.98      1385
+   r_arm_top       0.97      0.99      0.98      1315
+      steady       0.92      0.79      0.85      1327
+       tpose       0.99      0.95      0.97      1282
+
+    accuracy                           0.94      9267
+   macro avg       0.94      0.94      0.94      9267
+weighted avg       0.94      0.94      0.94      9267
+
+rf
+
+              precision    recall  f1-score   support
+
+         dab       0.97      0.88      0.92      1351
+   l_arm_mid       0.95      0.78      0.86      1314
+   l_arm_top       0.81      0.96      0.88      1293
+   r_arm_mid       0.99      0.94      0.96      1385
+   r_arm_top       0.93      0.99      0.96      1315
+      steady       0.99      1.00      0.99      1327
+       tpose       0.90      0.97      0.94      1282
+
+    accuracy                           0.93      9267
+   macro avg       0.94      0.93      0.93      9267
+weighted avg       0.94      0.93      0.93      9267
+
+svm
+
+              precision    recall  f1-score   support
+
+         dab       0.91      0.78      0.84      1351
+   l_arm_mid       0.72      0.75      0.74      1314
+   l_arm_top       0.96      0.95      0.96      1293
+   r_arm_mid       0.89      0.99      0.94      1385
+   r_arm_top       0.97      0.98      0.98      1315
+      steady       0.84      0.69      0.76      1327
+       tpose       0.68      0.79      0.73      1282
+
+    accuracy                           0.85      9267
+   macro avg       0.85      0.85      0.85      9267
+weighted avg       0.85      0.85      0.85      9267
+
+mlp
+
+              precision    recall  f1-score   support
+
+         dab       0.99      0.98      0.99      1351
+   l_arm_mid       0.95      0.95      0.95      1314
+   l_arm_top       0.97      0.98      0.98      1293
+   r_arm_mid       0.99      0.98      0.99      1385
+   r_arm_top       1.00      1.00      1.00      1315
+      steady       0.94      0.97      0.95      1327
+       tpose       0.98      0.95      0.97      1282
+
+    accuracy                           0.97      9267
+   macro avg       0.97      0.97      0.97      9267
+weighted avg       0.97      0.97      0.97      9267
+
diff --git a/extra/test_reports/Final/Custom dataset nite/Confusion matrices/conf_matrix_lr.png b/extra/test_reports/Final/Custom dataset nite/Confusion matrices/conf_matrix_lr.png
diff --git a/...a/test_reports/Final/Custom dataset nite/Confusion matrices/conf_matrix_mlp.png b/...a/test_reports/Final/Custom dataset nite/Confusion matrices/conf_matrix_mlp.png
diff --git a/extra/test_reports/Final/Custom dataset nite/Confusion matrices/conf_matrix_rc.png b/extra/test_reports/Final/Custom dataset nite/Confusion matrices/conf_matrix_rc.png
diff --git a/extra/test_reports/Final/Custom dataset nite/Confusion matrices/conf_matrix_rf.png b/extra/test_reports/Final/Custom dataset nite/Confusion matrices/conf_matrix_rf.png
diff --git a/...a/test_reports/Final/Custom dataset nite/Confusion matrices/conf_matrix_svm.png b/...a/test_reports/Final/Custom dataset nite/Confusion matrices/conf_matrix_svm.png
diff --git a/extra/test_reports/Final/Custom dataset nite/Learning curves/learning_lr.png b/extra/test_reports/Final/Custom dataset nite/Learning curves/learning_lr.png
diff --git a/extra/test_reports/Final/Custom dataset nite/Learning curves/learning_mlp.png b/extra/test_reports/Final/Custom dataset nite/Learning curves/learning_mlp.png
diff --git a/extra/test_reports/Final/Custom dataset nite/Learning curves/learning_rc.png b/extra/test_reports/Final/Custom dataset nite/Learning curves/learning_rc.png
diff --git a/extra/test_reports/Final/Custom dataset nite/Learning curves/learning_rf.png b/extra/test_reports/Final/Custom dataset nite/Learning curves/learning_rf.png
diff --git a/extra/test_reports/Final/Custom dataset nite/Learning curves/learning_svm.png b/extra/test_reports/Final/Custom dataset nite/Learning curves/learning_svm.png
diff --git a/extra/test_reports/Final/Custom dataset nite/test_reports.txt b/extra/test_reports/Final/Custom dataset nite/test_reports.txt
@@ -0,0 +1,80 @@
+lr
+
+              precision    recall  f1-score   support
+
+         dab       0.88      0.85      0.86       500
+   l_arm_mid       1.00      1.00      1.00       438
+   l_arm_top       0.90      0.66      0.76       326
+   r_arm_mid       0.85      0.98      0.91       373
+   r_arm_top       0.99      0.91      0.95       338
+      steady       0.83      0.94      0.88       622
+       tpose       1.00      0.99      0.99       385
+
+    accuracy                           0.91      2982
+   macro avg       0.92      0.90      0.91      2982
+weighted avg       0.91      0.91      0.91      2982
+
+rc
+
+              precision    recall  f1-score   support
+
+         dab       0.90      0.55      0.68       500
+   l_arm_mid       0.98      0.97      0.98       438
+   l_arm_top       0.90      0.22      0.36       326
+   r_arm_mid       0.79      0.97      0.87       373
+   r_arm_top       0.73      0.96      0.83       338
+      steady       0.69      0.97      0.81       622
+       tpose       0.99      0.99      0.99       385
+
+    accuracy                           0.82      2982
+   macro avg       0.86      0.81      0.79      2982
+weighted avg       0.85      0.82      0.80      2982
+
+rf
+
+              precision    recall  f1-score   support
+
+         dab       0.94      1.00      0.97       500
+   l_arm_mid       0.98      0.99      0.99       438
+   l_arm_top       1.00      0.92      0.96       326
+   r_arm_mid       1.00      0.98      0.99       373
+   r_arm_top       1.00      1.00      1.00       338
+      steady       0.95      1.00      0.97       622
+       tpose       1.00      0.92      0.96       385
+
+    accuracy                           0.98      2982
+   macro avg       0.98      0.97      0.98      2982
+weighted avg       0.98      0.98      0.98      2982
+
+svm
+
+              precision    recall  f1-score   support
+
+         dab       0.83      0.42      0.56       500
+   l_arm_mid       1.00      0.98      0.99       438
+   l_arm_top       0.43      0.36      0.39       326
+   r_arm_mid       0.71      0.96      0.81       373
+   r_arm_top       0.79      0.99      0.88       338
+      steady       0.69      0.73      0.71       622
+       tpose       0.87      0.98      0.92       385
+
+    accuracy                           0.76      2982
+   macro avg       0.76      0.77      0.75      2982
+weighted avg       0.77      0.76      0.75      2982
+
+mlp
+
+              precision    recall  f1-score   support
+
+         dab       0.91      0.90      0.91       500
+   l_arm_mid       1.00      0.98      0.99       438
+   l_arm_top       0.00      0.00      0.00       326
+   r_arm_mid       0.92      0.91      0.92       373
+   r_arm_top       0.96      0.91      0.93       338
+      steady       0.62      0.98      0.76       622
+       tpose       0.99      0.99      0.99       385
+
+    accuracy                           0.85      2982
+   macro avg       0.77      0.81      0.79      2982
+weighted avg       0.78      0.85      0.81      2982
+
diff --git a/main.py b/main.py
@@ -34,7 +34,7 @@ class Classifications(Enum):
       rf = 3
       gb = 4
       svm = 5
-      cnn = 6
+      mlp = 6
 
 def main(args):
 

diff --git a/model/kinect_depth/prediction_models/prediction_model_lr.pkl b/model/kinect_depth/prediction_models/prediction_model_lr.pkl
diff --git a/model/kinect_depth/prediction_models/prediction_model_mlp.pkl b/model/kinect_depth/prediction_models/prediction_model_mlp.pkl
diff --git a/model/kinect_depth/prediction_models/prediction_model_rc.pkl b/model/kinect_depth/prediction_models/prediction_model_rc.pkl
diff --git a/model/kinect_depth/prediction_models/prediction_model_rf.pkl b/model/kinect_depth/prediction_models/prediction_model_rf.pkl
diff --git a/model/kinect_depth/prediction_models/prediction_model_svm.pkl b/model/kinect_depth/prediction_models/prediction_model_svm.pkl
diff --git a/model/kinect_depth/run_kinect.py b/model/kinect_depth/run_kinect.py
@@ -98,7 +98,7 @@ def run(self, classificationModel: str):
                             gesture_class, gesture_prob = self.__use_gb(model, X, img)
                         elif classificationModel == 'svm':
                             gesture_class, gesture_prob = self.__use_svm(model, X, img)
-                        elif classificationModel == 'cnn':
+                        elif classificationModel == 'mlp':
                             gesture_class, gesture_prob = self.__use_cnn(model, X, img)
 
                         cv2.putText(img, 'CLASS', (95,12), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1, cv2.LINE_AA)

diff --git a/model/mediapipe/acquire_data_webcam.py b/model/mediapipe/acquire_data_webcam.py
@@ -134,7 +134,7 @@ def acquire_data(self, gesture: str):
             while cap.isOpened():
                 ret, image = cap.read()
                 if ret == True:
-                    image_flip, dark_image, bright_image, resized_image = self.__data_augmentation(image)
+                    image_flip, dark_image, bright_image, resized_image = self.__data_augmentation(image, gesture)
                     ifresults, diresults, biresults, riresults= holistic.process(image_flip), holistic.process(dark_image), holistic.process(bright_image), holistic.process(resized_image)
 
                     try:
@@ -173,8 +173,11 @@ def acquire_data(self, gesture: str):
         print('Processing done, kthxbye')
         sys.stdout.flush()
 
-    def __data_augmentation(self, image):
-        flip = cv2.flip(image, 1)
+    def __data_augmentation(self, image, gesture):
+        if gesture in ['l_arm_mid', 'l_arm_top', 'r_arm_mid', 'r_arm_top']:
+            flip = cv2.resize(image, (0,0), fx=uniform(0.9,1), fy=uniform(0.9,1))
+        else:
+            flip = cv2.flip(image, 1)
         dark = cv2.convertScaleAbs(image, alpha=uniform(0.5, 1), beta=0)
         bright = cv2.convertScaleAbs(image, alpha=uniform(1, 1.5))
         resized = cv2.resize(image, (0,0), fx=uniform(0.9,1), fy=uniform(0.9,1))

diff --git a/model/mediapipe/prediction_models/prediction_model_cnn.pkl b/model/mediapipe/prediction_models/prediction_model_cnn.pkl
diff --git a/model/mediapipe/prediction_models/prediction_model_gb.pkl b/model/mediapipe/prediction_models/prediction_model_gb.pkl
diff --git a/model/mediapipe/prediction_models/prediction_model_lr.pkl b/model/mediapipe/prediction_models/prediction_model_lr.pkl
diff --git a/model/mediapipe/prediction_models/prediction_model_mlp.pkl b/model/mediapipe/prediction_models/prediction_model_mlp.pkl
diff --git a/model/mediapipe/prediction_models/prediction_model_rc.pkl b/model/mediapipe/prediction_models/prediction_model_rc.pkl
diff --git a/model/mediapipe/prediction_models/prediction_model_rf.pkl b/model/mediapipe/prediction_models/prediction_model_rf.pkl
diff --git a/model/mediapipe/prediction_models/prediction_model_svm.pkl b/model/mediapipe/prediction_models/prediction_model_svm.pkl
diff --git a/model/mediapipe/run_mediapipe.py b/model/mediapipe/run_mediapipe.py
@@ -98,7 +98,7 @@ def run(self, classificationModel: str):
                         gesture_class, gesture_prob = self.__use_gb(model, X, image)
                     elif classificationModel == 'svm':
                         gesture_class, gesture_prob = self.__use_svm(model, X, image)
-                    elif classificationModel == 'cnn':
+                    elif classificationModel == 'mlp':
                         gesture_class, gesture_prob = self.__use_cnn(model, X, image)
 
                     if g_ass.addToBufferAndCheck(gesture_class, gesture_prob[np.argmax(gesture_prob)]):
-Original file line number
+Diff line change
@@ Expand Up / @@ -34,7 +34,7 @@ class Classifications(Enum): @@
           rf = 3
           gb = 4
           svm = 5
-          cnn = 6
+          mlp = 6
     def main(args):
@@ Expand Down @@