Merge pull request #269 from leondavi/FullFlowCI

saveToFile in Stats , loss_min baseline

src_py/apiServer/experiment_flow_debug.py

@@ -35,6 +35,7 @@ def print_test(in_str : str):
 
 experiment_inst = api_server_instance.get_experiment(experiment_name)
 exp_stats = Stats(experiment_inst)
+exp_stats.get_loss_min(saveToFile=True)
 loss = exp_stats.get_loss()
 loss_min = exp_stats.get_loss_min()
 conf = exp_stats.get_confusion_matrices()

src_py/apiServer/experiment_flow_test.py

@@ -13,7 +13,8 @@ def print_test(in_str : str):
 
 NERLNET_PATH = os.getenv('NERLNET_PATH')
 TESTS_PATH = os.getenv('TESTS_PATH')
-TESTS_BASELINE = os.getenv('TESTS_BASELINE')
+TESTS_BASELINE_ACC_STATS = os.getenv('TEST_BASELINE_ACC_STATS')
+TESTS_BASELINE_LOSS_MIN = os.getenv('TEST_BASELINE_LOSS_MIN')
 NERLNET_RUN_SCRIPT = "./NerlnetRun.sh --run-mode release"
 NERLNET_RUN_STOP_SCRIPT = "./NerlnetRun.sh --run-mode stop"
 NERLNET_RUNNING_TIMEOUT_SEC = int(os.getenv('NERLNET_RUNNING_TIMEOUT_SEC'))
@@ -63,21 +64,31 @@ def print_test(in_str : str):
     print_test(stdout)
 
 exp_stats = Stats(experiment_inst)
-data = exp_stats.get_loss_min()
+loss_min = exp_stats.get_loss_min(saveToFile=True)
 print_test("min loss of each worker")
-print(data)
+baseline_loss_min = import_dict_json(TESTS_BASELINE_LOSS_MIN)
+for worker in loss_min.keys():
+    diff = abs(loss_min[worker] - baseline_loss_min[worker])
+    if baseline_loss_min[worker] == 0:
+        error = diff
+    else:
+        error = diff/baseline_loss_min[worker]
+    print_test(f"worker: {worker}, diff: {diff} , error: {error}")
+    if error > TEST_ACCEPTABLE_MARGIN_OF_ERROR:
+        print(f"Anomaly failure detected")
+        print(f"Error: {error} , Acceptable error: {TEST_ACCEPTABLE_MARGIN_OF_ERROR}")
+        exit(1)
+
+
 
 conf = exp_stats.get_confusion_matrices()
 acc_stats = exp_stats.get_accuracy_stats(conf)
-baseline_acc_stats = import_dict_json(TESTS_BASELINE)
-diff_from_baseline = []
+baseline_acc_stats = import_dict_json(TESTS_BASELINE_ACC_STATS)
 for worker in acc_stats.keys():
     for j in acc_stats[worker].keys():
         diff = abs(acc_stats[worker][j]["F1"] - baseline_acc_stats[worker][str(j)]["F1"])
-        diff_from_baseline.append(diff/baseline_acc_stats[worker][str(j)]["F1"])
-anomaly_detected = not all([x < TEST_ACCEPTABLE_MARGIN_OF_ERROR for x in diff_from_baseline])
-if anomaly_detected:
-    print_test("Anomaly failure detected")
-    print_test(f"diff_from_baseline: {diff_from_baseline}")
-    exit(1)
-
+        error = diff/baseline_acc_stats[worker][str(j)]["F1"]
+        if error > TEST_ACCEPTABLE_MARGIN_OF_ERROR:
+            print_test("Anomaly failure detected")
+            print_test(f"diff_from_baseline: {diff}")
+            exit(1)

src_py/apiServer/stats.py

@@ -44,6 +44,9 @@ def get_loss(self , plot : bool = False , saveToFile : bool = False):
             plt.grid(visible=True, which='minor', linestyle='-', alpha=0.7)
             plt.show()
             plt.savefig(f'{EXPERIMENT_RESULTS_PATH}/{self.experiment.name}/Training/Loss_graph.png')
+
+        if saveToFile:
+            export_dict_json(f'{EXPERIMENT_RESULTS_PATH}/{self.exp_path}/loss.json', loss_dict)
         return loss_dict
 
     def get_loss_min(self , plot : bool = False , saveToFile : bool = False):
@@ -69,6 +72,9 @@ def get_loss_min(self , plot : bool = False , saveToFile : bool = False):
             plt.grid(visible=True, which='minor', linestyle='-', alpha=0.7)
             plt.show()
             plt.savefig(f'{EXPERIMENT_RESULTS_PATH}/{self.experiment.name}/Training/Min_loss_graph.png')
+
+        if saveToFile:
+            export_dict_json(f'{EXPERIMENT_RESULTS_PATH}/{self.exp_path}/min_loss.json', min_loss_dict)
         return min_loss_dict
 
     def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False , saveToFile : bool = False):
@@ -110,6 +116,9 @@ def get_confusion_matrices(self , normalize : bool = False ,plot : bool = False
             plt.subplots_adjust(wspace=1, hspace=0.15) 
             f.colorbar(disp.im_, ax=axes)
             plt.show()
+
+        if saveToFile:
+            export_dict_json(f'{EXPERIMENT_RESULTS_PATH}/{self.exp_path}/confusion_matrices.json', workers_confusion_matrices)
         return workers_confusion_matrices
 
     def get_accuracy_stats(self , confMatDict , show : bool = False , saveToFile : bool = False) -> dict:

tests/NerlnetFullFlowTest.sh

@@ -13,7 +13,8 @@ NERLNET_CONFIG_INPUT_DATA_DIR=$NERLNET_CONFIG_DIR/inputDataDir.nerlconfig
 NERLNET_CONFIG_INPUT_DATA_DIR_BACKUP=$NERLNET_CONFIG_DIR/inputDataDir.nerlconfig.bac
 
 TEST_INPUT_JSONS_FILES_DIR="$TESTS_PATH/inputJsonsFiles"
-export TESTS_BASELINE="$TEST_INPUT_JSONS_FILES_DIR/accuracy_stats_synt_1d_2c_4r_4w.json"
+export TEST_BASELINE_ACC_STATS="$TEST_INPUT_JSONS_FILES_DIR/accuracy_stats_synt_1d_2c_4r_4w.json"
+export TEST_BASELINE_LOSS_MIN="$TEST_INPUT_JSONS_FILES_DIR/min_loss_synt_1d_2c_4r_4w.json"
 
 TEST_ARCH_JSON_NOIP_0=$TEST_INPUT_JSONS_FILES_DIR/arch_test_synt_1d_2c_1s_4r_4w.json.noip
 TEST_ARCH_JSON_0=$TEST_INPUT_JSONS_FILES_DIR/arch_test_synt_1d_2c_1s_4r_4w.json

tests/inputJsonsFiles/min_loss_synt_1d_2c_4r_4w.json

@@ -0,0 +1,6 @@
+{
+    "w1": 0.0,
+    "w2": 0.0,
+    "w3": 0.0,
+    "w4": 0.0
+}