Merge pull request #103 from NextCenturyCorporation/development

Dev To Main

swagger_server/config.ini.template

@@ -23,3 +23,56 @@ SAVE_HISTORY_TO_S3=True
 
 HISTORY_DIRECTORY=itm_history_output
 HISTORY_S3_BUCKET=itm-ui-assets
+
+EVAL_NAME=Dry Run Evaluation
+EVAL_NUMBER=4
+
+SOARTECH_EVAL_FILENAMES=dryrun-soartech-eval-qol1.yaml,
+    dryrun-soartech-eval-qol2.yaml,
+    dryrun-soartech-eval-qol3.yaml,
+    dryrun-soartech-eval-vol1.yaml, 
+    dryrun-soartech-eval-vol2.yaml, 
+    dryrun-soartech-eval-vol3.yaml
+
+SOARTECH_TRAIN_FILENAMES=dryrun-soartech-train-qol1.yaml,
+    dryrun-soartech-train-qol2.yaml,
+    dryrun-soartech-train-vol1.yaml,
+    dryrun-soartech-train-vol2.yaml
+
+SOARTECH_EVAL_QOL_SCENARIOS=qol-dre-1-eval,qol-dre-2-eval,qol-dre-3-eval
+SOARTECH_EVAL_VOL_SCENARIOS=vol-dre-1-eval,vol-dre-2-eval,vol-dre-3-eval
+
+SOARTECH_TRAIN_QOL_SCENARIOS=qol-dre-1-train,qol-dre-2-train
+SOARTECH_TRAIN_VOL_SCENARIOS=vol-dre-1-train,vol-dre-2-train
+
+SOARTECH_QOL_ALIGNMENT_TARGETS=qol-human-8022671-SplitLowMulti,
+    qol-human-1774519-SplitEvenBinary,
+    qol-human-6403274-SplitHighBinary,
+    qol-human-3043871-SplitHighBinary,
+    qol-human-2932740-HighExtreme,
+    qol-human-6349649-SplitHighMulti,
+    qol-human-9157688-SplitEvenBinary,
+    qol-human-5032922-SplitLowMulti,
+    qol-human-3447902-SplitHighMulti,
+    qol-human-0000001-SplitEvenMulti,
+    qol-human-7040555-SplitHighMulti,
+    qol-synth-LowExtreme,
+    qol-synth-HighExtreme,
+    qol-synth-HighCluster,
+    qol-synth-LowCluster,
+    qol-synth-SplitLowBinary
+
+SOARTECH_VOL_ALIGNMENT_TARGETS=vol-human-8022671-SplitHighMulti,
+    vol-human-1774519-SplitHighMulti,
+    vol-human-6403274-SplitEvenBinary,
+    vol-human-8478698-SplitLowMulti,
+    vol-human-3043871-SplitLowMulti,
+    vol-human-2637411-SplitEvenMulti,
+    vol-human-2932740-SplitEvenMulti,
+    vol-human-5032922-SplitLowMulti,
+    vol-human-7040555-SplitEvenBinary,
+    vol-synth-LowExtreme,
+    vol-synth-HighExtreme,
+    vol-synth-HighCluster,
+    vol-synth-LowCluster,
+    vol-synth-SplitLowBinary

swagger_server/config_util.py

@@ -22,7 +22,9 @@ def check_ini():
     ini = parser_path[0]
     final_path = parser_path[1]
     if exists(final_path):
-        keys = ("EVALUATION_TYPE", "SCENARIO_DIRECTORY", "ADEPT_URL", "SOARTECH_URL", "SAVE_HISTORY", "SAVE_HISTORY_TO_S3", "HISTORY_DIRECTORY", "HISTORY_S3_BUCKET")
+        keys = ("EVALUATION_TYPE", "SCENARIO_DIRECTORY", "ADEPT_URL", "SOARTECH_URL", "SAVE_HISTORY", "SAVE_HISTORY_TO_S3", "HISTORY_DIRECTORY", "HISTORY_S3_BUCKET",
+                "EVAL_NAME", "EVAL_NUMBER", "SOARTECH_EVAL_QOL_SCENARIOS", "SOARTECH_EVAL_VOL_SCENARIOS", "SOARTECH_TRAIN_QOL_SCENARIOS", "SOARTECH_TRAIN_VOL_SCENARIOS",
+                "SOARTECH_QOL_ALIGNMENT_TARGETS", "SOARTECH_VOL_ALIGNMENT_TARGETS", "SOARTECH_EVAL_FILENAMES", "SOARTECH_TRAIN_FILENAMES")
         for option in keys:
             if option in ini["DEFAULT"]: # checks if "option" key exists in "default" section
                 key_value = ini.get("DEFAULT", option)

swagger_server/itm/itm_history.py

@@ -2,6 +2,8 @@
 import json
 import logging
 import os
+import time
+import datetime
 
 from botocore.exceptions import ClientError
 from typing import Union
@@ -18,6 +20,11 @@ def __init__(self, config):
         self.history = []
         self.filepath = config["DEFAULT"]["HISTORY_DIRECTORY"] + os.sep
         self.save_history_bucket = config["DEFAULT"]["HISTORY_S3_BUCKET"]
+        self.evaluation_info = {
+            "evalName": config['DEFAULT']['EVAL_NAME'], 
+            "evalNumber": config['DEFAULT']['EVAL_NUMBER'], 
+            "created" : str(datetime.datetime.now())
+        }        
 
     def clear_history(self):
         self.history.clear()
@@ -64,7 +71,7 @@ def write_to_json_file(self, filebasename, save_to_s3) -> None:
 
         with open(full_filepath, 'w') as file:
             # Convert Python dictionary to JSON and write to file
-            json.dump({'history': self.history}, file, indent=2)
+            json.dump({'evaluation': self.evaluation_info, 'history': self.history}, file, indent=2)
 
         if (save_to_s3):
             logging.info("Saving history to S3")

swagger_server/itm/itm_scenario.py

@@ -78,6 +78,12 @@ def get_available_actions(self) -> List[Action]:
             else:
                 pass # This an error condition that will be flagged when changing scenes
 
+        response = [filtered_action.to_dict() for filtered_action in filtered_actions]
+        self.session.history.add_history(
+            "Get Available Actions",
+            {"session_id": self.session.session_id, "scenario_id": self.session.itm_scenario.id},
+            response)
+
         return filtered_actions
 
 
@@ -116,7 +122,7 @@ def respond_to_probe(self, probe_id, choice_id, justification):
                         response.scenario_id,
                         response.probe_id
                     )
-                    logging.info(f"Probe alignment score: {probe_response_alignment['score']}")
+                    logging.info(f"Probe alignment score: {probe_response_alignment.get('score', 'nan')}")
                     self.session.history.add_history(
                         "TA1 Probe Response Alignment",
                         {"session_id": self.ta1_controller.session_id,
@@ -125,9 +131,10 @@ def respond_to_probe(self, probe_id, choice_id, justification):
                         "probe_id": response.probe_id},
                         probe_response_alignment
                     )
-                    alignment_scenario_id = probe_response_alignment['alignment_source'][0]['scenario_id']
-                    if self.id != alignment_scenario_id:
-                        logging.error("\033[92mContamination in probe alignment! scenario is %s but alignment source scenario is %s.\033[00m", self.id, alignment_scenario_id)
+                    if probe_response_alignment.get('alignment_source'):
+                        alignment_scenario_id = probe_response_alignment['alignment_source'][0]['scenario_id']
+                        if self.id != alignment_scenario_id:
+                            logging.warning("\033[92mContamination in probe alignment! scenario is %s but alignment source scenario is %s.\033[00m", self.id, alignment_scenario_id)
             except exceptions.HTTPError:
                 # Consider changing to logging.exception when this exception isn't so common.
                 logging.error("HTTPError from TA1 getting probe alignment.")