fixed schema to allow null values when some metrics are not calcula…

…ted.
isi-usc-edu · Dec 11, 2024 · d873376 · d873376
1 parent bec37be
commit d873376
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 10 deletions.
diff --git a/...ics.DF_QPE.f2d73e1f-3058-43c4-a634-b6c267c84ff1.32034c2e-2319-4da8-8d99-65d0a3807291.json b/...ics.DF_QPE.f2d73e1f-3058-43c4-a634-b6c267c84ff1.32034c2e-2319-4da8-8d99-65d0a3807291.json
@@ -3,7 +3,7 @@
     "performance_metrics_uuid": "32034c2e-2319-4da8-8d99-65d0a3807291",
     "solver_short_name": "DF_QPE",
     "solver_uuid": "f2d73e1f-3058-43c4-a634-b6c267c84ff1",
-    "creation_timestamp": "2024-12-10T12:17:57.483531+00:00",
+    "creation_timestamp": "2024-12-10T12:17:57.483531Z",
     "ml_metrics": {
         "solvability_ratio": null,
         "f1_score": null,

diff --git a/...est_energy.16537433-9f4c-4eae-a65d-787dc3b35b59.8297b0b2-d56c-4d49-98b4-b22f1cadbf2b.json b/...est_energy.16537433-9f4c-4eae-a65d-787dc3b35b59.8297b0b2-d56c-4d49-98b4-b22f1cadbf2b.json
@@ -3,7 +3,7 @@
     "performance_metrics_uuid": "8297b0b2-d56c-4d49-98b4-b22f1cadbf2b",
     "solver_short_name": "DMRG_Niagara_cluster_lowest_energy",
     "solver_uuid": "16537433-9f4c-4eae-a65d-787dc3b35b59",
-    "creation_timestamp": "2024-12-10T12:17:57.033953+00:00",
+    "creation_timestamp": "2024-12-10T12:17:57.033953Z",
     "ml_metrics": {
         "solvability_ratio": 0.7533,
         "f1_score": [

diff --git a/schemas/performance_metrics.schema.0.0.1.json b/schemas/performance_metrics.schema.0.0.1.json
@@ -55,17 +55,17 @@
                 "ml_metrics_calculator_version": {
                     "title": "ML performance metrics calculator version",
                     "description": "The version of the ML metrics calculator software.",
-                    "type": "string"
+                    "type": "number"
                 },
                 "solvability_ratio": {
                     "title": "ML Solvability Ratio",
                     "description": "TODO.  A number between 0-1.  A higher number is better.",
-                    "type": "number"
+                    "type":["null","number"]
                 },
                 "f1_score": {
                     "title": "F1 score",
                     "description": "TODO.  This is a measure of the goodness of the ML model, not the solver being evaluated.",
-                    "type": "object"
+                    "type": ["null","array"]
                 }
             }
         },
@@ -200,7 +200,7 @@
                     "solution_submitted_by_due_date": {
                         "title": "Solution Within Due Date?",
                         "description": "Each `problem_instance` file has a specified calendar due date.  This boolean to identifies if the solution was submitted by the due date.  If there was no due date specified, then this is `true`.",
-                        "type": "boolean"
+                        "type": ["null","boolean"]
                     }
                 }
             }
@@ -247,12 +247,12 @@
                     "solved_within_run_time": {
                         "title": "TODO",
                         "description": "a boolean indicating if the solver run time was within the run time requirement specified in the problem_instance.",
-                        "type": "boolean"
+                        "type": ["null","boolean"]
                     },
                     "solved_within_accuracy_requirement": {
                         "title": "TODO",
                         "description": "a boolean indicating if the solver run time was within the accuracy requirement specified in the problem_instance.",
-                        "type": "boolean"
+                        "type": ["null","boolean"]
                     },
                     "overall_run_time_seconds":{
                         "title": "TODO",

diff --git a/scripts/validate_json_file.py b/scripts/validate_json_file.py
@@ -26,7 +26,6 @@
 
 
 
-
 def main(args):
 
     # read the file you want to validate
@@ -75,7 +74,9 @@ def main(args):
         jsonschema.validate(instance=file_contents, schema=schema)
         print("JSON file is valid per the schema!\n\n")
     except Exception as e:
-        # print(f"Error: {e}") ## lots of output.
+        if args.verbose:
+            print(f"Error: {e}") ## lots of output...
+            print("\n\n")
         print(f"JSON file has FAILED schema validation!\n\n")
         sys.exit(1)
 
@@ -102,6 +103,13 @@ def main(args):
         help="(Optional) schema file.  If used, this will override whatever URL may be listed in the JSON file in the $schema field."
     )
 
+    parser.add_argument(
+        "-v","--verbose",
+        action="store_true",
+        default=False,
+        help="print verbose error message of why the file may have failed validation."
+    )
+
     args = parser.parse_args()
 
     main(args)