updated performance metrics script to process LRE files.

isi-usc-edu · Dec 9, 2024 · d3ec7ab · d3ec7ab
1 parent 3bbe48e
commit d3ec7ab
Show file tree

Hide file tree

Showing 8 changed files with 24,797 additions and 509 deletions.
diff --git a/...ics.DF QPE.f2d73e1f-3058-43c4-a634-b6c267c84ff1.19f79118-acab-4770-8ea6-0915f2712955.json b/...ics.DF QPE.f2d73e1f-3058-43c4-a634-b6c267c84ff1.19f79118-acab-4770-8ea6-0915f2712955.json
diff --git a/...308d61f8-a6b0-42b3-8e96-0012ba466553.json → ...d8dc20e9-0b6c-48b5-ab41-9637ef0ff4aa.json b/...308d61f8-a6b0-42b3-8e96-0012ba466553.json → ...d8dc20e9-0b6c-48b5-ab41-9637ef0ff4aa.json
@@ -1,14 +1,14 @@
 {
     "$schema": "https://raw.githubusercontent.com/isi-usc-edu/qb-gsee-benchmark/main/schemas/performance_metrics.schema.0.0.1.json",
-    "performance_metrics_uuid": "308d61f8-a6b0-42b3-8e96-0012ba466553",
+    "performance_metrics_uuid": "d8dc20e9-0b6c-48b5-ab41-9637ef0ff4aa",
     "solver_short_name": "DMRG_Niagara_cluster_lowest_energy",
     "solver_uuid": "16537433-9f4c-4eae-a65d-787dc3b35b59",
-    "creation_timestamp": "2024-12-05T22:23:21.568590+00:00",
+    "creation_timestamp": "2024-12-09T21:46:10.109534+00:00",
     "ml_metrics": {
-        "solvability_ratio": 0.1958,
+        "solvability_ratio": 0.2963,
         "f1_score": [
-            0.9733333333333334,
-            0.9787234042553191
+            0.9371428571428572,
+            0.9554655870445344
         ],
         "ml_metrics_calculator_version": 1
     },

diff --git a/scripts/README.md b/scripts/README.md
@@ -0,0 +1,2 @@
+This directory is primarily used by the benchmark maintainers (repo maintainers).  The casual benchmark enjoyer will probably not use any of the scripts or content here.
+
diff --git a/scripts/aggregated_solver_labels_2024-12-05-17-03.csv b/scripts/aggregated_solver_labels_2024-12-05-17-03.csv
diff --git a/scripts/aggregated_solver_labels_2024-12-09-16-35.csv b/scripts/aggregated_solver_labels_2024-12-09-16-35.csv
diff --git a/scripts/compute_all_performance_metrics_script.log.txt b/scripts/compute_all_performance_metrics_script.log.txt
diff --git a/scripts/compute_all_performance_metrics_script.py b/scripts/compute_all_performance_metrics_script.py
@@ -126,7 +126,7 @@ def identify_unique_participating_solvers(
 
 def get_solver_short_name(solver_uuid, solver_list) -> str:
     df = solver_list[solver_list["solver_uuid"] == solver_uuid]
-    return df["solver_short_name"][0]
+    return df["solver_short_name"].iloc[0]
 
 
 
@@ -182,6 +182,7 @@ def main(args):
         "solver_short_name",
         "solver_uuid",
         "solution_uuid",
+        "problem_instance_short_name",
         "problem_instance_uuid",
         "task_uuid",
         "instance_data_object_uuid",
@@ -191,13 +192,17 @@ def main(args):
         "solved_within_accuracy_requirement",
         "overall_run_time_seconds",
         "is_resource_estimate",
+        "num_logical_qubits",
+        "num_shots",
+        "num_T_gates_per_shot",
         "submitted_by_calendar_due_date",
         "label" # label True/False, that the Hamiltonian was solved.
     ]
     aggregated_results = pd.DataFrame(columns=aggregated_results_columns)
 
     for problem_instance in problem_instance_list:
         problem_instance_uuid = problem_instance["problem_instance_uuid"]
+        problem_instance_short_name = problem_instance["short_name"]
 
         for task in problem_instance["tasks"]:
             num_supporting_files = len(task["supporting_files"])
@@ -234,20 +239,16 @@ def main(args):
                     solution_list=solution_list
                 )
 
-                # the format of a results dictionary:
-                # results = { 
-                #   task_uuid,
-                #   energy,
-                #   energy_units,
-                #   run_time{overall_time{seconds}}
-                # }
 
+                # determine if it's a resource estimate
+                is_resource_estimate = None
                 for solution in solution_list:
                     if solution["solution_uuid"] == solution_uuid:
                         is_resource_estimate = solution["is_resource_estimate"]
                         break
 
 
+
                 # task-specific requirements
                 time_limit_seconds = task["requirements"]["time_limit_seconds"]
                 accuracy_tol = task["requirements"]["accuracy"]
@@ -263,6 +264,7 @@ def main(args):
                 if results is None:
                     # the solver did NOT submit a solution file for the problem_instance or Hamiltonian.
                     # mark it as failed.  TODO:  do something more nuanced with non-attempted problems in the future.
+                    is_resource_estimate = None
                     attempted = False
                     solved_within_run_time = False
                     solved_within_accuracy_requirement = False
@@ -277,25 +279,35 @@ def main(args):
                     overall_run_time_seconds = results["run_time"]["overall_time"]["seconds"]
                     solved_within_run_time = overall_run_time_seconds <= time_limit_seconds
 
-                    reported_energy = results["energy"]
-
-
-
-                    try:
-                        solved_within_accuracy_requirement = bool(np.abs(reported_energy - reference_energy) < accuracy_tol)
-                        # TODO:  account for differences in units.  E.g., Hartree vs. kCal/mol vs. other.
-                    except Exception as e:
-                        logging.error(f'Error: {e}', exc_info=True)
-                        logging.info(f"warning!  no energy target specified in task {task_uuid}")
-                        solved_within_accuracy_requirement = False
-
-                    calendar_due_date = problem_instance["calendar_due_date"]
-                    if calendar_due_date is None:
-                        submitted_by_calendar_due_date = True # no due date.
+                    if is_resource_estimate:
+                        num_logical_qubits = results["quantum_resources"]["logical"]["num_logical_qubits"]
+                        num_shots = results["quantum_resources"]["logical"]["num_shots"]
+                        num_T_gates_per_shot = results["quantum_resources"]["logical"]["num_T_gates_per_shot"]
+                        solved_within_accuracy_requirement = True # always true.  assume LREs solve to accuracy.
+                        submitted_by_calendar_due_date = None 
                     else:
-                        # TODO: issue-44.  handle case when more than one solution submitted by
-                        # one solver.  for now assume, solutions were submitted by due date.
-                        submitted_by_calendar_due_date = True # no due date.
+                        # NOT a logical resource estimate.
+                        num_logical_qubits = None
+                        num_shots = None
+                        num_T_gates_per_shot = None
+
+                        reported_energy = results["energy"]
+                        try:
+                            solved_within_accuracy_requirement = bool(np.abs(reported_energy - reference_energy) < accuracy_tol)
+                            # TODO:  account for differences in units.  E.g., Hartree vs. kCal/mol vs. other.
+                        except Exception as e:
+                            logging.error(f'Error: {e}', exc_info=True)
+                            logging.info(f"warning!  no energy target specified in task {task_uuid}")
+                            solved_within_accuracy_requirement = False
+
+                        calendar_due_date = problem_instance["calendar_due_date"]
+                        if calendar_due_date is None: 
+                            # no due date specified in problem_instance
+                            submitted_by_calendar_due_date = True
+                        else:
+                            # TODO: issue-44.  handle case when more than one solution submitted by
+                            # one solver.  for now assume solutions were submitted by due date.
+                            submitted_by_calendar_due_date = True 
 
                     label = solved_within_run_time and solved_within_accuracy_requirement
 
@@ -306,6 +318,7 @@ def main(args):
                     "solver_short_name":solver_short_name,
                     "solver_uuid":solver_uuid,
                     "solution_uuid":solution_uuid,
+                    "problem_instance_short_name":problem_instance_short_name,
                     "problem_instance_uuid":problem_instance_uuid,
                     "task_uuid":task_uuid,
                     "instance_data_object_uuid":instance_data_object_uuid,
@@ -315,6 +328,9 @@ def main(args):
                     "solved_within_accuracy_requirement":solved_within_accuracy_requirement,
                     "overall_run_time_seconds":overall_run_time_seconds,
                     "is_resource_estimate":is_resource_estimate,
+                    "num_logical_qubits":num_logical_qubits,
+                    "num_shots":num_shots,
+                    "num_T_gates_per_shot":num_T_gates_per_shot,
                     "submitted_by_calendar_due_date":submitted_by_calendar_due_date,
                     "label":label # label True/False, that the Hamiltonian was solved.
                 }])
@@ -353,22 +369,31 @@ def main(args):
         solver_labels_file_name = f"solver_labels.{solver_short_name}.{solver_uuid}.csv"
         solver_labels.to_csv(solver_labels_file_name)
 
-        logging.info(f"calculating ML scores for solver {solver_short_name}/{solver_uuid}...")
-
-        # TODO:  update remove the "partial_results" from the Hamiltonian features when completed.  Re-run.
-        ham_features_file="../Hamiltonian_features/experimental/fast_double_factorization_features/Hamiltonian_features.partial_results.csv"
-        solvability_ratio, f1_score = miniML(argparse.Namespace(
-            ham_features_file=ham_features_file,
-            config_file="../BubbleML/miniML/miniML_config.json",
-            solver_uuid=solver_uuid,
-            solver_labels_file=solver_labels_file_name,
-            verbose=False
-        ))
-        ml_scores[solver_uuid] = {
-            "solvability_ratio":solvability_ratio,
-            "f1_score":list(f1_score),
-            "ml_metrics_calculator_version":1
-        }
+
+        try:
+            logging.info(f"calculating ML scores for solver {solver_short_name}/{solver_uuid}...")
+
+            ham_features_file="../Hamiltonian_features/experimental/fast_double_factorization_features/Hamiltonian_features.csv"
+            solvability_ratio, f1_score = miniML(argparse.Namespace(
+                ham_features_file=ham_features_file,
+                config_file="../BubbleML/miniML/miniML_config.json",
+                solver_uuid=solver_uuid,
+                solver_labels_file=solver_labels_file_name,
+                verbose=False
+            ))
+            ml_scores[solver_uuid] = {
+                "solvability_ratio":solvability_ratio,
+                "f1_score":list(f1_score),
+                "ml_metrics_calculator_version":1
+            }
+        except Exception as e:
+            logging.error(f'Error: {e}', exc_info=True)
+            logging.info(f"bummer!  setting ml_scores to `None`.")
+            ml_scores[solver_uuid] = {
+                "solvability_ratio":None,
+                "f1_score":None,
+                "ml_metrics_calculator_version":1
+            }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		This directory is primarily used by the benchmark maintainers (repo maintainers). The casual benchmark enjoyer will probably not use any of the scripts or content here.