Automate parameter search measurements

Author: ottoblep

.gitignore

@@ -17,6 +17,7 @@ bin/
 build/
 develop-eggs/
 dist/
+results/
 eggs/
 lib/
 lib64/

apps/vss_generation_timetable_simulator_genetic_parameter_testing.cpp

@@ -17,6 +17,8 @@ int main(int argc, char** argv) {
 
   const std::string model_path  = args[1];
   const std::string output_path = args[2];
+  const std::string model_name =
+      model_path.substr(model_path.find_last_of("/"), model_path.length());
 
   cda_rail::Network network =
       cda_rail::Network::import_network(model_path + "/network");
@@ -69,7 +71,10 @@ int main(int argc, char** argv) {
         workers.pop_back();
       }
 
-      score_coll.export_csv(output_path + "/score_hist_" +
+      auto save_path =
+          output_path + "/results/genetic_params/crossover/" + model_name;
+      cda_rail::is_directory_and_create(save_path);
+      score_coll.export_csv(save_path + "/score_hist_" +
                             std::to_string(xover_fraction).substr(0, 5) +
                             ".csv");
     }
@@ -105,7 +110,10 @@ int main(int argc, char** argv) {
         workers.pop_back();
       }
 
-      score_coll.export_csv(output_path + "/score_hist_" +
+      auto save_path =
+          output_path + "/results/genetic_params/mut_rate/" + model_name;
+      cda_rail::is_directory_and_create(save_path);
+      score_coll.export_csv(save_path + "/score_hist_" +
                             std::to_string(mut_rate).substr(0, 5) + ".csv");
     }
   }
@@ -141,7 +149,10 @@ int main(int argc, char** argv) {
         workers.pop_back();
       }
 
-      score_coll.export_csv(output_path + "/score_hist_" +
+      auto save_path =
+          output_path + "/results/genetic_params/pop/" + model_name;
+      cda_rail::is_directory_and_create(save_path);
+      score_coll.export_csv(save_path + "/score_hist_" +
                             std::to_string(pop).substr(0, 5) + ".csv");
     }
   }
@@ -177,7 +188,10 @@ int main(int argc, char** argv) {
         workers.pop_back();
       }
 
-      score_coll.export_csv(output_path + "/score_hist_" +
+      auto save_path =
+          output_path + "/results/genetic_params/elite/" + model_name;
+      cda_rail::is_directory_and_create(save_path);
+      score_coll.export_csv(save_path + "/score_hist_" +
                             std::to_string(elite).substr(0, 5) + ".csv");
     }
   }

apps/vss_generation_timetable_simulator_greedy_parameter_testing.cpp

@@ -17,6 +17,8 @@ int main(int argc, char** argv) {
 
   const std::string model_path  = args[1];
   const std::string output_path = args[2];
+  const std::string model_name =
+      model_path.substr(model_path.find_last_of("/"), model_path.length());
 
   cda_rail::Network network =
       cda_rail::Network::import_network(model_path + "/network");
@@ -29,7 +31,7 @@ int main(int argc, char** argv) {
   if (processor_count == 0)
     processor_count = 1;
 
-  std::vector<size_t> test_timeouts = {5, 10, 50, 100, 250, 500};
+  std::vector<size_t> test_timeouts = {1, 2, 5, 10, 50, 100, 250};
 
   for (size_t train_to : test_timeouts) {
     cda_rail::sim::ScoreHistoryCollection score_coll;
@@ -59,7 +61,10 @@ int main(int argc, char** argv) {
       workers.pop_back();
     }
 
-    score_coll.export_csv(output_path + "/score_hist_" +
+    auto save_path =
+        output_path + "/results/greedy_params/stall_time/" + model_name;
+    cda_rail::is_directory_and_create(save_path);
+    score_coll.export_csv(save_path + "/score_hist_" +
                           std::to_string(train_to).substr(0, 5) + ".csv");
   }
 }

apps/vss_generation_timetable_simulator_local_parameter_testing.cpp

@@ -17,6 +17,8 @@ int main(int argc, char** argv) {
 
   const std::string model_path  = args[1];
   const std::string output_path = args[2];
+  const std::string model_name =
+      model_path.substr(model_path.find_last_of("/"), model_path.length());
 
   cda_rail::Network network =
       cda_rail::Network::import_network(model_path + "/network");
@@ -60,8 +62,10 @@ int main(int argc, char** argv) {
       workers.pop_back();
     }
 
+    auto save_path = output_path + "/results/local_params/multi/" + model_name;
+    cda_rail::is_directory_and_create(save_path);
     score_coll.export_csv(
-        output_path + "/score_hist_" +
+        save_path + "/score_hist_" +
         std::to_string(std::get<0>(param)).substr(0, 5) + "-" +
         std::to_string(std::get<1>(param)).substr(0, 5) + "-" +
         std::to_string((std::get<2>(param))).substr(0, 5) + ".csv");

apps/vss_generation_timetable_simulator_search_methods_testing.cpp

@@ -49,7 +49,7 @@ int main(int argc, char** argv) {
   };
 
   std::vector<std::string> methods = {"random", "random+local", "greedy",
-                                      "grasp", "genetic"};
+                                      "grasp"};
 
   for (std::string method : methods) {
     cda_rail::sim::ScoreHistoryCollection score_coll;
@@ -63,9 +63,9 @@ int main(int argc, char** argv) {
         int max_samples;
         if (method == "random" || method == "greedy" ||
             method == "random+local" || method == "grasp")
-          max_samples = 4;
+          max_samples = 1;
         else
-          max_samples = 2;
+          max_samples = 1;
 
         for (size_t sample = 0; sample < max_samples; sample++) {
           // Method here
@@ -74,15 +74,15 @@ int main(int argc, char** argv) {
                      cda_rail::sim::ScoreHistory>
               res;
           if (method == "random") {
-            res = solver.random_search(std::chrono::seconds{2500}, {});
+            res = solver.random_search(std::chrono::seconds{10}, {});
           } else if (method == "greedy") {
-            res = solver.greedy_search(std::chrono::seconds{2500}, {},
+            res = solver.greedy_search(std::chrono::seconds{10}, {},
                                        {std::chrono::milliseconds{50}});
           } else if (method == "random+local") {
-            res = solver.random_local_search(std::chrono::seconds{2500},
+            res = solver.random_local_search(std::chrono::seconds{10},
                                              loc_params);
           } else if (method == "grasp") {
-            res = solver.grasp_search(std::chrono::seconds{2500},
+            res = solver.grasp_search(std::chrono::seconds{10},
                                       {std::chrono::milliseconds{50}},
                                       loc_params);
           } else if (method == "genetic") {
@@ -109,8 +109,10 @@ int main(int argc, char** argv) {
       workers.pop_back();
     }
 
-    score_coll.export_csv(output_path + "/" + model_name + "_" + method +
-                          ".csv");
+    cda_rail::is_directory_and_create(output_path + "/results/methods/" +
+                                      model_name);
+    score_coll.export_csv(output_path + "/methods/" + model_name +
+                          "/score_hist_" + method + ".csv");
   }
 }
 

dev_scripts/launch_all_measures.sh

@@ -0,0 +1,6 @@
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+sh $SCRIPT_DIR/launch_genetic_paramsearch.sh
+sh $SCRIPT_DIR/launch_greedy_paramsearch.sh
+sh $SCRIPT_DIR/launch_local_paramsearch.sh
+sh $SCRIPT_DIR/launch_method_compare_all.sh