Corrected accuracy, ran some test experiments

src/utils/accuracy.py

@@ -24,10 +24,12 @@ def accuracy(predmodel, optmodel, dataloader):
     predmodel.eval()
     loss = 0
     optsum = 0
+    num_batches = 0
     # load data
     for data in dataloader:
         x, c, w, z = data
         batch_loss = 0
+        num_batches += 1
         # cuda
         if next(predmodel.parameters()).is_cuda:
             x, c, w, z = x.cuda(), c.cuda(), w.cuda(), z.cuda()
@@ -37,41 +39,44 @@ def accuracy(predmodel, optmodel, dataloader):
         # solve
         for j in range(cp.shape[0]):
             # accumulate loss
-            batch_loss += calAccuracy(optmodel, cp[j], w[j].to("cpu").detach().numpy(),j)
+            batch_loss += calAccuracy(optmodel, cp[j], w[j].to("cpu").detach().numpy(),c[j].to("cpu").detach().numpy())
         loss += batch_loss/cp.shape[0]
     # turn back train mode
     predmodel.train()
+    print(f'num batches is {num_batches}')
     # normalized
-    return loss # divide by batch size
+    return loss/num_batches # divide by number of batches
 
 
-def calAccuracy(optmodel, pred_cost, true_sol,idx):
+def calAccuracy(optmodel, pred_cost, true_sol, true_cost):
     """
-    A function to calculate normalized true regret for a batch
+    A function to calculate accuracy, where the prediction is considered "accurate" if
+    it yields almost the same objective function value as the true solution.
 
     Args:
         optmodel (optModel): optimization model
-        pred_cost (torch.tensor): predicted costs
+        pred_cost (torch.tensor): predicted cost vector
         true_sol (torch.tensor): true solution
+        true_cost: ground truth cost vector
 
     Returns:
-        acc: 1 if true_sol matches that computed using pred_cost to within a small tolerance.
+        acc: 1 if predicted solution yields almost the same objective function value as true_sol.
         
     """
     # opt sol for pred cost
     optmodel.setObj(pred_cost)
     sol, _ = optmodel.solve()
     # ## plotting utility
-    if idx % 25 == 0:
-        fig, axes = plt.subplots(1,2)
-        cax1 = axes[0].matshow(true_sol.reshape(12,12))
-        axes[0].set_title('True')
+    # if idx % 25 == 0:
+    #     fig, axes = plt.subplots(1,2)
+    #     cax1 = axes[0].matshow(true_sol.reshape(12,12))
+    #     axes[0].set_title('True')
 
-        cax2 = axes[1].matshow(sol.reshape(12,12))
-        axes[1].set_title('Predicted')
-        rand_number = random.randint(1,1000)
-        plt.savefig(os.path.join('./src/warcraft/imgs/','comparing_outputs_batch_id='+str(idx)+".png"))
-    if np.linalg.norm(sol - true_sol)< 1e-4:
+    #     cax2 = axes[1].matshow(sol.reshape(12,12))
+    #     axes[1].set_title('Predicted')
+    #     rand_number = random.randint(1,1000)
+    #     plt.savefig(os.path.join('./src/warcraft/imgs/','comparing_outputs_batch_id='+str(idx)+".png"))
+    if np.dot(true_cost, sol - true_sol) < 1e-3: # np.linalg.norm(sol - true_sol)< 1e-4:
         acc = 1
     else:
         acc = 0

src/warcraft/models.py

@@ -56,7 +56,7 @@ def _data_space_forward(self, d):
             return cost_vec
 
     def F(self, z, cost_vec):
-        return cost_vec + 0.0005*z
+        return cost_vec + 0.5*z
 
     def test_time_forward(self, d):
         return self._data_space_forward(d)

src/warcraft/trainer.py

@@ -46,7 +46,7 @@ def trainer(net, train_dataset, test_dataset, val_dataset, edges, grid_size, max
         raise TypeError("Please choose a supported model!")
 
     ## Initialize arrays that will be returned and checkpoint directory
-    val_loss_hist= []
+    val_regret_hist= []
     test_regret_hist = []
     test_acc_hist = []
     val_acc_hist = []
@@ -61,17 +61,17 @@ def trainer(net, train_dataset, test_dataset, val_dataset, edges, grid_size, max
     net.to('cpu')
     curr_val_acc = accuracy(net, net.shortest_path_solver, loader_val)
     val_acc_hist.append(curr_val_acc)
-    best_val_loss = metric(net, net.shortest_path_solver, loader_val)
+    best_val_regret = metric(net, net.shortest_path_solver, loader_val)
 
-    print('Initial validation regret is ', best_val_loss)
-    val_loss_hist.append(best_val_loss)
-    time_till_best_val_loss = 0
+    print('Initial validation regret is ', best_val_regret)
+    val_regret_hist.append(best_val_regret)
+    time_till_best_val_regret = 0
 
      ## Compute initial test loss
-    best_test_loss = metric(net,net.shortest_path_solver, loader_test)
+    best_test_regret = metric(net,net.shortest_path_solver, loader_test)
     best_test_acc = accuracy(net, net.shortest_path_solver, loader_test)
-    print('Initial test regret is ', best_test_loss)
-    test_regret_hist.append(best_test_loss)
+    print('Initial test regret is ', best_test_regret)
+    test_regret_hist.append(best_test_regret)
     test_acc_hist.append(best_test_acc)
 
     ## Train!
@@ -143,49 +143,50 @@ def trainer(net, train_dataset, test_dataset, val_dataset, edges, grid_size, max
         ## Now compute loss on validation set
         net.eval()
         net.to('cpu')
-        val_loss = metric(net, net.shortest_path_solver, loader_val)
+        val_regret = metric(net, net.shortest_path_solver, loader_val)
         val_acc = accuracy(net, net.shortest_path_solver, loader_val)
         val_acc_hist.append(val_acc)
-        val_loss_hist.append(val_loss)
+        val_regret_hist.append(val_regret)
         ## Do the same on test set for consistency with PyEPO
-        test_loss = metric(net, net.shortest_path_solver, loader_test)
+        test_regret = metric(net, net.shortest_path_solver, loader_test)
         test_acc = accuracy(net, net.shortest_path_solver, loader_test)
         test_acc_hist.append(test_acc)
-        test_regret_hist.append(test_loss)
+        test_regret_hist.append(test_regret)
 
         print('\n Current validation accuracy is ' + str(val_acc))
         if (epoch == int(max_epochs*0.6)) or (epoch == int(max_epochs*0.8)):
             for g in optimizer.param_groups:
                 g['lr'] /= 10
 
-        if val_loss < best_val_loss:
+        if val_regret < best_val_regret:
             state_save_name = checkpt_path+'best.pth'
             torch.save(net.state_dict(), state_save_name)
             # If we have achieved lowest validation thus far, this will be the model selected.
             # So, we compute test loss
-            best_test_loss = metric(net,net.shortest_path_solver, loader_test)
+            best_test_regret = metric(net,net.shortest_path_solver, loader_test)
             best_test_acc = accuracy(net, net.shortest_path_solver, loader_test)
-            best_val_loss = val_loss
-            print(f'Best test regret is {best_test_loss} achieved at epoch {epoch}')
-            time_till_best_val_loss = sum(epoch_time_hist)
+            best_val_regret = val_regret
+            print(f'Best test regret is {best_test_regret} achieved at epoch {epoch}')
+            print(f'Best test accuracy is {best_test_acc} achieved at epoch {epoch}')
+            time_till_best_val_regret = sum(epoch_time_hist)
 
-        print('epoch: ', epoch, 'validation regret is ', val_loss, 'epoch time: ', epoch_time)
+        print('epoch: ', epoch, 'validation regret is ', val_regret, 'epoch time: ', epoch_time)
 
     state_save_name = checkpt_path+'last.pth'
     torch.save(net.state_dict(), state_save_name)
-    if time_till_best_val_loss < 1e-6:
-        time_till_best_val_loss = sum(epoch_time_hist)
+    if time_till_best_val_regret < 1e-6:
+        time_till_best_val_regret = sum(epoch_time_hist)
 
     # Collect results
-    results = {"val_loss_hist": val_loss_hist,
+    results = {"val_regret_hist": val_regret_hist,
                "train_mse_loss_hist": train_mse_loss_hist,
                "val_acc_hist": val_acc_hist,
                "test_regret_hist": test_regret_hist,
                "test_acc_hist": test_acc_hist,
                "epoch_time_hist": epoch_time_hist,
-               "best_test_loss": best_test_loss,
+               "best_test_regret": best_test_regret,
                "best_test_acc": best_test_acc,
-               "time_till_best_val_loss":time_till_best_val_loss
+               "time_till_best_val_regret":time_till_best_val_regret
                 }
 
     return results