Train longer epochs (#9)

* initial commit

* working example

* add new models

* update rescaling

* add eval make cam

* minor updates

* add experiments

* update the experiments

* update tests

* update scripts

* update scripts and scales

* introduce bash script

* create a bash function

* initial commit

* working example

* add new models

* add eval make cam

* minor updates

* add experiments

* update the experiments

* update tests

* update scripts

* update scripts and scales

* introduce bash script

* run long_run (#12)

misc/cal_crf.py

@@ -1,5 +1,4 @@
 from functools import partial
-from functools import partial
 from multiprocessing import Pool
 
 import torch
@@ -59,6 +58,7 @@ def calculate_crf(
     dataset_train,
     dataset_valid,
     device,
+    crf_batch_size=500,
     *args,
     **kwargs
 ):
@@ -73,23 +73,39 @@ def calculate_crf(
             img_i = [img_ii.to(device) for img_ii in item["img"]]
             res = generate_pseudo_label(model, img_i, item["label"][0], item["size"])
             tr_results.append((idx.cpu().item(), res))
+            if len(tr_results) >= crf_batch_size:
+                with Pool(processes=4) as pool:
+                    pool.starmap(
+                        partial(dataset_train.update_cam, fg_thres=cfg.cam_eval_thres),
+                        tqdm(tr_results, total=len(tr_results)),
+                    )
+                tr_results = []
         print("\nCreating Pseudo Labels for validation")
         for item in tqdm(val_loader, total=len(val_loader.dataset)):
             idx = item["idx"][0]
             img_i = [img_ii.to(device) for img_ii in item["img"]]
             res = generate_pseudo_label(model, img_i, item["label"][0], item["size"])
             val_results.append((idx.cpu().item(), res))
+            if len(val_results) >= crf_batch_size:
+                with Pool(processes=4) as pool:
+                    pool.starmap(
+                        partial(dataset_valid.update_cam, fg_thres=cfg.cam_eval_thres),
+                        tqdm(val_results, total=len(val_results)),
+                    )
+                val_results = []
     print("Applying CRF to CAM results")
-    with Pool(processes=4) as pool:
-        pool.starmap(
-            partial(dataset_train.update_cam, fg_thres=cfg.cam_eval_thres),
-            tqdm(tr_results, total=len(tr_results)),
-        )
-    with Pool(processes=4) as pool:
-        pool.starmap(
-            partial(dataset_valid.update_cam, fg_thres=cfg.cam_eval_thres),
-            tqdm(val_results, total=len(val_results)),
-        )
+    if len(tr_results) > 0:
+        with Pool(processes=4) as pool:
+            pool.starmap(
+                partial(dataset_train.update_cam, fg_thres=cfg.cam_eval_thres),
+                tqdm(tr_results, total=len(tr_results)),
+            )
+    if len(val_results) > 0:
+        with Pool(processes=4) as pool:
+            pool.starmap(
+                partial(dataset_valid.update_cam, fg_thres=cfg.cam_eval_thres),
+                tqdm(val_results, total=len(val_results)),
+            )
 
     loader_train = DataLoader(
         dataset_train,

scripts/compare_backbone/compare_backbones_train.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+source $(dirname $0)/../train.sh
+
+run_experiment resnet50v1 256 96 25 50
+run_experiment resnet50v2 256 96 25 50
+run_experiment resnet50v3 256 64 25 50
+run_experiment resnet50v4 256 8 25 50
+
+run_experiment resnet50v1 320 64 25 50
+run_experiment resnet50v2 320 64 25 50
+run_experiment resnet50v3 320 32 25 50
+run_experiment resnet50v4 320 8 25 50
+
+run_experiment resnet50v1 448 32 25 50
+run_experiment resnet50v2 448 32 25 50
+run_experiment resnet50v3 448 16 25 50
+run_experiment resnet50v4 448 6 25 50
+
+run_experiment resnet50v1 512 32 25 50
+run_experiment resnet50v2 512 24 25 50
+run_experiment resnet50v3 512 12 25 50
+run_experiment resnet50v4 512 4 25 50

scripts/compare_threshold/train.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+run_experiment() {
+
+    CAM_EVAL_THRES=$1
+
+    BACKBONE=resnet50v2
+    python -m train \
+        backbone=${BACKBONE} \
+        train_list=$(pwd)/voc12/train.txt \
+        val_list=$(pwd)/voc12/val.txt \
+        crop_size=512 \
+        batch_size=24 \
+        hydra.run.dir=$(pwd)/results/threshold/${BACKBONE}/cam_eval_thres${CAM_EVAL_THRES}/train \
+        crf_freq=25 \
+        epochs=50 \
+        cam_eval_thres=${CAM_EVAL_THRES}
+
+}
+
+run_experiment 0.1
+run_experiment 0.15
+run_experiment 0.2
+run_experiment 0.25
+run_experiment 0.3

scripts/long_run/eval.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+CROP_SIZE=512
+BACKBONE=resnet50v2
+CAM_EVAL_THRESHOLD=0.15
+
+for model_name in 4 199 224 249 274 299 324 349 374 399 424 449 474 499 524 549 574; do
+    python -m eval_cam \
+        cam_out_dir=$(pwd)/results/pipeline/${BACKBONE}/crop${CROP_SIZE}/eval/make_cam/seg-model-${model_name}/cam_outputs \
+        hydra.run.dir=$(pwd)/results/pipeline/${BACKBONE}/crop${CROP_SIZE}/eval/eval_cam/seg-model-${model_name}_${CAM_EVAL_THRESHOLD} \
+        infer_set=train \
+        logs=$(pwd)/results/pipeline/${BACKBONE}/crop${CROP_SIZE}/eval/logs_${CAM_EVAL_THRESHOLD} \
+        last_epoch=300 \
+        cam_eval_thres=${CAM_EVAL_THRESHOLD}
+done

scripts/long_run/make_cam.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+BACKBONE=resnet50v2
+CROP_SIZE=512
+
+for model_name in 4 199 224 249 274 299 324 349 374 399 424 449 474 499 524 549 574; do
+    python -m make_cam \
+        backbone=${BACKBONE} \
+        weights=$(pwd)/results/pipeline/${BACKBONE}/crop${CROP_SIZE}/train/weights/seg-model-${model_name}.pt \
+        hydra.run.dir=$(pwd)/results/pipeline/${BACKBONE}/crop${CROP_SIZE}/eval/make_cam/seg-model-${model_name} \
+        infer_list=$(pwd)/voc12/train.txt \
+        crop_size=${CROP_SIZE}
+done

scripts/long_run/train.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source $(dirname $0)/../train.sh
+
+run_experiment resnet50v2 512 24 25 1000

scripts/train.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+run_experiment() {
+    BACKBONE=$1
+    CROP_SIZE=$2
+    BATCH_SIZE=$3
+    CRF_FREQ=$4
+    EPOCHS=$5
+
+    python -m train \
+        backbone=${BACKBONE} \
+        train_list=$(pwd)/voc12/train.txt \
+        val_list=$(pwd)/voc12/val.txt \
+        crop_size=${CROP_SIZE} \
+        batch_size=${BATCH_SIZE} \
+        hydra.run.dir=$(pwd)/results/pipeline/${BACKBONE}/crop${CROP_SIZE}/train \
+        crf_freq=${CRF_FREQ} \
+        epochs=${EPOCHS}
+}