incorporate kiana changs to viz

manipulathor_baselines/procthor_baselines/experiments/ithor/obj_nav_2camera_ithor_wide.py

@@ -85,6 +85,24 @@ class ithorObjectNavClipResnet50RGBOnly2CameraWideFOV(
     MAX_STEPS = 500
     if platform.system() == "Darwin":
         MAX_STEPS = 100
+        SENSORS += [
+            RGBSensorStretchKinectBigFov(
+                height=224,
+                width=224,
+                use_resnet_normalization=True,
+                mean=mean,
+                stdev=stdev,
+                uuid="rgb_lowres_arm_only_viz",
+            ),
+            RGBSensorThor(
+                height=224,
+                width=224,
+                use_resnet_normalization=True,
+                mean=mean,
+                stdev=stdev,
+                uuid="rgb_lowres_only_viz",
+            ),
+        ]
 
     TASK_SAMPLER = AllRoomsObjectNavTaskSampler
     TASK_TYPE = StretchObjectNavTask

manipulathor_baselines/procthor_baselines/experiments/ithor/obj_nav_2camera_procthor_narrow.py

@@ -70,6 +70,25 @@ class ProcTHORObjectNavClipResnet50RGBOnly2CameraNarrowFOV(
     MAX_STEPS = 500
     if platform.system() == "Darwin":
         MAX_STEPS = 100
+        SENSORS += [
+            RGBSensorStretchKinect(
+                height=224,
+                width=224,
+                use_resnet_normalization=True,
+                mean=mean,
+                stdev=stdev,
+                uuid="rgb_lowres_arm_only_viz",
+            ),
+            RGBSensorStretchIntel(
+                height=224,
+                width=224,
+                use_resnet_normalization=True,
+                mean=mean,
+                stdev=stdev,
+                uuid="rgb_lowres_only_viz",
+            ),
+        ]
+
 
     TASK_SAMPLER = ProcTHORObjectNavTaskSampler
     TASK_TYPE = StretchObjectNavTask

manipulathor_baselines/procthor_baselines/experiments/ithor/obj_nav_2camera_procthor_wide.py

@@ -68,17 +68,37 @@ class ProcTHORObjectNavClipResnet50RGBOnly2CameraWideFOV(
     ]
 
     MAX_STEPS = 500
+    NUM_PROCESSES = 40
+    # NUM_TRAIN_HOUSES = 50 #TODO: house > num_proc oversampling not implemented yet
+
     if platform.system() == "Darwin":
         MAX_STEPS = 100
+        NUM_TRAIN_HOUSES = 100
+        SENSORS += [
+            RGBSensorStretchKinectBigFov(
+                height=224,
+                width=224,
+                use_resnet_normalization=True,
+                mean=mean,
+                stdev=stdev,
+                uuid="rgb_lowres_arm_only_viz",
+            ),
+            RGBSensorThor(
+                height=224,
+                width=224,
+                use_resnet_normalization=True,
+                mean=mean,
+                stdev=stdev,
+                uuid="rgb_lowres_only_viz",
+            ),
+        ]
 
     TASK_SAMPLER = ProcTHORObjectNavTaskSampler
     TASK_TYPE = StretchObjectNavTask
     ENVIRONMENT_TYPE = StretchManipulaTHOREnvironment
     POTENTIAL_VISUALIZERS = [StretchObjNavImageVisualizer, TestMetricLogger]
 
 
-    NUM_PROCESSES = 40
-    # NUM_TRAIN_HOUSES = 50 #TODO: house > num_proc oversampling not implemented yet
 
 
     def __init__(self):

manipulathor_baselines/procthor_baselines/experiments/ithor/obj_nav_2camera_robothor_narrow.py

@@ -79,6 +79,24 @@ class RobothorObjectNavClipResnet50RGBOnly2CameraNarrowFOV(
     MAX_STEPS = 500
     if platform.system() == "Darwin":
         MAX_STEPS = 500
+        SENSORS += [
+            RGBSensorStretchKinect(
+                height=224,
+                width=224,
+                use_resnet_normalization=True,
+                mean=mean,
+                stdev=stdev,
+                uuid="rgb_lowres_arm_only_viz",
+            ),
+            RGBSensorStretchIntel(
+                height=224,
+                width=224,
+                use_resnet_normalization=True,
+                mean=mean,
+                stdev=stdev,
+                uuid="rgb_lowres_only_viz",
+            ),
+        ]
 
     TASK_SAMPLER = AllRoomsObjectNavTaskSampler
     TASK_TYPE = StretchObjectNavTask

manipulathor_baselines/procthor_baselines/models/clip_objnav_ncamera_model.py

@@ -4,6 +4,8 @@
 Facebook's Habitat.
 """
 from typing import Tuple, Optional, List, Dict, cast
+import platform
+from datetime import datetime
 
 import gym
 import torch
@@ -16,6 +18,9 @@
     FusionType,
 )
 
+from manipulathor_utils.debugger_util import ForkedPdb
+from utils.hacky_viz_utils import hacky_visualization
+
 
 class ResnetTensorNavNCameraActorCritic(VisualNavActorCritic):
     def __init__(
@@ -75,6 +80,8 @@ def __init__(
 
         self.train()
 
+        self.starting_time = datetime.now().strftime("{}_%m_%d_%Y_%H_%M_%S_%f".format(self.__class__.__name__))
+
     @property
     def is_blind(self) -> bool:
         """True if the model is blind (e.g. neither 'depth' or 'rgb' is an
@@ -84,6 +91,11 @@ def is_blind(self) -> bool:
     def forward_encoder(self, observations: ObservationType) -> torch.FloatTensor:
         return self.goal_visual_encoder(observations)
 
+    def forward(self, **kwargs): #TODO NOW remove
+        if platform.system() == "Darwin":
+            hacky_visualization(kwargs['observations'], base_directory_to_right_images=self.starting_time)
+        return super(ResnetTensorNavNCameraActorCritic, self).forward(**kwargs)
+
 
 class ResnetNCameraTensorGoalEncoder(nn.Module):
     def __init__(

manipulathor_baselines/procthor_baselines/models/clip_preprocessors.py

@@ -10,7 +10,7 @@
 from allenact.base_abstractions.preprocessor import Preprocessor
 from allenact.utils.misc_utils import prepare_locals_for_super
 from manipulathor_utils.debugger_util import ForkedPdb
-from utils.hacky_viz_utils import hacky_visualization
+# from utils.hacky_viz_utils import hacky_visualization
 from datetime import datetime
 
 
@@ -113,9 +113,9 @@ def to(self, device: torch.device) -> "ClipResNetPreprocessor":
 
     def process(self, obs: Dict[str, Any], *args: Any, **kwargs: Any) -> Any:
         x = obs[self.input_uuids[0]].to(self.device).permute(0, 3, 1, 2)  # bhwc -> bchw
-        if self.visualize:
-            obs['rgb_lowres'] = x.permute(0,2,3,1).unsqueeze(0)
-            hacky_visualization(obs, base_directory_to_right_images=self.starting_time)
+        # if self.visualize:
+        #     obs['rgb_lowres'] = x.permute(0,2,3,1).unsqueeze(0)
+        #     hacky_visualization(obs, base_directory_to_right_images=self.starting_time)
         # If the input is depth, repeat it across all 3 channels
         if x.shape[1] == 1:
             x = x.repeat(1, 3, 1, 1)

utils/hacky_viz_utils.py

@@ -23,6 +23,8 @@ def unnormalize_image(img):
 
     if 'only_detection_rgb_lowres' in observations:
         viz_image = observations['only_detection_rgb_lowres']
+    elif 'rgb_lowres_only_viz' in observations:
+        viz_image = observations['rgb_lowres_only_viz']
     elif 'rgb_lowres' in observations:
         viz_image = observations['rgb_lowres']
     else:
@@ -61,6 +63,10 @@ def normalize_depth(depth):
             kinect_image = observations['rgb_lowres_arm'].squeeze(0).squeeze(0)
             kinect_image = unnormalize_image(kinect_image)
             list_of_visualizations.append(kinect_image)
+        if 'rgb_lowres_arm_only_viz' in observations:
+            kinect_image = observations['rgb_lowres_arm_only_viz'].squeeze(0).squeeze(0)
+            kinect_image = unnormalize_image(kinect_image)
+            list_of_visualizations.append(kinect_image)
         if 'depth_lowres_arm' in observations:
             arm_depth = normalize_depth(observations['depth_lowres_arm'])
             list_of_visualizations.append(arm_depth)