From c357ebed60fcec8354bd45a02e36284c43fa1f91 Mon Sep 17 00:00:00 2001 From: Pascal Roth Date: Wed, 28 Aug 2024 10:24:52 +0200 Subject: [PATCH 1/6] rename depth --- source/extensions/omni.isaac.lab/docs/CHANGELOG.rst | 7 +++++++ .../omni/isaac/lab/sensors/camera/tiled_camera.py | 10 +++++----- .../omni.isaac.lab/test/sensors/test_tiled_camera.py | 6 +++--- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/source/extensions/omni.isaac.lab/docs/CHANGELOG.rst b/source/extensions/omni.isaac.lab/docs/CHANGELOG.rst index dbb34b986d..3c007bdc3c 100644 --- a/source/extensions/omni.isaac.lab/docs/CHANGELOG.rst +++ b/source/extensions/omni.isaac.lab/docs/CHANGELOG.rst @@ -1,3 +1,10 @@ +Changed +^^^^^^^ + +* Changed the data type "depth" to "distance_to_camera" in :class:`omni.isaac.lab.sensors.TiledCamera` class to be + consistent with all other cameras. + + Changelog --------- diff --git a/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py b/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py index c2f7197225..4f85e60a5e 100644 --- a/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py +++ b/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py @@ -37,7 +37,7 @@ class TiledCamera(Camera): The following sensor types are supported: - ``"rgb"``: A rendered color image. - - ``"depth"``: An image containing the distance to camera optical center. + - ``"distance_to_camera"``: An image containing the distance to camera optical center. .. attention:: Please note that the fidelity of RGB images may be lower than the standard camera sensor due to the @@ -54,7 +54,7 @@ class TiledCamera(Camera): cfg: TiledCameraCfg """The configuration parameters.""" - SUPPORTED_TYPES: set[str] = {"rgb", "depth"} + SUPPORTED_TYPES: set[str] = {"rgb", "distance_to_camera"} """The set of sensor types that are supported.""" def __init__(self, cfg: TiledCameraCfg): @@ -198,7 +198,7 @@ def _update_buffers_impl(self, env_ids: Sequence[int]): wp.from_torch(self._data.output[data_type]), # zero-copy alias *list(self._data.output[data_type].shape[1:]), # height, width, num_channels self._tiling_grid_shape()[0], # num_tiles_x - offset if data_type == "depth" else 0, + offset if data_type == "distance_to_camera" else 0, ], device=self.device, ) @@ -232,8 +232,8 @@ def _create_buffers(self): data_dict["rgb"] = torch.zeros( (self._view.count, self.cfg.height, self.cfg.width, 3), device=self.device ).contiguous() - if "depth" in self.cfg.data_types: - data_dict["depth"] = torch.zeros( + if "distance_to_camera" in self.cfg.data_types: + data_dict["distance_to_camera"] = torch.zeros( (self._view.count, self.cfg.height, self.cfg.width, 1), device=self.device ).contiguous() self._data.output = TensorDict(data_dict, batch_size=self._view.count, device=self.device) diff --git a/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py b/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py index 627ea7dd33..ed875b693a 100644 --- a/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py +++ b/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py @@ -43,7 +43,7 @@ def setUp(self): offset=TiledCameraCfg.OffsetCfg(pos=(0.0, 0.0, 4.0), rot=(0.0, 0.0, 1.0, 0.0), convention="ros"), prim_path="/World/Camera", update_period=0, - data_types=["rgb", "depth"], + data_types=["rgb", "distance_to_camera"], spawn=sim_utils.PinholeCameraCfg( focal_length=24.0, focus_distance=400.0, horizontal_aperture=20.955, clipping_range=(0.1, 1.0e5) ), @@ -224,7 +224,7 @@ def test_depth_only_camera(self): # Create camera camera_cfg = copy.deepcopy(self.camera_cfg) - camera_cfg.data_types = ["depth"] + camera_cfg.data_types = ["distance_to_camera"] camera_cfg.prim_path = "/World/Origin_.*/CameraSensor" camera = TiledCamera(camera_cfg) # Check simulation parameter is set correctly @@ -236,7 +236,7 @@ def test_depth_only_camera(self): # Check if camera prim is set correctly and that it is a camera prim self.assertEqual(camera._sensor_prims[1].GetPath().pathString, "/World/Origin_01/CameraSensor") self.assertIsInstance(camera._sensor_prims[0], UsdGeom.Camera) - self.assertListEqual(list(camera.data.output.keys()), ["depth"]) + self.assertListEqual(list(camera.data.output.keys()), ["distance_to_camera"]) # Simulate for a few steps # note: This is a workaround to ensure that the textures are loaded. From d380a0bb72a8731a184e8453703d90e170142451 Mon Sep 17 00:00:00 2001 From: Pascal Roth Date: Wed, 28 Aug 2024 11:20:18 +0200 Subject: [PATCH 2/6] make both depth and distance_to_camera work --- .../isaac/lab/sensors/camera/tiled_camera.py | 26 +++++++++++++----- .../test/sensors/test_tiled_camera.py | 27 +++++++++++++++++++ .../direct/cartpole/cartpole_camera_env.py | 4 +-- source/standalone/demos/cameras.py | 8 +++--- 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py b/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py index 4f85e60a5e..9ac42ed238 100644 --- a/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py +++ b/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py @@ -6,6 +6,7 @@ from __future__ import annotations import math +import carb import numpy as np import torch from collections.abc import Sequence @@ -38,6 +39,7 @@ class TiledCamera(Camera): - ``"rgb"``: A rendered color image. - ``"distance_to_camera"``: An image containing the distance to camera optical center. + - ``"depth"``: An alias for ``"distance_to_camera"``. .. attention:: Please note that the fidelity of RGB images may be lower than the standard camera sensor due to the @@ -54,7 +56,7 @@ class TiledCamera(Camera): cfg: TiledCameraCfg """The configuration parameters.""" - SUPPORTED_TYPES: set[str] = {"rgb", "distance_to_camera"} + SUPPORTED_TYPES: set[str] = {"rgb", "distance_to_camera", "depth"} """The set of sensor types that are supported.""" def __init__(self, cfg: TiledCameraCfg): @@ -157,12 +159,21 @@ def _initialize_impl(self): # start the orchestrator (if not already started) rep.orchestrator._orchestrator._is_started = True + # check the data_types and remove "depth" if "distance_to_camera" is requested too + if "depth" in self.cfg.data_types and "distance_to_camera" in self.cfg.data_types: + carb.log_warn("Both 'depth' and 'distance_to_camera' are requested which are the same. 'depth' will be ignored.") + self.cfg.data_types.remove("depth") + # NOTE: internally, "distance_to_camera" is named "depth", name is adjusted here + data_type = self.cfg.data_types.copy() + if "distance_to_camera" in data_type: + data_type.remove("distance_to_camera") + data_type.append("depth") # Create a tiled sensor from the camera prims rep_sensor = rep.create.tiled_sensor( cameras=self._view.prim_paths, camera_resolution=[self.image_shape[1], self.image_shape[0]], tiled_resolution=self._tiled_image_shape(), - output_types=self.cfg.data_types, + output_types=data_type, ) # Get render product render_prod_path = rep.create.render_product(camera=rep_sensor, resolution=self._tiled_image_shape()) @@ -198,7 +209,7 @@ def _update_buffers_impl(self, env_ids: Sequence[int]): wp.from_torch(self._data.output[data_type]), # zero-copy alias *list(self._data.output[data_type].shape[1:]), # height, width, num_channels self._tiling_grid_shape()[0], # num_tiles_x - offset if data_type == "distance_to_camera" else 0, + offset if data_type == "distance_to_camera" or data_type == "depth" else 0, ], device=self.device, ) @@ -232,10 +243,11 @@ def _create_buffers(self): data_dict["rgb"] = torch.zeros( (self._view.count, self.cfg.height, self.cfg.width, 3), device=self.device ).contiguous() - if "distance_to_camera" in self.cfg.data_types: - data_dict["distance_to_camera"] = torch.zeros( - (self._view.count, self.cfg.height, self.cfg.width, 1), device=self.device - ).contiguous() + for data_type in ["distance_to_camera", "depth"]: + if data_type in self.cfg.data_types: + data_dict[data_type] = torch.zeros( + (self._view.count, self.cfg.height, self.cfg.width, 1), device=self.device + ).contiguous() self._data.output = TensorDict(data_dict, batch_size=self._view.count, device=self.device) def _tiled_image_shape(self) -> tuple[int, int]: diff --git a/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py b/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py index ed875b693a..2a540820f5 100644 --- a/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py +++ b/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py @@ -216,6 +216,33 @@ def test_rgb_only_camera(self): self.assertGreater(im_data[1].mean().item(), 0.0) del camera + def test_data_types(self): + """Test single camera initialization.""" + # Create camera + camera_cfg_distance = copy.deepcopy(self.camera_cfg) + camera_cfg_distance.data_types = ["distance_to_camera"] + camera_cfg_distance.prim_path = "/World/CameraDistance" + camera_distance = TiledCamera(camera_cfg_distance) + camera_cfg_depth = copy.deepcopy(self.camera_cfg) + camera_cfg_depth.data_types = ["depth"] + camera_cfg_depth.prim_path = "/World/CameraDepth" + camera_depth = TiledCamera(camera_cfg_depth) + camera_cfg_both = copy.deepcopy(self.camera_cfg) + camera_cfg_both.data_types = ["distance_to_camera", "depth"] + camera_cfg_both.prim_path = "/World/CameraBoth" + camera_both = TiledCamera(camera_cfg_both) + # Play sim + self.sim.reset() + # Check if camera is initialized + self.assertTrue(camera_distance.is_initialized) + self.assertTrue(camera_depth.is_initialized) + self.assertTrue(camera_both.is_initialized) + self.assertListEqual(list(camera_distance.data.output.keys()), ["distance_to_camera"]) + self.assertListEqual(list(camera_depth.data.output.keys()), ["depth"]) + self.assertListEqual(list(camera_both.data.output.keys()), ["distance_to_camera"]) + + del camera_distance, camera_depth, camera_both + def test_depth_only_camera(self): """Test initialization with only depth.""" diff --git a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/direct/cartpole/cartpole_camera_env.py b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/direct/cartpole/cartpole_camera_env.py index e962ffc772..b23f1f97bd 100644 --- a/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/direct/cartpole/cartpole_camera_env.py +++ b/source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/direct/cartpole/cartpole_camera_env.py @@ -80,7 +80,7 @@ class CartpoleDepthCameraEnvCfg(CartpoleRGBCameraEnvCfg): tiled_camera: TiledCameraCfg = TiledCameraCfg( prim_path="/World/envs/env_.*/Camera", offset=TiledCameraCfg.OffsetCfg(pos=(-7.0, 0.0, 3.0), rot=(0.9945, 0.0, 0.1045, 0.0), convention="world"), - data_types=["depth"], + data_types=["distance_to_camera"], spawn=sim_utils.PinholeCameraCfg( focal_length=24.0, focus_distance=400.0, horizontal_aperture=20.955, clipping_range=(0.1, 20.0) ), @@ -172,7 +172,7 @@ def _apply_action(self) -> None: self._cartpole.set_joint_effort_target(self.actions, joint_ids=self._cart_dof_idx) def _get_observations(self) -> dict: - data_type = "rgb" if "rgb" in self.cfg.tiled_camera.data_types else "depth" + data_type = "rgb" if "rgb" in self.cfg.tiled_camera.data_types else "distance_to_camera" observations = {"policy": self._tiled_camera.data.output[data_type].clone()} if self.cfg.write_image_to_file: diff --git a/source/standalone/demos/cameras.py b/source/standalone/demos/cameras.py index 65159bc05d..7f140c9a9c 100644 --- a/source/standalone/demos/cameras.py +++ b/source/standalone/demos/cameras.py @@ -96,7 +96,7 @@ class SensorsSceneCfg(InteractiveSceneCfg): update_period=0.1, height=480, width=640, - data_types=["rgb", "depth"], + data_types=["rgb", "distance_to_camera"], spawn=None, # the camera is already spawned in the scene offset=TiledCameraCfg.OffsetCfg(pos=(0.510, 0.0, 0.015), rot=(0.5, -0.5, 0.5, -0.5), convention="ros"), ) @@ -221,7 +221,7 @@ def run_simulator(sim: sim_utils.SimulationContext, scene: InteractiveScene): print("-------------------------------") print(scene["tiled_camera"]) print("Received shape of rgb image: ", scene["tiled_camera"].data.output["rgb"].shape) - print("Received shape of depth image: ", scene["tiled_camera"].data.output["depth"].shape) + print("Received shape of depth image: ", scene["tiled_camera"].data.output["distance_to_camera"].shape) print("-------------------------------") print(scene["raycast_camera"]) print("Received shape of depth: ", scene["raycast_camera"].data.output["distance_to_image_plane"].shape) @@ -242,7 +242,7 @@ def run_simulator(sim: sim_utils.SimulationContext, scene: InteractiveScene): # compare generated Depth images across different cameras depth_images = [ scene["camera"].data.output["distance_to_image_plane"][0], - scene["tiled_camera"].data.output["depth"][0, ..., 0], + scene["tiled_camera"].data.output["distance_to_camera"][0, ..., 0], scene["raycast_camera"].data.output["distance_to_image_plane"][0], ] save_images_grid( @@ -250,7 +250,7 @@ def run_simulator(sim: sim_utils.SimulationContext, scene: InteractiveScene): cmap="turbo", subtitles=["Camera", "TiledCamera", "RaycasterCamera"], title="Depth Image: Cam0", - filename=os.path.join(output_dir, "depth", f"{count:04d}.jpg"), + filename=os.path.join(output_dir, "distance_to_camera", f"{count:04d}.jpg"), ) # save all tiled RGB images From 77a896dc09098743d964e61af5db28702c3abcd0 Mon Sep 17 00:00:00 2001 From: Pascal Roth Date: Wed, 28 Aug 2024 11:20:57 +0200 Subject: [PATCH 3/6] formatter --- .../omni/isaac/lab/sensors/camera/tiled_camera.py | 6 ++++-- .../omni.isaac.lab/test/sensors/test_tiled_camera.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py b/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py index 9ac42ed238..46ae1a4a27 100644 --- a/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py +++ b/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py @@ -6,13 +6,13 @@ from __future__ import annotations import math -import carb import numpy as np import torch from collections.abc import Sequence from tensordict import TensorDict from typing import TYPE_CHECKING, Any +import carb import omni.usd import warp as wp from omni.isaac.core.prims import XFormPrimView @@ -161,7 +161,9 @@ def _initialize_impl(self): rep.orchestrator._orchestrator._is_started = True # check the data_types and remove "depth" if "distance_to_camera" is requested too if "depth" in self.cfg.data_types and "distance_to_camera" in self.cfg.data_types: - carb.log_warn("Both 'depth' and 'distance_to_camera' are requested which are the same. 'depth' will be ignored.") + carb.log_warn( + "Both 'depth' and 'distance_to_camera' are requested which are the same. 'depth' will be ignored." + ) self.cfg.data_types.remove("depth") # NOTE: internally, "distance_to_camera" is named "depth", name is adjusted here data_type = self.cfg.data_types.copy() diff --git a/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py b/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py index 2a540820f5..d10926b414 100644 --- a/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py +++ b/source/extensions/omni.isaac.lab/test/sensors/test_tiled_camera.py @@ -240,7 +240,7 @@ def test_data_types(self): self.assertListEqual(list(camera_distance.data.output.keys()), ["distance_to_camera"]) self.assertListEqual(list(camera_depth.data.output.keys()), ["depth"]) self.assertListEqual(list(camera_both.data.output.keys()), ["distance_to_camera"]) - + del camera_distance, camera_depth, camera_both def test_depth_only_camera(self): From 831d566f021342cd5e746301f7f7c4a19d0061af Mon Sep 17 00:00:00 2001 From: Pascal Roth Date: Wed, 28 Aug 2024 11:22:18 +0200 Subject: [PATCH 4/6] changelog --- source/extensions/omni.isaac.lab/docs/CHANGELOG.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/extensions/omni.isaac.lab/docs/CHANGELOG.rst b/source/extensions/omni.isaac.lab/docs/CHANGELOG.rst index 3c007bdc3c..7f8cae5bce 100644 --- a/source/extensions/omni.isaac.lab/docs/CHANGELOG.rst +++ b/source/extensions/omni.isaac.lab/docs/CHANGELOG.rst @@ -1,8 +1,8 @@ -Changed -^^^^^^^ +Added +^^^^^ -* Changed the data type "depth" to "distance_to_camera" in :class:`omni.isaac.lab.sensors.TiledCamera` class to be - consistent with all other cameras. +* Added alternative data type "distance_to_camera" in :class:`omni.isaac.lab.sensors.TiledCamera` class to be + consistent with all other cameras (equal to type "depth"). Changelog From bb8307c7125427aa4a5f14a8d76a21a94e419226 Mon Sep 17 00:00:00 2001 From: Pascal Date: Thu, 29 Aug 2024 17:20:54 +0200 Subject: [PATCH 5/6] add note --- .../omni/isaac/lab/sensors/camera/tiled_camera.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py b/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py index 46ae1a4a27..447578113f 100644 --- a/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py +++ b/source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py @@ -57,7 +57,11 @@ class TiledCamera(Camera): """The configuration parameters.""" SUPPORTED_TYPES: set[str] = {"rgb", "distance_to_camera", "depth"} - """The set of sensor types that are supported.""" + """The set of sensor types that are supported. + + .. note:: + The ``"depth"`` type is an alias for ``"distance_to_camera"``. + """ def __init__(self, cfg: TiledCameraCfg): """Initializes the tiled camera sensor. From fc577ca382d0d462a8733e337a99a95fb0655e99 Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 3 Sep 2024 15:22:12 +0200 Subject: [PATCH 6/6] changlog --- source/extensions/omni.isaac.lab/config/extension.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/omni.isaac.lab/config/extension.toml b/source/extensions/omni.isaac.lab/config/extension.toml index 4265d7cd07..66d17de023 100644 --- a/source/extensions/omni.isaac.lab/config/extension.toml +++ b/source/extensions/omni.isaac.lab/config/extension.toml @@ -1,7 +1,7 @@ [package] # Note: Semantic Versioning is used: https://semver.org/ -version = "0.22.4" +version = "0.22.6" # Description title = "Isaac Lab framework for Robot Learning"