Change CameraData.info from list[dict] to flat dict[str, Any] to match Replicator annotator output

Author: nvsekkin

scripts/tutorials/04_sensors/run_ray_caster_camera.py

@@ -132,12 +132,10 @@ def run_simulator(sim: sim_utils.SimulationContext, scene_entities: dict):
             single_cam_data = convert_dict_to_backend(
                 {k: v[camera_index] for k, v in camera.data.output.items()}, backend="numpy"
             )
-            # Extract the other information
-            single_cam_info = camera.data.info[camera_index]
-
             # Pack data back into replicator format to save them using its writer
             rep_output = {"annotators": {}}
-            for key, data, info in zip(single_cam_data.keys(), single_cam_data.values(), single_cam_info.values()):
+            for key, data in single_cam_data.items():
+                info = camera.data.info.get(key)
                 if info is not None:
                     rep_output["annotators"][key] = {"render_product": {"data": data, **info}}
                 else:

scripts/tutorials/04_sensors/run_usd_camera.py

@@ -232,12 +232,10 @@ def run_simulator(sim: sim_utils.SimulationContext, scene_entities: dict):
                 {k: v[camera_index] for k, v in camera.data.output.items()}, backend="numpy"
             )
 
-            # Extract the other information
-            single_cam_info = camera.data.info[camera_index]
-
             # Pack data back into replicator format to save them using its writer
             rep_output = {"annotators": {}}
-            for key, data, info in zip(single_cam_data.keys(), single_cam_data.values(), single_cam_info.values()):
+            for key, data in single_cam_data.items():
+                info = camera.data.info.get(key)
                 if info is not None:
                     rep_output["annotators"][key] = {"render_product": {"data": data, **info}}
                 else:

source/isaaclab/docs/CHANGELOG.rst

@@ -11,11 +11,13 @@ Changed
   into a single implementation. :class:`Camera` now delegates all rendering to the
   :class:`~isaaclab.renderers.Renderer` abstraction (same approach :class:`TiledCamera` used).
   The public API is unchanged for :class:`Camera` users.
-* **Breaking (TiledCamera users):** :attr:`~isaaclab.sensors.camera.CameraData.info` now correctly
-  returns ``list[dict[str, Any]]`` (per-camera, then per-data-type) as documented in
-  :class:`~isaaclab.sensors.camera.CameraData`. :class:`TiledCamera` previously returned a flat
-  ``dict``, which violated the documented contract. Migration: replace
-  ``camera.data.info[data_type]`` with ``camera.data.info[cam_idx][data_type]``.
+* **Breaking:** :attr:`~isaaclab.sensors.camera.CameraData.info` is now a flat
+  ``dict[str, Any]`` keyed by data type (e.g. ``camera.data.info["semantic_segmentation"]``).
+  The metadata is shared across all cameras and identical to what the underlying renderer returns.
+
+  - **Camera users (old):** replace ``camera.data.info[cam_idx][data_type]`` with
+    ``camera.data.info[data_type]``.
+  - **TiledCamera users (old):** access pattern ``camera.data.info[data_type]`` is unchanged.
 
 Deprecated
 ^^^^^^^^^^

source/isaaclab/isaaclab/renderers/base_renderer.py

@@ -97,7 +97,8 @@ def read_output(self, render_data: Any, output_name: str, camera_data: Any) -> N
         """Read a named output from the renderer into the camera data container.
 
         Implementations should populate ``camera_data.output[output_name]`` with
-        rendered data and, when available, other metadata (ex. ``camera_data.info``).
+        rendered data and, when available, ``camera_data.info[output_name]`` with
+        annotator metadata (e.g. ``idToLabels`` for segmentation types).
 
         Args:
             render_data: The render data object from :meth:`create_render_data`.

source/isaaclab/isaaclab/sensors/camera/camera.py

@@ -567,7 +567,7 @@ def _create_buffers(self):
                 ).contiguous()
 
         self._data.output = data_dict
-        self._data.info = [{name: None for name in self.cfg.data_types} for _ in range(self._view.count)]
+        self._data.info = {name: None for name in self.cfg.data_types}
         self._renderer.set_outputs(self._render_data, self._data.output)
 
     def _update_intrinsic_matrices(self, env_ids: Sequence[int]):

source/isaaclab/isaaclab/sensors/camera/camera_data.py

@@ -59,7 +59,7 @@ class CameraData:
     .. _Replicator Documentation: https://docs.omniverse.nvidia.com/prod_extensions/prod_extensions/ext_replicator/annotators_details.html#annotator-output
     """
 
-    info: list[dict[str, Any]] = None
+    info: dict[str, Any] = None
     """The retrieved sensor info with sensor types as key.
 
     This contains extra information provided by the sensor such as semantic segmentation label mapping, prim paths.

source/isaaclab/isaaclab/sensors/ray_caster/ray_caster_camera.py

@@ -376,7 +376,7 @@ def _create_buffers(self):
         # -- output data
         # create the buffers to store the annotator data.
         self._data.output = {}
-        self._data.info = [{name: None for name in self.cfg.data_types}] * self._view.count
+        self._data.info = {name: None for name in self.cfg.data_types}
         for name in self.cfg.data_types:
             if name in ["distance_to_image_plane", "distance_to_camera"]:
                 shape = (self.cfg.pattern_cfg.height, self.cfg.pattern_cfg.width, 1)

source/isaaclab/test/sensors/test_camera.py

@@ -109,7 +109,7 @@ def test_camera_init(setup_sim_camera):
     assert camera.data.quat_w_opengl.shape == (1, 4)
     assert camera.data.intrinsic_matrices.shape == (1, 3, 3)
     assert camera.data.image_shape == (camera_cfg.height, camera_cfg.width)
-    assert camera.data.info == [{camera_cfg.data_types[0]: None}]
+    assert camera.data.info == {camera_cfg.data_types[0]: None}
 
     # Simulate physics
     for _ in range(10):
@@ -939,9 +939,9 @@ def test_camera_all_annotators(setup_camera_device, device):
     assert output["semantic_segmentation"].dtype == torch.uint8
     assert output["instance_segmentation_fast"].dtype == torch.uint8
     assert output["instance_id_segmentation_fast"].dtype == torch.uint8
-    assert isinstance(info[0]["semantic_segmentation"], dict)
-    assert isinstance(info[0]["instance_segmentation_fast"], dict)
-    assert isinstance(info[0]["instance_id_segmentation_fast"], dict)
+    assert isinstance(info["semantic_segmentation"], dict)
+    assert isinstance(info["instance_segmentation_fast"], dict)
+    assert isinstance(info["instance_id_segmentation_fast"], dict)
 
     del camera
 
@@ -972,7 +972,7 @@ def test_camera_segmentation_non_colorize(setup_camera_device, device):
     for seg_type in camera_cfg.data_types:
         assert camera.data.output[seg_type].shape == (num_cameras, camera_cfg.height, camera_cfg.width, 1)
         assert camera.data.output[seg_type].dtype == torch.int32
-        assert isinstance(camera.data.info[0][seg_type], dict)
+        assert isinstance(camera.data.info[seg_type], dict)
 
     del camera
 

source/isaaclab/test/sensors/test_multi_mesh_ray_caster_camera.py

@@ -138,7 +138,7 @@ def test_camera_init(setup_simulation):
     assert camera.data.quat_w_opengl.shape == (1, 4)
     assert camera.data.intrinsic_matrices.shape == (1, 3, 3)
     assert camera.data.image_shape == (camera_cfg.pattern_cfg.height, camera_cfg.pattern_cfg.width)
-    assert camera.data.info == [{camera_cfg.data_types[0]: None}]
+    assert camera.data.info == {camera_cfg.data_types[0]: None}
     # Simulate physics
     for _ in range(10):
         # perform rendering

source/isaaclab/test/sensors/test_multi_tiled_camera.py

@@ -245,9 +245,9 @@ def test_all_annotators_multi_tiled_camera(setup_camera):
         assert output["semantic_segmentation"].dtype == torch.uint8
         assert output["instance_segmentation_fast"].dtype == torch.uint8
         assert output["instance_id_segmentation_fast"].dtype == torch.uint8
-        assert isinstance(info[0]["semantic_segmentation"], dict)
-        assert isinstance(info[0]["instance_segmentation_fast"], dict)
-        assert isinstance(info[0]["instance_id_segmentation_fast"], dict)
+        assert isinstance(info["semantic_segmentation"], dict)
+        assert isinstance(info["instance_segmentation_fast"], dict)
+        assert isinstance(info["instance_id_segmentation_fast"], dict)
 
     for camera in tiled_cameras:
         del camera