feat(viewer): add support of rendering pointcloud segmentation (#261)

ktro2828 · mojomex · SamratThapa120 · web-flow · commit a62b4b342905 · 2026-02-03T07:26:38.000+09:00
* feat: add support of rendering segmentation pointcloud Signed-off-by: ktro2828 <kotaro.uetake@tier4.jp> * feat: separate lidarseg method Signed-off-by: ktro2828 <kotaro.uetake@tier4.jp> * chore(viewer): lidarseg tweaks (#256) * chore(deps): restrict rerun-sdk to versions >=0.20.0,<0.28.0 (#248) * Bump version from 0.5.2 to 0.5.3 (#247) * Bump version from 0.5.2 to 0.5.3 * Pin rerun-sdk dependency to version 0.20.0 * refactor: update custom validator definition (#250) * refactor: update custom validator definition Signed-off-by: ktro2828 <kotaro.uetake@tier4.jp> * Update t4_devkit/schema/tables/base.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Signed-off-by: ktro2828 <kotaro.uetake@tier4.jp> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix: modity type of size to tuple[int, int] (#251) Signed-off-by: ktro2828 <kotaro.uetake@tier4.jp> * fix: abstract away category indexing differences Semseg and non-semseg datasets behave differently (position-based vs. explicit indexing). Now computing index field in case position-based indexing is used. Signed-off-by: Max SCHMELLER <max.schmeller@tier4.jp> * fix(rendering): don't fail when map is not present Signed-off-by: Max SCHMELLER <max.schmeller@tier4.jp> * fix(rendering): fix iteration freezing when skipping a pointcloud without lidarseg file Signed-off-by: Max SCHMELLER <max.schmeller@tier4.jp> * feat(rendering): auto-enable SEGMENTATION coloring when lidarseg is available Signed-off-by: Max SCHMELLER <max.schmeller@tier4.jp> * docs: checkmark for semseg viz Signed-off-by: Max SCHMELLER <max.schmeller@tier4.jp> * fix(rendering): fix wrongly named `color_mode` args Signed-off-by: Max SCHMELLER <max.schmeller@tier4.jp> * refactor(compatibility): move to schema package Signed-off-by: Max SCHMELLER <max.schmeller@tier4.jp> --------- Signed-off-by: ktro2828 <kotaro.uetake@tier4.jp> Signed-off-by: Max SCHMELLER <max.schmeller@tier4.jp> Co-authored-by: Kotaro Uetake <60615504+ktro2828@users.noreply.github.com> Co-authored-by: Samrat Thapa <38401989+SamratThapa120@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor: remove lidarseg command from t4viz Signed-off-by: ktro2828 <kotaro.uetake@tier4.jp> * refactor: replace lidarseg existence checke Signed-off-by: ktro2828 <kotaro.uetake@tier4.jp> * feat: add support of rendering segmentation pointcloud Signed-off-by: ktro2828 <kotaro.uetake@tier4.jp> * fix: resolve invalid usage of future submitting and enable to raise exceptions Signed-off-by: ktro2828 <kotaro.uetake@tier4.jp> * docs: add visualization sample Signed-off-by: ktro2828 <kotaro.uetake@tier4.jp> * refactor: remove RerunViewer.render_pointcloud(...) Signed-off-by: ktro2828 <kotaro.uetake@tier4.jp> --------- Signed-off-by: ktro2828 <kotaro.uetake@tier4.jp> Signed-off-by: Max SCHMELLER <max.schmeller@tier4.jp> Co-authored-by: Max Schmeller <6088931+mojomex@users.noreply.github.com> Co-authored-by: Samrat Thapa <38401989+SamratThapa120@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
diff --git a/README.md b/README.md
@@ -59,7 +59,7 @@ More details, please refer to [`t4viz` CLI](./docs/cli/t4viz.md) or [API usage](
 | Feature | Task                        | Support |
 | :------ | :-------------------------- | :-----: |
 | 3D      | 3D Boxes                    |   ✅    |
-|         | PointCloud Segmentation     |         |
+|         | PointCloud Segmentation     |   ✅    |
 |         | Raw PointCloud              |   ✅    |
 |         | 3D Trajectories             |   ✅    |
 |         | TF Links                    |   ✅    |
diff --git a/docs/assets/render_lidarseg.png b/docs/assets/render_lidarseg.png
diff --git a/docs/index.md b/docs/index.md
@@ -17,7 +17,7 @@ More details, please refer to [`t4viz` CLI](./cli/t4viz.md) or [API usage](./tut
 | Feature | Task                        | Support |
 | :------ | :-------------------------- | :-----: |
 | 3D      | 3D Boxes                    |   ✅    |
-|         | PointCloud Segmentation     |         |
+|         | PointCloud Segmentation     |   ✅    |
 |         | Raw PointCloud              |   ✅    |
 |         | 3D Trajectories             |   ✅    |
 |         | TF Links                    |   ✅    |
diff --git a/docs/tutorials/render.md b/docs/tutorials/render.md
@@ -134,15 +134,32 @@ It allows you to render boxes by specifying elements of boxes directly:
 
 ```python
 from t4_devkit.dataclass import LidarPointCloud
+from t4_devkit.viewer import PointCloudColorMode
 # Point cloud channel name
 >>> lidar_channel = "LIDAR_TOP"
 # Load point cloud from file
 >>> pointcloud = LidarPointCloud.from_file(<PATH_TO_POINTCLOUD.pcd.bin>)
->>> viewer.render_pointcloud(seconds, lidar_channel, pointcloud)
+>>> color_mode = PointCloudColorMode.DISTANCE
+>>> viewer.render_pointcloud(seconds, lidar_channel, pointcloud, color_mode)
 ```
 
 ![Render Point Cloud](../assets/render_pointcloud.png)
 
+### Rendering LiDAR segmentation
+
+```python
+from t4_devkit.dataclass import SegmentationPointCloud
+from t4_devkit.viewer import PointCloudColorMode
+# Point cloud channel name
+>>> lidar_channel = "LIDAR_TOP"
+# Load point cloud and label from file
+>>> pointcloud = SegmentationPointCloud.from_file("<PATH_TO_POINTCLOUD.pcd.bin>", "<PATH_TO_LABEL.pcd.bin>")
+>>> color_mode = PointCloudColorMode.SEGMENTATION
+>>> viewer.render_pointcloud(seconds, lidar_channel, pointcloud, color_mode)
+```
+
+![Render LiDAR Segmentation](../assets/render_lidarseg.png)
+
 ### Rendering lanelet map
 
 ```python
diff --git a/t4_devkit/helper/rendering.py b/t4_devkit/helper/rendering.py
@@ -13,7 +13,7 @@
 
 from t4_devkit.common.geometry import view_points
 from t4_devkit.common.timestamp import microseconds2seconds, seconds2microseconds
-from t4_devkit.dataclass import LidarPointCloud, RadarPointCloud
+from t4_devkit.dataclass import LidarPointCloud, RadarPointCloud, SegmentationPointCloud
 from t4_devkit.schema import SensorModality
 from t4_devkit.viewer import (
     PointCloudColorMode,
@@ -56,9 +56,17 @@ def __init__(self, t4: Tier4) -> None:
         self._label2id: dict[str, int] = {
             category.name: category.index for category in self._t4.category
         }
+        self._sample_data_to_lidarseg_filename: dict[str, str] | None = (
+            {lidarseg.sample_data_token: lidarseg.filename for lidarseg in self._t4.lidarseg}
+            if self._t4.lidarseg
+            else None
+        )
 
         self._executor = concurrent.futures.ThreadPoolExecutor()
 
+    def _has_lidarseg(self) -> bool:
+        return self._sample_data_to_lidarseg_filename is not None
+
     def _init_viewer(
         self,
         app_id: str,
@@ -122,17 +130,24 @@ def render_scene(
         app_id = f"scene@{self._t4.dataset_id}"
         viewer = self._init_viewer(app_id, render_ann=True, save_dir=save_dir)
 
-        self._render_map(viewer)
+        self._try_render_map(viewer)
 
         scene: Scene = self._t4.scene[0]
         first_sample: Sample = self._t4.get("sample", scene.first_sample_token)
         max_timestamp_us = first_sample.timestamp + seconds2microseconds(max_time_seconds)
 
-        concurrent.futures.wait(
+        pointcloud_color_mode = (
+            PointCloudColorMode.SEGMENTATION
+            if self._has_lidarseg()
+            else PointCloudColorMode.DISTANCE
+        )
+
+        futures = (
             self._render_lidar_and_ego(
                 viewer=viewer,
                 first_lidar_tokens=first_lidar_tokens,
                 max_timestamp_us=max_timestamp_us,
+                color_mode=pointcloud_color_mode,
             )
             + self._render_radars(
                 viewer=viewer,
@@ -146,23 +161,23 @@ def render_scene(
             )
             + [
                 self._executor.submit(
-                    self._render_annotation3ds(
-                        viewer=viewer,
-                        first_sample_token=scene.first_sample_token,
-                        max_timestamp_us=max_timestamp_us,
-                        future_seconds=future_seconds,
-                    )
+                    self._render_annotation3ds,
+                    viewer=viewer,
+                    first_sample_token=scene.first_sample_token,
+                    max_timestamp_us=max_timestamp_us,
+                    future_seconds=future_seconds,
                 ),
                 self._executor.submit(
-                    self._render_annotation2ds(
-                        viewer=viewer,
-                        first_sample_token=scene.first_sample_token,
-                        max_timestamp_us=max_timestamp_us,
-                    )
+                    self._render_annotation2ds,
+                    viewer=viewer,
+                    first_sample_token=scene.first_sample_token,
+                    max_timestamp_us=max_timestamp_us,
                 ),
             ]
         )
 
+        _handle_futures(futures)
+
     def render_instance(
         self,
         instance_token: str | Sequence[str],
@@ -221,13 +236,20 @@ def render_instance(
         app_id = f"instance@{self._t4.dataset_id}"
         viewer = self._init_viewer(app_id, render_ann=True, save_dir=save_dir)
 
-        self._render_map(viewer)
+        self._try_render_map(viewer)
+
+        pointcloud_color_mode = (
+            PointCloudColorMode.SEGMENTATION
+            if self._has_lidarseg()
+            else PointCloudColorMode.DISTANCE
+        )
 
-        concurrent.futures.wait(
+        futures = (
             self._render_lidar_and_ego(
                 viewer=viewer,
                 first_lidar_tokens=first_lidar_tokens,
                 max_timestamp_us=max_timestamp_us,
+                color_mode=pointcloud_color_mode,
             )
             + self._render_radars(
                 viewer=viewer,
@@ -241,25 +263,25 @@ def render_instance(
             )
             + [
                 self._executor.submit(
-                    self._render_annotation3ds(
-                        viewer=viewer,
-                        first_sample_token=first_sample.token,
-                        max_timestamp_us=max_timestamp_us,
-                        future_seconds=future_seconds,
-                        instance_tokens=instance_tokens,
-                    )
+                    self._render_annotation3ds,
+                    viewer=viewer,
+                    first_sample_token=first_sample.token,
+                    max_timestamp_us=max_timestamp_us,
+                    future_seconds=future_seconds,
+                    instance_tokens=instance_tokens,
                 ),
                 self._executor.submit(
-                    self._render_annotation2ds(
-                        viewer=viewer,
-                        first_sample_token=first_sample.token,
-                        max_timestamp_us=max_timestamp_us,
-                        instance_tokens=instance_tokens,
-                    )
+                    self._render_annotation2ds,
+                    viewer=viewer,
+                    first_sample_token=first_sample.token,
+                    max_timestamp_us=max_timestamp_us,
+                    instance_tokens=instance_tokens,
                 ),
-            ],
+            ]
         )
 
+        _handle_futures(futures)
+
     def render_pointcloud(
         self,
         *,
@@ -282,7 +304,7 @@ def render_pointcloud(
         app_id = f"pointcloud@{self._t4.dataset_id}"
         viewer = self._init_viewer(app_id, render_ann=False, save_dir=save_dir)
 
-        self._render_map(viewer)
+        self._try_render_map(viewer)
 
         # search first lidar sample data token
         first_lidar_token: str | None = None
@@ -299,22 +321,24 @@ def render_pointcloud(
             max_time_seconds
         )
 
-        concurrent.futures.wait(
-            self._render_lidar_and_ego(
-                viewer=viewer,
-                first_lidar_tokens=[first_lidar_token],
-                max_timestamp_us=max_timestamp_us,
-            )
-            + self._render_points_on_cameras(
-                first_point_sample_data_token=first_lidar_token,
-                max_timestamp_us=max_timestamp_us,
-                min_dist=1.0,
-                ignore_distortion=ignore_distortion,
-            ),
+        # TODO: support rendering segmentation pointcloud on camera
+        futures = self._render_lidar_and_ego(
+            viewer=viewer,
+            first_lidar_tokens=[first_lidar_token],
+            max_timestamp_us=max_timestamp_us,
+        ) + self._render_points_on_cameras(
+            first_point_sample_data_token=first_lidar_token,
+            max_timestamp_us=max_timestamp_us,
+            min_dist=1.0,
+            ignore_distortion=ignore_distortion,
         )
 
-    def _render_map(self, viewer: RerunViewer) -> None:
+        _handle_futures(futures)
+
+    def _try_render_map(self, viewer: RerunViewer) -> None:
         lanelet_path = osp.join(self._t4.map_dir, "lanelet2_map.osm")
+        if not osp.exists(lanelet_path):
+            return
         viewer.render_map(lanelet_path)
 
     def _render_sensor_calibration(self, viewer: RerunViewer, sample_data_token: str) -> None:
@@ -337,6 +361,7 @@ def _render_lidar_and_ego(
         viewer: RerunViewer,
         first_lidar_tokens: list[str],
         max_timestamp_us: float,
+        *,
         color_mode: PointCloudColorMode = PointCloudColorMode.DISTANCE,
     ) -> list[Future]:
         def _render_single_lidar(first_lidar_token: str) -> None:
@@ -345,25 +370,39 @@ def _render_single_lidar(first_lidar_token: str) -> None:
             current_lidar_token = first_lidar_token
             while current_lidar_token != "":
                 sample_data: SampleData = self._t4.get("sample_data", current_lidar_token)
+                current_lidar_token = sample_data.next
 
                 if max_timestamp_us < sample_data.timestamp:
                     break
 
                 ego_pose: EgoPose = self._t4.get("ego_pose", sample_data.ego_pose_token)
                 viewer.render_ego(ego_pose=ego_pose)
 
-                pointcloud = LidarPointCloud.from_file(
-                    osp.join(self._t4.data_root, sample_data.filename)
-                )
+                # render segmentation pointcloud if available, otherwise render raw pointcloud
+                if color_mode == PointCloudColorMode.SEGMENTATION:
+                    if not (
+                        self._has_lidarseg()
+                        and sample_data.token in self._sample_data_to_lidarseg_filename
+                    ):
+                        continue
+
+                    label_filename = self._sample_data_to_lidarseg_filename[sample_data.token]
+                    pointcloud = SegmentationPointCloud.from_file(
+                        point_filepath=osp.join(self._t4.data_root, sample_data.filename),
+                        label_filepath=osp.join(self._t4.data_root, label_filename),
+                    )
+                else:
+                    pointcloud = LidarPointCloud.from_file(
+                        osp.join(self._t4.data_root, sample_data.filename)
+                    )
+
                 viewer.render_pointcloud(
                     seconds=microseconds2seconds(sample_data.timestamp),
                     channel=sample_data.channel,
                     pointcloud=pointcloud,
                     color_mode=color_mode,
                 )
 
-                current_lidar_token = sample_data.next
-
         return [self._executor.submit(_render_single_lidar, token) for token in first_lidar_tokens]
 
     def _render_radars(
@@ -699,3 +738,19 @@ def _append_mask(
         camera_masks[camera]["class_ids"] = [class_id]
         camera_masks[camera]["uuids"] = [uuid]
     return camera_masks
+
+
+def _handle_futures(futures: list[Future]) -> None:
+    """Wait for all futures and raise exception if any.
+
+    Args:
+        futures (list[Future]): List of futures.
+    """
+    if not futures:
+        return
+
+    concurrent.futures.wait(futures)
+    for future in futures:
+        if not future.done():
+            continue
+        future.result()
diff --git a/t4_devkit/viewer/color.py b/t4_devkit/viewer/color.py
@@ -20,6 +20,7 @@ class PointCloudColorMode(str, Enum):
 
     DISTANCE = "distance"
     INTENSITY = "intensity"
+    SEGMENTATION = "segmentation"
 
 
 def pointcloud_color(
diff --git a/t4_devkit/viewer/viewer.py b/t4_devkit/viewer/viewer.py
@@ -9,6 +9,7 @@
 
 from t4_devkit.common.converter import to_quaternion
 from t4_devkit.common.timestamp import microseconds2seconds
+from t4_devkit.dataclass import SegmentationPointCloud
 from t4_devkit.lanelet import LaneletParser
 from t4_devkit.schema import SensorModality
 
@@ -24,7 +25,7 @@
 from .record import BatchBox2D, BatchBox3D, BatchSegmentation2D
 
 if TYPE_CHECKING:
-    from t4_devkit.dataclass import Box2D, Box3D, Future, PointCloudLike
+    from t4_devkit.dataclass import Box2D, Box3D, Future, PointCloudLike, SegmentationPointCloud
     from t4_devkit.schema import CalibratedSensor, EgoPose, Sensor
     from t4_devkit.typing import (
         CameraIntrinsicLike,
@@ -426,11 +427,18 @@ def render_pointcloud(
         # TODO(ktro2828): add support of rendering pointcloud on images
         rr.set_time_seconds(self.config.timeline, seconds)
 
-        colors = pointcloud_color(pointcloud, color_mode=color_mode)
-        rr.log(
-            format_entity(self.config.ego_entity, channel),
-            rr.Points3D(pointcloud.points[:3].T, colors=colors),
-        )
+        if color_mode == PointCloudColorMode.SEGMENTATION:
+            assert isinstance(pointcloud, SegmentationPointCloud)
+            rr.log(
+                format_entity(self.config.ego_entity, channel),
+                rr.Points3D(pointcloud.points[:3].T, class_ids=pointcloud.labels),
+            )
+        else:
+            colors = pointcloud_color(pointcloud, color_mode=color_mode)
+            rr.log(
+                format_entity(self.config.ego_entity, channel),
+                rr.Points3D(pointcloud.points[:3].T, colors=colors),
+            )
 
     @_check_spatial2d
     def render_image(self, seconds: float, camera: str, image: str | NDArrayU8) -> None:
