Record Newton markers and overlays in videos

docs/source/api/index.rst

@@ -154,6 +154,7 @@ The following modules are available in the ``isaaclab_newton`` extension:
    renderers
    sensors
    sim.schemas
+   video_recording
 
 .. toctree::
    :hidden:

docs/source/api/lab_newton/isaaclab_newton.video_recording.rst

@@ -0,0 +1,4 @@
+isaaclab\_newton.video\_recording
+=================================
+
+.. automodule:: isaaclab_newton.video_recording

docs/source/how-to/record_video.rst

@@ -80,6 +80,32 @@ precedence and only one ``--video`` stream is recorded. Rerun records ``.rrd`` r
 the Rerun visualizer rather than producing ``--video`` clips, and Viser does not currently provide a
 ``--video`` recording backend.
 
+When the Newton visualizer selects the backend, video capture mirrors its effective
+``visible_env_indices`` and ``max_visible_envs`` selection. For example,
+``visible_env_indices=[0, 1, 2, 3]`` makes both the live Newton view and the recorded clip contain
+only those four simulation worlds. This mirroring does not apply when
+``VideoRecorderCfg.backend_source = "renderer"``, because renderer-selected capture is independent
+of active visualizers.
+
+When ``scene.env_spacing`` is zero, selected worlds still share the same simulated coordinates.
+Set :attr:`~isaaclab_visualizers.newton.NewtonVisualizerCfg.world_spacing` to add visual-only
+offsets [m]. The recorder mirrors this setting, so four worlds can be arranged in a compact 2-by-2
+grid without changing physics:
+
+.. code-block:: python
+
+    from isaaclab_visualizers.newton import NewtonVisualizerCfg
+
+    NewtonVisualizerCfg(
+        visible_env_indices=[0, 1, 2, 3],
+        world_spacing=(2.0, 2.0, 0.0),
+    )
+
+The Newton GL recorder also renders active :class:`~isaaclab.markers.VisualizationMarkers` when
+:attr:`~isaaclab.visualizers.VisualizerCfg.enable_markers` is enabled. This includes visual-only
+task geometry such as Dexsuite's colored table. Marker instances follow the same selected-world
+offsets as model geometry; set ``enable_markers=False`` to omit them from both views.
+
 Set ``VideoRecorderCfg.backend_source = "renderer"`` to ignore active visualizers and choose from the
 physics/renderer stack instead. In that mode, PhysX physics (``physics=physx``) or Isaac RTX
 (``renderer=isaacsim_rtx_renderer``) selects the Kit path. Newton physics (``physics=newton_mjwarp``) or
@@ -160,7 +186,8 @@ Summary
      - Visualizer ``eye`` / ``lookat`` copied to ``/OmniverseKit_Persp`` + Replicator RGB
    * - ``--visualizer newton`` with default ``backend_source``
      - Newton GL (``"newton_gl"``)
-     - Visualizer ``eye`` / ``lookat`` initially, then live Newton viewer camera sync per frame
+     - Visualizer ``eye`` / ``lookat`` initially, then live camera, visible-environment
+       selection, world-spacing, and marker-overlay sync per frame
 
 
 See also

source/isaaclab/changelog.d/newton-video-visible-envs.rst

@@ -0,0 +1,7 @@
+Fixed
+^^^^^
+
+* Fixed Newton training videos to honor the active visualizer's
+  :attr:`~isaaclab.visualizers.VisualizerCfg.visible_env_indices` and
+  :attr:`~isaaclab.visualizers.VisualizerCfg.max_visible_envs` settings, as well as
+  Newton visual world spacing and viewer-side markers.

source/isaaclab/isaaclab/envs/utils/video_recorder.py

@@ -213,7 +213,7 @@ def __init__(self, cfg: VideoRecorderCfg, scene: InteractiveScene):
                 self._capture = create_isaacsim_kit_perspective_video(kcfg)
 
     def _sync_newton_camera(self) -> None:
-        """Push the Newton visualizer's live camera pose into the capture object.
+        """Push the Newton visualizer's world layout and live camera pose into the capture object.
 
         Called once per :meth:`render_rgb_array` when a Newton visualizer is active.
         The live visualizer instance is resolved lazily (visualizers are initialised by
@@ -227,6 +227,34 @@ def _sync_newton_camera(self) -> None:
             if self._live_visualizer is None:
                 return
 
+        get_visualized_env_ids = getattr(self._live_visualizer, "get_visualized_env_ids", None)
+        if callable(get_visualized_env_ids):
+            visible_env_ids = get_visualized_env_ids()
+            max_visible_envs = getattr(self._live_visualizer.cfg, "max_visible_envs", None)
+            if visible_env_ids is not None:
+                visible_env_ids = list(visible_env_ids)
+                if max_visible_envs is not None:
+                    visible_env_ids = visible_env_ids[: max(0, int(max_visible_envs))]
+            elif max_visible_envs is not None and self._scene.num_envs > 0:
+                num_envs = max(0, int(self._scene.num_envs))
+                visible_env_ids = list(range(min(max(0, int(max_visible_envs)), num_envs)))
+
+            set_visible_worlds = getattr(self._capture, "set_visible_worlds", None)
+            if callable(set_visible_worlds):
+                set_visible_worlds(visible_env_ids)
+
+        world_spacing = getattr(self._live_visualizer.cfg, "world_spacing", None)
+        set_world_offsets = getattr(self._capture, "set_world_offsets", None)
+        if world_spacing is not None and callable(set_world_offsets):
+            set_world_offsets(world_spacing)
+
+        set_frame_overlay_callback = getattr(self._capture, "set_frame_overlay_callback", None)
+        render_markers = getattr(self._live_visualizer, "render_markers", None)
+        if callable(set_frame_overlay_callback):
+            enable_markers = bool(getattr(self._live_visualizer.cfg, "enable_markers", False))
+            callback = render_markers if enable_markers and callable(render_markers) else None
+            set_frame_overlay_callback(callback)
+
         viewer = getattr(self._live_visualizer, "_viewer", None)
         if viewer is None:
             return
@@ -248,8 +276,8 @@ def render_rgb_array(self) -> np.ndarray | None:
         if self._backend is None or self._capture is None:
             return None
         if self._matched_visualizer == "newton":
-            # Newton GL camera state lives in the capture object and must be synced each frame
-            # to follow interactive viewer movement.
+            # Newton GL world layout and camera state live in the capture object and must be synced
+            # each frame to follow the active visualizer.
             self._sync_newton_camera()
         # Kit capture uses the configured eye/lookat applied to the recording camera at construction time.
         return self._capture.render_rgb_array()

source/isaaclab/test/envs/test_video_recorder.py

@@ -7,7 +7,7 @@
 import math
 import sys
 from types import SimpleNamespace
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock, call, patch
 
 import numpy as np
 import pytest
@@ -326,10 +326,22 @@ def test_render_rgb_array_skips_sync_when_no_visualizer():
     mock_sync.assert_not_called()
 
 
-def _make_newton_visualizer(pos=(1.0, 2.0, 3.0), yaw_deg=45.0, pitch_deg=30.0):
+def _make_newton_visualizer(
+    pos=(1.0, 2.0, 3.0),
+    yaw_deg=45.0,
+    pitch_deg=30.0,
+    visible_env_ids=None,
+    max_visible_envs=None,
+    world_spacing=(0.0, 0.0, 0.0),
+    enable_markers=True,
+):
     """Return a mock that quacks like a NewtonVisualizer with a live camera."""
     viz = MagicMock()
     viz.cfg.visualizer_type = "newton"
+    viz.cfg.max_visible_envs = max_visible_envs
+    viz.cfg.world_spacing = world_spacing
+    viz.cfg.enable_markers = enable_markers
+    viz.get_visualized_env_ids.return_value = visible_env_ids
     cam = MagicMock()
     cam.pos = pos
     cam.yaw = yaw_deg
@@ -403,6 +415,117 @@ def test_sync_newton_camera_target_derived_from_pitch_yaw():
     assert abs(dz) < 1e-6
 
 
+def test_sync_newton_camera_forwards_visible_env_ids_before_camera():
+    """Resolved visualizer env ids reach the capture before its lazy camera initialization."""
+    recorder = _create_recorder(_backend="newton_gl", _matched_visualizer="newton")
+    newton_viz = _make_newton_visualizer(visible_env_ids=[0, 1, 2, 3], world_spacing=(2.0, 2.0, 0.0))
+    recorder._live_visualizer = newton_viz
+    recorder._scene.num_envs = 16
+
+    recorder._sync_newton_camera()
+
+    assert recorder._capture.method_calls[0] == call.set_visible_worlds([0, 1, 2, 3])
+    assert recorder._capture.method_calls[1] == call.set_world_offsets((2.0, 2.0, 0.0))
+    assert recorder._capture.method_calls[2] == call.set_frame_overlay_callback(newton_viz.render_markers)
+    assert recorder._capture.method_calls[3][0] == "update_camera"
+
+
+def test_sync_newton_camera_disables_marker_overlays_with_visualizer():
+    """The recording viewer omits markers when the active visualizer disables them."""
+    recorder = _create_recorder(_backend="newton_gl", _matched_visualizer="newton")
+    newton_viz = _make_newton_visualizer(enable_markers=False)
+    recorder._live_visualizer = newton_viz
+
+    recorder._sync_newton_camera()
+
+    recorder._capture.set_frame_overlay_callback.assert_called_once_with(None)
+
+
+def test_sync_newton_camera_truncates_explicit_visible_env_ids_to_cap():
+    """An explicit visualizer selection is truncated by max_visible_envs before capture."""
+    recorder = _create_recorder(_backend="newton_gl", _matched_visualizer="newton")
+    newton_viz = _make_newton_visualizer(visible_env_ids=[7, 2, 5], max_visible_envs=2)
+    recorder._live_visualizer = newton_viz
+    recorder._scene.num_envs = 8
+
+    recorder._sync_newton_camera()
+
+    recorder._capture.set_visible_worlds.assert_called_once_with([7, 2])
+
+
+@pytest.mark.parametrize(
+    ("num_envs", "max_visible_envs", "expected"),
+    [
+        (16, 4, [0, 1, 2, 3]),
+        (2, 4, [0, 1]),
+        (16, 0, []),
+        (0, 4, None),
+    ],
+)
+def test_sync_newton_camera_resolves_cap_only_visible_env_ids(num_envs, max_visible_envs, expected):
+    """A cap-only visualizer selection becomes a clamped contiguous env-id list."""
+    recorder = _create_recorder(_backend="newton_gl", _matched_visualizer="newton")
+    newton_viz = _make_newton_visualizer(visible_env_ids=None, max_visible_envs=max_visible_envs)
+    recorder._live_visualizer = newton_viz
+    recorder._scene.num_envs = num_envs
+
+    recorder._sync_newton_camera()
+
+    recorder._capture.set_visible_worlds.assert_called_once_with(expected)
+
+
+def test_sync_newton_camera_forwards_none_when_all_envs_are_visible():
+    """None is forwarded when neither an explicit selection nor a cap is configured."""
+    recorder = _create_recorder(_backend="newton_gl", _matched_visualizer="newton")
+    newton_viz = _make_newton_visualizer(visible_env_ids=None, max_visible_envs=None)
+    recorder._live_visualizer = newton_viz
+    recorder._scene.num_envs = 8
+
+    recorder._sync_newton_camera()
+
+    recorder._capture.set_visible_worlds.assert_called_once_with(None)
+
+
+def test_sync_newton_camera_preserves_empty_visible_env_selection():
+    """An explicit empty selection remains empty rather than becoming the all-envs sentinel."""
+    recorder = _create_recorder(_backend="newton_gl", _matched_visualizer="newton")
+    newton_viz = _make_newton_visualizer(visible_env_ids=[], max_visible_envs=None)
+    recorder._live_visualizer = newton_viz
+    recorder._scene.num_envs = 8
+
+    recorder._sync_newton_camera()
+
+    recorder._capture.set_visible_worlds.assert_called_once_with([])
+
+
+def test_sync_newton_camera_forwards_visibility_without_live_viewer():
+    """Visibility sync does not depend on the interactive viewer camera being initialized."""
+    recorder = _create_recorder(_backend="newton_gl", _matched_visualizer="newton")
+    newton_viz = _make_newton_visualizer(visible_env_ids=[0, 1, 2, 3])
+    newton_viz._viewer = None
+    recorder._live_visualizer = newton_viz
+    recorder._scene.num_envs = 8
+
+    recorder._sync_newton_camera()
+
+    recorder._capture.set_visible_worlds.assert_called_once_with([0, 1, 2, 3])
+    recorder._capture.update_camera.assert_not_called()
+
+
+def test_sync_newton_camera_supports_capture_without_visibility_setter():
+    """Custom legacy capture classes without set_visible_worlds still receive camera updates."""
+    recorder = _create_recorder(_backend="newton_gl", _matched_visualizer="newton")
+    newton_viz = _make_newton_visualizer(visible_env_ids=[0, 1])
+    recorder._live_visualizer = newton_viz
+    recorder._scene.num_envs = 8
+    update_camera = MagicMock()
+    recorder._capture = SimpleNamespace(update_camera=update_camera)
+
+    recorder._sync_newton_camera()
+
+    update_camera.assert_called_once()
+
+
 def test_sync_newton_camera_no_visualizer_does_not_raise():
     """_sync_newton_camera silently skips when no Newton visualizer is registered."""
     recorder = _create_recorder(_backend="newton_gl", _matched_visualizer="newton")

source/isaaclab/test/markers/test_visualization_markers.py

@@ -587,7 +587,39 @@ def test_newton_marker_render_filters_visible_envs(monkeypatch: pytest.MonkeyPat
 
     assert len(viewer.instances) == 1
     assert viewer.instances[0]["hidden"] is False
-    assert viewer.instances[0]["xforms"][:, 0].tolist() == [1.0, 3.0, 5.0, 7.0]
+    assert viewer.instances[0]["xforms"][:, 0].tolist() == [2.0, 3.0, 6.0, 7.0]
+
+
+def test_newton_marker_render_applies_visible_world_offsets(monkeypatch: pytest.MonkeyPatch):
+    _patch_newton_marker_render_deps(monkeypatch)
+    translations = torch.arange(8, dtype=torch.float32).unsqueeze(1).repeat(1, 3)
+    marker = _make_newton_marker_for_render(
+        marker_names=["arrow"],
+        translations=translations,
+        marker_indices=torch.zeros(8, dtype=torch.int32),
+    )
+    viewer = _FakeNewtonMarkerViewer()
+    viewer.world_offsets = torch.tensor([[0.0, 0.0, 0.0], [10.0, 0.0, 0.0], [20.0, 0.0, 0.0], [30.0, 0.0, 0.0]])
+
+    marker.render(viewer, visible_env_ids=[1, 3], num_envs=4)
+
+    assert viewer.instances[0]["xforms"][:, 0].tolist() == [12.0, 13.0, 36.0, 37.0]
+
+
+def test_newton_marker_render_applies_world_offsets_when_all_envs_visible(monkeypatch: pytest.MonkeyPatch):
+    _patch_newton_marker_render_deps(monkeypatch)
+    translations = torch.arange(8, dtype=torch.float32).unsqueeze(1).repeat(1, 3)
+    marker = _make_newton_marker_for_render(
+        marker_names=["arrow"],
+        translations=translations,
+        marker_indices=torch.zeros(8, dtype=torch.int32),
+    )
+    viewer = _FakeNewtonMarkerViewer()
+    viewer.world_offsets = torch.tensor([[0.0, 0.0, 0.0], [10.0, 0.0, 0.0], [20.0, 0.0, 0.0], [30.0, 0.0, 0.0]])
+
+    marker.render(viewer, visible_env_ids=None, num_envs=4)
+
+    assert viewer.instances[0]["xforms"][:, 0].tolist() == [0.0, 1.0, 12.0, 13.0, 24.0, 25.0, 36.0, 37.0]
 
 
 def test_newton_marker_render_routes_instances_by_prototype(monkeypatch: pytest.MonkeyPatch):

source/isaaclab_newton/changelog.d/newton-video-visible-envs.minor.rst

@@ -0,0 +1,8 @@
+Added
+^^^^^
+
+* Added :meth:`~isaaclab_newton.video_recording.NewtonGlPerspectiveVideo.set_visible_worlds`
+  and :meth:`~isaaclab_newton.video_recording.NewtonGlPerspectiveVideo.set_world_offsets` to
+  select and visually separate simulation worlds in Newton GL recordings.
+* Added :meth:`~isaaclab_newton.video_recording.NewtonGlPerspectiveVideo.set_frame_overlay_callback`
+  to render visualizer-owned overlays into recorded frames.