Add camera PPISP HDR renderer integration

Apply camera PPISP from renderer HDR buffers and verify RTX, OVRTX, and Newton HDR output plumbing for the camera pipeline.

Author: moennen

source/isaaclab/isaaclab/renderers/__init__.pyi

@@ -6,6 +6,7 @@
 __all__ = [
     "BaseRenderer",
     "CameraRenderSpec",
+    "CameraPPISPCfg",
     "RenderBufferKind",
     "RenderBufferSpec",
     "Renderer",
@@ -16,6 +17,7 @@ __all__ = [
 from .base_renderer import BaseRenderer
 from .camera_render_spec import CameraRenderSpec
 from .output_contract import RenderBufferKind, RenderBufferSpec
+from .camera_ppisp import CameraPPISPCfg
 from .renderer import Renderer
 from .renderer_cfg import RendererCfg
 from .render_context import RenderContext

source/isaaclab/isaaclab/renderers/camera_ppisp.py

@@ -0,0 +1,272 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Camera PPISP configuration and USD/SPG parsing helpers.
+
+The implementation follows the physically plausible ISP model described in
+https://arxiv.org/abs/2601.18336.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from isaaclab.utils import configclass
+
+
+CAMERA_PPISP_SHADER_NAME = "PPISP"
+
+CAMERA_PPISP_FLOAT2_INPUTS = {
+    "vignettingCenterR",
+    "vignettingCenterG",
+    "vignettingCenterB",
+    "colorLatentBlue",
+    "colorLatentRed",
+    "colorLatentGreen",
+    "colorLatentNeutral",
+}
+
+CAMERA_PPISP_DEFAULT_INPUTS: dict[str, float | tuple[float, float]] = {
+    "exposureOffset": 0.0,
+    "vignettingCenterR": (0.0, 0.0),
+    "vignettingAlpha1R": 0.0,
+    "vignettingAlpha2R": 0.0,
+    "vignettingAlpha3R": 0.0,
+    "vignettingCenterG": (0.0, 0.0),
+    "vignettingAlpha1G": 0.0,
+    "vignettingAlpha2G": 0.0,
+    "vignettingAlpha3G": 0.0,
+    "vignettingCenterB": (0.0, 0.0),
+    "vignettingAlpha1B": 0.0,
+    "vignettingAlpha2B": 0.0,
+    "vignettingAlpha3B": 0.0,
+    "colorLatentBlue": (0.0, 0.0),
+    "colorLatentRed": (0.0, 0.0),
+    "colorLatentGreen": (0.0, 0.0),
+    "colorLatentNeutral": (0.0, 0.0),
+    "crfToeR": 0.013659,
+    "crfShoulderR": 0.013659,
+    "crfGammaR": 0.378165,
+    "crfCenterR": 0.0,
+    "crfToeG": 0.013659,
+    "crfShoulderG": 0.013659,
+    "crfGammaG": 0.378165,
+    "crfCenterG": 0.0,
+    "crfToeB": 0.013659,
+    "crfShoulderB": 0.013659,
+    "crfGammaB": 0.378165,
+    "crfCenterB": 0.0,
+}
+
+
+def default_camera_ppisp_inputs() -> dict[str, float | tuple[float, float]]:
+    """Return a copy of the PPISP identity/default input dictionary."""
+    return dict(CAMERA_PPISP_DEFAULT_INPUTS)
+
+
+@configclass
+class CameraPPISPCfg:
+    """Configuration for PPISP post-processing.
+
+    PPISP inputs are static in IsaacLab. If imported from animated USD shader inputs,
+    the first authored time sample is used and later samples are ignored.
+    """
+
+    shader_prim_path: str | None = None
+    """Optional source USD shader prim path used to populate :attr:`inputs`."""
+
+    inputs: dict[str, float | tuple[float, float]] = field(default_factory=default_camera_ppisp_inputs)
+    """Flat PPISP shader input values keyed by USD input name."""
+
+
+@dataclass
+class RenderProductInfo:
+    """Parsed USD RenderProduct information used for PPISP validation."""
+
+    render_product_path: str
+    camera_paths: list[str]
+    resolution: tuple[int, int] | None
+    ordered_vars: list[str]
+    ppisp: CameraPPISPCfg | None
+    camera_xform_time_samples: list[float]
+
+
+def normalize_camera_ppisp_cfg(
+    ppisp_cfg: CameraPPISPCfg | dict[str, Any] | None,
+    stage: Any | None = None,
+) -> CameraPPISPCfg | None:
+    """Convert supported user PPISP representations to :class:`CameraPPISPCfg`."""
+    if ppisp_cfg is None:
+        return None
+    if isinstance(ppisp_cfg, CameraPPISPCfg):
+        input_overrides = dict(ppisp_cfg.inputs)
+        if ppisp_cfg.shader_prim_path and stage is not None:
+            ppisp_cfg = _merge_shader_inputs_with_cfg(ppisp_cfg, stage, input_overrides)
+        else:
+            ppisp_cfg.inputs = _normalized_inputs(input_overrides)
+        return ppisp_cfg
+    if isinstance(ppisp_cfg, dict):
+        input_overrides = ppisp_cfg.get(
+            "inputs", {key: value for key, value in ppisp_cfg.items() if key in CAMERA_PPISP_DEFAULT_INPUTS}
+        )
+        cfg = CameraPPISPCfg()
+        cfg.inputs = _normalized_inputs(input_overrides)
+        shader_prim_path = ppisp_cfg.get("shader_prim_path")
+        if shader_prim_path is not None:
+            cfg.shader_prim_path = str(shader_prim_path)
+            if stage is not None:
+                cfg = _merge_shader_inputs_with_cfg(cfg, stage, input_overrides)
+        return cfg
+    raise TypeError(f"Unsupported PPISP configuration type: {type(ppisp_cfg)!r}")
+
+
+def camera_ppisp_cfg_from_usd_shader(shader: Any) -> CameraPPISPCfg:
+    """Create :class:`CameraPPISPCfg` from a ``UsdShade.Shader`` prim.
+
+    Animated inputs are collapsed to their first authored time sample.
+    """
+    cfg = CameraPPISPCfg(shader_prim_path=str(shader.GetPath()))
+    values = default_camera_ppisp_inputs()
+    for input_name in values:
+        shader_input = shader.GetInput(input_name)
+        if not shader_input:
+            continue
+        attr = shader_input.GetAttr()
+        value = _read_first_authored_value(attr)
+        if value is not None:
+            values[input_name] = _normalize_input_value(input_name, value)
+    cfg.inputs = values
+    return cfg
+
+
+def camera_ppisp_cfg_from_usd_stage(stage: Any, shader_prim_path: str) -> CameraPPISPCfg:
+    """Create :class:`CameraPPISPCfg` from a shader prim path in a USD stage."""
+    from pxr import UsdShade
+
+    shader = UsdShade.Shader(stage.GetPrimAtPath(shader_prim_path))
+    if not shader:
+        raise ValueError(f"PPISP shader prim not found at path: {shader_prim_path}")
+    return camera_ppisp_cfg_from_usd_shader(shader)
+
+
+def parse_render_product(stage: Any, render_product_path: str) -> RenderProductInfo:
+    """Parse a USD RenderProduct and optional PPISP shader configuration."""
+    render_product = stage.GetPrimAtPath(render_product_path)
+    if not render_product.IsValid() or render_product.GetTypeName() != "RenderProduct":
+        raise ValueError(f"RenderProduct not found at path: {render_product_path}")
+
+    camera_rel = render_product.GetRelationship("camera")
+    camera_paths = [str(path) for path in camera_rel.GetTargets()] if camera_rel else []
+    if not camera_paths:
+        raise ValueError(f"RenderProduct at path '{render_product_path}' has no camera relationship targets.")
+
+    resolution = None
+    resolution_attr = render_product.GetAttribute("resolution")
+    if resolution_attr:
+        resolution_value = resolution_attr.Get()
+        if resolution_value is not None:
+            resolution = (int(resolution_value[0]), int(resolution_value[1]))
+
+    ordered_vars_rel = render_product.GetRelationship("orderedVars")
+    ordered_vars = [str(path) for path in ordered_vars_rel.GetTargets()] if ordered_vars_rel else []
+
+    ppisp = None
+    ppisp_prim = stage.GetPrimAtPath(f"{render_product_path}/{CAMERA_PPISP_SHADER_NAME}")
+    if ppisp_prim.IsValid():
+        from pxr import UsdShade
+
+        ppisp = camera_ppisp_cfg_from_usd_shader(UsdShade.Shader(ppisp_prim))
+
+    return RenderProductInfo(
+        render_product_path=render_product_path,
+        camera_paths=camera_paths,
+        resolution=resolution,
+        ordered_vars=ordered_vars,
+        ppisp=ppisp,
+        camera_xform_time_samples=collect_camera_xform_time_samples(stage, camera_paths),
+    )
+
+
+def parse_render_product_file(usd_path: str, render_product_path: str) -> RenderProductInfo:
+    """Open a USD file and parse a RenderProduct."""
+    from pxr import Usd
+
+    stage = Usd.Stage.Open(usd_path)
+    if stage is None:
+        raise RuntimeError(f"Failed to open USD stage at path: {usd_path}")
+    return parse_render_product(stage, render_product_path)
+
+
+def collect_camera_xform_time_samples(stage: Any, camera_paths: list[str]) -> list[float]:
+    """Collect authored xform time samples from cameras and inherited source cameras."""
+    time_samples = set()
+    for camera_path in camera_paths:
+        prim = stage.GetPrimAtPath(camera_path)
+        if not prim.IsValid():
+            continue
+        _collect_xform_attr_time_samples(prim, time_samples)
+        for inherited_path in prim.GetInherits().GetAllDirectInherits():
+            inherited_prim = stage.GetPrimAtPath(inherited_path)
+            if inherited_prim.IsValid():
+                _collect_xform_attr_time_samples(inherited_prim, time_samples)
+    if not time_samples:
+        start_time = stage.GetStartTimeCode()
+        end_time = stage.GetEndTimeCode()
+        if start_time != end_time:
+            time_samples.update([start_time, end_time])
+        else:
+            time_samples.add(start_time)
+    return sorted(time_samples)
+
+
+def _normalized_inputs(inputs: dict[str, Any]) -> dict[str, float | tuple[float, float]]:
+    values = default_camera_ppisp_inputs()
+    for input_name, value in inputs.items():
+        if input_name not in values:
+            raise ValueError(f"Unknown PPISP input: {input_name}")
+        values[input_name] = _normalize_input_value(input_name, value)
+    return values
+
+
+def _merge_shader_inputs_with_cfg(
+    ppisp_cfg: CameraPPISPCfg,
+    stage: Any,
+    input_overrides: dict[str, Any],
+) -> CameraPPISPCfg:
+    parsed_cfg = camera_ppisp_cfg_from_usd_stage(stage, ppisp_cfg.shader_prim_path)
+    if input_overrides != CAMERA_PPISP_DEFAULT_INPUTS:
+        parsed_cfg.inputs.update(_normalized_input_overrides(input_overrides))
+    return parsed_cfg
+
+
+def _normalized_input_overrides(inputs: dict[str, Any]) -> dict[str, float | tuple[float, float]]:
+    values = {}
+    for input_name, value in inputs.items():
+        if input_name not in CAMERA_PPISP_DEFAULT_INPUTS:
+            raise ValueError(f"Unknown PPISP input: {input_name}")
+        values[input_name] = _normalize_input_value(input_name, value)
+    return values
+
+
+def _normalize_input_value(input_name: str, value: Any) -> float | tuple[float, float]:
+    if input_name in CAMERA_PPISP_FLOAT2_INPUTS:
+        if len(value) != 2:
+            raise ValueError(f"PPISP input '{input_name}' expects two values.")
+        return (float(value[0]), float(value[1]))
+    return float(value)
+
+
+def _read_first_authored_value(attr: Any) -> Any:
+    time_samples = attr.GetTimeSamples()
+    if time_samples:
+        return attr.Get(time_samples[0])
+    return attr.Get()
+
+
+def _collect_xform_attr_time_samples(prim: Any, time_samples: set[float]) -> None:
+    for attr in prim.GetAttributes():
+        if attr.GetName().startswith("xformOp:"):
+            time_samples.update(attr.GetTimeSamples())