v2.54.0

ViroRenderer/VROARSession.h

@@ -319,7 +319,21 @@ class VROARSession {
      texture are updated after each call to updateFrame().
      */
     virtual std::shared_ptr<VROTexture> getCameraBackgroundTexture() = 0;
-
+
+    /*
+     Get the semantic texture for the current frame. Each pixel is a VROSemanticLabel
+     value (0-11, R8 format). Returns nullptr if semantic mode is not enabled or
+     the platform does not support it.
+     */
+    virtual std::shared_ptr<VROTexture> getSemanticTexture() { return nullptr; }
+
+    /*
+     Get the confidence texture for the current frame. Each pixel is a confidence value
+     (R8, 0=uncertain, 255=certain) corresponding to the label in getSemanticTexture().
+     Returns nullptr if not supported; callers should substitute a 1×1 white texture.
+     */
+    virtual std::shared_ptr<VROTexture> getSemanticConfidenceTexture() { return nullptr; }
+
     /*
      Invoke when the viewport changes. The AR engine may adjust its camera
      background and projection matrices in response to a viewport change.
@@ -557,6 +571,11 @@ class VROARSession {
         std::function<void(bool success, std::string error)> callback) {
         if (callback) callback(false, "Not supported");
     }
+    virtual void rvGetSceneAssets(
+        const std::string& sceneId,
+        std::function<void(bool success, std::string jsonData, std::string error)> callback) {
+        if (callback) callback(false, "", "Not supported");
+    }
 
     // ========================================================================
     // ReactVision Geospatial CRUD API

ViroRenderer/VRODriverOpenGL.h

@@ -205,7 +205,23 @@ class VRODriverOpenGL : public VRODriver, public std::enable_shared_from_this<VR
             }
         }
     }
-
+
+    /*
+     Invalidate the CPU-side GL state cache without issuing any GL calls. Must be called
+     after switching EGL contexts (e.g. to/from the recording surface) so that cached texture
+     bindings and shader state from the previous context are not mistakenly treated as still
+     valid in the new context. The new context has its own independent GL state, so all cache
+     entries must be considered stale.
+     */
+    void invalidateGLStateCache() {
+        _activeTextureUnit = 0;
+        for (int i = 0; i < kMaxTextureUnits; i++) {
+            _activeTextures[i].clear();
+        }
+        _boundRenderTarget.reset();
+        _boundShader.reset();
+    }
+
     void setDepthWritingEnabled(bool enabled) {
         if (_depthWritingEnabled == enabled) {
             return;

ViroRenderer/VROMaterial.cpp

@@ -33,6 +33,7 @@
 #include "VROSortKey.h"
 #include "VROThreadRestricted.h"
 #include "VROLog.h"
+#include "VROShaderModifier.h"
 #include <atomic>
 #include <algorithm>
 
@@ -380,3 +381,132 @@ void VROMaterial::setChromaKeyFilteringColor(VROVector3f color) {
 void VROMaterial::setNeedsToneMapping(bool needsToneMapping) {
     _needsToneMapping = needsToneMapping;
 }
+
+// ============================================================
+// Semantic Masking
+// ============================================================
+
+void VROMaterial::setSemanticMaskEnabled(bool enabled) {
+    if (_semanticMaskEnabled == enabled) {
+        return;
+    }
+    _semanticMaskEnabled = enabled;
+    applySemanticMaskModifier();
+}
+
+void VROMaterial::setSemanticMaskMode(VROSemanticMaskMode mode) {
+    _semanticMaskMode = mode;
+}
+
+void VROMaterial::setSemanticLabelMask(uint16_t mask) {
+    _semanticLabelMask = mask;
+}
+
+void VROMaterial::applySemanticMaskModifier() {
+    // Remove old modifier if present.
+    if (_semanticMaskModifier) {
+        removeShaderModifier(_semanticMaskModifier);
+        _semanticMaskModifier = nullptr;
+    }
+
+    if (!_semanticMaskEnabled) {
+        return;
+    }
+
+    // Fragment shader modifier lines. The VROShaderModifier constructor splits these:
+    // - lines starting with "uniform " → uniforms section
+    // - other lines → body section
+    //
+    // semantic_texture           : R8 label texture; each texel = VROSemanticLabel value (0-11).
+    // semantic_confidence_texture: R8 confidence texture; 0=uncertain, 1=certain (after /255).
+    // semantic_mask_mode : 0 = ShowOnly  — keep fragments where label IS in mask
+    //                      1 = Hide      — keep fragments where label is NOT in mask
+    //                      2 = Debug     — colorise by label (no masking)
+    // semantic_label_mask: bitmask float; bit N set → label N is selected.
+    // ar_viewport_size   : always-available built-in (vec3, xy = width/height in pixels).
+    //
+    // Alpha-blend approach (replaces discard):
+    //   ShowOnly: alpha *= conf  when matched, alpha *= 0 otherwise
+    //   Hide:     alpha *= 0     when matched, alpha *= 1 otherwise
+    //             (matched boundary pixels get alpha = 1-conf, giving soft hide edges)
+    // The material's default VROBlendMode::Alpha handles the compositing.
+    std::vector<std::string> lines = {
+        // parseCustomUniforms sees "uniform sampler2D" and places each sampler in
+        // _modifierSamplers; loadTextures() binds them to the matching VROGlobalTextureType.
+        "uniform sampler2D semantic_texture;",
+        "uniform sampler2D semantic_confidence_texture;",
+        // ar_viewport_size and ar_semantic_texture_transform are base program uniforms
+        // (VROShaderProgram::addUniform) bound unconditionally each frame.
+        // ar_semantic_texture_transform = getViewportToCameraImageTransform() in GL convention
+        // (y=0 bottom), matching gl_FragCoord directly — no Y-flip needed.
+        "uniform highp vec3 ar_viewport_size;",
+        "uniform highp mat4 ar_semantic_texture_transform;",
+        "uniform highp float semantic_mask_mode;",
+        "uniform highp float semantic_label_mask;",
+        // Compute shared UV once. No Y-flip: gl_FragCoord and the transform are both GL-convention.
+        "highp vec2 semUV = (ar_semantic_texture_transform * vec4(gl_FragCoord.xy / ar_viewport_size.xy, 0.0, 1.0)).xy;",
+        // Clamp inward by a small margin to skip the 1-2 pixel unlabeled border that ARCore's
+        // neural network outputs at the left/right edges of the landscape image. In portrait mode
+        // those edges map to thin horizontal lines at the top/bottom of the viewport. Without the
+        // clamp those border pixels (label=0) cut the sky effect with a transparent stripe.
+        // 0.005 ≈ half a pixel on a 192-px-tall semantic texture; harmless for center content.
+        "semUV = clamp(semUV, vec2(0.005), vec2(0.995));",
+        "bool semInBounds = (semUV.x >= 0.0 && semUV.x <= 1.0 && semUV.y >= 0.0 && semUV.y <= 1.0);",
+        "if (semInBounds) {",
+        "    highp float label_raw = texture(semantic_texture, semUV).r * 255.0;",
+        "    int semLabel = int(floor(label_raw + 0.5));",
+        "    bool semMatches = (semLabel >= 1 && semLabel <= 11 && (int(semantic_label_mask) & (1 << semLabel)) != 0);",
+        // Debug mode (semantic_mask_mode == 2): colorise by label for overlay visualisation.
+        // Blue = unlabeled, teal→orange gradient = classified pixels (fully opaque).
+        "    if (semantic_mask_mode >= 1.5 && semantic_mask_mode < 2.5) {",
+        "        highp float t = label_raw / 11.0;",
+        "        _output_color = vec4(t, float(semLabel > 0) * 0.8, 1.0 - t, 1.0);",
+        "    } else {",
+        // Confidence value from texture [0,1]. Used as alpha weight for soft edges.
+        // On iOS (no platform confidence) the texture is 1x1 white → conf = 1.0 → hard edges.
+        "        highp float conf = texture(semantic_confidence_texture, semUV).r;",
+        // ShowOnly (mode=0): show matched pixels with weight = conf; hide everything else.
+        // Hide (mode=1): hide matched pixels; show others at full opacity.
+        // ShowOnlySky (mode=3): like ShowOnly, but unlabeled pixels use viewport Y as a proxy
+        //   for sky context — upper half of screen shows sphere, lower half hides it. This
+        //   handles the ~10-15% border of the ARCore semantic image where the neural network
+        //   returns label=0 even though the camera sees real sky (or real ground). Regular
+        //   ShowOnly (mode=0) is unchanged so other semantic mask uses are not affected.
+        "        if (semantic_mask_mode < 0.5) {",
+        "            _output_color.a *= semMatches ? conf : 0.0;",
+        "        } else if (semantic_mask_mode > 2.5) {",
+        "            if (semMatches) {",
+        "                _output_color.a *= conf;",
+        "            } else if (semLabel == 0) {",
+        "                highp float normY = gl_FragCoord.y / ar_viewport_size.y;",
+        "                _output_color.a *= (normY > 0.5) ? 1.0 : 0.0;",
+        "            } else {",
+        "                _output_color.a *= 0.0;",
+        "            }",
+        "        } else {",
+        "            _output_color.a *= semMatches ? (1.0 - conf) : 1.0;",
+        "        }",
+        "    }",
+        "}"
+        // semInBounds == false: no semantic data available for this pixel → pass through unchanged.
+    };
+
+    _semanticMaskModifier = std::make_shared<VROShaderModifier>(
+        VROShaderEntryPoint::Fragment, lines);
+    _semanticMaskModifier->setName("semantic_mask");
+
+    // Capture current values by value — if mode/mask change, applySemanticMaskModifier()
+    // is called again, creating a new modifier with updated captures.
+    float modeVal = (float)static_cast<int>(_semanticMaskMode);
+    float maskVal = (float)_semanticLabelMask;
+    _semanticMaskModifier->setUniformBinder("semantic_mask_mode", VROShaderProperty::Float,
+        [modeVal](VROUniform *uniform, const VROGeometry *, const VROMaterial *) {
+            uniform->setFloat(modeVal);
+        });
+    _semanticMaskModifier->setUniformBinder("semantic_label_mask", VROShaderProperty::Float,
+        [maskVal](VROUniform *uniform, const VROGeometry *, const VROMaterial *) {
+            uniform->setFloat(maskVal);
+        });
+
+    addShaderModifier(_semanticMaskModifier);
+}

ViroRenderer/VROMaterial.h

@@ -37,6 +37,7 @@
 #include "VROStringUtil.h"
 #include "VROThreadRestricted.h"
 #include "VRODriver.h"
+#include "VROSemantics.h"
 
 enum class VROFace {
     Front,
@@ -361,6 +362,21 @@ class VROMaterial : public VROAnimatable, public VROThreadRestricted {
         return _needsToneMapping;
     }
 
+    /*
+     Semantic masking. When enabled, fragments are shown or hidden based on
+     the semantic label of the underlying real-world pixel (requires ARCore
+     scene semantics to be enabled on the AR session).
+     */
+    void setSemanticMaskEnabled(bool enabled);
+    bool isSemanticMaskEnabled() const { return _semanticMaskEnabled; }
+
+    void setSemanticMaskMode(VROSemanticMaskMode mode);
+    VROSemanticMaskMode getSemanticMaskMode() const { return _semanticMaskMode; }
+
+    // Bitmask of VROSemanticLabel values (bit N = label N). Use (1 << label_int) to set bits.
+    void setSemanticLabelMask(uint16_t mask);
+    uint16_t getSemanticLabelMask() const { return _semanticLabelMask; }
+
     /*
      Material rendering order; this should only be used to fix a rendering order between materials
      that are part of the same geometry. For cross-geometry rendering order, use
@@ -647,6 +663,19 @@ class VROMaterial : public VROAnimatable, public VROThreadRestricted {
      */
     bool _needsToneMapping;
 
+    /*
+     Semantic masking.
+     */
+    bool _semanticMaskEnabled = false;
+    VROSemanticMaskMode _semanticMaskMode = VROSemanticMaskMode::ShowOnly;
+    uint16_t _semanticLabelMask = 0;
+
+    // Shader modifier injected when semantic masking is enabled (kept to allow removal).
+    std::shared_ptr<VROShaderModifier> _semanticMaskModifier;
+
+    // Internal: inject or remove the semantic mask shader modifier.
+    void applySemanticMaskModifier();
+
     /*
      The rendering order of this material, which determines when it is rendered in relation to
      other materials. See VROSortKey for where this falls within the hierarchy of renderinng sort

ViroRenderer/VROMaterialShaderBinding.cpp

@@ -98,6 +98,9 @@ void VROMaterialShaderBinding::loadUniforms() {
     _occlusionZNearUniform = program->getUniform("occlusion_z_near");
     _occlusionZFarUniform = program->getUniform("occlusion_z_far");
 
+    // AR semantic uniform
+    _arSemanticTextureTransformUniform = program->getUniform("ar_semantic_texture_transform");
+
     // Camera texture transform (may be null if no modifier uses camera_texture)
     _cameraImageTransformUniform = program->getUniform("camera_image_transform");
 
@@ -166,6 +169,9 @@ void VROMaterialShaderBinding::loadTextures() {
             // This avoids conflict with AO maps on the material.
             _textures.emplace_back(VROGlobalTextureType::ARDepthMap);
         }
+        else if (sampler == "semantic_texture") {
+            _textures.emplace_back(VROGlobalTextureType::SemanticMap);
+        }
         else if (sampler == "emissive_texture") {
             _textures.emplace_back(_material.getEmission().getTexture());
         }
@@ -202,6 +208,14 @@ void VROMaterialShaderBinding::loadTextures() {
             _textures.emplace_back(VROGlobalTextureType::CameraBackground);
             continue;
         }
+        if (samplerName == "semantic_texture") {
+            _textures.emplace_back(VROGlobalTextureType::SemanticMap);
+            continue;
+        }
+        if (samplerName == "semantic_confidence_texture") {
+            _textures.emplace_back(VROGlobalTextureType::SemanticConfidence);
+            continue;
+        }
         // Guard: standard global samplers may be re-declared in modifier uniforms strings.
         // parseCustomUniforms now skips them (they're already in _samplers), but handle
         // them here as a belt-and-suspenders fallback so we never push a null local ref.
@@ -354,7 +368,19 @@ void VROMaterialShaderBinding::bindGeometryUniforms(float opacity, const VROGeom
 }
 
 void VROMaterialShaderBinding::bindOcclusionUniforms(const VRORenderContext &context) {
-    // Only bind if occlusion is enabled
+    // Semantic texture transform is independent of depth occlusion — bind unconditionally.
+    if (_arSemanticTextureTransformUniform != nullptr) {
+        _arSemanticTextureTransformUniform->setMat4(context.getSemanticTextureTransform());
+    }
+
+    // ar_viewport_size is needed by the semantic mask modifier even without depth occlusion.
+    // If occlusion IS enabled it will be set again below — that's fine, it's the same value.
+    if (_arViewportSizeUniform != nullptr) {
+        VROViewport viewport = context.getViewport();
+        _arViewportSizeUniform->setVec3({(float)viewport.getWidth(), (float)viewport.getHeight(), 0.0f});
+    }
+
+    // Only bind depth-occlusion-specific uniforms if occlusion is enabled
     if (!context.isOcclusionEnabled()) {
         return;
     }

ViroRenderer/VROMaterialShaderBinding.h

@@ -132,6 +132,9 @@ class VROMaterialShaderBinding {
     VROUniform *_occlusionZNearUniform;
     VROUniform *_occlusionZFarUniform;
 
+    // AR semantic uniform (independent of depth occlusion)
+    VROUniform *_arSemanticTextureTransformUniform;
+
     // Camera texture transform uniform
     VROUniform *_cameraImageTransformUniform;