feat: add depth mesh generation, fix android semantics

Author: doranteseduardo

ViroRenderer/VROARDepthMesh.cpp

@@ -0,0 +1,53 @@
+//
+//  VROARDepthMesh.cpp
+//  ViroRenderer
+//
+//  Copyright © 2024 Viro Media. All rights reserved.
+//
+//  Permission is hereby granted, free of charge, to any person obtaining
+//  a copy of this software and associated documentation files (the
+//  "Software"), to deal in the Software without restriction, including
+//  without limitation the rights to use, copy, modify, merge, publish,
+//  distribute, sublicense, and/or sell copies of the Software, and to
+//  permit persons to whom the Software is furnished to do so, subject to
+//  the following conditions:
+//
+//  The above copyright notice and this permission notice shall be included
+//  in all copies or substantial portions of the Software.
+//
+//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+//  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+//  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+//  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+//  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+//  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+//  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+#include "VROARDepthMesh.h"
+#include <numeric>
+
+VROARDepthMesh::VROARDepthMesh() {}
+
+VROARDepthMesh::VROARDepthMesh(std::vector<VROVector3f> vertices,
+                               std::vector<int> indices,
+                               std::vector<float> confidences)
+    : _vertices(std::move(vertices)),
+      _indices(std::move(indices)),
+      _confidences(std::move(confidences)) {
+}
+
+VROARDepthMesh::~VROARDepthMesh() {}
+
+float VROARDepthMesh::getAverageConfidence() const {
+    if (_confidences.empty()) {
+        return 0.0f;
+    }
+    float sum = std::accumulate(_confidences.begin(), _confidences.end(), 0.0f);
+    return sum / static_cast<float>(_confidences.size());
+}
+
+bool VROARDepthMesh::isValid() const {
+    return !_vertices.empty() &&
+           !_indices.empty() &&
+           _indices.size() % 3 == 0;
+}

ViroRenderer/VROARDepthMesh.h

@@ -0,0 +1,100 @@
+//
+//  VROARDepthMesh.h
+//  ViroRenderer
+//
+//  Copyright © 2024 Viro Media. All rights reserved.
+//
+//  Permission is hereby granted, free of charge, to any person obtaining
+//  a copy of this software and associated documentation files (the
+//  "Software"), to deal in the Software without restriction, including
+//  without limitation the rights to use, copy, modify, merge, publish,
+//  distribute, sublicense, and/or sell copies of the Software, and to
+//  permit persons to whom the Software is furnished to do so, subject to
+//  the following conditions:
+//
+//  The above copyright notice and this permission notice shall be included
+//  in all copies or substantial portions of the Software.
+//
+//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+//  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+//  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+//  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+//  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+//  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+//  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+#ifndef VROARDepthMesh_h
+#define VROARDepthMesh_h
+
+#include <vector>
+#include <memory>
+#include "VROVector3f.h"
+
+/**
+ * Represents a triangulated mesh generated from AR depth data.
+ * Used for physics collision and hit testing against real-world surfaces.
+ *
+ * The mesh is generated by sampling the depth image at regular intervals
+ * (controlled by stride) and connecting adjacent valid depth samples into
+ * triangles. Depth discontinuities are detected to avoid connecting
+ * surfaces that are far apart in depth (e.g., a table edge and the floor).
+ */
+class VROARDepthMesh {
+public:
+    VROARDepthMesh();
+
+    /**
+     * Construct a depth mesh from vertex and index data.
+     *
+     * @param vertices Array of 3D vertex positions in world space
+     * @param indices Triangle indices (every 3 values form a triangle)
+     * @param confidences Per-vertex confidence values (0.0-1.0)
+     */
+    VROARDepthMesh(std::vector<VROVector3f> vertices,
+                   std::vector<int> indices,
+                   std::vector<float> confidences);
+
+    ~VROARDepthMesh();
+
+    /**
+     * Get the vertex positions in world space.
+     */
+    const std::vector<VROVector3f>& getVertices() const { return _vertices; }
+
+    /**
+     * Get the triangle indices. Every 3 values form a triangle.
+     */
+    const std::vector<int>& getIndices() const { return _indices; }
+
+    /**
+     * Get the per-vertex confidence values (0.0-1.0).
+     */
+    const std::vector<float>& getConfidences() const { return _confidences; }
+
+    /**
+     * Get the number of vertices in the mesh.
+     */
+    int getVertexCount() const { return static_cast<int>(_vertices.size()); }
+
+    /**
+     * Get the number of triangles in the mesh.
+     */
+    int getTriangleCount() const { return static_cast<int>(_indices.size() / 3); }
+
+    /**
+     * Get the average confidence across all vertices.
+     */
+    float getAverageConfidence() const;
+
+    /**
+     * Check if the mesh is valid (has vertices and valid triangle indices).
+     */
+    bool isValid() const;
+
+private:
+    std::vector<VROVector3f> _vertices;
+    std::vector<int> _indices;
+    std::vector<float> _confidences;
+};
+
+#endif /* VROARDepthMesh_h */

ViroRenderer/VROARDepthMesh.mm

@@ -0,0 +1,53 @@
+//
+//  VROARDepthMesh.cpp
+//  ViroRenderer
+//
+//  Copyright © 2024 Viro Media. All rights reserved.
+//
+//  Permission is hereby granted, free of charge, to any person obtaining
+//  a copy of this software and associated documentation files (the
+//  "Software"), to deal in the Software without restriction, including
+//  without limitation the rights to use, copy, modify, merge, publish,
+//  distribute, sublicense, and/or sell copies of the Software, and to
+//  permit persons to whom the Software is furnished to do so, subject to
+//  the following conditions:
+//
+//  The above copyright notice and this permission notice shall be included
+//  in all copies or substantial portions of the Software.
+//
+//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+//  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+//  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+//  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+//  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+//  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+//  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+#include "VROARDepthMesh.h"
+#include <numeric>
+
+VROARDepthMesh::VROARDepthMesh() {}
+
+VROARDepthMesh::VROARDepthMesh(std::vector<VROVector3f> vertices,
+                               std::vector<int> indices,
+                               std::vector<float> confidences)
+    : _vertices(std::move(vertices)),
+      _indices(std::move(indices)),
+      _confidences(std::move(confidences)) {
+}
+
+VROARDepthMesh::~VROARDepthMesh() {}
+
+float VROARDepthMesh::getAverageConfidence() const {
+    if (_confidences.empty()) {
+        return 0.0f;
+    }
+    float sum = std::accumulate(_confidences.begin(), _confidences.end(), 0.0f);
+    return sum / static_cast<float>(_confidences.size());
+}
+
+bool VROARDepthMesh::isValid() const {
+    return !_vertices.empty() &&
+           !_indices.empty() &&
+           _indices.size() % 3 == 0;
+}

ViroRenderer/VROARFrame.h

@@ -39,6 +39,7 @@
 class VROARCamera;
 class VROARAnchor;
 class VROARHitTestResult;
+class VROARDepthMesh;
 class VROTexture;
 enum class VROARHitTestResultType;
 enum class VROCameraOrientation;
@@ -148,6 +149,22 @@ class VROARFrame {
      */
     virtual int getDepthImageHeight() const { return 0; }
 
+    /**
+     * Generate a triangulated mesh from depth data for physics collision.
+     * The mesh vertices are in world space coordinates.
+     *
+     * @param stride Sample every Nth pixel (1=full resolution, 4=recommended for performance)
+     * @param minConfidence Skip pixels with confidence below this threshold (0.0-1.0)
+     * @param maxDepth Skip pixels with depth beyond this distance in meters
+     * @return Shared pointer to depth mesh, or nullptr if depth data is unavailable
+     */
+    virtual std::shared_ptr<VROARDepthMesh> generateDepthMesh(
+        int stride = 4,
+        float minConfidence = 0.3f,
+        float maxDepth = 5.0f) {
+        return nullptr;
+    }
+
     /*
      Returns the transform matrix to convert from camera texture coordinates
      to depth texture coordinates. The depth map may have a different

ViroRenderer/VROARPointCloud.h

@@ -28,8 +28,8 @@
 
 #include <cstdint>
 #include <vector>
+#include "VROVector4f.h"
 
-class VROVector4f;
 class VROMatrix3f;
 
 // TODO: possible need to guard uint64_t usage with #ifdef UINT64_MAX, seems fine on iOS/Android

ViroRenderer/VROARScene.cpp

@@ -37,6 +37,7 @@
 #include "VROMaterial.h"
 #include "VROImageUtil.h"
 #include "VROARCamera.h"
+#include "VROPencil.h"
 
 VROARScene::~VROARScene() {
     // no-op, we define this here vs in header because for some reason if you want
@@ -137,6 +138,7 @@ void VROARScene::setDriver(std::shared_ptr<VRODriver> driver) {
 
 void VROARScene::updateParticles(const VRORenderContext &context) {
     updatePointCloud();
+    updateWorldMesh();
     VROScene::updateParticles(context);
 }
 
@@ -279,3 +281,81 @@ void VROARScene::willDisappear() {
         _declarativeSession->sceneWillDisappear();
     }
 }
+
+#pragma mark - World Mesh
+
+void VROARScene::setWorldMeshEnabled(bool enabled) {
+    _worldMeshEnabled = enabled;
+
+    if (enabled && !_worldMesh) {
+        // Create world mesh manager with our physics world
+        _worldMesh = std::make_shared<VROARWorldMesh>(getPhysicsWorld());
+        _worldMesh->setConfig(_worldMeshConfig);
+
+        // Set up callback to notify delegate
+        std::weak_ptr<VROARSceneDelegate> weakDelegate = _delegate;
+        _worldMesh->setUpdateCallback([weakDelegate](const VROWorldMeshStats& stats) {
+            std::shared_ptr<VROARSceneDelegate> delegate = weakDelegate.lock();
+            if (delegate) {
+                delegate->onWorldMeshUpdated(stats);
+            }
+        });
+    }
+
+    if (_worldMesh) {
+        _worldMesh->setEnabled(enabled);
+    }
+
+    pinfo("VROARScene: World mesh %s", enabled ? "enabled" : "disabled");
+}
+
+bool VROARScene::isWorldMeshEnabled() const {
+    return _worldMeshEnabled;
+}
+
+void VROARScene::setWorldMeshConfig(const VROWorldMeshConfig& config) {
+    _worldMeshConfig = config;
+
+    if (_worldMesh) {
+        _worldMesh->setConfig(config);
+    }
+}
+
+VROWorldMeshStats VROARScene::getWorldMeshStats() const {
+    if (_worldMesh) {
+        return _worldMesh->getStats();
+    }
+    return VROWorldMeshStats();
+}
+
+void VROARScene::updateWorldMesh() {
+    if (!_worldMesh || !_worldMeshEnabled) {
+        return;
+    }
+
+    std::shared_ptr<VROARSession> arSession = _arSession.lock();
+    if (!arSession) {
+        return;
+    }
+
+    std::unique_ptr<VROARFrame> &frame = arSession->getLastFrame();
+    if (!frame) {
+        return;
+    }
+
+    _worldMesh->updateFromFrame(frame);
+}
+
+void VROARScene::computePhysics(const VRORenderContext &context) {
+    // Call base class physics computation
+    VROScene::computePhysics(context);
+
+    // Draw world mesh wireframe if enabled
+    debugDrawWorldMesh(context);
+}
+
+void VROARScene::debugDrawWorldMesh(const VRORenderContext &context) {
+    if (_worldMesh && _worldMeshEnabled) {
+        _worldMesh->debugDraw(context.getPencil());
+    }
+}