FCE-3493: Add ability to use custom camera / custom source as a video track

FishjamReactNativeWebrtc.podspec

@@ -33,7 +33,7 @@ Pod::Spec.new do |s|
   # (miniaudio is not ABI-compatible across differing configs).
   s.pod_target_xcconfig = {
     'CLANG_CXX_LANGUAGE_STANDARD' => 'c++20',
-    'HEADER_SEARCH_PATHS' => '"$(PODS_TARGET_SRCROOT)/common/cpp/vendor" "$(PODS_TARGET_SRCROOT)/common/cpp/fishjam-audio"',
+    'HEADER_SEARCH_PATHS' => '"$(PODS_TARGET_SRCROOT)/common/cpp/vendor" "$(PODS_TARGET_SRCROOT)/common/cpp/fishjam-audio" "$(PODS_TARGET_SRCROOT)/common/cpp/fishjam-video"',
     'GCC_PREPROCESSOR_DEFINITIONS' =>
       'MA_NO_DEVICE_IO=1 MA_NO_DECODING=1 MA_NO_ENCODING=1 MA_NO_GENERATION=1 ' \
       'MA_NO_RESOURCE_MANAGER=1 MA_NO_NODE_GRAPH=1 $(inherited)',

android/build.gradle

@@ -19,7 +19,7 @@ android {
     }
 
     defaultConfig {
-        minSdkVersion 24
+        minSdkVersion safeExtGet('minSdkVersion', 24)
         targetSdkVersion safeExtGet('targetSdkVersion', 24)
         versionCode 1
         versionName "1.0"
@@ -28,7 +28,15 @@ android {
         externalNativeBuild {
             cmake {
                 cppFlags "-std=c++20"
-                arguments "-DANDROID_STL=c++_shared"
+                // The custom-video-track lib references AHardwareBuffer_* (__INTRODUCED_IN(26))
+                // while this package is minSdk 24. ANDROID_WEAK_API_DEFS makes those symbols
+                // weak so the lib is loadable even on API 24-25, with the __builtin_available
+                // guards gating every call at runtime. The API-26 gate in
+                // WebRTCModule.installCustomVideoJSI (and createCustomVideoTrack) remains the
+                // primary guard; weak linking is insurance so any future code path that loads
+                // the lib on an older device degrades gracefully instead of throwing
+                // UnsatisfiedLinkError.
+                arguments "-DANDROID_STL=c++_shared", "-DANDROID_WEAK_API_DEFS=ON"
             }
         }
 

android/src/main/cpp/CMakeLists.txt

@@ -53,3 +53,46 @@ target_link_libraries(fishjam-webrtc-audio
     android
     log
 )
+
+# --- Custom video track ----------------------------------------------------
+#
+# Loaded from Java via System.loadLibrary("webrtc-custom-video-track") by
+# AHardwareBufferAllocator, CustomVideoFrameDelivery and FJVideoPushInstaller.
+#   - ahardware_buffer_alloc.cpp : allocate GPU_FRAMEBUFFER | GPU_SAMPLED_IMAGE AHBs
+#                                 that react-native-webgpu (Dawn) renders into.
+#   - custom_video_gl.cpp       : import each AHB as a GL_TEXTURE_EXTERNAL_OES
+#                                 texture (AHB -> EGLImage -> OES) and wait the
+#                                 sync-fd GPU fence so libwebrtc can encode it.
+#   - FJVideoPushJSI.cpp        : shared JSI core installing the per-frame push
+#                                 global and forwarding each frame to a deliver cb.
+#   - FJVideoPushInstaller.cpp  : fbjni HybridClass wiring the JS CallInvoker to the
+#                                 shared core and routing frames back to Java.
+add_library(webrtc-custom-video-track
+    SHARED
+    "${CMAKE_CURRENT_SOURCE_DIR}/ahardware_buffer_alloc.cpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/custom_video_gl.cpp"
+    "${FJ_COMMON_CPP_DIR}/fishjam-video/FJVideoPushJSI.cpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/FJVideoPushInstaller.cpp"
+)
+
+set_target_properties(webrtc-custom-video-track PROPERTIES
+    CXX_STANDARD 20
+    CXX_STANDARD_REQUIRED ON
+)
+
+target_include_directories(webrtc-custom-video-track
+    PRIVATE
+    "${FJ_COMMON_CPP_DIR}/fishjam-video"
+)
+
+# ReactAndroid / fbjni prefab packages are already located above (for the audio
+# lib) and remain in scope here.
+target_link_libraries(webrtc-custom-video-track
+    ReactAndroid::jsi
+    ReactAndroid::reactnative
+    fbjni::fbjni
+    EGL
+    GLESv2
+    android
+    log
+)

android/src/main/cpp/FJVideoPushInstaller.cpp

@@ -0,0 +1,75 @@
+#include "FJVideoPushInstaller.h"
+
+// facebook::jni::ThreadScope is declared in <fbjni/detail/Environment.h>, pulled
+// in transitively by <fbjni/fbjni.h> (included from FJVideoPushInstaller.h).
+
+namespace jni = facebook::jni;
+
+namespace fishjam {
+
+FJVideoPushInstaller::FJVideoPushInstaller(jni::alias_ref<jhybridobject> javaThis,
+                                           std::shared_ptr<FJVideoPush> push)
+    : javaPart_(jni::make_global(javaThis)), push_(std::move(push)) {
+    // Route every JS-pushed frame back to the Java peer, which forwards it to the
+    // matching CustomVideoFrameDelivery. `nativeBuffer` is the forwarding
+    // discriminator: non-zero => a finished AHardwareBuffer* to forward
+    // (bufferIndex/fence unused); zero => pooled delivery of `bufferIndex` with the
+    // sync-fd carried in fenceHandle (fenceSignaledValue unused on Android — the fd
+    // already encodes the signal — but forwarded for parity with the shared
+    // DeliverFn contract).
+    //
+    // deliver_ now runs on whatever thread called sink.push. For a VisionCamera
+    // frame-processor that is a worklet thread NOT attached to the JVM, so we
+    // establish a ThreadScope (attach the current thread for the duration of the
+    // JNI dispatch) before touching any JNIEnv. ThreadScope is a no-op on threads
+    // already attached (e.g. the RN JS thread), so the pooled path pays nothing.
+    // Capturing the global_ref keeps the Java peer alive for the callback lifetime.
+    auto javaPart = javaPart_;
+    push_->setDeliver([javaPart](const std::string &trackId, int bufferIndex, uint64_t nativeBuffer,
+                                 uint64_t timestampNs, int rotation, uint64_t fenceHandle,
+                                 uint64_t fenceSignaledValue) {
+        facebook::jni::ThreadScope threadScope;
+        static const auto deliverFrame =
+            javaPart->getClass()
+                ->getMethod<void(jni::alias_ref<jstring>, jint, jlong, jlong, jint, jlong, jlong)>(
+                    "deliverFrame");
+        deliverFrame(javaPart, jni::make_jstring(trackId), static_cast<jint>(bufferIndex),
+                     static_cast<jlong>(nativeBuffer), static_cast<jlong>(timestampNs),
+                     static_cast<jint>(rotation), static_cast<jlong>(fenceHandle),
+                     static_cast<jlong>(fenceSignaledValue));
+    });
+}
+
+jni::local_ref<FJVideoPushInstaller::jhybriddata> FJVideoPushInstaller::initHybrid(
+    jni::alias_ref<jhybridobject> javaThis,
+    jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> callInvokerHolder) {
+    // FJVideoPush only needs the JS CallInvoker; it acquires the jsi::Runtime
+    // itself inside invokeAsync, so no runtime pointer is required here.
+    auto callInvoker = callInvokerHolder->cthis()->getCallInvoker();
+    return makeCxxInstance(javaThis, std::make_shared<FJVideoPush>(callInvoker));
+}
+
+void FJVideoPushInstaller::installPush() {
+    // FJVideoPush::install sets the global on the JS thread and then runs this
+    // callback (also on the JS thread). We notify the Java peer there so the
+    // Promise resolves strictly after the global exists. Capturing the global_ref
+    // keeps the Java peer alive until the callback runs.
+    auto javaPart = javaPart_;
+    push_->install([javaPart] {
+        static const auto onPushInstalled = javaPart->getClass()->getMethod<void()>("onPushInstalled");
+        onPushInstalled(javaPart);
+    });
+}
+
+void FJVideoPushInstaller::registerNatives() {
+    registerHybrid({
+        makeNativeMethod("initHybrid", FJVideoPushInstaller::initHybrid),
+        makeNativeMethod("installPush", FJVideoPushInstaller::installPush),
+    });
+}
+
+}  // namespace fishjam
+
+JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
+    return facebook::jni::initialize(vm, [] { fishjam::FJVideoPushInstaller::registerNatives(); });
+}

android/src/main/cpp/FJVideoPushInstaller.h

@@ -0,0 +1,42 @@
+// fbjni HybridClass backing com.oney.WebRTCModule.FJVideoPushInstaller.
+//
+// Installs the JS global `__fishjamWebrtcGetCustomVideoSink` on the JS thread
+// (via the CallInvoker), then notifies the Java peer so the install Promise
+// resolves only once the global actually exists. Each frame pushed through a
+// sink obtained from that global is forwarded back to the Java peer's
+// deliverFrame(...), which routes it to the matching CustomVideoFrameDelivery.
+#pragma once
+
+#include <ReactCommon/CallInvokerHolder.h>
+#include <fbjni/fbjni.h>
+
+#include <memory>
+
+#include "FJVideoPushJSI.h"
+
+namespace fishjam {
+
+class FJVideoPushInstaller : public facebook::jni::HybridClass<FJVideoPushInstaller> {
+   public:
+    static constexpr auto kJavaDescriptor = "Lcom/oney/WebRTCModule/FJVideoPushInstaller;";
+
+    static facebook::jni::local_ref<jhybriddata> initHybrid(
+        facebook::jni::alias_ref<jhybridobject> javaThis,
+        facebook::jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> callInvokerHolder);
+
+    static void registerNatives();
+
+    // Sets the JS global on the JS thread, then calls the Java peer's
+    // onPushInstalled() once it is in place.
+    void installPush();
+
+   private:
+    friend HybridBase;
+
+    facebook::jni::global_ref<javaobject> javaPart_;
+    std::shared_ptr<FJVideoPush> push_;
+
+    FJVideoPushInstaller(facebook::jni::alias_ref<jhybridobject> javaThis, std::shared_ptr<FJVideoPush> push);
+};
+
+}  // namespace fishjam

android/src/main/cpp/ahardware_buffer_alloc.cpp

@@ -0,0 +1,73 @@
+// AHardwareBuffer (AHB) allocation primitives for the Android custom-video-track.
+//
+// Allocates an AHardwareBuffer that is BOTH GPU-renderable (GPU_FRAMEBUFFER) and
+// GPU-sampleable (GPU_SAMPLED_IMAGE), hands its pointer to JS as a jlong, and
+// releases it later. JS/WebGPU imports that pointer via react-native-webgpu's
+// importSharedTextureMemory({handle}) and renders into it; the rendered AHB is
+// then delivered to WebRTC by custom_video_gl.cpp. Pooling (N buffers at stable
+// indices) lives one layer up, in CustomVideoBufferPool.java.
+//
+// The AHardwareBuffer_* APIs are __INTRODUCED_IN(26), so every call is wrapped in
+// __builtin_available(android 26, *) to compile against minSdk 24. Callers
+// (WebRTCModule / GetUserMediaImpl) reject on SDK_INT < 26 before reaching here,
+// so the runtime-unavailable branches are never taken on older devices.
+
+#include <android/hardware_buffer.h>
+#include <jni.h>
+
+extern "C" {
+
+// Allocates a single RGBA8 AHardwareBuffer usable as both a GPU render target
+// and a GPU sampled image, and returns the AHardwareBuffer* reinterpreted as a
+// jlong (64-bit). Returns 0 on failure. The caller owns the reference taken by
+// AHardwareBuffer_allocate; balance it with releaseAHB().
+//
+// Usage flags:
+//   AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE  -> importable + sampleable by Dawn
+//   AHARDWAREBUFFER_USAGE_GPU_FRAMEBUFFER    -> renderable (RENDER_ATTACHMENT)
+// No CPU usage is requested: this is a pure-GPU buffer.
+JNIEXPORT jlong JNICALL
+Java_com_oney_WebRTCModule_AHardwareBufferAllocator_allocateFramebufferAHB(
+        JNIEnv* /* env */, jclass /* clazz */, jint width, jint height) {
+    if (width <= 0 || height <= 0) {
+        return 0;
+    }
+
+    if (__builtin_available(android 26, *)) {
+        AHardwareBuffer_Desc desc = {};
+        desc.width = static_cast<uint32_t>(width);
+        desc.height = static_cast<uint32_t>(height);
+        desc.layers = 1;
+        desc.format = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
+        desc.usage = AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE | AHARDWAREBUFFER_USAGE_GPU_FRAMEBUFFER;
+
+        AHardwareBuffer* buffer = nullptr;
+        int status = AHardwareBuffer_allocate(&desc, &buffer);
+        if (status != 0 || buffer == nullptr) {
+            return 0;
+        }
+
+        return reinterpret_cast<jlong>(buffer);
+    }
+    return 0;
+}
+
+// Releases the reference taken in allocateFramebufferAHB (the +1 from
+// AHardwareBuffer_allocate). Safe to call once per allocated handle; passing 0
+// is a no-op.
+//
+// NOTE: the raw AHB pointer handed to JS (as a long) is a dangling pointer once
+// this returns — it must NOT be imported/used by JS/WebGPU after releaseAHB().
+// The Java pool nulls its handle here so no further frame can reference it.
+JNIEXPORT void JNICALL
+Java_com_oney_WebRTCModule_AHardwareBufferAllocator_releaseAHB(
+        JNIEnv* /* env */, jclass /* clazz */, jlong handle) {
+    if (handle == 0) {
+        return;
+    }
+    if (__builtin_available(android 26, *)) {
+        AHardwareBuffer_release(reinterpret_cast<AHardwareBuffer*>(handle));
+    }
+}
+
+}  // extern "C"