update

Author: younghyopark

.DS_Store

@@ -1,12 +1,16 @@
 VisionProTeleop
 ===========
 
+> **🎉 UPDATE: Now supporting Low-Latency Video Streaming!** You can now stream back your robot's camera feed back to Vision Pro via webRTC protocol, alongside the original hand tracking data stream. 
+No complicated network setting required. Download the app, `pip install avp_stream`, and you're done!
+
 ![CleanShot 2024-03-03 at 13 55 11@2x](https://github.com/Improbable-AI/VisionProTeleop/assets/68195716/d87a906c-ccf3-4e2d-bd25-a66dc0df803b)
 
 
 
-Wanna use your new Apple Vision Pro to control your robot?  Wanna record how you navigate and manipulate the world to train your robot? 
-This VisionOS app and python library streams your Head + Wrist + Hand Tracking result via gRPC over a WiFi network, so any robots connected to the same wifi network can subscribe and use. 
+Wanna use your new Apple Vision Pro to control your robot?  Wanna record how you navigate and manipulate the world?  
+
+This VisionOS app and python library streams your Head + Wrist + Hand Tracking result via gRPC over a WiFi network, so any robots connected to the same wifi network can subscribe and use. **It can also stream stereo (or mono) camera feeds from your robot, back to the Vision Pro.**
 
 > **For a more detailed explanation, check out this short [paper](./assets/short_paper_new.pdf).**
 
@@ -51,13 +55,57 @@ Then, add this code snippet to any of your projects you were developing:
 ```python
 from avp_stream import VisionProStreamer
 avp_ip = "10.31.181.201"   # example IP 
-s = VisionProStreamer(ip = avp_ip, record = True)
+s = VisionProStreamer(ip = avp_ip)
+
+while True:
+    r = s.latest
+    print(r['head'], r['right_wrist'], r['right_fingers'])
+```
+
+### Step 4. [🎉NEW FEATURE] Stream video feeds from your robot back to Vision Pro! 
+
+Streaming your robot's video feed back to Vision Pro requires one additional line: `start_video_streaming`. 
+
+```python
+from avp_stream import VisionProStreamer
+avp_ip = "10.31.181.201"   # example IP 
+s = VisionProStreamer(ip = avp_ip)
+
+# you can simply start a video stream 
+# by defining which video device you want to use
+s.start_video_streaming(device="/dev/video0", format="v4l2", \
+                        size="640x480", fps=30)
 
 while True:
     r = s.latest
     print(r['head'], r['right_wrist'], r['right_fingers'])
 ```
 
+You can also: 
+
+- image-process the camera frames before you send it to Vision Pro 
+  ```python
+  s = VisionProStreamer(ip = avp_ip)
+  # define your own image processing function, and register
+  s.register_frame_callback(my_own_processor)
+  s.start_video_streaming(device="/dev/video0", format="v4l2", \
+                        size="640x480", fps=30)
+  ```
+- work without a physical camera and send over synthetically generated frames (i.e., rendered from simulation)
+  ```python
+  s = VisionProStreamer(ip = avp_ip)
+  # define your own image generating function, and register
+  s.register_frame_callback(synthetic_frame_generator)
+  s.start_video_streaming(device = None, format = None, \
+                          size="1280x720", fps=60)
+  ```
+
+which is explained in detail in [examples](examples) folder. 
+
+> **NOTE**:  Finding the right combination of `device`, `format`, `size`, and `fps` can sometime be tricky, since camera models only support certain combination. Also, depending on your operation system and how you plugged the camera, the `device` might have been mounted on a different directory. 
+
+
+
 
 
 ## Available Data
@@ -90,6 +138,8 @@ r['left_wrist_roll']: float
 ```
 
 
+
+
 ### Axis Convention
 
 Refer to the image below to see how the axis are defined for your head, wrist, and fingers. 
@@ -104,6 +154,19 @@ Refer to the image below to see how the axis are defined for your head, wrist, a
 Refer to the image above to see what order the joints are represented in each hand's skeleton. 
 
 
+## App Features
+
+### Status Window 
+
+
+
+### Modifying the Video Viewport
+
+
+
+### Temporarily Hiding Video Stream  
+
+
 ## Acknowledgements 
 
 We acknowledge support from Hyundai Motor Company and ARO MURI grant number W911NF-23-1-0277. 

README.md

@@ -30,6 +30,7 @@
 		C1893C762B93C1AC00F2269D /* handtracking.grpc.swift in Sources */ = {isa = PBXBuildFile; fileRef = C1893C742B93C1AC00F2269D /* handtracking.grpc.swift */; };
 		C1893C772B93C1AC00F2269D /* handtracking.pb.swift in Sources */ = {isa = PBXBuildFile; fileRef = C1893C752B93C1AC00F2269D /* handtracking.pb.swift */; };
 		C1DA80C42ECE377600B55CC2 /* StatusView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C1DA80C32ECE377600B55CC2 /* StatusView.swift */; };
+		C1DA80C72ECE8C6F00B55CC2 /* ViewControlSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = C1DA80C62ECE8C6F00B55CC2 /* ViewControlSettings.swift */; };
 		C1E05F782ECD84DF0018C506 /* LiveKitWebRTC in Frameworks */ = {isa = PBXBuildFile; productRef = C1E05F772ECD84DF0018C506 /* LiveKitWebRTC */; };
 		C1E05F7C2ECD850B0018C506 /* ImmersiveView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C1E05F7A2ECD850B0018C506 /* ImmersiveView.swift */; };
 		C1E05F7D2ECD850B0018C506 /* ImageData.swift in Sources */ = {isa = PBXBuildFile; fileRef = C1E05F792ECD850B0018C506 /* ImageData.swift */; };
@@ -65,8 +66,9 @@
 		16EE5E9A2B576DB000D354ED /* 🧩Model.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "🧩Model.swift"; sourceTree = "<group>"; };
 		C1893C742B93C1AC00F2269D /* handtracking.grpc.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = handtracking.grpc.swift; path = avp_stream/grpc_msg/handtracking.grpc.swift; sourceTree = SOURCE_ROOT; };
 		C1893C752B93C1AC00F2269D /* handtracking.pb.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = handtracking.pb.swift; path = avp_stream/grpc_msg/handtracking.pb.swift; sourceTree = SOURCE_ROOT; };
-		C19896F42B7D5099003BAF99 /* VisionProTeleop.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = VisionProTeleop.entitlements; sourceTree = "<group>"; };
 		C1DA80C32ECE377600B55CC2 /* StatusView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StatusView.swift; sourceTree = "<group>"; };
+		C1DA80C52ECE821800B55CC2 /* VisionProTeleop.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = VisionProTeleop.entitlements; sourceTree = "<group>"; };
+		C1DA80C62ECE8C6F00B55CC2 /* ViewControlSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewControlSettings.swift; sourceTree = "<group>"; };
 		C1E05F792ECD850B0018C506 /* ImageData.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageData.swift; sourceTree = "<group>"; };
 		C1E05F7A2ECD850B0018C506 /* ImmersiveView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImmersiveView.swift; sourceTree = "<group>"; };
 		C1E05F7B2ECD850B0018C506 /* WebRTCClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebRTCClient.swift; sourceTree = "<group>"; };
@@ -142,9 +144,9 @@
 		165ADB492B4B71B0008A756F /* Tracking Streamer */ = {
 			isa = PBXGroup;
 			children = (
+				C1DA80C52ECE821800B55CC2 /* VisionProTeleop.entitlements */,
 				C1893C742B93C1AC00F2269D /* handtracking.grpc.swift */,
 				C1893C752B93C1AC00F2269D /* handtracking.pb.swift */,
-				C19896F42B7D5099003BAF99 /* VisionProTeleop.entitlements */,
 				165ADB4E2B4B71B0008A756F /* App.swift */,
 				165ADB502B4B71B0008A756F /* ContentView.swift */,
 				C1E05F792ECD850B0018C506 /* ImageData.swift */,
@@ -153,6 +155,7 @@
 				16688DE52B59F35F004CE12B /* 🥽AppModel.swift */,
 				1642FAB82B4D6CAA0084F9ED /* 🌐RealityView.swift */,
 				C1DA80C32ECE377600B55CC2 /* StatusView.swift */,
+				C1DA80C62ECE8C6F00B55CC2 /* ViewControlSettings.swift */,
 				16EE5E982B576DA800D354ED /* 🧩Name.swift */,
 				16EE5E9A2B576DB000D354ED /* 🧩Model.swift */,
 				160ED7972B4B74C5002AD987 /* 🧑HeadTrackingComponent&System.swift */,
@@ -261,6 +264,7 @@
 				16EE5E992B576DA800D354ED /* 🧩Name.swift in Sources */,
 				160ED7982B4B74C5002AD987 /* 🧑HeadTrackingComponent&System.swift in Sources */,
 				1642FAB92B4D6CAA0084F9ED /* 🌐RealityView.swift in Sources */,
+				C1DA80C72ECE8C6F00B55CC2 /* ViewControlSettings.swift in Sources */,
 				1642FAB52B4D54A60084F9ED /* 🛠️SettingPanel.swift in Sources */,
 				16253CFB2B4E2FCB0028F0E2 /* 📏Unit.swift in Sources */,
 				165ADB512B4B71B0008A756F /* ContentView.swift in Sources */,
@@ -411,19 +415,23 @@
 				CODE_SIGN_ENTITLEMENTS = "Tracking Streamer/VisionProTeleop.entitlements";
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 1;
+				CURRENT_PROJECT_VERSION = 2;
 				DEVELOPMENT_ASSET_PATHS = "";
 				DEVELOPMENT_TEAM = ATTMC2WVK2;
 				ENABLE_PREVIEWS = YES;
 				GENERATE_INFOPLIST_FILE = YES;
 				INFOPLIST_FILE = "$(TARGET_NAME)/Supporting files/Info.plist";
 				INFOPLIST_KEY_CFBundleDisplayName = "Tracking Streamer";
+				INFOPLIST_KEY_ITSAppUsesNonExemptEncryption = NO;
 				INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
+				INFOPLIST_KEY_NSCameraUsageDescription = "This app might use your cameras to sense the surrounding environments. ";
+				INFOPLIST_KEY_NSHandsTrackingUsageDescription = "This app tracks your hand to teleoperate a robot. ";
+				INFOPLIST_KEY_NSWorldSensingUsageDescription = "This app senses your surronuding world to reconstruct a simulation.";
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				MARKETING_VERSION = 1.0;
+				MARKETING_VERSION = 2.0;
 				PRODUCT_BUNDLE_IDENTIFIER = improbable.younghyo.DexTeleop2;
 				PRODUCT_NAME = "$(TARGET_NAME)";
 				PROVISIONING_PROFILE_SPECIFIER = "";
@@ -443,19 +451,23 @@
 				CODE_SIGN_ENTITLEMENTS = "Tracking Streamer/VisionProTeleop.entitlements";
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 1;
+				CURRENT_PROJECT_VERSION = 2;
 				DEVELOPMENT_ASSET_PATHS = "";
 				DEVELOPMENT_TEAM = ATTMC2WVK2;
 				ENABLE_PREVIEWS = YES;
 				GENERATE_INFOPLIST_FILE = YES;
 				INFOPLIST_FILE = "$(TARGET_NAME)/Supporting files/Info.plist";
 				INFOPLIST_KEY_CFBundleDisplayName = "Tracking Streamer";
+				INFOPLIST_KEY_ITSAppUsesNonExemptEncryption = NO;
 				INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
+				INFOPLIST_KEY_NSCameraUsageDescription = "This app might use your cameras to sense the surrounding environments. ";
+				INFOPLIST_KEY_NSHandsTrackingUsageDescription = "This app tracks your hand to teleoperate a robot. ";
+				INFOPLIST_KEY_NSWorldSensingUsageDescription = "This app senses your surronuding world to reconstruct a simulation.";
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				MARKETING_VERSION = 1.0;
+				MARKETING_VERSION = 2.0;
 				PRODUCT_BUNDLE_IDENTIFIER = improbable.younghyo.DexTeleop2;
 				PRODUCT_NAME = "$(TARGET_NAME)";
 				PROVISIONING_PROFILE_SPECIFIER = "";

Tracking Streamer.xcodeproj/project.pbxproj

@@ -149,17 +149,14 @@ func getIPAddress() -> String {
 
             guard let interface = ptr?.pointee else { return "" }
             let addrFamily = interface.ifa_addr.pointee.sa_family
-            if addrFamily == UInt8(AF_INET) || addrFamily == UInt8(AF_INET6) {
-
-                // wifi = ["en0"]
-                // wired = ["en2", "en3", "en4"]
-                // cellular = ["pdp_ip0","pdp_ip1","pdp_ip2","pdp_ip3"]
-
+            if addrFamily == UInt8(AF_INET) {
+                // Only check en0 (WiFi interface)
                 let name: String = String(cString: (interface.ifa_name))
-                if  name == "en0" || name == "en2" || name == "en3" || name == "en4" || name == "pdp_ip0" || name == "pdp_ip1" || name == "pdp_ip2" || name == "pdp_ip3" {
+                if name == "en0" {
                     var hostname = [CChar](repeating: 0, count: Int(NI_MAXHOST))
                     getnameinfo(interface.ifa_addr, socklen_t((interface.ifa_addr.pointee.sa_len)), &hostname, socklen_t(hostname.count), nil, socklen_t(0), NI_NUMERICHOST)
                     address = String(cString: hostname)
+                    break  // Found en0, no need to continue
                 }
             }
         }