Add glass-to-glass latency benchmark documentation and comparison image

- Introduced a new markdown file `benchmark.md` detailing the glass-to-glass latency measurement methodology, test configurations, and results for video streaming performance.
- Added a new image file `comparison.png` for visual representation of benchmark results.

Author: younghyopark

README.md

@@ -29,12 +29,22 @@ This VisionOS app and python library streams your Head + Wrist + Hand Tracking r
 
 [![Star History Chart](https://api.star-history.com/svg?repos=improbable-ai/visionproteleop&type=date&legend=top-left)](https://www.star-history.com/#improbable-ai/visionproteleop&type=date&legend=top-left)
 
+
+## Benchmark Results
+
+We performed comprehensive glass-to-glass latency measurements to evaluate the end-to-end performance of our video streaming system. The results show consistently low latency across all tested resolutions, with wired connections achieving **~20ms** at lower resolutions and wireless connections maintaining **~50-100ms** even at 4K.
+
+For detailed methodology, test configurations, and complete results, see the **[Benchmark Documentation](docs/benchmark.md)**.
+
+![](comparison.png)
+
+
 ## How to Use
 
 If you use this repository in your work, consider citing:
 
     @software{park2024avp,
-        title={Using Apple Vision Pro to Train and Control Robots},
+        title={Using Apple Vision Pro to Train and Control Robots},`
         author={Park, Younghyo and Agrawal, Pulkit},
         year={2024},
         url = {https://github.com/Improbable-AI/VisionProTeleop},

Tracking Streamer.xcodeproj/project.xcworkspace/xcuserdata/yhpark.xcuserdatad/UserInterfaceState.xcuserstate

@@ -45,6 +45,7 @@ struct StatusOverlay: View {
     @State private var webrtcConnected: Bool = false
     @State private var hidePreviewTask: Task<Void, Never>?
     @State private var showStatusPositionControls: Bool = false
+    @State private var showExitConfirmation: Bool = false
 
     init(hasFrames: Binding<Bool> = .constant(false), showVideoStatus: Bool = true, isMinimized: Binding<Bool> = .constant(false), showViewControls: Binding<Bool> = .constant(false), previewZDistance: Binding<Float?> = .constant(nil), previewActive: Binding<Bool> = .constant(false), userInteracted: Binding<Bool> = .constant(false), videoMinimized: Binding<Bool> = .constant(false), videoFixed: Binding<Bool> = .constant(false), previewStatusPosition: Binding<(x: Float, y: Float)?> = .constant(nil), previewStatusActive: Binding<Bool> = .constant(false)) {
         self._hasFrames = hasFrames
@@ -161,7 +162,7 @@ struct StatusOverlay: View {
             .buttonStyle(.plain)
 
             Button {
-                exit(0)
+                showExitConfirmation = true
             } label: {
                 ZStack {
                     Circle()
@@ -173,10 +174,17 @@ struct StatusOverlay: View {
                 }
             }
             .buttonStyle(.plain)
+            .confirmationDialog("Are you sure you want to exit?", isPresented: $showExitConfirmation, titleVisibility: .visible) {
+                Button("Exit", role: .destructive) {
+                    exit(0)
+                }
+                Button("Cancel", role: .cancel) {}
+            }
         }
         .padding(30)
         .background(Color.black.opacity(0.6))
         .cornerRadius(36)
+        .fixedSize()
     }
 
     private var expandedView: some View {
@@ -206,7 +214,7 @@ struct StatusOverlay: View {
                 Spacer()
 
                 Button {
-                    exit(0)
+                    showExitConfirmation = true
                 } label: {
                     ZStack {
                         Circle()
@@ -218,6 +226,12 @@ struct StatusOverlay: View {
                     }
                 }
                 .buttonStyle(.plain)
+                .confirmationDialog("Are you sure you want to exit?", isPresented: $showExitConfirmation, titleVisibility: .visible) {
+                    Button("Exit", role: .destructive) {
+                        exit(0)
+                    }
+                    Button("Cancel", role: .cancel) {}
+                }
             }
 
             Divider()
@@ -853,6 +867,7 @@ struct StatusPreviewView: View {
         .padding(30)
         .background(Color.black.opacity(0.6))
         .cornerRadius(36)
+        .fixedSize()
         .opacity(0.5)  // 50% transparent
     }
 }

Tracking Streamer/StatusView.swift

@@ -36,11 +36,19 @@
         }
       }
     },
+    "Are you sure you want to exit?" : {
+      "comment" : "A confirmation dialog that appears when the user taps the exit button.",
+      "isCommentAutoGenerated" : true
+    },
     "Audio:" : {
 
     },
     "Auto-Perpendicular" : {
 
+    },
+    "Cancel" : {
+      "comment" : "A button that cancels the exit confirmation dialog.",
+      "isCommentAutoGenerated" : true
     },
     "Connected" : {
       "comment" : "The text that appears when the user's WebRTC connection is established.",

Tracking Streamer/Supporting files/Localizable.xcstrings

@@ -82,6 +82,8 @@ def main():
     parser.add_argument("--resolution", type=str, default="640x480", help="Streaming resolution (WIDTHxHEIGHT)")
     parser.add_argument("--sweep", action="store_true", help="Sweep over common resolutions")
     parser.add_argument("--trials", type=int, default=1000, help="Number of trials per resolution in sweep mode")
+    parser.add_argument("--warmup", type=int, default=100, help="Number of warmup frames before collecting data")
+    parser.add_argument("--stereo", action="store_true", help="Enable stereo video streaming (doubles width)")
     args = parser.parse_args()
 
     print(f"Connecting to Vision Pro at {args.ip}...")
@@ -97,14 +99,26 @@ def __init__(self):
             self.collecting = False
             self.target_trials = 0
             self.current_resolution = ""
+            self.last_frame_time = 0
+            self.warmup_frames = 0
+            self.in_warmup = False
 
-    state = LatencyState()
+    state = LatencyState()    
 
     def frame_callback(frame):
         # 1. Get current timestamp
         now = time.perf_counter()
         if state.sequence_id == 0:
             state.streamer.reset_benchmark_epoch(now)
+            state.last_frame_time = now
+        else:
+            # Sleep to maintain target FPS (30fps = 33.33ms per frame)
+            elapsed = now - state.last_frame_time
+            target_frame_time = 1.0 / args.fps
+            # if elapsed < target_frame_time:
+            #     time.sleep(target_frame_time - elapsed)
+            now = time.perf_counter()
+            state.last_frame_time = now
 
         timestamp_ms = int((now - state.streamer._benchmark_epoch) * 1000)
 
@@ -120,17 +134,32 @@ def frame_callback(frame):
     streamer.register_frame_callback(frame_callback)
 
     # Define resolutions for sweep
-    resolutions = [
-        ("240p", "426x240"),
-        ("360p", "640x360"),
-        ("720p", "1280x720"),
-        ("1080p", "1920x1080"),
-        ("2160p", "3840x2160")
-    ]
+    if args.stereo:
+        # Double the width for stereo mode
+        resolutions = [
+            ("240p", "852x240"),
+            ("360p", "1280x360"),
+            ("720p", "2560x720"),
+            ("1080p", "3840x1080"),
+            ("2160p", "7680x2160")
+        ]
+    else:
+        resolutions = [
+            ("240p", "426x240"),
+            ("360p", "640x360"),
+            ("720p", "1280x720"),
+            ("1080p", "1920x1080"),
+            ("2160p", "3840x2160")
+        ]
 
     if not args.sweep:
         # Single resolution mode
-        streamer.start_video_streaming(device=None, size=args.resolution, fps=args.fps)
+        resolution = args.resolution
+        if args.stereo:
+            # Double the width for stereo
+            width, height = map(int, resolution.split('x'))
+            resolution = f"{width * 2}x{height}"
+        streamer.start_video_streaming(device=None, size=resolution, fps=args.fps, stereo_video=args.stereo)
         print("Streaming started. Measuring latency...")
         try:
             while True:
@@ -141,7 +170,7 @@ def frame_callback(frame):
         # Sweep mode
         # Start with the first resolution
         first_res_name, first_res_size = resolutions[0]
-        streamer.start_video_streaming(device=None, size=first_res_size, fps=args.fps)
+        streamer.start_video_streaming(device=None, size=first_res_size, fps=args.fps, stereo_video=args.stereo)
 
         print(f"Starting sweep over {len(resolutions)} resolutions: {[r[0] for r in resolutions]}")
 
@@ -151,7 +180,9 @@ def frame_callback(frame):
             "test_config": {
                 "ip": args.ip,
                 "fps": args.fps,
-                "trials_per_resolution": args.trials
+                "trials_per_resolution": args.trials,
+                "warmup_frames": args.warmup,
+                "stereo": args.stereo
             },
             "resolutions": {}
         }
@@ -169,6 +200,33 @@ def frame_callback(frame):
             state.latencies = []
             state.collecting = True
             state.target_trials = args.trials
+            state.warmup_frames = args.warmup
+            state.in_warmup = True
+            
+            # Warmup phase
+            print(f"🔥 Warming up ({args.warmup} frames)...")
+            warmup_start_seq = state.sequence_id
+            warmup_pbar = tqdm(total=args.warmup, desc="Warmup", unit="frames")
+            
+            warmup_check_seq = warmup_start_seq
+            warmup_collected = 0
+            
+            while warmup_collected < args.warmup:
+                event = streamer.wait_for_benchmark_event(warmup_check_seq, timeout=1.0)
+                
+                if event:
+                    warmup_collected += 1
+                    warmup_pbar.update(1)
+                    warmup_check_seq += 1
+                else:
+                    # Check if we should skip
+                    if state.sequence_id > warmup_check_seq + 100:
+                        warmup_pbar.write(f"⚠️  Skipping missing warmup sequence {warmup_check_seq}")
+                        warmup_check_seq += 1
+            
+            warmup_pbar.close()
+            state.in_warmup = False
+            print(f"✅ Warmup complete. Starting data collection...")
 
             # Collect samples
             # We need to capture the return events.

avp_stream/latency_test.py

@@ -0,0 +1,130 @@
+import json
+import argparse
+import matplotlib.pyplot as plt
+import numpy as np
+from pathlib import Path
+
+def load_benchmark(filepath):
+    """Load a benchmark JSON file."""
+    with open(filepath, 'r') as f:
+        return json.load(f)
+
+def plot_benchmarks(json_files, output_path=None):
+    """
+    Plot multiple benchmark results comparing mean latency and jitter across resolutions.
+    
+    Args:
+        json_files: List of paths to benchmark JSON files
+        output_path: Optional path to save the plot (if None, displays interactively)
+    """
+    # Color palette for different benchmark files
+    colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', 
+              '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']
+    
+    # Resolution order and labels
+    resolution_order = ['240p', '360p', '720p', '1080p', '2160p']
+    resolution_labels = {
+        '240p': '240p\n(426×240)',
+        '360p': '360p\n(640×360)',
+        '720p': '720p\n(1280×720)',
+        '1080p': '1080p\n(1920×1080)',
+        '2160p': '2160p\n(3840×2160)'
+    }
+    
+    # Load all benchmark data
+    benchmarks = []
+    for filepath in json_files:
+        data = load_benchmark(filepath)
+        label = Path(filepath).stem  # Use filename without extension as label
+        benchmarks.append({
+            'label': label,
+            'data': data
+        })
+    
+    # Create figure with single plot
+    fig, ax2 = plt.subplots(1, 1, figsize=(10, 6))
+    # fig.suptitle('Glass-to-Glass Latency Benchmark Comparison', fontsize=16, fontweight='bold')
+    
+    x_positions = np.arange(len(resolution_order))
+    
+    # Plot jitter (standard deviation) as line plot with shaded region
+    for idx, benchmark in enumerate(benchmarks):
+        means = []
+        stds = []
+        p95s = []
+        p99s = []
+        
+        for res in resolution_order:
+            if res in benchmark['data']['resolutions']:
+                res_data = benchmark['data']['resolutions'][res]
+                if 'error' not in res_data:
+                    means.append(res_data['mean_ms'])
+                    stds.append(res_data['std_ms'])
+                    p95s.append(res_data.get('p95_ms', 0))
+                    p99s.append(res_data.get('p99_ms', 0))
+                else:
+                    means.append(0)
+                    stds.append(0)
+                    p95s.append(0)
+                    p99s.append(0)
+            else:
+                means.append(0)
+                stds.append(0)
+                p95s.append(0)
+                p99s.append(0)
+        
+        means = np.array(means)
+        stds = np.array(stds)
+        
+        color = colors[idx % len(colors)]
+        
+        # Plot mean line
+        ax2.plot(x_positions, means, marker='o', linewidth=2, 
+                label=f'{benchmark["label"]}', color=color, markersize=8)
+        
+        # Plot shaded region for ±1 std deviation (no legend entry)
+        ax2.fill_between(x_positions, means - stds, means + stds, 
+                         alpha=0.3, color=color)
+    
+    ax2.set_xlabel('Resolution', fontsize=12, fontweight='bold')
+    ax2.set_ylabel('Latency (ms)', fontsize=12, fontweight='bold')
+    ax2.set_title('Glass-to-Glass Latency Test', fontsize=13, fontweight='bold', pad=10)
+    ax2.set_xticks(x_positions)
+    ax2.set_xticklabels([resolution_labels[r] for r in resolution_order])
+    ax2.legend(loc='upper left', framealpha=0.9, fontsize=9)
+    ax2.grid(True, alpha=0.3, linestyle='--')
+    ax2.set_axisbelow(True)
+    
+    # Add test configuration info as text
+    if benchmarks:
+        config = benchmarks[0]['data']['test_config']
+        info_text = f"Test Config: {config['trials_per_resolution']} trials/resolution. For stereo tests, width is doubled."
+        if 'warmup_frames' in config:
+            info_text += f", {config['warmup_frames']} warmup frames"
+        fig.text(0.5, 0.02, info_text, ha='center', fontsize=10, style='italic', color='gray')
+    
+    plt.tight_layout(rect=[0, 0.04, 1, 0.96])
+    
+    if output_path:
+        plt.savefig(output_path, dpi=300, bbox_inches='tight')
+        print(f"✅ Plot saved to: {output_path}")
+    else:
+        plt.show()
+
+def main():
+    parser = argparse.ArgumentParser(description="Plot multiple benchmark results for comparison")
+    parser.add_argument('json_files', nargs='+', help='Paths to benchmark JSON files')
+    parser.add_argument('--output', '-o', type=str, help='Output path for the plot (PNG/PDF)')
+    args = parser.parse_args()
+    
+    # Verify all files exist
+    for filepath in args.json_files:
+        if not Path(filepath).exists():
+            print(f"❌ Error: File not found: {filepath}")
+            return
+    
+    print(f"📊 Plotting {len(args.json_files)} benchmark file(s)...")
+    plot_benchmarks(args.json_files, args.output)
+
+if __name__ == "__main__":
+    main()