Benchmarks (#925)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: pblazej

.github/workflows/benchmark.yaml

@@ -0,0 +1,46 @@
+name: Benchmarks
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  benchmark:
+    name: Benchmark
+    runs-on: macos-latest
+    timeout-minutes: 60
+    env:
+      LK_BENCHMARK: 1
+      # LIVEKIT_URL: ${{ secrets.BENCHMARK_URL }}
+      # LIVEKIT_API_KEY: ${{ secrets.BENCHMARK_API_KEY }}
+      # LIVEKIT_API_SECRET: ${{ secrets.BENCHMARK_API_SECRET }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install dependencies
+        run: brew install livekit jemalloc swiftly && swiftly init --quiet-shell-followup --skip-install -y
+
+      - name: Run LiveKit Server
+        run: livekit-server --dev &
+
+      - uses: maxim-lobanov/setup-xcode@v1
+        with:
+          xcode-version: latest
+
+      - name: Run Benchmarks
+        working-directory: Benchmarks
+        run: |
+          swiftly run +xcode swift package --disable-sandbox \
+            benchmark baseline update current
+
+      - name: Benchmark Report
+        working-directory: Benchmarks
+        run: |
+          swiftly run +xcode swift package --disable-sandbox \
+            benchmark baseline read current \
+            --format markdown \
+            | tee -a "$GITHUB_STEP_SUMMARY"

AGENTS.md

@@ -12,6 +12,12 @@ xcodebuild build -scheme LiveKit -destination 'platform=macOS'
 # Run tests (requires local server: livekit-server --dev, install via brew install livekit)
 xcodebuild test -scheme LiveKit -only-testing LiveKitCoreTests -destination 'platform=macOS'
 
+# Build benchmarks
+cd Benchmarks && swiftly run +xcode swift build
+
+# Run benchmarks (requires local server: livekit-server --dev)
+cd Benchmarks && LK_BENCHMARK=1 swiftly run +xcode swift package --disable-sandbox benchmark
+
 # List available simulators for platform-specific builds
 xcrun simctl list devices
 ```

Benchmarks/LiveKitBenchmark/BenchmarkSupport/BenchmarkConfig.swift

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+
+/// Configuration for benchmark runs, read from environment variables.
+struct BenchmarkConfig {
+    let url: String
+    let apiKey: String
+    let apiSecret: String
+    let mode: InfrastructureMode
+
+    enum InfrastructureMode {
+        case local
+        case cloud(region: String?)
+    }
+
+    /// Read benchmark configuration from environment variables.
+    ///
+    /// Falls back to local defaults (`ws://localhost:7880`, `devkey`/`secret`)
+    /// when environment variables are not set.
+    ///
+    /// Override with:
+    ///   - `LIVEKIT_URL`: WebSocket URL (e.g., `wss://my-project.livekit.cloud`)
+    ///   - `LIVEKIT_API_KEY`: API key for token generation
+    ///   - `LIVEKIT_API_SECRET`: API secret for token generation
+    ///   - `LIVEKIT_BENCHMARK_REGION`: Server region (only used for cloud)
+    static func fromEnvironment() -> BenchmarkConfig {
+        let url = ProcessInfo.processInfo.environment["LIVEKIT_URL"] ?? "ws://localhost:7880"
+        let apiKey = ProcessInfo.processInfo.environment["LIVEKIT_API_KEY"] ?? "devkey"
+        let apiSecret = ProcessInfo.processInfo.environment["LIVEKIT_API_SECRET"] ?? "secret"
+
+        // Auto-detect mode from URL
+        let mode: InfrastructureMode = if url.hasPrefix("ws://") || url.contains("localhost") {
+            .local
+        } else {
+            .cloud(region: ProcessInfo.processInfo.environment["LIVEKIT_BENCHMARK_REGION"])
+        }
+
+        return BenchmarkConfig(
+            url: url,
+            apiKey: apiKey,
+            apiSecret: apiSecret,
+            mode: mode
+        )
+    }
+}

Benchmarks/LiveKitBenchmark/BenchmarkSupport/BenchmarkTracer.swift

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+import LiveKit
+
+/// A ``Tracing`` implementation that retains completed spans for benchmark analysis.
+///
+/// The default ``LoggingTracer`` logs spans when they end. This implementation
+/// keeps them so benchmarks can extract timing data after operations complete.
+///
+/// Inject via `LiveKitSDK.setTracing()` before running benchmarks.
+
+extension Span {
+    /// All events as label → milliseconds relative to `start`.
+    var splitMilliseconds: [String: Int64] {
+        var result = [String: Int64]()
+        for entry in entries {
+            result[entry.label] = Int64((entry.time - start) * 1000)
+        }
+        return result
+    }
+}
+
+final class BenchmarkTracer: Tracing, @unchecked Sendable {
+    private let _completedSpans = StateSync<[String: Span]>([:])
+
+    @discardableResult
+    func beginSpan(_ name: String) -> Span {
+        let span = Span(label: name)
+        span.onEnd = { [weak self] span in
+            self?._completedSpans.mutate { $0[name] = span }
+        }
+        return span
+    }
+
+    /// Retrieve the most recently completed span with the given name.
+    func completedSpan(_ name: String) -> Span? {
+        _completedSpans.read { $0[name] }
+    }
+
+    /// Clear all completed spans.
+    func reset() {
+        _completedSpans.mutate { $0.removeAll() }
+    }
+}

Benchmarks/LiveKitBenchmark/BenchmarkSupport/EchoParticipant.swift

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+import LiveKit
+
+/// A second SDK instance that acts as an echo participant for
+/// data channel and RPC benchmarks.
+///
+/// Connects to the same room and echoes:
+/// - Data channel messages back to the sender
+/// - RPC calls back with the same payload
+///
+/// Records its own processing timestamps for overhead decomposition.
+final class EchoParticipant: Sendable {
+    let room: Room
+
+    /// Processing timestamps: (receive time, echo sent time) in microseconds
+    /// relative to an arbitrary monotonic epoch.
+    struct ProcessingTimestamp {
+        let receiveUs: Int64
+        let echoSentUs: Int64
+        var overheadUs: Int64 { echoSentUs - receiveUs }
+    }
+
+    private let _timestamps = StateSync<[ProcessingTimestamp]>([])
+    // Strong reference to keep the delegate alive (MulticastDelegate uses weak references)
+    private nonisolated(unsafe) var _echoDelegate: AnyObject?
+
+    var processingTimestamps: [ProcessingTimestamp] {
+        _timestamps.copy()
+    }
+
+    init() {
+        room = Room()
+    }
+
+    /// Connect to the specified room and set up echo handlers.
+    func connect(url: String, token: String) async throws {
+        try await room.connect(url: url, token: token)
+    }
+
+    /// Register the echo RPC handler.
+    ///
+    /// - Parameter delay: Optional delay in nanoseconds to simulate processing (for BM-RPC-003)
+    func registerEchoRpc(delay: UInt64 = 0) async throws {
+        try await room.registerRpcMethod("echo") { data in
+            if delay > 0 {
+                try await Task.sleep(nanoseconds: delay)
+            }
+            return data.payload
+        }
+    }
+
+    /// Set up data echo handler that echoes received data back to the sender.
+    func setupDataEcho() {
+        let delegate = DataEchoDelegate(room: room, timestamps: _timestamps)
+        _echoDelegate = delegate
+        room.delegates.add(delegate: delegate)
+    }
+
+    func disconnect() async {
+        await room.disconnect()
+    }
+
+    func clearTimestamps() {
+        _timestamps.mutate { $0.removeAll() }
+    }
+}
+
+/// Delegate that echoes data channel messages.
+private final class DataEchoDelegate: NSObject, RoomDelegate, @unchecked Sendable {
+    private let room: Room
+    private let timestamps: StateSync<[EchoParticipant.ProcessingTimestamp]>
+
+    init(room: Room, timestamps: StateSync<[EchoParticipant.ProcessingTimestamp]>) {
+        self.room = room
+        self.timestamps = timestamps
+        super.init()
+    }
+
+    func room(_: Room, participant _: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType _: EncryptionType) {
+        let recvTime = Int64(ProcessInfo.processInfo.systemUptime * 1_000_000)
+        Task {
+            try? await room.localParticipant.publish(
+                data: data,
+                options: .init(topic: topic)
+            )
+            let sentTime = Int64(ProcessInfo.processInfo.systemUptime * 1_000_000)
+            timestamps.mutate {
+                $0.append(EchoParticipant.ProcessingTimestamp(
+                    receiveUs: recvTime,
+                    echoSentUs: sentTime
+                ))
+            }
+        }
+    }
+}

Benchmarks/LiveKitBenchmark/BenchmarkSupport/TokenGenerator.swift

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+import LiveKitUniFFI
+
+/// Generates LiveKit access tokens for benchmark participants.
+///
+/// Uses LiveKitUniFFI's `tokenGenerate` function, which provides the same
+/// HMAC-SHA256 JWT signing as the server SDKs.
+struct TokenGenerator {
+    let apiKey: String
+    let apiSecret: String
+
+    /// Generate a LiveKit access token.
+    ///
+    /// - Parameters:
+    ///   - roomName: Room to grant access to
+    ///   - identity: Participant identity
+    ///   - canPublish: Whether the participant can publish tracks
+    ///   - canSubscribe: Whether the participant can subscribe to tracks
+    ///   - ttl: Token time-to-live in seconds (default: 300s / 5 minutes)
+    func generate(
+        roomName: String,
+        identity: String,
+        canPublish: Bool = true,
+        canSubscribe: Bool = true,
+        ttl: TimeInterval = 300
+    ) -> String {
+        let grants = VideoGrants(
+            roomCreate: false,
+            roomList: false,
+            roomRecord: false,
+            roomAdmin: false,
+            roomJoin: true,
+            room: roomName,
+            destinationRoom: "",
+            canPublish: canPublish,
+            canSubscribe: canSubscribe,
+            canPublishData: true,
+            canPublishSources: [],
+            canUpdateOwnMetadata: false,
+            ingressAdmin: false,
+            hidden: false,
+            recorder: false
+        )
+
+        let options = TokenOptions(
+            ttl: ttl,
+            videoGrants: grants,
+            identity: identity,
+            name: identity
+        )
+
+        let credentials = ApiCredentials(
+            key: apiKey,
+            secret: apiSecret
+        )
+
+        do {
+            return try tokenGenerate(options: options, credentials: credentials)
+        } catch {
+            fatalError("Failed to generate token: \(error)")
+        }
+    }
+}