Merge remote-tracking branch 'origin/main' into dz/simulate-scenario

Author: davidzhao

.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# CODEOWNERS for python-sdks
+# These owners will be requested for review on all PRs
+
+* @cloudwebrtc @lukasIO @xianshijing-lk

README.md

@@ -274,6 +274,20 @@ LiveKit is a dynamic realtime environment and calls can fail for various reasons
 
 You may throw errors of the type `RpcError` with a string `message` in an RPC method handler and they will be received on the caller's side with the message intact. Other errors will not be transmitted and will instead arrive to the caller as `1500` ("Application Error"). Other built-in errors are detailed in `RpcError`.
 
+## Hardware video codec support
+
+The underlying Rust SDK ships with platform-specific hardware-accelerated encoders/decoders, These are used automatically when available and compatible with the runtime environment (OS, drivers, GPU, and codec).
+
+| Platform                       | Codec(s)   | Encoder | Decoder | Backend                                |
+| ------------------------------ | ---------- | ------- | ------- | -------------------------------------- |
+| macOS                          | H264, H265 | ✓       | ✓       | VideoToolbox                           |
+| Linux (AMD GPU)                | H264       | ✓       |         | VAAPI                                  |
+| Linux x64 (NVIDIA GPU)         | H264, H265 | ✓       | ✓       | NVENC / NVDEC (NVIDIA Video Codec SDK) |
+
+Software encoders (libvpx for VP8/VP9, libaom for AV1, OpenH264 for H264) are used as a fallback when hardware acceleration is not available.
+
+> **Note:** Availability depends on the specific machine configuration, including GPU model, driver support, and runtime environment.
+
 ## Examples
 
 - [Facelandmark](https://github.com/livekit/python-sdks/tree/main/examples/face_landmark): Use mediapipe to detect face landmarks (eyes, nose ...)

examples/e2ee.py

@@ -1,15 +1,19 @@
 import asyncio
 import logging
+import os
 from signal import SIGINT, SIGTERM
 
 import numpy as np
 from livekit import rtc
 
-URL = "ws://localhost:7880"
-TOKEN = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE5MDY2MTMyODgsImlzcyI6IkFQSVRzRWZpZFpqclFvWSIsIm5hbWUiOiJuYXRpdmUiLCJuYmYiOjE2NzI2MTMyODgsInN1YiI6Im5hdGl2ZSIsInZpZGVvIjp7InJvb20iOiJ0ZXN0Iiwicm9vbUFkbWluIjp0cnVlLCJyb29tQ3JlYXRlIjp0cnVlLCJyb29tSm9pbiI6dHJ1ZSwicm9vbUxpc3QiOnRydWV9fQ.uSNIangMRu8jZD5mnRYoCHjcsQWCrJXgHCs0aNIgBFY"  # noqa
+URL = os.environ.get("LIVEKIT_URL", "ws://localhost:7880")
+TOKEN = os.environ.get(
+    "LIVEKIT_TOKEN",
+    "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE5MDY2MTMyODgsImlzcyI6IkFQSVRzRWZpZFpqclFvWSIsIm5hbWUiOiJuYXRpdmUiLCJuYmYiOjE2NzI2MTMyODgsInN1YiI6Im5hdGl2ZSIsInZpZGVvIjp7InJvb20iOiJ0ZXN0Iiwicm9vbUFkbWluIjp0cnVlLCJyb29tQ3JlYXRlIjp0cnVlLCJyb29tSm9pbiI6dHJ1ZSwicm9vbUxpc3QiOnRydWV9fQ.uSNIangMRu8jZD5mnRYoCHjcsQWCrJXgHCs0aNIgBFY",
+)  # noqa
 
-# ("livekitrocks") this is our shared key, it must match the one used by your clients
-SHARED_KEY = b"livekitrocks"
+# This shared key must match the one used by your clients
+SHARED_KEY = os.environ.get("E2EE_SHARED_KEY", "livekitrocks").encode()
 
 WIDTH, HEIGHT = 1280, 720
 
@@ -106,7 +110,7 @@ def on_e2ee_state_changed(participant: rtc.Participant, state: rtc.EncryptionSta
         e2ee_options.key_provider_options.shared_key = SHARED_KEY
 
         await room.connect(
-            URL, TOKEN, options=rtc.RoomOptions(auto_subscribe=True, e2ee=e2ee_options)
+            URL, TOKEN, options=rtc.RoomOptions(auto_subscribe=True, encryption=e2ee_options)
         )
 
         logging.info("connected to room %s", room.name)

livekit-api/pyproject.toml

@@ -30,7 +30,7 @@ dependencies = [
     "aiohttp>=3.9.0",
     "protobuf>=4",
     "types-protobuf>=4",
-    "livekit-protocol>=1.1.5,<2.0.0",
+    "livekit-protocol>=1.1.8,<2.0.0",
 ]
 
 [project.urls]

livekit-protocol/generate_proto.sh

@@ -38,6 +38,7 @@ protoc \
     $API_PROTOCOL/livekit_analytics.proto \
     $API_PROTOCOL/livekit_rtc.proto \
     $API_PROTOCOL/agent/livekit_agent_session.proto \
+    $API_PROTOCOL/agent/livekit_agent_inference.proto \
     $API_PROTOCOL/agent/livekit_agent_text.proto \
     $API_PROTOCOL/agent/livekit_agent_dev.proto \
     $API_PROTOCOL/logger/options.proto \
@@ -87,6 +88,8 @@ mv "$API_OUT_PYTHON/livekit_connector_pb2.py" "$API_OUT_PYTHON/connector.py"
 mv "$API_OUT_PYTHON/livekit_connector_pb2.pyi" "$API_OUT_PYTHON/connector.pyi"
 
 mkdir -p "$API_OUT_PYTHON/agent_pb"
+mv "$API_OUT_PYTHON/agent/livekit_agent_inference_pb2.py" "$API_OUT_PYTHON/agent_pb/agent_inference.py"
+mv "$API_OUT_PYTHON/agent/livekit_agent_inference_pb2.pyi" "$API_OUT_PYTHON/agent_pb/agent_inference.pyi"
 mv "$API_OUT_PYTHON/agent/livekit_agent_session_pb2.py" "$API_OUT_PYTHON/agent_pb/agent_session.py"
 mv "$API_OUT_PYTHON/agent/livekit_agent_session_pb2.pyi" "$API_OUT_PYTHON/agent_pb/agent_session.pyi"
 mv "$API_OUT_PYTHON/agent/livekit_agent_text_pb2.py" "$API_OUT_PYTHON/agent_pb/agent_text.py"
@@ -98,7 +101,7 @@ mkdir -p "$API_OUT_PYTHON/logger_pb"
 mv "$API_OUT_PYTHON/logger/options_pb2.py" "$API_OUT_PYTHON/logger_pb/options.py"
 mv "$API_OUT_PYTHON/logger/options_pb2.pyi" "$API_OUT_PYTHON/logger_pb/options.pyi"
 
-find "$API_OUT_PYTHON" -name '*.py' -o -name '*.pyi' | xargs perl -i -pe 's|^(import (livekit_egress_pb2\|livekit_room_pb2\|livekit_webhook_pb2\|livekit_ingress_pb2\|livekit_models_pb2\|livekit_agent_pb2\|livekit_agent_dispatch_pb2\|livekit_analytics_pb2\|livekit_sip_pb2\|livekit_metrics_pb2\|livekit_rtc_pb2\|livekit_connector_whatsapp_pb2\|livekit_connector_twilio_pb2\|livekit_connector_pb2\|livekit_agent_session_pb2\|livekit_agent_dev_pb2\|options_pb2))|from . $1|g'
+find "$API_OUT_PYTHON" -name '*.py' -o -name '*.pyi' | xargs perl -i -pe 's|^(import (livekit_egress_pb2\|livekit_room_pb2\|livekit_webhook_pb2\|livekit_ingress_pb2\|livekit_models_pb2\|livekit_agent_pb2\|livekit_agent_dispatch_pb2\|livekit_analytics_pb2\|livekit_sip_pb2\|livekit_metrics_pb2\|livekit_rtc_pb2\|livekit_connector_whatsapp_pb2\|livekit_connector_twilio_pb2\|livekit_connector_pb2\|livekit_agent_session_pb2\|livekit_agent_inference_pb2\|livekit_agent_dev_pb2\|livekit_agent_text_pb2\|options_pb2))|from . $1|g'
 
 find "$API_OUT_PYTHON" -name '*.py' -o -name '*.pyi' | xargs perl -i -pe 's|livekit_(\w+)_pb2|${1}|g'
 
@@ -112,4 +115,4 @@ find "$API_OUT_PYTHON" -mindepth 2 -name '*.py' -o -name '*.pyi' | xargs perl -i
 find "$API_OUT_PYTHON"/agent_pb -name '*.py' -o -name '*.pyi' | xargs perl -i -pe 's|from agent import (agent_\w+) as ([^ ]+)|from . import $1 as $2|g'
 
 # fixes - error: ClassVar can only be used for assignments in class body  [misc]
-perl -i -pe 's|^(\w+_FIELD_NUMBER): _ClassVar\[int\]|$1: int|g' "$API_OUT_PYTHON/logger_pb/options.pyi"
+perl -i -pe 's|^(\w+_FIELD_NUMBER): _ClassVar\[int\]|$1: int|g' "$API_OUT_PYTHON/logger_pb/options.pyi"