Merge pull request #3 from mehtarac/realtime_mp

(feat)bidirectional_streaming: add openai realtime model provider

Author: mehtarac

pyproject.toml

@@ -54,12 +54,24 @@ sagemaker = [
     "boto3-stubs[sagemaker-runtime]>=1.26.0,<2.0.0",
     "openai>=1.68.0,<2.0.0",  # SageMaker uses OpenAI-compatible interface
 ]
+bidirectional-streaming-nova = [
+    "pyaudio>=0.2.13",
+    "rx>=3.2.0",
+    "smithy-aws-core>=0.0.1",
+    "pytz",
+    "aws_sdk_bedrock_runtime",
+]
+bidirectional-streaming-openai = [
+    "pyaudio>=0.2.13",
+    "websockets>=12.0,<14.0",
+]
 bidirectional-streaming = [
     "pyaudio>=0.2.13",
     "rx>=3.2.0",
     "smithy-aws-core>=0.0.1",
     "pytz",
     "aws_sdk_bedrock_runtime",
+    "websockets>=12.0,<14.0",
 ]
 otel = ["opentelemetry-exporter-otlp-proto-http>=1.30.0,<2.0.0"]
 docs = [

src/strands/experimental/bidirectional_streaming/__init__.py

@@ -8,6 +8,7 @@
 
 # Model providers - What users need to create models
 from .models.novasonic import NovaSonicBidirectionalModel
+from .models.openai import OpenAIRealtimeBidirectionalModel
 
 # Event types - For type hints and event handling
 from .types.bidirectional_streaming import (
@@ -17,20 +18,26 @@
     InterruptionDetectedEvent,
     TextOutputEvent,
     UsageMetricsEvent,
+    VoiceActivityEvent,
 )
 
 __all__ = [
     # Main interface
     "BidirectionalAgent",
+    
     # Model providers
     "NovaSonicBidirectionalModel",
+    "OpenAIRealtimeBidirectionalModel",
+    
     # Event types
     "AudioInputEvent",
-    "AudioOutputEvent",
+    "AudioOutputEvent", 
     "TextOutputEvent",
     "InterruptionDetectedEvent",
     "BidirectionalStreamEvent",
+    "VoiceActivityEvent",
     "UsageMetricsEvent",
+    
     # Model interface
     "BidirectionalModel",
     "BidirectionalModelSession",

src/strands/experimental/bidirectional_streaming/agent/agent.py

@@ -33,7 +33,6 @@
 from ..models.bidirectional_model import BidirectionalModel
 from ..types.bidirectional_streaming import AudioInputEvent, BidirectionalStreamEvent
 
-
 logger = logging.getLogger(__name__)
 
 _DEFAULT_AGENT_NAME = "Strands Agents"

src/strands/experimental/bidirectional_streaming/event_loop/bidirectional_event_loop.py

@@ -23,7 +23,6 @@
 from ....types.tools import ToolResult, ToolUse
 from ..models.bidirectional_model import BidirectionalModelSession
 
-
 logger = logging.getLogger(__name__)
 
 # Session constants

src/strands/experimental/bidirectional_streaming/models/__init__.py

@@ -2,10 +2,13 @@
 
 from .bidirectional_model import BidirectionalModel, BidirectionalModelSession
 from .novasonic import NovaSonicBidirectionalModel, NovaSonicSession
+from .openai import OpenAIRealtimeBidirectionalModel, OpenAIRealtimeSession
 
 __all__ = [
-    "BidirectionalModel",
-    "BidirectionalModelSession",
-    "NovaSonicBidirectionalModel",
+    "BidirectionalModel", 
+    "BidirectionalModelSession", 
+    "NovaSonicBidirectionalModel", 
     "NovaSonicSession",
+    "OpenAIRealtimeBidirectionalModel",
+    "OpenAIRealtimeSession"
 ]

src/strands/experimental/bidirectional_streaming/models/novasonic.py

@@ -23,7 +23,7 @@
 
 from aws_sdk_bedrock_runtime.client import BedrockRuntimeClient, InvokeModelWithBidirectionalStreamOperationInput
 from aws_sdk_bedrock_runtime.config import Config, HTTPAuthSchemeResolver, SigV4AuthScheme
-from aws_sdk_bedrock_runtime.models import BidirectionalInputPayloadPart, InvokeModelWithBidirectionalStreamInputChunk
+from aws_sdk_bedrock_runtime.models import BidirectionalInputPayloadPart, InvokeModelWithBidirectionalStreamInputChunk, InvokeModelWithBidirectionalStreamOperationOutput
 from smithy_aws_core.identity.environment import EnvironmentCredentialsResolver
 
 from ....types.content import Messages
@@ -35,9 +35,8 @@
     BidirectionalConnectionStartEvent,
     InterruptionDetectedEvent,
     TextOutputEvent,
-    UsageMetricsEvent
+    UsageMetricsEvent,
 )
-
 from .bidirectional_model import BidirectionalModel, BidirectionalModelSession
 
 logger = logging.getLogger(__name__)
@@ -81,11 +80,11 @@ class NovaSonicSession(BidirectionalModelSession):
     interface.
     """
 
-    def __init__(self, stream: any, config: dict[str, any]) -> None:
+    def __init__(self, stream: InvokeModelWithBidirectionalStreamOperationOutput, config: dict[str, any]) -> None:
         """Initialize Nova Sonic connection.
 
         Args:
-            stream: Nova Sonic bidirectional stream.
+            stream: Nova Sonic bidirectional stream operation output from AWS SDK.
             config: Model configuration.
         """
         self.stream = stream
@@ -487,14 +486,14 @@ def _convert_nova_event(self, nova_event: dict[str, any]) -> dict[str, any] | No
 
             return {"interruptionDetected": interruption}
 
-        # Handle usage events (ignore)
+        # Handle usage events - convert to standardized format
         elif "usageEvent" in nova_event:
             usage_data = nova_event["usageEvent"]
             usage_metrics: UsageMetricsEvent = {
-                "totalTokens": usage_data.get("totalTokens"),
-                "inputTokens": usage_data.get("totalInputTokens"),
-                "outputTokens": usage_data.get("totalOutputTokens"),
-                "audioTokens": usage_data.get("details", {}).get("total", {}).get("output", {}).get("speechTokens"),
+                "totalTokens": usage_data.get("totalTokens", 0),
+                "inputTokens": usage_data.get("totalInputTokens", 0),
+                "outputTokens": usage_data.get("totalOutputTokens", 0),
+                "audioTokens": usage_data.get("details", {}).get("total", {}).get("output", {}).get("speechTokens", 0)
             }
             return {"usageMetrics": usage_metrics}