feat: update pyaudio listener and speaker with device_index argument

Author: thirdgerb

RELEASES.md

@@ -4,6 +4,11 @@
 
 first release version.
 
+## v0.1.7
+
+* update speaker and listener with pyaudio device_index argument
+* streamlit_app.yml add options about audio_input and audio_output
+
 ## v0.1.6
 
 * upgrade openai package to 1.59, support develop message.

ghostos/framework/audio/pyaudio_io/__init__.py

@@ -1,19 +1,42 @@
+from typing import Union
 from ghostos.abcd.realtime import Speaker, Listener
 
 
-def get_pyaudio_pcm16_listener(rate: int = 24000, interval: float = 0.5) -> Listener:
+def get_pyaudio_pcm16_listener(
+        rate: int = 24000,
+        interval: float = 0.5,
+        channels: int = 1,
+        chunk_size: int = 1024,
+        input_device_index: Union[int, None] = None,
+) -> Listener:
     try:
         import pyaudio
     except ImportError:
         raise ImportError(f"pyaudio package is required. run `pip install ghostos[audio]`")
     from ghostos.framework.audio.pyaudio_io.listener import PyAudioPCM16Listener
-    return PyAudioPCM16Listener(rate, interval=interval)
+    return PyAudioPCM16Listener(
+        rate=rate,
+        interval=interval,
+        channels=channels,
+        chunk_size=chunk_size,
+        input_device_index=input_device_index,
+    )
 
 
-def get_pyaudio_pcm16_speaker(rate: int = 24000, buffer_size: int = 1024 * 5) -> Speaker:
+def get_pyaudio_pcm16_speaker(
+        rate: int = 24000,
+        buffer_size: int = 1024 * 5,
+        channels: int = 1,
+        output_device_index: Union[int, None] = None,
+) -> Speaker:
     try:
         import pyaudio
     except ImportError:
         raise ImportError(f"pyaudio package is required. run `pip install ghostos[audio]`")
     from ghostos.framework.audio.pyaudio_io.speaker import PyAudioPCM16Speaker
-    return PyAudioPCM16Speaker(rate, buffer_size)
+    return PyAudioPCM16Speaker(
+        rate=rate,
+        buffer_size=buffer_size,
+        channels=channels,
+        output_device_index=output_device_index,
+    )

ghostos/framework/audio/pyaudio_io/listener.py

@@ -3,7 +3,7 @@
 except ImportError:
     raise ImportError(f"Pyaudio is required, please install pyaudio or ghostos[audio] first")
 
-from typing import Callable, Union
+from typing import Callable, Optional
 from ghostos.abcd.realtime import Listener, Listening
 from threading import Thread, Event
 from io import BytesIO
@@ -16,14 +16,22 @@
 
 class PyAudioPCM16Listener(Listener):
 
-    def __init__(self, rate: int = 24000, chunk_size: int = CHUNK, interval: float = 0.5):
+    def __init__(
+            self,
+            rate: int = 24000,
+            chunk_size: int = CHUNK,
+            interval: float = 0.5,
+            channels: int = CHANNELS,
+            input_device_index: Optional[int] = None,
+    ):
         self.rate = rate
         self.chunk_size = chunk_size
         self.stream = PyAudio().open(
             format=paInt16,
-            channels=1,
+            channels=channels,
             rate=self.rate,
             input=True,
+            input_device_index=input_device_index,
         )
         self.interval = interval
 

ghostos/framework/audio/pyaudio_io/speaker.py

@@ -10,14 +10,21 @@
 
 class PyAudioPCM16Speaker(Speaker):
 
-    def __init__(self, rate: int = 24000, buffer_size: int = 4096):
+    def __init__(
+            self,
+            rate: int = 24000,
+            buffer_size: int = 4096,
+            channels: int = 1,
+            output_device_index: Union[int, None] = None,
+    ):
         self.rate = rate
         self.buffer_size = buffer_size
         self.stream = PyAudio().open(
             format=paInt16,
-            channels=1,
+            channels=channels,
             rate=self.rate,
             output=True,
+            output_device_index=output_device_index,
         )
 
     def speak(self, queue: Callable[[], Union[bytes, None]]) -> Speaking:

ghostos/ghosts/moss_agent/agent.py

@@ -207,9 +207,12 @@ def _get_instruction_prompter(self, session: Session, runtime: MossRuntime) -> P
                 title="Meta Instruction",
                 content=AGENT_META_INTRODUCTION,
             ).with_children(
+                # ghostos meta instruction.
                 TextPrmt(title="GhostOS", content=GHOSTOS_INTRODUCTION),
+                # the information about moss
                 TextPrmt(title="MOSS", content=MOSS_INTRODUCTION),
-                # code context
+
+                # the moss providing context prompter.
                 get_moss_context_prompter("Code Context", runtime),
             ),
             # agent prompt

ghostos/prototypes/streamlitapp/pages/chat_with_ghost.py

@@ -255,8 +255,22 @@ def get_realtime_app(conversation: Conversation) -> Optional[RealtimeApp]:
 
     from ghostos.framework.audio import get_pyaudio_pcm16_speaker, get_pyaudio_pcm16_listener
     from ghostos.framework.openai_realtime import get_openai_realtime_app
-    speaker = get_pyaudio_pcm16_speaker()
-    listener = get_pyaudio_pcm16_listener()
+    app_conf = get_app_conf()
+    audio_input = app_conf.audio_input
+    audio_output = app_conf.audio_output
+    speaker = get_pyaudio_pcm16_speaker(
+        rate=audio_output.sample_rate,
+        buffer_size=audio_output.buffer_size,
+        channels=audio_output.channels,
+        output_device_index=audio_output.output_device_index,
+    )
+    listener = get_pyaudio_pcm16_listener(
+        rate=audio_input.sample_rate,
+        interval=audio_input.interval,
+        channels=audio_input.channels,
+        chunk_size=audio_input.chunk_size,
+        input_device_index=audio_input.input_device_index,
+    )
     vad_mode = True
     return get_openai_realtime_app(conversation, vad_mode=vad_mode, listener=listener, speaker=speaker)
 

ghostos/prototypes/streamlitapp/resources.py

@@ -1,15 +1,14 @@
-from typing import Optional, Dict, Tuple, List
+from typing import Optional, Dict, Tuple, List, Union
 
 from enum import Enum
-from pydantic import Field
+from pydantic import BaseModel, Field
 import streamlit as st
 from ghostos.container import Container
 from ghostos.prototypes.streamlitapp.utils.session import Singleton
 from ghostos.contracts.configs import YamlConfig, Configs
 from ghostos.contracts.assets import ImageAssets, FileInfo, AudioAssets
 from ghostos.contracts.documents import DocumentRegistry, Documents
 from PIL.Image import Image as ImageType
-from ghostos.core.messages.message_classes import ImageAssetMessage
 from ghostos.helpers import GHOSTOS_DOMAIN, uuid
 from streamlit.runtime.uploaded_file_manager import DeletedFile, UploadedFile
 
@@ -19,6 +18,21 @@ def get_container() -> Container:
     return Singleton.get(Container, st.session_state)
 
 
+class AudioInputConf(BaseModel):
+    sample_rate: int = Field(24000)
+    interval: float = Field(0.5)
+    channels: int = Field(1)
+    chunk_size: int = Field(1024)
+    input_device_index: Union[int, None] = Field(None)
+
+
+class AudioOutputConf(BaseModel):
+    sample_rate: int = Field(24000)
+    channels: int = Field(1)
+    buffer_size: int = Field(1024 * 5)
+    output_device_index: Union[int, None] = Field(None)
+
+
 class AppConf(YamlConfig):
     relative_path = "streamlit_app.yml"
 
@@ -29,6 +43,9 @@ class AppConf(YamlConfig):
         default_factory=dict,
     )
 
+    audio_input: AudioInputConf = Field(default_factory=AudioInputConf)
+    audio_output: AudioOutputConf = Field(default_factory=AudioOutputConf)
+
     class BoolOpts(str, Enum):
         HELP_MODE = "ghostos.streamlit.app.help_mode"
         """global help mode"""

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ghostos"
-version = "0.1.6"
+version = "0.1.7"
 description = "A framework offers an operating system simulator with a Python Code Interface for AI Agents"
 authors = ["zhuming <thirdgerb@gmail.com>", "Nile Zhou <nilezhou123@gmail.com>"]
 license = "MIT"