Add Sesame CSM TTS and Whisper STT containers

- Create Sesame CSM TTS container with FastAPI server
- Create Whisper STT container with WebSocket support
- Add comprehensive GitHub Actions testing for speech services
- Implement Deepgram-compatible API endpoints
- Use tiny Whisper model for faster testing

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

Author: YedPool

.github/workflows/test-docker.yml

@@ -295,4 +295,114 @@ jobs:
         
         echo "🧹 Cleaning up..."
         docker stop token-server-test || true
-        docker rm token-server-test || true
+        docker rm token-server-test || true
+
+  test-speech-services:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      
+    - name: Build Whisper STT Docker image
+      run: |
+        cd whisper-stt
+        docker build --progress=plain -t whisper-stt:test .
+        
+    - name: Build Sesame TTS Docker image
+      run: |
+        cd sesame-tts
+        docker build --progress=plain -t sesame-tts:test .
+        
+    - name: Test Whisper STT container startup
+      run: |
+        # Start Whisper STT container
+        docker run -d --name whisper-stt-test \
+          -p 8002:8002 \
+          -e WHISPER_MODEL=tiny \
+          whisper-stt:test
+          
+        # Show immediate logs
+        echo "📋 Initial Whisper STT logs:"
+        docker logs whisper-stt-test || true
+          
+        # Wait for container to start
+        sleep 30
+        
+        # Check if container is still running
+        if docker ps | grep -q whisper-stt-test; then
+          echo "✅ Whisper STT container started successfully"
+        else
+          echo "❌ Whisper STT container failed to start"
+          docker logs whisper-stt-test
+          exit 1
+        fi
+        
+    - name: Test Whisper STT health endpoint
+      run: |
+        # Wait up to 60 seconds for STT server to be ready
+        timeout=60
+        elapsed=0
+        
+        while [ $elapsed -lt $timeout ]; do
+          if curl -s http://localhost:8002/health > /dev/null 2>&1; then
+            echo "✅ Whisper STT health endpoint responding"
+            curl -s http://localhost:8002/health | jq .
+            break
+          fi
+          echo "⏳ Waiting for Whisper STT... ($elapsed/$timeout seconds)"
+          sleep 5
+          elapsed=$((elapsed + 5))
+        done
+        
+        if [ $elapsed -ge $timeout ]; then
+          echo "❌ Whisper STT health endpoint failed to respond"
+          docker logs whisper-stt-test
+          exit 1
+        fi
+        
+    - name: Test Sesame TTS container startup (without model loading)
+      run: |
+        # Start Sesame TTS container (will fail without HF token but should start server)
+        docker run -d --name sesame-tts-test \
+          -p 8001:8001 \
+          sesame-tts:test
+          
+        # Show immediate logs
+        echo "📋 Initial Sesame TTS logs:"
+        docker logs sesame-tts-test || true
+          
+        # Wait for container
+        sleep 20
+        
+        # Check container status (it may exit due to missing HF token, that's expected)
+        echo "📊 Sesame TTS container status:"
+        docker ps -a | grep sesame-tts-test || true
+        
+        echo "📋 Full Sesame TTS logs:"
+        docker logs sesame-tts-test || true
+        
+    - name: Test service endpoints info
+      run: |
+        echo "🔍 Testing Whisper STT root endpoint..."
+        curl -s http://localhost:8002/ | jq . || true
+        
+        echo "📊 Container resource usage:"
+        docker stats --no-stream whisper-stt-test || true
+        
+    - name: Speech services logs and cleanup
+      if: always()
+      run: |
+        echo "📋 Whisper STT final logs:"
+        docker logs whisper-stt-test || true
+        
+        echo "📋 Sesame TTS final logs:"
+        docker logs sesame-tts-test || true
+        
+        echo "🧹 Cleaning up..."
+        docker stop whisper-stt-test sesame-tts-test || true
+        docker rm whisper-stt-test sesame-tts-test || true

sesame-tts/Dockerfile

@@ -0,0 +1,37 @@
+# Sesame CSM TTS Docker Image
+FROM python:3.10-slim
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    curl \
+    git \
+    ffmpeg \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /app
+
+# Copy Python requirements and install dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY main.py .
+
+# Set environment variables
+ENV NO_TORCH_COMPILE=1
+ENV TRANSFORMERS_CACHE=/app/.cache
+ENV HF_HOME=/app/.cache
+
+# Create cache directory
+RUN mkdir -p /app/.cache
+
+# Expose FastAPI port
+EXPOSE 8001
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=15s --start-period=60s --retries=3 \
+    CMD curl -f http://localhost:8001/health || exit 1
+
+# Start FastAPI server
+CMD ["python", "main.py"]

sesame-tts/main.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+"""
+Sesame CSM TTS Server
+Provides text-to-speech using Sesame's Conversational Speech Model
+"""
+
+import os
+import logging
+import io
+import base64
+from typing import Optional
+from datetime import datetime
+
+import torch
+import torchaudio
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from transformers import AutoProcessor, CsmForConditionalGeneration
+import uvicorn
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Create FastAPI app
+app = FastAPI(
+    title="Sesame CSM TTS Server",
+    description="Text-to-speech using Sesame's Conversational Speech Model",
+    version="1.0.0"
+)
+
+# Add CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["GET", "POST"],
+    allow_headers=["*"],
+)
+
+class TTSRequest(BaseModel):
+    text: str
+    speaker: Optional[int] = 0
+    max_audio_length_ms: Optional[int] = 10000
+    return_format: Optional[str] = "wav"  # wav, mp3, base64
+
+class TTSResponse(BaseModel):
+    audio_data: str  # base64 encoded audio
+    sample_rate: int
+    format: str
+    duration_ms: float
+
+class TTSServer:
+    def __init__(self):
+        self.model = None
+        self.processor = None
+        self.device = self._detect_device()
+        self.sample_rate = 24000  # CSM default sample rate
+        
+        logger.info(f"Using device: {self.device}")
+        
+    def _detect_device(self):
+        """Detect the best available device."""
+        if torch.cuda.is_available():
+            return "cuda"
+        elif torch.backends.mps.is_available():
+            return "mps"
+        else:
+            return "cpu"
+    
+    def load_model(self):
+        """Load the Sesame CSM model."""
+        try:
+            model_id = "sesame/csm-1b"
+            logger.info(f"Loading Sesame CSM model: {model_id}")
+            
+            # Load processor and model
+            self.processor = AutoProcessor.from_pretrained(model_id)
+            self.model = CsmForConditionalGeneration.from_pretrained(
+                model_id,
+                device_map=self.device,
+                torch_dtype=torch.float16 if self.device == "cuda" else torch.float32
+            )
+            
+            logger.info("✓ Sesame CSM model loaded successfully")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Failed to load model: {e}")
+            return False
+    
+    def generate_audio(self, text: str, speaker: int = 0, max_audio_length_ms: int = 10000):
+        """Generate audio from text using CSM."""
+        try:
+            if not self.model or not self.processor:
+                raise ValueError("Model not loaded")
+            
+            logger.info(f"Generating audio for text: '{text[:50]}...'")
+            
+            # Prepare inputs
+            inputs = self.processor(
+                text=text,
+                speaker_id=speaker,
+                return_tensors="pt"
+            ).to(self.device)
+            
+            # Generate audio
+            with torch.no_grad():
+                audio_codes = self.model.generate(
+                    **inputs,
+                    max_new_tokens=max_audio_length_ms // 25,  # Rough estimate
+                    do_sample=True,
+                    temperature=0.7
+                )
+            
+            # Decode audio
+            audio_array = self.processor.decode(audio_codes[0])
+            
+            # Ensure audio is on CPU and in correct format
+            if isinstance(audio_array, torch.Tensor):
+                audio_array = audio_array.cpu().float()
+            
+            logger.info(f"✓ Generated audio: {audio_array.shape} samples at {self.sample_rate}Hz")
+            return audio_array
+            
+        except Exception as e:
+            logger.error(f"Audio generation failed: {e}")
+            raise
+    
+    def audio_to_base64(self, audio_tensor, format="wav"):
+        """Convert audio tensor to base64 encoded string."""
+        try:
+            # Create a bytes buffer
+            buffer = io.BytesIO()
+            
+            # Save audio to buffer
+            torchaudio.save(
+                buffer, 
+                audio_tensor.unsqueeze(0) if audio_tensor.dim() == 1 else audio_tensor,
+                self.sample_rate,
+                format=format
+            )
+            
+            # Get bytes and encode to base64
+            buffer.seek(0)
+            audio_bytes = buffer.getvalue()
+            audio_base64 = base64.b64encode(audio_bytes).decode('utf-8')
+            
+            return audio_base64
+            
+        except Exception as e:
+            logger.error(f"Audio encoding failed: {e}")
+            raise
+
+# Global TTS server instance
+tts_server = TTSServer()
+
+@app.on_event("startup")
+async def startup_event():
+    """Load model on startup."""
+    logger.info("🚀 Starting Sesame CSM TTS Server")
+    
+    if not tts_server.load_model():
+        logger.error("❌ Failed to load TTS model")
+        raise RuntimeError("Model loading failed")
+    
+    logger.info("✅ TTS Server ready!")
+
+@app.post("/v1/speak", response_model=TTSResponse)
+async def speak(request: TTSRequest, http_request: Request):
+    """Generate speech from text (Deepgram-compatible endpoint)."""
+    
+    client_ip = http_request.client.host if http_request.client else "unknown"
+    logger.info(f"TTS request from {client_ip}: '{request.text[:50]}...'")
+    
+    try:
+        # Generate audio
+        audio_tensor = tts_server.generate_audio(
+            text=request.text,
+            speaker=request.speaker,
+            max_audio_length_ms=request.max_audio_length_ms
+        )
+        
+        # Convert to base64
+        audio_base64 = tts_server.audio_to_base64(audio_tensor, request.return_format)
+        
+        # Calculate duration
+        duration_ms = len(audio_tensor) / tts_server.sample_rate * 1000
+        
+        return TTSResponse(
+            audio_data=audio_base64,
+            sample_rate=tts_server.sample_rate,
+            format=request.return_format,
+            duration_ms=duration_ms
+        )
+        
+    except Exception as e:
+        logger.error(f"TTS generation failed: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/tts")
+async def tts_simple(request: TTSRequest):
+    """Simple TTS endpoint."""
+    return await speak(request, None)
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint."""
+    model_loaded = tts_server.model is not None
+    return {
+        "status": "healthy" if model_loaded else "unhealthy",
+        "service": "sesame-csm-tts",
+        "timestamp": datetime.utcnow().isoformat() + "Z",
+        "model_loaded": model_loaded,
+        "device": tts_server.device,
+        "sample_rate": tts_server.sample_rate
+    }
+
+@app.get("/")
+async def root():
+    """Root endpoint with API information."""
+    return {
+        "service": "Sesame CSM TTS Server",
+        "version": "1.0.0",
+        "description": "Text-to-speech using Sesame's Conversational Speech Model",
+        "endpoints": {
+            "POST /v1/speak": "Generate speech (Deepgram-compatible)",
+            "POST /tts": "Simple TTS generation",
+            "GET /health": "Health check",
+            "GET /": "This information"
+        },
+        "docs": "/docs"
+    }
+
+if __name__ == "__main__":
+    port = int(os.getenv("PORT", 8001))
+    logger.info(f"Starting Sesame CSM TTS Server on port {port}")
+    uvicorn.run(app, host="0.0.0.0", port=port)