Add audio/video streaming and web tester for telemetry

Introduces bidirectional audio and video streaming using GStreamer pipelines, with new modules for audio (src/audio.py) and video (src/video.py). Adds a new main entrypoint (main.py) to orchestrate telemetry, audio, and video processes. Refactors telemetry.py to src/data.py and enhances UDP socket priority and stats reporting. Adds a web-tester Flask/SocketIO backend and car frontend for browser-based audio/video testing, including Docker/test scripts and documentation updates for hardware setup and environment variables.

Author: haoruizhou

test_local.sh

@@ -2,23 +2,46 @@ FROM python:3.11-slim
 
 WORKDIR /app
 
-# Install system dependencies for CAN and networking
+# Install system dependencies for CAN, networking, and GStreamer (Audio/Video)
 RUN apt-get update && apt-get install -y \
     can-utils \
     iproute2 \
+    python3-gi \
+    python3-gst-1.0 \
+    gir1.2-gstreamer-1.0 \
+    gstreamer1.0-tools \
+    gstreamer1.0-plugins-base \
+    gstreamer1.0-plugins-good \
+    gstreamer1.0-plugins-bad \
+    gstreamer1.0-plugins-ugly \
+    gstreamer1.0-libav \
+    gstreamer1.0-alsa \
+    libgirepository1.0-dev \
+    libgirepository-2.0-dev \
+    libcairo2-dev \
+    pkg-config \
+    gcc \
+    python3-dev \
+    libglib2.0-dev \
+    libgl1 \
+    gobject-introspection \
     && rm -rf /var/lib/apt/lists/*
 
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-COPY telemetry.py .
+# Copy source code
+COPY src/ ./src/
+COPY web-tester/ ./web-tester/
+COPY main.py .
 
 # Use environment variables for configuration
 ENV ROLE=auto
 ENV REMOTE_IP=192.168.1.100
 ENV UDP_PORT=5005
 ENV TCP_PORT=5006
 ENV REDIS_URL=redis://localhost:6379/0
+ENV ENABLE_AUDIO=true
+ENV ENABLE_VIDEO=true
 
-CMD ["python", "telemetry.py"]
-
+CMD ["python", "main.py"]

universal-telemetry-software/Dockerfile

@@ -12,18 +12,44 @@ This is a unified firmware/software for both the Car (Source) and Base Station (
   - **Sequence Tracking** on the base station.
   - **TCP Retransmission**: Every 10s, the base station requests missing packets from the car.
 - **Frontend Integration**: Publishes JSON messages to Redis for the `redis_ws_bridge`.
+- **Bidirectional Audio**: Low-latency voice comms using OPUS/UDP.
 
-## RPi Setup (CAN)
-Ensure your CAN hat (e.g., MCP2515) is configured in `/boot/config.txt`:
+## Hardware Setup (RPi 5)
+
+### 1. CAN Bus
+Ensure your CAN hat (e.g., MCP2515) is configured in `/boot/config.txt`. Note that RPi 5 uses a different SPI controller sometimes, but standard overlays usually work:
 ```text
 dtparam=spi=on
 dtoverlay=mcp2515-can0,oscillator=16000000,interrupt=25
 ```
-Then initialize the interface:
+Initialize:
 ```bash
 sudo ip link set can0 up type can bitrate 500000
 ```
 
+### 2. Audio (No 3.5mm Jack)
+The RPi 5 has no analog audio. You must use a digital interface.
+
+#### Option A: I2S Audio HAT (Recommended for FSAE)
+Use a HAT like the **WM8960** or **HiFiBerry**. These use the GPIO header, are mechanically secure, and have low latency.
+1. Install drivers (see HAT vendor docs).
+2. Check device index: `aplay -l` and `arecord -l`.
+3. Configure Env Vars:
+   ```bash
+   export AUDIO_SOURCE="alsasrc device=hw:0,0"
+   export AUDIO_SINK="alsasink device=hw:0,0"
+   ```
+
+#### Option B: USB Audio Adapter
+Easiest to test, but **vibration risk** in car. Use hot glue or strain relief!
+1. Plug in USB Sound Card.
+2. Check device index: `aplay -l` (usually card 1).
+3. Configure Env Vars:
+   ```bash
+   export AUDIO_SOURCE="alsasrc device=hw:1,0"
+   export AUDIO_SINK="alsasink device=hw:1,0"
+   ```
+
 ## Deployment
 1. Set the `REMOTE_IP` in `docker-compose.yml` to the IP of the other side.
 2. Run:
@@ -37,3 +63,6 @@ docker-compose up -d
 - `UDP_PORT`: Default `5005`.
 - `TCP_PORT`: Default `5006`.
 - `REDIS_URL`: Default `redis://localhost:6379/0`.
+- `ENABLE_AUDIO`: `true` (default).
+- `AUDIO_SOURCE`: GStreamer source element (default `autoaudiosrc`).
+- `AUDIO_SINK`: GStreamer sink element (default `autoaudiosink`).

universal-telemetry-software/README.md

@@ -0,0 +1,96 @@
+import os
+import time
+import multiprocessing
+import logging
+from src.data import TelemetryNode
+from src.video import run_video
+from src.audio import run_audio
+import asyncio
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger("Main")
+
+def start_telemetry():
+    # Set High Priority (Critical for CAN)
+    try:
+        os.nice(-10)
+        logger.info("Telemetry process priority set to -10 (High)")
+    except PermissionError:
+        logger.warning("Could not set Telemetry priority (needs root/CAP_SYS_NICE)")
+
+    # Telemetry is asyncio based
+    node = TelemetryNode()
+    asyncio.run(node.start())
+
+def start_video(role, remote_ip):
+    # Set Lower Priority (Video can drop frames if needed)
+    try:
+        os.nice(5)
+    except PermissionError:
+        pass
+    run_video(role, remote_ip)
+
+def start_audio(role, remote_ip):
+    # Set Medium Priority (Audio needs low latency)
+    try:
+        os.nice(-5)
+        logger.info("Audio process priority set to -5 (Medium)")
+    except PermissionError:
+        logger.warning("Could not set Audio priority")
+    run_audio(role, remote_ip)
+
+if __name__ == "__main__":
+    logger.info("Universal Telemetry Software Starting...")
+    
+    # Configuration
+    role = os.getenv("ROLE", "auto")
+    remote_ip = os.getenv("REMOTE_IP", "127.0.0.1")
+    enable_video = os.getenv("ENABLE_VIDEO", "true").lower() == "true"
+    enable_audio = os.getenv("ENABLE_AUDIO", "true").lower() == "true"
+
+    # Note: Telemetry needs to run first or alone to detect role if "auto"
+    # But for simplicity, if "auto", we might need logic in main to detect first?
+    # TelemetryNode detects it internally. Ideally, we detect once here.
+    
+    if role == "auto":
+        # Quick detection logic (duplicated for now from telemetry.py for init)
+        try:
+            import can
+            with can.interface.Bus(channel='can0', bustype='socketcan') as bus:
+                role = "car"
+        except:
+            role = "base"
+        logger.info(f"Auto-detected Role: {role}")
+
+    processes = []
+
+    # 1. Telemetry (Critical)
+    p_telemetry = multiprocessing.Process(target=start_telemetry, name="Telemetry")
+    p_telemetry.start()
+    processes.append(p_telemetry)
+
+    # 2. Video (Optional)
+    if enable_video:
+        p_video = multiprocessing.Process(target=start_video, args=(role, remote_ip), name="Video")
+        p_video.start()
+        processes.append(p_video)
+
+    # 3. Audio (Optional)
+    if enable_audio:
+        p_audio = multiprocessing.Process(target=start_audio, args=(role, remote_ip), name="Audio")
+        p_audio.start()
+        processes.append(p_audio)
+
+    try:
+        while True:
+            time.sleep(1)
+            # Monitor children
+            for p in processes:
+                if not p.is_alive():
+                    logger.error(f"Process {p.name} died!")
+                    # Optional: Restart logic
+    except KeyboardInterrupt:
+        logger.info("Shutting down...")
+        for p in processes:
+            p.terminate()
+            p.join()

universal-telemetry-software/assets/test-video.mp4

@@ -1,3 +1,8 @@
 python-can
 redis
-websockets
+websockets
+PyGObject
+Flask
+Flask-SocketIO
+eventlet
+opencv-python-headless