add gemini demo (#62)

Author: czerwiukk

gemini-demo/.dockerignore

@@ -0,0 +1,7 @@
+node_modules/
+dist/
+.env
+*.log
+.DS_Store
+backend/yarn.lock
+web/yarn.lock

gemini-demo/.env.example

@@ -0,0 +1,4 @@
+FISHJAM_ID=
+FISHJAM_MANAGEMENT_TOKEN=
+GEMINI_API_KEY=
+VITE_FISHJAM_ID=

gemini-demo/.gitignore

@@ -0,0 +1,6 @@
+node_modules/
+.yarn
+dist/
+.env
+*.log
+.DS_Store

gemini-demo/.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

gemini-demo/README.md

@@ -0,0 +1,57 @@
+# Gemini Demo
+
+A minimal example of a video call with a Gemini Live AI agent using Fishjam Cloud.
+
+## What it does
+
+- Create a room and join a video call
+- Spawn a Gemini Live voice agent with a custom system prompt
+- The agent joins the call, listens to participants, and responds with voice
+- Supports Google Search for real-time information
+
+## Setup
+
+1. Copy `.env.example` to `.env` and fill in your credentials:
+
+```
+cp .env.example .env
+```
+
+2. Install dependencies:
+
+```
+cd backend && npm install
+cd ../web && npm install
+```
+
+3. Start the backend:
+
+```
+cd backend && npm run start
+```
+
+4. Start the frontend (in another terminal):
+
+```
+cd web && npm run start
+```
+
+5. Open http://localhost:5173
+
+## Architecture
+
+```
+backend/src/main.ts   - Fastify + tRPC server, Fishjam SDK, Gemini Live API
+web/src/App.tsx        - React frontend with Fishjam React Client
+web/src/trpc.ts        - tRPC client setup
+```
+
+### Audio flow
+
+```
+Peer audio (16kHz) → Fishjam Agent → Gemini Live API
+                                          ↓
+                    Fishjam Agent Track ← Gemini response (24kHz)
+                          ↓
+                    All peers hear the agent
+```

gemini-demo/backend/.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

gemini-demo/backend/Dockerfile

@@ -0,0 +1,37 @@
+FROM node:24-alpine AS builder
+
+WORKDIR /app
+
+RUN apk add --no-cache python3 make g++
+RUN corepack enable
+
+COPY package.json yarn.lock .yarnrc.yml ./
+COPY backend ./backend
+COPY web ./web
+
+RUN yarn install --immutable
+
+RUN yarn workspace gemini-demo-backend build
+
+FROM node:24-alpine AS runner
+
+WORKDIR /app
+
+RUN apk add --no-cache dumb-init python3 make g++
+RUN corepack enable
+
+COPY package.json yarn.lock .yarnrc.yml ./
+COPY backend ./backend
+COPY web ./web
+
+RUN yarn install --immutable
+
+COPY --from=builder /app/backend/dist ./backend/dist
+
+RUN addgroup -g 1001 -S nodejs && adduser -S nodejs -u 1001
+USER nodejs
+
+EXPOSE 8000
+
+ENTRYPOINT ["dumb-init", "--"]
+CMD ["node", "backend/dist/main.js"]

gemini-demo/backend/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "gemini-demo-backend",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./src/main.ts"
+    }
+  },
+  "types": "./src/main.ts",
+  "dependencies": {
+    "@fastify/cors": "^11.1.0",
+    "@fishjam-cloud/js-server-sdk": "^0.25.4",
+    "@google/genai": "^1.44.0",
+    "@trpc/server": "^11.6.0",
+    "dotenv": "^17.2.3",
+    "fastify": "^5.6.1",
+    "pino-pretty": "^13.1.1",
+    "ws": "^8.18.0",
+    "zod": "^4.1.11"
+  },
+  "devDependencies": {
+    "@tsconfig/node24": "^24.0.1",
+    "@types/node": "^24.5.2",
+    "@types/ws": "^8.18.1",
+    "tsx": "^4.20.6",
+    "typescript": "^5.9.2"
+  },
+  "scripts": {
+    "start": "tsx watch src/main.ts",
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc --noEmit"
+  },
+  "packageManager": "yarn@4.12.0"
+}

gemini-demo/backend/src/agents.ts

@@ -0,0 +1,96 @@
+import { LiveServerMessage, Modality } from "@google/genai";
+import * as FishjamGemini from "@fishjam-cloud/js-server-sdk/gemini";
+import { type TrackId, type RoomId } from "@fishjam-cloud/js-server-sdk";
+
+import { fishjam, genai } from "./clients.js";
+
+export const createAgent = async (
+  roomId: RoomId,
+  systemInstruction: string,
+) => {
+  const { agent: fishjamAgent } = await fishjam.createAgent(roomId, {
+    output: FishjamGemini.geminiInputAudioSettings,
+  });
+
+  const agentTrack = fishjamAgent.createTrack(
+    FishjamGemini.geminiOutputAudioSettings,
+  );
+
+  let cleanup: CallableFunction | undefined;
+
+  const session = await genai.live.connect({
+    model: "gemini-3.1-flash-live-preview",
+    config: {
+      responseModalities: [Modality.AUDIO],
+      systemInstruction,
+      tools: [
+        { googleSearch: {} },
+        {
+          functionDeclarations: [
+            {
+              name: "disconnect",
+              description: `Disconnect yourself from the room. 
+                            Use this when the user asks you to disconnect.`,
+            },
+          ],
+        },
+      ],
+    },
+    callbacks: {
+      onmessage: async (message: LiveServerMessage) => {
+        if (message.data) {
+          const audio = Buffer.from(message.data, "base64");
+          fishjamAgent.sendData(agentTrack.id, audio);
+        }
+
+        if (message.serverContent?.interrupted) {
+          fishjamAgent.interruptTrack(agentTrack.id);
+        }
+
+        message.toolCall?.functionCalls?.forEach((call) => {
+          if (call.name === "disconnect") {
+            cleanup?.();
+          }
+        });
+      },
+    },
+  });
+
+  const room = await fishjam.getRoom(roomId);
+
+  const interval = setInterval(async () => {
+    const humanPeerVideoTrack = room.peers
+      .find(({ type }) => type === "webrtc")
+      ?.tracks.find(({ type }) => type === "video");
+
+    if (!humanPeerVideoTrack?.id) return;
+
+    const image = await fishjamAgent.captureImage(
+      humanPeerVideoTrack.id as TrackId,
+    );
+
+    session.sendRealtimeInput({
+      video: {
+        data: Buffer.from(image.data).toString("base64"),
+        mimeType: image.contentType,
+      },
+    });
+  }, 1000);
+
+  fishjamAgent.on("trackData", ({ data }) => {
+    session.sendRealtimeInput({
+      audio: {
+        data: Buffer.from(data).toString("base64"),
+        mimeType: FishjamGemini.inputMimeType,
+      },
+    });
+  });
+
+  cleanup = () => {
+    clearInterval(interval);
+    session.close();
+    fishjamAgent.deleteTrack(agentTrack.id);
+    fishjamAgent.removeAllListeners("trackData");
+    fishjamAgent.disconnect();
+  };
+};

gemini-demo/backend/src/clients.ts

@@ -0,0 +1,13 @@
+import { FishjamClient } from "@fishjam-cloud/js-server-sdk";
+import * as FishjamGemini from "@fishjam-cloud/js-server-sdk/gemini";
+
+import config from "./config.js";
+
+export const fishjam = new FishjamClient({
+  fishjamId: config.FISHJAM_ID,
+  managementToken: config.FISHJAM_MANAGEMENT_TOKEN,
+});
+
+export const genai = FishjamGemini.createClient({
+  apiKey: config.GEMINI_API_KEY,
+});