Gemini integration (#218)

## Description

Adds the Gemini integration.
It's available to import from the `@fishjam-cloud/js-server-sdk/gemini`
path.
Transcription example has been adjusted as well.

## Documentation impact

- [ ] Documentation update required
- [ ] Documentation updated [in another PR](_)
- [x] No documentation update required

## Types of changes

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to
      not work as expected)

Author: czerwiukk

examples/transcription/src/const.ts

@@ -1 +1 @@
-export const TRANSCRIPTION_MODEL = 'gemini-live-2.5-flash-preview';
+export const TRANSCRIPTION_MODEL = 'gemini-2.5-flash-native-audio-preview-09-2025';

examples/transcription/src/service/transcription.ts

@@ -1,6 +1,5 @@
 import {
   FishjamConfig,
-  FishjamAgent,
   FishjamWSNotifier,
   PeerConnected,
   PeerDisconnected,
@@ -9,6 +8,7 @@ import {
   FishjamClient,
   RoomId,
 } from '@fishjam-cloud/js-server-sdk';
+import GeminiIntegration from '@fishjam-cloud/js-server-sdk/gemini';
 import { GoogleGenAI, LiveServerMessage, Modality, Session } from '@google/genai';
 import { TRANSCRIPTION_MODEL } from '../const';
 
@@ -20,7 +20,7 @@ export class TranscriptionService {
   fishjamClient: FishjamClient;
 
   constructor(fishjamConfig: FishjamConfig, geminiKey: string) {
-    this.ai = new GoogleGenAI({ apiKey: geminiKey });
+    this.ai = GeminiIntegration.createClient({ apiKey: geminiKey });
     this.fishjamConfig = fishjamConfig;
     this.fishjamClient = new FishjamClient(fishjamConfig);
     this.initFishjam();
@@ -51,7 +51,7 @@ export class TranscriptionService {
         agent: agent,
       } = await this.fishjamClient.createAgent(
         message.roomId,
-        {},
+        { output: GeminiIntegration.geminiInputAudioSettings },
         {
           onClose: (code, reason) => console.log(`Fishjam agent websocket closed. code: ${code}, reason: ${reason}`),
           onError: (error) => console.error('Fishjam agent websocket error: %O', error),
@@ -67,7 +67,7 @@ export class TranscriptionService {
     const session = await this.ai.live.connect({
       model: TRANSCRIPTION_MODEL,
       config: {
-        responseModalities: [Modality.TEXT],
+        responseModalities: [Modality.AUDIO],
         inputAudioTranscription: {},
       },
       callbacks: {
@@ -115,10 +115,11 @@ export class TranscriptionService {
     const { data, peerId } = message;
 
     const session = this.peerSessions.get(peerId);
+
     session?.sendRealtimeInput({
       audio: {
         data: data.toBase64(),
-        mimeType: 'audio/pcm;rate=16000',
+        mimeType: GeminiIntegration.inputMimeType,
       },
     });
   }

packages/js-server-sdk/package.json

@@ -24,7 +24,8 @@
     "dist"
   ],
   "exports": {
-    ".": "./dist/index.js"
+    ".": "./dist/index.js",
+    "./gemini": "./dist/integrations/gemini.js"
   },
   "scripts": {
     "build": "tsup --dts-resolve",
@@ -37,6 +38,7 @@
   "tsup": {
     "entry": [
       "src/index.ts",
+      "src/integrations/gemini.ts",
       "src/proto.ts"
     ],
     "noExternal": [
@@ -54,6 +56,14 @@
     "axios": "^1.7.9",
     "uuid": "^11.1.0"
   },
+  "peerDependencies": {
+    "@google/genai": "^1.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@google/genai": {
+      "optional": true
+    }
+  },
   "devDependencies": {
     "@fishjam-cloud/fishjam-openapi": "workspace:*",
     "@fishjam-cloud/fishjam-proto": "workspace:*",

packages/js-server-sdk/src/integrations/gemini.ts

@@ -0,0 +1,60 @@
+import type { GoogleGenAI, GoogleGenAIOptions } from '@google/genai' with { 'resolution-mode': 'import' };
+import fishjamSDK from '../../package.json';
+import type { PeerOptionsAgentOutput } from '@fishjam-cloud/fishjam-openapi';
+import type { AudioCodecParameters } from '../agent';
+
+const SDK_NAME = 'fishjam-js-server-sdk';
+
+/**
+ * A collection of settings for Google Gemini integration.
+ */
+export default {
+  /**
+   * Creates a GoogleGenAI client.
+   * This function dynamically imports the "@google/genai" module,
+   * so it will only be loaded when this function is called.
+   *
+   * @param options Configuration for the GoogleGenAI client.
+   * @returns A GoogleGenAI instance.
+   */
+  createClient: (options: GoogleGenAIOptions): GoogleGenAI => {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const { GoogleGenAI } = require('@google/genai');
+    const trackingHeader = { 'X-Goog-Api-Client': `${SDK_NAME}/${fishjamSDK.version}` };
+    const finalOptions = {
+      ...options,
+      httpOptions: {
+        ...options.httpOptions,
+        headers: {
+          ...options.httpOptions?.headers,
+          ...trackingHeader,
+        },
+      },
+    };
+    return new GoogleGenAI(finalOptions);
+  },
+
+  /**
+   * Predefined audio settings for the agent's output track,
+   * configured for Gemini's 24kHz audio output.
+   */
+  geminiOutputAudioSettings: {
+    encoding: 'pcm16',
+    channels: 1,
+    sampleRate: 24000,
+  } as const satisfies AudioCodecParameters,
+
+  /**
+   * Predefined audio settings for subscribing to room audio,
+   * configured for Gemini's 16kHz audio input.
+   */
+  geminiInputAudioSettings: {
+    audioFormat: 'pcm16',
+    audioSampleRate: 16000,
+  } as const satisfies PeerOptionsAgentOutput,
+
+  /**
+   * The MIME type for the audio data sent to Gemini.
+   */
+  inputMimeType: 'audio/pcm;rate=16000' as const,
+};

packages/js-server-sdk/tsconfig.json

@@ -6,9 +6,9 @@
     "lib": ["es2023"],
     "module": "node16",
     "target": "es2022",
-
     "strict": true,
     "esModuleInterop": true,
+    "resolveJsonModule": true,
     "skipLibCheck": true,
     "declaration": true,
     "moduleResolution": "node16",

yarn.lock

@@ -858,6 +858,11 @@ __metadata:
     tsup: "npm:^8.4.0"
     typed-emitter: "npm:^2.1.0"
     uuid: "npm:^11.1.0"
+  peerDependencies:
+    "@google/genai": ^1.0.0
+  peerDependenciesMeta:
+    "@google/genai":
+      optional: true
   languageName: unknown
   linkType: soft