refactor: new voice chat sagas and add livekit voice chat (#443)

* w

* checkpoint

* wip

* wip

* split voice chat sagas

* wip

* wip

* checkpoint: new voice sagas working with opus

* delete old voice sagas

* add missing libopus.wasm.js
add livekit voice handler

* fix

* change token handle

* update unity-renderer

* livekit with custom and temporal server to get token

* improve livekit voice chat

* room names by realm and island

* todo comment

* better logs

* fix: remove initialization of voice chat sagas from kernel (#453)

* feat: add spatial sound (need more test)
feat: add ff for livekit connection (independent from transport)

* checkpoint

* working for chrome

* remove logs

* typo

* fix ff

* fix

* fix

* remove undefined type

* use ff

* minor issues (#465)

Co-authored-by: menduz <github@menduz.com>

Author: kuruk-mm

package-lock.json

@@ -27,7 +27,7 @@
   "devDependencies": {
     "@dcl/eslint-config": "^1.0.5-20220608210046.commit-d51d9ec",
     "@dcl/posix": "^1.0.3",
-    "@dcl/unity-renderer": "^1.0.41205-20220623213209.commit-67ba18f",
+    "@dcl/unity-renderer": "^1.0.49884-20220818143841.commit-421e39d",
     "@microsoft/api-documenter": "^7.2.2",
     "@microsoft/api-extractor": "^7.1.7",
     "@tweenjs/tween.js": "^17.2.0",
@@ -98,6 +98,7 @@
     "gifuct-js": "^2.0.0",
     "google-protobuf": "^3.6.1",
     "hls.js": "^0.14.17",
+    "livekit-client": "^1.2.10",
     "mitt": "^3.0.0",
     "mz-observable": "^1.0.1",
     "p-queue": "^7.1.0",

package.json

@@ -2,7 +2,8 @@ export default class Html {
   static loopbackAudioElement() {
     const audio = document.getElementById('voice-chat-audio')
     if (!audio) {
-      console.error('There is no #voice-chat-audio element to use with VoiceChat')
+      console.error('There is no #voice-chat-audio element to use with VoiceChat. Returning a `new Audio()`')
+      return new Audio()
     }
     return audio as HTMLAudioElement | undefined
   }

packages/shared/Html.ts

@@ -1,61 +1,5 @@
 import { action } from 'typesafe-actions'
-import type { VoiceCommunicator } from 'voice-chat-codec/VoiceCommunicator'
 import type { CommsContext } from './context'
-import type { VoicePolicy } from './types'
-
-export const VOICE_PLAYING_UPDATE = '[COMMS] voicePlayingUpdate'
-export const voicePlayingUpdate = (userId: string, playing: boolean) =>
-  action(VOICE_PLAYING_UPDATE, { userId, playing })
-export type VoicePlayingUpdate = ReturnType<typeof voicePlayingUpdate>
-
-export const INIT_VOICE_COMMUNICATOR = '[COMMS] setVoiceCommunicator'
-export const setVoiceCommunicator = (voiceCommunicator: VoiceCommunicator | null) =>
-  action(INIT_VOICE_COMMUNICATOR, { voiceCommunicator })
-
-/**
- * Action to join voice chat
- */
-export const JOIN_VOICE_CHAT = '[COMMS] joinVoiceChat'
-export const joinVoiceChat = () => action(JOIN_VOICE_CHAT)
-
-/**
- * Action to leave voice chat
- */
-export const LEAVE_VOICE_CHAT = '[COMMS] leaveVoiceChat'
-export const leaveVoiceChat = () => action(LEAVE_VOICE_CHAT)
-
-/**
- * Action to trigger voice chat recording
- */
-export const SET_VOICE_CHAT_RECORDING = '[COMMS] setVoiceChatRecording'
-export const setVoiceChatRecording = (recording: boolean) => action(SET_VOICE_CHAT_RECORDING, { recording })
-export type SetVoiceChatRecording = ReturnType<typeof setVoiceChatRecording>
-
-/**
- * Action to toggle voice chat recording
- */
-export const TOGGLE_VOICE_CHAT_RECORDING = '[COMMS] toggleVoiceChatRecording'
-export const toggleVoiceChatRecording = () => action(TOGGLE_VOICE_CHAT_RECORDING)
-export type ToggleVoiceChatRecording = ReturnType<typeof toggleVoiceChatRecording>
-
-/**
- * Action triggered when recording starts or stops
- */
-export const VOICE_RECORDING_UPDATE = '[COMMS] voiceRecordingUpdate'
-export const voiceRecordingUpdate = (recording: boolean) => action(VOICE_RECORDING_UPDATE, { recording })
-export type VoiceRecordingUpdate = ReturnType<typeof voiceRecordingUpdate>
-
-export const SET_VOICE_VOLUME = '[COMMS] setVoiceVolume'
-export const setVoiceVolume = (volume: number) => action(SET_VOICE_VOLUME, { volume })
-export type SetVoiceVolume = ReturnType<typeof setVoiceVolume>
-
-export const SET_VOICE_MUTE = '[COMMS] setVoiceMute'
-export const setVoiceMute = (mute: boolean) => action(SET_VOICE_MUTE, { mute })
-export type SetVoiceMute = ReturnType<typeof setVoiceMute>
-
-export const SET_VOICE_POLICY = '[COMMS] setVoicePolicy'
-export const setVoicePolicy = (voicePolicy: VoicePolicy) => action(SET_VOICE_POLICY, { voicePolicy })
-export type SetVoicePolicy = ReturnType<typeof setVoicePolicy>
 
 export const SET_COMMS_ISLAND = '[COMMS] setCommsIsland'
 export const setCommsIsland = (island: string | undefined) => action(SET_COMMS_ISLAND, { island })

packages/shared/comms/actions.ts

@@ -23,7 +23,7 @@ import { messageReceived } from '../chat/actions'
 import { getBannedUsers } from 'shared/meta/selectors'
 import { getIdentity } from 'shared/session'
 import { CommsContext } from './context'
-import { processVoiceFragment } from './voice-over-comms'
+import { processVoiceFragment } from 'shared/voiceChat/handlers'
 import future, { IFuture } from 'fp-future'
 import { handleCommsDisconnection } from './actions'
 import { Avatar } from '@dcl/schemas'
@@ -34,7 +34,7 @@ import { trackEvent } from 'shared/analytics'
 import { ProfileType } from 'shared/profiles/types'
 import { ensureAvatarCompatibilityFormat } from 'shared/profiles/transformations/profileToServerFormat'
 import { scenesSubscribedToCommsEvents } from './sceneSubscriptions'
-import { isBlockedOrBanned } from './voice-selectors'
+import { isBlockedOrBanned } from 'shared/voiceChat/selectors'
 
 const receiveProfileOverCommsChannel = new Observable<Avatar>()
 const sendMyProfileOverCommsChannel = new Observable<Record<string, never>>()

packages/shared/comms/handlers.ts

@@ -2,22 +2,12 @@ import { AnyAction } from 'redux'
 
 import { COMMS_ESTABLISHED } from 'shared/loading/types'
 
-import { CommsState, VoicePolicy } from './types'
-import {
-  INIT_VOICE_COMMUNICATOR,
-  SET_COMMS_ISLAND,
-  SET_VOICE_CHAT_RECORDING,
-  SET_VOICE_POLICY,
-  SET_WORLD_CONTEXT,
-  TOGGLE_VOICE_CHAT_RECORDING
-} from './actions'
+import { CommsState } from './types'
+import { SET_COMMS_ISLAND, SET_WORLD_CONTEXT } from './actions'
 
 const INITIAL_COMMS: CommsState = {
   initialized: false,
-  voiceChatRecording: false,
-  voicePolicy: VoicePolicy.ALLOW_ALL,
-  context: undefined,
-  voiceCommunicator: null
+  context: undefined
 }
 
 export function commsReducer(state?: CommsState, action?: AnyAction): CommsState {
@@ -30,15 +20,6 @@ export function commsReducer(state?: CommsState, action?: AnyAction): CommsState
   switch (action.type) {
     case COMMS_ESTABLISHED:
       return { ...state, initialized: true }
-    case INIT_VOICE_COMMUNICATOR:
-      return { ...state, voiceCommunicator: action.payload.voiceCommunicator }
-    case SET_VOICE_CHAT_RECORDING:
-      if (action.payload.recording === state.voiceChatRecording) return state
-      return { ...state, voiceChatRecording: action.payload.recording }
-    case TOGGLE_VOICE_CHAT_RECORDING:
-      return { ...state, voiceChatRecording: !state.voiceChatRecording }
-    case SET_VOICE_POLICY:
-      return { ...state, voicePolicy: action.payload.voicePolicy }
     case SET_COMMS_ISLAND:
       return { ...state, island: action.payload.island }
     case SET_WORLD_CONTEXT:

packages/shared/comms/reducer.ts

@@ -4,7 +4,6 @@ import { commsEstablished, FATAL_ERROR } from 'shared/loading/types'
 import { CommsContext, commsLogger } from './context'
 import { getCommsContext, getRealm } from './selectors'
 import { BEFORE_UNLOAD } from 'shared/protocol/actions'
-import { voiceSaga } from './voice-sagas'
 import {
   HandleCommsDisconnection,
   HANDLE_COMMS_DISCONNECTION,
@@ -192,8 +191,6 @@ function* handleNewCommsContext() {
       // disconnect previous context
       yield call(disconnectContext, oldContext)
     }
-
-    yield call(voiceSaga)
   })
 }
 

packages/shared/comms/sagas.ts

@@ -1,23 +1,10 @@
 import { Realm } from 'shared/dao/types'
-import { isFeatureToggleEnabled } from 'shared/selectors'
-import { VOICE_CHAT_FEATURE_TOGGLE } from 'shared/types'
-import { lastPlayerScene } from 'shared/world/sceneState'
-import type { VoiceCommunicator } from 'voice-chat-codec/VoiceCommunicator'
 import type { CommsContext } from './context'
 import { RootCommsState } from './types'
 
-export const isVoiceChatRecording = (store: RootCommsState) => store.comms.voiceChatRecording
-export const getVoicePolicy = (store: RootCommsState) => store.comms.voicePolicy
 export const getCommsIsland = (store: RootCommsState): string | undefined => store.comms.island
 export const getRealm = (store: RootCommsState): Realm | undefined => store.comms.context?.realm
 export const getCommsContext = (state: RootCommsState): CommsContext | undefined => state.comms.context
-export const getVoiceCommunicator = (store: RootCommsState): VoiceCommunicator | null => {
-  return store.comms.voiceCommunicator
-}
-
-export function isVoiceChatAllowedByCurrentScene() {
-  return isFeatureToggleEnabled(VOICE_CHAT_FEATURE_TOGGLE, lastPlayerScene?.entity.metadata)
-}
 
 export function sameRealm(realm1: Realm, realm2: Realm) {
   return (