Skip to content

Commit 558fa39

Browse files
mudlermudler
authored
feat: add audio support (#316)
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 33ffe06 commit 558fa39

6 files changed

Lines changed: 396 additions & 107 deletions

File tree

‎core/agent/agent.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package agent
22

33
import (
4+
"bytes"
45
"context"
56
"errors"
67
"fmt"
8+
"io"
79
"os"
810
"regexp"
911
"strings"
@@ -220,6 +222,42 @@ func (a *Agent) Enqueue(j *types.Job) {
220222
a.jobQueue <- j
221223
}
222224

225+
func (a *Agent) Transcribe(ctx context.Context, file string) (string, error) {
226+
resp, err := a.client.CreateTranscription(ctx,
227+
openai.AudioRequest{
228+
Model: a.options.LLMAPI.TranscriptionModel,
229+
Language: a.options.LLMAPI.TranscriptionLanguage,
230+
FilePath: file,
231+
},
232+
)
233+
if err != nil {
234+
return "", err
235+
}
236+
return resp.Text, nil
237+
}
238+
239+
func (a *Agent) TTS(ctx context.Context, text string) ([]byte, error) {
240+
if a.options.LLMAPI.TTSModel == "" {
241+
return nil, fmt.Errorf("TTS model is not set")
242+
}
243+
resp, err := a.client.CreateSpeech(ctx,
244+
openai.CreateSpeechRequest{
245+
Model: openai.SpeechModel(a.options.LLMAPI.TTSModel),
246+
Input: text,
247+
ResponseFormat: openai.SpeechResponseFormatMp3,
248+
},
249+
)
250+
if err != nil {
251+
return nil, err
252+
}
253+
defer resp.Close()
254+
255+
buf := bytes.NewBuffer(nil)
256+
io.Copy(buf, resp)
257+
258+
return buf.Bytes(), nil
259+
}
260+
223261
func (a *Agent) askLLM(ctx context.Context, conversation []openai.ChatCompletionMessage, maxRetries int) (openai.ChatCompletionMessage, error) {
224262
var resp openai.ChatCompletionResponse
225263
var err error

‎core/agent/options.go‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,13 @@ import (
1212
type Option func(*options) error
1313

1414
type llmOptions struct {
15-
APIURL string
16-
APIKey string
17-
Model string
18-
MultimodalModel string
15+
APIURL string
16+
APIKey string
17+
Model string
18+
MultimodalModel string
19+
TranscriptionModel string
20+
TranscriptionLanguage string
21+
TTSModel string
1922
}
2023

2124
type options struct {
@@ -80,8 +83,11 @@ func defaultOptions() *options {
8083
maxEvaluationLoops: 2,
8184
enableEvaluation: false,
8285
LLMAPI: llmOptions{
83-
APIURL: "http://localhost:8080",
84-
Model: "gpt-4",
86+
APIURL: "http://localhost:8080",
87+
Model: "gpt-4",
88+
TranscriptionModel: "whisper-1",
89+
TranscriptionLanguage: "",
90+
TTSModel: "tts-1",
8591
},
8692
character: Character{
8793
Name: "",
@@ -425,3 +431,24 @@ func EnableEvaluation() Option {
425431
return nil
426432
}
427433
}
434+
435+
func WithTranscriptionModel(model string) Option {
436+
return func(o *options) error {
437+
o.LLMAPI.TranscriptionModel = model
438+
return nil
439+
}
440+
}
441+
442+
func WithTranscriptionLanguage(language string) Option {
443+
return func(o *options) error {
444+
o.LLMAPI.TranscriptionLanguage = language
445+
return nil
446+
}
447+
}
448+
449+
func WithTTSModel(model string) Option {
450+
return func(o *options) error {
451+
o.LLMAPI.TTSModel = model
452+
return nil
453+
}
454+
}

‎core/state/config.go‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,13 +48,16 @@ type AgentConfig struct {
4848

4949
Description string `json:"description" form:"description"`
5050

51-
Model string `json:"model" form:"model"`
52-
MultimodalModel string `json:"multimodal_model" form:"multimodal_model"`
53-
APIURL string `json:"api_url" form:"api_url"`
54-
APIKey string `json:"api_key" form:"api_key"`
55-
LocalRAGURL string `json:"local_rag_url" form:"local_rag_url"`
56-
LocalRAGAPIKey string `json:"local_rag_api_key" form:"local_rag_api_key"`
57-
LastMessageDuration string `json:"last_message_duration" form:"last_message_duration"`
51+
Model string `json:"model" form:"model"`
52+
MultimodalModel string `json:"multimodal_model" form:"multimodal_model"`
53+
TranscriptionModel string `json:"transcription_model" form:"transcription_model"`
54+
TranscriptionLanguage string `json:"transcription_language" form:"transcription_language"`
55+
TTSModel string `json:"tts_model" form:"tts_model"`
56+
APIURL string `json:"api_url" form:"api_url"`
57+
APIKey string `json:"api_key" form:"api_key"`
58+
LocalRAGURL string `json:"local_rag_url" form:"local_rag_url"`
59+
LocalRAGAPIKey string `json:"local_rag_api_key" form:"local_rag_api_key"`
60+
LastMessageDuration string `json:"last_message_duration" form:"last_message_duration"`
5861

5962
Name string `json:"name" form:"name"`
6063
HUD bool `json:"hud" form:"hud"`
@@ -146,6 +149,27 @@ func NewAgentConfigMeta(
146149
DefaultValue: "",
147150
Tags: config.Tags{Section: "ModelSettings"},
148151
},
152+
{
153+
Name: "transcription_model",
154+
Label: "Transcription Model",
155+
Type: "text",
156+
DefaultValue: "",
157+
Tags: config.Tags{Section: "ModelSettings"},
158+
},
159+
{
160+
Name: "transcription_language",
161+
Label: "Transcription Language",
162+
Type: "text",
163+
DefaultValue: "",
164+
Tags: config.Tags{Section: "ModelSettings"},
165+
},
166+
{
167+
Name: "tts_model",
168+
Label: "TTS Model",
169+
Type: "text",
170+
DefaultValue: "",
171+
Tags: config.Tags{Section: "ModelSettings"},
172+
},
149173
{
150174
Name: "api_url",
151175
Label: "API URL",

‎core/state/pool.go‎

Lines changed: 77 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -26,21 +26,21 @@ import (
2626

2727
type AgentPool struct {
2828
sync.Mutex
29-
file string
30-
pooldir string
31-
pool AgentPoolData
32-
agents map[string]*Agent
33-
managers map[string]sse.Manager
34-
agentStatus map[string]*Status
35-
apiURL, defaultModel, defaultMultimodalModel string
36-
mcpBoxURL string
37-
imageModel, localRAGAPI, localRAGKey, apiKey string
38-
availableActions func(*AgentConfig) func(ctx context.Context, pool *AgentPool) []types.Action
39-
connectors func(*AgentConfig) []Connector
40-
dynamicPrompt func(*AgentConfig) func(ctx context.Context, pool *AgentPool) []DynamicPrompt
41-
filters func(*AgentConfig) types.JobFilters
42-
timeout string
43-
conversationLogs string
29+
file string
30+
pooldir string
31+
pool AgentPoolData
32+
agents map[string]*Agent
33+
managers map[string]sse.Manager
34+
agentStatus map[string]*Status
35+
apiURL, defaultModel, defaultMultimodalModel, defaultTTSModel string
36+
mcpBoxURL, defaultTranscriptionModel, defaultTranscriptionLanguage string
37+
imageModel, localRAGAPI, localRAGKey, apiKey string
38+
availableActions func(*AgentConfig) func(ctx context.Context, pool *AgentPool) []types.Action
39+
connectors func(*AgentConfig) []Connector
40+
dynamicPrompt func(*AgentConfig) func(ctx context.Context, pool *AgentPool) []DynamicPrompt
41+
filters func(*AgentConfig) types.JobFilters
42+
timeout string
43+
conversationLogs string
4444
}
4545

4646
type Status struct {
@@ -74,7 +74,7 @@ func loadPoolFromFile(path string) (*AgentPoolData, error) {
7474
}
7575

7676
func NewAgentPool(
77-
defaultModel, defaultMultimodalModel, imageModel, apiURL, apiKey, directory, mcpBoxURL string,
77+
defaultModel, defaultMultimodalModel, defaultTranscriptionModel, defaultTranscriptionLanguage, defaultTTSModel, imageModel, apiURL, apiKey, directory, mcpBoxURL string,
7878
LocalRAGAPI string,
7979
availableActions func(*AgentConfig) func(ctx context.Context, pool *AgentPool) []types.Action,
8080
connectors func(*AgentConfig) []Connector,
@@ -96,25 +96,28 @@ func NewAgentPool(
9696
if _, err := os.Stat(poolfile); err != nil {
9797
// file does not exist, create a new pool
9898
return &AgentPool{
99-
file: poolfile,
100-
pooldir: directory,
101-
apiURL: apiURL,
102-
defaultModel: defaultModel,
103-
defaultMultimodalModel: defaultMultimodalModel,
104-
mcpBoxURL: mcpBoxURL,
105-
imageModel: imageModel,
106-
localRAGAPI: LocalRAGAPI,
107-
apiKey: apiKey,
108-
agents: make(map[string]*Agent),
109-
pool: make(map[string]AgentConfig),
110-
agentStatus: make(map[string]*Status),
111-
managers: make(map[string]sse.Manager),
112-
connectors: connectors,
113-
availableActions: availableActions,
114-
dynamicPrompt: promptBlocks,
115-
filters: filters,
116-
timeout: timeout,
117-
conversationLogs: conversationPath,
99+
file: poolfile,
100+
pooldir: directory,
101+
apiURL: apiURL,
102+
defaultModel: defaultModel,
103+
defaultMultimodalModel: defaultMultimodalModel,
104+
defaultTranscriptionModel: defaultTranscriptionModel,
105+
defaultTranscriptionLanguage: defaultTranscriptionLanguage,
106+
defaultTTSModel: defaultTTSModel,
107+
mcpBoxURL: mcpBoxURL,
108+
imageModel: imageModel,
109+
localRAGAPI: LocalRAGAPI,
110+
apiKey: apiKey,
111+
agents: make(map[string]*Agent),
112+
pool: make(map[string]AgentConfig),
113+
agentStatus: make(map[string]*Status),
114+
managers: make(map[string]sse.Manager),
115+
connectors: connectors,
116+
availableActions: availableActions,
117+
dynamicPrompt: promptBlocks,
118+
filters: filters,
119+
timeout: timeout,
120+
conversationLogs: conversationPath,
118121
}, nil
119122
}
120123

@@ -123,25 +126,28 @@ func NewAgentPool(
123126
return nil, err
124127
}
125128
return &AgentPool{
126-
file: poolfile,
127-
apiURL: apiURL,
128-
pooldir: directory,
129-
defaultModel: defaultModel,
130-
defaultMultimodalModel: defaultMultimodalModel,
131-
mcpBoxURL: mcpBoxURL,
132-
imageModel: imageModel,
133-
apiKey: apiKey,
134-
agents: make(map[string]*Agent),
135-
managers: make(map[string]sse.Manager),
136-
agentStatus: map[string]*Status{},
137-
pool: *poolData,
138-
connectors: connectors,
139-
localRAGAPI: LocalRAGAPI,
140-
dynamicPrompt: promptBlocks,
141-
filters: filters,
142-
availableActions: availableActions,
143-
timeout: timeout,
144-
conversationLogs: conversationPath,
129+
file: poolfile,
130+
apiURL: apiURL,
131+
pooldir: directory,
132+
defaultModel: defaultModel,
133+
defaultMultimodalModel: defaultMultimodalModel,
134+
defaultTranscriptionModel: defaultTranscriptionModel,
135+
defaultTranscriptionLanguage: defaultTranscriptionLanguage,
136+
defaultTTSModel: defaultTTSModel,
137+
mcpBoxURL: mcpBoxURL,
138+
imageModel: imageModel,
139+
apiKey: apiKey,
140+
agents: make(map[string]*Agent),
141+
managers: make(map[string]sse.Manager),
142+
agentStatus: map[string]*Status{},
143+
pool: *poolData,
144+
connectors: connectors,
145+
localRAGAPI: LocalRAGAPI,
146+
dynamicPrompt: promptBlocks,
147+
filters: filters,
148+
availableActions: availableActions,
149+
timeout: timeout,
150+
conversationLogs: conversationPath,
145151
}, nil
146152
}
147153

@@ -334,11 +340,25 @@ func (a *AgentPool) startAgentWithConfig(name string, config *AgentConfig, obs O
334340
ctx := context.Background()
335341
model := a.defaultModel
336342
multimodalModel := a.defaultMultimodalModel
343+
transcriptionModel := a.defaultTranscriptionModel
344+
transcriptionLanguage := a.defaultTranscriptionLanguage
345+
ttsModel := a.defaultTTSModel
337346

338347
if config.MultimodalModel != "" {
339348
multimodalModel = config.MultimodalModel
340349
}
341350

351+
if config.TranscriptionModel != "" {
352+
transcriptionModel = config.TranscriptionModel
353+
}
354+
355+
if config.TranscriptionLanguage != "" {
356+
transcriptionLanguage = config.TranscriptionLanguage
357+
}
358+
if config.TTSModel != "" {
359+
ttsModel = config.TTSModel
360+
}
361+
342362
if config.Model != "" {
343363
model = config.Model
344364
} else {
@@ -419,6 +439,9 @@ func (a *AgentPool) startAgentWithConfig(name string, config *AgentConfig, obs O
419439
WithLLMAPIURL(a.apiURL),
420440
WithContext(ctx),
421441
WithMCPServers(config.MCPServers...),
442+
WithTranscriptionModel(transcriptionModel),
443+
WithTranscriptionLanguage(transcriptionLanguage),
444+
WithTTSModel(ttsModel),
422445
WithPeriodicRuns(config.PeriodicRuns),
423446
WithPermanentGoal(config.PermanentGoal),
424447
WithMCPSTDIOServers(config.MCPSTDIOServers...),

‎main.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ import (
1313

1414
var baseModel = os.Getenv("LOCALAGI_MODEL")
1515
var multimodalModel = os.Getenv("LOCALAGI_MULTIMODAL_MODEL")
16+
var transcriptionModel = os.Getenv("LOCALAGI_TRANSCRIPTION_MODEL")
17+
var transcriptionLanguage = os.Getenv("LOCALAGI_TRANSCRIPTION_LANGUAGE")
18+
var ttsModel = os.Getenv("LOCALAGI_TTS_MODEL")
1619
var apiURL = os.Getenv("LOCALAGI_LLM_API_URL")
1720
var apiKey = os.Getenv("LOCALAGI_LLM_API_KEY")
1821
var timeout = os.Getenv("LOCALAGI_TIMEOUT")
@@ -60,6 +63,9 @@ func main() {
6063
pool, err := state.NewAgentPool(
6164
baseModel,
6265
multimodalModel,
66+
transcriptionModel,
67+
transcriptionLanguage,
68+
ttsModel,
6369
imageModel,
6470
apiURL,
6571
apiKey,

0 commit comments

Comments
 (0)