feat(realtime): LocalAI Assistant ("Manage Mode") for the Talk page

Mirrors the chat-page metadata.localai_assistant flow so users can ask the
realtime model what's loaded / installed / configured. Tools are run
server-side via the same in-process MCP holder that powers the chat
modality — no transport switch, no proxy, no new wire protocol.

Wire:
- core/http/endpoints/openai/realtime.go:
  - RealtimeSessionOptions{LocalAIAssistant,IsAdmin}; isCurrentUserAdmin
    helper mirrors chat.go's requireAssistantAccess (no-op when auth
    disabled, else requires auth.RoleAdmin).
  - Session grows AssistantExecutor mcpTools.ToolExecutor.
  - runRealtimeSession, when opts.LocalAIAssistant is set: gate on admin,
    fail closed if DisableLocalAIAssistant or the holder has no tools,
    DiscoverTools and inject into session.Tools, prepend
    holder.SystemPrompt() to instructions.
  - Tool-call dispatch loop: when AssistantExecutor.IsTool(name), run
    ExecuteTool inproc, append a FunctionCallOutput to conv.Items, skip
    the function_call_arguments client emit (the client can't execute
    these — it doesn't know about them). After the loop, if any
    assistant tool ran, trigger another response so the model speaks the
    result. Mirrors chat's agentic loop, driven server-side rather than
    via client round-trip.

- core/http/endpoints/openai/realtime_webrtc.go: RealtimeCallRequest
  gains `localai_assistant` (JSON omitempty). Handshake calls
  isCurrentUserAdmin and builds RealtimeSessionOptions.

- core/http/react-ui/src/pages/Talk.jsx: admin-only "Manage Mode"
  checkbox under the Tools dropdown; passes localai_assistant: true to
  realtimeApi.call's body, captured in the connect callback's deps.

Mirroring chat's pattern means the in-process MCP tools surface "just
works" for the Talk page without exposing a Streamable-HTTP MCP endpoint
(which was the alternative). Clients with their own MCP servers can
still use the existing ClientMCPDropdown path in parallel; the realtime
handler distinguishes them by AssistantExecutor.IsTool() at dispatch
time.

Assisted-by: claude-code:claude-opus-4-7-1m [Claude Code]
Signed-off-by: Richard Palethorpe <io@richiejp.com>

Author: richiejp

core/http/endpoints/openai/realtime.go

@@ -20,6 +20,8 @@ import (
 	"github.com/mudler/LocalAI/core/backend"
 
 	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/http/auth"
+	mcpTools "github.com/mudler/LocalAI/core/http/endpoints/mcp"
 	"github.com/mudler/LocalAI/core/http/endpoints/openai/types"
 	"github.com/mudler/LocalAI/core/schema"
 	"github.com/mudler/LocalAI/core/templates"
@@ -86,6 +88,14 @@ type Session struct {
 	// pairs are kept together so we never feed an orphaned tool result.
 	MaxHistoryItems int
 
+	// AssistantExecutor is non-nil when the session opted into the in-process
+	// LocalAI Assistant tool surface. Tool calls whose name matches this
+	// executor's catalog are run inproc and their output is fed back to the
+	// model server-side; the client never sees a function_call_arguments
+	// event for those. Mirrors the chat handler's metadata.localai_assistant
+	// path.
+	AssistantExecutor mcpTools.ToolExecutor
+
 	// Response cancellation: protects activeResponseCancel/activeResponseDone
 	responseMu           sync.Mutex
 	activeResponseCancel context.CancelFunc
@@ -211,6 +221,19 @@ func RealtimeTranscriptionSession(application *application.Application) echo.Han
 	}
 }
 
+// RealtimeSessionOptions bundles per-session knobs decoded from the WS query
+// string (or the WebRTC handshake body). Mirrors what chat.go pulls off
+// `metadata.localai_assistant` — admin-only opt-in to the in-process
+// management tool surface.
+type RealtimeSessionOptions struct {
+	LocalAIAssistant bool
+	// AuthEnabled mirrors chat.go's requireAssistantAccess gate. We resolve
+	// admin role at handshake time (where the echo.Context has the auth
+	// cookie/Bearer) and drop the result here so runRealtimeSession can
+	// decide without holding onto the request.
+	IsAdmin bool
+}
+
 func Realtime(application *application.Application) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		ws, err := upgrader.Upgrade(c.Response(), c.Request(), nil)
@@ -224,18 +247,33 @@ func Realtime(application *application.Application) echo.HandlerFunc {
 
 		// Extract query parameters from Echo context before passing to websocket handler
 		model := c.QueryParam("model")
+		opts := RealtimeSessionOptions{
+			LocalAIAssistant: c.QueryParam("localai_assistant") == "1" || c.QueryParam("localai_assistant") == "true",
+			IsAdmin:          isCurrentUserAdmin(c, application),
+		}
 
-		registerRealtime(application, model)(ws)
+		registerRealtime(application, model, opts)(ws)
 		return nil
 	}
 }
 
-func registerRealtime(application *application.Application, model string) func(c *websocket.Conn) {
+// isCurrentUserAdmin replicates the chat-side admin check at the realtime
+// handshake. When auth is disabled, every caller is treated as admin (same
+// as chat's requireAssistantAccess).
+func isCurrentUserAdmin(c echo.Context, application *application.Application) bool {
+	if application == nil || application.ApplicationConfig() == nil || !application.ApplicationConfig().Auth.Enabled {
+		return true
+	}
+	user := auth.GetUser(c)
+	return user != nil && user.Role == auth.RoleAdmin
+}
+
+func registerRealtime(application *application.Application, model string, opts RealtimeSessionOptions) func(c *websocket.Conn) {
 	return func(conn *websocket.Conn) {
 		t := NewWebSocketTransport(conn)
 		evaluator := application.TemplatesEvaluator()
 		xlog.Debug("Realtime WebSocket connection established", "address", conn.RemoteAddr().String(), "model", model)
-		runRealtimeSession(application, t, model, evaluator)
+		runRealtimeSession(application, t, model, evaluator, opts)
 	}
 }
 
@@ -306,7 +344,7 @@ func prepareRealtimeConfig(cfg *config.ModelConfig) (errCode, errMsg string, ok
 
 // runRealtimeSession runs the main event loop for a realtime session.
 // It is transport-agnostic and works with both WebSocket and WebRTC.
-func runRealtimeSession(application *application.Application, t Transport, model string, evaluator *templates.Evaluator) {
+func runRealtimeSession(application *application.Application, t Transport, model string, evaluator *templates.Evaluator, opts RealtimeSessionOptions) {
 	cl := application.ModelConfigLoader()
 	cfg, err := cl.LoadModelConfigFileByNameDefaultOptions(model, application.ApplicationConfig())
 	if err != nil {
@@ -321,16 +359,72 @@ func runRealtimeSession(application *application.Application, t Transport, model
 		return
 	}
 
+	// LocalAI Assistant opt-in: gate on admin (same rule as chat.go's
+	// requireAssistantAccess) and grab the process-wide holder's executor.
+	// We collect tools + system prompt here and merge them into the session
+	// below so they're live from the first response.create.
+	var assistantTools []types.ToolUnion
+	var assistantSystemPrompt string
+	var assistantExecutor mcpTools.ToolExecutor
+	if opts.LocalAIAssistant {
+		if !opts.IsAdmin {
+			sendError(t, "forbidden", "localai_assistant requires admin", "", "")
+			return
+		}
+		appCfg := application.ApplicationConfig()
+		if appCfg != nil && appCfg.DisableLocalAIAssistant {
+			sendError(t, "unavailable", "LocalAI Assistant is disabled on this server", "", "")
+			return
+		}
+		holder := application.LocalAIAssistant()
+		if holder == nil || !holder.HasTools() {
+			sendError(t, "unavailable", "LocalAI Assistant is not available on this server", "", "")
+			return
+		}
+		exec := holder.Executor()
+		fns, discErr := exec.DiscoverTools(context.Background())
+		if discErr != nil {
+			xlog.Error("realtime: failed to discover LocalAI Assistant tools", "error", discErr)
+			sendError(t, "tool_discovery_failed", "failed to discover assistant tools: "+discErr.Error(), "", "")
+			return
+		}
+		assistantExecutor = exec
+		assistantSystemPrompt = holder.SystemPrompt()
+		assistantTools = make([]types.ToolUnion, 0, len(fns))
+		for _, fn := range fns {
+			fnCopy := fn
+			assistantTools = append(assistantTools, types.ToolUnion{
+				Function: &types.ToolFunction{
+					Name:        fnCopy.Name,
+					Description: fnCopy.Description,
+					Parameters:  fnCopy.Parameters,
+				},
+			})
+		}
+		xlog.Debug("realtime: LocalAI Assistant tools injected", "count", len(fns))
+	}
+
 	sttModel := cfg.Pipeline.Transcription
 
+	// Compose the system prompt: prepend the assistant prompt when we have
+	// one (it teaches the model the safety rules and tool recipes), then the
+	// session's default voice instructions. Order matches chat.go's
+	// hasSystemMessage check — assistant prompt comes first.
+	instructions := defaultInstructions
+	if assistantSystemPrompt != "" {
+		instructions = assistantSystemPrompt + "\n\n" + defaultInstructions
+	}
+
 	sessionID := generateSessionID()
 	session := &Session{
 		ID:                sessionID,
 		TranscriptionOnly: false,
 		Model:             model,
 		Voice:             cfg.TTSConfig.Voice,
-		Instructions:      defaultInstructions,
+		Instructions:      instructions,
 		ModelConfig:       cfg,
+		Tools:             assistantTools,
+		AssistantExecutor: assistantExecutor,
 		TurnDetection: &types.TurnDetectionUnion{
 			ServerVad: &types.ServerVad{
 				Threshold:         0.5,
@@ -1647,8 +1741,16 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation,
 		})
 	}
 
-	// Handle Tool Calls
+	// Handle Tool Calls. Two paths:
+	//   - LocalAI Assistant tools (session.AssistantExecutor.IsTool) run
+	//     server-side; we append both the call and its output to conv.Items
+	//     and re-trigger a follow-up response so the model can speak the
+	//     result. The client only sees observability events.
+	//   - All other tools follow the standard OpenAI flow: emit
+	//     function_call_arguments.done and wait for the client to send
+	//     conversation.item.create back.
 	xlog.Debug("About to handle tool calls", "finalToolCallsCount", len(finalToolCalls))
+	executedAssistantTool := false
 	for i, tc := range finalToolCalls {
 		toolCallID := generateItemID()
 		callID := "call_" + generateUniqueID() // OpenAI uses call_xyz
@@ -1680,6 +1782,34 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation,
 			Item:            fcItem,
 		})
 
+		serverSide := session.AssistantExecutor != nil && session.AssistantExecutor.IsTool(tc.Name)
+		if serverSide {
+			output, execErr := session.AssistantExecutor.ExecuteTool(ctx, tc.Name, tc.Arguments)
+			if execErr != nil {
+				output = "Error: " + execErr.Error()
+				xlog.Error("realtime: assistant tool execution failed", "tool", tc.Name, "error", execErr)
+			}
+			foItem := types.MessageItemUnion{
+				FunctionCallOutput: &types.MessageItemFunctionCallOutput{
+					ID:     generateItemID(),
+					CallID: callID,
+					Output: output,
+					Status: types.ItemStatusCompleted,
+				},
+			}
+			conv.Lock.Lock()
+			conv.Items = append(conv.Items, &foItem)
+			conv.Lock.Unlock()
+			sendEvent(t, types.ResponseOutputItemDoneEvent{
+				ServerEventBase: types.ServerEventBase{},
+				ResponseID:      responseID,
+				OutputIndex:     outputIndex,
+				Item:            fcItem,
+			})
+			executedAssistantTool = true
+			continue
+		}
+
 		sendEvent(t, types.ResponseFunctionCallArgumentsDeltaEvent{
 			ServerEventBase: types.ServerEventBase{},
 			ResponseID:      responseID,
@@ -1715,6 +1845,13 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation,
 			Status: types.ResponseStatusCompleted,
 		},
 	})
+
+	// If we executed any assistant tools inproc, run another response cycle
+	// so the model can speak the result. Mirrors the chat-side agentic loop
+	// but driven server-side rather than by client round-trip.
+	if executedAssistantTool {
+		triggerResponse(ctx, session, conv, t, nil)
+	}
 }
 
 // Helper functions to generate unique IDs

core/http/endpoints/openai/realtime_webrtc.go

@@ -15,6 +15,10 @@ import (
 type RealtimeCallRequest struct {
 	SDP   string `json:"sdp"`
 	Model string `json:"model"`
+	// LocalAIAssistant opts the session into the in-process admin tool
+	// surface (same modality as the chat page's "Manage Mode"). Admin-only;
+	// the realtime entry point gates it the same way the chat handler does.
+	LocalAIAssistant bool `json:"localai_assistant,omitempty"`
 }
 
 // RealtimeCallResponse is the JSON response for POST /v1/realtime/calls.
@@ -165,9 +169,13 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc {
 
 		// Start the realtime session in a goroutine
 		evaluator := application.TemplatesEvaluator()
+		opts := RealtimeSessionOptions{
+			LocalAIAssistant: req.LocalAIAssistant,
+			IsAdmin:          isCurrentUserAdmin(c, application),
+		}
 		go func() {
 			defer transport.Close()
-			runRealtimeSession(application, transport, req.Model, evaluator)
+			runRealtimeSession(application, transport, req.Model, evaluator, opts)
 		}()
 
 		return c.JSON(http.StatusCreated, RealtimeCallResponse{

core/http/react-ui/src/pages/Talk.jsx

@@ -5,6 +5,7 @@ import ModelSelector from '../components/ModelSelector'
 import ClientMCPDropdown from '../components/ClientMCPDropdown'
 import { useMCPClient } from '../hooks/useMCPClient'
 import { loadClientMCPServers } from '../utils/mcpClientStorage'
+import { useAuth } from '../context/AuthContext'
 
 const STATUS_STYLES = {
   disconnected: { icon: 'fa-solid fa-circle', color: 'var(--color-text-secondary)', bg: 'transparent' },
@@ -58,6 +59,12 @@ export default function Talk() {
     getConnectedTools,
   } = useMCPClient()
 
+  // LocalAI Assistant ("Manage Mode") — mirrors the chat-page toggle.
+  // Admin-only; the realtime endpoint enforces the gate too. When on, the
+  // backend mounts the in-process MCP admin tool surface for this session.
+  const { isAdmin } = useAuth()
+  const [manageMode, setManageMode] = useState(false)
+
   // Diagnostics
   const [diagVisible, setDiagVisible] = useState(false)
 
@@ -336,6 +343,7 @@ export default function Talk() {
       const data = await realtimeApi.call({
         sdp: pc.localDescription.sdp,
         model: selectedModel,
+        localai_assistant: manageMode,
       })
 
       await pc.setRemoteDescription({ type: 'answer', sdp: data.sdp })
@@ -344,7 +352,7 @@ export default function Talk() {
       updateStatus('error', 'Connection failed: ' + err.message)
       disconnect()
     }
-  }, [selectedModel, diagVisible, handleServerEvent, updateStatus, addToast])
+  }, [selectedModel, manageMode, diagVisible, handleServerEvent, updateStatus, addToast])
 
   // ── Disconnect ──
   const disconnect = useCallback(() => {
@@ -620,6 +628,26 @@ export default function Talk() {
               connectionStatuses={connectionStatuses}
               getConnectedTools={getConnectedTools}
             />
+            {isAdmin && (
+              <label style={{
+                display: 'flex', alignItems: 'center', gap: 'var(--spacing-xs)',
+                marginTop: 'var(--spacing-xs)', fontSize: '0.8125rem',
+                cursor: isConnected ? 'not-allowed' : 'pointer',
+                color: isConnected ? 'var(--color-text-secondary)' : 'var(--color-text)',
+              }}>
+                <input
+                  type="checkbox"
+                  checked={manageMode}
+                  disabled={isConnected}
+                  onChange={(e) => setManageMode(e.target.checked)}
+                />
+                <i className="fas fa-user-shield" style={{ color: 'var(--color-primary)' }} />
+                Manage Mode
+                <span style={{ color: 'var(--color-text-secondary)', fontSize: '0.75rem' }}>
+                  — let the model query LocalAI (models, backends, system info)
+                </span>
+              </label>
+            )}
           </div>
 
           {/* Pipeline details */}