feat(realtime): splice filled slot values into classifier replies

A classifier option's spoken reply can now reference its tool's argument
slots ("Going forward {{distance}} {{units}}."): the values inferred by
the slot-fill completion — or the recovery defaults — are spliced into
the reply as plain text before it is emitted, so what the assistant says
confirms what it actually inferred. Placeholders without a value stay
literal, and options without slots are untouched.

FillToolArguments now returns the raw slot values alongside the spliced
arguments JSON to make the reply templating possible.

Assisted-by: Claude:claude-fable-5
Signed-off-by: Richard Palethorpe <io@richiejp.com>

Author: richiejp

core/config/meta/registry.go

@@ -647,7 +647,7 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 		"pipeline.classifier.options": {
 			Section:     "pipeline",
 			Label:       "Classifier Options",
-			Description: "The intents the classifier scores each turn against. Each option has an id (also the scored route label — keep it short), a description of when it applies, an optional canned spoken reply, and an optional canned tool call {name, arguments}. A tool may also declare slots ([{name, type: number|enum|string, values, default, hint}]) whose \"{{name}}\" placeholders in arguments are filled by a short grammar-constrained completion when the option wins — the hybrid between prefill-only classification and full generation (requires completion in the scoring model's known_usecases). Clients can replace the list per session via session.update localai_classifier.",
+			Description: "The intents the classifier scores each turn against. Each option has an id (also the scored route label — keep it short), a description of when it applies, an optional canned spoken reply, and an optional canned tool call {name, arguments}. A tool may also declare slots ([{name, type: number|enum|string, values, default, hint}]) whose \"{{name}}\" placeholders in arguments (and, optionally, the reply) are filled by a short grammar-constrained completion when the option wins — the hybrid between prefill-only classification and full generation (requires completion in the scoring model's known_usecases). Clients can replace the list per session via session.update localai_classifier.",
 			Component:   "json-editor",
 			Order:       92,
 		},

core/http/endpoints/openai/realtime.go

@@ -283,9 +283,10 @@ type Model interface {
 	// FillToolArguments completes the chosen option's argument slots with a
 	// short grammar-constrained completion that continues the exact scoring
 	// prompt (so the backend's prompt cache stays warm) and returns the
-	// spliced tool-arguments JSON — the hybrid between prefill-only
-	// classification and full generation.
-	FillToolArguments(ctx context.Context, messages schema.Messages, options []types.ClassifierOption, normalization string, chosen *types.ClassifierOption) (string, error)
+	// spliced tool-arguments JSON plus the raw slot values (for reply
+	// templating) — the hybrid between prefill-only classification and full
+	// generation.
+	FillToolArguments(ctx context.Context, messages schema.Messages, options []types.ClassifierOption, normalization string, chosen *types.ClassifierOption) (string, map[string]string, error)
 	PredictConfig() *config.ModelConfig
 	// Warmup eagerly loads the pipeline's sub-model backends into memory so the
 	// first realtime turn doesn't pay each backend's cold-start load cost. Loads

core/http/endpoints/openai/realtime_classifier.go

@@ -306,10 +306,11 @@ func classifierRespond(ctx context.Context, session *Session, conv *Conversation
 	// (error and no complete default set) is handled like a scoring
 	// failure.
 	filledArgs := ""
+	var fillValues map[string]string
 	var fillLatency time.Duration
 	if chosen != nil {
 		var ferr error
-		filledArgs, fillLatency, ferr = fillChosenArguments(ctx, session, cc, msgs, chosen)
+		filledArgs, fillValues, fillLatency, ferr = fillChosenArguments(ctx, session, cc, msgs, chosen)
 		if ferr != nil {
 			if cc.FallbackMode() == types.ClassifierFallbackGenerate {
 				xlog.Warn("realtime classifier: slot fill failed; falling back to generation", "error", ferr)
@@ -363,7 +364,10 @@ func classifierRespond(ctx context.Context, session *Session, conv *Conversation
 	var toolCalls []functions.FuncCallResults
 	switch {
 	case chosen != nil:
-		reply = chosen.Reply
+		// The reply may template the filled slot values ("Going forward
+		// {{distance}} {{units}}.") so what is spoken confirms what was
+		// actually inferred.
+		reply = chosen.SpliceReply(fillValues)
 		if chosen.Tool != nil {
 			toolCalls = []functions.FuncCallResults{{Name: chosen.Tool.Name, Arguments: filledArgs}}
 		}
@@ -476,34 +480,35 @@ func parseSlotValues(chosenID, firstSlot, generated string, slots []types.Classi
 
 // fillChosenArguments resolves a winning option's tool arguments: canned
 // options pass through, slotted options run the fill completion with a
-// default-value recovery when inference fails. The error return is reserved
-// for unrecoverable failures (no complete default set).
-func fillChosenArguments(ctx context.Context, session *Session, cc *types.ClassifierConfig, msgs schema.Messages, chosen *types.ClassifierOption) (args string, latency time.Duration, err error) {
+// default-value recovery when inference fails. The slot values ride along
+// so the caller can splice them into the spoken reply too. The error return
+// is reserved for unrecoverable failures (no complete default set).
+func fillChosenArguments(ctx context.Context, session *Session, cc *types.ClassifierConfig, msgs schema.Messages, chosen *types.ClassifierOption) (args string, values map[string]string, latency time.Duration, err error) {
 	if chosen.Tool == nil {
-		return "", 0, nil
+		return "", nil, 0, nil
 	}
 	if len(chosen.Tool.Slots) == 0 {
 		if len(chosen.Tool.Arguments) > 0 {
-			return string(chosen.Tool.Arguments), 0, nil
+			return string(chosen.Tool.Arguments), nil, 0, nil
 		}
-		return "{}", 0, nil
+		return "{}", nil, 0, nil
 	}
 	start := time.Now()
-	args, err = session.ModelInterface.FillToolArguments(ctx, msgs, cc.Options, cc.Normalization, chosen)
+	args, values, err = session.ModelInterface.FillToolArguments(ctx, msgs, cc.Options, cc.Normalization, chosen)
 	latency = time.Since(start)
 	if err == nil {
-		return args, latency, nil
+		return args, values, latency, nil
 	}
 	xlog.Warn("realtime classifier: slot fill failed; trying slot defaults", "option", chosen.ID, "error", err)
 	defaults, derr := chosen.Tool.SlotDefaults()
 	if derr != nil {
-		return "", latency, err
+		return "", nil, latency, err
 	}
 	args, derr = chosen.Tool.SpliceArguments(defaults)
 	if derr != nil {
-		return "", latency, err
+		return "", nil, latency, err
 	}
-	return args, latency, nil
+	return args, defaults, latency, nil
 }
 
 // classifierPolicyDescription renders an option's scoring description,

core/http/endpoints/openai/realtime_classifier_test.go

@@ -52,6 +52,18 @@ func classifierResultEvents(t *fakeTransport) []types.ClassifierResultEvent {
 	return out
 }
 
+// replyTexts collects the assistant reply text of every completed output
+// item — what a classifier response actually "spoke".
+func replyTexts(t *fakeTransport) []string {
+	var out []string
+	for _, e := range t.events {
+		if ev, ok := e.(types.ResponseOutputTextDoneEvent); ok {
+			out = append(out, ev.Text)
+		}
+	}
+	return out
+}
+
 var _ = Describe("classifierConfigFromPipeline", func() {
 	It("returns nil for an absent block", func() {
 		cc, err := classifierConfigFromPipeline(nil)
@@ -503,7 +515,7 @@ func slottedTestConfig(threshold float64, fallback *types.ClassifierFallback, de
 			{
 				ID:          "up",
 				Description: "the user asks the drone to fly up",
-				Reply:       "Going up.",
+				Reply:       "Going up {{distance}} {{units}}.",
 				Tool: &types.ClassifierTool{
 					Name:      "move",
 					Arguments: json.RawMessage(`{"direction":"up","distance":"{{distance}}","units":"{{units}}"}`),
@@ -579,7 +591,8 @@ var _ = Describe("classifierRespond slot filling", func() {
 	It("emits the filled tool arguments and reports them in the result event", func() {
 		m := &fakeModel{
 			classifyScores: []router.LabelScore{{Label: "up", Score: 0.9}, {Label: "greeting", Score: 0.1}},
-			fillArgs:       `{"direction":"up","distance":3,"units":"m"}`,
+			fillArgs:       `{"direction":"up","distance":3,"units":"meters"}`,
+			fillValues:     map[string]string{"distance": "3", "units": "meters"},
 		}
 		session := classifierTestSession(m)
 		conv := &Conversation{}
@@ -595,15 +608,32 @@ var _ = Describe("classifierRespond slot filling", func() {
 		results := classifierResultEvents(t)
 		Expect(results).To(HaveLen(1))
 		Expect(results[0].ChosenID).To(Equal("up"))
-		Expect(results[0].Arguments).To(MatchJSON(`{"direction":"up","distance":3,"units":"m"}`))
+		Expect(results[0].Arguments).To(MatchJSON(`{"direction":"up","distance":3,"units":"meters"}`))
 
 		var fcArgs string
 		for _, e := range t.events {
 			if done, ok := e.(types.ResponseFunctionCallArgumentsDoneEvent); ok {
 				fcArgs = done.Arguments
 			}
 		}
-		Expect(fcArgs).To(MatchJSON(`{"direction":"up","distance":3,"units":"m"}`))
+		Expect(fcArgs).To(MatchJSON(`{"direction":"up","distance":3,"units":"meters"}`))
+	})
+
+	It("splices the filled values into a templated reply", func() {
+		m := &fakeModel{
+			classifyScores: []router.LabelScore{{Label: "up", Score: 0.9}, {Label: "greeting", Score: 0.1}},
+			fillArgs:       `{"direction":"up","distance":3,"units":"meters"}`,
+			fillValues:     map[string]string{"distance": "3", "units": "meters"},
+		}
+		session := classifierTestSession(m)
+		conv := &Conversation{}
+		t := &fakeTransport{}
+		r := &liveResponse{id: "resp-slot-reply"}
+
+		handled := classifierRespond(context.Background(), session, conv, t, r, slottedTestConfig(0.35, nil, false), classifierTestHistory, nil, 0)
+
+		Expect(handled).To(BeTrue())
+		Expect(replyTexts(t)).To(ConsistOf("Going up 3 meters."))
 	})
 
 	It("recovers with slot defaults when filling fails", func() {
@@ -626,6 +656,7 @@ var _ = Describe("classifierRespond slot filling", func() {
 			}
 		}
 		Expect(fcArgs).To(MatchJSON(`{"direction":"up","distance":1,"units":"m"}`))
+		Expect(replyTexts(t)).To(ConsistOf("Going up 1 m."), "the default-recovery reply confirms the defaults")
 	})
 
 	It("fails the response when filling fails and a slot has no default", func() {

core/http/endpoints/openai/realtime_doubles_test.go

@@ -109,10 +109,12 @@ type fakeModel struct {
 	classifyCalls       int
 	lastClassifyOptions []types.ClassifierOption
 
-	// FillToolArguments scripting: fillArgs is returned verbatim; fillErr
-	// fails the call. fillCalls counts invocations and lastFillChosen
-	// records which option's slots the handler asked to fill.
+	// FillToolArguments scripting: fillArgs/fillValues are returned
+	// verbatim; fillErr fails the call. fillCalls counts invocations and
+	// lastFillChosen records which option's slots the handler asked to
+	// fill.
 	fillArgs       string
+	fillValues     map[string]string
 	fillErr        error
 	fillCalls      int
 	lastFillChosen *types.ClassifierOption
@@ -127,13 +129,13 @@ type fakeModel struct {
 	lastMessages schema.Messages
 }
 
-func (m *fakeModel) FillToolArguments(_ context.Context, msgs schema.Messages, options []types.ClassifierOption, _ string, chosen *types.ClassifierOption) (string, error) {
+func (m *fakeModel) FillToolArguments(_ context.Context, msgs schema.Messages, options []types.ClassifierOption, _ string, chosen *types.ClassifierOption) (string, map[string]string, error) {
 	m.fillCalls++
 	m.lastFillChosen = chosen
 	if m.fillErr != nil {
-		return "", m.fillErr
+		return "", nil, m.fillErr
 	}
-	return m.fillArgs, nil
+	return m.fillArgs, m.fillValues, nil
 }
 
 func (m *fakeModel) ClassifyTurn(_ context.Context, msgs schema.Messages, options []types.ClassifierOption, _ string) ([]router.LabelScore, error) {

core/http/endpoints/openai/realtime_model.go

@@ -110,8 +110,8 @@ func (m *transcriptOnlyModel) ClassifyTurn(ctx context.Context, messages schema.
 	return nil, fmt.Errorf("classifier mode not supported in transcript-only mode")
 }
 
-func (m *transcriptOnlyModel) FillToolArguments(ctx context.Context, messages schema.Messages, options []types.ClassifierOption, normalization string, chosen *types.ClassifierOption) (string, error) {
-	return "", fmt.Errorf("classifier mode not supported in transcript-only mode")
+func (m *transcriptOnlyModel) FillToolArguments(ctx context.Context, messages schema.Messages, options []types.ClassifierOption, normalization string, chosen *types.ClassifierOption) (string, map[string]string, error) {
+	return "", nil, fmt.Errorf("classifier mode not supported in transcript-only mode")
 }
 
 func (m *transcriptOnlyModel) TTS(ctx context.Context, text, voice, language string) (string, *proto.Result, error) {
@@ -499,18 +499,18 @@ func (m *wrappedModel) ClassifyTurn(ctx context.Context, messages schema.Message
 // re-opened at its first slot, with a grammar pinning everything but the
 // slot values. Deterministic (temperature 0), a couple dozen tokens at
 // most.
-func (m *wrappedModel) FillToolArguments(ctx context.Context, messages schema.Messages, options []types.ClassifierOption, normalization string, chosen *types.ClassifierOption) (string, error) {
+func (m *wrappedModel) FillToolArguments(ctx context.Context, messages schema.Messages, options []types.ClassifierOption, normalization string, chosen *types.ClassifierOption) (string, map[string]string, error) {
 	if chosen == nil || chosen.Tool == nil || len(chosen.Tool.Slots) == 0 {
-		return "", fmt.Errorf("classifier: option has no slots to fill")
+		return "", nil, fmt.Errorf("classifier: option has no slots to fill")
 	}
 	slots := chosen.Tool.Slots
 	classifier, err := m.classifierFor(options, normalization)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
 	prompt, err := classifier.SlotFillPrompt(classifierProbe(messages), chosen.ID, slots[0].Name)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
 
 	// The scoring config, narrowed to a deterministic constrained
@@ -519,7 +519,7 @@ func (m *wrappedModel) FillToolArguments(ctx context.Context, messages schema.Me
 	// score].
 	cfg := *m.scoreConfig()
 	if !cfg.HasUsecases(config.FLAG_COMPLETION) {
-		return "", fmt.Errorf("classifier: slot filling requires completion in the scoring model's known_usecases")
+		return "", nil, fmt.Errorf("classifier: slot filling requires completion in the scoring model's known_usecases")
 	}
 	cfg.Grammar = slotFillGrammar(slots)
 	maxTokens := 16 + 16*len(slots)
@@ -529,17 +529,21 @@ func (m *wrappedModel) FillToolArguments(ctx context.Context, messages schema.Me
 
 	fn, err := backend.ModelInference(ctx, prompt, nil, nil, nil, nil, m.modelLoader, &cfg, m.confLoader, m.appConfig, nil, "", "", nil, nil, nil, nil)
 	if err != nil {
-		return "", fmt.Errorf("classifier: slot fill inference: %w", err)
+		return "", nil, fmt.Errorf("classifier: slot fill inference: %w", err)
 	}
 	resp, err := fn()
 	if err != nil {
-		return "", fmt.Errorf("classifier: slot fill inference: %w", err)
+		return "", nil, fmt.Errorf("classifier: slot fill inference: %w", err)
 	}
 	values, err := parseSlotValues(chosen.ID, slots[0].Name, resp.Response, slots)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
-	return chosen.Tool.SpliceArguments(values)
+	args, err := chosen.Tool.SpliceArguments(values)
+	if err != nil {
+		return "", nil, err
+	}
+	return args, values, nil
 }
 
 func (m *wrappedModel) Warmup(ctx context.Context) error {

core/http/endpoints/openai/types/classifier.go

@@ -212,6 +212,26 @@ func (t *ClassifierTool) SpliceArguments(values map[string]string) (string, erro
 	return args, nil
 }
 
+// SpliceReply fills "{{name}}" placeholders in the option's spoken reply
+// with the same slot values that filled the tool arguments, as plain text
+// ("Going {{distance}} {{units}}." → "Going 3 meters."), so the reply can
+// confirm what was actually inferred. Values are optional in the reply:
+// placeholders without a value stay literal, and options without slots (or
+// a nil value set) return the reply verbatim.
+func (o *ClassifierOption) SpliceReply(values map[string]string) string {
+	reply := o.Reply
+	if o.Tool == nil || len(values) == 0 {
+		return reply
+	}
+	for i := range o.Tool.Slots {
+		s := &o.Tool.Slots[i]
+		if v, ok := values[s.Name]; ok && v != "" {
+			reply = strings.ReplaceAll(reply, slotPlaceholder(s.Name), v)
+		}
+	}
+	return reply
+}
+
 // SlotDefaults returns every slot's default value, or an error naming the
 // first slot without one — the fill-failure path either recovers with a
 // complete default set or not at all.

core/http/endpoints/openai/types/classifier_test.go

@@ -267,4 +267,27 @@ var _ = Describe("ClassifierTool slots", func() {
 			Expect(err).To(MatchError(ContainSubstring(`"distance"`)))
 		})
 	})
+
+	Describe("SpliceReply", func() {
+		option := func(reply string, t *types.ClassifierTool) *types.ClassifierOption {
+			return &types.ClassifierOption{ID: "up", Description: "d", Reply: reply, Tool: t}
+		}
+
+		It("substitutes slot values as plain text", func() {
+			o := option("Going up {{distance}} {{units}}.", tool(numberSlot, enumSlot))
+			Expect(o.SpliceReply(map[string]string{"distance": "3.5", "units": "m"})).To(Equal("Going up 3.5 m."))
+		})
+
+		It("leaves placeholders without a value literal", func() {
+			o := option("Going up {{distance}} {{units}}.", tool(numberSlot, enumSlot))
+			Expect(o.SpliceReply(map[string]string{"distance": "3"})).To(Equal("Going up 3 {{units}}."))
+		})
+
+		It("returns the reply verbatim without slots or values", func() {
+			o := option("Going up {{distance}}.", nil)
+			Expect(o.SpliceReply(map[string]string{"distance": "3"})).To(Equal("Going up {{distance}}."))
+			slotted := option("Going up {{distance}}.", tool(numberSlot))
+			Expect(slotted.SpliceReply(nil)).To(Equal("Going up {{distance}}."))
+		})
+	})
 })

docs/content/features/openai-realtime.md

@@ -242,7 +242,9 @@ tool:
       hint: assume m when the user gives no units
 ```
 
-Slot declarations (and hints) are appended to the option's description in the shared system prompt, so they also inform scoring and cost no extra per-turn tokens. The `localai.classifier.result` event carries the final `arguments` and a `fill_latency_ms`. On an inference failure the slots' defaults apply; if any slot lacks a default the response fails (or falls through to generation with `fallback.mode: generate`). Slot filling requires `completion` in the scoring model's `known_usecases` alongside `score`.
+The option's spoken `reply` can reference the same placeholders — `reply: "Going forward {{distance}} {{units}}."` — and the filled values are spliced in as plain text before the reply is emitted, so what the assistant says confirms what it inferred. Reply placeholders are optional (one that names no slot stays literal).
+
+Slot declarations (and hints) are appended to the option's description in the shared system prompt, so they also inform scoring and cost no extra per-turn tokens. The `localai.classifier.result` event carries the final `arguments` and a `fill_latency_ms`. On an inference failure the slots' defaults apply (and template the reply); if any slot lacks a default the response fails (or falls through to generation with `fallback.mode: generate`). Slot filling requires `completion` in the scoring model's `known_usecases` alongside `score`.
 
 The concrete scoring model must declare `score` in `known_usecases`. A single llama.cpp model can serve ordinary inference and classification concurrently by declaring multiple use cases, for example `known_usecases: [chat, completion, score]`; LocalAI reserves the scoring slots only when `score` is present. Scoring also requires the unified KV cache, which is enabled by default, so a score-enabled model cannot set `kv_unified:false`.