fix(realtime): gate scoring capacity by model usecase

richiejp · richiejp · commit 625f9a916edf · 2026-07-20T14:07:10.000+01:00
Reserve llama.cpp scoring slots only for models that explicitly declare the score usecase, while allowing score to coexist with chat and completion. Reject incompatible unified-KV settings and classifier activation on models without scoring capacity.

Propagate application defaults when resolving realtime and preload pipeline stages so unset thread counts are resolved consistently without overriding explicit model settings.

Assisted-by: Codex:gpt-5
Signed-off-by: Richard Palethorpe &lt;io@richiejp.com&gt;
diff --git a/backend/backend.proto b/backend/backend.proto
@@ -448,6 +448,11 @@ message ModelOptions {
   // Proxy carries the cloud-proxy backend's per-model configuration.
   // Empty for non-proxy backends.
   ProxyOptions Proxy = 74;
+
+  // EnableScore reserves backend resources for the Score RPC. It is derived
+  // from the model's explicit `known_usecases: [score]` declaration so models
+  // that never score retain their ordinary serving footprint.
+  bool EnableScore = 75;
 }
 
 // ProxyOptions configures the cloud-proxy backend. UpstreamURL and
diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp
@@ -1336,7 +1336,8 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
     // KV cache — with per-sequence streams the extra ids would shrink every
     // sequence's context to n_ctx / n_seq_max. Decided after both option
     // passes so an explicit kv_unified:false wins and disables forking.
-    params.n_seq_score_forks = params.kv_unified ? SERVER_SCORE_FORK_SEQS : 0;
+    params.score_enabled = request->enablescore();
+    params.n_seq_score_forks = params.score_enabled && params.kv_unified ? SERVER_SCORE_FORK_SEQS : 0;
 
     // Terminate/pad the override vectors only after BOTH the named-option loop
     // and the generic passthrough (common_params_parse above) have pushed their
@@ -1394,6 +1395,14 @@ class BackendServiceImpl final : public backend::Backend::Service {
         common_params params;
         params_parse(ctx_server, request, params);
 
+        if (params.score_enabled && !params.kv_unified) {
+            const std::string error_msg =
+                "Score requires the unified KV cache; remove kv_unified:false or remove score from known_usecases";
+            result->set_message(error_msg);
+            result->set_success(false);
+            return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, error_msg);
+        }
+
         common_init();
         // Ensure debug logs are enabled after common_init() sets up logging
         common_log_set_verbosity_thold(params.verbosity);
@@ -2889,6 +2898,10 @@ class BackendServiceImpl final : public backend::Backend::Service {
         if (params_base.model.path.empty()) {
             return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, "Model not loaded");
         }
+        if (!params_base.score_enabled) {
+            return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION,
+                "Score was not enabled when the model was loaded; add score to known_usecases");
+        }
         if (request->candidates_size() == 0) {
             return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "candidates must be non-empty");
         }
diff --git a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch
@@ -17,25 +17,31 @@ diff --git a/common/common.h b/common/common.h
 index 1535317..8cdd664 100644
 --- a/common/common.h
 +++ b/common/common.h
-@@ -454,6 +454,7 @@ struct common_params {
+@@ -454,6 +454,8 @@ struct common_params {
      int32_t n_keep                =     0; // number of tokens to keep from initial prompt
      int32_t n_chunks              =    -1; // max number of chunks to process (-1 = unlimited)
      int32_t n_parallel            =     1; // number of parallel sequences to decode
 +    int32_t n_seq_score_forks     =     0; // extra seq ids beyond n_parallel, reserved for server score-task forks
++    bool    score_enabled          = false; // reserve server resources for the Score task type
      int32_t n_sequences           =     1; // number of sequences to decode
      int32_t n_outputs_max         =     0; // max outputs in a batch (0 = n_batch)
      int32_t grp_attn_n            =     1; // group-attention factor
 diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
 index 98d0cca..ac9bcac 100644
 --- a/tools/server/server-context.cpp
 +++ b/tools/server/server-context.cpp
-@@ -49,7 +49,11 @@ static uint32_t server_n_outputs_max(const common_params & params) {
+@@ -49,7 +49,16 @@ static uint32_t server_n_outputs_max(const common_params & params) {
  
      const uint32_t n_outputs_per_seq = 1 + common_speculative_n_max(&params.speculative);
  
 -    const uint64_t n_outputs = (uint64_t) params.n_parallel * n_outputs_per_seq;
 +    // score tasks (SERVER_TASK_TYPE_SCORE) output logits for every candidate
 +    // token, so reserve room for a bounded candidate tail per parallel slot
++    if (!params.score_enabled) {
++        return std::max<uint32_t>(1, std::min<uint64_t>(n_batch,
++            (uint64_t) params.n_parallel * n_outputs_per_seq));
++    }
++
 +    const uint32_t n_outputs_score_seq = 1 + SERVER_SCORE_MAX_CAND_TOKENS;
 +
 +    const uint64_t n_outputs = (uint64_t) params.n_parallel * std::max(n_outputs_per_seq, n_outputs_score_seq);
diff --git a/core/backend/options.go b/core/backend/options.go
@@ -422,6 +422,7 @@ func grpcModelOpts(c config.ModelConfig, modelPath string) *pb.ModelOptions {
 		Options:              withCompanionArtifactOptions(c.Options, c.Artifacts),
 		Overrides:            c.Overrides,
 		EngineArgs:           engineArgsJSON,
+		EnableScore:          c.HasUsecases(config.FLAG_SCORE),
 		CLIPSkip:             int32(c.Diffusers.ClipSkip),
 		ControlNet:           c.Diffusers.ControlNet,
 		ContextSize:          int32(ctxSize),
diff --git a/core/backend/options_internal_test.go b/core/backend/options_internal_test.go
@@ -120,6 +120,7 @@ var _ = Describe("grpcModelOpts NBatch", func() {
 		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &ctx}}
 		opts := grpcModelOpts(cfg, "/tmp/models")
 		Expect(opts.NBatch).To(BeEquivalentTo(512))
+		Expect(opts.EnableScore).To(BeFalse())
 	})
 
 	It("sizes the batch to the context window for score models", func() {
@@ -128,6 +129,14 @@ var _ = Describe("grpcModelOpts NBatch", func() {
 		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &ctx}, KnownUsecases: &scoreUsecase}
 		opts := grpcModelOpts(cfg, "/tmp/models")
 		Expect(opts.NBatch).To(BeEquivalentTo(4096))
+		Expect(opts.EnableScore).To(BeTrue())
+	})
+
+	It("enables score resources for a model with multiple usecases", func() {
+		usecases := config.FLAG_CHAT | config.FLAG_SCORE
+		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &ctx}, KnownUsecases: &usecases}
+		opts := grpcModelOpts(cfg, "/tmp/models")
+		Expect(opts.EnableScore).To(BeTrue())
 	})
 
 	It("keeps an explicit batch over the score default", func() {
diff --git a/core/backend/preload.go b/core/backend/preload.go
@@ -28,7 +28,7 @@ func PreloadModelByName(ctx context.Context, cl *config.ModelConfigLoader, ml *m
 		return nil, err
 	}
 
-	stages, err := pipelineStages(cl, &cfg.Pipeline, ml.ModelPath)
+	stages, err := pipelineStages(cl, &cfg.Pipeline, ml.ModelPath, appConfig.ToConfigLoaderOptions()...)
 	if err != nil {
 		return nil, err
 	}
@@ -59,7 +59,7 @@ var loadStage = PreloadModel
 // pipeline itself uses. A stage that fails to resolve is a misconfiguration,
 // so it fails fast rather than being deferred to load. A pipeline with no
 // stages set returns nil, which callers treat as "not a pipeline".
-func pipelineStages(cl *config.ModelConfigLoader, p *config.Pipeline, modelPath string) ([]PreloadStage, error) {
+func pipelineStages(cl *config.ModelConfigLoader, p *config.Pipeline, modelPath string, opts ...config.ConfigLoaderOption) ([]PreloadStage, error) {
 	voiceRec := ""
 	if p.VoiceRecognition != nil {
 		voiceRec = p.VoiceRecognition.Model
@@ -76,7 +76,7 @@ func pipelineStages(cl *config.ModelConfigLoader, p *config.Pipeline, modelPath
 		if s.name == "" {
 			continue
 		}
-		cfg, err := cl.LoadResolvedModelConfig(s.name, modelPath)
+		cfg, err := cl.LoadResolvedModelConfig(s.name, modelPath, opts...)
 		if err != nil {
 			return nil, fmt.Errorf("%s (%s): %w", s.role, s.name, err)
 		}
diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go
@@ -30,6 +30,7 @@ const (
 	UsecaseFaceRecognition     = "face_recognition"
 	UsecaseSpeakerRecognition  = "speaker_recognition"
 	UsecaseTokenClassify       = "token_classify"
+	UsecaseScore               = "score"
 )
 
 // GRPCMethod identifies a Backend service RPC from backend.proto.
@@ -60,6 +61,7 @@ const (
 	MethodVoiceEmbed         GRPCMethod = "VoiceEmbed"
 	MethodVoiceAnalyze       GRPCMethod = "VoiceAnalyze"
 	MethodTokenClassify      GRPCMethod = "TokenClassify"
+	MethodScore              GRPCMethod = "Score"
 )
 
 // UsecaseInfo describes a single known_usecase value and how it maps
@@ -192,6 +194,11 @@ var UsecaseInfoMap = map[string]UsecaseInfo{
 		GRPCMethod:  MethodTokenClassify,
 		Description: "Per-token classification (NER) via the TokenClassify RPC — the PII detector tier. Declared explicitly via known_usecases; never auto-guessed, since the token-classification head is not useful as general generation or embeddings.",
 	},
+	UsecaseScore: {
+		Flag:        FLAG_SCORE,
+		GRPCMethod:  MethodScore,
+		Description: "Joint log-probability scoring of candidate continuations via the Score RPC. Declared explicitly via known_usecases and usable alongside generation usecases.",
+	},
 }
 
 // BackendCapability describes which gRPC methods and usecases a backend supports.
@@ -241,8 +248,8 @@ func referenceVoiceCloning() *VoiceCloningCapability {
 var BackendCapabilities = map[string]BackendCapability{
 	// --- LLM / text generation backends ---
 	"llama-cpp": {
-		GRPCMethods:      []GRPCMethod{MethodPredict, MethodPredictStream, MethodEmbedding, MethodTokenizeString},
-		PossibleUsecases: []string{UsecaseChat, UsecaseCompletion, UsecaseEdit, UsecaseEmbeddings, UsecaseTokenize, UsecaseVision},
+		GRPCMethods:      []GRPCMethod{MethodPredict, MethodPredictStream, MethodEmbedding, MethodTokenizeString, MethodScore},
+		PossibleUsecases: []string{UsecaseChat, UsecaseCompletion, UsecaseEdit, UsecaseEmbeddings, UsecaseTokenize, UsecaseVision, UsecaseScore},
 		DefaultUsecases:  []string{UsecaseChat},
 		AcceptsImages:    true, // requires mmproj
 		Description:      "llama.cpp GGUF models — LLM inference with optional vision via mmproj",
diff --git a/core/config/model_config_loader.go b/core/config/model_config_loader.go
@@ -201,8 +201,8 @@ func (bcl *ModelConfigLoader) LoadModelConfigFileByNameDefaultOptions(modelName
 // survives unresolved into model loading and fails downstream — notably in
 // distributed mode with "backend name is empty". Mirrors the top-level alias
 // resolution in core/http/middleware/request.go.
-func (bcl *ModelConfigLoader) LoadResolvedModelConfig(modelName, modelPath string) (*ModelConfig, error) {
-	cfg, err := bcl.LoadModelConfigFileByName(modelName, modelPath)
+func (bcl *ModelConfigLoader) LoadResolvedModelConfig(modelName, modelPath string, opts ...ConfigLoaderOption) (*ModelConfig, error) {
+	cfg, err := bcl.LoadModelConfigFileByName(modelName, modelPath, opts...)
 	if err != nil {
 		return nil, err
 	}
diff --git a/core/config/model_config_loader_resolve_test.go b/core/config/model_config_loader_resolve_test.go
@@ -49,4 +49,21 @@ alias: real-llm
 		Expect(direct.Backend).To(Equal("llama-cpp"))
 		Expect(direct.Name).To(Equal("real-llm"))
 	})
+
+	It("applies loader defaults while preserving explicit model threads", func() {
+		tmpDir := GinkgoT().TempDir()
+		Expect(os.WriteFile(filepath.Join(tmpDir, "defaulted.yaml"), []byte("name: defaulted\nbackend: llama-cpp\n"), 0644)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(tmpDir, "explicit.yaml"), []byte("name: explicit\nbackend: llama-cpp\nthreads: 3\n"), 0644)).To(Succeed())
+
+		cl := config.NewModelConfigLoader(tmpDir)
+		defaulted, err := cl.LoadResolvedModelConfig("defaulted", tmpDir, config.LoadOptionThreads(11))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(defaulted.Threads).NotTo(BeNil())
+		Expect(*defaulted.Threads).To(Equal(11))
+
+		explicit, err := cl.LoadResolvedModelConfig("explicit", tmpDir, config.LoadOptionThreads(11))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(explicit.Threads).NotTo(BeNil())
+		Expect(*explicit.Threads).To(Equal(3))
+	})
 })
diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go
@@ -975,6 +975,10 @@ func runRealtimeSession(application *application.Application, t Transport, model
 
 		case types.ResponseCreateEvent:
 			xlog.Debug("recv", "message", string(msg))
+			if err := validateClassifierActivation(session.ModelInterface, e.Response.LocalAIClassifier); err != nil {
+				sendError(t, "invalid_request_error", "Invalid response classifier: "+err.Error(), "", e.EventID)
+				continue
+			}
 
 			// Handle optional items to add to context
 			if len(e.Response.Input) > 0 {
@@ -1254,7 +1258,7 @@ func updateSession(session *Session, update *types.SessionUnion, cl *config.Mode
 		// Replace-not-merge, like tools: the client owns the whole option
 		// list. Invalid configs reject the update without touching the
 		// session's current classifier.
-		if err := rt.LocalAIClassifier.Validate(); err != nil {
+		if err := validateClassifierActivation(session.ModelInterface, rt.LocalAIClassifier); err != nil {
 			return err
 		}
 		session.Classifier = rt.LocalAIClassifier
diff --git a/core/http/endpoints/openai/realtime_classifier.go b/core/http/endpoints/openai/realtime_classifier.go
@@ -85,6 +85,39 @@ func resolveClassifier(sessionCfg *types.ClassifierConfig, overrides *types.Resp
 	return sessionCfg
 }
 
+// validateClassifierActivation verifies both the wire config and the concrete
+// backend selected to score it. Scoring capacity is reserved at model load
+// only for configs that explicitly declare the score usecase, so accepting an
+// active classifier on any other model would defer a deterministic failure to
+// the first response.
+func validateClassifierActivation(m Model, cc *types.ClassifierConfig) error {
+	if cc == nil {
+		return nil
+	}
+	if err := cc.Validate(); err != nil {
+		return err
+	}
+	if !cc.Active() {
+		return nil
+	}
+	wm, ok := m.(*wrappedModel)
+	if !ok {
+		return fmt.Errorf("classifier: the session model does not support scoring")
+	}
+	cfg := wm.scoreConfig()
+	if cfg == nil || !cfg.HasUsecases(config.FLAG_SCORE) {
+		name := ""
+		if cfg != nil {
+			name = cfg.Name
+		}
+		return fmt.Errorf("classifier: scoring model %q must declare known_usecases: [score]", name)
+	}
+	if cfg.HasRouter() {
+		return fmt.Errorf("classifier: scoring model %q is a router; configure a concrete pipeline.classifier.model", cfg.Name)
+	}
+	return nil
+}
+
 // trimClassifierHistory drops system messages (the classifier builds its
 // own option-list system prompt) and selects what gets scored.
 // historyItems <= 0 (the default): only the latest user message. Positive
diff --git a/core/http/endpoints/openai/realtime_classifier_test.go b/core/http/endpoints/openai/realtime_classifier_test.go
@@ -93,6 +93,35 @@ var _ = Describe("classifierConfigFromPipeline", func() {
 	})
 })
 
+var _ = Describe("validateClassifierActivation", func() {
+	It("accepts a combined inference and score model", func() {
+		usecases := config.FLAG_CHAT | config.FLAG_SCORE
+		m := &wrappedModel{LLMConfig: &config.ModelConfig{KnownUsecases: &usecases}}
+		Expect(validateClassifierActivation(m, classifierTestConfig(0.4, nil))).To(Succeed())
+	})
+
+	It("rejects an active classifier when the model does not declare score", func() {
+		usecases := config.FLAG_CHAT
+		m := &wrappedModel{LLMConfig: &config.ModelConfig{KnownUsecases: &usecases}}
+		Expect(validateClassifierActivation(m, classifierTestConfig(0.4, nil))).To(MatchError(ContainSubstring("known_usecases")))
+	})
+
+	It("rejects a router config as the concrete scoring model", func() {
+		usecases := config.FLAG_SCORE
+		m := &wrappedModel{LLMConfig: &config.ModelConfig{
+			KnownUsecases: &usecases,
+			Router:        config.RouterConfig{Candidates: []config.RouterCandidate{{Model: "target"}}},
+		}}
+		Expect(validateClassifierActivation(m, classifierTestConfig(0.4, nil))).To(MatchError(ContainSubstring("concrete")))
+	})
+
+	It("allows disabling classification without score support", func() {
+		disabled := false
+		m := &wrappedModel{LLMConfig: &config.ModelConfig{}}
+		Expect(validateClassifierActivation(m, &types.ClassifierConfig{Enabled: &disabled})).To(Succeed())
+	})
+})
+
 var _ = Describe("resolveClassifier", func() {
 	It("uses the session config when no override is present", func() {
 		sess := classifierTestConfig(0, nil)
diff --git a/core/http/endpoints/openai/realtime_model.go b/core/http/endpoints/openai/realtime_model.go
diff --git a/core/http/endpoints/openai/realtime_voicegate.go b/core/http/endpoints/openai/realtime_voicegate.go
diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md
diff --git a/docs/content/features/text-generation.md b/docs/content/features/text-generation.md
diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go

Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@ func PreloadModelByName(ctx context.Context, cl config.ModelConfigLoader, ml m`
`28`	`28`	`return nil, err`
`29`	`29`	`}`
`30`	`30`
`31`		`- stages, err := pipelineStages(cl, &cfg.Pipeline, ml.ModelPath)`
	`31`	`+ stages, err := pipelineStages(cl, &cfg.Pipeline, ml.ModelPath, appConfig.ToConfigLoaderOptions()...)`
`32`	`32`	`if err != nil {`
`33`	`33`	`return nil, err`
`34`	`34`	`}`
`@@ -59,7 +59,7 @@ var loadStage = PreloadModel`
`59`	`59`	`// pipeline itself uses. A stage that fails to resolve is a misconfiguration,`
`60`	`60`	`// so it fails fast rather than being deferred to load. A pipeline with no`
`61`	`61`	`// stages set returns nil, which callers treat as "not a pipeline".`
`62`		`-func pipelineStages(cl config.ModelConfigLoader, p config.Pipeline, modelPath string) ([]PreloadStage, error) {`
	`62`	`+func pipelineStages(cl config.ModelConfigLoader, p config.Pipeline, modelPath string, opts ...config.ConfigLoaderOption) ([]PreloadStage, error) {`
`63`	`63`	`voiceRec := ""`
`64`	`64`	`if p.VoiceRecognition != nil {`
`65`	`65`	`voiceRec = p.VoiceRecognition.Model`
`@@ -76,7 +76,7 @@ func pipelineStages(cl config.ModelConfigLoader, p config.Pipeline, modelPath`
`76`	`76`	`if s.name == "" {`
`77`	`77`	`continue`
`78`	`78`	`}`
`79`		`- cfg, err := cl.LoadResolvedModelConfig(s.name, modelPath)`
	`79`	`+ cfg, err := cl.LoadResolvedModelConfig(s.name, modelPath, opts...)`
`80`	`80`	`if err != nil {`
`81`	`81`	`return nil, fmt.Errorf("%s (%s): %w", s.role, s.name, err)`
`82`	`82`	`}`