fix(distributed): reject wrong-model requests on the remaining modalities (#10990)

#10970 gave the four PredictOptions RPCs a model-identity check so a
backend reached through a stale distributed route rejects the request
instead of answering from whatever model it holds (#10952). Every other
modality shares that exposure: the route is cached by host:port, a worker
can recycle a stopped backend's port for another model's backend, and a
liveness-only probe cannot tell a stale row from a valid one.

Extends the same mechanism to the 21 remaining request messages that reach
a backend through the router, using the pattern #10970 established rather
than a parallel one:

- proto: ModelIdentity on each modality request message.
- controller: populated from ModelConfig.Model at the call site that also
  builds ModelOptions, so load-time and request-time values are equal by
  construction.
- backends: one generic guard in pkg/grpc/server.go (27 Go backends), the
  method set in backend/python/common (36 Python backends), llama-cpp
  (AudioTranscription/Stream, Rerank, Score) and privacy-filter
  (TokenClassify).
- reconcile already drops the stale row on IsModelMismatch; no change.

TTSRequest and SoundGenerationRequest get a SEPARATE ModelIdentity field
rather than reusing their existing `model`: FileStagingClient rewrites
`model` to a worker-local path, so comparing it would reject valid
requests in exactly the configuration this guards.

AudioEncode/AudioDecode are deliberately left unguarded: the opus codec
backend is loaded from a literal rather than a ModelConfig, so no value
carries the equality guarantee the comparison depends on. The four
bidirectional stream RPCs are out of scope; they bypass reconcile.

Empty means skip on both sides, so an old controller, an old backend, and
the bare request structs in tests/e2e-backends all keep working.


Assisted-by: Claude Code:claude-opus-4-8 [Read] [Edit] [Bash]

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

Author: localai-bot

backend/backend.proto

@@ -136,6 +136,10 @@ message MetricsResponse {
 message TokenClassifyRequest {
   string text = 1;
   float threshold = 2;
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 3;
 }
 
 // TokenClassifyEntity is one detected entity span. Byte offsets are
@@ -173,6 +177,10 @@ message ScoreRequest {
   // candidates differ in length and the consumer wants a per-token
   // measure comparable across them (PMI-style scoring).
   bool length_normalize = 4;
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 5;
 }
 
 // CandidateScore is one row in the ScoreResponse, matching by index
@@ -204,6 +212,10 @@ message RerankRequest {
   string query = 1;
   repeated string documents = 2;
   int32 top_n = 3;
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 4;
 }
 
 message RerankResult {
@@ -336,7 +348,14 @@ message PredictOptions {
   // paths (core/services/nodes/file_staging_client.go), so in distributed mode
   // they already differ from the load-time value and comparing them would
   // reject valid requests. Extending identity to those RPCs needs a separate
-  // field carrying the untranslated value.
+  // field carrying the untranslated value - which is exactly what
+  // TTSRequest.ModelIdentity and SoundGenerationRequest.ModelIdentity are.
+  //
+  // Every other request message that reaches a backend through the distributed
+  // router now carries the same ModelIdentity field, populated from the same
+  // ModelConfig.Model. FileStagingClient rewrites Src/Dst/Voice/Model/
+  // StartImage/EndImage/Audio and never ModelIdentity, so what the backend
+  // compares is always what the controller sent.
   string ModelIdentity = 54;
 
   // 24 was never assigned; reserve it so it is not silently reused.
@@ -510,6 +529,10 @@ message TranscriptRequest {
   float temperature = 8;
   repeated string timestamp_granularities = 9;
   bool stream = 10;
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 11;
 }
 
 message TranscriptResult {
@@ -588,6 +611,10 @@ message GenerateImageRequest {
 
   // Reference images for models that support them (e.g., Flux Kontext)
   repeated string ref_images = 12;
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 13;
 }
 
 message GenerateVideoRequest {
@@ -607,6 +634,10 @@ message GenerateVideoRequest {
   // Backend-specific per-request generation parameters. Values are strings
   // and are validated/coerced by the selected backend.
   map<string, string> params = 14;
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 15;
 }
 
 message TTSRequest {
@@ -624,10 +655,26 @@ message TTSRequest {
   // (e.g. Chatterbox exaggeration/cfg_weight/temperature). Values are strings and
   // coerced by the backend; unset leaves the backend's configured defaults.
   map<string, string> params = 7;
+  // ModelIdentity is a SEPARATE field from `model` above and carries the
+  // UNTRANSLATED controller-side ModelConfig.Model, so a backend can reject a
+  // request that reached it through a stale distributed route (#10952).
+  //
+  // `model` cannot be reused for this: FileStagingClient.TTS/.TTSStream and the
+  // SoundGeneration path rewrite it into a worker-local absolute path
+  // (core/services/nodes/file_staging_client.go), while the load-time value is
+  // untranslated. In distributed mode - exactly the configuration this guards -
+  // the two already differ, so comparing them would reject valid requests.
+  //
+  // Empty means "no identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 8;
 }
 
 message VADRequest {
   repeated float audio = 1;
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 2;
 }
 
 message VADSegment {
@@ -659,6 +706,10 @@ message DiarizeRequest {
   float  min_duration_on = 8;          // discard segments shorter than this (seconds); 0 = backend default
   float  min_duration_off = 9;         // merge gaps shorter than this (seconds); 0 = backend default
   bool   include_text = 10;            // when the backend can emit per-segment transcript for free, ask it to populate `text`
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 11;
 }
 
 message DiarizeSegment {
@@ -693,6 +744,18 @@ message SoundGenerationRequest {
   optional string language = 14;
   optional string timesignature = 15;
   optional bool instrumental = 17;
+  // ModelIdentity is a SEPARATE field from `model` above and carries the
+  // UNTRANSLATED controller-side ModelConfig.Model, so a backend can reject a
+  // request that reached it through a stale distributed route (#10952).
+  //
+  // `model` cannot be reused for this: FileStagingClient.TTS/.TTSStream and the
+  // SoundGeneration path rewrite it into a worker-local absolute path
+  // (core/services/nodes/file_staging_client.go), while the load-time value is
+  // untranslated. In distributed mode - exactly the configuration this guards -
+  // the two already differ, so comparing them would reject valid requests.
+  //
+  // Empty means "no identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 18;
 }
 
 message TokenizationResponse {
@@ -732,6 +795,10 @@ message DetectOptions {
   repeated float points = 3;   // Point coordinates as [x1, y1, label1, x2, y2, label2, ...] (label: 1=pos, 0=neg)
   repeated float boxes = 4;    // Box coordinates as [x1, y1, x2, y2, ...]
   float threshold = 5;         // Detection confidence threshold
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 6;
 }
 
 message Detection {
@@ -754,6 +821,10 @@ message SoundDetectionRequest {
   string src = 1;       // audio file path (LocalAI writes the upload to disk)
   int32 top_k = 2;      // number of top tags to return (0 = all classes)
   float threshold = 3;  // optional: drop tags scoring below this
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 4;
 }
 
 message SoundClass {
@@ -778,6 +849,10 @@ message DepthRequest {
   bool include_points = 7;         // back-project to a 3D point cloud (DualDPT)
   float points_conf_thresh = 8;    // keep points with confidence >= this threshold
   repeated string exports = 9;     // requested exports: "glb", "colmap"
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 10;
 }
 
 message DepthResponse {
@@ -809,6 +884,10 @@ message FaceVerifyRequest {
   string img2 = 2;              // base64-encoded image
   float  threshold = 3;         // cosine-distance threshold; 0 = use backend default
   bool   anti_spoofing = 4;     // run MiniFASNet liveness on each image; failed liveness forces verified=false
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 5;
 }
 
 message FaceVerifyResponse {
@@ -830,6 +909,10 @@ message FaceAnalyzeRequest {
   string          img = 1;          // base64-encoded image
   repeated string actions = 2;      // subset of ["age","gender","emotion","race"]; empty = all-supported
   bool            anti_spoofing = 3;
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 4;
 }
 
 message FaceAnalysis {
@@ -862,6 +945,10 @@ message VoiceVerifyRequest {
   string audio2 = 2;            // path to second audio clip
   float  threshold = 3;         // cosine-distance threshold; 0 = use backend default
   bool   anti_spoofing = 4;     // reserved for future AASIST bolt-on
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 5;
 }
 
 message VoiceVerifyResponse {
@@ -876,6 +963,10 @@ message VoiceVerifyResponse {
 message VoiceAnalyzeRequest {
   string          audio = 1;        // path to audio clip
   repeated string actions = 2;      // subset of ["age","gender","emotion"]; empty = all-supported
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 3;
 }
 
 message VoiceAnalysis {
@@ -894,6 +985,10 @@ message VoiceAnalyzeResponse {
 
 message VoiceEmbedRequest {
   string audio = 1;              // path to audio clip
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 2;
 }
 
 message VoiceEmbedResponse {
@@ -988,6 +1083,10 @@ message AudioTransformRequest {
   string reference_path = 2;         // optional auxiliary; empty => zero-fill
   string dst = 3;                    // required, output file path
   map<string, string> params = 4;    // backend-specific tuning
+  // ModelIdentity names the model this request is for; see
+  // PredictOptions.ModelIdentity for the full rationale. Empty means "no
+  // identity supplied" and backends MUST skip the check.
+  string ModelIdentity = 5;
 }
 
 message AudioTransformResult {

backend/cpp/llama-cpp/grpc-server.cpp

@@ -1416,7 +1416,11 @@ class BackendServiceImpl final : public backend::Backend::Service {
     // field and for the synthetic PredictOptions this server builds internally
     // for ASR, and the loaded side is empty when such a controller performed
     // the load. A false rejection is worse than the miss it prevents.
-    grpc::Status checkModelIdentity(const backend::PredictOptions* request) {
+    // Templated over the request type: every guarded request message exposes
+    // modelidentity(), and one body keeps the rule identical across modalities
+    // rather than repeating it per RPC.
+    template <typename Request>
+    grpc::Status checkModelIdentity(const Request* request) {
         if (request == nullptr || request->modelidentity().empty()) {
             return grpc::Status::OK;
         }
@@ -2846,6 +2850,8 @@ class BackendServiceImpl final : public backend::Backend::Service {
     }
 
     grpc::Status Rerank(ServerContext* context, const backend::RerankRequest* request, backend::RerankResult* rerankResult) override {
+        auto identity = checkModelIdentity(request);
+        if (!identity.ok()) return identity;
         if (!params_base.embedding || params_base.pooling_type != LLAMA_POOLING_TYPE_RANK) {
             return grpc::Status(grpc::StatusCode::UNIMPLEMENTED, "This server does not support reranking. Start it with `--reranking` and without `--embedding`");
         }
@@ -2970,6 +2976,8 @@ class BackendServiceImpl final : public backend::Backend::Service {
     grpc::Status Score(ServerContext* context, const backend::ScoreRequest* request, backend::ScoreResponse* response) override {
         auto auth = checkAuth(context);
         if (!auth.ok()) return auth;
+        auto identity = checkModelIdentity(request);
+        if (!identity.ok()) return identity;
         if (params_base.model.path.empty()) {
             return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, "Model not loaded");
         }
@@ -3428,6 +3436,8 @@ class BackendServiceImpl final : public backend::Backend::Service {
                                     backend::TranscriptResult* response) override {
         auto auth = checkAuth(context);
         if (!auth.ok()) return auth;
+        auto identity = checkModelIdentity(request);
+        if (!identity.ok()) return identity;
 
         backend::Reply reply;
         grpc::Status st = runTranscriptionAsCompletion(context, request, &reply);
@@ -3446,6 +3456,8 @@ class BackendServiceImpl final : public backend::Backend::Service {
                                           grpc::ServerWriter<backend::TranscriptStreamResponse>* writer) override {
         auto auth = checkAuth(context);
         if (!auth.ok()) return auth;
+        auto identity = checkModelIdentity(request);
+        if (!identity.ok()) return identity;
 
         // Buffered streaming: run the transcription as a normal chat
         // completion, then emit one delta + one final event. Real

backend/cpp/privacy-filter/grpc-server.cpp

@@ -41,6 +41,11 @@ namespace {
 // per loaded model. g_mu guards (re)load against in-flight classification.
 std::mutex          g_mu;
 pf_ctx *            g_ctx = nullptr;
+// The ModelOptions.Model this process loaded, compared against
+// TokenClassifyRequest.ModelIdentity so a request that arrived through a stale
+// distributed route is rejected rather than answered from the wrong model
+// (#10952). Guarded by g_mu like the rest of the engine state.
+std::string         g_loaded_model_identity;
 std::atomic<Server *> g_server{nullptr};
 
 // Resolve the device string the engine expects ("cpu" / "gpu" / "cuda" /
@@ -113,17 +118,48 @@ class PrivacyFilterBackend final : public backend::Backend::Service {
         }
 
         g_ctx = ctx;
+        // Record what we loaded so TokenClassify can reject a request meant
+        // for a different model. request->model(), not modelfile(): it is the
+        // value the controller also sends as ModelIdentity, and the two are
+        // read from the same ModelConfig.Model (#10952).
+        g_loaded_model_identity = request->model();
         result->set_success(true);
         result->set_message("privacy-filter loaded (" + device + ")");
         return GStatus::OK;
     }
 
+    // checkModelIdentity mirrors pkg/grpc/server.go,
+    // backend/python/common/model_identity.py and the llama-cpp server. In
+    // distributed mode a worker can recycle a stopped backend's gRPC port for
+    // another model's backend, and the controller's liveness-only probe cannot
+    // tell a stale cached route from a valid one, so the backend has to catch
+    // it. Either side empty means "skip": the request side is empty for a
+    // controller that predates the field, the loaded side when such a
+    // controller performed the load. A false rejection is worse than the miss.
+    // Callers must already hold g_mu.
+    GStatus checkModelIdentity(const backend::TokenClassifyRequest * request) {
+        if (request == nullptr || request->modelidentity().empty()) {
+            return GStatus::OK;
+        }
+        if (g_loaded_model_identity.empty() ||
+            g_loaded_model_identity == request->modelidentity()) {
+            return GStatus::OK;
+        }
+        // NOT_FOUND plus this exact sentinel is the cross-language contract
+        // the router matches on (grpcerrors.ModelMismatchSentinel).
+        return GStatus(StatusCode::NOT_FOUND,
+                       "privacy-filter: model identity mismatch: loaded \"" +
+                       g_loaded_model_identity + "\", requested \"" +
+                       request->modelidentity() + "\"");
+    }
+
     GStatus TokenClassify(ServerContext *, const backend::TokenClassifyRequest * request,
                           backend::TokenClassifyResponse * response) override {
         std::lock_guard<std::mutex> lock(g_mu);
         if (!g_ctx) {
             return GStatus(StatusCode::FAILED_PRECONDITION, "Model not loaded");
         }
+        if (GStatus id = checkModelIdentity(request); !id.ok()) return id;
 
         const std::string & text = request->text();
         if (text.empty()) {