feat(ai-proxy): add per-protocol request_body override and rename max_tokens mapping to llm_options (#13269)

Author: nic-6443

apisix/plugins/ai-protocols/init.lua

@@ -22,6 +22,8 @@
 
 local converters = require("apisix.plugins.ai-protocols.converters")
 local ipairs = ipairs
+local pairs = pairs
+local table = table
 
 local _M = {}
 
@@ -66,6 +68,18 @@ end
 
 
 
+--- Get the list of all registered protocol names.
+-- @return table Array of protocol names
+function _M.names()
+    local names = {}
+    for name in pairs(registered) do
+        names[#names + 1] = name
+    end
+    table.sort(names)
+    return names
+end
+
+
 --- Find a converter that can bridge from client_protocol to a protocol
 -- supported by the driver. Delegates to the converters registry.
 -- @param client_protocol string The detected client protocol

apisix/plugins/ai-providers/base.lua

@@ -36,6 +36,7 @@ local transport_http = require("apisix.plugins.ai-transport.http")
 local transport_auth = require("apisix.plugins.ai-transport.auth")
 local log_sanitize = require("apisix.utils.log-sanitize")
 local protocols = require("apisix.plugins.ai-protocols")
+local deep_merge = require("apisix.plugins.ai-proxy.merge").deep_merge
 local ngx = ngx
 local ngx_now = ngx.now
 local tonumber = tonumber
@@ -183,12 +184,21 @@ function _M.build_request(self, ctx, conf, request_body, opts)
         end
     end
 
-    -- Apply request body override via provider capability hook
-    if opts.override_request_body then
+    -- Apply llm_options via provider capability hook (always force-overwrites)
+    if opts.override_llm_options then
         local cap = self.capabilities and self.capabilities[ctx.ai_target_protocol]
         if cap and cap.rewrite_request_body then
-            cap.rewrite_request_body(request_body, opts.override_request_body,
-                                     opts.request_body_force_override)
+            cap.rewrite_request_body(request_body, opts.override_llm_options, true)
+        end
+    end
+
+    -- Apply per-target-protocol request body override (deep merge)
+    if opts.request_body_override_map then
+        local patch = opts.request_body_override_map[ctx.ai_target_protocol]
+        if patch then
+            core.log.info("applying request_body override for target protocol '",
+                          ctx.ai_target_protocol, "'")
+            request_body = deep_merge(request_body, patch, opts.request_body_force_override)
         end
     end
     params.body = request_body

apisix/plugins/ai-proxy/base.lua

@@ -125,7 +125,9 @@ function _M.before_proxy(conf, ctx, on_error)
             model_options = ai_instance.options,
             conf = ai_instance.provider_conf or {},
             auth = ai_instance.auth,
-            override_request_body =
+            override_llm_options =
+                core.table.try_read_attr(ai_instance, "override", "llm_options"),
+            request_body_override_map =
                 core.table.try_read_attr(ai_instance, "override", "request_body"),
             request_body_force_override =
                 core.table.try_read_attr(ai_instance, "override", "request_body_force_override"),

apisix/plugins/ai-proxy/merge.lua

@@ -0,0 +1,75 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+--- Deep-merge helper for ai-proxy request body overrides.
+-- Semantics:
+--   * Both sides are plain objects (string-keyed tables) -> recursive merge.
+--   * Otherwise (scalar, array, type mismatch, cjson.empty_array/empty_object)
+--     -> patch value replaces target value wholesale.
+-- This matches RFC 7396 JSON Merge Patch minus null-deletion.
+
+local core = require("apisix.core")
+local pairs = pairs
+local next = next
+local type = type
+local getmetatable = getmetatable
+
+local _M = {}
+
+
+-- Returns true when tbl is a plain object (string keys only, or empty) that
+-- we should recurse into. Arrays (cjson array_mt) and cjson sentinels are
+-- treated as "replace wholesale".
+local function is_plain_object(tbl)
+    if type(tbl) ~= "table" then
+        return false
+    end
+    local mt = getmetatable(tbl)
+    if mt == core.json.array_mt then
+        return false
+    end
+    local k = next(tbl)
+    if k == nil then
+        return true
+    end
+    return type(k) == "string"
+end
+
+
+local function deep_merge(target, patch, force)
+    if not is_plain_object(patch) then
+        return patch
+    end
+    if not is_plain_object(target) then
+        -- target is not an object but patch is; patch wins only if force or
+        -- target is nil (which the caller handles)
+        return patch
+    end
+    for k, v in pairs(patch) do
+        if is_plain_object(v) and is_plain_object(target[k]) then
+            -- Both sides are objects: always recurse regardless of force
+            deep_merge(target[k], v, force)
+        elseif target[k] == nil or force then
+            target[k] = v
+        end
+    end
+    return target
+end
+_M.deep_merge = deep_merge
+
+
+return _M

apisix/plugins/ai-proxy/schema.lua

@@ -16,6 +16,8 @@
 --
 local schema_def = require("apisix.schema_def")
 local ai_providers_schema = require("apisix.plugins.ai-providers.schema")
+local protocols = require("apisix.plugins.ai-protocols")
+local ipairs = ipairs
 
 local _M = {}
 
@@ -72,33 +74,59 @@ local model_options_schema = {
     additionalProperties = true,
 }
 
+-- Build per-target-protocol request body override schema.
+-- Each registered protocol gets an optional "any-shape object" entry.
+local request_body_override_properties = {}
+for _, proto_name in ipairs(protocols.names()) do
+    request_body_override_properties[proto_name] = {
+        type = "object",
+        description = "Deep-merged into the outgoing request body when the "
+            .. "target protocol is '" .. proto_name .. "'.",
+        additionalProperties = true,
+    }
+end
+
+local request_body_override_schema = {
+    type = "object",
+    description = "Per target-protocol request body overrides. Keys are target "
+        .. "protocol names; values are partial request bodies that are "
+        .. "deep-merged into the outgoing body (objects merged recursively, "
+        .. "arrays and scalars replaced wholesale).",
+    properties = request_body_override_properties,
+    additionalProperties = false,
+}
+
+local llm_options_schema = {
+    type = "object",
+    properties = {
+        max_tokens = {
+            type = "integer",
+            minimum = 1,
+            description = "Maximum number of output tokens. APISIX automatically "
+                .. "maps this to the correct field name for the target provider "
+                .. "(e.g. max_completion_tokens for OpenAI, max_output_tokens "
+                .. "for Responses API). Always force-overwrites the client value.",
+        },
+    },
+    additionalProperties = false,
+}
+
 local override_schema = {
     type = "object",
     properties = {
         endpoint = {
             type = "string",
             description = "To be specified to override the endpoint of the AI Instance",
         },
-        request_body = {
-            type = "object",
-            properties = {
-                max_tokens = {
-                    type = "integer",
-                    minimum = 1,
-                    description = "Maximum number of output tokens. APISIX automatically "
-                        .. "maps this to the correct field name for the target provider "
-                        .. "(e.g. max_completion_tokens for OpenAI, max_output_tokens "
-                        .. "for Responses API).",
-                },
-            },
-            additionalProperties = false,
-        },
+        llm_options = llm_options_schema,
+        request_body = request_body_override_schema,
         request_body_force_override = {
             type = "boolean",
             default = false,
             description = "When false (default), client request body fields take "
-                .. "priority and override values only fill in missing fields. "
-                .. "When true, override values forcefully overwrite client fields.",
+                .. "priority and request_body override values only fill in "
+                .. "missing fields. When true, request_body override values "
+                .. "forcefully overwrite client fields.",
         },
     },
 }

docs/en/latest/plugins/ai-proxy-multi.md

@@ -81,9 +81,10 @@ In addition, the Plugin also supports logging LLM request information in the acc
 | logging.payloads                    | boolean        | False    | false                           |              | If true, log request and response payload. |
 | instances.override                    | object         | False    |                                   |              | Override setting. |
 | instances.override.endpoint           | string         | False    |                                   |              | LLM provider endpoint to replace the default endpoint with. If not configured, the Plugin uses the default OpenAI endpoint `https://api.openai.com/v1/chat/completions`. |
-| instances.override.request_body       | object         | False    |                                   |              | Request body overrides. See [Provider-aware `max_tokens` mapping](./ai-proxy.md#provider-aware-max_tokens-mapping) in the `ai-proxy` documentation for how the contained fields are forwarded to each provider. |
-| instances.override.request_body.max_tokens | integer    | False    |                                   | ≥ 1          | Maximum number of output tokens. APISIX automatically maps this to the provider-specific field name (e.g. `max_completion_tokens` for OpenAI Chat Completions, `max_output_tokens` for OpenAI Responses API, `max_tokens` for most other providers). By default, client request fields take priority and the override value only fills in when the client did not set it; set `instances.override.request_body_force_override` to `true` to forcefully overwrite the client value. |
-| instances.override.request_body_force_override | boolean | False | false |                            | When `false` (default), client request body fields take priority and `instances.override.request_body` values only fill in missing fields. When `true`, `instances.override.request_body` values forcefully overwrite client request body fields. |
+| instances.override.llm_options           | object         | False    |                                   |              | Provider-aware LLM options. See [Provider-aware `max_tokens` mapping](./ai-proxy.md#provider-aware-max_tokens-mapping) in the `ai-proxy` documentation. |
+| instances.override.llm_options.max_tokens | integer    | False    |                                   | ≥ 1          | Maximum number of output tokens. APISIX automatically maps this to the provider-specific field name. Always force-overwrites the client value. |
+| instances.override.request_body       | object         | False    |                                   |              | Per target-protocol request body overrides. See [Per-protocol request body override](./ai-proxy.md#per-protocol-request-body-override) in the `ai-proxy` documentation. |
+| instances.override.request_body_force_override | boolean | False | false |                            | When `false` (default), client request body fields take priority and `instances.override.request_body` values only fill in missing fields. When `true`, `instances.override.request_body` values forcefully overwrite client fields. Does not affect `instances.override.llm_options`. |
 | instances.checks                              | object         | False    |                                   |              | Health check configurations. Note that at the moment, OpenAI, DeepSeek, and AIMLAPI do not provide an official health check endpoint. Other LLM services that you can configure under `openai-compatible` provider may have available health check endpoints. |
 | instances.checks.active                       | object         | True     |                                   |              | Active health check configurations. |
 | instances.checks.active.type                  | string         | False    | http                            | [http, https, tcp] | Type of health check connection. |

docs/en/latest/plugins/ai-proxy.md

@@ -66,9 +66,10 @@ In addition, the Plugin also supports logging LLM request information in the acc
 | options.model   | string  | False    |         |                                          | Name of the LLM model, such as `gpt-4` or `gpt-3.5`. Refer to the LLM provider's API documentation for available models. |
 | override        | object  | False    |         |                                          | Override setting. |
 | override.endpoint | string | False    |         |                                          | Custom LLM provider endpoint, required when `provider` is `openai-compatible`. |
-| override.request_body | object | False  |         |                                          | Request body overrides. See [Provider-aware `max_tokens` mapping](#provider-aware-max_tokens-mapping) for how the contained fields are forwarded to each provider. |
-| override.request_body.max_tokens | integer | False  |         | ≥ 1                                | Maximum number of output tokens. APISIX automatically maps this to the provider-specific field name (e.g. `max_completion_tokens` for OpenAI Chat Completions, `max_output_tokens` for OpenAI Responses API, `max_tokens` for most other providers). By default, client request fields take priority and the override value only fills in when the client did not set it; set `override.request_body_force_override` to `true` to forcefully overwrite the client value. |
-| override.request_body_force_override | boolean | False | false |                                    | When `false` (default), client request body fields take priority and `override.request_body` values only fill in missing fields. When `true`, `override.request_body` values forcefully overwrite client request body fields. |
+| override.llm_options | object | False  |         |                                          | Provider-aware LLM options. See [Provider-aware `max_tokens` mapping](#provider-aware-max_tokens-mapping). |
+| override.llm_options.max_tokens | integer | False  |         | ≥ 1                                | Maximum number of output tokens. APISIX automatically maps this to the provider-specific field name (e.g. `max_completion_tokens` for OpenAI Chat Completions, `max_output_tokens` for OpenAI Responses API, `max_tokens` for most other providers). Always force-overwrites the client value. |
+| override.request_body | object | False  |         |                                          | Per target-protocol request body overrides. Keys are target protocol names (`openai-chat`, `openai-responses`, `openai-embeddings`, `anthropic-messages`); values are partial request bodies that are deep-merged into the outgoing body (objects merged recursively, arrays and scalars replaced wholesale). See [Per-protocol request body override](#per-protocol-request-body-override). |
+| override.request_body_force_override | boolean | False | false |                                    | When `false` (default), client request body fields take priority and `override.request_body` values only fill in missing fields. When `true`, `override.request_body` values forcefully overwrite client fields. Does not affect `override.llm_options`, which always force-overwrites. |
 | logging        | object  | False    |         |                                          | Logging configurations. Does not affect `error.log`. |
 | logging.summaries | boolean | False | false |                                          | If true, logs request LLM model, duration, request, and response tokens. |
 | logging.payloads  | boolean | False | false |                                          | If true, logs request and response payload. |
@@ -82,7 +83,7 @@ In addition, the Plugin also supports logging LLM request information in the acc
 
 ## Provider-aware `max_tokens` mapping
 
-LLM providers and API endpoints disagree on the field name used to cap the number of output tokens. Configuring `override.request_body.max_tokens` lets you set a single value in APISIX and have it forwarded under the field name expected by each provider/endpoint.
+LLM providers and API endpoints disagree on the field name used to cap the number of output tokens. Configuring `override.llm_options.max_tokens` lets you set a single value in APISIX and have it forwarded under the field name expected by each provider/endpoint. `llm_options` always force-overwrites the client value.
 
 The table below shows, for each `provider` and target API endpoint, the upstream field name APISIX rewrites `max_tokens` to. A `—` means the provider does not expose that endpoint.
 
@@ -100,10 +101,21 @@ The table below shows, for each `provider` and target API endpoint, the upstream
 
 ¹ When `provider` is `openai` and the target is the Chat Completions endpoint, APISIX always rewrites to `max_completion_tokens` and removes any `max_tokens` field from the request body — `max_tokens` has been deprecated in favor of `max_completion_tokens` by OpenAI.
 
+## Per-protocol request body override
+
+`override.request_body` provides fine-grained, per-protocol control over the outgoing request body. Keys are target protocol names (`openai-chat`, `openai-responses`, `openai-embeddings`, `anthropic-messages`); values are partial JSON objects that are deep-merged into the outgoing body after protocol conversion.
+
+Merge semantics:
+
+- Both sides are plain objects (string-keyed) → recursive merge.
+- Otherwise (scalar, array, type mismatch) → patch value replaces target value wholesale.
+
 Priority between client request and override is controlled by `override.request_body_force_override`:
 
-- `false` (default): if the client request body already sets the corresponding field, it is preserved; the override value only fills in when the field is missing.
-- `true`: the override value forcefully overwrites the field in the client request body.
+- `false` (default): if the client request body already sets the field, it is preserved; the override value only fills in when the field is missing.
+- `true`: the override value forcefully overwrites the client field.
+
+When both `llm_options` and `request_body` are configured, `llm_options` is applied first (always force), then `request_body` deep-merges on top. This means `request_body` can override fields set by `llm_options`.
 
 ## Examples