uts

Author: mkulakow

demos/continuous_batching/agentic_ai/README.md

@@ -330,7 +330,7 @@ Pull and start OVMS:
 ```bash
 mkdir -p ${HOME}/models
 docker run -d --user $(id -u):$(id -g) --rm -p 8000:8000 -v ${HOME}/models:/models --device /dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) openvino/model_server:weekly \
---rest_port 8000 --model_repository_path /models --source_model Junrui2021/Qwen3-VL-8B-Instruct-int4 --tool_parser hermes3 --target_device GPU --task text_generation --pipeline_type VLM_CB --allowed_media_domains raw.githubusercontent.com
+--rest_port 8122 --model_repository_path /models --source_model Junrui2021/Qwen3-VL-8B-Instruct-int4 --model_name ovms-model --tool_parser hermes3 --target_device GPU --task text_generation --pipeline_type VLM_CB --allowed_media_domains raw.githubusercontent.com
 ```
 
 Use MCP server, with additional image of Gdańsk old town. VLM model deduces location and calls `get_weather` tool to summarize the weather conditions in the city.

src/llm/apis/openai_responses.cpp

@@ -224,13 +224,20 @@ static absl::StatusOr<ResponsesInputItemKind> classifyInputItem(const rapidjson:
 // "Message has tool role, but there was no previous assistant message with a
 // tool call!").
 //
+// Reasoning that is not followed by an assistant or function_call item is
+// emitted as a standalone assistant turn with empty content and the buffered
+// reasoning attached as `reasoning_content`. This preserves the model's
+// chain-of-thought across turns even when the prior turn produced no visible
+// output.
+//
 // The algorithm is sink-agnostic; concrete output (ov::genai::ChatHistory vs a
 // rapidjson messages array) is provided by the Sink template parameter, which
 // must implement:
 //   absl::Status extractContent(itemObj, index, std::string& outText);
 //   void emitToolMessage(callId, output);
 //   void emitMessage(role, contentText, reasoning);  // reasoning empty -> skip
 //   void emitAssistantWithToolCalls(contentText, reasoning, toolCalls);
+//   void emitStandaloneReasoning(reasoning);  // assistant turn carrying only reasoning_content
 //   absl::Status onMissingRole(itemObj);
 template <typename Sink>
 class ResponsesInputBuilder {
@@ -313,7 +320,8 @@ class ResponsesInputBuilder {
             return absl::OkStatus();
         }
         // Non-assistant items must not absorb pending tool_calls; flush first.
-        // (flushPendingFunctionCalls also clears any orphan reasoning content.)
+        // (flushPendingFunctionCalls also emits any standalone reasoning content
+        // as a standalone assistant turn.)
         if (role != "assistant") {
             flushPendingFunctionCalls("");
         }
@@ -329,7 +337,16 @@ class ResponsesInputBuilder {
 
     void flushPendingFunctionCalls(const std::string& assistantText) {
         if (pendingFunctionCalls.empty()) {
-            pendingReasoningContent.clear();
+            // No tool calls, but possibly buffered reasoning to flush as a
+            // standalone assistant turn carrying only reasoning_content (no
+            // `content` field at all, so templates that gate on `message.content`
+            // skip the content branch and templates that gate on
+            // `message.reasoning_content` still see the buffered text).
+            if (!pendingReasoningContent.empty()) {
+                std::string reasoning = std::move(pendingReasoningContent);
+                pendingReasoningContent.clear();
+                sink.emitStandaloneReasoning(reasoning);
+            }
             return;
         }
         std::string reasoning = std::move(pendingReasoningContent);
@@ -427,6 +444,15 @@ class ChatHistorySink {
         chatHistory.last()["tool_calls"] = rapidJsonValueToJsonContainer(toolCallsArray);
     }
 
+    // Emit an assistant turn that carries only reasoning_content (no content,
+    // no tool_calls). Used when reasoning is not followed by an assistant or
+    // function_call item.
+    void emitStandaloneReasoning(const std::string& reasoning) {
+        chatHistory.push_back({});
+        chatHistory.last()["role"] = "assistant";
+        chatHistory.last()["reasoning_content"] = reasoning;
+    }
+
     absl::Status onMissingRole(const rapidjson::Value::ConstObject&) {
         return absl::InvalidArgumentError("input item role is missing or invalid");
     }
@@ -515,6 +541,15 @@ class ProcessedJsonSink {
         messagesArray.PushBack(msgObj, alloc);
     }
 
+    // Emit an assistant turn that carries only reasoning_content (no content,
+    // no tool_calls). See ChatHistorySink::emitStandaloneReasoning for rationale.
+    void emitStandaloneReasoning(const std::string& reasoning) {
+        rapidjson::Value msgObj(rapidjson::kObjectType);
+        msgObj.AddMember("role", rapidjson::Value("assistant", alloc), alloc);
+        msgObj.AddMember("reasoning_content", rapidjson::Value(reasoning.c_str(), alloc), alloc);
+        messagesArray.PushBack(msgObj, alloc);
+    }
+
     void emitAssistantWithToolCalls(const std::string& contentText, const std::string& reasoning,
         const std::vector<const rapidjson::Value*>& toolCalls) {
         rapidjson::Value msgObj(rapidjson::kObjectType);