fix: normalize named tool_choice to required + filtered tools for OpenAI-compatible providers (#528)

LM Studio (and Ollama) reject the named tool_choice dict format
{"type": "function", "function": {"name": "..."}} with HTTP 400.

The reflect agent uses this format on iterations 0-2 to force sequential
tool selection, causing reflect to fail entirely on LM Studio.

The fix converts named tool_choice dicts to tool_choice="required" with
the tools list filtered to just the requested tool — semantically identical
and accepted by all providers including LM Studio and Ollama.

Closes #520

Author: nicoloboschi

hindsight-api/hindsight_api/engine/providers/openai_compatible_llm.py

@@ -522,6 +522,19 @@ async def call_with_tools(
         """
         start_time = time.time()
 
+        # Normalize named tool_choice dicts to "required" + filter tools.
+        # Some providers (e.g. LM Studio, Ollama) reject the OpenAI named format
+        # {"type": "function", "function": {"name": "..."}}.  The semantics are
+        # identical to tool_choice="required" with the tools list restricted to
+        # just the requested tool, so we apply that transformation universally.
+        if isinstance(tool_choice, dict) and tool_choice.get("type") == "function":
+            forced_name = tool_choice.get("function", {}).get("name")
+            if forced_name:
+                filtered = [t for t in tools if t.get("function", {}).get("name") == forced_name]
+                if filtered:
+                    tools = filtered
+                tool_choice = "required"
+
         # Build call parameters
         call_params: dict[str, Any] = {
             "model": self.model,

hindsight-api/tests/test_lmstudio_tool_choice.py

@@ -0,0 +1,321 @@
+"""
+Reproduce issue #520: Reflect fails with LM Studio due to unsupported tool_choice format.
+
+The reflect agent forces tool selection via named tool_choice dicts on the first few iterations:
+  {"type": "function", "function": {"name": "search_mental_models"}}
+
+LM Studio (and Ollama) reject this format with HTTP 400:
+  "Tool choice of type 'function' is not supported. Use 'auto', 'none', or 'required'."
+
+The fix should convert named tool_choice to "required" and filter the tools list
+to only the requested tool for providers that don't support named tool_choice.
+"""
+
+import json
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from openai import APIStatusError
+
+from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
+
+# Reflect agent tools (subset matching what agent.py uses)
+REFLECT_TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "search_mental_models",
+            "description": "Search consolidated mental models",
+            "parameters": {
+                "type": "object",
+                "properties": {"query": {"type": "string"}},
+                "required": ["query"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "search_observations",
+            "description": "Search raw observations",
+            "parameters": {
+                "type": "object",
+                "properties": {"query": {"type": "string"}},
+                "required": ["query"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "recall",
+            "description": "Recall semantic memories",
+            "parameters": {
+                "type": "object",
+                "properties": {"query": {"type": "string"}},
+                "required": ["query"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "done",
+            "description": "Finish and return the answer",
+            "parameters": {
+                "type": "object",
+                "properties": {"answer": {"type": "string"}},
+                "required": ["answer"],
+            },
+        },
+    },
+]
+
+
+def _make_lmstudio_llm() -> OpenAICompatibleLLM:
+    return OpenAICompatibleLLM(
+        provider="lmstudio",
+        api_key="local",
+        base_url="http://localhost:1234/v1",
+        model="openai/gpt-oss-20b",
+    )
+
+
+def _lmstudio_400_error(msg: str = "Tool choice of type 'function' is not supported. Use 'auto', 'none', or 'required'.") -> APIStatusError:
+    """Simulate the HTTP 400 LM Studio returns for unsupported tool_choice format."""
+    mock_response = MagicMock()
+    mock_response.status_code = 400
+    mock_response.headers = {}
+    return APIStatusError(
+        message=msg,
+        response=mock_response,
+        body={"error": {"message": msg, "type": "invalid_request_error"}},
+    )
+
+
+def _make_tool_call_response(tool_name: str, arguments: dict) -> MagicMock:
+    """Build a mock successful tool call response from the LLM API."""
+    mock_tc = MagicMock()
+    mock_tc.id = "call_abc123"
+    mock_tc.function.name = tool_name
+    mock_tc.function.arguments = json.dumps(arguments)
+
+    mock_response = MagicMock()
+    mock_response.usage.prompt_tokens = 120
+    mock_response.usage.completion_tokens = 40
+    mock_response.usage.total_tokens = 160
+    mock_response.choices[0].finish_reason = "tool_calls"
+    mock_response.choices[0].message.content = None
+    mock_response.choices[0].message.tool_calls = [mock_tc]
+    return mock_response
+
+
+class TestLMStudioNamedToolChoiceBug:
+    """
+    Reproduces issue #520.
+
+    The reflect agent (agent.py lines 546-555) sets tool_choice to a named dict
+    on the first iterations to force sequential retrieval:
+
+        iteration=0, has_mental_models=True  → {"type": "function", "function": {"name": "search_mental_models"}}
+        iteration=0, has_mental_models=False → {"type": "function", "function": {"name": "search_observations"}}
+        iteration=1, has_mental_models=True  → {"type": "function", "function": {"name": "search_observations"}}
+        iteration=1 or (2 with models)       → {"type": "function", "function": {"name": "recall"}}
+
+    LM Studio rejects these dict formats with HTTP 400.
+    """
+
+    @pytest.mark.asyncio
+    async def test_lmstudio_named_tool_choice_no_longer_causes_400(self):
+        """
+        Regression test for issue #520: named tool_choice dict is converted to
+        "required" + filtered tools before the API call, so LM Studio never
+        sees the unsupported format and the 400 error no longer occurs.
+        """
+        llm = _make_lmstudio_llm()
+        named_tool_choice = {"type": "function", "function": {"name": "search_mental_models"}}
+        success_response = _make_tool_call_response("search_mental_models", {"query": "user name"})
+
+        with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create:
+            mock_create.return_value = success_response
+
+            # Should succeed — no 400 because the dict is converted before sending
+            result = await llm.call_with_tools(
+                messages=[{"role": "user", "content": "What is the user's name?"}],
+                tools=REFLECT_TOOLS,
+                tool_choice=named_tool_choice,
+                max_retries=0,
+            )
+
+        assert len(result.tool_calls) == 1
+        assert result.tool_calls[0].name == "search_mental_models"
+
+        sent_kwargs = mock_create.call_args.kwargs
+        assert sent_kwargs["tool_choice"] == "required"
+        assert len(sent_kwargs["tools"]) == 1
+        assert sent_kwargs["tools"][0]["function"]["name"] == "search_mental_models"
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize(
+        "forced_tool_name",
+        ["search_mental_models", "search_observations", "recall"],
+    )
+    async def test_all_reflect_forced_tools_fail_on_lmstudio(self, forced_tool_name: str):
+        """
+        Each named tool_choice the reflect agent uses on iterations 0-2 triggers
+        the same 400 error on LM Studio.
+        """
+        llm = _make_lmstudio_llm()
+        named_tool_choice = {"type": "function", "function": {"name": forced_tool_name}}
+
+        with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create:
+            mock_create.side_effect = _lmstudio_400_error()
+
+            with pytest.raises(APIStatusError) as exc_info:
+                await llm.call_with_tools(
+                    messages=[{"role": "user", "content": "Test query"}],
+                    tools=REFLECT_TOOLS,
+                    tool_choice=named_tool_choice,
+                    max_retries=0,
+                )
+
+            assert exc_info.value.status_code == 400
+
+    @pytest.mark.asyncio
+    async def test_lmstudio_string_tool_choice_works_fine(self):
+        """
+        String tool_choice values ("auto", "none", "required") ARE supported by LM Studio.
+        Only the dict format {"type": "function", "function": {"name": "..."}} fails.
+        This test confirms the control case works.
+        """
+        llm = _make_lmstudio_llm()
+        success_response = _make_tool_call_response("search_mental_models", {"query": "user name"})
+
+        with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create:
+            mock_create.return_value = success_response
+
+            result = await llm.call_with_tools(
+                messages=[{"role": "user", "content": "What is the user's name?"}],
+                tools=REFLECT_TOOLS,
+                tool_choice="required",  # string form — LM Studio accepts this
+                max_retries=0,
+            )
+
+        assert len(result.tool_calls) == 1
+        assert result.tool_calls[0].name == "search_mental_models"
+
+        # Confirm "required" was sent, not a dict
+        sent_kwargs = mock_create.call_args.kwargs
+        assert sent_kwargs["tool_choice"] == "required"
+
+
+class TestExpectedFixBehavior:
+    """
+    Tests that document the EXPECTED behavior after the fix is applied.
+
+    For lmstudio (and ollama) providers, when tool_choice is a named dict:
+      {"type": "function", "function": {"name": "search_mental_models"}}
+
+    The fix should:
+      1. Convert tool_choice to "required"
+      2. Filter tools to only the requested tool
+
+    These tests currently FAIL (because the fix is not yet implemented).
+    After the fix is applied, they should PASS.
+    """
+
+    @pytest.mark.asyncio
+    async def test_fix_converts_named_tool_choice_to_required(self):
+        """
+        After fix: named tool_choice dict is converted to "required" for lmstudio.
+        The API receives tool_choice="required" instead of the unsupported dict.
+        """
+        llm = _make_lmstudio_llm()
+        named_tool_choice = {"type": "function", "function": {"name": "search_mental_models"}}
+        success_response = _make_tool_call_response("search_mental_models", {"query": "user name"})
+
+        with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create:
+            mock_create.return_value = success_response
+
+            result = await llm.call_with_tools(
+                messages=[{"role": "user", "content": "What is the user's name?"}],
+                tools=REFLECT_TOOLS,
+                tool_choice=named_tool_choice,
+                max_retries=0,
+            )
+
+        assert len(result.tool_calls) == 1
+        assert result.tool_calls[0].name == "search_mental_models"
+
+        sent_kwargs = mock_create.call_args.kwargs
+        # Fix: dict was converted to "required"
+        assert sent_kwargs["tool_choice"] == "required", (
+            f"Expected tool_choice='required', got {sent_kwargs['tool_choice']!r}"
+        )
+        # Fix: tools filtered to just the requested one
+        assert len(sent_kwargs["tools"]) == 1
+        assert sent_kwargs["tools"][0]["function"]["name"] == "search_mental_models"
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize(
+        "forced_tool_name",
+        ["search_mental_models", "search_observations", "recall"],
+    )
+    async def test_fix_filters_tools_to_requested_tool(self, forced_tool_name: str):
+        """
+        After fix: tools list is filtered to only the forced tool so the model
+        can only call that one tool (equivalent to the named tool_choice behavior).
+        """
+        llm = _make_lmstudio_llm()
+        named_tool_choice = {"type": "function", "function": {"name": forced_tool_name}}
+        success_response = _make_tool_call_response(forced_tool_name, {"query": "test"})
+
+        with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create:
+            mock_create.return_value = success_response
+
+            await llm.call_with_tools(
+                messages=[{"role": "user", "content": "Test query"}],
+                tools=REFLECT_TOOLS,
+                tool_choice=named_tool_choice,
+                max_retries=0,
+            )
+
+        sent_kwargs = mock_create.call_args.kwargs
+        assert sent_kwargs["tool_choice"] == "required"
+        assert len(sent_kwargs["tools"]) == 1
+        assert sent_kwargs["tools"][0]["function"]["name"] == forced_tool_name
+
+    @pytest.mark.asyncio
+    async def test_fix_also_applies_to_openai_provider(self):
+        """
+        The fix is generalized: all providers convert named tool_choice to
+        "required" + filtered tools.  OpenAI natively supports the dict format
+        too, so the behaviour is semantically identical either way.
+        """
+        from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM
+
+        openai_llm = OpenAICompatibleLLM(
+            provider="openai",
+            api_key="sk-test",
+            base_url="",
+            model="gpt-4o-mini",
+        )
+
+        named_tool_choice = {"type": "function", "function": {"name": "search_mental_models"}}
+        success_response = _make_tool_call_response("search_mental_models", {"query": "test"})
+
+        with patch.object(openai_llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create:
+            mock_create.return_value = success_response
+
+            await openai_llm.call_with_tools(
+                messages=[{"role": "user", "content": "Test"}],
+                tools=REFLECT_TOOLS,
+                tool_choice=named_tool_choice,
+                max_retries=0,
+            )
+
+        sent_kwargs = mock_create.call_args.kwargs
+        # Generalized fix applies to OpenAI too
+        assert sent_kwargs["tool_choice"] == "required"
+        assert len(sent_kwargs["tools"]) == 1
+        assert sent_kwargs["tools"][0]["function"]["name"] == "search_mental_models"