Merge pull request #366 from CyberAgentAILab/fix/assert-genai-edit-square-output

Fix structured json fallback

Author: proboscis

packages/openai_support/src/pinjected_openai/openrouter/README.md

@@ -2,6 +2,17 @@
 
 This document tracks the availability of `providers` and `top_provider` fields across various models in the OpenRouter API.
 
+## Structured Output Observations (November 2025)
+
+Recent live tests against OpenRouter's `/chat/completions` endpoint with `response_format` enabled show the following behaviour:
+
+- ✅ `openai/gpt-4o` and `openai/gpt-4o-mini` return valid JSON payloads (e.g. `{"answer":"Paris","confidence":0.99}`) that can be parsed directly.
+- ✅ `google/gemini-2.0-flash-001` also returns well-formed JSON that satisfies our `SimpleResponse` schema.
+- ⚠️ `openai/gpt-5`, `openai/gpt-5-nano`, and `openai/gpt-5-mini` respond with an **empty string** for `choices[0].message.content`, even though the models advertise `response_format`/`structured_outputs` support. Downstream parsing therefore fails with a validation error.
+- ❌ `openai/gpt-4-turbo` currently rejects structured-output requests with `HTTP 400 Bad Request` when `response_format` is supplied.
+
+Until OpenRouter/OpenAI fix the GPT-5 and GPT-4-turbo behaviour, our e2e tests intentionally fail when they encounter these empty-string / error responses.
+
 ## Field Availability by Model
 
 | Model | has_providers | has_top_provider |
@@ -136,4 +147,4 @@ The API shows a clear transition pattern:
         "can_stream": true
     }
 ]
-```
+```

packages/openai_support/src/pinjected_openai/openrouter/instances.py

@@ -20,8 +20,7 @@ class NoEndpointsFoundError(Exception):
 class StructuredLLM(Protocol):
     async def __call__(
         self, text: str, images=None, response_format: type[BaseModel] | None = None
-    ):
-        pass
+    ) -> BaseModel: ...
 
 
 class ASllmOpenrouterProtocol(Protocol):
@@ -98,6 +97,9 @@ async def a_sllm_openrouter(  # noqa: PINJ045
 a_cached_structured_llm__claude_sonnet_3_5: IProxy[StructuredLLM] = async_cached(
     lzma_sqlite(injected("cache_root_path") / "claude_sonnet_3_5.sqlite")
 )(Injected.partial(a_sllm_openrouter, model="anthropic/claude-3.5-sonnet"))
+a_cached_structured_llm__gpt4o_mini: IProxy[StructuredLLM] = async_cached(
+    lzma_sqlite(injected("cache_root_path") / "gpt4o_mini_structured.sqlite")
+)(Injected.partial(a_sllm_openrouter, model="openai/gpt-4o-mini"))
 a_cached_sllm_gpt4o__openrouter: IProxy = async_cached(
     sqlite_dict(injected("cache_root_path") / "gpt4o.sqlite")
 )(Injected.partial(a_openrouter_base_chat_completion, model="openai/gpt-4o"))
@@ -120,7 +122,7 @@ async def a_sllm_openrouter(  # noqa: PINJ045
 __design__ = design(
     overrides=design(
         a_llm_for_json_schema_example=a_cached_sllm_gpt4o__openrouter,
-        a_structured_llm_for_json_fix=a_cached_sllm_gpt4o_mini__openrouter,
+        a_structured_llm_for_json_fix=a_cached_structured_llm__gpt4o_mini,
         # openai_config=injected('openai_config__personal')
     )
 )

packages/openai_support/src/pinjected_openai/openrouter/instances.pyi

@@ -1,4 +1,4 @@
-from typing import overload
+from typing import overload, Protocol
 from pydantic import BaseModel
 from pinjected import IProxy
 
@@ -21,13 +21,21 @@ a_cached_structured_llm__gemini_flash_2_0: IProxy[StructuredLLM]
 a_cached_structured_llm__deepseek_chat: IProxy[StructuredLLM]
 a_cached_structured_llm__gemini_flash_thinking_2_0: IProxy[StructuredLLM]
 a_cached_structured_llm__claude_sonnet_3_5: IProxy[StructuredLLM]
+a_cached_structured_llm__gpt4o_mini: IProxy[StructuredLLM]
 a_cached_sllm_gpt4o__openrouter: IProxy
 a_cached_sllm_gpt4o_mini__openrouter: IProxy
 test_cached_sllm_gpt4o_mini: IProxy
 test_cached_sllm_gpt4o: IProxy
 test_gemini_flash_2_0_structured: IProxy
 
-class StructuredLLM: ...
+class StructuredLLM(Protocol):
+    async def __call__(
+        self,
+        text: str,
+        images=...,
+        response_format: type[BaseModel] | None = ...,
+    ) -> BaseModel: ...
+
 class NoEndpointsFoundError: ...
 
 # Additional symbols: