Skip to content

Commit cc2588b

Browse files
daewoongohdaewoongoh
authored
fix(litellm): preserve reasoning_content for known reasoning model families (#899)
* fix(litellm): preserve reasoning_content for known reasoning model families LiteLLM's /v1/model/info never reports reasoning capability flags, so infer preserveReasoning from the model alias/routed-model name and use convertToR1Format to keep reasoning_content intact across tool-call turns. Signed-off-by: daewoongoh <dw.oh@samsung.com> * test(litellm): cover preserveReasoning inference and message conversion branching Add unit coverage for LITELLM_PRESERVE_REASONING_PATTERN matching per model family, getLiteLLMModels setting preserveReasoning on matched models, and LiteLLMHandler.createMessage branching between convertToR1Format and convertToOpenAiMessages based on info.preserveReasoning. Signed-off-by: daewoongoh <dw.oh@samsung.com> * refactor(litellm): replace preserveReasoning regex with an explicit model id list Regex-based family matching could over-match unrelated aliases sharing a substring (e.g. glm-5-flash matching a glm-5 prefix), a gap flagged in PR #899 review. LITELLM_PRESERVE_REASONING_MODEL_IDS now lists the exact model ids that set preserveReasoning: true in their native provider config, matched via isLiteLLMPreserveReasoningModel() against the final segment of model_name/litellm_params.model. Also fixes a test in litellm.spec.ts where the model_name alias itself matched a known id, so the assertion didn't actually prove litellm_params.model was being checked. Signed-off-by: daewoongoh <dw.oh@samsung.com> --------- Signed-off-by: daewoongoh <dw.oh@samsung.com>
1 parent 3024768 commit cc2588b

6 files changed

Lines changed: 400 additions & 3 deletions

File tree

‎packages/types/src/__tests__/lite-llm.test.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import { isLiteLLMPreserveReasoningModel, LITELLM_PRESERVE_REASONING_MODEL_IDS } from "../providers/lite-llm.js"
2+
3+
describe("LiteLLM preserveReasoning model detection", () => {
4+
it("matches every explicitly listed model id", () => {
5+
for (const modelId of LITELLM_PRESERVE_REASONING_MODEL_IDS) {
6+
expect(isLiteLLMPreserveReasoningModel(modelId)).toBe(true)
7+
}
8+
})
9+
10+
it("does not contain duplicate model ids", () => {
11+
expect(new Set(LITELLM_PRESERVE_REASONING_MODEL_IDS).size).toBe(LITELLM_PRESERVE_REASONING_MODEL_IDS.length)
12+
})
13+
14+
it("matches provider-prefixed routed model names by their final segment", () => {
15+
expect(isLiteLLMPreserveReasoningModel("deepseek/deepseek-reasoner")).toBe(true)
16+
expect(isLiteLLMPreserveReasoningModel("bedrock/moonshot.kimi-k2-thinking")).toBe(true)
17+
expect(isLiteLLMPreserveReasoningModel("fireworks_ai/accounts/fireworks/models/kimi-k2p7-code")).toBe(true)
18+
})
19+
20+
it("matches case-insensitively", () => {
21+
expect(isLiteLLMPreserveReasoningModel("MiniMax-M2.7-Highspeed")).toBe(true)
22+
expect(isLiteLLMPreserveReasoningModel("GLM-5.2")).toBe(true)
23+
})
24+
25+
it("does not match model ids that merely contain a known family as a substring", () => {
26+
expect(isLiteLLMPreserveReasoningModel("deepseek-v4-mini")).toBe(false)
27+
expect(isLiteLLMPreserveReasoningModel("mimo-v2.6")).toBe(false)
28+
expect(isLiteLLMPreserveReasoningModel("kimi-k2.6")).toBe(false)
29+
expect(isLiteLLMPreserveReasoningModel("kimi-k2.7-code")).toBe(false)
30+
expect(isLiteLLMPreserveReasoningModel("minimax-m4")).toBe(false)
31+
expect(isLiteLLMPreserveReasoningModel("minimax-m1")).toBe(false)
32+
expect(isLiteLLMPreserveReasoningModel("glm-4.7-flash")).toBe(false)
33+
expect(isLiteLLMPreserveReasoningModel("glm-4.7-flashx")).toBe(false)
34+
expect(isLiteLLMPreserveReasoningModel("glm-5-flash")).toBe(false)
35+
expect(isLiteLLMPreserveReasoningModel("glm-4.8")).toBe(false)
36+
expect(isLiteLLMPreserveReasoningModel("qwen3.5-plus")).toBe(false)
37+
expect(isLiteLLMPreserveReasoningModel("qwen3.7-mini")).toBe(false)
38+
expect(isLiteLLMPreserveReasoningModel("qwen3.6-max")).toBe(false)
39+
})
40+
41+
it("does not match unrelated model names", () => {
42+
expect(isLiteLLMPreserveReasoningModel("gpt-4")).toBe(false)
43+
expect(isLiteLLMPreserveReasoningModel("claude-3-opus")).toBe(false)
44+
expect(isLiteLLMPreserveReasoningModel("")).toBe(false)
45+
expect(isLiteLLMPreserveReasoningModel(undefined)).toBe(false)
46+
})
47+
})

‎packages/types/src/providers/lite-llm.ts‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,83 @@ export const litellmDefaultModelInfo: ModelInfo = {
1313
cacheWritesPrice: 3.75,
1414
cacheReadsPrice: 0.3,
1515
}
16+
17+
/**
18+
* LiteLLM is a gateway: it fronts arbitrary underlying models and its
19+
* `/v1/model/info` response carries no reasoning-related capability flags
20+
* (no `preserveReasoning` equivalent). The underlying model identity is only
21+
* visible as text in the model alias (`model_name`) or the routed target
22+
* (`litellm_params.model`, e.g. `deepseek/deepseek-reasoner`,
23+
* `bedrock/moonshot.kimi-k2-thinking`, `fireworks_ai/.../kimi-k2p7-code`).
24+
*
25+
* Rather than matching model-family substrings with a regex (which can
26+
* over-match unrelated aliases, e.g. a family fragment appearing inside a
27+
* longer unrelated model id), this is an explicit list of the exact model
28+
* ids that set `preserveReasoning: true` in their native provider config
29+
* (see deepseek.ts, mimo.ts, moonshot.ts, bedrock.ts, fireworks.ts, zai.ts,
30+
* minimax.ts, opencode-go.ts). The same behavior is inferred for a
31+
* LiteLLM-routed alias of the same underlying model. Keep this list in sync
32+
* with those registries. This is still best-effort: unrecognized aliases or
33+
* renamed deployments will not match, and callers should treat it as a
34+
* heuristic, not a source of truth.
35+
*/
36+
export const LITELLM_PRESERVE_REASONING_MODEL_IDS = [
37+
// deepseek.ts
38+
"deepseek-v4-flash",
39+
"deepseek-v4-pro",
40+
"deepseek-reasoner",
41+
42+
// mimo.ts, opencode-go.ts
43+
"mimo-v2.5",
44+
"mimo-v2.5-pro",
45+
46+
// moonshot.ts, bedrock.ts, fireworks.ts
47+
"kimi-k2-thinking",
48+
"moonshot.kimi-k2-thinking",
49+
"kimi-k2p7-code",
50+
51+
// zai.ts
52+
"glm-4.7",
53+
"glm-5",
54+
"glm-5.1",
55+
"glm-5.2",
56+
"glm-5-turbo",
57+
58+
// bedrock.ts, minimax.ts, opencode-go.ts
59+
"minimax.minimax-m2",
60+
"minimax-m2",
61+
"minimax-m2-stable",
62+
"minimax-m2.1",
63+
"minimax-m2.1-highspeed",
64+
"minimax-m2.5",
65+
"minimax-m2.5-highspeed",
66+
"minimax-m2.7",
67+
"minimax-m2.7-highspeed",
68+
"minimax-m3",
69+
70+
// opencode-go.ts
71+
"qwen3.6-plus",
72+
"qwen3.7-plus",
73+
"qwen3.7-max",
74+
] as const
75+
76+
const LITELLM_PRESERVE_REASONING_MODEL_ID_SET = new Set<string>(LITELLM_PRESERVE_REASONING_MODEL_IDS)
77+
78+
/**
79+
* Checks whether `modelName` (a LiteLLM alias or routed `litellm_params.model`
80+
* value) identifies a model that requires `preserveReasoning: true`.
81+
* Provider-prefixed routed names (e.g. `deepseek/deepseek-reasoner`,
82+
* `fireworks_ai/accounts/fireworks/models/kimi-k2p7-code`) are matched by
83+
* their final slash-delimited segment.
84+
*/
85+
export function isLiteLLMPreserveReasoningModel(modelName: string | undefined): boolean {
86+
const normalized = modelName?.trim().toLowerCase()
87+
88+
if (!normalized) {
89+
return false
90+
}
91+
92+
const modelId = normalized.split("/").pop()
93+
94+
return modelId !== undefined && LITELLM_PRESERVE_REASONING_MODEL_ID_SET.has(modelId)
95+
}

‎src/api/providers/__tests__/lite-llm.spec.ts‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1124,6 +1124,145 @@ describe("LiteLLMHandler", () => {
11241124
})
11251125
})
11261126

1127+
describe("preserveReasoning message conversion", () => {
1128+
const mockStream = {
1129+
async *[Symbol.asyncIterator]() {
1130+
yield {
1131+
choices: [{ delta: { content: "ok" } }],
1132+
usage: { prompt_tokens: 1, completion_tokens: 1 },
1133+
}
1134+
},
1135+
}
1136+
1137+
it("uses convertToR1Format (merging tool-result text) when the model info sets preserveReasoning", async () => {
1138+
const optionsWithReasoning: ApiHandlerOptions = {
1139+
...mockOptions,
1140+
litellmModelId: "deepseek-reasoner-alias",
1141+
}
1142+
handler = new LiteLLMHandler(optionsWithReasoning)
1143+
1144+
vi.spyOn(handler as any, "fetchModel").mockResolvedValue({
1145+
id: "deepseek-reasoner-alias",
1146+
info: { ...litellmDefaultModelInfo, preserveReasoning: true },
1147+
})
1148+
1149+
const systemPrompt = "You are a helpful assistant"
1150+
const messages: Anthropic.Messages.MessageParam[] = [
1151+
{ role: "user", content: "Hello" },
1152+
{
1153+
role: "assistant",
1154+
content: [
1155+
{ type: "text", text: "I'll help." },
1156+
{ type: "tool_use", id: "toolu_123", name: "read_file", input: { path: "test.txt" } },
1157+
],
1158+
// DeepSeek-style interleaved thinking: must be echoed back in the next request.
1159+
reasoning_content: "Let me check the file first.",
1160+
} as Anthropic.Messages.MessageParam & { reasoning_content: string },
1161+
{
1162+
role: "user",
1163+
content: [
1164+
{ type: "tool_result", tool_use_id: "toolu_123", content: "file contents" },
1165+
{ type: "text", text: "Thanks, continue." },
1166+
],
1167+
},
1168+
]
1169+
1170+
mockCreate.mockReturnValue({
1171+
withResponse: vi.fn().mockResolvedValue({ data: mockStream }),
1172+
})
1173+
1174+
const generator = handler.createMessage(systemPrompt, messages)
1175+
for await (const _chunk of generator) {
1176+
// Consume
1177+
}
1178+
1179+
const createCall = mockCreate.mock.calls[0][0]
1180+
1181+
// convertToR1Format with mergeToolResultText folds the trailing text into the
1182+
// tool message instead of appending a separate user message.
1183+
const toolMessage = createCall.messages.find((msg: any) => msg.role === "tool")
1184+
expect(toolMessage).toBeDefined()
1185+
expect(toolMessage.content).toBe("file contents\n\nThanks, continue.")
1186+
1187+
const trailingUserMessage = createCall.messages.find(
1188+
(msg: any) => msg.role === "user" && msg.content === "Thanks, continue.",
1189+
)
1190+
expect(trailingUserMessage).toBeUndefined()
1191+
1192+
// The whole point of routing through convertToR1Format: reasoning_content
1193+
// must survive on the assistant message so the model doesn't reject the
1194+
// follow-up request for missing prior reasoning.
1195+
const assistantMessage = createCall.messages.find(
1196+
(msg: any) => msg.role === "assistant" && msg.tool_calls?.length > 0,
1197+
)
1198+
expect(assistantMessage).toBeDefined()
1199+
expect(assistantMessage.reasoning_content).toBe("Let me check the file first.")
1200+
})
1201+
1202+
it("uses convertToOpenAiMessages (no merging) when the model info does not set preserveReasoning", async () => {
1203+
vi.spyOn(handler as any, "fetchModel").mockResolvedValue({
1204+
id: litellmDefaultModelId,
1205+
info: { ...litellmDefaultModelInfo, preserveReasoning: undefined },
1206+
})
1207+
1208+
const systemPrompt = "You are a helpful assistant"
1209+
// Task.buildCleanConversationHistory() (src/core/task/Task.ts) already strips the
1210+
// reasoning content block before messages reach this handler when the model's
1211+
// preserveReasoning is not true, so no reasoning_content/reasoning block is present
1212+
// on the assistant message here — this input reflects what the handler actually
1213+
// receives in that case.
1214+
const messages: Anthropic.Messages.MessageParam[] = [
1215+
{ role: "user", content: "Hello" },
1216+
{
1217+
role: "assistant",
1218+
content: [
1219+
{ type: "text", text: "I'll help." },
1220+
{ type: "tool_use", id: "toolu_123", name: "read_file", input: { path: "test.txt" } },
1221+
],
1222+
},
1223+
{
1224+
role: "user",
1225+
content: [
1226+
{ type: "tool_result", tool_use_id: "toolu_123", content: "file contents" },
1227+
{ type: "text", text: "Thanks, continue." },
1228+
],
1229+
},
1230+
]
1231+
1232+
mockCreate.mockReturnValue({
1233+
withResponse: vi.fn().mockResolvedValue({ data: mockStream }),
1234+
})
1235+
1236+
const generator = handler.createMessage(systemPrompt, messages)
1237+
for await (const _chunk of generator) {
1238+
// Consume
1239+
}
1240+
1241+
const createCall = mockCreate.mock.calls[0][0]
1242+
1243+
const toolMessage = createCall.messages.find((msg: any) => msg.role === "tool")
1244+
expect(toolMessage).toBeDefined()
1245+
expect(toolMessage.content).toBe("file contents")
1246+
1247+
const trailingUserMessage = createCall.messages.find(
1248+
(msg: any) =>
1249+
msg.role === "user" &&
1250+
(msg.content === "Thanks, continue." ||
1251+
(Array.isArray(msg.content) &&
1252+
msg.content.some((part: any) => part.text === "Thanks, continue."))),
1253+
)
1254+
expect(trailingUserMessage).toBeDefined()
1255+
1256+
// No reasoning_content is sent to the API in this branch, matching what
1257+
// buildCleanConversationHistory already stripped upstream.
1258+
const assistantMessage = createCall.messages.find(
1259+
(msg: any) => msg.role === "assistant" && msg.tool_calls?.length > 0,
1260+
)
1261+
expect(assistantMessage).toBeDefined()
1262+
expect(assistantMessage.reasoning_content).toBeUndefined()
1263+
})
1264+
})
1265+
11271266
describe("session ID header", () => {
11281267
const mockStream = {
11291268
async *[Symbol.asyncIterator]() {

‎src/api/providers/fetchers/__tests__/litellm.spec.ts‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,4 +697,117 @@ describe("getLiteLLMModels", () => {
697697
description: "model-with-only-max-output-tokens via LiteLLM proxy",
698698
})
699699
})
700+
701+
describe("preserveReasoning inference", () => {
702+
it("sets preserveReasoning: true when only the routed model (not the alias) matches a known reasoning model id", async () => {
703+
const mockResponse = {
704+
data: {
705+
data: [
706+
{
707+
model_name: "my-deepseek-alias",
708+
model_info: {
709+
max_tokens: 8192,
710+
max_input_tokens: 128000,
711+
},
712+
litellm_params: {
713+
model: "deepseek/deepseek-reasoner",
714+
},
715+
},
716+
{
717+
model_name: "my-kimi-alias",
718+
model_info: {
719+
max_tokens: 8192,
720+
max_input_tokens: 128000,
721+
},
722+
litellm_params: {
723+
model: "bedrock/moonshot.kimi-k2-thinking",
724+
},
725+
},
726+
],
727+
},
728+
}
729+
730+
mockedAxios.get.mockResolvedValue(mockResponse)
731+
732+
const result = await getLiteLLMModels("test-api-key", "http://localhost:4000")
733+
734+
expect(result["my-deepseek-alias"]).toMatchObject({ preserveReasoning: true })
735+
expect(result["my-kimi-alias"]).toMatchObject({ preserveReasoning: true })
736+
})
737+
738+
it("omits preserveReasoning when the routed model does not match a known reasoning model id", async () => {
739+
const mockResponse = {
740+
data: {
741+
data: [
742+
{
743+
model_name: "gpt-4-turbo",
744+
model_info: {
745+
max_tokens: 8192,
746+
max_input_tokens: 128000,
747+
},
748+
litellm_params: {
749+
model: "openai/gpt-4-turbo",
750+
},
751+
},
752+
],
753+
},
754+
}
755+
756+
mockedAxios.get.mockResolvedValue(mockResponse)
757+
758+
const result = await getLiteLLMModels("test-api-key", "http://localhost:4000")
759+
760+
expect(result["gpt-4-turbo"]).not.toHaveProperty("preserveReasoning")
761+
})
762+
763+
it("matches against the model alias even when the routed model name does not match", async () => {
764+
const mockResponse = {
765+
data: {
766+
data: [
767+
{
768+
model_name: "glm-5.2",
769+
model_info: {
770+
max_tokens: 8192,
771+
max_input_tokens: 128000,
772+
},
773+
litellm_params: {
774+
model: "zai/some-custom-deployment",
775+
},
776+
},
777+
],
778+
},
779+
}
780+
781+
mockedAxios.get.mockResolvedValue(mockResponse)
782+
783+
const result = await getLiteLLMModels("test-api-key", "http://localhost:4000")
784+
785+
expect(result["glm-5.2"]).toMatchObject({ preserveReasoning: true })
786+
})
787+
788+
it("does not match a model id that merely contains a known family as a substring", async () => {
789+
const mockResponse = {
790+
data: {
791+
data: [
792+
{
793+
model_name: "glm-5-flash",
794+
model_info: {
795+
max_tokens: 8192,
796+
max_input_tokens: 128000,
797+
},
798+
litellm_params: {
799+
model: "zai/glm-5-flash",
800+
},
801+
},
802+
],
803+
},
804+
}
805+
806+
mockedAxios.get.mockResolvedValue(mockResponse)
807+
808+
const result = await getLiteLLMModels("test-api-key", "http://localhost:4000")
809+
810+
expect(result["glm-5-flash"]).not.toHaveProperty("preserveReasoning")
811+
})
812+
})
700813
})

0 commit comments

Comments
 (0)