fix: guard recursive Gemini schema refs

Author: roomote

src/api/providers/__tests__/gemini-handler.spec.ts

@@ -629,5 +629,57 @@ describe("GeminiHandler backend support", () => {
 			})
 			expect(schema.required).toEqual(expect.arrayContaining(["a", "b"]))
 		})
+
+		it("should stop recursive $ref expansion before the sanitized schema becomes cyclic", async () => {
+			const options = { apiProvider: "gemini" } as ApiHandlerOptions
+			const handler = new GeminiHandler(options)
+			const stub = vi.fn().mockReturnValue((async function* () {})())
+			// @ts-ignore access private client
+			handler["client"].models.generateContentStream = stub
+
+			await handler
+				.createMessage("test", [] as any, {
+					taskId: "test-task",
+					tools: [
+						{
+							type: "function",
+							function: {
+								name: "recursive_ref_tool",
+								description: "Tool with recursive $ref",
+								parameters: {
+									type: "object",
+									$defs: {
+										Node: {
+											type: "object",
+											properties: {
+												value: { type: "string" },
+												next: { $ref: "#/$defs/Node" },
+											},
+											required: ["value"],
+										},
+									},
+									properties: {
+										root: { $ref: "#/$defs/Node" },
+									},
+									required: ["root"],
+								},
+							},
+						},
+					],
+				})
+				.next()
+
+			const schema = stub.mock.calls[0][0].config.tools[0].functionDeclarations[0].parametersJsonSchema
+			expect(() => JSON.stringify(schema)).not.toThrow()
+			expect(JSON.stringify(schema)).not.toContain("$ref")
+			expect(schema.properties.root).toEqual({
+				type: "object",
+				properties: {
+					value: { type: "string" },
+					next: {},
+				},
+				required: ["value"],
+			})
+		})
 	})
 })

src/api/providers/gemini.ts

@@ -46,13 +46,17 @@ const GEMINI_SCHEMA_COMPATIBILITY_DROP_KEYS = new Set([
 	"definitions",
 ])
 
-function sanitizeSchemaForGemini(schema: unknown, defs?: Record<string, unknown>): unknown {
+function sanitizeSchemaForGemini(
+	schema: unknown,
+	defs?: Record<string, unknown>,
+	activeRefs: Set<string> = new Set(),
+): unknown {
 	if (!schema || typeof schema !== "object") {
 		return schema
 	}
 
 	if (Array.isArray(schema)) {
-		return schema.map((item) => sanitizeSchemaForGemini(item, defs))
+		return schema.map((item) => sanitizeSchemaForGemini(item, defs, activeRefs))
 	}
 
 	const source = schema as Record<string, unknown>
@@ -68,7 +72,19 @@ function sanitizeSchemaForGemini(schema: unknown, defs?: Record<string, unknown>
 		if (match) {
 			const resolved = resolvedDefs[match[1]]
 			if (resolved !== undefined) {
-				return sanitizeSchemaForGemini(resolved, resolvedDefs)
+				// Recursive MCP schemas are valid JSON Schema but not something Gemini
+				// can consume directly. Stop at the recursive edge so we still send a
+				// finite, serializable schema instead of overflowing the stack.
+				if (activeRefs.has(match[1])) {
+					return {}
+				}
+
+				activeRefs.add(match[1])
+				try {
+					return sanitizeSchemaForGemini(resolved, resolvedDefs, activeRefs)
+				} finally {
+					activeRefs.delete(match[1])
+				}
 			}
 		}
 	}
@@ -84,12 +100,12 @@ function sanitizeSchemaForGemini(schema: unknown, defs?: Record<string, unknown>
 				: true
 		})
 		nullable = nullable || variants.length < composition.length
-		Object.assign(result, sanitizeSchemaForGemini(variants[0] ?? {}, resolvedDefs))
+		Object.assign(result, sanitizeSchemaForGemini(variants[0] ?? {}, resolvedDefs, activeRefs))
 	}
 
 	if (Array.isArray(source.allOf)) {
 		for (const variant of source.allOf) {
-			const sanitized = sanitizeSchemaForGemini(variant, resolvedDefs)
+			const sanitized = sanitizeSchemaForGemini(variant, resolvedDefs, activeRefs)
 			if (sanitized && typeof sanitized === "object" && !Array.isArray(sanitized)) {
 				const s = sanitized as Record<string, unknown>
 				// Deep-merge properties so later allOf fragments don't overwrite
@@ -116,7 +132,7 @@ function sanitizeSchemaForGemini(schema: unknown, defs?: Record<string, unknown>
 		}
 
 		if (key === "properties" && value && typeof value === "object" && !Array.isArray(value)) {
-			const sanitizedProperties = sanitizeSchemaForGemini(value, resolvedDefs)
+			const sanitizedProperties = sanitizeSchemaForGemini(value, resolvedDefs, activeRefs)
 			if (sanitizedProperties && typeof sanitizedProperties === "object" && !Array.isArray(sanitizedProperties)) {
 				result.properties = {
 					...(result.properties as Record<string, unknown> | undefined),
@@ -143,7 +159,7 @@ function sanitizeSchemaForGemini(schema: unknown, defs?: Record<string, unknown>
 			continue
 		}
 
-		result[key] = sanitizeSchemaForGemini(value, resolvedDefs)
+		result[key] = sanitizeSchemaForGemini(value, resolvedDefs, activeRefs)
 	}
 
 	if (nullable) {