Skip to content

Commit 5e08868

Browse files
HavenDVclaude
HavenDV
and
claude
committed
feat: Add tool calling and embedding tests for all CustomProviders + Azure MEAI
- Tool calling tests for 13 providers (all except Perplexity which lacks support) - EmbeddingGenerator tests for 7 providers (DeepInfra, Together, Mistral, Cohere, Fireworks, Nebius, GitHub Models, Azure) - Azure CustomProvider support in GetAuthorizedChatApi helper Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 03f9bcb commit 5e08868

3 files changed

Lines changed: 303 additions & 2 deletions

File tree

‎src/tests/OpenAI.IntegrationTests/Tests.ChatClient.CustomProviders.cs‎

Lines changed: 140 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ namespace tryAGI.OpenAI.IntegrationTests;
88
/// </summary>
99
public partial class Tests
1010
{
11+
// ═══════════════════════════════════════════════════════════════
12+
// Shared helpers
13+
// ═══════════════════════════════════════════════════════════════
14+
1115
private async Task ChatClient_CustomProvider_GetResponseAsync(CustomProvider provider)
1216
{
1317
var (api, model) = GetAuthorizedChatApi(provider);
@@ -43,7 +47,89 @@ [new Meai.ChatMessage(Meai.ChatRole.User, "Count from 1 to 5.")],
4347
Console.WriteLine($"[{provider}] Got {updates.Count} streaming updates");
4448
}
4549

46-
// --- DeepInfra ---
50+
private async Task ChatClient_CustomProvider_ToolCallingAsync(CustomProvider provider)
51+
{
52+
var (api, model) = GetAuthorizedChatApi(provider);
53+
using var _ = api;
54+
Meai.IChatClient chatClient = api;
55+
56+
var tool = Meai.AIFunctionFactory.Create(
57+
(string city) => city switch
58+
{
59+
"Paris" => "22°C, sunny",
60+
"London" => "15°C, cloudy",
61+
_ => "Unknown",
62+
},
63+
name: "GetWeather",
64+
description: "Gets the current weather for a city");
65+
66+
var response = await chatClient.GetResponseAsync(
67+
[new Meai.ChatMessage(Meai.ChatRole.User, "What's the weather in Paris?")],
68+
new Meai.ChatOptions
69+
{
70+
ModelId = model,
71+
Tools = [tool],
72+
});
73+
74+
response.Should().NotBeNull();
75+
response.FinishReason.Should().Be(Meai.ChatFinishReason.ToolCalls);
76+
77+
var functionCall = response.Messages
78+
.SelectMany(m => m.Contents)
79+
.OfType<Meai.FunctionCallContent>()
80+
.FirstOrDefault();
81+
82+
functionCall.Should().NotBeNull();
83+
functionCall!.Name.Should().Be("GetWeather");
84+
Console.WriteLine($"[{provider}] Tool call: {functionCall.Name}({string.Join(", ", functionCall.Arguments?.Select(kv => $"{kv.Key}={kv.Value}") ?? [])})");
85+
}
86+
87+
// ═══════════════════════════════════════════════════════════════
88+
// Azure
89+
// ═══════════════════════════════════════════════════════════════
90+
91+
[TestMethod]
92+
public async Task ChatClient_Azure_GetResponse()
93+
{
94+
var (api, model) = GetAuthorizedChatApi(CustomProvider.Azure);
95+
using var _ = api;
96+
Meai.IChatClient chatClient = api;
97+
98+
var response = await chatClient.GetResponseAsync(
99+
[new Meai.ChatMessage(Meai.ChatRole.User, "Say hello in exactly 3 words.")],
100+
new Meai.ChatOptions { ModelId = model });
101+
102+
response.Should().NotBeNull();
103+
response.Messages.Should().NotBeEmpty();
104+
Console.WriteLine($"[Azure] {response.Messages[0].Text}");
105+
}
106+
107+
[TestMethod]
108+
public async Task ChatClient_Azure_Streaming()
109+
{
110+
var (api, model) = GetAuthorizedChatApi(CustomProvider.Azure);
111+
using var _ = api;
112+
Meai.IChatClient chatClient = api;
113+
114+
var updates = new List<Meai.ChatResponseUpdate>();
115+
await foreach (var update in chatClient.GetStreamingResponseAsync(
116+
[new Meai.ChatMessage(Meai.ChatRole.User, "Count from 1 to 5.")],
117+
new Meai.ChatOptions { ModelId = model }))
118+
{
119+
updates.Add(update);
120+
}
121+
122+
updates.Should().NotBeEmpty();
123+
Console.WriteLine($"[Azure] Got {updates.Count} streaming updates");
124+
}
125+
126+
[TestMethod]
127+
public Task ChatClient_Azure_ToolCalling() =>
128+
ChatClient_CustomProvider_ToolCallingAsync(CustomProvider.Azure);
129+
130+
// ═══════════════════════════════════════════════════════════════
131+
// DeepInfra
132+
// ═══════════════════════════════════════════════════════════════
47133

48134
[TestMethod]
49135
public Task ChatClient_DeepInfra_GetResponse() =>
@@ -53,6 +139,10 @@ public Task ChatClient_DeepInfra_GetResponse() =>
53139
public Task ChatClient_DeepInfra_Streaming() =>
54140
ChatClient_CustomProvider_StreamingAsync(CustomProvider.DeepInfra);
55141

142+
[TestMethod]
143+
public Task ChatClient_DeepInfra_ToolCalling() =>
144+
ChatClient_CustomProvider_ToolCallingAsync(CustomProvider.DeepInfra);
145+
56146
// --- Groq ---
57147

58148
[TestMethod]
@@ -63,6 +153,10 @@ public Task ChatClient_Groq_GetResponse() =>
63153
public Task ChatClient_Groq_Streaming() =>
64154
ChatClient_CustomProvider_StreamingAsync(CustomProvider.Groq);
65155

156+
[TestMethod]
157+
public Task ChatClient_Groq_ToolCalling() =>
158+
ChatClient_CustomProvider_ToolCallingAsync(CustomProvider.Groq);
159+
66160
// --- OpenRouter ---
67161

68162
[TestMethod]
@@ -73,6 +167,10 @@ public Task ChatClient_OpenRouter_GetResponse() =>
73167
public Task ChatClient_OpenRouter_Streaming() =>
74168
ChatClient_CustomProvider_StreamingAsync(CustomProvider.OpenRouter);
75169

170+
[TestMethod]
171+
public Task ChatClient_OpenRouter_ToolCalling() =>
172+
ChatClient_CustomProvider_ToolCallingAsync(CustomProvider.OpenRouter);
173+
76174
// --- Fireworks ---
77175

78176
[TestMethod]
@@ -83,6 +181,10 @@ public Task ChatClient_Fireworks_GetResponse() =>
83181
public Task ChatClient_Fireworks_Streaming() =>
84182
ChatClient_CustomProvider_StreamingAsync(CustomProvider.Fireworks);
85183

184+
[TestMethod]
185+
public Task ChatClient_Fireworks_ToolCalling() =>
186+
ChatClient_CustomProvider_ToolCallingAsync(CustomProvider.Fireworks);
187+
86188
// --- Together ---
87189

88190
[TestMethod]
@@ -93,6 +195,10 @@ public Task ChatClient_Together_GetResponse() =>
93195
public Task ChatClient_Together_Streaming() =>
94196
ChatClient_CustomProvider_StreamingAsync(CustomProvider.Together);
95197

198+
[TestMethod]
199+
public Task ChatClient_Together_ToolCalling() =>
200+
ChatClient_CustomProvider_ToolCallingAsync(CustomProvider.Together);
201+
96202
// --- DeepSeek ---
97203

98204
[TestMethod]
@@ -103,6 +209,10 @@ public Task ChatClient_DeepSeek_GetResponse() =>
103209
public Task ChatClient_DeepSeek_Streaming() =>
104210
ChatClient_CustomProvider_StreamingAsync(CustomProvider.DeepSeek);
105211

212+
[TestMethod]
213+
public Task ChatClient_DeepSeek_ToolCalling() =>
214+
ChatClient_CustomProvider_ToolCallingAsync(CustomProvider.DeepSeek);
215+
106216
// --- XAi (Grok) ---
107217

108218
[TestMethod]
@@ -113,7 +223,11 @@ public Task ChatClient_XAi_GetResponse() =>
113223
public Task ChatClient_XAi_Streaming() =>
114224
ChatClient_CustomProvider_StreamingAsync(CustomProvider.XAi);
115225

116-
// --- Perplexity ---
226+
[TestMethod]
227+
public Task ChatClient_XAi_ToolCalling() =>
228+
ChatClient_CustomProvider_ToolCallingAsync(CustomProvider.XAi);
229+
230+
// --- Perplexity (no tool calling support) ---
117231

118232
[TestMethod]
119233
public Task ChatClient_Perplexity_GetResponse() =>
@@ -133,6 +247,10 @@ public Task ChatClient_SambaNova_GetResponse() =>
133247
public Task ChatClient_SambaNova_Streaming() =>
134248
ChatClient_CustomProvider_StreamingAsync(CustomProvider.SambaNova);
135249

250+
[TestMethod]
251+
public Task ChatClient_SambaNova_ToolCalling() =>
252+
ChatClient_CustomProvider_ToolCallingAsync(CustomProvider.SambaNova);
253+
136254
// --- Mistral ---
137255

138256
[TestMethod]
@@ -143,6 +261,10 @@ public Task ChatClient_Mistral_GetResponse() =>
143261
public Task ChatClient_Mistral_Streaming() =>
144262
ChatClient_CustomProvider_StreamingAsync(CustomProvider.Mistral);
145263

264+
[TestMethod]
265+
public Task ChatClient_Mistral_ToolCalling() =>
266+
ChatClient_CustomProvider_ToolCallingAsync(CustomProvider.Mistral);
267+
146268
// --- Cerebras ---
147269

148270
[TestMethod]
@@ -153,6 +275,10 @@ public Task ChatClient_Cerebras_GetResponse() =>
153275
public Task ChatClient_Cerebras_Streaming() =>
154276
ChatClient_CustomProvider_StreamingAsync(CustomProvider.Cerebras);
155277

278+
[TestMethod]
279+
public Task ChatClient_Cerebras_ToolCalling() =>
280+
ChatClient_CustomProvider_ToolCallingAsync(CustomProvider.Cerebras);
281+
156282
// --- Cohere ---
157283

158284
[TestMethod]
@@ -163,6 +289,10 @@ public Task ChatClient_Cohere_GetResponse() =>
163289
public Task ChatClient_Cohere_Streaming() =>
164290
ChatClient_CustomProvider_StreamingAsync(CustomProvider.Cohere);
165291

292+
[TestMethod]
293+
public Task ChatClient_Cohere_ToolCalling() =>
294+
ChatClient_CustomProvider_ToolCallingAsync(CustomProvider.Cohere);
295+
166296
// --- Nebius ---
167297

168298
[TestMethod]
@@ -173,6 +303,10 @@ public Task ChatClient_Nebius_GetResponse() =>
173303
public Task ChatClient_Nebius_Streaming() =>
174304
ChatClient_CustomProvider_StreamingAsync(CustomProvider.Nebius);
175305

306+
[TestMethod]
307+
public Task ChatClient_Nebius_ToolCalling() =>
308+
ChatClient_CustomProvider_ToolCallingAsync(CustomProvider.Nebius);
309+
176310
// --- GitHub Models ---
177311

178312
[TestMethod]
@@ -182,4 +316,8 @@ public Task ChatClient_GitHub_GetResponse() =>
182316
[TestMethod]
183317
public Task ChatClient_GitHub_Streaming() =>
184318
ChatClient_CustomProvider_StreamingAsync(CustomProvider.GitHub);
319+
320+
[TestMethod]
321+
public Task ChatClient_GitHub_ToolCalling() =>
322+
ChatClient_CustomProvider_ToolCallingAsync(CustomProvider.GitHub);
185323
}

‎src/tests/OpenAI.IntegrationTests/Tests.EmbeddingGenerator.CustomProviders.cs‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
using Meai = Microsoft.Extensions.AI;
2+
3+
namespace tryAGI.OpenAI.IntegrationTests;
4+
5+
/// <summary>
6+
/// Tests MEAI IEmbeddingGenerator interface across CustomProviders that support embeddings.
7+
/// Each provider requires its own API key environment variable.
8+
/// </summary>
9+
public partial class Tests
10+
{
11+
private async Task EmbeddingGenerator_CustomProvider_GenerateAsync(
12+
CustomProvider provider, string embeddingModel)
13+
{
14+
var (api, _) = GetAuthorizedChatApi(provider);
15+
using var _ = api;
16+
Meai.IEmbeddingGenerator<string, Meai.Embedding<float>> generator = api;
17+
18+
var result = await generator.GenerateAsync(
19+
["Hello, world!"],
20+
new Meai.EmbeddingGenerationOptions { ModelId = embeddingModel });
21+
22+
result.Should().NotBeNull();
23+
result.Should().HaveCount(1);
24+
result[0].Vector.Length.Should().BeGreaterThan(0);
25+
Console.WriteLine($"[{provider}] Embedding dimensions: {result[0].Vector.Length}");
26+
}
27+
28+
private async Task EmbeddingGenerator_CustomProvider_BatchAsync(
29+
CustomProvider provider, string embeddingModel)
30+
{
31+
var (api, _) = GetAuthorizedChatApi(provider);
32+
using var _ = api;
33+
Meai.IEmbeddingGenerator<string, Meai.Embedding<float>> generator = api;
34+
35+
var result = await generator.GenerateAsync(
36+
["First sentence.", "Second sentence.", "Third sentence."],
37+
new Meai.EmbeddingGenerationOptions { ModelId = embeddingModel });
38+
39+
result.Should().HaveCount(3);
40+
foreach (var embedding in result)
41+
{
42+
embedding.Vector.Length.Should().BeGreaterThan(0);
43+
}
44+
Console.WriteLine($"[{provider}] Batch: {result.Count} embeddings, {result[0].Vector.Length} dimensions");
45+
}
46+
47+
// --- DeepInfra ---
48+
49+
[TestMethod]
50+
public Task EmbeddingGenerator_DeepInfra_Generate() =>
51+
EmbeddingGenerator_CustomProvider_GenerateAsync(
52+
CustomProvider.DeepInfra, "BAAI/bge-en-icl");
53+
54+
[TestMethod]
55+
public Task EmbeddingGenerator_DeepInfra_Batch() =>
56+
EmbeddingGenerator_CustomProvider_BatchAsync(
57+
CustomProvider.DeepInfra, "BAAI/bge-en-icl");
58+
59+
// --- Together ---
60+
61+
[TestMethod]
62+
public Task EmbeddingGenerator_Together_Generate() =>
63+
EmbeddingGenerator_CustomProvider_GenerateAsync(
64+
CustomProvider.Together, "BAAI/bge-large-en-v1.5");
65+
66+
[TestMethod]
67+
public Task EmbeddingGenerator_Together_Batch() =>
68+
EmbeddingGenerator_CustomProvider_BatchAsync(
69+
CustomProvider.Together, "BAAI/bge-large-en-v1.5");
70+
71+
// --- Mistral ---
72+
73+
[TestMethod]
74+
public Task EmbeddingGenerator_Mistral_Generate() =>
75+
EmbeddingGenerator_CustomProvider_GenerateAsync(
76+
CustomProvider.Mistral, "mistral-embed");
77+
78+
[TestMethod]
79+
public Task EmbeddingGenerator_Mistral_Batch() =>
80+
EmbeddingGenerator_CustomProvider_BatchAsync(
81+
CustomProvider.Mistral, "mistral-embed");
82+
83+
// --- Cohere ---
84+
85+
[TestMethod]
86+
public Task EmbeddingGenerator_Cohere_Generate() =>
87+
EmbeddingGenerator_CustomProvider_GenerateAsync(
88+
CustomProvider.Cohere, "embed-english-v3.0");
89+
90+
[TestMethod]
91+
public Task EmbeddingGenerator_Cohere_Batch() =>
92+
EmbeddingGenerator_CustomProvider_BatchAsync(
93+
CustomProvider.Cohere, "embed-english-v3.0");
94+
95+
// --- Fireworks ---
96+
97+
[TestMethod]
98+
public Task EmbeddingGenerator_Fireworks_Generate() =>
99+
EmbeddingGenerator_CustomProvider_GenerateAsync(
100+
CustomProvider.Fireworks, "nomic-ai/nomic-embed-text-v1.5");
101+
102+
[TestMethod]
103+
public Task EmbeddingGenerator_Fireworks_Batch() =>
104+
EmbeddingGenerator_CustomProvider_BatchAsync(
105+
CustomProvider.Fireworks, "nomic-ai/nomic-embed-text-v1.5");
106+
107+
// --- Nebius ---
108+
109+
[TestMethod]
110+
public Task EmbeddingGenerator_Nebius_Generate() =>
111+
EmbeddingGenerator_CustomProvider_GenerateAsync(
112+
CustomProvider.Nebius, "BAAI/bge-en-icl");
113+
114+
[TestMethod]
115+
public Task EmbeddingGenerator_Nebius_Batch() =>
116+
EmbeddingGenerator_CustomProvider_BatchAsync(
117+
CustomProvider.Nebius, "BAAI/bge-en-icl");
118+
119+
// --- Azure ---
120+
121+
[TestMethod]
122+
public async Task EmbeddingGenerator_Azure_Generate()
123+
{
124+
var (api, _) = GetAuthorizedChatApi(CustomProvider.Azure);
125+
using var _ = api;
126+
Meai.IEmbeddingGenerator<string, Meai.Embedding<float>> generator = api;
127+
128+
var embeddingModel = Environment.GetEnvironmentVariable("AZURE_OPENAI_EMBEDDING_MODEL")
129+
?? "text-embedding-3-small";
130+
131+
var result = await generator.GenerateAsync(
132+
["Hello, world!"],
133+
new Meai.EmbeddingGenerationOptions { ModelId = embeddingModel });
134+
135+
result.Should().NotBeNull();
136+
result.Should().HaveCount(1);
137+
result[0].Vector.Length.Should().BeGreaterThan(0);
138+
Console.WriteLine($"[Azure] Embedding dimensions: {result[0].Vector.Length}");
139+
}
140+
141+
// --- GitHub Models ---
142+
143+
[TestMethod]
144+
public Task EmbeddingGenerator_GitHub_Generate() =>
145+
EmbeddingGenerator_CustomProvider_GenerateAsync(
146+
CustomProvider.GitHub, "text-embedding-3-small");
147+
148+
[TestMethod]
149+
public Task EmbeddingGenerator_GitHub_Batch() =>
150+
EmbeddingGenerator_CustomProvider_BatchAsync(
151+
CustomProvider.GitHub, "text-embedding-3-small");
152+
}

0 commit comments

Comments
 (0)