Skip to content

Commit 81dccf6

Browse files
HavenDVHavenDV
committed
fix: treat suspended DeepInfra runs as inconclusive
1 parent 8fc0892 commit 81dccf6

4 files changed

Lines changed: 193 additions & 132 deletions

File tree

‎src/tests/OpenAI.IntegrationTests/Tests.Chat.cs‎

Lines changed: 52 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -23,20 +23,23 @@ public partial class Tests
2323
[DataRow(CustomProvider.OllamaCloud)]
2424
public async Task GenerateFiveRandomWords(CustomProvider customProvider)
2525
{
26-
var pair = GetAuthorizedChatApi(customProvider);
27-
using var api = pair.Api;
28-
29-
string response = await api.Chat.CreateChatCompletionAsync(
30-
new CreateChatCompletionRequest
31-
{
32-
CreateChatCompletionRequestVariant2 = new CreateChatCompletionRequestVariant2
26+
string response = await ExecuteProviderAwareAsync(customProvider, async () =>
27+
{
28+
var pair = GetAuthorizedChatApi(customProvider);
29+
using var api = pair.Api;
30+
31+
return await api.Chat.CreateChatCompletionAsync(
32+
new CreateChatCompletionRequest
3333
{
34-
Messages = ["Generate five random words."],
35-
Model = pair.Model,
36-
}
37-
});
38-
response.Should().NotBeEmpty();
34+
CreateChatCompletionRequestVariant2 = new CreateChatCompletionRequestVariant2
35+
{
36+
Messages = ["Generate five random words."],
37+
Model = pair.Model,
38+
}
39+
});
40+
}).ConfigureAwait(false);
3941

42+
response.Should().NotBeEmpty();
4043
Console.WriteLine(response);
4144
}
4245

@@ -60,23 +63,26 @@ public async Task GenerateFiveRandomWords(CustomProvider customProvider)
6063
[DataRow(CustomProvider.OllamaCloud)]
6164
public async Task GenerateFiveRandomWordsAsStream(CustomProvider customProvider)
6265
{
63-
var pair = GetAuthorizedChatApi(customProvider);
64-
using var api = pair.Api;
65-
66-
var enumerable = api.Chat.CreateChatCompletionAsStreamAsync(
67-
new CreateChatCompletionRequest
68-
{
69-
CreateChatCompletionRequestVariant2 = new CreateChatCompletionRequestVariant2
70-
{
71-
Messages = ["Generate five random words."],
72-
Model = pair.Model,
73-
}
74-
});
75-
76-
await foreach (string response in enumerable)
66+
await ExecuteProviderAwareAsync(customProvider, async () =>
7767
{
78-
Console.WriteLine(response);
79-
}
68+
var pair = GetAuthorizedChatApi(customProvider);
69+
using var api = pair.Api;
70+
71+
var enumerable = api.Chat.CreateChatCompletionAsStreamAsync(
72+
new CreateChatCompletionRequest
73+
{
74+
CreateChatCompletionRequestVariant2 = new CreateChatCompletionRequestVariant2
75+
{
76+
Messages = ["Generate five random words."],
77+
Model = pair.Model,
78+
}
79+
});
80+
81+
await foreach (string response in enumerable)
82+
{
83+
Console.WriteLine(response);
84+
}
85+
}).ConfigureAwait(false);
8086
}
8187

8288
[TestMethod]
@@ -96,29 +102,26 @@ public async Task GenerateFiveRandomWordsAsStream(CustomProvider customProvider)
96102
[DataRow(CustomProvider.OllamaCloud)]
97103
public async Task GenerateFiveRandomWordsAsJsonObject(CustomProvider customProvider)
98104
{
99-
var pair = GetAuthorizedChatApi(customProvider);
100-
using var api = pair.Api;
101-
102-
string response = await api.Chat.CreateChatCompletionAsync(
103-
new CreateChatCompletionRequest
104-
{
105-
CreateChatCompletionRequestVariant2 = new CreateChatCompletionRequestVariant2
105+
string response = await ExecuteProviderAwareAsync(customProvider, async () =>
106+
{
107+
var pair = GetAuthorizedChatApi(customProvider);
108+
using var api = pair.Api;
109+
110+
return await api.Chat.CreateChatCompletionAsync(
111+
new CreateChatCompletionRequest
106112
{
107-
Messages = ["Generate five random words as json."],
108-
Model = pair.Model,
109-
ResponseFormat = new ResponseFormatJsonObject
113+
CreateChatCompletionRequestVariant2 = new CreateChatCompletionRequestVariant2
110114
{
111-
Type = ResponseFormatJsonObjectType.JsonObject,
112-
},
113-
}
114-
});
115-
// messages: ["Generate five random words as json."],
116-
// model: pair.Model,
117-
// responseFormat: new ResponseFormatJsonObject
118-
// {
119-
// Type = ResponseFormatJsonObjectType.JsonObject,
120-
// },
121-
// user: "tryAGI.OpenAI.IntegrationTests.Tests.GenerateSomeJson");
115+
Messages = ["Generate five random words as json."],
116+
Model = pair.Model,
117+
ResponseFormat = new ResponseFormatJsonObject
118+
{
119+
Type = ResponseFormatJsonObjectType.JsonObject,
120+
},
121+
}
122+
});
123+
}).ConfigureAwait(false);
124+
122125
response.Should().NotBeEmpty();
123126

124127
Console.WriteLine(response);

‎src/tests/OpenAI.IntegrationTests/Tests.ChatClient.CustomProviders.cs‎

Lines changed: 67 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -15,74 +15,83 @@ public partial class Tests
1515

1616
private async Task ChatClient_CustomProvider_GetResponseAsync(CustomProvider provider)
1717
{
18-
var (api, model) = GetAuthorizedChatApi(provider);
19-
using var _ = api;
20-
Meai.IChatClient chatClient = api;
21-
22-
var response = await chatClient.GetResponseAsync(
23-
[new Meai.ChatMessage(Meai.ChatRole.User, "Say hello in exactly 3 words.")],
24-
new Meai.ChatOptions { ModelId = model });
25-
26-
response.Should().NotBeNull();
27-
response.Messages.Should().NotBeEmpty();
28-
var text = response.Messages[0].Text;
29-
text.Should().NotBeNullOrWhiteSpace();
30-
Console.WriteLine($"[{provider}] {text}");
18+
await ExecuteProviderAwareAsync(provider, async () =>
19+
{
20+
var (api, model) = GetAuthorizedChatApi(provider);
21+
using var _ = api;
22+
Meai.IChatClient chatClient = api;
23+
24+
var response = await chatClient.GetResponseAsync(
25+
[new Meai.ChatMessage(Meai.ChatRole.User, "Say hello in exactly 3 words.")],
26+
new Meai.ChatOptions { ModelId = model });
27+
28+
response.Should().NotBeNull();
29+
response.Messages.Should().NotBeEmpty();
30+
var text = response.Messages[0].Text;
31+
text.Should().NotBeNullOrWhiteSpace();
32+
Console.WriteLine($"[{provider}] {text}");
33+
}).ConfigureAwait(false);
3134
}
3235

3336
private async Task ChatClient_CustomProvider_StreamingAsync(CustomProvider provider)
3437
{
35-
var (api, model) = GetAuthorizedChatApi(provider);
36-
using var _ = api;
37-
Meai.IChatClient chatClient = api;
38-
39-
var updates = new List<Meai.ChatResponseUpdate>();
40-
await foreach (var update in chatClient.GetStreamingResponseAsync(
41-
[new Meai.ChatMessage(Meai.ChatRole.User, "Count from 1 to 5.")],
42-
new Meai.ChatOptions { ModelId = model }))
38+
await ExecuteProviderAwareAsync(provider, async () =>
4339
{
44-
updates.Add(update);
45-
}
40+
var (api, model) = GetAuthorizedChatApi(provider);
41+
using var _ = api;
42+
Meai.IChatClient chatClient = api;
43+
44+
var updates = new List<Meai.ChatResponseUpdate>();
45+
await foreach (var update in chatClient.GetStreamingResponseAsync(
46+
[new Meai.ChatMessage(Meai.ChatRole.User, "Count from 1 to 5.")],
47+
new Meai.ChatOptions { ModelId = model }))
48+
{
49+
updates.Add(update);
50+
}
4651

47-
updates.Should().NotBeEmpty();
48-
Console.WriteLine($"[{provider}] Got {updates.Count} streaming updates");
52+
updates.Should().NotBeEmpty();
53+
Console.WriteLine($"[{provider}] Got {updates.Count} streaming updates");
54+
}).ConfigureAwait(false);
4955
}
5056

5157
private async Task ChatClient_CustomProvider_ToolCallingAsync(CustomProvider provider)
5258
{
53-
var (api, model) = GetAuthorizedChatApi(provider);
54-
using var _ = api;
55-
Meai.IChatClient chatClient = api;
56-
57-
var tool = Meai.AIFunctionFactory.Create(
58-
(string city) => city switch
59-
{
60-
"Paris" => "22°C, sunny",
61-
"London" => "15°C, cloudy",
62-
_ => "Unknown",
63-
},
64-
name: "GetWeather",
65-
description: "Gets the current weather for a city");
66-
67-
var response = await chatClient.GetResponseAsync(
68-
[new Meai.ChatMessage(Meai.ChatRole.User, "What's the weather in Paris?")],
69-
new Meai.ChatOptions
70-
{
71-
ModelId = model,
72-
Tools = [tool],
73-
});
74-
75-
response.Should().NotBeNull();
76-
response.FinishReason.Should().Be(Meai.ChatFinishReason.ToolCalls);
77-
78-
var functionCall = response.Messages
79-
.SelectMany(m => m.Contents)
80-
.OfType<Meai.FunctionCallContent>()
81-
.FirstOrDefault();
82-
83-
functionCall.Should().NotBeNull();
84-
functionCall!.Name.Should().Be("GetWeather");
85-
Console.WriteLine($"[{provider}] Tool call: {functionCall.Name}({string.Join(", ", functionCall.Arguments?.Select(kv => $"{kv.Key}={kv.Value}") ?? [])})");
59+
await ExecuteProviderAwareAsync(provider, async () =>
60+
{
61+
var (api, model) = GetAuthorizedChatApi(provider);
62+
using var _ = api;
63+
Meai.IChatClient chatClient = api;
64+
65+
var tool = Meai.AIFunctionFactory.Create(
66+
(string city) => city switch
67+
{
68+
"Paris" => "22°C, sunny",
69+
"London" => "15°C, cloudy",
70+
_ => "Unknown",
71+
},
72+
name: "GetWeather",
73+
description: "Gets the current weather for a city");
74+
75+
var response = await chatClient.GetResponseAsync(
76+
[new Meai.ChatMessage(Meai.ChatRole.User, "What's the weather in Paris?")],
77+
new Meai.ChatOptions
78+
{
79+
ModelId = model,
80+
Tools = [tool],
81+
});
82+
83+
response.Should().NotBeNull();
84+
response.FinishReason.Should().Be(Meai.ChatFinishReason.ToolCalls);
85+
86+
var functionCall = response.Messages
87+
.SelectMany(m => m.Contents)
88+
.OfType<Meai.FunctionCallContent>()
89+
.FirstOrDefault();
90+
91+
functionCall.Should().NotBeNull();
92+
functionCall!.Name.Should().Be("GetWeather");
93+
Console.WriteLine($"[{provider}] Tool call: {functionCall.Name}({string.Join(", ", functionCall.Arguments?.Select(kv => $"{kv.Key}={kv.Value}") ?? [])})");
94+
}).ConfigureAwait(false);
8695
}
8796

8897
// ═══════════════════════════════════════════════════════════════

‎src/tests/OpenAI.IntegrationTests/Tests.EmbeddingGenerator.CustomProviders.cs‎

Lines changed: 31 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -12,37 +12,43 @@ public partial class Tests
1212
private async Task EmbeddingGenerator_CustomProvider_GenerateAsync(
1313
CustomProvider provider, string embeddingModel)
1414
{
15-
var (api, _) = GetAuthorizedChatApi(provider);
16-
using var _ = api;
17-
Meai.IEmbeddingGenerator<string, Meai.Embedding<float>> generator = api;
18-
19-
var result = await generator.GenerateAsync(
20-
["Hello, world!"],
21-
new Meai.EmbeddingGenerationOptions { ModelId = embeddingModel });
22-
23-
result.Should().NotBeNull();
24-
result.Should().HaveCount(1);
25-
result[0].Vector.Length.Should().BeGreaterThan(0);
26-
Console.WriteLine($"[{provider}] Embedding dimensions: {result[0].Vector.Length}");
15+
await ExecuteProviderAwareAsync(provider, async () =>
16+
{
17+
var (api, _) = GetAuthorizedChatApi(provider);
18+
using var _ = api;
19+
Meai.IEmbeddingGenerator<string, Meai.Embedding<float>> generator = api;
20+
21+
var result = await generator.GenerateAsync(
22+
["Hello, world!"],
23+
new Meai.EmbeddingGenerationOptions { ModelId = embeddingModel });
24+
25+
result.Should().NotBeNull();
26+
result.Should().HaveCount(1);
27+
result[0].Vector.Length.Should().BeGreaterThan(0);
28+
Console.WriteLine($"[{provider}] Embedding dimensions: {result[0].Vector.Length}");
29+
}).ConfigureAwait(false);
2730
}
2831

2932
private async Task EmbeddingGenerator_CustomProvider_BatchAsync(
3033
CustomProvider provider, string embeddingModel)
3134
{
32-
var (api, _) = GetAuthorizedChatApi(provider);
33-
using var _ = api;
34-
Meai.IEmbeddingGenerator<string, Meai.Embedding<float>> generator = api;
35-
36-
var result = await generator.GenerateAsync(
37-
["First sentence.", "Second sentence.", "Third sentence."],
38-
new Meai.EmbeddingGenerationOptions { ModelId = embeddingModel });
39-
40-
result.Should().HaveCount(3);
41-
foreach (var embedding in result)
35+
await ExecuteProviderAwareAsync(provider, async () =>
4236
{
43-
embedding.Vector.Length.Should().BeGreaterThan(0);
44-
}
45-
Console.WriteLine($"[{provider}] Batch: {result.Count} embeddings, {result[0].Vector.Length} dimensions");
37+
var (api, _) = GetAuthorizedChatApi(provider);
38+
using var _ = api;
39+
Meai.IEmbeddingGenerator<string, Meai.Embedding<float>> generator = api;
40+
41+
var result = await generator.GenerateAsync(
42+
["First sentence.", "Second sentence.", "Third sentence."],
43+
new Meai.EmbeddingGenerationOptions { ModelId = embeddingModel });
44+
45+
result.Should().HaveCount(3);
46+
foreach (var embedding in result)
47+
{
48+
embedding.Vector.Length.Should().BeGreaterThan(0);
49+
}
50+
Console.WriteLine($"[{provider}] Batch: {result.Count} embeddings, {result[0].Vector.Length} dimensions");
51+
}).ConfigureAwait(false);
4652
}
4753

4854
// --- DeepInfra ---

‎src/tests/OpenAI.IntegrationTests/Tests.Helpers.cs‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,49 @@ namespace tryAGI.OpenAI.IntegrationTests;
33
[TestClass]
44
public partial class Tests
55
{
6+
private static async Task ExecuteProviderAwareAsync(CustomProvider provider, Func<Task> action)
7+
{
8+
try
9+
{
10+
await action().ConfigureAwait(false);
11+
}
12+
catch (ApiException exception)
13+
{
14+
RethrowIfKnownProviderAvailabilityIssue(provider, exception);
15+
throw;
16+
}
17+
}
18+
19+
private static async Task<T> ExecuteProviderAwareAsync<T>(CustomProvider provider, Func<Task<T>> action)
20+
{
21+
try
22+
{
23+
return await action().ConfigureAwait(false);
24+
}
25+
catch (ApiException exception)
26+
{
27+
RethrowIfKnownProviderAvailabilityIssue(provider, exception);
28+
throw;
29+
}
30+
}
31+
32+
private static void RethrowIfKnownProviderAvailabilityIssue(CustomProvider provider, ApiException exception)
33+
{
34+
if (provider != CustomProvider.DeepInfra)
35+
{
36+
return;
37+
}
38+
39+
var responseText = exception.ResponseBody ?? exception.Message;
40+
if (exception.StatusCode == System.Net.HttpStatusCode.PaymentRequired &&
41+
responseText.Contains("inference suspended", StringComparison.OrdinalIgnoreCase))
42+
{
43+
throw new AssertInconclusiveException(
44+
"DeepInfra inference is suspended for the configured account. Skipping provider compatibility checks until billing access is restored.",
45+
exception);
46+
}
47+
}
48+
649
private static OpenAiClient GetAuthenticatedClient() => GetAuthorizedApi();
750

851
private static OpenAiClient GetAuthorizedApi()

0 commit comments

Comments
 (0)