Skip to content

Commit 9268144

Browse files
NeoXiderNeoXider
committed
feat: add portable endpoint readiness probes
1 parent 222e6ea commit 9268144

23 files changed

Lines changed: 914 additions & 208 deletions

‎Assets/CoreAI.Demos/QwenDemo/QwenDemoShared.cs‎

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88
using System.Threading;
99
using System.Threading.Tasks;
1010
using CoreAI.Ai;
11+
using CoreAI.Infrastructure.Llm;
1112
using UnityEngine;
12-
using UnityEngine.Networking;
1313
using Debug = UnityEngine.Debug;
1414

1515
namespace CoreAI.ExampleGame.QwenDemo
@@ -373,21 +373,16 @@ private static T Read<T>(object source, string memberName)
373373

374374
private static async Task<bool> ProbeHttpAsync(int port, CancellationToken cancellationToken)
375375
{
376-
using UnityWebRequest request = new(
377-
$"http://localhost:{port}/v1/chat/completions", UnityWebRequest.kHttpVerbPOST);
378-
request.uploadHandler = new UploadHandlerRaw(Encoding.UTF8.GetBytes("{}"));
379-
request.downloadHandler = new DownloadHandlerBuffer();
380-
request.SetRequestHeader("Content-Type", "application/json");
381-
request.timeout = 2;
382-
UnityWebRequestAsyncOperation operation = request.SendWebRequest();
383-
while (!operation.isDone)
384-
{
385-
cancellationToken.ThrowIfCancellationRequested();
386-
await Task.Yield();
387-
}
388-
389-
long status = request.responseCode;
390-
return status is >= 200 and < 500 && status is not 401 and not 403 and not 404;
376+
ILlmEndpointReadinessProbe probe = new UnityWebRequestOpenAiReadinessProbe();
377+
LlmEndpointReadinessResult result = await probe.ProbeAsync(
378+
new LlmEndpointReadinessRequest
379+
{
380+
BaseUrl = $"http://localhost:{port}/v1",
381+
TimeoutSeconds = 2,
382+
Mode = LlmEndpointReadinessMode.CompletionsOnly
383+
},
384+
cancellationToken);
385+
return result.IsReady;
391386
}
392387
}
393388

‎Assets/CoreAI/CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
profile takes precedence over agent, role, default, and legacy routing.
1111
- `ILlmEndpointSecretProvider` keeps credential resolution behind a portable host boundary; persisted
1212
descriptors contain a `SecretReference`, never the session credential.
13+
- Portable `ILlmEndpointReadinessProbe` request/result contracts, shared OpenAI status policy, and
14+
`HttpClientOpenAiReadinessProbe` let ordinary .NET hosts validate endpoints without referencing Unity.
1315

1416
### Changed
1517

‎Assets/CoreAI/Docs/LLM_ROUTING.md‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,14 +73,21 @@ Lesson and practice orchestrators can keep routing portable while adding per-tur
7373
## Host Boundary
7474

7575
`CoreAI.Core` ships the portable OpenAI-compatible HTTP path: `IOpenAiHttpTransport`,
76-
`HttpClientOpenAiTransport`, and `MeaiOpenAiChatClient`. A plain .NET host can construct that client directly
77-
without Unity. Loopback URLs bypass the system proxy so local model sockets remain local; external URLs retain
78-
the platform proxy policy.
76+
`HttpClientOpenAiTransport`, and `MeaiOpenAiChatClient`. It also ships
77+
`ILlmEndpointReadinessProbe`, `LlmEndpointReadinessRequest`, the shared status policy, and
78+
`HttpClientOpenAiReadinessProbe`. A plain .NET host can therefore construct both the chat client and endpoint
79+
readiness pipeline directly without Unity. `ModelsThenCompletions` checks `/models` first and falls back to
80+
the completion handler only when `/models` is unsupported (`404`/`405`); `CompletionsOnly` is available for
81+
embedded servers that expose no models route. Loopback URLs bypass the system proxy so local model sockets
82+
remain local; external URLs retain the platform proxy policy.
83+
Readiness probes do not follow redirects and never treat `3xx` as a ready handler, preventing credentials
84+
from crossing origins during endpoint activation.
7985

8086
`CoreAiUnity` owns Unity runtime integration: endpoint registry lifecycle and persistence, `CoreAISettingsAsset`,
8187
LLMUnity native model startup/readiness, WebGL `UnityWebRequest`/Fetch transports, Hub/Chat UI, and VContainer
82-
registration. It selects the Unity/WebGL transport adapter where required; it does not own the portable HTTP
83-
client contract or implementation.
88+
registration. It supplies `UnityWebRequestOpenAiReadinessProbe` for Unity/WebGL and injects it into endpoint
89+
activation. Search/configuration of `LLMAgent`, `LLM.WaitUntilReady()`, host leases, and llama.cpp unload stay
90+
strictly Unity-owned; the HTTP readiness contract and policy do not depend on Unity.
8491

8592
Production games such as RedoSchool should put provider keys and quota enforcement behind `ServerManagedApi`. The Unity client sends a user/session token to the backend; the backend performs entitlement, calls the provider, records usage, and returns stable provider errors.
8693

‎Assets/CoreAI/Runtime/Core/AssemblyInfo.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System.Runtime.CompilerServices;
22

33
[assembly: InternalsVisibleTo("CoreAI.Tests")]
4+
[assembly: InternalsVisibleTo("CoreAI.Core.Tests")]
45
[assembly: InternalsVisibleTo("CoreAI.Mods")]
56
[assembly: InternalsVisibleTo("CoreAI.Mods.Tests")]

‎Assets/CoreAI/Runtime/Core/Features/Llm/HttpClientOpenAiReadinessProbe.cs‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
using System;
2+
using System.Net.Http;
3+
using System.Net.Http.Headers;
4+
using System.Text;
5+
using System.Threading;
6+
using System.Threading.Tasks;
7+
using CoreAI.Ai;
8+
9+
namespace CoreAI.Infrastructure.Llm
10+
{
11+
/// <summary>Portable <see cref="HttpClient"/> readiness adapter for ordinary .NET hosts.</summary>
12+
public sealed class HttpClientOpenAiReadinessProbe : ILlmEndpointReadinessProbe
13+
{
14+
private static readonly Lazy<HttpClient> s_loopbackClient =
15+
new(() => CreateClient(bypassProxy: true));
16+
private static readonly Lazy<HttpClient> s_externalClient =
17+
new(() => CreateClient(bypassProxy: false));
18+
19+
private readonly HttpClient _client;
20+
21+
public HttpClientOpenAiReadinessProbe()
22+
{
23+
}
24+
25+
internal HttpClientOpenAiReadinessProbe(HttpClient client)
26+
{
27+
_client = client ?? throw new ArgumentNullException(nameof(client));
28+
}
29+
30+
public async Task<LlmEndpointReadinessResult> ProbeAsync(
31+
LlmEndpointReadinessRequest request,
32+
CancellationToken cancellationToken = default)
33+
{
34+
if (request == null)
35+
{
36+
throw new ArgumentNullException(nameof(request));
37+
}
38+
39+
if (!TryParseHttpBaseUri(request.BaseUrl, out Uri baseUri))
40+
{
41+
return Failed(0, "Endpoint base URL is invalid.");
42+
}
43+
44+
int timeoutSeconds = Math.Max(1, request.TimeoutSeconds);
45+
using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(timeoutSeconds));
46+
using CancellationTokenSource linked =
47+
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeout.Token);
48+
49+
try
50+
{
51+
if (request.Mode == LlmEndpointReadinessMode.ModelsThenCompletions)
52+
{
53+
LlmEndpointReadinessResult models = await SendAsync(
54+
BuildRoute(baseUri, "models"),
55+
HttpMethod.Get,
56+
request.ApiKey,
57+
true,
58+
linked.Token).ConfigureAwait(false);
59+
if (models.IsReady || !LlmEndpointReadinessPolicy.ShouldTryCompletions(models.StatusCode))
60+
{
61+
return models;
62+
}
63+
}
64+
65+
return await SendAsync(
66+
BuildRoute(baseUri, "chat/completions"),
67+
HttpMethod.Post,
68+
request.ApiKey,
69+
false,
70+
linked.Token).ConfigureAwait(false);
71+
}
72+
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
73+
{
74+
return Failed(0, $"Endpoint readiness timed out after {timeoutSeconds}s.");
75+
}
76+
catch (HttpRequestException ex)
77+
{
78+
return Failed(0, "Endpoint readiness transport failed: " + ex.GetType().Name + ".");
79+
}
80+
}
81+
82+
private static bool TryParseHttpBaseUri(string value, out Uri uri)
83+
{
84+
bool valid = Uri.TryCreate((value ?? "").TrimEnd('/'), UriKind.Absolute, out uri) &&
85+
(uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps) &&
86+
string.IsNullOrEmpty(uri.UserInfo) &&
87+
string.IsNullOrEmpty(uri.Query) &&
88+
string.IsNullOrEmpty(uri.Fragment);
89+
return valid;
90+
}
91+
92+
private static Uri BuildRoute(Uri baseUri, string route)
93+
{
94+
UriBuilder builder = new(baseUri)
95+
{
96+
Path = baseUri.AbsolutePath.TrimEnd('/') + "/" + route,
97+
Query = "",
98+
Fragment = ""
99+
};
100+
return builder.Uri;
101+
}
102+
103+
private async Task<LlmEndpointReadinessResult> SendAsync(
104+
Uri uri,
105+
HttpMethod method,
106+
string apiKey,
107+
bool modelsRoute,
108+
CancellationToken cancellationToken)
109+
{
110+
using HttpRequestMessage message = new(method, uri);
111+
if (method == HttpMethod.Post)
112+
{
113+
message.Content = new StringContent("{}", Encoding.UTF8, "application/json");
114+
}
115+
116+
if (!string.IsNullOrWhiteSpace(apiKey))
117+
{
118+
message.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiKey.Trim());
119+
}
120+
121+
HttpClient client = _client ?? GetSharedClient(uri);
122+
using HttpResponseMessage response = await client.SendAsync(
123+
message,
124+
HttpCompletionOption.ResponseHeadersRead,
125+
cancellationToken).ConfigureAwait(false);
126+
int status = (int)response.StatusCode;
127+
bool ready = modelsRoute
128+
? status is >= 200 and < 300
129+
: LlmEndpointReadinessPolicy.IsHandlerReached(status);
130+
return ready
131+
? new LlmEndpointReadinessResult { IsReady = true, StatusCode = status }
132+
: Failed(status, $"Endpoint readiness probe failed ({status}).");
133+
}
134+
135+
private static LlmEndpointReadinessResult Failed(int statusCode, string error) => new()
136+
{
137+
IsReady = false,
138+
StatusCode = statusCode,
139+
Error = error ?? ""
140+
};
141+
142+
private static HttpClient GetSharedClient(Uri uri) =>
143+
uri.IsLoopback ? s_loopbackClient.Value : s_externalClient.Value;
144+
145+
private static HttpClient CreateClient(bool bypassProxy)
146+
{
147+
HttpClientHandler handler = new();
148+
handler.AllowAutoRedirect = false;
149+
if (bypassProxy)
150+
{
151+
try
152+
{
153+
handler.UseProxy = false;
154+
}
155+
catch
156+
{
157+
// WHY: runtimes without a writable proxy setting retain their platform default.
158+
}
159+
}
160+
161+
return new HttpClient(handler) { Timeout = Timeout.InfiniteTimeSpan };
162+
}
163+
}
164+
}

‎Assets/CoreAI/Runtime/Core/Features/Llm/HttpClientOpenAiReadinessProbe.cs.meta‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Assets/CoreAI/Runtime/Core/Features/LlmRouting/LlmEndpointReadiness.cs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
using System;
2+
using System.Threading;
3+
using System.Threading.Tasks;
4+
5+
namespace CoreAI.Ai
6+
{
7+
/// <summary>OpenAI-compatible route used to prove that an endpoint handler is reachable.</summary>
8+
public enum LlmEndpointReadinessMode
9+
{
10+
/// <summary>Checks <c>/models</c>, then falls back to <c>/chat/completions</c> when unsupported.</summary>
11+
ModelsThenCompletions = 0,
12+
13+
/// <summary>Checks the completions handler directly, as required by embedded llama.cpp hosts.</summary>
14+
CompletionsOnly = 1
15+
}
16+
17+
/// <summary>Portable input for an OpenAI-compatible endpoint readiness probe.</summary>
18+
public sealed class LlmEndpointReadinessRequest
19+
{
20+
public string BaseUrl { get; set; } = "";
21+
public string ApiKey { get; set; } = "";
22+
public int TimeoutSeconds { get; set; } = 5;
23+
public LlmEndpointReadinessMode Mode { get; set; }
24+
}
25+
26+
/// <summary>Portable readiness outcome without host-specific HTTP types.</summary>
27+
public sealed class LlmEndpointReadinessResult
28+
{
29+
public bool IsReady { get; set; }
30+
public int StatusCode { get; set; }
31+
public string Error { get; set; } = "";
32+
}
33+
34+
/// <summary>Host-supplied probe for OpenAI-compatible endpoint readiness.</summary>
35+
public interface ILlmEndpointReadinessProbe
36+
{
37+
Task<LlmEndpointReadinessResult> ProbeAsync(
38+
LlmEndpointReadinessRequest request,
39+
CancellationToken cancellationToken = default);
40+
}
41+
42+
/// <summary>Shared status policy used by .NET and Unity HTTP adapters.</summary>
43+
public static class LlmEndpointReadinessPolicy
44+
{
45+
public static bool IsHandlerReached(long status) =>
46+
status is >= 200 and < 500 &&
47+
status is not (>= 300 and < 400) &&
48+
status is not 401 and not 403 and not 404;
49+
50+
public static bool ShouldTryCompletions(long modelsStatus) =>
51+
modelsStatus is 404 or 405;
52+
}
53+
}

‎Assets/CoreAI/Runtime/Core/Features/LlmRouting/LlmEndpointReadiness.cs.meta‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)