Collapse LLM inference callback public API to LlmRequestHandler

Hide the redundant low-level provider interface and adapter from the public
surface in both SDKs; the sole public extension point is now the
LlmRequestHandler base class. Replace the LlmInferenceConfig provider factory
with a direct handler instance (the provider is client-global, constructed
once with no args).

.NET: ILlmInferenceProvider + the LlmInferenceRequest/ResponseInit/ResponseSink
DTOs become internal; LlmRequestHandler implements the interface explicitly so
OnLlmRequestAsync leaves its public surface. LlmInferenceConfig.Handler replaces
the Func<LlmRequestHandler> factory.

TS: stop exporting LlmInferenceProvider and createLlmInferenceAdapter from
index.ts; LlmInferenceConfig.handler replaces createLlmInferenceProvider. The
request/sink DTOs stay exported as onLlmRequest's contract (TS lacks explicit
interface implementation). E2E providers become LlmRequestHandler subclasses
overriding onLlmRequest.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: SteveSandersonMS

dotnet/src/Client.cs

@@ -1692,18 +1692,15 @@ await Rpc.SessionFs.SetProviderAsync(
     /// </summary>
     private ClientGlobalApiHandlers? BuildClientGlobalApis()
     {
-        var factory = _options.LlmInference?.CreateLlmInferenceProvider;
-        if (factory is null)
+        var handler = _options.LlmInference?.Handler;
+        if (handler is null)
         {
             return null;
         }
 
-        var provider = factory()
-            ?? throw new InvalidOperationException("LlmInferenceConfig.CreateLlmInferenceProvider returned null.");
-
         return new ClientGlobalApiHandlers
         {
-            LlmInference = new LlmInferenceAdapter(provider, () => _serverRpc),
+            LlmInference = new LlmInferenceAdapter(handler, () => _serverRpc),
         };
     }
 

dotnet/src/LlmInferenceProvider.cs

@@ -42,8 +42,7 @@ public enum LlmInferenceTransport
 /// (no provider type, endpoint kind, or wire API); consumers that need that
 /// information derive it from the URL / headers themselves.
 /// </remarks>
-[Experimental(Diagnostics.Experimental)]
-public sealed class LlmInferenceRequest
+internal sealed class LlmInferenceRequest
 {
     /// <summary>Opaque runtime-minted id, stable across the request lifecycle.</summary>
     public required string RequestId { get; init; }
@@ -100,8 +99,7 @@ public sealed class LlmInferenceRequest
 }
 
 /// <summary>Response head passed to <see cref="LlmInferenceResponseSink.StartAsync"/>.</summary>
-[Experimental(Diagnostics.Experimental)]
-public sealed class LlmInferenceResponseInit
+internal sealed class LlmInferenceResponseInit
 {
     /// <summary>HTTP status code (101 acknowledges a WebSocket upgrade).</summary>
     public int Status { get; init; }
@@ -119,8 +117,7 @@ public sealed class LlmInferenceResponseInit
 /// exactly one of <see cref="EndAsync"/> or <see cref="ErrorAsync"/>. Calling
 /// out of order throws.
 /// </summary>
-[Experimental(Diagnostics.Experimental)]
-public abstract class LlmInferenceResponseSink
+internal abstract class LlmInferenceResponseSink
 {
     /// <summary>Sends the response head (status + headers) back to the runtime.</summary>
     public abstract Task StartAsync(LlmInferenceResponseInit init);
@@ -139,24 +136,23 @@ public abstract class LlmInferenceResponseSink
 }
 
 /// <summary>
-/// Implemented by SDK consumers to service the LLM inference requests the
-/// runtime would otherwise issue itself. The same callback handles both
-/// buffered and streaming responses — the consumer just calls
+/// Internal seam implemented by <see cref="LlmRequestHandler"/> and consumed by
+/// <see cref="LlmInferenceAdapter"/>. The single callback handles both buffered
+/// and streaming responses — the implementer calls
 /// <see cref="LlmInferenceResponseSink.WriteAsync(ReadOnlyMemory{byte})"/> zero
 /// or more times before <see cref="LlmInferenceResponseSink.EndAsync"/>.
 /// </summary>
 /// <remarks>
-/// Prefer subclassing <see cref="LlmRequestHandler"/> for a transparent
-/// pass-through starting point; implement this interface directly only when you
-/// need full control over the raw byte streams.
+/// Not part of the public API: consumers subclass <see cref="LlmRequestHandler"/>
+/// rather than implementing this directly. It exists so the adapter can drive any
+/// handler through one uniform entry point.
 /// </remarks>
-[Experimental(Diagnostics.Experimental)]
-public interface ILlmInferenceProvider
+internal interface ILlmInferenceProvider
 {
     /// <summary>
-    /// Invoked by the runtime once per outbound LLM request the consumer has
-    /// opted to handle. The consumer is responsible for eventually calling
-    /// either <see cref="LlmInferenceResponseSink.EndAsync"/> or
+    /// Invoked by the adapter once per outbound LLM request. The implementer is
+    /// responsible for eventually calling either
+    /// <see cref="LlmInferenceResponseSink.EndAsync"/> or
     /// <see cref="LlmInferenceResponseSink.ErrorAsync"/>; failing to do so leaks
     /// runtime state. Throwing surfaces a transport-level failure to the runtime
     /// (equivalent to <c>ResponseBody.ErrorAsync(...)</c> when

dotnet/src/LlmRequestHandler.cs

@@ -56,9 +56,8 @@ public readonly struct LlmWebSocketMessage(ReadOnlyMemory<byte> data, bool isBin
 
 /// <summary>
 /// Base class for SDK consumers who want to observe or mutate the LLM inference
-/// requests the runtime issues. Implements <see cref="ILlmInferenceProvider"/>,
-/// so an instance can be returned directly from
-/// <see cref="LlmInferenceConfig.CreateLlmInferenceProvider"/>.
+/// requests the runtime issues. An instance is returned directly from
+/// <see cref="LlmInferenceConfig.Handler"/>.
 /// </summary>
 /// <remarks>
 /// <para>
@@ -80,8 +79,7 @@ public readonly struct LlmWebSocketMessage(ReadOnlyMemory<byte> data, bool isBin
 /// — observe or mutate WebSocket messages in either direction.</item>
 /// </list>
 /// <para>
-/// The same subclass handles both transports —
-/// <see cref="OnLlmRequestAsync"/> dispatches on
+/// The same subclass handles both transports — dispatch keys on
 /// <see cref="LlmInferenceRequest.Transport"/>.
 /// </para>
 /// </remarks>
@@ -106,7 +104,7 @@ public class LlmRequestHandler : ILlmInferenceProvider
     };
 
     /// <inheritdoc />
-    public async Task OnLlmRequestAsync(LlmInferenceRequest request)
+    async Task ILlmInferenceProvider.OnLlmRequestAsync(LlmInferenceRequest request)
     {
         ArgumentNullException.ThrowIfNull(request);
 

dotnet/src/Types.cs

@@ -495,13 +495,12 @@ public sealed class SessionFsConfig
 public sealed class LlmInferenceConfig
 {
     /// <summary>
-    /// Factory invoked once when the client connects, producing the provider that
-    /// will service every intercepted model-layer request for the lifetime of the
-    /// connection. Return a <see cref="LlmRequestHandler"/> subclass for a
-    /// transparent pass-through starting point, or any
-    /// <see cref="ILlmInferenceProvider"/> for full control.
+    /// Handler that services every intercepted model-layer request for the
+    /// lifetime of the client connection. Subclass <see cref="LlmRequestHandler"/>
+    /// and override its hooks to observe, mutate, or fully replace each
+    /// request/response.
     /// </summary>
-    public Func<ILlmInferenceProvider>? CreateLlmInferenceProvider { get; set; }
+    public LlmRequestHandler? Handler { get; set; }
 }
 
 /// <summary>

dotnet/test/E2E/LlmInferenceE2EProvider.cs

@@ -3,6 +3,8 @@
  *--------------------------------------------------------------------------------------------*/
 
 using System.Collections.Concurrent;
+using System.Net;
+using System.Net.Http;
 using System.Text;
 using System.Text.RegularExpressions;
 
@@ -11,19 +13,27 @@ namespace GitHub.Copilot.Test.E2E;
 #pragma warning disable GHCP001 // The LLM inference surface is intentionally experimental.
 
 /// <summary>
-/// An <see cref="ILlmInferenceProvider"/> for e2e tests that records every
-/// intercepted request (url + threaded session id) and fabricates well-formed
-/// responses for every model-layer endpoint, so an agent turn completes
-/// entirely off-network — no upstream server and no CAPI proxy acting as the
-/// inference endpoint.
+/// A <see cref="LlmRequestHandler"/> subclass for e2e tests that records every
+/// intercepted request (url + threaded session id) and fully replaces the
+/// upstream call with a fabricated, well-formed response for every model-layer
+/// endpoint, so an agent turn completes entirely off-network — no upstream
+/// server and no CAPI proxy acting as the inference endpoint.
 /// </summary>
 /// <remarks>
+/// <para>
+/// This exercises the public extension surface end to end: a consumer subclasses
+/// <see cref="LlmRequestHandler"/> and overrides <see cref="ForwardAsync"/> to
+/// short-circuit the upstream HTTP call with any <see cref="HttpResponseMessage"/>
+/// it likes. The base class streams that response back to the runtime.
+/// </para>
+/// <para>
 /// All response bodies are emitted as raw JSON string literals rather than via
 /// <c>JsonSerializer</c>: the test project disables reflection-based STJ on
 /// net8.0 (<c>JsonSerializerIsReflectionEnabledByDefault=false</c>), so
 /// serializing anonymous types would throw at runtime.
+/// </para>
 /// </remarks>
-internal sealed class RecordingInferenceProvider : ILlmInferenceProvider
+internal sealed class RecordingInferenceProvider : LlmRequestHandler
 {
     internal const string SyntheticText = "OK from the synthetic stream.";
 
@@ -36,18 +46,22 @@ internal sealed class RecordingInferenceProvider : ILlmInferenceProvider
     public IReadOnlyList<InterceptedRequest> InferenceRequests =>
         [.. _records.Where(r => IsInferenceUrl(r.Url))];
 
-    public async Task OnLlmRequestAsync(LlmInferenceRequest request)
+    protected override async Task<HttpResponseMessage> ForwardAsync(HttpRequestMessage request, LlmRequestContext ctx)
     {
-        _records.Enqueue(new InterceptedRequest(request.Url, request.SessionId));
-
-        if (IsInferenceUrl(request.Url))
-        {
-            await HandleInferenceAsync(request).ConfigureAwait(false);
-        }
-        else
-        {
-            await HandleNonInferenceModelTrafficAsync(request).ConfigureAwait(false);
-        }
+        var url = request.RequestUri!.ToString();
+        _records.Enqueue(new InterceptedRequest(url, ctx.SessionId));
+
+        var bodyText = request.Content is null
+            ? string.Empty
+#if NET8_0_OR_GREATER
+            : await request.Content.ReadAsStringAsync(ctx.CancellationToken).ConfigureAwait(false);
+#else
+            : await request.Content.ReadAsStringAsync().ConfigureAwait(false);
+#endif
+
+        return IsInferenceUrl(url)
+            ? BuildInferenceResponse(url, bodyText)
+            : BuildNonInferenceResponse(url);
     }
 
     internal static bool IsInferenceUrl(string url)
@@ -59,116 +73,67 @@ internal static bool IsInferenceUrl(string url)
             || u.EndsWith("/messages", StringComparison.Ordinal);
     }
 
-    private static async Task<string> DrainRequestAsync(LlmInferenceRequest req)
+    /// <summary>
+    /// Synthesizes a well-formed inference response so the agent turn completes.
+    /// The runtime selects <c>/responses</c> for both the CAPI and BYOK sessions
+    /// here; <c>/chat/completions</c> is handled too for robustness.
+    /// </summary>
+    private static HttpResponseMessage BuildInferenceResponse(string url, string bodyText)
     {
-        using var buffer = new MemoryStream();
-        await foreach (var chunk in req.RequestBody.ConfigureAwait(false))
+        var wantsStream = WantsStreamRegex.IsMatch(bodyText);
+        var u = url.ToLowerInvariant();
+
+        if (u.Contains("/responses", StringComparison.Ordinal))
         {
-            if (chunk.Length > 0)
-            {
-                buffer.Write(chunk.ToArray(), 0, chunk.Length);
-            }
+            return wantsStream
+                ? Sse(string.Concat(ResponsesStreamEvents))
+                : Json(BufferedResponseJson);
         }
 
-        return Encoding.UTF8.GetString(buffer.ToArray());
-    }
-
-    private static async Task RespondBufferedAsync(LlmInferenceRequest req, int status, string contentType, string body)
-    {
-        await DrainRequestAsync(req).ConfigureAwait(false);
-        await req.ResponseBody.StartAsync(new LlmInferenceResponseInit
-        {
-            Status = status,
-            Headers = Headers(contentType),
-        }).ConfigureAwait(false);
-        if (body.Length > 0)
+        if (u.Contains("/chat/completions", StringComparison.Ordinal) && wantsStream)
         {
-            await req.ResponseBody.WriteAsync(body).ConfigureAwait(false);
+            return Sse(string.Concat(ChatCompletionStreamEvents));
         }
 
-        await req.ResponseBody.EndAsync().ConfigureAwait(false);
+        // /chat/completions non-streaming (and any other inference url) — buffered JSON.
+        return Json(BufferedChatCompletionJson);
     }
 
     /// <summary>
     /// Serves the non-inference model-layer GETs/POSTs the runtime issues
     /// (catalog, model session, policy). These flow through the same callback
     /// but carry no session id (they happen outside an agent turn).
     /// </summary>
-    private static async Task HandleNonInferenceModelTrafficAsync(LlmInferenceRequest req)
+    private static HttpResponseMessage BuildNonInferenceResponse(string url)
     {
-        var url = req.Url.ToLowerInvariant();
-        if (url.EndsWith("/models", StringComparison.Ordinal))
+        var u = url.ToLowerInvariant();
+        if (u.EndsWith("/models", StringComparison.Ordinal))
         {
-            await RespondBufferedAsync(req, 200, "application/json", ModelCatalogJson).ConfigureAwait(false);
-            return;
+            return Json(ModelCatalogJson);
         }
 
-        if (url.Contains("/models/session", StringComparison.Ordinal))
+        if (u.Contains("/models/session", StringComparison.Ordinal))
         {
-            await RespondBufferedAsync(req, 200, "application/json", "{}").ConfigureAwait(false);
-            return;
+            return Json("{}");
         }
 
-        if (url.Contains("/policy", StringComparison.Ordinal))
+        if (u.Contains("/policy", StringComparison.Ordinal))
         {
-            await RespondBufferedAsync(req, 200, "application/json", "{\"state\":\"enabled\"}").ConfigureAwait(false);
-            return;
+            return Json("{\"state\":\"enabled\"}");
         }
 
-        await RespondBufferedAsync(req, 200, "application/json", "{}").ConfigureAwait(false);
+        return Json("{}");
     }
 
-    /// <summary>
-    /// Synthesizes a well-formed inference response so the agent turn completes.
-    /// The runtime selects <c>/responses</c> for both the CAPI and BYOK sessions
-    /// here; <c>/chat/completions</c> is handled too for robustness.
-    /// </summary>
-    private static async Task HandleInferenceAsync(LlmInferenceRequest req)
+    private static HttpResponseMessage Json(string body) => new(HttpStatusCode.OK)
     {
-        var bodyText = await DrainRequestAsync(req).ConfigureAwait(false);
-        var wantsStream = WantsStreamRegex.IsMatch(bodyText);
-        var url = req.Url.ToLowerInvariant();
-
-        if (url.Contains("/responses", StringComparison.Ordinal))
-        {
-            if (!wantsStream)
-            {
-                await req.ResponseBody.StartAsync(new LlmInferenceResponseInit { Status = 200, Headers = Headers("application/json") }).ConfigureAwait(false);
-                await req.ResponseBody.WriteAsync(BufferedResponseJson).ConfigureAwait(false);
-                await req.ResponseBody.EndAsync().ConfigureAwait(false);
-                return;
-            }
-
-            await req.ResponseBody.StartAsync(new LlmInferenceResponseInit { Status = 200, Headers = Headers("text/event-stream") }).ConfigureAwait(false);
-            foreach (var sseEvent in ResponsesStreamEvents)
-            {
-                await req.ResponseBody.WriteAsync(sseEvent).ConfigureAwait(false);
-            }
-
-            await req.ResponseBody.EndAsync().ConfigureAwait(false);
-            return;
-        }
-
-        if (url.Contains("/chat/completions", StringComparison.Ordinal) && wantsStream)
-        {
-            await req.ResponseBody.StartAsync(new LlmInferenceResponseInit { Status = 200, Headers = Headers("text/event-stream") }).ConfigureAwait(false);
-            foreach (var sseEvent in ChatCompletionStreamEvents)
-            {
-                await req.ResponseBody.WriteAsync(sseEvent).ConfigureAwait(false);
-            }
-
-            await req.ResponseBody.EndAsync().ConfigureAwait(false);
-            return;
-        }
-
-        // /chat/completions non-streaming — buffered JSON.
-        await req.ResponseBody.StartAsync(new LlmInferenceResponseInit { Status = 200, Headers = Headers("application/json") }).ConfigureAwait(false);
-        await req.ResponseBody.WriteAsync(BufferedChatCompletionJson).ConfigureAwait(false);
-        await req.ResponseBody.EndAsync().ConfigureAwait(false);
-    }
+        Content = new StringContent(body, Encoding.UTF8, "application/json"),
+    };
 
-    private static Dictionary<string, IReadOnlyList<string>> Headers(string contentType) =>
-        new() { ["content-type"] = [contentType] };
+    private static HttpResponseMessage Sse(string body) => new(HttpStatusCode.OK)
+    {
+        Content = new StringContent(body, Encoding.UTF8, "text/event-stream"),
+    };
 
     private static readonly string[] ResponsesStreamEvents =
     [

dotnet/test/E2E/LlmInferenceSessionIdE2ETests.cs

@@ -26,7 +26,7 @@ private CopilotClient CreateClientWith(RecordingInferenceProvider provider) =>
             Connection = RuntimeConnection.ForStdio(),
             LlmInference = new LlmInferenceConfig
             {
-                CreateLlmInferenceProvider = () => provider,
+                Handler = provider,
             },
         });