Skip to content

Commit 815bbd0

Browse files
Refine LLM inference callback handlers
Collapse the HTTP callback seam to SendRequest/sendRequest, replace websocket hooks with per-connection handlers, and update tests to use the forwarding handler model. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 2eb2754 commit 815bbd0

8 files changed

Lines changed: 863 additions & 676 deletions

File tree

dotnet/src/LlmRequestHandler.cs

Lines changed: 470 additions & 183 deletions
Large diffs are not rendered by default.

dotnet/test/E2E/LlmInferenceE2EProvider.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ namespace GitHub.Copilot.Test.E2E;
2222
/// <remarks>
2323
/// <para>
2424
/// This exercises the public extension surface end to end: a consumer subclasses
25-
/// <see cref="LlmRequestHandler"/> and overrides <see cref="ForwardAsync"/> to
25+
/// <see cref="LlmRequestHandler"/> and overrides <see cref="SendRequestAsync"/> to
2626
/// short-circuit the upstream HTTP call with any <see cref="HttpResponseMessage"/>
2727
/// it likes. The base class streams that response back to the runtime.
2828
/// </para>
@@ -46,7 +46,7 @@ internal sealed class RecordingInferenceProvider : LlmRequestHandler
4646
public IReadOnlyList<InterceptedRequest> InferenceRequests =>
4747
[.. _records.Where(r => IsInferenceUrl(r.Url))];
4848

49-
protected override async Task<HttpResponseMessage> ForwardAsync(HttpRequestMessage request, LlmRequestContext ctx)
49+
protected override async Task<HttpResponseMessage> SendRequestAsync(HttpRequestMessage request, LlmRequestContext ctx)
5050
{
5151
var url = request.RequestUri!.ToString();
5252
_records.Enqueue(new InterceptedRequest(url, ctx.SessionId));

dotnet/test/Unit/LlmInference/LlmInferenceHandlerTests.cs

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -46,23 +46,20 @@ private static LlmInferenceRequest HttpRequest(
4646
};
4747

4848
/// <summary>A handler whose upstream call is a canned delegate (no network).</summary>
49-
private sealed class StubHandler(Func<HttpRequestMessage, HttpResponseMessage> forward) : LlmRequestHandler
49+
private sealed class StubHandler(Func<HttpRequestMessage, HttpResponseMessage> send) : LlmRequestHandler
5050
{
51-
protected override Task<HttpResponseMessage> ForwardAsync(HttpRequestMessage request, LlmRequestContext ctx) =>
52-
Task.FromResult(forward(request));
51+
protected override Task<HttpResponseMessage> SendRequestAsync(HttpRequestMessage request, LlmRequestContext ctx) =>
52+
Task.FromResult(send(request));
5353
}
5454

55-
/// <summary>A handler that adds a header in <c>TransformRequestAsync</c>.</summary>
56-
private sealed class HeaderMutatingHandler(Func<HttpRequestMessage, HttpResponseMessage> forward) : LlmRequestHandler
55+
/// <summary>A handler that adds a header before calling <c>base.SendRequestAsync</c>.</summary>
56+
private sealed class HeaderMutatingHandler(Func<HttpRequestMessage, HttpResponseMessage> send) : LlmRequestHandler
5757
{
58-
protected override Task<HttpRequestMessage> TransformRequestAsync(HttpRequestMessage request, LlmRequestContext ctx)
58+
protected override Task<HttpResponseMessage> SendRequestAsync(HttpRequestMessage request, LlmRequestContext ctx)
5959
{
6060
request.Headers.TryAddWithoutValidation("authorization", "Bearer swapped-token");
61-
return Task.FromResult(request);
61+
return Task.FromResult(send(request));
6262
}
63-
64-
protected override Task<HttpResponseMessage> ForwardAsync(HttpRequestMessage request, LlmRequestContext ctx) =>
65-
Task.FromResult(forward(request));
6663
}
6764

6865
[Fact]

nodejs/src/index.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,10 @@ export {
2828
approveAll,
2929
convertMcpCallToolResult,
3030
createSessionFsAdapter,
31+
CopilotWebSocketHandler,
32+
ForwardingWebSocketHandler,
3133
LlmRequestHandler,
32-
wrapGlobalWebSocket,
34+
LlmWebSocketCloseStatus,
3335
SYSTEM_MESSAGE_SECTIONS,
3436
} from "./types.js";
3537
// Re-export the generated session-event types (every *Event interface and
@@ -128,7 +130,6 @@ export type {
128130
LlmInferenceResponseInit,
129131
LlmInferenceResponseSink,
130132
LlmRequestContext,
131-
LlmWebSocketUpstream,
132133
SystemMessageAppendConfig,
133134
SystemMessageConfig,
134135
SystemMessageCustomizeConfig,

0 commit comments

Comments
 (0)