feat: add OpenAI responses continuation middleware and integrate with agent

Author: NoOne7135

agent/middleware/openAiResponsesContinuation.ts

@@ -0,0 +1,92 @@
+import { AIMessage } from "@langchain/core/messages";
+import { createMiddleware } from "langchain";
+
+type OpenAiResponsesMetadata = {
+  id?: string;
+};
+
+type OpenAiResponsesContext = {
+  sessionId: string;
+  turnId: string;
+};
+
+function getTurnKey(context: OpenAiResponsesContext) {
+  return `${context.sessionId}:${context.turnId}`;
+}
+
+function getResponseId(message: AIMessage) {
+  const metadata = message.response_metadata as OpenAiResponsesMetadata | undefined;
+  return metadata?.id ?? null;
+}
+
+function getPreviousResponseId(modelSettings?: Record<string, unknown>) {
+  return (modelSettings as { previous_response_id?: string } | undefined)
+    ?.previous_response_id;
+}
+
+function getContinuationMessages<T extends { response_metadata?: unknown }>(
+  messages: T[],
+  previousResponseId: string,
+) {
+  let continuationStartIndex: number | null = null;
+
+  for (let index = messages.length - 1; index >= 0; index -= 1) {
+    const message = messages[index];
+
+    if (
+      AIMessage.isInstance(message) &&
+      (message.response_metadata as OpenAiResponsesMetadata | undefined)?.id ===
+        previousResponseId
+    ) {
+      continuationStartIndex = index + 1;
+      break;
+    }
+  }
+
+  if (continuationStartIndex === null) {
+    return null;
+  }
+
+  return messages.slice(continuationStartIndex);
+}
+
+export function createOpenAiResponsesContinuationMiddleware() {
+  const responseIdsByTurn = new Map<string, string>();
+
+  return createMiddleware({
+    name: "OpenAiResponsesContinuationMiddleware",
+    async wrapModelCall(request, handler) {
+      const context = request.runtime.context as OpenAiResponsesContext;
+      const turnKey = getTurnKey(context);
+      const previousResponseId =
+        getPreviousResponseId(request.modelSettings) ??
+        responseIdsByTurn.get(turnKey);
+      const continuationMessages = previousResponseId
+        ? getContinuationMessages(request.messages, previousResponseId)
+        : null;
+
+      const response = await handler(
+        previousResponseId && continuationMessages
+          ? {
+              ...request,
+              messages: continuationMessages,
+              modelSettings: {
+                ...request.modelSettings,
+                previous_response_id: previousResponseId,
+              },
+            }
+          : request,
+      ) as AIMessage;
+
+      const responseId = getResponseId(response);
+
+      if (responseId) {
+        responseIdsByTurn.set(turnKey, responseId);
+      } else {
+        responseIdsByTurn.delete(turnKey);
+      }
+
+      return response;
+    },
+  });
+}

agent/middleware/sequenceDebug.ts

@@ -23,6 +23,8 @@ export type SequenceDebug = {
   prompt: string;
   reasoning: string;
   text: string;
+  cachedTokens: number;
+  responseId: string | null;
   toolCalls: SequenceDebugToolCall[];
   endedAt: string;
   resultType: SequenceDebugResultType;
@@ -37,9 +39,21 @@ type PendingSequenceDebug = Omit<SequenceDebug, "toolCalls" | "endedAt" | "resul
 type SequenceDebugModelCall = {
   reasoning: string;
   text: string;
+  cachedTokens: number;
+  responseId: string | null;
   resultType: SequenceDebugResultType;
 };
 
+type OpenAiUsageMetadata = {
+  input_token_details?: {
+    cache_read?: number;
+  };
+};
+
+type OpenAiResponseMetadata = {
+  id?: string;
+};
+
 export type SequenceDebugModelCallSink = {
   handleModelCallStart: (prompt: string) => void;
   handleModelCallComplete: (params: SequenceDebugModelCall) => void;
@@ -58,6 +72,8 @@ function createPendingSequenceDebug(sequenceId: number): PendingSequenceDebug {
     prompt: "",
     reasoning: "",
     text: "",
+    cachedTokens: 0,
+    responseId: null,
     toolCalls: [],
     pendingToolCalls: 0,
     resultType: null,
@@ -83,6 +99,8 @@ function finalizeSequenceDebug(sequence: PendingSequenceDebug): SequenceDebug {
     prompt: sequence.prompt,
     reasoning: sequence.reasoning,
     text: sequence.text,
+    cachedTokens: sequence.cachedTokens,
+    responseId: sequence.responseId,
     toolCalls: sequence.toolCalls.map(({ completed: _completed, ...toolCall }) => toolCall),
     endedAt: new Date().toISOString(),
     resultType: sequence.resultType ?? "final_text",
@@ -172,6 +190,12 @@ function extractSequenceResponseDebug(message: AIMessage): SequenceDebugModelCal
   return {
     reasoning,
     text: textFromBlocks || (typeof message.content === "string" ? message.content : ""),
+    cachedTokens:
+      (message.usage_metadata as OpenAiUsageMetadata | undefined)
+        ?.input_token_details?.cache_read ?? 0,
+    responseId:
+      (message.response_metadata as OpenAiResponseMetadata | undefined)?.id ??
+      null,
     resultType: hasToolCallSignal(message) ? "tool_calls" : "final_text",
   };
 }
@@ -216,6 +240,8 @@ export function createSequenceDebugCollector(): SequenceDebugCollector {
       const sequenceDebug = ensureSequenceDebug();
       sequenceDebug.reasoning = params.reasoning;
       sequenceDebug.text = params.text;
+      sequenceDebug.cachedTokens = params.cachedTokens;
+      sequenceDebug.responseId = params.responseId;
       sequenceDebug.resultType = params.resultType;
 
       if (

agent/simpleAgent.ts

@@ -11,6 +11,7 @@ import {
   createSequenceDebugMiddleware,
   type SequenceDebugModelCallSink,
 } from "./middleware/sequenceDebug.js";
+import { createOpenAiResponsesContinuationMiddleware } from "./middleware/openAiResponsesContinuation.js";
 import type { ApiBasedTool } from "../apiBasedTools.js";
 import type { ToolCallEventSink } from "./toolCallEvents.js";
 
@@ -205,6 +206,10 @@ export function createAgentChatModel(params: {
     maxTokens: params.maxTokens,
     useResponsesApi: true,
     outputVersion: "v1",
+
+    promptCacheKey: `adminforth-agent:${model}:system-v1:tools-v1`,
+    promptCacheRetention: "in_memory" as "in_memory",
+
     ...(reasoning ? { reasoning } : {}),
     ...(typeof options.timeoutMs === "number"
       ? { timeout: options.timeoutMs }
@@ -250,16 +255,19 @@ export async function callAgent(params: {
 
   const tools = await createAgentTools(customComponentsDir, apiBasedTools);
   const apiBasedToolsMiddleware = createApiBasedToolsMiddleware(apiBasedTools);
+  const openAiResponsesContinuationMiddleware =
+    createOpenAiResponsesContinuationMiddleware();
   const sequenceDebugMiddleware = createSequenceDebugMiddleware(
     sequenceDebugSink,
   );
 
   const middleware = [
     apiBasedToolsMiddleware,
+    openAiResponsesContinuationMiddleware,
     sequenceDebugMiddleware,
     summarizationMiddleware({
       model: summaryModel,
-      trigger: { tokens: 1024 * 8 },
+      trigger: { tokens: 1024 * 128 },
       keep: { messages: 10 },
     }),
   ] as const;