Skip to content

Commit 4dfc810

Browse files
tomer-qualifire-aitomer-qualifire-ai
authored
SDK Ramp Up - Support new request/response model (#20)
1 parent 8981f83 commit 4dfc810

11 files changed

Lines changed: 2698 additions & 97 deletions

File tree

‎package.json‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "qualifire",
3-
"version": "0.0.3",
3+
"version": "0.1.0",
44
"description": "Qualifire client SDK",
55
"main": "./lib/index.js",
66
"files": [
@@ -114,9 +114,13 @@
114114
]
115115
},
116116
"dependencies": {
117+
"@ai-sdk/provider-utils": "^3.0.7",
118+
"@anthropic-ai/sdk": "^0.60.0",
119+
"@google/genai": "^1.16.0",
117120
"@traceloop/node-server-sdk": "^0.13.3",
121+
"ai": "^5.0.22",
118122
"openai": "^4.28.4",
119-
"zod": "^3.22.4"
123+
"zod": "^4.0.0"
120124
},
121-
"packageManager": "pnpm@9.6.0+sha512.38dc6fba8dba35b39340b9700112c2fe1e12f10b17134715a4aa98ccf7bb035e76fd981cf0bb384dfa98f8d6af5481c2bef2f4266a24bfa20c34eb7147ce0b5e"
125+
"packageManager": "pnpm@10.14.0"
122126
}

‎pnpm-lock.yaml‎

Lines changed: 389 additions & 63 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/frameworks/canonical.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
import { EvaluationProxyAPIRequest } from '../types';
2+
3+
export interface CanonicalEvaluationStrategy<RequestType, ResponseType> {
4+
convertToQualifireEvaluationRequest(
5+
request: RequestType,
6+
response: ResponseType
7+
): Promise<EvaluationProxyAPIRequest>;
8+
}

‎src/frameworks/claude/claude-converter.ts‎

Lines changed: 292 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,292 @@
1+
import { Message, type MessageCreateParams, type MessageStreamParams } from '@anthropic-ai/sdk/resources';
2+
import {
3+
MessageStreamEvent,
4+
RawMessageStreamEvent,
5+
type ContentBlockParam,
6+
type InputJSONDelta,
7+
type RawContentBlockDeltaEvent,
8+
type RawContentBlockStartEvent,
9+
type RawMessageStartEvent,
10+
type TextBlock,
11+
type TextBlockParam,
12+
type TextDelta,
13+
type ThinkingBlock,
14+
type ToolResultBlockParam,
15+
type ToolUseBlock
16+
} from '@anthropic-ai/sdk/resources/messages';
17+
import {
18+
EvaluationProxyAPIRequest,
19+
LLMMessage,
20+
LLMToolCall,
21+
LLMToolDefinition,
22+
} from '../../types';
23+
import { CanonicalEvaluationStrategy } from '../canonical';
24+
25+
type AnthropicCreateAPIResponsesType = Message | RawMessageStreamEvent;
26+
27+
type AnthropicAPIRequestsType = MessageCreateParams
28+
type AnthropicAPIResponsesType =
29+
| AnthropicCreateAPIResponsesType
30+
| MessageStreamParams;
31+
32+
export class ClaudeCanonicalEvaluationStrategy
33+
implements
34+
CanonicalEvaluationStrategy<
35+
AnthropicAPIRequestsType,
36+
AnthropicAPIResponsesType
37+
> {
38+
async convertToQualifireEvaluationRequest(
39+
request: AnthropicAPIRequestsType,
40+
response: AnthropicAPIResponsesType
41+
): Promise<EvaluationProxyAPIRequest> {
42+
const {
43+
messages: requestMessages,
44+
available_tools: requestAvailableTools,
45+
} = this.convertRequest(request);
46+
47+
const messages: LLMMessage[] = requestMessages || [];
48+
const availableTools: LLMToolDefinition[] = requestAvailableTools || [];
49+
50+
// Avoid undefined response
51+
if (!response) {
52+
return {
53+
messages,
54+
available_tools: availableTools,
55+
};
56+
}
57+
58+
// Check if response is streaming or non-streaming
59+
if (Array.isArray(response)) {
60+
const streamingResultMessages = await this.handleStreaming(response as Array<MessageStreamEvent>);
61+
messages.push(...streamingResultMessages);
62+
} else {
63+
const nonStreamingResultMessages = await this.handleNonStreamingResponse(
64+
response
65+
);
66+
messages.push(...nonStreamingResultMessages);
67+
}
68+
69+
return {
70+
messages,
71+
available_tools: availableTools,
72+
};
73+
}
74+
75+
convertRequest(request: any): EvaluationProxyAPIRequest {
76+
const messages: LLMMessage[] = [];
77+
const availableTools: LLMToolDefinition[] = [];
78+
79+
// Handle Claude system message first (if present)
80+
if (request?.system) {
81+
messages.push({
82+
role: 'system',
83+
content: request.system as string,
84+
});
85+
}
86+
87+
// Handle Claude request messages
88+
if (request?.messages) {
89+
messages.push(
90+
...this.convertClaudeMessagesToLLMMessages(request.messages as Array<Message>)
91+
);
92+
}
93+
94+
// Handle tools
95+
if (request?.tools) {
96+
for (const tool of request.tools) {
97+
availableTools.push({
98+
name: tool.name,
99+
description: tool.description,
100+
parameters: tool.input_schema?.properties || {},
101+
});
102+
}
103+
}
104+
105+
return {
106+
messages,
107+
available_tools: availableTools,
108+
};
109+
}
110+
111+
private async handleStreaming(response: Array<MessageStreamEvent>): Promise<LLMMessage[]> {
112+
const messages: LLMMessage[] = [];
113+
114+
let role: string | undefined;
115+
let accumulatedContent: string[] = [];
116+
let accumulatedToolName: string | undefined;
117+
let accumulatedToolId: string | undefined;
118+
let accumulatedToolInput: string[] = [];
119+
for (const responseEvent of response) {
120+
switch (responseEvent.type) {
121+
case 'message_start':
122+
const rawMessageStartEvent = responseEvent as RawMessageStartEvent;
123+
role = rawMessageStartEvent.message.role;
124+
accumulatedContent = [];
125+
accumulatedToolName = undefined;
126+
accumulatedToolId = undefined;
127+
accumulatedToolInput = [];
128+
break;
129+
case 'content_block_start':
130+
const rawContentBlockStartEvent = responseEvent as RawContentBlockStartEvent;
131+
switch (rawContentBlockStartEvent.content_block.type) {
132+
case 'text':
133+
const textBlock = rawContentBlockStartEvent.content_block as TextBlock;
134+
accumulatedContent.push(textBlock.text)
135+
break;
136+
case 'tool_use':
137+
const toolUseBlock = rawContentBlockStartEvent.content_block as ToolUseBlock;
138+
accumulatedToolId = toolUseBlock.id
139+
accumulatedToolName = toolUseBlock.name
140+
accumulatedToolInput = []
141+
break;
142+
case 'thinking':
143+
const thinkingBlock = rawContentBlockStartEvent.content_block as ThinkingBlock;
144+
accumulatedContent.push(thinkingBlock.thinking)
145+
break;
146+
default:
147+
console.debug(`Invalid content block type: ${responseEvent}`);
148+
}
149+
break;
150+
case 'content_block_delta':
151+
const rawContentBlockDeltaEvent = responseEvent as RawContentBlockDeltaEvent;
152+
switch (rawContentBlockDeltaEvent.delta.type) {
153+
case 'text_delta':
154+
const textDelta = rawContentBlockDeltaEvent.delta as TextDelta;
155+
accumulatedContent.push(textDelta.text)
156+
break;
157+
case 'input_json_delta':
158+
const inputJsonDelta = rawContentBlockDeltaEvent.delta as InputJSONDelta;
159+
accumulatedToolInput.push(inputJsonDelta.partial_json)
160+
break;
161+
default:
162+
console.debug(`Invalid delta type: ${rawContentBlockDeltaEvent}`);
163+
}
164+
break;
165+
case 'message_stop':
166+
let finalContent: string | undefined;
167+
if (accumulatedContent.length > 0) {
168+
finalContent = accumulatedContent.join('').trim();
169+
}
170+
let finalTool: LLMToolCall | undefined;
171+
if (accumulatedToolName) {
172+
finalTool = {
173+
id: accumulatedToolId,
174+
name: accumulatedToolName,
175+
arguments: JSON.parse(accumulatedToolInput.join('')),
176+
};
177+
};
178+
if (!role) {
179+
console.debug(`role was not set`);
180+
continue;
181+
}
182+
messages.push({
183+
role: role == 'model' ? 'assistant' : role,
184+
content: finalContent ?? undefined,
185+
tool_calls: finalTool ? [finalTool] : undefined,
186+
});
187+
role = undefined;
188+
accumulatedContent = [];
189+
accumulatedToolName = undefined;
190+
accumulatedToolId = undefined;
191+
accumulatedToolInput = [];
192+
break;
193+
case 'content_block_stop':
194+
case 'message_delta':
195+
break;
196+
default:
197+
console.debug(`Invalid event: ${responseEvent}`);
198+
}
199+
}
200+
return messages;
201+
}
202+
203+
private async handleNonStreamingResponse(
204+
response: any
205+
): Promise<LLMMessage[]> {
206+
const messages: LLMMessage[] = [];
207+
208+
if (response.role !== 'assistant') {
209+
throw new Error(
210+
`Response role must be 'assistant'. Make sure to use response
211+
from anthropic.messages.create() when not using streaming.`);
212+
}
213+
messages.push(...this.convertClaudeMessagesToLLMMessages([response] as Array<Message>));
214+
215+
return messages;
216+
}
217+
218+
// Claude-specific function to convert Response API messages to LLM messages
219+
private convertClaudeMessagesToLLMMessages(messages: Array<Message>): LLMMessage[] {
220+
const extractedMessages: LLMMessage[] = [];
221+
222+
for (const message of messages) {
223+
if (typeof message.content === 'string') {
224+
const llmMessage: LLMMessage = {
225+
role: message.role,
226+
content: message.content,
227+
};
228+
extractedMessages.push(llmMessage);
229+
continue;
230+
}
231+
const aggregatedContent: string[] = [];
232+
const aggregatedToolCalls: LLMToolCall[] = [];
233+
let role: string = message.role;
234+
if (!message.content) {
235+
continue;
236+
}
237+
for (const part of (message.content as Array<ContentBlockParam>)) {
238+
switch (part.type) {
239+
case 'tool_use':
240+
const toolUseBlock = part as ToolUseBlock;
241+
aggregatedToolCalls.push({
242+
name: toolUseBlock.name,
243+
arguments: toolUseBlock.input as Record<string, any>,
244+
id: toolUseBlock.id,
245+
});
246+
break;
247+
case 'tool_result':
248+
role = 'tool'; // Claude expects 'user' role for tool results. But Qualifire treats tool as results as it is sent from 'tool'
249+
const toolResultBlock = part as ToolResultBlockParam;
250+
if (typeof toolResultBlock.content === 'string') {
251+
aggregatedContent.push(toolResultBlock.content)
252+
} else {
253+
(toolResultBlock.content as Array<ContentBlockParam>).filter(part => part.type === 'text').forEach(part => {
254+
const textPart = part as TextBlockParam;
255+
aggregatedContent.push(textPart.text)
256+
})
257+
}
258+
break;
259+
case 'text':
260+
const textBlock = part as TextBlock;
261+
aggregatedContent.push(textBlock.text)
262+
break;
263+
default:
264+
console.debug(
265+
'Invalid Claude output: message - ' +
266+
JSON.stringify(message) +
267+
' part - ' +
268+
JSON.stringify(part)
269+
);
270+
}
271+
}
272+
273+
// If we accumulated aggregatedContent or aggregatedToolCalls, add the message
274+
if (aggregatedContent.length > 0 || aggregatedToolCalls.length > 0) {
275+
let accumulatedMessage: LLMMessage = {
276+
role,
277+
};
278+
279+
if (aggregatedContent.length > 0) {
280+
accumulatedMessage.content = aggregatedContent.join('');
281+
}
282+
283+
// Only add aggregatedToolCalls property for assistant messages
284+
if (aggregatedToolCalls.length > 0) {
285+
accumulatedMessage.tool_calls = aggregatedToolCalls;
286+
}
287+
extractedMessages.push(accumulatedMessage);
288+
}
289+
}
290+
return extractedMessages;
291+
}
292+
}

0 commit comments

Comments
 (0)