Hotfix | New evaluations-api schema fix (#21)

* fix: new Evaluations-api schema fixes

* Bump version

* Bump version

Author: yuval-qf

package.json

@@ -1,6 +1,6 @@
 {
   "name": "qualifire",
-  "version": "0.1.0",
+  "version": "1.3.0",
   "description": "Qualifire client SDK",
   "main": "./lib/index.js",
   "files": [
@@ -123,4 +123,4 @@
     "zod": "^4.0.0"
   },
   "packageManager": "pnpm@10.14.0"
-}
+}

src/frameworks/claude/claude-converter.ts

@@ -119,43 +119,43 @@ export class ClaudeCanonicalEvaluationStrategy
     for (const responseEvent of response) {
       switch (responseEvent.type) {
         case 'message_start':
-          const rawMessageStartEvent = responseEvent as RawMessageStartEvent;
+          const rawMessageStartEvent = responseEvent ;
           role = rawMessageStartEvent.message.role;
           accumulatedContent = [];
           accumulatedToolName = undefined;
           accumulatedToolId = undefined;
           accumulatedToolInput = [];
           break;
         case 'content_block_start':
-          const rawContentBlockStartEvent = responseEvent as RawContentBlockStartEvent;
+          const rawContentBlockStartEvent = responseEvent ;
           switch (rawContentBlockStartEvent.content_block.type) {
             case 'text':
-              const textBlock = rawContentBlockStartEvent.content_block as TextBlock;  
+              const textBlock = rawContentBlockStartEvent.content_block ;  
               accumulatedContent.push(textBlock.text)
               break;
             case 'tool_use':
-              const toolUseBlock = rawContentBlockStartEvent.content_block as ToolUseBlock;
+              const toolUseBlock = rawContentBlockStartEvent.content_block ;
               accumulatedToolId = toolUseBlock.id
               accumulatedToolName = toolUseBlock.name
               accumulatedToolInput = []
               break;
             case 'thinking':
-              const thinkingBlock = rawContentBlockStartEvent.content_block as ThinkingBlock;
+              const thinkingBlock = rawContentBlockStartEvent.content_block ;
               accumulatedContent.push(thinkingBlock.thinking)
               break;
             default:
               console.debug(`Invalid content block type: ${responseEvent}`);
           }
           break;
         case 'content_block_delta':
-          const rawContentBlockDeltaEvent = responseEvent as RawContentBlockDeltaEvent;
+          const rawContentBlockDeltaEvent = responseEvent ;
           switch (rawContentBlockDeltaEvent.delta.type) {
               case 'text_delta':
-                const textDelta = rawContentBlockDeltaEvent.delta as TextDelta;
+                const textDelta = rawContentBlockDeltaEvent.delta ;
                 accumulatedContent.push(textDelta.text)
                 break;
               case 'input_json_delta':
-                const inputJsonDelta = rawContentBlockDeltaEvent.delta as InputJSONDelta;
+                const inputJsonDelta = rawContentBlockDeltaEvent.delta ;
                 accumulatedToolInput.push(inputJsonDelta.partial_json)
                 break;
               default:
@@ -176,7 +176,7 @@ export class ClaudeCanonicalEvaluationStrategy
             };
           };
           if (!role) {
-            console.debug(`role was not set`);
+            console.debug('role was not set');
             continue;
           }
           messages.push({
@@ -246,7 +246,7 @@ export class ClaudeCanonicalEvaluationStrategy
             break;
           case 'tool_result':
             role = 'tool'; // Claude expects 'user' role for tool results. But Qualifire treats tool as results as it is sent from 'tool'
-            const toolResultBlock = part as ToolResultBlockParam;
+            const toolResultBlock = part ;
             if (typeof toolResultBlock.content === 'string') {
               aggregatedContent.push(toolResultBlock.content)
             } else {
@@ -272,7 +272,7 @@ export class ClaudeCanonicalEvaluationStrategy
 
       // If we accumulated aggregatedContent or aggregatedToolCalls, add the message
       if (aggregatedContent.length > 0 || aggregatedToolCalls.length > 0) {
-        let accumulatedMessage: LLMMessage = {
+        const accumulatedMessage: LLMMessage = {
           role,
         };
 

src/frameworks/gemini/gemini-converter.ts

@@ -72,7 +72,7 @@ export class GeminiAICanonicalEvaluationStrategy
       }
     }
 
-    let contents: Array<Content> = [];
+    const contents: Array<Content> = [];
     if (request?.config?.systemInstruction) {
       /*
         Gemini request.contents is an object called contentListUnions which can be
@@ -154,9 +154,9 @@ export class GeminiAICanonicalEvaluationStrategy
   ): Promise<LLMMessage[]> {
     const messages: LLMMessage[] = [];
 
-    let accumulatedContentParts: string[] = [];
+    const accumulatedContentParts: string[] = [];
     let currentRole = 'assistant';
-    let toolCalls: LLMToolCall[] = [];
+    const toolCalls: LLMToolCall[] = [];
     for (const chunk of response) {
       if (chunk?.candidates && chunk.candidates.length > 0) {
         if (chunk.candidates.length > 1) {
@@ -216,8 +216,8 @@ function convertContentToLLMMessage(content: any): LLMMessage | null {
 
   // In Gemini role is optional, but by default the api is changing it to 'user' when no role is provided
   let role = content.role || 'user';
-  let textContent: string[] = [];
-  let tool_calls: LLMToolCall[] = [];
+  const textContent: string[] = [];
+  const tool_calls: LLMToolCall[] = [];
 
   // Process all parts and aggregate them
   for (const part of content.parts) {
@@ -304,7 +304,7 @@ function convertContentListUnionsToContentList(
   if (inputs.every(isContent)) {
     convertedContents = inputs as Array<Content>;
   } else if (inputs.every(isPartOrString)) {
-    let partInputs: Array<Part> = [];
+    const partInputs: Array<Part> = [];
     for (const partOrString of inputs) {
       if (typeof partOrString === 'string') {
         partInputs.push({ text: partOrString });

src/frameworks/openai/openai-converter.ts

@@ -131,7 +131,7 @@ export class OpenAICanonicalEvaluationStrategy
 
     if (request?.messages) {
       messages.push(
-        ...this.convertRequestMessagesForChatCompletions(request.messages as ChatCompletionMessageParam[])
+        ...this.convertRequestMessagesForChatCompletions(request.messages )
       );
     }
 
@@ -158,25 +158,25 @@ export class OpenAICanonicalEvaluationStrategy
         case 'system':
         case 'developer':
         case 'user':
-          let textMessage = message as ChatCompletionDeveloperMessageParam // This works the same for System user and developer. Used this type just out of comfort
+          const textMessage = message as ChatCompletionDeveloperMessageParam // This works the same for System user and developer. Used this type just out of comfort
           if (typeof textMessage.content === 'string') {
-            content = textMessage.content as string;
+            content = textMessage.content ;
           } else {
-            content = (textMessage.content as ChatCompletionContentPartText[]).filter((part) => part.type === 'text').map((part) => part.text).join('')
+            content = (textMessage.content ).filter((part) => part.type === 'text').map((part) => part.text).join('')
           }
           break;
         case 'assistant':
-          let assistantMessage = message as ChatCompletionAssistantMessageParam
+          const assistantMessage = message 
           if (assistantMessage.content) {
             if (typeof assistantMessage.content === 'string') {
               content = assistantMessage.content;
             } else {
-              content = (assistantMessage.content as Array<ChatCompletionContentPartText | ChatCompletionContentPartRefusal>).filter((part) => part.type === 'text').map((part) => (part as ChatCompletionContentPartText).text).join('')
+              content = (assistantMessage.content ).filter((part) => part.type === 'text').map((part) => (part as ChatCompletionContentPartText).text).join('')
             }
           }
           if (assistantMessage.tool_calls) {
             toolCalls = assistantMessage.tool_calls
-            .filter((toolCalls) => toolCalls.type === "function")
+            .filter((toolCalls) => toolCalls.type === 'function')
             .map((toolCalls) => {
                   // It's of type ChatCompletionMessageFunctionToolCall but it's not exported
                   let toolArguments = {}
@@ -195,12 +195,12 @@ export class OpenAICanonicalEvaluationStrategy
           }
           break;
         case 'tool':
-          let toolMessage = message as ChatCompletionToolMessageParam
+          const toolMessage = message 
           if (toolMessage.content) {
             if (typeof toolMessage.content === 'string') {
               content = toolMessage.content;
             } else {
-              content = (toolMessage.content as Array<ChatCompletionContentPartText>).filter((part) => part.type === 'text').map((part) => part.text).join('')
+              content = (toolMessage.content ).filter((part) => part.type === 'text').map((part) => part.text).join('')
             }
           }
           break;
@@ -520,7 +520,7 @@ export class OpenAICanonicalEvaluationStrategy
   private async handleChatCompletionsNonStreaming(
     response: ChatCompletion
   ): Promise<LLMMessage[]> {
-    response = response as ChatCompletion;
+    response = response ;
 
     const messages: LLMMessage[] = [];
 
@@ -565,7 +565,7 @@ export class OpenAICanonicalEvaluationStrategy
 
     if (response.output) {
       messages.push(
-        ...convertResponsesAPIMessagesToLLMMessages(response.output as Array<ResponseOutputItem>)
+        ...convertResponsesAPIMessagesToLLMMessages(response.output )
       );
     } else {
       console.debug(
@@ -619,8 +619,8 @@ function convertResponsesAPIMessagesToLLMMessages(
       continue
     }
 
-    let aggregatedContent: string[] = [];
-    let aggregatedToolCalls: LLMToolCall[] = [];
+    const aggregatedContent: string[] = [];
+    const aggregatedToolCalls: LLMToolCall[] = [];
     let role = message.role;
 
     for (const contentElement of message.content) {

src/frameworks/vercelai/vercelai-converter.ts

@@ -210,14 +210,14 @@ export class VercelAICanonicalEvaluationStrategy
     for (const message of messages) {
       switch (message.role) {
         case 'system':
-          const systemMessage = message as SystemModelMessage;
+          const systemMessage = message ;
           extractedMessages.push({
             role: 'system',
             content: systemMessage.content,
           });
           break;
         case 'user':
-          const userMessage = message as UserModelMessage;
+          const userMessage = message ;
           if (typeof userMessage.content === 'string') {
             extractedMessages.push({
               role: 'user',
@@ -237,7 +237,7 @@ export class VercelAICanonicalEvaluationStrategy
           }
           break;
         case 'assistant':
-          const assistantMessage = message as AssistantModelMessage;
+          const assistantMessage = message ;
           if (typeof assistantMessage.content === 'string') {
             extractedMessages.push({
               role: 'assistant',
@@ -264,7 +264,7 @@ export class VercelAICanonicalEvaluationStrategy
           }
           break;
         case 'tool':
-          const toolMessage = message as ToolModelMessage;
+          const toolMessage = message ;
           const toolContent = extractToolContent(toolMessage);
           extractedMessages.push({
             role: 'tool',
@@ -292,7 +292,7 @@ function extractToolContent(toolMessage: ToolModelMessage): string {
   }
   const firstContent = content[0];
   if (firstContent.type !== 'tool-result') {
-    console.debug(`Expected tool-result in the first content of a tool message. Seems like invalid tool was given.`);
+    console.debug('Expected tool-result in the first content of a tool message. Seems like invalid tool was given.');
     return '';
   }
   // LanguageModelV2ToolResultOutput is not exported unfortunaly so we have to use this type
@@ -316,12 +316,12 @@ function extractToolContent(toolMessage: ToolModelMessage): string {
 }
 
 function extractToolCalls(content: ToolCallPart[]): LLMToolCall[] {
-  let toolCalls: LLMToolCall[] = [];
+  const toolCalls: LLMToolCall[] = [];
   for (const part of content) {
     if (part.type !== 'tool-call') {
       continue;
     }
-    let toolCall: unknown = {
+    const toolCall: unknown = {
       name: part.toolName,
       arguments: part.input,
       id: part.toolCallId,

src/index.ts

@@ -1,15 +1,13 @@
 import * as traceloop from '@traceloop/node-server-sdk';
+import { CanonicalEvaluationStrategy } from './frameworks/canonical';
 import { ClaudeCanonicalEvaluationStrategy } from './frameworks/claude/claude-converter';
 import { GeminiAICanonicalEvaluationStrategy } from './frameworks/gemini/gemini-converter';
 import { OpenAICanonicalEvaluationStrategy } from './frameworks/openai/openai-converter';
 import { VercelAICanonicalEvaluationStrategy } from './frameworks/vercelai/vercelai-converter';
 import { EvaluationProxyAPIRequest, EvaluationProxyAPIRequestSchema, EvaluationRequestV2Schema, type EvaluationRequestV2, type EvaluationResponse, type Framework } from './types';
-import { CanonicalEvaluationStrategy } from './frameworks/canonical';
 
 export type {
-  EvaluationRequestV2,
-  EvaluationProxyAPIRequest,
-  EvaluationResponse, Framework, LLMMessage
+  EvaluationProxyAPIRequest, EvaluationRequestV2, EvaluationResponse, Framework, LLMMessage, ModelMode, PolicyTarget
 } from './types';
 
 
@@ -102,15 +100,12 @@ export class Qualifire {
    *  framework: 'openai',
    *  request: openaiRequest, // As given to openaiClient.chat.completions.create(), openaiClient.responses.create()
    *  response: openaiResponse, // Response as returned by openaiClient.chat.completions.create() or openaiClient.responses.create()
-   *  dangerousContentCheck: true,
+   *  contentModerationCheck: true,
    *  groundingCheck: true,
    *  hallucinationsCheck: true,
-   *  harassmentCheck: true,
-   *  hateSpeechCheck: true,
    *  instructionsFollowingCheck: true,
    *  piiCheck: true,
    *  promptInjections: true,
-   *  sexualContentCheck: true,
    *  toolSelectionQualityCheck: false,
    * });
    * 
@@ -158,7 +153,7 @@ export class Qualifire {
    *     { role: 'user', content: 'What is the capital of France?' },
    *     { role: 'assistant', content: 'Paris' }
    *   ],
-   *   dangerousContentCheck: true,
+   *   contentModerationCheck: true,
    *   hallucinationsCheck: true,
    * });
    * ```
@@ -202,21 +197,29 @@ export class Qualifire {
    * Evaluates using direct messages without conversion (overrides request/response if both are provided)
    */
   private evaluateWithBackwardCompatibility = async (evaluationProxyAPIRequest: EvaluationProxyAPIRequest): Promise<EvaluationResponse | undefined> => {
+    // Compute contentModerationCheck from deprecated fields or use the new field
+    const contentModerationCheck = evaluationProxyAPIRequest.contentModerationCheck || 
+      evaluationProxyAPIRequest.dangerous_content_check || 
+      evaluationProxyAPIRequest.dangerousContentCheck || 
+      evaluationProxyAPIRequest.harassment_check || 
+      evaluationProxyAPIRequest.harassmentCheck || 
+      evaluationProxyAPIRequest.hate_speech_check || 
+      evaluationProxyAPIRequest.hateSpeechCheck || 
+      evaluationProxyAPIRequest.sexual_content_check || 
+      evaluationProxyAPIRequest.sexualContentCheck;
+
     const url = `${this.baseUrl}/api/evaluation/evaluate`;
     const body = {
       input: evaluationProxyAPIRequest.input,
       output: evaluationProxyAPIRequest.output,
       messages: evaluationProxyAPIRequest.messages,
       available_tools: evaluationProxyAPIRequest.available_tools,
-      dangerous_content_check: evaluationProxyAPIRequest.dangerous_content_check || evaluationProxyAPIRequest.dangerousContentCheck,
+      content_moderation_check: contentModerationCheck,
       grounding_check: evaluationProxyAPIRequest.grounding_check || evaluationProxyAPIRequest.groundingCheck,
       hallucinations_check: evaluationProxyAPIRequest.hallucinations_check || evaluationProxyAPIRequest.hallucinationsCheck,
-      harassment_check: evaluationProxyAPIRequest.harassment_check || evaluationProxyAPIRequest.harassmentCheck,
-      hate_speech_check: evaluationProxyAPIRequest.hate_speech_check || evaluationProxyAPIRequest.hateSpeechCheck,
       instructions_following_check: evaluationProxyAPIRequest.instructions_following_check || evaluationProxyAPIRequest.instructionsFollowingCheck,
       pii_check: evaluationProxyAPIRequest.pii_check || evaluationProxyAPIRequest.piiCheck,
       prompt_injections: evaluationProxyAPIRequest.prompt_injections || evaluationProxyAPIRequest.promptInjections,
-      sexual_content_check: evaluationProxyAPIRequest.sexual_content_check || evaluationProxyAPIRequest.sexualContentCheck,
       syntax_checks: evaluationProxyAPIRequest.syntax_checks || evaluationProxyAPIRequest.syntaxChecks,
       tool_selection_quality_check: evaluationProxyAPIRequest.tool_selection_quality_check || evaluationProxyAPIRequest.toolSelectionQualityCheck,
       assertions: evaluationProxyAPIRequest.assertions,
@@ -245,6 +248,13 @@ export class Qualifire {
    * Evaluates using framework converters for request/response
    */
   private evaluateWithConverters = async (EvaluationRequestV2: EvaluationRequestV2): Promise<EvaluationResponse | undefined> => {
+    // Compute contentModerationCheck from deprecated fields or use the new field
+    const contentModerationCheck = EvaluationRequestV2.contentModerationCheck || 
+      EvaluationRequestV2.dangerousContentCheck || 
+      EvaluationRequestV2.harassmentCheck || 
+      EvaluationRequestV2.hateSpeechCheck || 
+      EvaluationRequestV2.sexualContentCheck;
+
     const frameworkConverters: Record<Framework, () => CanonicalEvaluationStrategy<any, any>> = {
       'openai': () => new OpenAICanonicalEvaluationStrategy(),
       'vercelai': () => new VercelAICanonicalEvaluationStrategy(),
@@ -268,15 +278,12 @@ export class Qualifire {
     const body = {
       messages: evaluationRequest.messages,
       available_tools: evaluationRequest.available_tools,
-      dangerous_content_check: EvaluationRequestV2.dangerousContentCheck,
+      content_moderation_check: contentModerationCheck,
       grounding_check: EvaluationRequestV2.groundingCheck,
       hallucinations_check: EvaluationRequestV2.hallucinationsCheck,
-      harassment_check: EvaluationRequestV2.harassmentCheck,
-      hate_speech_check: EvaluationRequestV2.hateSpeechCheck,
       instructions_following_check: EvaluationRequestV2.instructionsFollowingCheck,
       pii_check: EvaluationRequestV2.piiCheck,
       prompt_injections: EvaluationRequestV2.promptInjections,
-      sexual_content_check: EvaluationRequestV2.sexualContentCheck,
       syntax_checks: EvaluationRequestV2.syntaxChecks,
       tool_selection_quality_check: EvaluationRequestV2.toolSelectionQualityCheck,
       assertions: EvaluationRequestV2.assertions,