[FIRE-987] feature: support new tool_use_quality_check & tuq_mode fields and deprecate tool_selection_quality_check & tsq_mode fields

package.json

@@ -1,6 +1,6 @@
 {
   "name": "qualifire",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Qualifire client SDK",
   "main": "./lib/index.js",
   "files": [

src/index.ts

@@ -4,13 +4,25 @@ import { ClaudeCanonicalEvaluationStrategy } from './frameworks/claude/claude-co
 import { GeminiAICanonicalEvaluationStrategy } from './frameworks/gemini/gemini-converter';
 import { OpenAICanonicalEvaluationStrategy } from './frameworks/openai/openai-converter';
 import { VercelAICanonicalEvaluationStrategy } from './frameworks/vercelai/vercelai-converter';
-import { EvaluationProxyAPIRequest, EvaluationProxyAPIRequestSchema, EvaluationRequestV2Schema, type EvaluationRequestV2, type EvaluationResponse, type Framework } from './types';
+import {
+  EvaluationProxyAPIRequest,
+  EvaluationProxyAPIRequestSchema,
+  EvaluationRequestV2Schema,
+  type EvaluationRequestV2,
+  type EvaluationResponse,
+  type Framework,
+} from './types';
 
 export type {
-  EvaluationProxyAPIRequest, EvaluationRequestV2, EvaluationResponse, Framework, LLMMessage, ModelMode, PolicyTarget
+  EvaluationProxyAPIRequest,
+  EvaluationRequestV2,
+  EvaluationResponse,
+  Framework,
+  LLMMessage,
+  ModelMode,
+  PolicyTarget,
 } from './types';
 
-
 /**
  * Represents the Qualifire SDK.
  */
@@ -61,7 +73,7 @@ export class Qualifire {
    * This function supports two modes:
    * 1. Request-Response mode: If `request`, `response` and `framework` are provided, they fully analyzed by the Qualifire API
    * 2. Fine-grained messages mode: If `messages`, `input` or `output` are provided, they are sent specifically to the Qualifire API
-   * 
+   *
    * Supported frameworks are: openai, vercelai, gemini, claude
    * Note: Direct messages are deprecated, but still supported for backward compatibility.
    *
@@ -91,11 +103,11 @@ export class Qualifire {
    *    },
    *  ],
    * };
-   * 
+   *
    * const openAiResponse = await openaiClient.chat.completions.create(
    *   openAiRequest
    * );
-   * 
+   *
    * const qualifireResponse = await qualifireClient.evaluate({
    *  framework: 'openai',
    *  request: openaiRequest, // As given to openaiClient.chat.completions.create(), openaiClient.responses.create()
@@ -106,9 +118,9 @@ export class Qualifire {
    *  instructionsFollowingCheck: true,
    *  piiCheck: true,
    *  promptInjections: true,
-   *  toolSelectionQualityCheck: false,
+   *  toolUseQualityCheck: false, // Use this instead of deprecated toolSelectionQualityCheck
    * });
-   * 
+   *
    * // If you are using streaming mode.
    * const openAiRequestStream = {
    *  stream: true,
@@ -129,11 +141,11 @@ export class Qualifire {
    *    },
    *  ],
    * };
-   * 
+   *
    * const openAiResponseStream = await openaiClient.chat.completions.create(
    *   openAiRequestStream
    * );
-   * 
+   *
    * let ResponseChunks: any[] = [];
    * for await (const chunk of openAiResponseStream) {
    *   ResponseChunks.push(chunk);
@@ -146,7 +158,7 @@ export class Qualifire {
    *   groundingCheck: true,
    *   promptInjections: true,
    * });
-   * 
+   *
    * // Fine-grained messages mode
    * const response2 = await qualifire.evaluate({
    *   messages: [
@@ -157,7 +169,7 @@ export class Qualifire {
    *   hallucinationsCheck: true,
    * });
    * ```
-   * 
+   *
    * // A typical output for qualifire response would be:
    * Qualifire response: {
    *  "status": "failed",
@@ -178,34 +190,45 @@ export class Qualifire {
    *  ]
    * }
    */
-  evaluate = async (evaluationRequest: EvaluationProxyAPIRequest | EvaluationRequestV2): Promise<EvaluationResponse | undefined> => {
+  evaluate = async (
+    evaluationRequest: EvaluationProxyAPIRequest | EvaluationRequestV2
+  ): Promise<EvaluationResponse | undefined> => {
     // If messages are provided directly, use them as-is without conversion
-    const parseEvaluationProxyAPIRequest = EvaluationProxyAPIRequestSchema.safeParse(evaluationRequest)
+    const parseEvaluationProxyAPIRequest =
+      EvaluationProxyAPIRequestSchema.safeParse(evaluationRequest);
     if (parseEvaluationProxyAPIRequest.success) {
-      return this.evaluateWithBackwardCompatibility(parseEvaluationProxyAPIRequest.data as EvaluationProxyAPIRequest);
+      return this.evaluateWithBackwardCompatibility(
+        parseEvaluationProxyAPIRequest.data as EvaluationProxyAPIRequest
+      );
     }
 
-    const parseEvaluationRequestResultV2 = EvaluationRequestV2Schema.safeParse(evaluationRequest)
+    const parseEvaluationRequestResultV2 =
+      EvaluationRequestV2Schema.safeParse(evaluationRequest);
     if (parseEvaluationRequestResultV2.success) {
-      return this.evaluateWithConverters(parseEvaluationRequestResultV2.data );
+      return this.evaluateWithConverters(parseEvaluationRequestResultV2.data);
     }
-
-    throw new Error(`Invalid evaluation request format: ${JSON.stringify(evaluationRequest)}`);
+
+    throw new Error(
+      `Invalid evaluation request format: ${JSON.stringify(evaluationRequest)}`
+    );
   };
 
   /**
    * Evaluates using direct messages without conversion (overrides request/response if both are provided)
    */
-  private evaluateWithBackwardCompatibility = async (evaluationProxyAPIRequest: EvaluationProxyAPIRequest): Promise<EvaluationResponse | undefined> => {
+  private evaluateWithBackwardCompatibility = async (
+    evaluationProxyAPIRequest: EvaluationProxyAPIRequest
+  ): Promise<EvaluationResponse | undefined> => {
     // Compute contentModerationCheck from deprecated fields or use the new field
-    const contentModerationCheck = evaluationProxyAPIRequest.contentModerationCheck || 
-      evaluationProxyAPIRequest.dangerous_content_check || 
-      evaluationProxyAPIRequest.dangerousContentCheck || 
-      evaluationProxyAPIRequest.harassment_check || 
-      evaluationProxyAPIRequest.harassmentCheck || 
-      evaluationProxyAPIRequest.hate_speech_check || 
-      evaluationProxyAPIRequest.hateSpeechCheck || 
-      evaluationProxyAPIRequest.sexual_content_check || 
+    const contentModerationCheck =
+      evaluationProxyAPIRequest.contentModerationCheck ||
+      evaluationProxyAPIRequest.dangerous_content_check ||
+      evaluationProxyAPIRequest.dangerousContentCheck ||
+      evaluationProxyAPIRequest.harassment_check ||
+      evaluationProxyAPIRequest.harassmentCheck ||
+      evaluationProxyAPIRequest.hate_speech_check ||
+      evaluationProxyAPIRequest.hateSpeechCheck ||
+      evaluationProxyAPIRequest.sexual_content_check ||
       evaluationProxyAPIRequest.sexualContentCheck;
 
     const url = `${this.baseUrl}/api/evaluation/evaluate`;
@@ -215,13 +238,30 @@ export class Qualifire {
       messages: evaluationProxyAPIRequest.messages,
       available_tools: evaluationProxyAPIRequest.available_tools,
       content_moderation_check: contentModerationCheck,
-      grounding_check: evaluationProxyAPIRequest.grounding_check || evaluationProxyAPIRequest.groundingCheck,
-      hallucinations_check: evaluationProxyAPIRequest.hallucinations_check || evaluationProxyAPIRequest.hallucinationsCheck,
-      instructions_following_check: evaluationProxyAPIRequest.instructions_following_check || evaluationProxyAPIRequest.instructionsFollowingCheck,
-      pii_check: evaluationProxyAPIRequest.pii_check || evaluationProxyAPIRequest.piiCheck,
-      prompt_injections: evaluationProxyAPIRequest.prompt_injections || evaluationProxyAPIRequest.promptInjections,
-      syntax_checks: evaluationProxyAPIRequest.syntax_checks || evaluationProxyAPIRequest.syntaxChecks,
-      tool_selection_quality_check: evaluationProxyAPIRequest.tool_selection_quality_check || evaluationProxyAPIRequest.toolSelectionQualityCheck,
+      grounding_check:
+        evaluationProxyAPIRequest.grounding_check ||
+        evaluationProxyAPIRequest.groundingCheck,
+      hallucinations_check:
+        evaluationProxyAPIRequest.hallucinations_check ||
+        evaluationProxyAPIRequest.hallucinationsCheck,
+      instructions_following_check:
+        evaluationProxyAPIRequest.instructions_following_check ||
+        evaluationProxyAPIRequest.instructionsFollowingCheck,
+      pii_check:
+        evaluationProxyAPIRequest.pii_check ||
+        evaluationProxyAPIRequest.piiCheck,
+      prompt_injections:
+        evaluationProxyAPIRequest.prompt_injections ||
+        evaluationProxyAPIRequest.promptInjections,
+      syntax_checks:
+        evaluationProxyAPIRequest.syntax_checks ||
+        evaluationProxyAPIRequest.syntaxChecks,
+      tool_use_quality_check:
+        evaluationProxyAPIRequest.toolUseQualityCheck ||
+        evaluationProxyAPIRequest.toolSelectionQualityCheck ||
+        evaluationProxyAPIRequest.tool_selection_quality_check,
+      tuq_mode:
+        evaluationProxyAPIRequest.tuqMode ?? evaluationProxyAPIRequest.tsqMode,
       assertions: evaluationProxyAPIRequest.assertions,
     };
 
@@ -247,32 +287,45 @@ export class Qualifire {
   /**
    * Evaluates using framework converters for request/response
    */
-  private evaluateWithConverters = async (EvaluationRequestV2: EvaluationRequestV2): Promise<EvaluationResponse | undefined> => {
+  private evaluateWithConverters = async (
+    EvaluationRequestV2: EvaluationRequestV2
+  ): Promise<EvaluationResponse | undefined> => {
     // Compute contentModerationCheck from deprecated fields or use the new field
-    const contentModerationCheck = EvaluationRequestV2.contentModerationCheck || 
-      EvaluationRequestV2.dangerousContentCheck || 
-      EvaluationRequestV2.harassmentCheck || 
-      EvaluationRequestV2.hateSpeechCheck || 
+    const contentModerationCheck =
+      EvaluationRequestV2.contentModerationCheck ||
+      EvaluationRequestV2.dangerousContentCheck ||
+      EvaluationRequestV2.harassmentCheck ||
+      EvaluationRequestV2.hateSpeechCheck ||
       EvaluationRequestV2.sexualContentCheck;
 
-    const frameworkConverters: Record<Framework, () => CanonicalEvaluationStrategy<any, any>> = {
-      'openai': () => new OpenAICanonicalEvaluationStrategy(),
-      'vercelai': () => new VercelAICanonicalEvaluationStrategy(),
-      'gemini': () => new GeminiAICanonicalEvaluationStrategy(),
-      'claude': () => new ClaudeCanonicalEvaluationStrategy(),
+    const frameworkConverters: Record<
+      Framework,
+      () => CanonicalEvaluationStrategy<any, any>
+    > = {
+      openai: () => new OpenAICanonicalEvaluationStrategy(),
+      vercelai: () => new VercelAICanonicalEvaluationStrategy(),
+      gemini: () => new GeminiAICanonicalEvaluationStrategy(),
+      claude: () => new ClaudeCanonicalEvaluationStrategy(),
     };
 
     const supportedFrameworks = Object.keys(frameworkConverters);
     const converterFactory = frameworkConverters[EvaluationRequestV2.framework];
-    
+
     if (!converterFactory) {
-      throw new Error(`Unsupported framework: ${EvaluationRequestV2.framework}. Supported frameworks: ${supportedFrameworks.join(', ')}`);
+      throw new Error(
+        `Unsupported framework: ${
+          EvaluationRequestV2.framework
+        }. Supported frameworks: ${supportedFrameworks.join(', ')}`
+      );
     }
 
     const requestConverter = converterFactory();
 
-
-    const evaluationRequest = await requestConverter.convertToQualifireEvaluationRequest(EvaluationRequestV2.request, EvaluationRequestV2.response)
+    const evaluationRequest =
+      await requestConverter.convertToQualifireEvaluationRequest(
+        EvaluationRequestV2.request,
+        EvaluationRequestV2.response
+      );
 
     const url = `${this.baseUrl}/api/evaluation/evaluate`;
     const body = {
@@ -281,11 +334,16 @@ export class Qualifire {
       content_moderation_check: contentModerationCheck,
       grounding_check: EvaluationRequestV2.groundingCheck,
       hallucinations_check: EvaluationRequestV2.hallucinationsCheck,
-      instructions_following_check: EvaluationRequestV2.instructionsFollowingCheck,
+      instructions_following_check:
+        EvaluationRequestV2.instructionsFollowingCheck,
       pii_check: EvaluationRequestV2.piiCheck,
       prompt_injections: EvaluationRequestV2.promptInjections,
       syntax_checks: EvaluationRequestV2.syntaxChecks,
-      tool_selection_quality_check: EvaluationRequestV2.toolSelectionQualityCheck,
+      tool_use_quality_check:
+        EvaluationRequestV2.toolUseQualityCheck ||
+        EvaluationRequestV2.toolSelectionQualityCheck ||
+        EvaluationRequestV2.tool_selection_quality_check,
+      tuq_mode: EvaluationRequestV2.tuqMode ?? EvaluationRequestV2.tsqMode,
       assertions: EvaluationRequestV2.assertions,
     };
 

src/types.ts

@@ -2,15 +2,15 @@ import { z } from 'zod';
 
 // Framework type based on supported frameworks
 const FrameworkEnum = ['openai', 'vercelai', 'gemini', 'claude'] as const;
-export type Framework = typeof FrameworkEnum[number];
+export type Framework = (typeof FrameworkEnum)[number];
 
 // Model mode enum
 const ModelModeEnum = ['speed', 'balanced', 'quality'] as const;
-export type ModelMode = typeof ModelModeEnum[number];
+export type ModelMode = (typeof ModelModeEnum)[number];
 
 // Policy target enum
 const PolicyTargetEnum = ['input', 'output', 'both'] as const;
-export type PolicyTarget = typeof PolicyTargetEnum[number];
+export type PolicyTarget = (typeof PolicyTargetEnum)[number];
 
 export const messageSchema = z.object({
   role: z.string(),
@@ -96,7 +96,9 @@ export const EvaluationRequestV2Schema = z.object({
   sexualContentCheck: z.boolean().default(false).optional(),
   contentModerationCheck: z.boolean().default(false).optional(),
   syntaxChecks: z.record(z.string(), SyntaxCheckArgsSchema).optional(),
+  /** @deprecated Use toolUseQualityCheck instead */
   toolSelectionQualityCheck: z.boolean().default(false).optional(),
+  toolUseQualityCheck: z.boolean().default(false).optional(),
   assertions: z.array(z.string()).optional(),
   /** @deprecated Automatically added from the request*/
   available_tools: z.array(LLMToolDefinitionSchema).optional(),
@@ -120,9 +122,11 @@ export const EvaluationRequestV2Schema = z.object({
   sexual_content_check: z.boolean().default(false).optional(),
   /** @deprecated Use syntaxChecks instead */
   syntax_checks: z.record(z.string(), SyntaxCheckArgsSchema).optional(),
-  /** @deprecated Use toolSelectionQualityCheck instead */
+  /** @deprecated Use toolUseQualityCheck instead */
   tool_selection_quality_check: z.boolean().default(false).optional(),
-  tsqMode: z.enum(ModelModeEnum).default('balanced').optional(),
+  /** @deprecated Use tuqMode instead */
+  tsqMode: z.enum(ModelModeEnum).optional(),
+  tuqMode: z.enum(ModelModeEnum).optional(),
   consistencyMode: z.enum(ModelModeEnum).default('balanced').optional(),
   assertionsMode: z.enum(ModelModeEnum).default('balanced').optional(),
   groundingMode: z.enum(ModelModeEnum).default('balanced').optional(),
@@ -180,8 +184,12 @@ export const EvaluationProxyAPIRequestSchema = z
     sexualContentCheck: z.boolean().default(false).optional(),
     contentModerationCheck: z.boolean().default(false).optional(),
     syntaxChecks: z.record(z.string(), SyntaxCheckArgsSchema).optional(),
+    /** @deprecated Use toolUseQualityCheck instead */
     toolSelectionQualityCheck: z.boolean().default(false).optional(),
-    tsqMode: z.enum(ModelModeEnum).default('balanced').optional(),
+    toolUseQualityCheck: z.boolean().default(false).optional(),
+    /** @deprecated Use tuqMode instead */
+    tsqMode: z.enum(ModelModeEnum).optional(),
+    tuqMode: z.enum(ModelModeEnum).optional(),
     consistencyMode: z.enum(ModelModeEnum).default('balanced').optional(),
     assertionsMode: z.enum(ModelModeEnum).default('balanced').optional(),
     groundingMode: z.enum(ModelModeEnum).default('balanced').optional(),
@@ -206,24 +214,28 @@ export const EvaluationProxyAPIRequestSchema = z
       });
     }
 
-    // Validation: tool_selection_quality_check requires messages and available_tools
-    if (data.tool_selection_quality_check) {
+    // Validation: tool_selection_quality_check or toolSelectionQualityCheck or toolUseQualityCheck requires messages and available_tools
+    if (
+      data.tool_selection_quality_check ||
+      data.toolSelectionQualityCheck ||
+      data.toolUseQualityCheck
+    ) {
       const hasAvailableTools =
         Array.isArray(data.available_tools) && data.available_tools.length > 0;
 
       if (!hasMessages) {
         ctx.addIssue({
           code: z.ZodIssueCode.custom,
           message:
-            'messages must be provided when tool_selection_quality_check is true',
+            'messages must be provided when tool quality check is enabled',
           path: ['messages'],
         });
       }
       if (!hasAvailableTools) {
         ctx.addIssue({
           code: z.ZodIssueCode.custom,
           message:
-            'available_tools must be provided when tool_selection_quality_check is true',
+            'available_tools must be provided when tool quality check is enabled',
           path: ['available_tools'],
         });
       }