chore: sync from agents-private

.changeset/little-terms-lead.md

@@ -0,0 +1,7 @@
+---
+"@inkeep/agents-core": patch
+"@inkeep/agents-manage-ui": patch
+"@inkeep/agents-api": patch
+---
+
+Evaluation failure event added

agents-api/__snapshots__/openapi.json

@@ -12401,7 +12401,8 @@
                 "conversation.created",
                 "conversation.updated",
                 "feedback.created",
-                "event.created"
+                "event.created",
+                "evaluation.failed"
               ],
               "type": "string"
             },
@@ -12470,7 +12471,8 @@
                 "conversation.created",
                 "conversation.updated",
                 "feedback.created",
-                "event.created"
+                "event.created",
+                "evaluation.failed"
               ],
               "type": "string"
             },
@@ -12584,7 +12586,8 @@
                 "conversation.created",
                 "conversation.updated",
                 "feedback.created",
-                "event.created"
+                "event.created",
+                "evaluation.failed"
               ],
               "type": "string"
             },

agents-api/src/domains/evals/workflow/functions/evaluateConversation.ts

@@ -1,5 +1,6 @@
 import {
   createEvaluationResult,
+  evaluatePassCriteria,
   generateId,
   getAgentIdsForEvaluators,
   getConversation,
@@ -13,6 +14,7 @@ import { manageDbClient } from '../../../../data/db';
 import manageDbPool from '../../../../data/db/manageDbPool';
 import runDbClient from '../../../../data/db/runDbClient';
 import { getLogger } from '../../../../logger';
+import { emitEvaluationFailedWebhook } from '../../../run/services/WebhookDeliveryService';
 import { EvaluationService } from '../../services/EvaluationService';
 
 const logger = getLogger('workflow-evaluate-conversation');
@@ -97,16 +99,67 @@ async function executeEvaluatorStep(
       projectId,
     });
 
+    const outputData = (output as { output?: Record<string, unknown> })?.output ?? {};
+    const passCriteriaResult = evaluatePassCriteria(evaluator.passCriteria ?? null, outputData);
+    const verdict = passCriteriaResult.status;
+
+    if (passCriteriaResult.configurationErrors?.length) {
+      logger.error(
+        {
+          evaluatorId: evaluator.id,
+          configurationErrors: passCriteriaResult.configurationErrors,
+          availableOutputFields: Object.keys(outputData),
+        },
+        'Evaluator pass criteria misconfigured'
+      );
+    }
+
     const updated = await updateEvaluationResult(runDbClient)({
       scopes: { tenantId, projectId, evaluationResultId: evalResult.id },
       data: { output: output as any },
     });
 
     logger.info(
-      { conversationId, evaluatorId: evaluator.id, resultId: evalResult.id },
+      { conversationId, evaluatorId: evaluator.id, resultId: evalResult.id, verdict },
       'Evaluation completed successfully'
     );
 
+    try {
+      const failedConditions = (passCriteriaResult.failedConditions ?? []).map((c) => {
+        const val = outputData[c.field];
+        return {
+          field: c.field,
+          operator: c.operator,
+          value: c.value,
+          actual: typeof val === 'number' ? val : 0,
+        };
+      });
+
+      await emitEvaluationFailedWebhook({
+        runDbClient,
+        tenantId,
+        projectId,
+        verdict,
+        failedConditions,
+        evaluationResult: {
+          id: evalResult.id,
+          evaluatorId: evaluator.id,
+          conversationId,
+          evaluationRunId,
+        },
+        evaluator: { id: evaluator.id, name: evaluator.name },
+      });
+    } catch (emitErr) {
+      logger.warn(
+        {
+          error: emitErr instanceof Error ? emitErr.message : String(emitErr),
+          evaluatorId: evaluator.id,
+          resultId: evalResult.id,
+        },
+        'Failed to emit evaluation.failed webhook'
+      );
+    }
+
     return updated;
   } catch (error) {
     const errorMessage = error instanceof Error ? error.message : 'Unknown error';

agents-api/src/domains/evals/workflow/functions/runDatasetItem.ts

@@ -10,6 +10,7 @@ import {
   addConversationIdToInvocation,
   createDatasetRunConversationRelation,
   createEvaluationResult,
+  evaluatePassCriteria,
   generateId,
   getAgentIdsForEvaluators,
   getConversation,
@@ -29,6 +30,7 @@ import manageDbPool from '../../../../data/db/manageDbPool';
 import runDbClient from '../../../../data/db/runDbClient';
 import { getLogger } from '../../../../logger';
 import { executeAgentAsync } from '../../../run/services/TriggerService';
+import { emitEvaluationFailedWebhook } from '../../../run/services/WebhookDeliveryService';
 import { EvaluationService } from '../../services/EvaluationService';
 
 const logger = getLogger('workflow-run-dataset-item');
@@ -333,16 +335,67 @@ async function executeEvaluatorStep(
       expectedOutput,
     });
 
+    const outputData = (output as { output?: Record<string, unknown> })?.output ?? {};
+    const passCriteriaResult = evaluatePassCriteria(evaluator.passCriteria ?? null, outputData);
+    const verdict = passCriteriaResult.status;
+
+    if (passCriteriaResult.configurationErrors?.length) {
+      logger.error(
+        {
+          evaluatorId: evaluator.id,
+          configurationErrors: passCriteriaResult.configurationErrors,
+          availableOutputFields: Object.keys(outputData),
+        },
+        'Evaluator pass criteria misconfigured'
+      );
+    }
+
     await updateEvaluationResult(runDbClient)({
       scopes: { tenantId, projectId, evaluationResultId: evalResult.id },
       data: { output: output as any },
     });
 
     logger.info(
-      { conversationId, evaluatorId: evaluator.id, resultId: evalResult.id },
+      { conversationId, evaluatorId: evaluator.id, resultId: evalResult.id, verdict },
       'Evaluation completed'
     );
 
+    try {
+      const failedConditions = (passCriteriaResult.failedConditions ?? []).map((c) => {
+        const val = outputData[c.field];
+        return {
+          field: c.field,
+          operator: c.operator,
+          value: c.value,
+          actual: typeof val === 'number' ? val : 0,
+        };
+      });
+
+      await emitEvaluationFailedWebhook({
+        runDbClient,
+        tenantId,
+        projectId,
+        verdict,
+        failedConditions,
+        evaluationResult: {
+          id: evalResult.id,
+          evaluatorId: evaluator.id,
+          conversationId,
+          evaluationRunId,
+        },
+        evaluator: { id: evaluator.id, name: evaluator.name },
+      });
+    } catch (emitErr) {
+      logger.warn(
+        {
+          error: emitErr instanceof Error ? emitErr.message : String(emitErr),
+          evaluatorId: evaluator.id,
+          resultId: evalResult.id,
+        },
+        'Failed to emit evaluation.failed webhook'
+      );
+    }
+
     return evalResult.id;
   } catch (error) {
     const errorMessage = error instanceof Error ? error.message : 'Unknown error';

agents-api/src/domains/manage/routes/improvements.ts

@@ -14,6 +14,7 @@ import {
   doltPreviewMergeConflicts,
   doltPreviewMergeConflictsSummary,
   EvalSummaryResponseSchema,
+  evaluatePassCriteria,
   generateId,
   getDatasetById,
   getEvaluatorsByIds,
@@ -53,33 +54,6 @@ import { continueImprovement, triggerImprovement } from '../../run/services/Impr
 
 const logger = getLogger('improvements');
 
-type PassCondition = {
-  field: string;
-  operator: string;
-  value: number;
-};
-
-function evaluateCondition(cond: PassCondition, outputData: Record<string, unknown>): boolean {
-  const val = outputData[cond.field];
-  if (typeof val !== 'number') return false;
-  switch (cond.operator) {
-    case '>':
-      return val > cond.value;
-    case '<':
-      return val < cond.value;
-    case '>=':
-      return val >= cond.value;
-    case '<=':
-      return val <= cond.value;
-    case '=':
-      return val === cond.value;
-    case '!=':
-      return val !== cond.value;
-    default:
-      return false;
-  }
-}
-
 type ImprovementListEntry = {
   branchName: string;
   status: string;
@@ -1088,23 +1062,11 @@ app.openapi(
         const evaluator = evaluatorMap.get(result.evaluatorId);
         const evaluatorName = evaluator?.name ?? result.evaluatorId;
 
-        let passed: 'passed' | 'failed' | 'no_criteria' | 'pending' = 'pending';
+        let passed: EvalSummaryResult['passed'] = 'pending';
         if (result.output) {
-          const outputData = (result.output as { output?: Record<string, unknown> })?.output;
-          const criteria = (evaluator?.passCriteria ?? null) as {
-            operator?: 'and' | 'or';
-            conditions?: PassCondition[];
-          } | null;
-
-          if (outputData && criteria?.conditions?.length) {
-            const allPass =
-              criteria.operator === 'and'
-                ? criteria.conditions.every((cond) => evaluateCondition(cond, outputData))
-                : criteria.conditions.some((cond) => evaluateCondition(cond, outputData));
-            passed = allPass ? 'passed' : 'failed';
-          } else if (outputData) {
-            passed = 'no_criteria';
-          }
+          const outputData =
+            ((result.output as Record<string, unknown>)?.output as Record<string, unknown>) ?? {};
+          passed = evaluatePassCriteria(evaluator?.passCriteria ?? null, outputData).status;
         }
 
         return {