Custom eval sample fix (#45989)

* Sample-Fix

* fix samples

Author: w-javed

sdk/ai/azure-ai-projects/samples/evaluations/custom_evaluators/answer_length_evaluator/answer_length_evaluator.py

@@ -2,11 +2,14 @@
 
 
 class AnswerLengthEvaluator:
-    def __init__(self, *, model_config):
-        self.model_config = model_config
+    def __init__(self, *, config: str, threshold, **kwargs):
+        self.config = config
+        self.threshold = threshold
 
     def __call__(self, *args, **kwargs):
-        return {"result": evaluate_answer_length(kwargs.get("response"))}
+        return {
+            "result": evaluate_answer_length(kwargs.get("response")), 
+        }
 
 
 def evaluate_answer_length(answer: str | None):

sdk/ai/azure-ai-projects/samples/evaluations/custom_evaluators/friendly_evaluator/common_util/util.py

@@ -41,10 +41,11 @@ def build_evaluation_messages(query: str, response: str) -> list:
     ]
 
 
-def parse_evaluation_result(raw_result: str) -> dict:
+def parse_evaluation_result(raw_result: str, threshold: int = 3) -> dict:
     """Parse the LLM's JSON response into a structured evaluation result.
 
     :param raw_result: The raw string output from the LLM.
+    :param threshold: The minimum score to be considered "Pass".
     :return: A dict with score, label, reason, and explanation.
     """
     import json
@@ -56,16 +57,16 @@ def parse_evaluation_result(raw_result: str) -> dict:
             text = text.split("\n", 1)[1] if "\n" in text else text[3:]
             text = text.rsplit("```", 1)[0]
         result = json.loads(text.strip())
-        score = int(result.get("score", 3))
+        score = int(result.get("score", threshold))
         return {
             "score": max(1, min(5, score)),
-            "label": result.get("label", "Pass" if score >= 3 else "Fail"),
+            "label": result.get("label", "Pass" if score >= threshold else "Fail"),
             "reason": result.get("reason", "No reason provided"),
             "explanation": result.get("explanation", "No explanation provided"),
         }
     except (json.JSONDecodeError, ValueError, KeyError):
         return {
-            "score": 3,
+            "score": threshold,
             "label": "Pass",
             "reason": "Could not parse LLM response",
             "explanation": f"Raw LLM output: {raw_result}",

sdk/ai/azure-ai-projects/samples/evaluations/custom_evaluators/friendly_evaluator/friendly_evaluator.py

@@ -15,10 +15,12 @@ class FriendlyEvaluator:
         - azure_deployment: The deployment/model name.
         - api_version: The API version (default: "2024-06-01").
         - api_key: (Optional) The API key. If not provided, DefaultAzureCredential is used.
+    :param threshold: The minimum score (1-5) to be considered "Pass" (default: 3).
     """
 
-    def __init__(self, *, model_config: dict):
+    def __init__(self, *, model_config: dict, threshold: int = 3, **kwargs):
         self.model_config = model_config
+        self.threshold = threshold
         api_key = model_config.get("api_key")
 
         if api_key:
@@ -61,4 +63,4 @@ def __call__(self, *, query: str, response: str, **kwargs) -> dict:
         raw_result = completion.choices[0].message.content
         if raw_result is None:
             raise ValueError("No content in completion response")
-        return parse_evaluation_result(raw_result)
+        return parse_evaluation_result(raw_result, self.threshold)

sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_upload_custom_evaluator.py

@@ -81,8 +81,11 @@
             entry_point="answer_length_evaluator:AnswerLengthEvaluator",
             init_parameters={
                 "type": "object",
-                "properties": {"model_config": {"type": "string"}},
-                "required": ["model_config"],
+                "properties": {
+                    "config": {"type": "string"}, 
+                    "threshold": {"type": "number"}
+                },
+                "required": ["config", "threshold"],
             },
             data_schema={
                 "type": "object",
@@ -93,7 +96,7 @@
                 "required": ["query", "response"],
             },
             metrics={
-                "score": EvaluatorMetric(
+                "result": EvaluatorMetric(
                     type=EvaluatorMetricType.ORDINAL,
                     desirable_direction=EvaluatorMetricDirection.INCREASE,
                     min_value=1,
@@ -138,7 +141,8 @@
             "name": evaluator_name,
             "evaluator_name": evaluator_name,
             "initialization_parameters": {
-                "model_config": f"{model_deployment_name}",
+                "config": "example config value",
+                "threshold": 3,
             },
         }
     ]

sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_upload_friendly_evaluator.py

@@ -99,9 +99,10 @@
                             "api_key": {"type": "string"},
                         },
                         "required": ["azure_endpoint", "api_key"],
-                    }
+                    },
+                    "threshold": {"type": "number"},
                 },
-                "required": ["model_config"],
+                "required": ["model_config", "threshold"],
             },
             data_schema={
                 "type": "object",
@@ -157,11 +158,8 @@
             "name": evaluator_name,
             "evaluator_name": evaluator_name,
             "initialization_parameters": {
-                "model_config": {
-                    "azure_endpoint": azure_openai_endpoint,
-                    "api_key": f"{azure_openai_api_key}",
-                    "api_version": "2024-06-01",
-                },
+                 "deployment_name": f"{model_deployment_name}", # provide model_config or, deployment name passed is used to construct the model_config for the evaluator. 
+                 "threshold": 3,
             },
         }
     ]