Merge pull request #6 from mesameki/codespace-sturdy-space-umbrella-wgxvgx7w9jxcg457

EvaluationAirlift Changes (Custom Eval, More Built-In Eval, Simulation, ...)

Author: qubitron

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/devcontainers/python:3.10
+FROM mcr.microsoft.com/devcontainers/python:3.11
 
 WORKDIR /
 
@@ -11,5 +11,5 @@ ENV LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
 RUN apt update && apt install -y fuse
 
 # Install Python SDK
-COPY src/requirements.txt .
-RUN pip install -r requirements.txt 
+COPY src/requirements.freeze.txt .
+RUN pip install -r requirements.freeze.txt 

.devcontainer/devcontainer.json

@@ -20,10 +20,12 @@
 				"ms-azuretools.azure-dev"
 			]
 		}
-	}
+	},
 
 	// Features to add to the dev container. More info: https://containers.dev/features.
-	// "features": {},
+	"features": {
+		"ghcr.io/devcontainers/features/azure-cli:1": {}
+	},
 
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [],
@@ -36,4 +38,4 @@
 
 	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
 	// "remoteUser": "root"
-}
+}

README.md

@@ -48,6 +48,7 @@ def get_chat_response(chat_input: str, chat_history: list = []) -> ChatResponse:
         searchQuery = intentPrompty(query=chat_input, chat_history=chat_history)
 
     # retrieve relevant documents and context given chat_history and current user query (chat_input)
+    print(os.environ["AZURE_OPENAI_API_KEY"])
     documents = get_documents(searchQuery, 3)
 
     # send query + document context to chat completion for a response
@@ -57,7 +58,7 @@ def get_chat_response(chat_input: str, chat_history: list = []) -> ChatResponse:
         'parameters': { 
             'max_tokens': 256,
             'temperature': 0.2,
-            'stream': True # always stream responses, consumers/clients should handle streamed response
+            'stream': False # always stream responses, consumers/clients should handle streamed response
         }
     })
     result = chatPrompty(

src/copilot_flow/copilot.py

@@ -0,0 +1,8 @@
+from .friendliness import FriendlinessEvaluator
+from .completeness import CompletenessEvaluator
+ 
+__all__ = [
+    "FriendlinessEvaluator",
+    "CompletenessEvaluator"
+]
+ 

src/custom_evaluators/__init__.py

@@ -0,0 +1,76 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+import re
+import json
+
+import numpy as np
+
+from promptflow.client import load_flow
+from promptflow.core import AzureOpenAIModelConfiguration
+
+
+class CompletenessEvaluator:
+    def __init__(self, model_config: AzureOpenAIModelConfiguration, prompty_filename: str = "completeness.prompty"):
+        """
+        Initialize an evaluator configured for a specific Azure OpenAI model.
+
+        :param model_config: Configuration for the Azure OpenAI model.
+        :type model_config: AzureOpenAIModelConfiguration
+
+        **Usage**
+
+        .. code-block:: python
+
+            eval_fn = CompletenessEvaluator(model_config)
+            result = eval_fn(
+                question="What is (3+1)-4?",
+                answer="First, the result within the first bracket is 3+1 = 4; then the next step is 4-4=0. The answer is 0",
+                truth="0")
+        """
+        # TODO: Remove this block once the bug is fixed
+        # https://msdata.visualstudio.com/Vienna/_workitems/edit/3151324
+        if model_config.api_version is None:
+            model_config.api_version = "2024-02-15-preview"
+
+        prompty_model_config = {"configuration": model_config}
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, prompty_filename)
+        self._flow = load_flow(source=prompty_path, model=prompty_model_config)
+
+    def __call__(self, *, question: str, answer: str, truth: str, **kwargs):
+        """Evaluate correctness of the answer in the context.
+
+        :param answer: The answer to be evaluated.
+        :type answer: str
+        :param context: The context in which the answer is evaluated.
+        :type context: str
+        :return: The correctness score.
+        :rtype: dict
+        """
+        # Validate input parameters
+        question = str(question or "")
+        answer = str(answer or "")
+        truth = str(truth or "")
+
+        if not (answer.strip()) or not (question.strip()):
+            raise ValueError("All inputs including 'question', 'answer' must be non-empty strings.")
+
+        # Run the evaluation flow
+        llm_output = self._flow(question=question, answer=answer, truth=truth)
+
+        score = np.nan
+        reason = ""
+        try:
+            json_output = json.loads(llm_output)
+            score = json_output["SCORE"]
+            reason = json_output["REASON"]
+        except Exception as e:
+            if llm_output:
+                match = re.findall(r"\d$", llm_output)
+                if match:
+                    score = float(match[-1])
+
+        return {"gpt_completeness": float(score), "gpt_completeness_reason": reason}

src/custom_evaluators/completeness.py

@@ -0,0 +1,94 @@
+---
+name: Reasonableness
+description: Evaluates reasonableness score for QA scenario
+model:
+  api: chat
+  configuration:
+    type: azure_openai
+    azure_deployment: ${env:AZURE_DEPLOYMENT}
+    api_key: ${env:AZURE_OPENAI_API_KEY}
+    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
+  parameters:
+    temperature: 0.0
+    max_tokens: 100
+    top_p: 1.0
+    presence_penalty: 0
+    frequency_penalty: 0
+    seed: 0
+    response_format:
+      type: text
+
+
+inputs:
+  question:
+    type: string
+  answer:
+    type: string
+  truth:
+    type: string
+---
+system:
+You are an AI assistant. You will be given the definition of an evaluation metric for assessing the quality of an answer in a question-answering task. Your job is to compute an accurate evaluation score using the provided evaluation metric.
+user:
+You are an expert specialized in quality and safety evaluation of responses from intelligent assistant systems to user queries. Given some inputs, your objective is to measure whether the generated answer is complete or not, in reference to the ground truth. The metric is based on the prompt template below, where an answer is considered complete if it doesn't miss a statement from the ground truth.  
+
+Use the following steps to respond to inputs.  
+
+Step 1: Extract all statements from TRUTH. If truth is an empty string, skip all remaining steps and output {"REASON": "No missing statements found.", "SCORE": 5}. 
+
+Step 2: Extract all statements from ANSWER.
+
+Step 3: Pay extra attention to statements that involve numbers, dates, or proper nouns. Reason step-by-step and identify whether ANSWER misses any of the statements in TRUTH. Output those missing statements in REASON.
+
+Step 4: Rate the completeness of ANSWER between one to five stars using the following scale:
+
+One star: ANSWER is missing all of the statements in TRUTH.
+
+Two stars: ANSWER has some statements, but it is missing all the critical statements necessary to answer the question.
+
+Three stars: ANSWER has some statements, but it is missing some critical statements necessary to answer the question.
+
+Four stars: ANSWER has most of the statements, but it is missing few statements which are not important to answer the question. 
+
+Five stars: ANSWER has all of the statements in the TRUTH. 
+
+Please assign a rating between 1 and 5 based on the completeness the response. Output the rating in SCORE.
+
+Independent Examples:
+## Example Task #1 Input:
+{"QUESTION": "What color does TrailBlaze Hiking Pants come in?", "ANSWER": "Khaki", "TRUTH": "Khaki"}
+## Example Task #1 Output:
+{"REASON": "No missing statements found.", "SCORE": 5}
+## Example Task #2 Input:
+{"QUESTION": "What color does TrailBlaze Hiking Pants come in?", "ANSWER": "Red", "TRUTH": "Khaki"}
+## Example Task #2 Output:
+{"REASON": "missing statements: \n1. Khaki", "SCORE": 1}
+## Example Task #3 Input:
+{"QUESTION": "What purchases did Sarah Lee make and at what price point?", "ANSWER": "['TrailMaster X4 Tent: $$250', 'CozyNights Sleeping Bag: $$100', 'TrailBlaze Hiking Pants: $$75', 'RainGuard Hiking Jacket: $$110']", "TRUTH": "['TrailMaster X4 Tent: $$250', 'CozyNights Sleeping Bag: $$100', 'TrailBlaze Hiking Pants: $$75', 'TrekMaster Camping Chair: $$100', 'SkyView 2-Person Tent: $$200', 'RainGuard Hiking Jacket: $$110', 'CompactCook Camping Stove: $$60']"}
+## Example Task #3 Output:
+{"REASON": "missing statements: \n1. 'TrekMaster Camping Chair: $$100'\n2.'SkyView 2-Person Tent: $$200'\n3. 'CompactCook Camping Stove: $$60'", "SCORE": 3}
+## Example Task #4 Input:
+{"QUESTION": "How many TrailMaster X4 Tents did John Smith bought?", "ANSWER": "1", "TRUTH": "2"}
+## Example Task #4 Output:
+{"REASON": "missing statements: \n1. 2 tents were purchased by John Smith.", "SCORE": 1}
+## Example Task #5 Input:
+{"QUESTION": "How water-proof are TrailBlazeMaster pants?", "ANSWER": "They are perfectly water-proof in all weather conditions", "TRUTH": "They are mostly water-proof except for rare, extreme weather conditions like hurricanes."}
+## Example Task #5 Output:
+{"REASON": "missing statements: \n1. Rare, extreme weather conditions like hurricanes would make TrailBlazeMaster pants not water-proof.", "SCORE": 4}
+## Example Task #6 Input:
+{"QUESTION": "How water-proof are TrailBlazeMaster pants?", "ANSWER": "They are perfectly water-proof in all weather conditions", "TRUTH": "They are slightly water-proof."}
+## Example Task #6 Output:
+{"REASON": "missing statements: \n1. TrailBlazeMaster pants are only slightly water-proof.", "SCORE": 2}
+## Example Task #7 Input:
+{"QUESTION": "Is a Belgium a country?", "ANSWER": "Sorry I cannot assist with that.", "TRUTH": "Sorry, I cannot answer any questions unrelated to sports gear."}
+## Example Task #7 Output:
+{"REASON": "No missing statements found.", "SCORE": 5}
+## Example Task #8 Input:
+{"QUESTION": "Is a Belgium a country?", "ANSWER": "Sorry I cannot provide answers unrelated to sports/gear", "TRUTH": ""}
+## Example Task #8 Output:
+{"REASON": "No missing statements found.", "SCORE": 5}
+
+## Actual Task Input:
+{"QUESTION": {{question}}, "ANSWER": {{answer}}, "TRUTH": {{truth}}}
+Reminder: The return values for each task should be an integer between 1 and 5. Do not repeat TRUTH, ANSWER or QUESTION.
+## Actual Task Output:

src/custom_evaluators/completeness_cot.prompty

@@ -0,0 +1,51 @@
+---
+name: Friendliness Evaluator
+description: Friendliness Evaluator to measure warmth and approachability of answers.
+model:
+  api: chat
+  configuration:
+    type: azure_openai
+    azure_deployment: gpt-4
+    api_key: ${env:AZURE_OPENAI_API_KEY}
+    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
+  parameters:
+    temperature: 0.1
+    response_format: { "type": "text" }
+inputs:
+  response:
+    type: string
+outputs:
+  score:
+    type: int
+  explanation:
+    type: string
+---
+
+system:
+Friendliness assesses the warmth and approachability of the answer. Rate the friendliness of the response between one to five stars using the following scale:
+
+One star: the answer is unfriendly or hostile
+
+Two stars: the answer is mostly unfriendly
+
+Three stars: the answer is neutral
+
+Four stars: the answer is mostly friendly
+
+Five stars: the answer is very friendly
+
+Please assign a rating between 1 and 5 based on the tone and demeanor of the response.
+
+**Example 1**
+generated_query: I just dont feel like helping you! Your questions are getting very annoying.
+output: 
+{"score": 1, "explanation": "The response is not warm and is resisting to be providing helpful information."}
+**Example 2**
+generated_query: I'm sorry this watch is not working for you. Very happy to assist you with a replacement. 
+output: 
+{"score": 5, "explanation": "The response is warm and empathetic, offering a resolution with care."}
+
+
+**Here the actual conversation to be scored:**
+generated_query: {{response}}
+output:

src/custom_evaluators/friendliness.prompty

@@ -0,0 +1,13 @@
+import os
+import json
+from promptflow.client import load_flow
+
+class FriendlinessEvaluator:
+    def __init__(self):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, "friendliness.prompty")
+        self._flow = load_flow(source=prompty_path)
+
+    def __call__(self, *, response: str, **kwargs):
+        response =  self._flow(response=response)
+        return json.loads(response)