diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py index a4b4df20c7..b7a2041fa1 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py @@ -19,7 +19,7 @@ _convert_remote_eval_params_to_grader ) from save_evaluation import load_evaluator -from model_target import ModelTarget +from model_target import ModelTarget, get_key_from_dict AZURE_ENDPOINT = "AzureEndpoint" API_KEY = "ApiKey" @@ -119,33 +119,26 @@ def create_model_target_and_data_mapping(command_line_args): except json.JSONDecodeError as e: raise RuntimeError(f"Invalid JSON in eval_target: {e}") - model_config = target_config.get(MODEL_CONFIG, {}) + model_config = get_key_from_dict(target_config, MODEL_CONFIG, {}) - api_key_env = model_config.get(API_KEY, "") + api_key_env = get_key_from_dict(model_config, API_KEY, "") api_key_value = os.environ.get(api_key_env.upper(), "") + if not api_key_value: raise RuntimeError(f"API key environment variable '{api_key_env.upper()}' is missing or empty!") - model_config_type = str(model_config.get(TYPE, "")) + model_config_type = str(get_key_from_dict(model_config, TYPE, "")) logger.info(f" - Type: {model_config_type}") - endpoint = model_config.get(AZURE_ENDPOINT, "") - model_params = target_config.get(MODEL_PARAMS, {}).copy() - data_mapping = model_params.pop(DATA_MAPPING, None) + endpoint = get_key_from_dict(model_config, AZURE_ENDPOINT, "") + model_params = get_key_from_dict(target_config, MODEL_PARAMS, {}).copy() + data_mapping = get_key_from_dict(model_params, DATA_MAPPING, None) if data_mapping is None: data_mapping = DEFAULT_DATA_MAPPING logger.info(f"Using default dataMapping: {data_mapping}") - system_message = target_config.get(SYSTEM_MESSAGE, "") - few_shot_examples = target_config.get(FEW_SHOT_EXAMPLES, []) - - logger.info("Creating ModelTarget with values:") - logger.info(f" - Endpoint: {endpoint}") - logger.info(f" - ApiKey: {'[HIDDEN]' if api_key_value else 'MISSING'}") - logger.info(f" - ModelParams: {model_params}") - logger.info(f" - SystemMessage: {system_message}") - logger.info(f" - FewShotExamples: {few_shot_examples}") - logger.info(f" - DataMapping: {data_mapping}") + system_message = get_key_from_dict(target_config, SYSTEM_MESSAGE, "") + few_shot_examples = get_key_from_dict(target_config, FEW_SHOT_EXAMPLES, []) model_target = ModelTarget( endpoint=endpoint, @@ -154,6 +147,7 @@ def create_model_target_and_data_mapping(command_line_args): system_message=system_message, few_shot_examples=few_shot_examples, ) + logger.info("Created ModelTarget.") return (model_target, data_mapping) diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/model_target.py b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/model_target.py index 8e6ac8b058..7b158d10ac 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/model_target.py +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/model_target.py @@ -5,10 +5,23 @@ import requests import logging + +USER = "User" +ASSISTANT = "Assistant" + + logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) +def get_key_from_dict(d, key, default=None): + """Get a particular Key from given dict.""" + for k, v in d.items(): + if k.lower() == key.lower(): + return v + return default + + class ModelTarget: """Represents a model target and runs the chat/completion API on a query based on the target.""" @@ -29,10 +42,12 @@ def generate_response(self, query, context=None, **kwargs): messages = [{"role": "system", "content": self.system_message}] for example in self.few_shot_examples: - if 'User' in example: - messages.append({"role": "user", "content": example['User']}) - if 'Assistant' in example: - messages.append({"role": "assistant", "content": example['Assistant']}) + user_content = get_key_from_dict(example, USER, None) + if user_content is not None: + messages.append({"role": "user", "content": user_content}) + assistant_content = get_key_from_dict(example, ASSISTANT, None) + if assistant_content is not None: + messages.append({"role": "assistant", "content": assistant_content}) if context: messages.append({"role": "user", "content": f"{context} {query}"})