From 199424f02b35c088ea4d4dc650dd987dde83b90c Mon Sep 17 00:00:00 2001 From: Ethan Winters Date: Fri, 25 Apr 2025 11:32:20 -0700 Subject: [PATCH 1/4] add aoai changes --- .../context/evaluate_on_data.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py index a456587779..a4b4df20c7 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py @@ -15,6 +15,9 @@ from azure.ai.ml.identity import AzureMLOnBehalfOfCredential from azure.ai.evaluation import evaluate +from azure.ai.evaluation._evaluate._evaluate_aoai import ( + _convert_remote_eval_params_to_grader +) from save_evaluation import load_evaluator from model_target import ModelTarget @@ -84,10 +87,14 @@ def initialize_evaluators(command_line_args): for evaluator_name, evaluator in evaluators_o.items(): init_params = evaluator["InitParams"] update_value_in_dict(init_params, "AZURE_OPENAI_API_KEY", lambda x: os.environ[x.upper()]) - flow = load_evaluator("./" + evaluator_name) - if any(rai_eval in evaluator["Id"] for rai_eval in rai_evaluators): - init_params["credential"] = AzureMLOnBehalfOfCredential() - evaluators[evaluator_name] = flow(**init_params) + if evaluator["Id"].startswith("aoai://"): + grader = _convert_remote_eval_params_to_grader(evaluator["Id"], init_params) + evaluators[evaluator_name] = grader + else: + flow = load_evaluator("./" + evaluator_name) + if any(rai_eval in evaluator["Id"] for rai_eval in rai_evaluators): + init_params["credential"] = AzureMLOnBehalfOfCredential() + evaluators[evaluator_name] = flow(**init_params) return evaluators From 4bc433eabb2b0d50384617125979891a6bf86975 Mon Sep 17 00:00:00 2001 From: Ethan Winters Date: Tue, 6 May 2025 13:45:51 -0700 Subject: [PATCH 2/4] upgrade --- .../environments/evaluations-built-in/context/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt index 3f1035a430..1133b452b0 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt @@ -1,4 +1,4 @@ -azure-ai-evaluation==1.2.0 +azure-ai-evaluation==1.6.0 azureml-mlflow=={{latest-pypi-version}} azure-ai-ml=={{latest-pypi-version}} From e98ba9eed0b068f3ed6d5ca2c9ef6d91eb635aff Mon Sep 17 00:00:00 2001 From: Ethan Winters Date: Tue, 6 May 2025 14:55:14 -0700 Subject: [PATCH 3/4] pin otel --- .../environments/evaluations-built-in/context/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt index 1133b452b0..a86dbb2482 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt @@ -4,7 +4,7 @@ azure-ai-ml=={{latest-pypi-version}} openai=={{latest-pypi-version}} opentelemetry-api=={{latest-pypi-version}} -opentelemetry-sdk=={{latest-pypi-version}} +opentelemetry-sdk==1.31.0 azure-monitor-query=={{latest-pypi-version}} croniter=={{latest-pypi-version}} azure-monitor-opentelemetry=={{latest-pypi-version}} From 4694dfde2f223eec712f12e5e9cd78b6a3593a87 Mon Sep 17 00:00:00 2001 From: Ethan Winters Date: Tue, 6 May 2025 15:01:25 -0700 Subject: [PATCH 4/4] pin otel api --- .../environments/evaluations-built-in/context/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt index a86dbb2482..1b42f019d8 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt @@ -3,7 +3,7 @@ azureml-mlflow=={{latest-pypi-version}} azure-ai-ml=={{latest-pypi-version}} openai=={{latest-pypi-version}} -opentelemetry-api=={{latest-pypi-version}} +opentelemetry-api==1.31.0 opentelemetry-sdk==1.31.0 azure-monitor-query=={{latest-pypi-version}} croniter=={{latest-pypi-version}}