From 56e6f99fac82de311066f73baad4428337516c2c Mon Sep 17 00:00:00 2001 From: Elaine Wan Date: Mon, 2 Jun 2025 12:34:18 -0400 Subject: [PATCH 01/13] add obo env variable --- .../evaluations-built-in/context/evaluate_on_data.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py index 8528226e92..23edc01fc2 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py @@ -43,9 +43,7 @@ GENERATED_RESPONSE_KEY = "generated_response" GENERATED_RESPONSE_MAPPING = f"${{data.{GENERATED_RESPONSE_KEY}}}" -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - +os.environ["AZUREML_OBO_ENABLED"] = "True" def update_value_in_dict(d, key_substring, new_func): """Recursively search for a value containing 'key_substring' and apply 'new_func' to modify it.""" From 76d089e2e3afb925b109d482cb2d35e0f1727d9d Mon Sep 17 00:00:00 2001 From: Elaine Wan Date: Mon, 2 Jun 2025 12:35:33 -0400 Subject: [PATCH 02/13] add back logger --- .../evaluations-built-in/context/evaluate_on_data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py index 23edc01fc2..4700fd3ee0 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py @@ -21,6 +21,7 @@ from save_evaluation import load_evaluator from model_target import ModelTarget, get_key_from_dict +os.environ["AZUREML_OBO_ENABLED"] = "True" AZURE_ENDPOINT = "AzureEndpoint" API_KEY = "ApiKey" AZURE_DEPLOYMENT = "AzureDeployment" @@ -43,7 +44,8 @@ GENERATED_RESPONSE_KEY = "generated_response" GENERATED_RESPONSE_MAPPING = f"${{data.{GENERATED_RESPONSE_KEY}}}" -os.environ["AZUREML_OBO_ENABLED"] = "True" +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) def update_value_in_dict(d, key_substring, new_func): """Recursively search for a value containing 'key_substring' and apply 'new_func' to modify it.""" From 881a6c4f444bc70e270a25bd65e7fe8a06d20482 Mon Sep 17 00:00:00 2001 From: Elaine Wan Date: Tue, 3 Jun 2025 12:02:03 -0400 Subject: [PATCH 03/13] add testing --- .../workflows/evaluation-on-cloud-tests.yaml | 61 +++++++++++++++++++ .../tests/test_evaluate_on_data.py | 18 ++++++ 2 files changed, 79 insertions(+) create mode 100644 .github/workflows/evaluation-on-cloud-tests.yaml create mode 100644 assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py diff --git a/.github/workflows/evaluation-on-cloud-tests.yaml b/.github/workflows/evaluation-on-cloud-tests.yaml new file mode 100644 index 0000000000..5de554a5f2 --- /dev/null +++ b/.github/workflows/evaluation-on-cloud-tests.yaml @@ -0,0 +1,61 @@ +name: evaluation-on-cloud-unittests + +on: + pull_request: + branches: + - main + paths: + - assets/evaluation_on_cloud/** + - .github/workflows/evaluation-on-cloud-unittests.yaml + workflow_dispatch: + +env: + conda_env_prefix: /opt/conda/envs/eval_on_cloud + eval_on_cloud_unittests_dir: assets/evaluation_on_cloud/environments/evaluations-built-in/tests + eval_on_cloud_conda_yaml: assets/evaluation_on_cloud/environments/evaluations-built-in/context/conda.yaml + pytest_reports: pytest-reports + +jobs: + run_unit_tests: + name: Run + runs-on: ubuntu-latest + + permissions: + # Required for EnricoMi/publish-unit-test-result-action + checks: write + issues: read + pull-requests: write + + steps: + - name: Clone branch + uses: actions/checkout@v3 + + - name: Use Python 3.10 or newer + uses: actions/setup-python@v4 + with: + python-version: '>=3.10' + + - name: Create conda env + run: | + set -ex + apt-get update && apt-get upgrade && apt-get install -y build-essential + echo $conda_env_prefix + sed -i 's/- pip:/- pip:\n - --extra-index-url https:\/\/download.pytorch.org\/whl\/cpu/' $eval_on_cloud_conda_yaml + sed -i 's/=={{latest-pypi-version}}//g' $eval_on_cloud_conda_yaml + sed -i 's/=={{latest-pypi-version:pre}}//g' $eval_on_cloud_conda_yaml + echo " - pytest" >> $eval_on_cloud_conda_yaml + echo " - pytest-xdist" >> $eval_on_cloud_conda_yaml + cat $eval_on_cloud_conda_yaml + conda env create -p $conda_env_prefix -f $eval_on_cloud_conda_yaml + echo "conda env successfully created at $conda_env_prefix" + conda list -p $conda_env_prefix + + - name: Execute tests + run: conda run -p $conda_env_prefix python -m pytest $eval_on_cloud_tests_dir --tb=native --junitxml=$pytest_reports/test-result.xml -x -n 1 -ra --show-capture=no + + - name: Publish test results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + check_name: Test Results for ${{ github.workflow }} + junit_files: ${{ env.pytest_reports }}/**/*.xml diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py new file mode 100644 index 0000000000..f16c5628b1 --- /dev/null +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py @@ -0,0 +1,18 @@ +import sys +from pathlib import Path +import os +import importlib + +def test_evaluate_obo_sets_env_var(): + script_path = Path(__file__).parent.parent / "context" / "evaluate_on_data.py" + module_name = "evaluate_on_data" + sys.path.insert(0, str(script_path.parent)) + + if module_name in sys.modules: + del sys.modules[module_name] + + assert os.environ.get("AZUREML_OBO_ENABLED") is None + + importlib.import_module(module_name) + + assert os.environ.get("AZUREML_OBO_ENABLED") == "True" From 6ed0f3d5809d7ca4bbaa606ba966f1079cd0f309 Mon Sep 17 00:00:00 2001 From: Elaine Wan Date: Tue, 3 Jun 2025 12:05:05 -0400 Subject: [PATCH 04/13] update naming --- .github/workflows/evaluation-on-cloud-tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/evaluation-on-cloud-tests.yaml b/.github/workflows/evaluation-on-cloud-tests.yaml index 5de554a5f2..e29c8c3611 100644 --- a/.github/workflows/evaluation-on-cloud-tests.yaml +++ b/.github/workflows/evaluation-on-cloud-tests.yaml @@ -1,4 +1,4 @@ -name: evaluation-on-cloud-unittests +name: evaluation-on-cloud-tests on: pull_request: @@ -6,12 +6,12 @@ on: - main paths: - assets/evaluation_on_cloud/** - - .github/workflows/evaluation-on-cloud-unittests.yaml + - .github/workflows/evaluation-on-cloud-tests.yaml workflow_dispatch: env: conda_env_prefix: /opt/conda/envs/eval_on_cloud - eval_on_cloud_unittests_dir: assets/evaluation_on_cloud/environments/evaluations-built-in/tests + eval_on_cloud_tests_dir: assets/evaluation_on_cloud/environments/evaluations-built-in/tests eval_on_cloud_conda_yaml: assets/evaluation_on_cloud/environments/evaluations-built-in/context/conda.yaml pytest_reports: pytest-reports From ab869a8f8918ec79c1241dfdf5ec93c58aa89bf8 Mon Sep 17 00:00:00 2001 From: Elaine Wan Date: Tue, 3 Jun 2025 12:16:05 -0400 Subject: [PATCH 05/13] add conda.yml --- .../evaluations-built-in/context/conda.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 assets/evaluation_on_cloud/environments/evaluations-built-in/context/conda.yaml diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/conda.yaml b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/conda.yaml new file mode 100644 index 0000000000..8bbaa0ae52 --- /dev/null +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/conda.yaml @@ -0,0 +1,14 @@ +name: eval_on_cloud +channels: + - defaults + - conda-forge +dependencies: + - python=3.10 + - pip + - pip: + - -r assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt + - pandas + - requests + - promptflow + - pytest + - pytest-xdist From 741a6d9de5df752797633b384d34bb19ece37e29 Mon Sep 17 00:00:00 2001 From: Elaine Wan Date: Tue, 3 Jun 2025 12:17:51 -0400 Subject: [PATCH 06/13] add copyright --- .../evaluations-built-in/tests/test_evaluate_on_data.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py index f16c5628b1..dc23367d20 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + import sys from pathlib import Path import os From 941330e5ce66b8095cb39a19bbd20010a8fd8a9a Mon Sep 17 00:00:00 2001 From: Elaine Wan Date: Tue, 3 Jun 2025 12:26:11 -0400 Subject: [PATCH 07/13] fix conda + syntax --- .../environments/evaluations-built-in/context/conda.yaml | 4 ++-- .../evaluations-built-in/context/evaluate_on_data.py | 1 + .../evaluations-built-in/tests/test_evaluate_on_data.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/conda.yaml b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/conda.yaml index 8bbaa0ae52..267d5f64d4 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/conda.yaml +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/conda.yaml @@ -5,10 +5,10 @@ channels: dependencies: - python=3.10 - pip + - pandas + - requests - pip: - -r assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt - - pandas - - requests - promptflow - pytest - pytest-xdist diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py index 4700fd3ee0..58e6fb392d 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py @@ -47,6 +47,7 @@ logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) + def update_value_in_dict(d, key_substring, new_func): """Recursively search for a value containing 'key_substring' and apply 'new_func' to modify it.""" for key, value in d.items(): diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py index dc23367d20..2d4abca300 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py @@ -6,6 +6,7 @@ import os import importlib + def test_evaluate_obo_sets_env_var(): script_path = Path(__file__).parent.parent / "context" / "evaluate_on_data.py" module_name = "evaluate_on_data" From 33c80e78431703f7d5d4376a8288e67fccf01106 Mon Sep 17 00:00:00 2001 From: Elaine Wan Date: Tue, 3 Jun 2025 12:37:41 -0400 Subject: [PATCH 08/13] add docstrings --- .../evaluations-built-in/tests/test_evaluate_on_data.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py index 2d4abca300..cb92f2629c 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py @@ -1,6 +1,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +""" +Tests for evaluate_on_data.py +""" + import sys from pathlib import Path import os @@ -8,6 +12,9 @@ def test_evaluate_obo_sets_env_var(): + """ + Test that importing evaluate_on_data.py sets the AZUREML_OBO_ENABLED environment variable to "True" + """ script_path = Path(__file__).parent.parent / "context" / "evaluate_on_data.py" module_name = "evaluate_on_data" sys.path.insert(0, str(script_path.parent)) From 6766436f65ce6a98f73cf0efc9a637dd7aa59429 Mon Sep 17 00:00:00 2001 From: Elaine Wan Date: Tue, 3 Jun 2025 12:41:13 -0400 Subject: [PATCH 09/13] fix yml --- .github/workflows/evaluation-on-cloud-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/evaluation-on-cloud-tests.yaml b/.github/workflows/evaluation-on-cloud-tests.yaml index e29c8c3611..ff5b7e58aa 100644 --- a/.github/workflows/evaluation-on-cloud-tests.yaml +++ b/.github/workflows/evaluation-on-cloud-tests.yaml @@ -40,7 +40,7 @@ jobs: set -ex apt-get update && apt-get upgrade && apt-get install -y build-essential echo $conda_env_prefix - sed -i 's/- pip:/- pip:\n - --extra-index-url https:\/\/download.pytorch.org\/whl\/cpu/' $eval_on_cloud_conda_yaml + sed -i 's/- pip:/- pip:\n $eval_on_cloud_conda_yaml sed -i 's/=={{latest-pypi-version}}//g' $eval_on_cloud_conda_yaml sed -i 's/=={{latest-pypi-version:pre}}//g' $eval_on_cloud_conda_yaml echo " - pytest" >> $eval_on_cloud_conda_yaml From 51af1015093a14ef0f25cb34dca7aa873a81869a Mon Sep 17 00:00:00 2001 From: Elaine Wan Date: Tue, 3 Jun 2025 12:42:55 -0400 Subject: [PATCH 10/13] fix docstrings :/ --- .../evaluations-built-in/tests/test_evaluate_on_data.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py index cb92f2629c..be6cdd6b88 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py @@ -1,9 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -""" -Tests for evaluate_on_data.py -""" +"""Tests for evaluate_on_data.py.""" import sys from pathlib import Path @@ -12,9 +10,8 @@ def test_evaluate_obo_sets_env_var(): - """ - Test that importing evaluate_on_data.py sets the AZUREML_OBO_ENABLED environment variable to "True" - """ + """Test that importing evaluate_on_data.py sets the AZUREML_OBO_ENABLED environment variable to "True".""" + script_path = Path(__file__).parent.parent / "context" / "evaluate_on_data.py" module_name = "evaluate_on_data" sys.path.insert(0, str(script_path.parent)) From ed9b8d2e3b6ea2625bc22b4ff78adf068d459a9d Mon Sep 17 00:00:00 2001 From: Elaine Wan Date: Tue, 3 Jun 2025 12:46:59 -0400 Subject: [PATCH 11/13] more formatting stuff :( --- .../evaluations-built-in/tests/test_evaluate_on_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py index be6cdd6b88..b088b31469 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py @@ -11,7 +11,6 @@ def test_evaluate_obo_sets_env_var(): """Test that importing evaluate_on_data.py sets the AZUREML_OBO_ENABLED environment variable to "True".""" - script_path = Path(__file__).parent.parent / "context" / "evaluate_on_data.py" module_name = "evaluate_on_data" sys.path.insert(0, str(script_path.parent)) From 0b83eb47e77d5d5a644e201149fde9d28a1f7c2b Mon Sep 17 00:00:00 2001 From: Elaine Wan Date: Tue, 3 Jun 2025 12:54:09 -0400 Subject: [PATCH 12/13] fix yml --- .github/workflows/evaluation-on-cloud-tests.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/evaluation-on-cloud-tests.yaml b/.github/workflows/evaluation-on-cloud-tests.yaml index ff5b7e58aa..9d7b00de51 100644 --- a/.github/workflows/evaluation-on-cloud-tests.yaml +++ b/.github/workflows/evaluation-on-cloud-tests.yaml @@ -40,11 +40,7 @@ jobs: set -ex apt-get update && apt-get upgrade && apt-get install -y build-essential echo $conda_env_prefix - sed -i 's/- pip:/- pip:\n $eval_on_cloud_conda_yaml - sed -i 's/=={{latest-pypi-version}}//g' $eval_on_cloud_conda_yaml - sed -i 's/=={{latest-pypi-version:pre}}//g' $eval_on_cloud_conda_yaml - echo " - pytest" >> $eval_on_cloud_conda_yaml - echo " - pytest-xdist" >> $eval_on_cloud_conda_yaml + sed -i 's/=={{latest-pypi-version}}//g' assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt cat $eval_on_cloud_conda_yaml conda env create -p $conda_env_prefix -f $eval_on_cloud_conda_yaml echo "conda env successfully created at $conda_env_prefix" From 77fa0d5fbec56afd89fc188d73de243653f6a46d Mon Sep 17 00:00:00 2001 From: Elaine Wan Date: Tue, 3 Jun 2025 13:01:39 -0400 Subject: [PATCH 13/13] move files, update filepaths --- .github/workflows/evaluation-on-cloud-tests.yaml | 2 +- .../evaluations-built-in/{context => tests}/conda.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename assets/evaluation_on_cloud/environments/evaluations-built-in/{context => tests}/conda.yaml (65%) diff --git a/.github/workflows/evaluation-on-cloud-tests.yaml b/.github/workflows/evaluation-on-cloud-tests.yaml index 9d7b00de51..84f536eb74 100644 --- a/.github/workflows/evaluation-on-cloud-tests.yaml +++ b/.github/workflows/evaluation-on-cloud-tests.yaml @@ -12,7 +12,7 @@ on: env: conda_env_prefix: /opt/conda/envs/eval_on_cloud eval_on_cloud_tests_dir: assets/evaluation_on_cloud/environments/evaluations-built-in/tests - eval_on_cloud_conda_yaml: assets/evaluation_on_cloud/environments/evaluations-built-in/context/conda.yaml + eval_on_cloud_conda_yaml: assets/evaluation_on_cloud/environments/evaluations-built-in/tests/conda.yaml pytest_reports: pytest-reports jobs: diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/conda.yaml b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/conda.yaml similarity index 65% rename from assets/evaluation_on_cloud/environments/evaluations-built-in/context/conda.yaml rename to assets/evaluation_on_cloud/environments/evaluations-built-in/tests/conda.yaml index 267d5f64d4..4dffe17ad5 100644 --- a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/conda.yaml +++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/conda.yaml @@ -8,7 +8,7 @@ dependencies: - pandas - requests - pip: - - -r assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt + - -r ../context/requirements.txt - promptflow - pytest - pytest-xdist