diff --git a/.github/workflows/evaluation-on-cloud-tests.yaml b/.github/workflows/evaluation-on-cloud-tests.yaml
new file mode 100644
index 0000000000..84f536eb74
--- /dev/null
+++ b/.github/workflows/evaluation-on-cloud-tests.yaml
@@ -0,0 +1,57 @@
+name: evaluation-on-cloud-tests
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - assets/evaluation_on_cloud/**
+      - .github/workflows/evaluation-on-cloud-tests.yaml
+  workflow_dispatch:
+
+env:
+  conda_env_prefix: /opt/conda/envs/eval_on_cloud
+  eval_on_cloud_tests_dir: assets/evaluation_on_cloud/environments/evaluations-built-in/tests
+  eval_on_cloud_conda_yaml: assets/evaluation_on_cloud/environments/evaluations-built-in/tests/conda.yaml
+  pytest_reports: pytest-reports
+
+jobs:
+  run_unit_tests:
+    name: Run
+    runs-on: ubuntu-latest
+
+    permissions:
+      # Required for EnricoMi/publish-unit-test-result-action
+      checks: write
+      issues: read
+      pull-requests: write
+
+    steps:
+      - name: Clone branch
+        uses: actions/checkout@v3
+
+      - name: Use Python 3.10 or newer
+        uses: actions/setup-python@v4
+        with:
+          python-version: '>=3.10'
+
+      - name: Create conda env
+        run: |
+          set -ex
+          apt-get update && apt-get upgrade && apt-get install -y build-essential
+          echo $conda_env_prefix
+          sed -i 's/=={{latest-pypi-version}}//g' assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt
+          cat $eval_on_cloud_conda_yaml
+          conda env create -p $conda_env_prefix -f $eval_on_cloud_conda_yaml
+          echo "conda env successfully created at $conda_env_prefix"
+          conda list -p $conda_env_prefix
+
+      - name: Execute tests
+        run: conda run -p $conda_env_prefix python -m pytest $eval_on_cloud_tests_dir --tb=native --junitxml=$pytest_reports/test-result.xml -x -n 1 -ra --show-capture=no
+
+      - name: Publish test results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          check_name: Test Results for ${{ github.workflow }}
+          junit_files: ${{ env.pytest_reports }}/**/*.xml
diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py
index 8528226e92..58e6fb392d 100644
--- a/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py
+++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/context/evaluate_on_data.py
@@ -21,6 +21,7 @@
 from save_evaluation import load_evaluator
 from model_target import ModelTarget, get_key_from_dict
 
+os.environ["AZUREML_OBO_ENABLED"] = "True"
 AZURE_ENDPOINT = "AzureEndpoint"
 API_KEY = "ApiKey"
 AZURE_DEPLOYMENT = "AzureDeployment"
diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/conda.yaml b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/conda.yaml
new file mode 100644
index 0000000000..4dffe17ad5
--- /dev/null
+++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/conda.yaml
@@ -0,0 +1,14 @@
+name: eval_on_cloud
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - python=3.10
+  - pip
+  - pandas
+  - requests
+  - pip:
+      - -r ../context/requirements.txt
+      - promptflow
+      - pytest
+      - pytest-xdist
diff --git a/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py
new file mode 100644
index 0000000000..b088b31469
--- /dev/null
+++ b/assets/evaluation_on_cloud/environments/evaluations-built-in/tests/test_evaluate_on_data.py
@@ -0,0 +1,25 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""Tests for evaluate_on_data.py."""
+
+import sys
+from pathlib import Path
+import os
+import importlib
+
+
+def test_evaluate_obo_sets_env_var():
+    """Test that importing evaluate_on_data.py sets the AZUREML_OBO_ENABLED environment variable to "True"."""
+    script_path = Path(__file__).parent.parent / "context" / "evaluate_on_data.py"
+    module_name = "evaluate_on_data"
+    sys.path.insert(0, str(script_path.parent))
+
+    if module_name in sys.modules:
+        del sys.modules[module_name]
+
+    assert os.environ.get("AZUREML_OBO_ENABLED") is None
+
+    importlib.import_module(module_name)
+
+    assert os.environ.get("AZUREML_OBO_ENABLED") == "True"