Python: Lab: Updates to GAIA module (microsoft#1763)

* Lab: Updates to GAIA module

* update

* emoj!

* fix lint

* update lab test workflow to only trigger for python changes

* lint

* lint

* Fix broken OpenAI agents JS documentation link

Author: ekzhu

.github/workflows/python-lab-tests.yml

@@ -16,8 +16,34 @@ env:
   UV_CACHE_DIR: /tmp/.uv-cache
 
 jobs:
+  paths-filter:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+    outputs:
+      pythonChanges: ${{ steps.filter.outputs.python}}
+    steps:
+      - uses: actions/checkout@v5
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            python:
+              - 'python/**'
+      # run only if 'python' files were changed
+      - name: python tests
+        if: steps.filter.outputs.python == 'true'
+        run: echo "Python file"
+      # run only if not 'python' files were changed
+      - name: not python tests
+        if: steps.filter.outputs.python != 'true'
+        run: echo "NOT python file"
+
   python-lab-tests:
     name: Python Lab Tests
+    needs: paths-filter
+    if: needs.paths-filter.outputs.pythonChanges == 'true'
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: true

docs/decisions/0006-userapproval.md

@@ -22,7 +22,6 @@ This document aims to provide options and capture the decision on how to model t
 See various features that would need to be supported via this type of mechanism, plus how various other frameworks support this:
 
 - Also see [dotnet issue 6492](https://github.com/dotnet/extensions/issues/6492), which discusses the need for a similar pattern in the context of MCP approvals.
-- Also see [the openai RunToolApprovalItem](https://openai.github.io/openai-agents-js/openai/agents/classes/runtoolapprovalitem/).
 - Also see [the openai human-in-the-loop guide](https://openai.github.io/openai-agents-js/guides/human-in-the-loop/#approval-requests).
 - Also see [the openai MCP guide](https://openai.github.io/openai-agents-js/guides/mcp/#optional-approval-flow).
 - Also see [MCP Approval Requests from OpenAI](https://platform.openai.com/docs/guides/tools-remote-mcp#approvals).

python/packages/lab/gaia/README.md

@@ -43,21 +43,6 @@ async def main() -> None:
 
 See the [gaia_sample.py](./samples/gaia_sample.py) for more detail.
 
-### Run the evaluation
-
-Run the evaluation script using `uv`:
-
-```bash
-uv run python run_gaia.py
-```
-
-By default, the script will first look for cached GAIA data in the `data_gaia_hub` directory,
-and download it if not found.
-The result will be saved to `gaia_results_<timestamp>.jsonl`.
-
-**Don't run the script inside this directory because it will confuse the local `agent_framework` namespace
-package with the real one.**
-
 ## View results
 
 We provide a console viewer for reading GAIA results:

python/packages/lab/gaia/agent_framework_lab_gaia/gaia.py

@@ -54,17 +54,29 @@ def setup_observability(self) -> None:
         if not self.enable_tracing:
             return
 
-        from agent_framework.observability import setup_observability
+        # If only file tracing is requested (no OTLP or Application Insights),
+        # skip the default setup_observability which adds console exporter
+        if self.trace_to_file and not self.otlp_endpoint and not self.applicationinsights_connection_string:
+            # Set up minimal tracing with only file export
+            from opentelemetry.sdk.trace import TracerProvider
+            from opentelemetry.trace import set_tracer_provider
+
+            tracer_provider = TracerProvider()
+            set_tracer_provider(tracer_provider)
+            self._setup_file_export()
+        else:
+            # Use full observability setup for OTLP/AppInsights
+            from agent_framework.observability import setup_observability
 
-        setup_observability(
-            enable_sensitive_data=True,  # Enable for detailed task traces
-            otlp_endpoint=self.otlp_endpoint,
-            applicationinsights_connection_string=self.applicationinsights_connection_string,
-        )
+            setup_observability(
+                enable_sensitive_data=True,  # Enable for detailed task traces
+                otlp_endpoint=self.otlp_endpoint,
+                applicationinsights_connection_string=self.applicationinsights_connection_string,
+            )
 
-        # Set up local file export if requested
-        if self.trace_to_file:
-            self._setup_file_export()
+            # Set up local file export if requested
+            if self.trace_to_file:
+                self._setup_file_export()
 
     def _setup_file_export(self) -> None:
         """Set up local file export for traces."""
@@ -204,29 +216,87 @@ def _load_gaia_local(repo_dir: Path, wanted_levels: list[int] | None = None, max
     """Load GAIA tasks from local repository directory."""
     tasks: list[Task] = []
 
-    for p in repo_dir.rglob("metadata.jsonl"):
-        for rec in _read_jsonl(p):
-            # Robustly extract fields used across variants
-            q = rec.get("Question") or rec.get("question") or rec.get("query") or rec.get("prompt")
-            ans = rec.get("Final answer") or rec.get("answer") or rec.get("final_answer")
-            qid = str(
-                rec.get("task_id")
-                or rec.get("question_id")
-                or rec.get("id")
-                or rec.get("uuid")
-                or f"{p.stem}:{len(tasks)}"
-            )
-            lvl = rec.get("Level") or rec.get("level")
-            fname = rec.get("file_name") or rec.get("filename") or None
+    # First try to load from parquet files (new format)
+    # Prioritize validation split over test split (validation has answers)
+    parquet_files = sorted(
+        repo_dir.rglob("metadata*.parquet"), key=lambda p: (0 if "validation" in str(p) else 1, str(p))
+    )
 
-            # Only evaluate examples with public answers (dev/validation split)
-            if not q or ans is None:
-                continue
+    for p in parquet_files:
+        try:
+            import pyarrow.parquet as pq
+
+            table = pq.read_table(p)
+            for row in table.to_pylist():
+                # Robustly extract fields used across variants
+                q = row.get("Question") or row.get("question") or row.get("query") or row.get("prompt")
+                ans = row.get("Final answer") or row.get("answer") or row.get("final_answer")
+                qid = str(
+                    row.get("task_id")
+                    or row.get("question_id")
+                    or row.get("id")
+                    or row.get("uuid")
+                    or f"{p.stem}:{len(tasks)}"
+                )
+                lvl = row.get("Level") or row.get("level")
 
-            if wanted_levels and (lvl not in wanted_levels):
-                continue
+                # Convert level to int if it's a string
+                def _parse_level(lvl: Any) -> int | None:
+                    """Parse level value to integer if possible."""
+                    if isinstance(lvl, int):
+                        return lvl
+                    if isinstance(lvl, str) and lvl.isdigit():
+                        return int(lvl)
+                    return None
 
-            tasks.append(Task(task_id=qid, question=q, answer=str(ans), level=lvl, file_name=fname, metadata=rec))
+                lvl = _parse_level(lvl)
+                fname = row.get("file_name") or row.get("filename") or None
+
+                # Only evaluate examples with public answers (dev/validation split)
+                # Skip if no question, no answer, or answer is placeholder like "?"
+                if not q or ans is None or str(ans).strip() in ["?", ""]:
+                    continue
+
+                if wanted_levels and (lvl not in wanted_levels):
+                    continue
+
+                tasks.append(Task(task_id=qid, question=q, answer=str(ans), level=lvl, file_name=fname, metadata=row))
+        except ImportError:
+            print("Warning: pyarrow not installed. Install with: pip install pyarrow")
+            continue
+        except Exception as e:
+            print(f"Warning: Could not load parquet file {p}: {e}")
+            continue
+
+    # Fall back to jsonl files (old format) if no parquet files found
+    if not tasks:
+        for p in repo_dir.rglob("metadata.jsonl"):
+            for rec in _read_jsonl(p):
+                # Robustly extract fields used across variants
+                q = rec.get("Question") or rec.get("question") or rec.get("query") or rec.get("prompt")
+                ans = rec.get("Final answer") or rec.get("answer") or rec.get("final_answer")
+                qid = str(
+                    rec.get("task_id")
+                    or rec.get("question_id")
+                    or rec.get("id")
+                    or rec.get("uuid")
+                    or f"{p.stem}:{len(tasks)}"
+                )
+                lvl = rec.get("Level") or rec.get("level")
+                # Convert level to int if it's a string
+                if isinstance(lvl, str) and lvl.isdigit():
+                    lvl = int(lvl)
+                fname = rec.get("file_name") or rec.get("filename") or None
+
+                # Only evaluate examples with public answers (dev/validation split)
+                # Skip if no question, no answer, or answer is placeholder like "?"
+                if not q or ans is None or str(ans).strip() in ["?", ""]:
+                    continue
+
+                if wanted_levels and (lvl not in wanted_levels):
+                    continue
+
+                tasks.append(Task(task_id=qid, question=q, answer=str(ans), level=lvl, file_name=fname, metadata=rec))
 
     # Shuffle to help with rate-limits and fairness if max_n is provided
     random.shuffle(tasks)
@@ -290,7 +360,6 @@ def _ensure_data(self) -> Path:
                 "with access to gaia-benchmark/GAIA."
             )
 
-        print(f"Downloading GAIA dataset to {self.data_dir}...")
         from huggingface_hub import snapshot_download
 
         local_dir = snapshot_download(  # type: ignore
@@ -438,8 +507,6 @@ async def run(
                     "Make sure you have dataset access and selected valid levels."
                 )
 
-            print(f"Running {len(tasks)} GAIA tasks (levels={levels}) with {parallel} parallel workers...")
-
             # Update benchmark span with task info
             if benchmark_span:
                 benchmark_span.set_attributes({
@@ -473,17 +540,12 @@ async def run(
                     "gaia.benchmark.avg_runtime_seconds": avg_runtime,
                 })
 
-            print("\nGAIA Benchmark Results:")
-            print(f"Accuracy: {accuracy:.3f} ({correct}/{len(results)})")
-            print(f"Average runtime: {avg_runtime:.2f}s")
-
             # Save results if requested
             if out:
                 with self.tracer.start_as_current_span(
                     "gaia.results.save", kind=SpanKind.INTERNAL, attributes={"gaia.results.output_file": out}
                 ):
                     self._save_results(results, out)
-                    print(f"Results saved to {out}")
 
             return results
 

python/packages/lab/gaia/samples/azure_ai_agent.py

@@ -0,0 +1,67 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Azure AI Agent factory for GAIA benchmark.
+
+This module provides a factory function to create an Azure AI agent
+configured for GAIA benchmark tasks.
+
+Required Environment Variables:
+    AZURE_AI_PROJECT_ENDPOINT: Azure AI project endpoint URL
+    AZURE_AI_MODEL_DEPLOYMENT_NAME: Name of the model deployment to use
+
+Optional Environment Variables:
+    BING_CONNECTION_NAME: Name of the Bing connection for web search
+        OR
+    BING_CONNECTION_ID: ID of the Bing connection for web search
+
+Authentication:
+    Uses Azure CLI credentials via AzureCliCredential.
+    Run `az login` before executing to authenticate.
+
+Example:
+    export AZURE_AI_PROJECT_ENDPOINT="https://your-project.azure.com"
+    export AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o"
+    export BING_CONNECTION_NAME="bing-grounding-connection"
+    az login
+"""
+
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+from agent_framework import ChatAgent, HostedCodeInterpreterTool, HostedWebSearchTool
+from agent_framework.azure import AzureAIAgentClient
+from azure.identity.aio import AzureCliCredential
+
+
+@asynccontextmanager
+async def create_gaia_agent() -> AsyncIterator[ChatAgent]:
+    """Create an Azure AI agent configured for GAIA benchmark tasks.
+
+    The agent is configured with:
+    - Bing Search tool for web information retrieval
+    - Code Interpreter tool for calculations and data analysis
+
+    Yields:
+        ChatAgent: A configured agent ready to run GAIA tasks.
+
+    Example:
+        async with create_gaia_agent() as agent:
+            result = await agent.run("What is the capital of France?")
+            print(result.text)
+    """
+    async with (
+        AzureCliCredential() as credential,
+        AzureAIAgentClient(async_credential=credential).create_agent(
+            name="GaiaAgent",
+            instructions="Solve tasks to your best ability. Use Bing Search to find "
+            "information and Code Interpreter to perform calculations and data analysis.",
+            tools=[
+                HostedWebSearchTool(
+                    name="Bing Grounding Search",
+                    description="Search the web for current information using Bing",
+                ),
+                HostedCodeInterpreterTool(),
+            ],
+        ) as agent,
+    ):
+        yield agent