Python: Add experimental decorator to all Evals pieces (microsoft#5040)

* Initial plan

* feat(python): add experimental decorator to all Evals pieces

Agent-Logs-Url: https://github.com/microsoft/agent-framework/sessions/99d71249-a5d6-4977-a5b5-6ffe0a3be2bc

Co-authored-by: TaoChenOSU <12570346+TaoChenOSU@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: TaoChenOSU <12570346+TaoChenOSU@users.noreply.github.com>

Author: Copilot

python/packages/core/agent_framework/_evaluation.py

@@ -53,6 +53,7 @@
     runtime_checkable,
 )
 
+from ._feature_stage import ExperimentalFeature, experimental
 from ._tools import FunctionTool
 from ._types import AgentResponse, Message
 
@@ -64,13 +65,15 @@
 logger = logging.getLogger(__name__)
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 class EvalNotPassedError(Exception):
     """Raised when evaluation results contain failures."""
 
 
 # region Core types
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 @runtime_checkable
 class ConversationSplitter(Protocol):
     """Strategy for splitting a conversation into (query, response) messages.
@@ -103,6 +106,7 @@ def split_before_memory(conversation):
     def __call__(self, conversation: list[Message]) -> tuple[list[Message], list[Message]]: ...
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 class ConversationSplit(str, Enum):
     """Built-in conversation split strategies.
 
@@ -131,6 +135,7 @@ def __call__(self, conversation: list[Message]) -> tuple[list[Message], list[Mes
         return _BUILT_IN_SPLITTERS[self](conversation)
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 @dataclass
 class ExpectedToolCall:
     """A tool call that an agent is expected to make.
@@ -173,6 +178,7 @@ def _split_full(conversation: list[Message]) -> tuple[list[Message], list[Messag
 }
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 class EvalItem:
     """A single item to be evaluated.
 
@@ -295,6 +301,7 @@ def per_turn_items(
 # region Score and result types
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 @dataclass
 class EvalScoreResult:
     """Result from a single evaluator on a single item.
@@ -312,6 +319,7 @@ class EvalScoreResult:
     sample: dict[str, Any] | None = None
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 @dataclass
 class EvalItemResult:
     """Per-item result from an evaluation run.
@@ -358,6 +366,7 @@ def is_failed(self) -> bool:
         return self.status == "fail"
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 class EvalResults:
     """Results from an evaluation run by a single provider.
 
@@ -493,6 +502,7 @@ def raise_for_status(self, msg: str | None = None) -> None:
 # region Evaluator protocol
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 @runtime_checkable
 class Evaluator(Protocol):
     """Protocol for evaluation providers.
@@ -543,6 +553,7 @@ async def evaluate(
 # region Converter
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 class AgentEvalConverter:
     """Converts agent-framework types to evaluation format.
 
@@ -846,6 +857,7 @@ def _extract_overall_query(workflow_result: WorkflowRunResult) -> str | None:
 # region Local evaluation checks
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 @dataclass
 class CheckResult:
     """Result of a single check on a single evaluation item.
@@ -870,6 +882,7 @@ class CheckResult:
 """
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 def keyword_check(*keywords: str, case_sensitive: bool = False) -> EvalCheck:
     """Check that the response contains all specified keywords.
 
@@ -897,6 +910,7 @@ def _check(item: EvalItem) -> CheckResult:
     return _check
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") -> EvalCheck:
     """Check that specific tools were called during the conversation.
 
@@ -979,6 +993,7 @@ def _extract_tool_calls(item: EvalItem) -> list[tuple[str, dict[str, Any] | None
     return calls
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 def tool_calls_present(item: EvalItem) -> CheckResult:
     """Check that all expected tool calls were made (unordered, extras OK).
 
@@ -1020,6 +1035,7 @@ def tool_calls_present(item: EvalItem) -> CheckResult:
     )
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 def tool_call_args_match(item: EvalItem) -> CheckResult:
     """Check that expected tool calls match on name and arguments.
 
@@ -1220,6 +1236,7 @@ def evaluator(fn: Callable[..., Any], /) -> EvalCheck: ...
 def evaluator(*, name: str | None = None) -> Callable[[Callable[..., Any]], EvalCheck]: ...
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 def evaluator(
     fn: Callable[..., Any] | None = None,
     *,
@@ -1322,6 +1339,7 @@ async def _run_check(check_fn: EvalCheck, item: EvalItem) -> CheckResult:
     return result
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 class LocalEvaluator:
     """Evaluation provider that runs checks locally without API calls.
 
@@ -1431,6 +1449,7 @@ async def evaluate(
 # region Public orchestration functions
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 async def evaluate_agent(
     *,
     agent: SupportsAgentRun | None = None,
@@ -1634,6 +1653,7 @@ async def evaluate_agent(
     return await _run_evaluators(evaluators, items, eval_name=name)
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 async def evaluate_workflow(
     *,
     workflow: Workflow,

python/packages/core/agent_framework/_feature_stage.py

@@ -46,6 +46,7 @@ class ExperimentalFeature(str, Enum):
     on enum membership or attribute presence over time.
     """
 
+    EVALS = "EVALS"
     SKILLS = "SKILLS"
 
 

python/packages/foundry/agent_framework_foundry/_foundry_evals.py

@@ -40,6 +40,7 @@
     EvalResults,
     EvalScoreResult,
 )
+from agent_framework._feature_stage import ExperimentalFeature, experimental
 from openai import AsyncOpenAI
 
 from ._chat_client import FoundryChatClient
@@ -491,6 +492,7 @@ async def _evaluate_via_responses_impl(
 # ---------------------------------------------------------------------------
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 class FoundryEvals:
     """Evaluation provider backed by Microsoft Foundry.
 
@@ -727,6 +729,7 @@ async def _evaluate_via_dataset(
 # ---------------------------------------------------------------------------
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 async def evaluate_traces(
     *,
     evaluators: Sequence[str] | None = None,
@@ -817,6 +820,7 @@ async def evaluate_traces(
     return await _poll_eval_run(oai_client, eval_obj.id, run.id, poll_interval, timeout)
 
 
+@experimental(feature_id=ExperimentalFeature.EVALS)
 async def evaluate_foundry_target(
     *,
     target: dict[str, Any],