Skip to content

Commit 5505e1d

Browse files
aprilk-msCopilot
aprilk-ms
and
Copilot
authored
[ai-projects] Use typed EvaluatorGenerationJob in rubric samples (#47271)
* [ai-projects] Use typed EvaluatorGenerationJob in rubric samples The service contract nests the job inputs under an inputs field on EvaluatorGenerationJob. The four rubric-evaluator-generation samples were passing flat dicts that the SDK was tolerating but the rolling-out service change requires the nested form. Convert all four samples to use the typed EvaluatorGenerationJob / EvaluatorGenerationInputs / *EvaluatorGenerationJobSource models, and drop the stale top-level `name` field which has no home in the new contract. For the traces source, switch from int unix timestamps to datetime values (the SDK model serializes them as unix-timestamp). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * [ai-projects] CHANGELOG: note typed EvaluatorGenerationJob in rubric samples Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent e6848a3 commit 5505e1d

5 files changed

Lines changed: 130 additions & 110 deletions

‎sdk/ai/azure-ai-projects/CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
* Added `sample_routines_with_schedule_trigger.py` to demonstrate triggering a routine on a recurring cron schedule via `ScheduleRoutineTrigger`.
1010
* Updated `sample_dataset_generation_job_traces_for_evaluation.py` and `sample_dataset_generation_job_traces_for_finetuning.py` to create a temporary agent, seed conversations, retry the data generation job over the trace window, and clean up all created resources.
1111
* Updated `sample_memory_crud.py` and `sample_memory_crud_async.py` to demonstrate memory item CRUD (`create_memory`, `get_memory`, `update_memory`, `list_memories`, `delete_memory`) in addition to memory store CRUD.
12+
* Updated the rubric evaluator generation samples (`sample_rubric_evaluator_generation_basic.py`, `sample_rubric_evaluator_generation_iterate.py`, `sample_rubric_evaluator_generation_lifecycle.py`, `sample_rubric_evaluator_generation_all_sources.py`) to use the typed `EvaluatorGenerationJob` / `EvaluatorGenerationInputs` / `*EvaluatorGenerationJobSource` models. The job inputs are now nested under `inputs` per the service contract, and the traces source uses `datetime` values for `start_time` / `end_time`.
1213

1314
## 2.2.0 (2026-05-29)
1415

‎sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py‎

Lines changed: 59 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -50,14 +50,24 @@
5050
import os
5151
import time
5252
import uuid
53-
from datetime import datetime, timezone
54-
from typing import Any, Dict, List, cast
53+
from datetime import datetime, timedelta, timezone
54+
from typing import List, cast
5555

5656
from dotenv import load_dotenv
5757

5858
from azure.identity import DefaultAzureCredential
5959
from azure.ai.projects import AIProjectClient
60-
from azure.ai.projects.models import JobStatus, RubricBasedEvaluatorDefinition
60+
from azure.ai.projects.models import (
61+
AgentEvaluatorGenerationJobSource,
62+
DatasetEvaluatorGenerationJobSource,
63+
EvaluatorGenerationInputs,
64+
EvaluatorGenerationJob,
65+
EvaluatorGenerationJobSource,
66+
JobStatus,
67+
PromptEvaluatorGenerationJobSource,
68+
RubricBasedEvaluatorDefinition,
69+
TracesEvaluatorGenerationJobSource,
70+
)
6171

6272
load_dotenv()
6373

@@ -85,51 +95,49 @@
8595
AIProjectClient(endpoint=endpoint, credential=credential) as project_client,
8696
):
8797
# 1. Combined Prompt + Agent + Dataset generation job.
88-
multi_sources: List[Dict[str, Any]] = [
89-
{
90-
"type": "Prompt",
91-
"description": "Inline application overview.",
92-
"prompt": (
98+
multi_sources: List[EvaluatorGenerationJobSource] = [
99+
PromptEvaluatorGenerationJobSource(
100+
description="Inline application overview.",
101+
prompt=(
93102
"You are evaluating a customer-support assistant that helps users "
94103
"manage their accounts, troubleshoot issues, and place orders. The "
95104
"assistant uses tools for account lookup, password reset, and order "
96105
"creation. It must confirm intent before performing destructive "
97106
"actions and maintain a patient, professional tone."
98107
),
99-
}
108+
),
100109
]
101110
if agent_name:
102111
multi_sources.append(
103-
{
104-
"type": "Agent",
105-
"description": "Agent metadata enriches the rubric with tool and instruction signals.",
106-
"agent_name": agent_name,
107-
}
112+
AgentEvaluatorGenerationJobSource(
113+
description="Agent metadata enriches the rubric with tool and instruction signals.",
114+
agent_name=agent_name,
115+
)
108116
)
109117
else:
110118
print("Skipping Agent source (FOUNDRY_AGENT_NAME not set).")
111119

112120
if dataset_name and dataset_version:
113121
multi_sources.append(
114-
{
115-
"type": "Dataset",
116-
"description": "Reference examples ground dimensions in real data.",
117-
"name": dataset_name,
118-
"version": dataset_version,
119-
}
122+
DatasetEvaluatorGenerationJobSource(
123+
description="Reference examples ground dimensions in real data.",
124+
name=dataset_name,
125+
version=dataset_version,
126+
)
120127
)
121128
else:
122129
print("Skipping Dataset source (FOUNDRY_REFERENCE_DATASET_NAME / _VERSION not set).")
123130

124131
multi_job = project_client.beta.evaluators.create_generation_job(
125-
job={
126-
"model": model_name,
127-
"name": "Multi-source generation",
128-
"evaluator_name": multi_name,
129-
"evaluator_display_name": "Customer Support Quality (multi-source)",
130-
"evaluator_description": "Generated from prompt, agent, and dataset signals.",
131-
"sources": multi_sources,
132-
},
132+
job=EvaluatorGenerationJob(
133+
inputs=EvaluatorGenerationInputs(
134+
model=model_name,
135+
evaluator_name=multi_name,
136+
evaluator_display_name="Customer Support Quality (multi-source)",
137+
evaluator_description="Generated from prompt, agent, and dataset signals.",
138+
sources=multi_sources,
139+
),
140+
),
133141
operation_id=f"rubric-multi-{short}",
134142
)
135143

@@ -159,32 +167,31 @@
159167
if not agent_name:
160168
print("Skipping traces job (requires FOUNDRY_AGENT_NAME for both the traces source and companion).")
161169
else:
162-
now = int(time.time())
163-
start_time = now - traces_window_days * 24 * 3600
164-
end_time = now + 600 # small padding for clock skew
170+
now = datetime.now(tz=timezone.utc)
171+
start_time = now - timedelta(days=traces_window_days)
172+
end_time = now + timedelta(seconds=600) # small padding for clock skew
165173

166174
traces_job = project_client.beta.evaluators.create_generation_job(
167-
job={
168-
"model": model_name,
169-
"name": "Traces-source generation",
170-
"evaluator_name": traces_name,
171-
"evaluator_display_name": "Customer Support Quality (from traces)",
172-
"evaluator_description": "Generated from real Application Insights conversation traces.",
173-
"sources": [
174-
{
175-
"type": "traces",
176-
"description": "Application Insights conversation traces for the agent.",
177-
"agent_name": agent_name,
178-
"start_time": start_time,
179-
"end_time": end_time,
180-
},
181-
{
182-
"type": "Agent",
183-
"description": "Companion source (service rejects traces-only).",
184-
"agent_name": agent_name,
185-
},
186-
],
187-
},
175+
job=EvaluatorGenerationJob(
176+
inputs=EvaluatorGenerationInputs(
177+
model=model_name,
178+
evaluator_name=traces_name,
179+
evaluator_display_name="Customer Support Quality (from traces)",
180+
evaluator_description="Generated from real Application Insights conversation traces.",
181+
sources=[
182+
TracesEvaluatorGenerationJobSource(
183+
description="Application Insights conversation traces for the agent.",
184+
agent_name=agent_name,
185+
start_time=start_time,
186+
end_time=end_time,
187+
),
188+
AgentEvaluatorGenerationJobSource(
189+
description="Companion source (service rejects traces-only).",
190+
agent_name=agent_name,
191+
),
192+
],
193+
),
194+
),
188195
operation_id=f"rubric-traces-{short}",
189196
)
190197

‎sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,10 @@
6060
from azure.identity import DefaultAzureCredential
6161
from azure.ai.projects import AIProjectClient
6262
from azure.ai.projects.models import (
63+
EvaluatorGenerationInputs,
64+
EvaluatorGenerationJob,
6365
JobStatus,
66+
PromptEvaluatorGenerationJobSource,
6467
RubricBasedEvaluatorDefinition,
6568
TestingCriterionAzureAIEvaluator,
6669
)
@@ -86,31 +89,31 @@
8689
):
8790
# 1. Generate an evaluator from a single `Prompt` source.
8891
job = project_client.beta.evaluators.create_generation_job(
89-
job={
90-
"model": model_name,
91-
"name": "Reservation Quality (Generated)",
92-
"evaluator_name": evaluator_name,
93-
"evaluator_display_name": "Reservation Quality (Generated)",
94-
"evaluator_description": "Quality evaluator generated from a prompt describing a restaurant reservation assistant.",
95-
"sources": [
96-
{
97-
"type": "Prompt",
98-
"description": "Application overview - purpose, capabilities, and tools.",
99-
"prompt": (
100-
"You are evaluating a restaurant reservation assistant. The assistant helps "
101-
"users create, modify, and cancel reservations at participating restaurants. "
102-
"It can:\n"
103-
" - Search for restaurants by name, cuisine, or neighborhood.\n"
104-
" - Check table availability for a requested date, time, and party size.\n"
105-
" - Create, update, and cancel reservations on behalf of the user.\n"
106-
" - Send SMS or email confirmations through a notifications tool.\n"
107-
"It must always confirm the user's intent before committing changes, "
108-
"ask follow-up questions when details are missing, and maintain a polite "
109-
"restaurant-host tone."
92+
job=EvaluatorGenerationJob(
93+
inputs=EvaluatorGenerationInputs(
94+
model=model_name,
95+
evaluator_name=evaluator_name,
96+
evaluator_display_name="Reservation Quality (Generated)",
97+
evaluator_description="Quality evaluator generated from a prompt describing a restaurant reservation assistant.",
98+
sources=[
99+
PromptEvaluatorGenerationJobSource(
100+
description="Application overview - purpose, capabilities, and tools.",
101+
prompt=(
102+
"You are evaluating a restaurant reservation assistant. The assistant helps "
103+
"users create, modify, and cancel reservations at participating restaurants. "
104+
"It can:\n"
105+
" - Search for restaurants by name, cuisine, or neighborhood.\n"
106+
" - Check table availability for a requested date, time, and party size.\n"
107+
" - Create, update, and cancel reservations on behalf of the user.\n"
108+
" - Send SMS or email confirmations through a notifications tool.\n"
109+
"It must always confirm the user's intent before committing changes, "
110+
"ask follow-up questions when details are missing, and maintain a polite "
111+
"restaurant-host tone."
112+
),
110113
),
111-
}
112-
],
113-
},
114+
],
115+
),
116+
),
114117
# `operation_id` makes the call idempotent - re-submitting the same id returns the existing job.
115118
operation_id=f"rubric-eval-basic-{short}",
116119
)

‎sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py‎

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,10 @@
4949
from azure.ai.projects.models import (
5050
EvaluatorCategory,
5151
EvaluatorDefinitionType,
52+
EvaluatorGenerationInputs,
53+
EvaluatorGenerationJob,
5254
JobStatus,
55+
PromptEvaluatorGenerationJobSource,
5356
RubricBasedEvaluatorDefinition,
5457
)
5558

@@ -72,25 +75,25 @@
7275
):
7376
# 1. Generate v1 of the evaluator from a single `Prompt` source.
7477
job = project_client.beta.evaluators.create_generation_job(
75-
job={
76-
"model": model_name,
77-
"name": "Reservation Quality (iterate)",
78-
"evaluator_name": evaluator_name,
79-
"evaluator_display_name": "Reservation Quality (iterate)",
80-
"evaluator_description": "Starting point for human-in-the-loop iteration.",
81-
"sources": [
82-
{
83-
"type": "Prompt",
84-
"description": "Inline application overview.",
85-
"prompt": (
86-
"You are evaluating a restaurant reservation assistant that creates, "
87-
"modifies, and cancels reservations. It uses tools for restaurant "
88-
"lookup, availability checking, and notifications. It must confirm "
89-
"user intent before committing changes."
78+
job=EvaluatorGenerationJob(
79+
inputs=EvaluatorGenerationInputs(
80+
model=model_name,
81+
evaluator_name=evaluator_name,
82+
evaluator_display_name="Reservation Quality (iterate)",
83+
evaluator_description="Starting point for human-in-the-loop iteration.",
84+
sources=[
85+
PromptEvaluatorGenerationJobSource(
86+
description="Inline application overview.",
87+
prompt=(
88+
"You are evaluating a restaurant reservation assistant that creates, "
89+
"modifies, and cancels reservations. It uses tools for restaurant "
90+
"lookup, availability checking, and notifications. It must confirm "
91+
"user intent before committing changes."
92+
),
9093
),
91-
}
92-
],
93-
},
94+
],
95+
),
96+
),
9497
operation_id=f"rubric-iterate-{short}",
9598
)
9699

‎sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py‎

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,13 @@
5050
from azure.core.exceptions import ResourceNotFoundError
5151
from azure.identity import DefaultAzureCredential
5252
from azure.ai.projects import AIProjectClient
53-
from azure.ai.projects.models import JobStatus, PageOrder
53+
from azure.ai.projects.models import (
54+
EvaluatorGenerationInputs,
55+
EvaluatorGenerationJob,
56+
JobStatus,
57+
PageOrder,
58+
PromptEvaluatorGenerationJobSource,
59+
)
5460

5561
load_dotenv()
5662

@@ -66,21 +72,21 @@
6672

6773
TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED}
6874

69-
# Shared job body used both for the initial create and the idempotency replay.
70-
job_body = {
71-
"model": model_name,
72-
"name": "Lifecycle demo",
73-
"evaluator_name": evaluator_name,
74-
"evaluator_display_name": "Lifecycle demo",
75-
"evaluator_description": "Minimal job used to demonstrate the LRO + list/delete lifecycle.",
76-
"sources": [
77-
{
78-
"type": "Prompt",
79-
"description": "Inline application overview.",
80-
"prompt": "You are evaluating a simple Q&A assistant that answers factual questions clearly and concisely.",
81-
}
82-
],
83-
}
75+
# Shared job used both for the initial create and the idempotency replay.
76+
job_body = EvaluatorGenerationJob(
77+
inputs=EvaluatorGenerationInputs(
78+
model=model_name,
79+
evaluator_name=evaluator_name,
80+
evaluator_display_name="Lifecycle demo",
81+
evaluator_description="Minimal job used to demonstrate the LRO + list/delete lifecycle.",
82+
sources=[
83+
PromptEvaluatorGenerationJobSource(
84+
description="Inline application overview.",
85+
prompt="You are evaluating a simple Q&A assistant that answers factual questions clearly and concisely.",
86+
),
87+
],
88+
),
89+
)
8490

8591
with (
8692
DefaultAzureCredential() as credential,

0 commit comments

Comments
 (0)