Skip to content

Commit 07ade2e

Browse files
jsondaicopybara-github
jsondai
authored and
copybara-github
committed
feat: GenAI Client(evals) - add create EvaluationExperiment
PiperOrigin-RevId: 953047538
1 parent 52b1c24 commit 07ade2e

7 files changed

Lines changed: 697 additions & 82 deletions

File tree

‎agentplatform/_genai/evals.py‎

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,29 @@
4343
logger = logging.getLogger("agentplatform_genai.evals")
4444

4545

46+
def _CreateEvaluationExperimentParameters_to_vertex(
47+
from_object: Union[dict[str, Any], object],
48+
parent_object: Optional[dict[str, Any]] = None,
49+
) -> dict[str, Any]:
50+
to_object: dict[str, Any] = {}
51+
if getv(from_object, ["display_name"]) is not None:
52+
setv(to_object, ["displayName"], getv(from_object, ["display_name"]))
53+
54+
if getv(from_object, ["labels"]) is not None:
55+
setv(to_object, ["labels"], getv(from_object, ["labels"]))
56+
57+
if getv(from_object, ["merge_strategy"]) is not None:
58+
setv(to_object, ["mergeStrategy"], getv(from_object, ["merge_strategy"]))
59+
60+
if getv(from_object, ["metadata"]) is not None:
61+
setv(to_object, ["metadata"], getv(from_object, ["metadata"]))
62+
63+
if getv(from_object, ["config"]) is not None:
64+
setv(to_object, ["config"], getv(from_object, ["config"]))
65+
66+
return to_object
67+
68+
4669
def _CreateEvaluationItemParameters_to_vertex(
4770
from_object: Union[dict[str, Any], object],
4871
parent_object: Optional[dict[str, Any]] = None,
@@ -1070,6 +1093,104 @@ def _UnifiedMetric_to_vertex(
10701093

10711094
class Evals(_api_module.BaseModule):
10721095

1096+
def create_evaluation_experiment(
1097+
self,
1098+
*,
1099+
display_name: Optional[str] = None,
1100+
labels: Optional[dict[str, str]] = None,
1101+
merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None,
1102+
metadata: Optional[dict[str, Any]] = None,
1103+
config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None,
1104+
) -> types.EvaluationExperiment:
1105+
"""
1106+
Creates an EvaluationExperiment.
1107+
1108+
Args:
1109+
display_name: The display name of the evaluation experiment.
1110+
labels: Labels for the evaluation experiment.
1111+
merge_strategy: Merge strategy for the evaluation experiment.
1112+
metadata: Metadata about the evaluation experiment, can be used by the
1113+
caller to store additional tracking information about the experiment.
1114+
config: Optional configuration for the create operation.
1115+
1116+
Returns:
1117+
The created evaluation experiment.
1118+
1119+
.. code-block:: python
1120+
1121+
eval_experiment = client.evals.create_evaluation_experiment(
1122+
display_name="my-experiment"
1123+
)
1124+
1125+
"""
1126+
1127+
parameter_model = types._CreateEvaluationExperimentParameters(
1128+
display_name=display_name,
1129+
labels=labels,
1130+
merge_strategy=merge_strategy,
1131+
metadata=metadata,
1132+
config=config,
1133+
)
1134+
1135+
request_url_dict: Optional[dict[str, str]]
1136+
if not self._api_client.vertexai:
1137+
raise ValueError(
1138+
"This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode."
1139+
)
1140+
else:
1141+
request_dict = _CreateEvaluationExperimentParameters_to_vertex(
1142+
parameter_model
1143+
)
1144+
request_url_dict = request_dict.get("_url")
1145+
if request_url_dict:
1146+
path = "evaluationExperiments".format_map(request_url_dict)
1147+
else:
1148+
path = "evaluationExperiments"
1149+
1150+
query_params = request_dict.get("_query")
1151+
if query_params:
1152+
path = f"{path}?{urlencode(query_params)}"
1153+
# TODO: remove the hack that pops config.
1154+
request_dict.pop("config", None)
1155+
1156+
http_options: Optional[types.HttpOptions] = None
1157+
if (
1158+
parameter_model.config is not None
1159+
and parameter_model.config.http_options is not None
1160+
):
1161+
http_options = parameter_model.config.http_options
1162+
1163+
request_dict = _common.convert_to_dict(request_dict)
1164+
request_dict = _common.encode_unserializable_types(request_dict)
1165+
1166+
response = self._api_client.request("post", path, request_dict, http_options)
1167+
1168+
response_dict = {} if not response.body else json.loads(response.body)
1169+
1170+
return_value = types.EvaluationExperiment._from_response(
1171+
response=response_dict,
1172+
kwargs=(
1173+
{
1174+
"config": {
1175+
"response_schema": getattr(
1176+
parameter_model.config, "response_schema", None
1177+
),
1178+
"response_json_schema": getattr(
1179+
parameter_model.config, "response_json_schema", None
1180+
),
1181+
"include_all_fields": getattr(
1182+
parameter_model.config, "include_all_fields", None
1183+
),
1184+
}
1185+
}
1186+
if getattr(parameter_model, "config", None)
1187+
else {}
1188+
),
1189+
)
1190+
1191+
self._api_client._verify_response(return_value)
1192+
return return_value
1193+
10731194
def _create_evaluation_item(
10741195
self,
10751196
*,
@@ -3403,6 +3524,106 @@ def delete_evaluation_metric(
34033524

34043525
class AsyncEvals(_api_module.BaseModule):
34053526

3527+
async def create_evaluation_experiment(
3528+
self,
3529+
*,
3530+
display_name: Optional[str] = None,
3531+
labels: Optional[dict[str, str]] = None,
3532+
merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None,
3533+
metadata: Optional[dict[str, Any]] = None,
3534+
config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None,
3535+
) -> types.EvaluationExperiment:
3536+
"""
3537+
Creates an EvaluationExperiment.
3538+
3539+
Args:
3540+
display_name: The display name of the evaluation experiment.
3541+
labels: Labels for the evaluation experiment.
3542+
merge_strategy: Merge strategy for the evaluation experiment.
3543+
metadata: Metadata about the evaluation experiment, can be used by the
3544+
caller to store additional tracking information about the experiment.
3545+
config: Optional configuration for the create operation.
3546+
3547+
Returns:
3548+
The created evaluation experiment.
3549+
3550+
.. code-block:: python
3551+
3552+
eval_experiment = client.evals.create_evaluation_experiment(
3553+
display_name="my-experiment"
3554+
)
3555+
3556+
"""
3557+
3558+
parameter_model = types._CreateEvaluationExperimentParameters(
3559+
display_name=display_name,
3560+
labels=labels,
3561+
merge_strategy=merge_strategy,
3562+
metadata=metadata,
3563+
config=config,
3564+
)
3565+
3566+
request_url_dict: Optional[dict[str, str]]
3567+
if not self._api_client.vertexai:
3568+
raise ValueError(
3569+
"This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode."
3570+
)
3571+
else:
3572+
request_dict = _CreateEvaluationExperimentParameters_to_vertex(
3573+
parameter_model
3574+
)
3575+
request_url_dict = request_dict.get("_url")
3576+
if request_url_dict:
3577+
path = "evaluationExperiments".format_map(request_url_dict)
3578+
else:
3579+
path = "evaluationExperiments"
3580+
3581+
query_params = request_dict.get("_query")
3582+
if query_params:
3583+
path = f"{path}?{urlencode(query_params)}"
3584+
# TODO: remove the hack that pops config.
3585+
request_dict.pop("config", None)
3586+
3587+
http_options: Optional[types.HttpOptions] = None
3588+
if (
3589+
parameter_model.config is not None
3590+
and parameter_model.config.http_options is not None
3591+
):
3592+
http_options = parameter_model.config.http_options
3593+
3594+
request_dict = _common.convert_to_dict(request_dict)
3595+
request_dict = _common.encode_unserializable_types(request_dict)
3596+
3597+
response = await self._api_client.async_request(
3598+
"post", path, request_dict, http_options
3599+
)
3600+
3601+
response_dict = {} if not response.body else json.loads(response.body)
3602+
3603+
return_value = types.EvaluationExperiment._from_response(
3604+
response=response_dict,
3605+
kwargs=(
3606+
{
3607+
"config": {
3608+
"response_schema": getattr(
3609+
parameter_model.config, "response_schema", None
3610+
),
3611+
"response_json_schema": getattr(
3612+
parameter_model.config, "response_json_schema", None
3613+
),
3614+
"include_all_fields": getattr(
3615+
parameter_model.config, "include_all_fields", None
3616+
),
3617+
}
3618+
}
3619+
if getattr(parameter_model, "config", None)
3620+
else {}
3621+
),
3622+
)
3623+
3624+
self._api_client._verify_response(return_value)
3625+
return return_value
3626+
34063627
async def _create_evaluation_item(
34073628
self,
34083629
*,

‎agentplatform/_genai/types/__init__.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
from .common import _CreateAgentEngineTaskRequestParameters
3737
from .common import _CreateDatasetParameters
3838
from .common import _CreateDatasetVersionParameters
39+
from .common import _CreateEvaluationExperimentParameters
3940
from .common import _CreateEvaluationItemParameters
4041
from .common import _CreateEvaluationMetricParameters
4142
from .common import _CreateEvaluationRunParameters
@@ -342,6 +343,9 @@
342343
from .common import CreateDatasetVersionConfig
343344
from .common import CreateDatasetVersionConfigDict
344345
from .common import CreateDatasetVersionConfigOrDict
346+
from .common import CreateEvaluationExperimentConfig
347+
from .common import CreateEvaluationExperimentConfigDict
348+
from .common import CreateEvaluationExperimentConfigOrDict
345349
from .common import CreateEvaluationItemConfig
346350
from .common import CreateEvaluationItemConfigDict
347351
from .common import CreateEvaluationItemConfigOrDict
@@ -2013,6 +2017,12 @@
20132017
"ListAgentEngineTaskEventsResponse",
20142018
"ListAgentEngineTaskEventsResponseDict",
20152019
"ListAgentEngineTaskEventsResponseOrDict",
2020+
"CreateEvaluationExperimentConfig",
2021+
"CreateEvaluationExperimentConfigDict",
2022+
"CreateEvaluationExperimentConfigOrDict",
2023+
"EvaluationExperiment",
2024+
"EvaluationExperimentDict",
2025+
"EvaluationExperimentOrDict",
20162026
"CreateEvaluationItemConfig",
20172027
"CreateEvaluationItemConfigDict",
20182028
"CreateEvaluationItemConfigOrDict",
@@ -2337,9 +2347,6 @@
23372347
"GetEvaluationExperimentConfig",
23382348
"GetEvaluationExperimentConfigDict",
23392349
"GetEvaluationExperimentConfigOrDict",
2340-
"EvaluationExperiment",
2341-
"EvaluationExperimentDict",
2342-
"EvaluationExperimentOrDict",
23432350
"GetEvaluationMetricConfig",
23442351
"GetEvaluationMetricConfigDict",
23452352
"GetEvaluationMetricConfigOrDict",
@@ -3669,10 +3676,10 @@
36693676
"VersionState",
36703677
"QuotaState",
36713678
"FeedbackType",
3679+
"EvaluationExperimentMergeStrategy",
36723680
"EvaluationItemType",
36733681
"SamplingMethod",
36743682
"EvaluationRunState",
3675-
"EvaluationExperimentMergeStrategy",
36763683
"OptimizeTarget",
36773684
"MemoryMetadataMergeStrategy",
36783685
"GenerateMemoriesResponseGeneratedMemoryAction",
@@ -3708,6 +3715,7 @@
37083715
"_CreateAgentEngineTaskRequestParameters",
37093716
"_AppendAgentEngineTaskEventRequestParameters",
37103717
"_ListAgentEngineTaskEventsRequestParameters",
3718+
"_CreateEvaluationExperimentParameters",
37113719
"_CreateEvaluationItemParameters",
37123720
"_CreateEvaluationMetricParameters",
37133721
"_CreateEvaluationRunParameters",

0 commit comments

Comments
 (0)