Skip to content

Commit 6053106

Browse files
vertex-sdk-botcopybara-github
authored andcommitted
feat:Add metric_resource_name to evaluation types and methods
PiperOrigin-RevId: 877145985
1 parent 6b5cc8f commit 6053106

2 files changed

Lines changed: 46 additions & 0 deletions

File tree

vertexai/_genai/evals.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,12 @@ def _EvaluationRunMetric_from_vertex(
309309
["metric_config"],
310310
_UnifiedMetric_from_vertex(getv(from_object, ["metricConfig"]), to_object),
311311
)
312+
if getv(from_object, ["metricResourceName"]) is not None:
313+
setv(
314+
to_object,
315+
["metric_resource_name"],
316+
getv(from_object, ["metricResourceName"]),
317+
)
312318

313319
return to_object
314320

@@ -327,6 +333,12 @@ def _EvaluationRunMetric_to_vertex(
327333
["metricConfig"],
328334
_UnifiedMetric_to_vertex(getv(from_object, ["metric_config"]), to_object),
329335
)
336+
if getv(from_object, ["metric_resource_name"]) is not None:
337+
setv(
338+
to_object,
339+
["metricResourceName"],
340+
getv(from_object, ["metric_resource_name"]),
341+
)
330342

331343
return to_object
332344

@@ -415,6 +427,12 @@ def _GenerateInstanceRubricsRequest_to_vertex(
415427
getv(from_object, ["rubric_generation_spec"]), to_object
416428
),
417429
)
430+
if getv(from_object, ["metric_resource_name"]) is not None:
431+
setv(
432+
to_object,
433+
["metricResourceName"],
434+
getv(from_object, ["metric_resource_name"]),
435+
)
418436

419437
if getv(from_object, ["config"]) is not None:
420438
setv(to_object, ["config"], getv(from_object, ["config"]))
@@ -574,6 +592,12 @@ def _RubricGenerationSpec_to_vertex(
574592
["rubricTypeOntology"],
575593
getv(from_object, ["rubric_type_ontology"]),
576594
)
595+
if getv(from_object, ["metric_resource_name"]) is not None:
596+
setv(
597+
to_object,
598+
["metricResourceName"],
599+
getv(from_object, ["metric_resource_name"]),
600+
)
577601

578602
return to_object
579603

@@ -949,6 +973,7 @@ def _generate_rubrics(
949973
self,
950974
*,
951975
contents: list[genai_types.ContentOrDict],
976+
metric_resource_name: Optional[str] = None,
952977
predefined_rubric_generation_spec: Optional[
953978
types.PredefinedMetricSpecOrDict
954979
] = None,
@@ -963,6 +988,7 @@ def _generate_rubrics(
963988
contents=contents,
964989
predefined_rubric_generation_spec=predefined_rubric_generation_spec,
965990
rubric_generation_spec=rubric_generation_spec,
991+
metric_resource_name=metric_resource_name,
966992
config=config,
967993
)
968994

@@ -1462,6 +1488,7 @@ def generate_rubrics(
14621488
generator_model_config: Optional["genai_types.AutoraterConfigOrDict"] = None,
14631489
rubric_content_type: Optional["types.RubricContentType"] = None,
14641490
rubric_type_ontology: Optional[list[str]] = None,
1491+
metric_resource_name: Optional[str] = None,
14651492
predefined_spec_name: Optional[Union[str, "types.PrebuiltMetric"]] = None,
14661493
metric_spec_parameters: Optional[dict[str, Any]] = None,
14671494
config: Optional[types.RubricGenerationConfigOrDict] = None,
@@ -1497,6 +1524,7 @@ def generate_rubrics(
14971524
generated. Only used if `prompt_template` is provided.
14981525
rubric_type_ontology: Optional. A pre-defined list of allowed types
14991526
for generated rubrics. Only used if `prompt_template` is provided.
1527+
metric_resource_name: Optional. The resource name of the metric definition.
15001528
predefined_spec_name: Optional. The name of a Predefined Metric to use
15011529
for rubric generation (e.g., "general_quality_v1") or a types.PrebuiltMetric object.
15021530
Mutually exclusive with `prompt_template` and its related parameters.
@@ -1625,6 +1653,7 @@ def generate_rubrics(
16251653
contents=contents,
16261654
rubric_generation_spec=rubric_gen_spec,
16271655
predefined_rubric_generation_spec=predefined_spec,
1656+
metric_resource_name=metric_resource_name,
16281657
config=config,
16291658
)
16301659
rubric_group = {rubric_group_name: response.generated_rubrics}

vertexai/_genai/types/common.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2243,6 +2243,10 @@ class RubricGenerationSpec(_common.BaseModel):
22432243
If this field is provided, it implies `include_rubric_type` should be true,
22442244
and the generated rubric types should be chosen from this ontology.""",
22452245
)
2246+
metric_resource_name: Optional[str] = Field(
2247+
default=None,
2248+
description="""The resource name of the metric definition.""",
2249+
)
22462250

22472251

22482252
class RubricGenerationSpecDict(TypedDict, total=False):
@@ -2265,6 +2269,8 @@ class RubricGenerationSpecDict(TypedDict, total=False):
22652269
If this field is provided, it implies `include_rubric_type` should be true,
22662270
and the generated rubric types should be chosen from this ontology."""
22672271

2272+
metric_resource_name: Optional[str]
2273+
22682274

22692275
RubricGenerationSpecOrDict = Union[RubricGenerationSpec, RubricGenerationSpecDict]
22702276

@@ -2451,6 +2457,10 @@ class EvaluationRunMetric(_common.BaseModel):
24512457
metric_config: Optional[UnifiedMetric] = Field(
24522458
default=None, description="""The unified metric used for evaluation run."""
24532459
)
2460+
metric_resource_name: Optional[str] = Field(
2461+
default=None,
2462+
description="""The resource name of the metric definition.""",
2463+
)
24542464

24552465

24562466
class EvaluationRunMetricDict(TypedDict, total=False):
@@ -2462,6 +2472,8 @@ class EvaluationRunMetricDict(TypedDict, total=False):
24622472
metric_config: Optional[UnifiedMetricDict]
24632473
"""The unified metric used for evaluation run."""
24642474

2475+
metric_resource_name: Optional[str]
2476+
24652477

24662478
EvaluationRunMetricOrDict = Union[EvaluationRunMetric, EvaluationRunMetricDict]
24672479

@@ -5231,6 +5243,10 @@ class _GenerateInstanceRubricsRequest(_common.BaseModel):
52315243
default=None,
52325244
description="""Specification for how the rubrics should be generated.""",
52335245
)
5246+
metric_resource_name: Optional[str] = Field(
5247+
default=None,
5248+
description="""The resource name of the metric definition.""",
5249+
)
52345250
config: Optional[RubricGenerationConfig] = Field(default=None, description="""""")
52355251

52365252

@@ -5251,6 +5267,7 @@ class _GenerateInstanceRubricsRequestDict(TypedDict, total=False):
52515267
rubric_generation_spec: Optional[RubricGenerationSpecDict]
52525268
"""Specification for how the rubrics should be generated."""
52535269

5270+
metric_resource_name: Optional[str]
52545271
config: Optional[RubricGenerationConfigDict]
52555272
""""""
52565273

0 commit comments

Comments
 (0)