Skip to content

Commit 585545f

Browse files
jsondaicopybara-github
jsondai
authored and
copybara-github
committed
chore: GenAI Client(evals) - add EvaluationExperiment type
PiperOrigin-RevId: 934627629
1 parent 2f5c8f8 commit 585545f

4 files changed

Lines changed: 172 additions & 0 deletions

File tree

‎agentplatform/_genai/types/__init__.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -505,6 +505,10 @@
505505
from .common import EvaluationDataset
506506
from .common import EvaluationDatasetDict
507507
from .common import EvaluationDatasetOrDict
508+
from .common import EvaluationExperiment
509+
from .common import EvaluationExperimentDict
510+
from .common import EvaluationExperimentMergeStrategy
511+
from .common import EvaluationExperimentOrDict
508512
from .common import EvaluationInstance
509513
from .common import EvaluationInstanceDict
510514
from .common import EvaluationInstanceOrDict
@@ -3284,6 +3288,9 @@
32843288
"ObservabilityEvalCase",
32853289
"ObservabilityEvalCaseDict",
32863290
"ObservabilityEvalCaseOrDict",
3291+
"EvaluationExperiment",
3292+
"EvaluationExperimentDict",
3293+
"EvaluationExperimentOrDict",
32873294
"RubricGroup",
32883295
"RubricGroupDict",
32893296
"RubricGroupOrDict",
@@ -3380,6 +3387,7 @@
33803387
"GenerateMemoriesResponseGeneratedMemoryAction",
33813388
"SkillRevisionState",
33823389
"PromptOptimizerMethod",
3390+
"EvaluationExperimentMergeStrategy",
33833391
"OptimizationMethod",
33843392
"PromptData",
33853393
"PromptDataDict",

‎agentplatform/_genai/types/common.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -535,6 +535,17 @@ class PromptOptimizerMethod(_common.CaseInSensitiveEnum):
535535
"""The data driven prompt optimizer designer for prompts from Android core API."""
536536

537537

538+
class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum):
539+
"""Merge strategy for the evaluation experiment."""
540+
541+
MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED"
542+
"""Unspecified merge strategy."""
543+
SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY"
544+
"""Default. Runs are treated as an independent, sequential history."""
545+
SHARED_RESULT_SET = "SHARED_RESULT_SET"
546+
"""Runs are parallel iterations contributing to a shared result set."""
547+
548+
538549
class OptimizationMethod(_common.CaseInSensitiveEnum):
539550
"""The method for data driven prompt optimization."""
540551

@@ -22747,6 +22758,73 @@ class ObservabilityEvalCaseDict(TypedDict, total=False):
2274722758
ObservabilityEvalCaseOrDict = Union[ObservabilityEvalCase, ObservabilityEvalCaseDict]
2274822759

2274922760

22761+
class EvaluationExperiment(_common.BaseModel):
22762+
"""Represents an experiment for iterating on and visualizing evaluation runs."""
22763+
22764+
name: Optional[str] = Field(
22765+
default=None,
22766+
description="""The resource name of the EvaluationExperiment. Format:
22767+
`projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""",
22768+
)
22769+
display_name: Optional[str] = Field(
22770+
default=None, description="""The display name of the evaluation experiment."""
22771+
)
22772+
evaluation_runs: Optional[list[str]] = Field(
22773+
default=None,
22774+
description="""The EvaluationRuns that are part of this experiment.""",
22775+
)
22776+
labels: Optional[dict[str, str]] = Field(
22777+
default=None, description="""Labels for the evaluation experiment."""
22778+
)
22779+
merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field(
22780+
default=None, description="""Merge strategy for the evaluation experiment."""
22781+
)
22782+
metadata: Optional[dict[str, Any]] = Field(
22783+
default=None,
22784+
description="""Metadata about the evaluation experiment, can be used by the caller
22785+
to store additional tracking information about the experiment.""",
22786+
)
22787+
create_time: Optional[datetime.datetime] = Field(
22788+
default=None, description="""Timestamp when this experiment was created."""
22789+
)
22790+
update_time: Optional[datetime.datetime] = Field(
22791+
default=None, description="""Timestamp when this experiment was last updated."""
22792+
)
22793+
22794+
22795+
class EvaluationExperimentDict(TypedDict, total=False):
22796+
"""Represents an experiment for iterating on and visualizing evaluation runs."""
22797+
22798+
name: Optional[str]
22799+
"""The resource name of the EvaluationExperiment. Format:
22800+
`projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`."""
22801+
22802+
display_name: Optional[str]
22803+
"""The display name of the evaluation experiment."""
22804+
22805+
evaluation_runs: Optional[list[str]]
22806+
"""The EvaluationRuns that are part of this experiment."""
22807+
22808+
labels: Optional[dict[str, str]]
22809+
"""Labels for the evaluation experiment."""
22810+
22811+
merge_strategy: Optional[EvaluationExperimentMergeStrategy]
22812+
"""Merge strategy for the evaluation experiment."""
22813+
22814+
metadata: Optional[dict[str, Any]]
22815+
"""Metadata about the evaluation experiment, can be used by the caller
22816+
to store additional tracking information about the experiment."""
22817+
22818+
create_time: Optional[datetime.datetime]
22819+
"""Timestamp when this experiment was created."""
22820+
22821+
update_time: Optional[datetime.datetime]
22822+
"""Timestamp when this experiment was last updated."""
22823+
22824+
22825+
EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict]
22826+
22827+
2275022828
class RubricGroup(_common.BaseModel):
2275122829
"""A group of rubrics.
2275222830

‎vertexai/_genai/types/__init__.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,10 @@
443443
from .common import EvaluationDataset
444444
from .common import EvaluationDatasetDict
445445
from .common import EvaluationDatasetOrDict
446+
from .common import EvaluationExperiment
447+
from .common import EvaluationExperimentDict
448+
from .common import EvaluationExperimentMergeStrategy
449+
from .common import EvaluationExperimentOrDict
446450
from .common import EvaluationInstance
447451
from .common import EvaluationInstanceDict
448452
from .common import EvaluationInstanceOrDict
@@ -2634,6 +2638,10 @@
26342638
"EvaluateMethodConfig",
26352639
"EvaluateMethodConfigDict",
26362640
"EvaluateMethodConfigOrDict",
2641+
"EvaluationExperiment",
2642+
"EvaluationExperimentDict",
2643+
"EvaluationExperimentOrDict",
2644+
"EvaluationExperimentMergeStrategy",
26372645
"EvaluateDatasetConfig",
26382646
"EvaluateDatasetConfigDict",
26392647
"EvaluateDatasetConfigOrDict",

‎vertexai/_genai/types/common.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,17 @@ class PromptOptimizerMethod(_common.CaseInSensitiveEnum):
498498
"""The data driven prompt optimizer designer for prompts from Android core API."""
499499

500500

501+
class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum):
502+
"""Merge strategy for the evaluation experiment."""
503+
504+
MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED"
505+
"""Unspecified merge strategy."""
506+
SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY"
507+
"""Default. Runs are treated as an independent, sequential history."""
508+
SHARED_RESULT_SET = "SHARED_RESULT_SET"
509+
"""Runs are parallel iterations contributing to a shared result set."""
510+
511+
501512
class OptimizationMethod(_common.CaseInSensitiveEnum):
502513
"""The method for data driven prompt optimization."""
503514

@@ -19121,6 +19132,73 @@ class ObservabilityEvalCaseDict(TypedDict, total=False):
1912119132
ObservabilityEvalCaseOrDict = Union[ObservabilityEvalCase, ObservabilityEvalCaseDict]
1912219133

1912319134

19135+
class EvaluationExperiment(_common.BaseModel):
19136+
"""Represents an experiment for iterating on and visualizing evaluation runs."""
19137+
19138+
name: Optional[str] = Field(
19139+
default=None,
19140+
description="""The resource name of the EvaluationExperiment. Format:
19141+
`projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""",
19142+
)
19143+
display_name: Optional[str] = Field(
19144+
default=None, description="""The display name of the evaluation experiment."""
19145+
)
19146+
evaluation_runs: Optional[list[str]] = Field(
19147+
default=None,
19148+
description="""The EvaluationRuns that are part of this experiment.""",
19149+
)
19150+
labels: Optional[dict[str, str]] = Field(
19151+
default=None, description="""Labels for the evaluation experiment."""
19152+
)
19153+
merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field(
19154+
default=None, description="""Merge strategy for the evaluation experiment."""
19155+
)
19156+
metadata: Optional[dict[str, Any]] = Field(
19157+
default=None,
19158+
description="""Metadata about the evaluation experiment, can be used by the caller
19159+
to store additional tracking information about the experiment.""",
19160+
)
19161+
create_time: Optional[datetime.datetime] = Field(
19162+
default=None, description="""Timestamp when this experiment was created."""
19163+
)
19164+
update_time: Optional[datetime.datetime] = Field(
19165+
default=None, description="""Timestamp when this experiment was last updated."""
19166+
)
19167+
19168+
19169+
class EvaluationExperimentDict(TypedDict, total=False):
19170+
"""Represents an experiment for iterating on and visualizing evaluation runs."""
19171+
19172+
name: Optional[str]
19173+
"""The resource name of the EvaluationExperiment. Format:
19174+
`projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`."""
19175+
19176+
display_name: Optional[str]
19177+
"""The display name of the evaluation experiment."""
19178+
19179+
evaluation_runs: Optional[list[str]]
19180+
"""The EvaluationRuns that are part of this experiment."""
19181+
19182+
labels: Optional[dict[str, str]]
19183+
"""Labels for the evaluation experiment."""
19184+
19185+
merge_strategy: Optional[EvaluationExperimentMergeStrategy]
19186+
"""Merge strategy for the evaluation experiment."""
19187+
19188+
metadata: Optional[dict[str, Any]]
19189+
"""Metadata about the evaluation experiment, can be used by the caller
19190+
to store additional tracking information about the experiment."""
19191+
19192+
create_time: Optional[datetime.datetime]
19193+
"""Timestamp when this experiment was created."""
19194+
19195+
update_time: Optional[datetime.datetime]
19196+
"""Timestamp when this experiment was last updated."""
19197+
19198+
19199+
EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict]
19200+
19201+
1912419202
class RubricGroup(_common.BaseModel):
1912519203
"""A group of rubrics.
1912619204

0 commit comments

Comments
 (0)