Skip to content

Commit 0665566

Browse files
vertex-sdk-botcopybara-github
vertex-sdk-bot
authored and
copybara-github
committed
feat: Add Observability GenAI data format converter for evals
PiperOrigin-RevId: 802668772
1 parent 910016d commit 0665566

4 files changed

Lines changed: 518 additions & 14 deletions

File tree

‎tests/unit/vertexai/genai/test_evals.py‎

Lines changed: 292 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
from vertexai import _genai
2626
from vertexai._genai import _evals_data_converters
2727
from vertexai._genai import _evals_metric_handlers
28+
from vertexai._genai import _observability_data_converter
2829
from vertexai._genai import evals
2930
from vertexai._genai import types as vertexai_genai_types
3031
from google.genai import client
@@ -1921,6 +1922,277 @@ def test_convert_skips_missing_request_or_response(self):
19211922
assert len(result_dataset.eval_cases) == 0
19221923

19231924

1925+
class TestObservabilityDataConverter:
1926+
"""Unit tests for the ObservabilityDataConverter class."""
1927+
1928+
def setup_method(self):
1929+
self.converter = _observability_data_converter.ObservabilityDataConverter()
1930+
1931+
def test_convert_simple_request_response(self):
1932+
raw_data = [
1933+
{
1934+
"format": "observability",
1935+
"request": [
1936+
{"role": "user", "parts": [{"content": "Hello", "type": "text"}]}
1937+
],
1938+
"response": [
1939+
{
1940+
"role": "system",
1941+
"parts": [{"content": "Hi", "type": "text"}],
1942+
}
1943+
],
1944+
}
1945+
]
1946+
result_dataset = self.converter.convert(raw_data)
1947+
1948+
assert isinstance(result_dataset, vertexai_genai_types.EvaluationDataset)
1949+
assert len(result_dataset.eval_cases) == 1
1950+
1951+
eval_case = result_dataset.eval_cases[0]
1952+
assert eval_case.prompt == genai_types.Content(
1953+
parts=[genai_types.Part(text="Hello")], role="user"
1954+
)
1955+
assert len(eval_case.responses) == 1
1956+
assert eval_case.responses[0].response == genai_types.Content(
1957+
parts=[genai_types.Part(text="Hi")], role="system"
1958+
)
1959+
assert eval_case.reference is None
1960+
assert eval_case.system_instruction is None
1961+
assert not eval_case.conversation_history
1962+
1963+
def test_convert_with_system_instruction(self):
1964+
raw_data = [
1965+
{
1966+
"format": "observability",
1967+
"request": [
1968+
{"role": "user", "parts": [{"content": "Hello", "type": "text"}]}
1969+
],
1970+
"response": [
1971+
{
1972+
"role": "system",
1973+
"parts": [{"content": "Hi", "type": "text"}],
1974+
}
1975+
],
1976+
"system_instruction": {
1977+
"role": "user",
1978+
"parts": [{"content": "Be helpful", "type": "text"}],
1979+
},
1980+
}
1981+
]
1982+
result_dataset = self.converter.convert(raw_data)
1983+
eval_case = result_dataset.eval_cases[0]
1984+
assert eval_case.system_instruction == genai_types.Content(
1985+
parts=[genai_types.Part(text="Be helpful")], role="user"
1986+
)
1987+
1988+
def test_convert_with_conversation_history(self):
1989+
raw_data = [
1990+
{
1991+
"format": "observability",
1992+
"request": [
1993+
{"role": "user", "parts": [{"content": "Hello", "type": "text"}]},
1994+
{"role": "system", "parts": [{"content": "Hi", "type": "text"}]},
1995+
{
1996+
"role": "user",
1997+
"parts": [
1998+
{"content": "What's the meaning of life?", "type": "text"}
1999+
],
2000+
},
2001+
],
2002+
"response": [
2003+
{
2004+
"role": "system",
2005+
"parts": [{"content": "42.", "type": "text"}],
2006+
}
2007+
],
2008+
}
2009+
]
2010+
2011+
result_dataset = self.converter.convert(raw_data)
2012+
eval_case = result_dataset.eval_cases[0]
2013+
2014+
assert eval_case.prompt == genai_types.Content(
2015+
parts=[genai_types.Part(text="What's the meaning of life?")], role="user"
2016+
)
2017+
2018+
assert len(eval_case.conversation_history) == 2
2019+
assert eval_case.conversation_history[0] == vertexai_genai_types.Message(
2020+
content=genai_types.Content(
2021+
parts=[genai_types.Part(text="Hello")], role="user"
2022+
),
2023+
turn_id="0",
2024+
author="user",
2025+
)
2026+
assert eval_case.conversation_history[1] == vertexai_genai_types.Message(
2027+
content=genai_types.Content(
2028+
parts=[genai_types.Part(text="Hi")], role="system"
2029+
),
2030+
turn_id="1",
2031+
author="system",
2032+
)
2033+
2034+
def test_convert_multiple_request_response(self):
2035+
raw_data = [
2036+
{
2037+
"format": "observability",
2038+
"request": [
2039+
{"role": "user", "parts": [{"content": "Hello", "type": "text"}]}
2040+
],
2041+
"response": [
2042+
{
2043+
"role": "system",
2044+
"parts": [{"content": "Hi", "type": "text"}],
2045+
}
2046+
],
2047+
},
2048+
{
2049+
"format": "observability",
2050+
"request": [
2051+
{"role": "user", "parts": [{"content": "Goodbye", "type": "text"}]}
2052+
],
2053+
"response": [
2054+
{
2055+
"role": "system",
2056+
"parts": [{"content": "Bye", "type": "text"}],
2057+
}
2058+
],
2059+
},
2060+
]
2061+
result_dataset = self.converter.convert(raw_data)
2062+
2063+
assert isinstance(result_dataset, vertexai_genai_types.EvaluationDataset)
2064+
assert len(result_dataset.eval_cases) == 2
2065+
2066+
eval_case = result_dataset.eval_cases[0]
2067+
assert eval_case.prompt == genai_types.Content(
2068+
parts=[genai_types.Part(text="Hello")], role="user"
2069+
)
2070+
assert eval_case.responses[0].response == genai_types.Content(
2071+
parts=[genai_types.Part(text="Hi")], role="system"
2072+
)
2073+
2074+
eval_case = result_dataset.eval_cases[1]
2075+
assert eval_case.prompt == genai_types.Content(
2076+
parts=[genai_types.Part(text="Goodbye")], role="user"
2077+
)
2078+
assert eval_case.responses[0].response == genai_types.Content(
2079+
parts=[genai_types.Part(text="Bye")], role="system"
2080+
)
2081+
2082+
def test_convert_skips_unknown_part_type(self):
2083+
raw_data = [
2084+
{
2085+
"format": "observability",
2086+
"request": [
2087+
{
2088+
"role": "user",
2089+
"parts": [
2090+
{"content": 123, "type": ""},
2091+
{"content": 456},
2092+
{"content": "Hello", "type": "text"},
2093+
],
2094+
}
2095+
],
2096+
"response": [
2097+
{
2098+
"role": "system",
2099+
"parts": [{"content": "Hi", "type": "text"}],
2100+
}
2101+
],
2102+
}
2103+
]
2104+
2105+
result_dataset = self.converter.convert(raw_data)
2106+
eval_case = result_dataset.eval_cases[0]
2107+
2108+
assert eval_case.prompt == genai_types.Content(
2109+
parts=[genai_types.Part(text="Hello")], role="user"
2110+
)
2111+
2112+
def test_convert_skips_missing_request(self):
2113+
raw_data = [
2114+
{
2115+
"format": "observability",
2116+
"response": [
2117+
{
2118+
"role": "system",
2119+
"parts": [{"content": "Hi", "type": "text"}],
2120+
}
2121+
],
2122+
}
2123+
]
2124+
result_dataset = self.converter.convert(raw_data)
2125+
assert not result_dataset.eval_cases
2126+
2127+
def test_convert_skips_missing_response(self):
2128+
raw_data = [
2129+
{
2130+
"format": "observability",
2131+
"request": [
2132+
{"role": "user", "parts": [{"content": "Hello", "type": "text"}]}
2133+
],
2134+
}
2135+
]
2136+
result_dataset = self.converter.convert(raw_data)
2137+
assert not result_dataset.eval_cases
2138+
2139+
def test_convert_tool_call_parts(self):
2140+
raw_data = [
2141+
{
2142+
"format": "observability",
2143+
"request": [
2144+
{
2145+
"role": "user",
2146+
"parts": [
2147+
{
2148+
"type": "tool_call",
2149+
"id": "tool_id",
2150+
"name": "tool_name",
2151+
"arguments": {"param": "1"},
2152+
}
2153+
],
2154+
}
2155+
],
2156+
"response": [
2157+
{
2158+
"role": "system",
2159+
"parts": [
2160+
{
2161+
"type": "tool_call_response",
2162+
"id": "tool_id",
2163+
"result": {"field": "2"},
2164+
}
2165+
],
2166+
}
2167+
],
2168+
}
2169+
]
2170+
result_dataset = self.converter.convert(raw_data)
2171+
2172+
eval_case = result_dataset.eval_cases[0]
2173+
assert eval_case.prompt == genai_types.Content(
2174+
parts=[
2175+
genai_types.Part(
2176+
function_call=genai_types.FunctionCall(
2177+
id="tool_id", name="tool_id", args={"param": "1"}
2178+
)
2179+
)
2180+
],
2181+
role="user",
2182+
)
2183+
assert len(eval_case.responses) == 1
2184+
assert eval_case.responses[0].response == genai_types.Content(
2185+
parts=[
2186+
genai_types.Part(
2187+
function_response=genai_types.FunctionResponse(
2188+
id="tool_id", name="tool_id", response={"field": "2"}
2189+
)
2190+
)
2191+
],
2192+
role="system",
2193+
)
2194+
2195+
19242196
class TestMetric:
19252197
"""Unit tests for the Metric class."""
19262198

@@ -2953,6 +3225,26 @@ def test_auto_detect_openai_schema(self):
29533225
== _evals_data_converters.EvalDatasetSchema.OPENAI
29543226
)
29553227

3228+
def test_auto_detect_observability_schema(self):
3229+
raw_data = [
3230+
{
3231+
"format": "observability",
3232+
"request": [
3233+
{"role": "user", "parts": [{"content": "Hello", "type": "text"}]}
3234+
],
3235+
"response": [
3236+
{
3237+
"role": "system",
3238+
"parts": [{"content": "Hi", "type": "text"}],
3239+
}
3240+
],
3241+
}
3242+
]
3243+
assert (
3244+
_evals_data_converters.auto_detect_dataset_schema(raw_data)
3245+
== _evals_data_converters.EvalDatasetSchema.OBSERVABILITY
3246+
)
3247+
29563248
def test_auto_detect_unknown_schema(self):
29573249
raw_data = [{"foo": "bar"}]
29583250
assert (

0 commit comments

Comments
 (0)